From 28b6da6e736ce5dc618e7c757e50eb6f1ac47546 Mon Sep 17 00:00:00 2001
From: magic_rb <richard@brezak.sk>
Date: Fri, 12 Jul 2024 16:44:02 +0200
Subject: [PATCH] Make the cut off point for combining build reports
 customizable.

Signed-off-by: magic_rb <richard@brezak.sk>
---
 buildbot_nix/__init__.py | 48 ++++++++++++++++++++++++++--------------
 buildbot_nix/models.py   |  1 +
 nix/master.nix           | 12 ++++++++++
 3 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py
index fec1b28..36cc22e 100644
--- a/buildbot_nix/__init__.py
+++ b/buildbot_nix/__init__.py
@@ -165,7 +165,11 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
     project: GitProject
 
     def __init__(
-        self, project: GitProject, supported_systems: list[str], **kwargs: Any
+        self,
+        project: GitProject,
+        supported_systems: list[str],
+        job_report_limit: int | None,
+        **kwargs: Any,
     ) -> None:
         kwargs = self.setupShellMixin(kwargs)
         super().__init__(**kwargs)
@@ -173,6 +177,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
         self.observer = logobserver.BufferLogObserver()
         self.addLogObserver("stdio", self.observer)
         self.supported_systems = supported_systems
+        self.job_report_limit = job_report_limit
 
     @defer.inlineCallbacks
     def run(self) -> Generator[Any, object, Any]:
@@ -205,29 +210,35 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
             self.number_of_jobs = len(filtered_jobs)
 
             self.build.addStepsAfterCurrentStep(
-                [  # noqa: RUF005
+                [
                     BuildTrigger(
                         self.project,
                         builds_scheduler=f"{project_id}-nix-build",
                         skipped_builds_scheduler=f"{project_id}-nix-skipped-build",
                         name="build flake",
                         jobs=filtered_jobs,
-                        report_status=(self.number_of_jobs <= 2),
+                        report_status=(
+                            self.job_report_limit is None
+                            or self.number_of_jobs <= self.job_report_limit
+                        ),
                     ),
                 ]
-                + [
-                    Trigger(
-                        waitForFinish=True,
-                        schedulerNames=[f"{project_id}-nix-build-combined"],
-                        haltOnFailure=True,
-                        flunkOnFailure=True,
-                        sourceStamps=[],
-                        alwaysUseLatest=False,
-                        updateSourceStamp=False,
-                    ),
-                ]
-                if self.number_of_jobs > 2
-                else [],
+                + (
+                    [
+                        Trigger(
+                            waitForFinish=True,
+                            schedulerNames=[f"{project_id}-nix-build-combined"],
+                            haltOnFailure=True,
+                            flunkOnFailure=True,
+                            sourceStamps=[],
+                            alwaysUseLatest=False,
+                            updateSourceStamp=False,
+                        ),
+                    ]
+                    if self.job_report_limit is not None
+                    and self.number_of_jobs > self.job_report_limit
+                    else []
+                ),
             )
 
         return result
@@ -388,6 +399,7 @@ def nix_eval_config(
     eval_lock: MasterLock,
     worker_count: int,
     max_memory_size: int,
+    job_report_limit: int | None,
 ) -> BuilderConfig:
     """Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
     For each evaluated attribute a new build pipeline is started.
@@ -413,6 +425,7 @@ def nix_eval_config(
             env={},
             name="evaluate flake",
             supported_systems=supported_systems,
+            job_report_limit=job_report_limit,
             command=[
                 "nix-eval-jobs",
                 "--workers",
@@ -655,6 +668,7 @@ def config_for_project(
     nix_eval_max_memory_size: int,
     eval_lock: MasterLock,
     post_build_steps: list[steps.BuildStep],
+    job_report_limit: int | None,
     outputs_path: Path | None = None,
     build_retries: int = 1,
 ) -> None:
@@ -730,6 +744,7 @@ def config_for_project(
                 worker_names,
                 git_url=project.get_project_url(),
                 supported_systems=nix_supported_systems,
+                job_report_limit=job_report_limit,
                 worker_count=nix_eval_worker_count,
                 max_memory_size=nix_eval_max_memory_size,
                 eval_lock=eval_lock,
@@ -946,6 +961,7 @@ class NixConfigurator(ConfiguratorBase):
                 self.config.eval_max_memory_size,
                 eval_lock,
                 [x.to_buildstep() for x in self.config.post_build_steps],
+                self.config.job_report_limit,
                 self.config.outputs_path,
                 self.config.build_retries,
             )
diff --git a/buildbot_nix/models.py b/buildbot_nix/models.py
index 2acdcd9..6308b71 100644
--- a/buildbot_nix/models.py
+++ b/buildbot_nix/models.py
@@ -179,6 +179,7 @@ class BuildbotNixConfig(BaseModel):
     outputs_path: Path | None
     url: str
     post_build_steps: list[PostBuildStep]
+    job_report_limit: int | None
 
     @property
     def nix_workers_secret(self) -> str:
diff --git a/nix/master.nix b/nix/master.nix
index 17f11f0..aaead46 100644
--- a/nix/master.nix
+++ b/nix/master.nix
@@ -338,6 +338,17 @@ in
         default = null;
         example = "/var/www/buildbot/nix-outputs";
       };
+
+      jobReportLimit = lib.mkOption {
+        type = lib.types.nullOr lib.types.ints.positive;
+        description = ''
+          The max number of build jobs per `nix-eval` `buildbot-nix` will report to backends (GitHub, Gitea, etc.).
+          If set to `null`, report everything, if set to `n` (some positive intereger), report builds individually
+          as long as the number of builds is less than or equal to `n`, then report builds using a combined
+          `nix-build-combined` build.
+        '';
+        default = 50;
+      };
     };
   };
   config = lib.mkMerge [
@@ -469,6 +480,7 @@ in
                 outputs_path = cfg.outputsPath;
                 url = config.services.buildbot-nix.master.webhookBaseUrl;
                 post_build_steps = cfg.postBuildSteps;
+                job_report_limit=if cfg.jobReportLimit == null then "None" else builtins.toJSON cfg.jobReportLimit;
               }}").read_text()))
             )
           ''