diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 637a881..fec1b28 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -59,12 +59,14 @@ class BuildTrigger(Trigger): builds_scheduler: str, skipped_builds_scheduler: str, jobs: list[dict[str, Any]], + report_status: bool, **kwargs: Any, ) -> None: if "name" not in kwargs: kwargs["name"] = "trigger" self.project = project self.jobs = jobs + self.report_status = report_status self.config = None self.builds_scheduler = builds_scheduler self.skipped_builds_scheduler = skipped_builds_scheduler @@ -102,6 +104,7 @@ class BuildTrigger(Trigger): props.setProperty("virtual_builder_name", name, source) props.setProperty("status_name", f"nix-build .#checks.{attr}", source) props.setProperty("virtual_builder_tags", "", source) + props.setProperty("report_status", self.report_status, source) drv_path = job.get("drvPath") system = job.get("system") @@ -145,6 +148,15 @@ class BuildTrigger(Trigger): return {"step": f"({', '.join(summary)})"} +class NixBuildCombined(steps.BuildStep): + """Shows the error message of a failed evaluation.""" + + name = "nix-build-combined" + + def run(self) -> Generator[Any, object, Any]: + return self.build.results + + class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): """Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for every attribute. @@ -190,16 +202,32 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): if not system or system in self.supported_systems: # report eval errors filtered_jobs.append(job) + self.number_of_jobs = len(filtered_jobs) + self.build.addStepsAfterCurrentStep( - [ + [ # noqa: RUF005 BuildTrigger( self.project, builds_scheduler=f"{project_id}-nix-build", skipped_builds_scheduler=f"{project_id}-nix-skipped-build", name="build flake", jobs=filtered_jobs, + report_status=(self.number_of_jobs <= 2), ), - ], + ] + + [ + Trigger( + waitForFinish=True, + schedulerNames=[f"{project_id}-nix-build-combined"], + haltOnFailure=True, + flunkOnFailure=True, + sourceStamps=[], + alwaysUseLatest=False, + updateSourceStamp=False, + ), + ] + if self.number_of_jobs > 2 + else [], ) return result @@ -600,6 +628,23 @@ def nix_register_gcroot_config( factory=factory, ) +def nix_build_combined_config( + project: GitProject, + worker_names: list[str], +) -> BuilderConfig: + factory = util.BuildFactory() + factory.addStep(NixBuildCombined()) + + return util.BuilderConfig( + name=f"{project.name}/nix-build-combined", + project=project.name, + workernames=worker_names, + collapseRequests=False, + env={}, + factory=factory, + properties=dict(status_name="nix-build-combined"), + ) + def config_for_project( config: dict[str, Any], @@ -653,6 +698,11 @@ def config_for_project( name=f"{project.project_id}-nix-skipped-build", builderNames=[f"{project.name}/nix-skipped-build"], ), + # this is triggered from `nix-eval` when the build contains too many outputs + schedulers.Triggerable( + name=f"{project.project_id}-nix-build-combined", + builderNames=[f"{project.name}/nix-build-combined"], + ), schedulers.Triggerable( name=f"{project.project_id}-nix-register-gcroot", builderNames=[f"{project.name}/nix-register-gcroot"], @@ -693,6 +743,7 @@ def config_for_project( ), nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]), nix_register_gcroot_config(project, worker_names), + nix_build_combined_config(project, worker_names), ], ) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 12957e2..b605d72 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from itertools import starmap from pathlib import Path -from typing import Any +from typing import Any, Callable from buildbot.config.builder import BuilderConfig from buildbot.plugins import util @@ -20,6 +20,7 @@ from buildbot.www.oauth2 import GitHubAuth from pydantic import BaseModel, ConfigDict, Field from twisted.logger import Logger from twisted.python import log +from twisted.internet import defer from .common import ( ThreadDeferredBuildStep, @@ -309,6 +310,32 @@ class GithubAuthBackend(ABC): ) -> list[BuildStep]: pass +class ModifyingGitHubStatusPush(GitHubStatusPush): + def checkConfig(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + self.modifyingFilter = modifyingFilter + + return super().checkConfig(**kwargs) + + def reconfigService(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + self.modifyingFilter = modifyingFilter + + return super().reconfigService(**kwargs) + + @defer.inlineCallbacks + def sendMessage(self, reports: Any) -> Any: + reports = self.modifyingFilter(reports) + if reports is None: + return + + result = yield super().sendMessage(reports) + return result + +def filter_for_combined_builds(reports: Any) -> Any | None: + properties = reports[0]["builds"][0]["properties"] + + if "report_status" in properties and not properties["report_status"][0]: + return None + return reports class GithubLegacyAuthBackend(GithubAuthBackend): auth_type: GitHubLegacyConfig @@ -329,12 +356,13 @@ class GithubLegacyAuthBackend(GithubAuthBackend): return [GitHubLegacySecretService(self.token)] def create_reporter(self) -> ReporterBase: - return GitHubStatusPush( + return ModifyingGitHubStatusPush( token=self.token.get(), # Since we dynamically create build steps, # we use `virtual_builder_name` in the webinterface # so that we distinguish what has beeing build context=Interpolate("buildbot/%(prop:status_name)s"), + modifyingFilter=filter_for_combined_builds, ) def create_reload_builder_steps( @@ -416,12 +444,13 @@ class GithubAppAuthBackend(GithubAuthBackend): self.project_id_map[props["projectname"]] ].get() - return GitHubStatusPush( + return ModifyingGitHubStatusPush( token=WithProperties("%(github_token)s", github_token=get_github_token), # Since we dynamically create build steps, # we use `virtual_builder_name` in the webinterface # so that we distinguish what has beeing build context=Interpolate("buildbot/%(prop:status_name)s"), + modifyingFilter=filter_for_combined_builds, ) def create_reload_builder_steps( diff --git a/nix/master.nix b/nix/master.nix index 43c3161..17f11f0 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -482,7 +482,9 @@ in dbUrl = config.services.buildbot-nix.master.dbUrl; package = cfg.buildbotNixpkgs.buildbot.overrideAttrs (old: { - patches = old.patches ++ [ ./0001-master-reporters-github-render-token-for-each-reques.patch ]; + patches = old.patches ++ [ + ./0001-master-reporters-github-render-token-for-each-reques.patch + ]; }); pythonPackages = let