From 8a6d34b83cd946f210c5948171cec3eafe4080ed Mon Sep 17 00:00:00 2001 From: magic_rb Date: Tue, 16 Jul 2024 15:06:13 +0200 Subject: [PATCH] Create gcroots on skipped builds too Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 116 ++++++++++++++---- ...-the-category-of-PR-changes-to-pull-.patch | 27 ++++ ...-branch-to-the-PR-branch-not-the-bas.patch | 30 +++++ nix/master.nix | 32 +++-- 4 files changed, 172 insertions(+), 33 deletions(-) create mode 100644 nix/0001-GiteaHandler-set-the-category-of-PR-changes-to-pull-.patch create mode 100644 nix/0002-GiteaHandler-set-branch-to-the-PR-branch-not-the-bas.patch diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 6526b88..04ed450 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -105,6 +105,18 @@ class BuildTrigger(Trigger): props.setProperty("status_name", f"nix-build .#checks.{attr}", source) props.setProperty("virtual_builder_tags", "", source) + drv_path = job.get("drvPath") + system = job.get("system") + out_path = job.get("outputs", {}).get("out") + + props.setProperty("attr", attr, source) + props.setProperty("system", system, source) + props.setProperty("drv_path", drv_path, source) + props.setProperty("out_path", out_path, source) + props.setProperty("default_branch", self.project.default_branch, source) + # we use this to identify builds when running a retry + props.setProperty("build_uuid", str(uuid.uuid4()), source) + if error is not None: props.setProperty("error", error, source) triggered_schedulers.append((self.skipped_builds_scheduler, props)) @@ -114,20 +126,9 @@ class BuildTrigger(Trigger): triggered_schedulers.append((self.skipped_builds_scheduler, props)) continue - drv_path = job.get("drvPath") - system = job.get("system") - out_path = job.get("outputs", {}).get("out") - build_props.setProperty(f"{attr}-out_path", out_path, source) build_props.setProperty(f"{attr}-drv_path", drv_path, source) - props.setProperty("attr", attr, source) - props.setProperty("system", system, source) - props.setProperty("drv_path", drv_path, source) - props.setProperty("out_path", out_path, source) - # we use this to identify builds when running a retry - props.setProperty("build_uuid", str(uuid.uuid4()), source) - triggered_schedulers.append((self.builds_scheduler, props)) return triggered_schedulers @@ -442,6 +443,21 @@ class CachixConfig: return env +@defer.inlineCallbacks +def do_register_gcroot_if(s: steps.BuildStep) -> Generator[Any, object, Any]: + gc_root = yield util.Interpolate( + "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s" + ).getRenderingFor(s.getProperties()) + out_path = yield util.Property("out_path").getRenderingFor(s.getProperties()) + default_branch = yield util.Property("default_branch").getRenderingFor( + s.getProperties() + ) + + return s.getProperty("branch") == str(default_branch) and not ( + Path(str(gc_root)).exists() and Path(str(gc_root)).readlink() == str(out_path) + ) + + def nix_build_config( project: GitProject, worker_names: list[str], @@ -492,19 +508,19 @@ def nix_build_config( ) factory.addStep( - steps.ShellCommand( + Trigger( name="Register gcroot", - command=[ - "nix-store", - "--add-root", - # FIXME: cleanup old build attributes - util.Interpolate( - "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s", - ), - "-r", - util.Property("out_path"), + waitForFinish=True, + schedulerNames=[ + f"{slugify_project_name(project.name)}-nix-register-gcroot" ], - doStepIf=lambda s: s.getProperty("branch") == project.default_branch, + haltOnFailure=True, + flunkOnFailure=True, + sourceStamps=[], + alwaysUseLatest=False, + updateSourceStamp=False, + doStepIf=do_register_gcroot_if, + copy_properties=["out_path", "attr"], ), ) factory.addStep( @@ -553,6 +569,24 @@ def nix_skipped_build_config( hideStepIf=lambda _, s: s.getProperty("error"), ), ) + + # if the change got pulled in from a PR, the roots haven't been created yet + factory.addStep( + Trigger( + name="Register gcroot", + waitForFinish=True, + schedulerNames=[ + f"{slugify_project_name(project.name)}-nix-register-gcroot" + ], + haltOnFailure=True, + flunkOnFailure=True, + sourceStamps=[], + alwaysUseLatest=False, + updateSourceStamp=False, + doStepIf=do_register_gcroot_if, + copy_properties=["out_path", "attr"], + ), + ) return util.BuilderConfig( name=f"{project.name}/nix-skipped-build", project=project.name, @@ -563,6 +597,39 @@ def nix_skipped_build_config( ) +def nix_register_gcroot_config( + project: GitProject, + worker_names: list[str], +) -> BuilderConfig: + factory = util.BuildFactory() + + # if the change got pulled in from a PR, the roots haven't been created yet + factory.addStep( + steps.ShellCommand( + name="Register gcroot", + command=[ + "nix-store", + "--add-root", + # FIXME: cleanup old build attributes + util.Interpolate( + "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s", + ), + "-r", + util.Property("out_path"), + ], + ), + ) + + return util.BuilderConfig( + name=f"{project.name}/nix-register-gcroot", + project=project.name, + workernames=worker_names, + collapseRequests=False, + env={}, + factory=factory, + ) + + def config_for_project( config: dict[str, Any], project: GitProject, @@ -615,6 +682,10 @@ def config_for_project( name=f"{project.project_id}-nix-skipped-build", builderNames=[f"{project.name}/nix-skipped-build"], ), + schedulers.Triggerable( + name=f"{project.project_id}-nix-register-gcroot", + builderNames=[f"{project.name}/nix-register-gcroot"], + ), # allow to manually trigger a nix-build schedulers.ForceScheduler( name=f"{project.project_id}-force", @@ -650,6 +721,7 @@ def config_for_project( retries=build_retries, ), nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]), + nix_register_gcroot_config(project, worker_names), ], ) diff --git a/nix/0001-GiteaHandler-set-the-category-of-PR-changes-to-pull-.patch b/nix/0001-GiteaHandler-set-the-category-of-PR-changes-to-pull-.patch new file mode 100644 index 0000000..bb63506 --- /dev/null +++ b/nix/0001-GiteaHandler-set-the-category-of-PR-changes-to-pull-.patch @@ -0,0 +1,27 @@ +From 0cab84b1a8b27b097a980fb6a4de09001eec79dc Mon Sep 17 00:00:00 2001 +From: magic_rb +Date: Mon, 15 Jul 2024 15:58:50 +0200 +Subject: [PATCH] `GiteaHandler` set the category of PR changes to "pull", + match `GitHubHandler` + +Signed-off-by: magic_rb +--- + buildbot_gitea/webhook.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/buildbot_gitea/webhook.py b/buildbot_gitea/webhook.py +index 444f20e..468365c 100644 +--- a/buildbot_gitea/webhook.py ++++ b/buildbot_gitea/webhook.py +@@ -103,7 +103,7 @@ class GiteaHandler(BaseHookHandler): + 'revlink': pull_request['html_url'], + 'repository': head['repo']['ssh_url'], + 'project': repository['full_name'], +- 'category': event_type, ++ 'category': "pull", + 'properties': { + 'event': event_type, + 'base_branch': base['ref'], +-- +2.44.1 + diff --git a/nix/0002-GiteaHandler-set-branch-to-the-PR-branch-not-the-bas.patch b/nix/0002-GiteaHandler-set-branch-to-the-PR-branch-not-the-bas.patch new file mode 100644 index 0000000..06f198e --- /dev/null +++ b/nix/0002-GiteaHandler-set-branch-to-the-PR-branch-not-the-bas.patch @@ -0,0 +1,30 @@ +From 7be1b81102e71617ddc568f7957ccc1429f0513d Mon Sep 17 00:00:00 2001 +From: magic_rb +Date: Mon, 15 Jul 2024 15:44:15 +0200 +Subject: [PATCH 2/3] `GiteaHandler`, set `branch` to the PR branch not the + base + +Signed-off-by: magic_rb +--- + buildbot_gitea/webhook.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/buildbot_gitea/webhook.py b/buildbot_gitea/webhook.py +index a63cfad..17d4809 100644 +--- a/buildbot_gitea/webhook.py ++++ b/buildbot_gitea/webhook.py +@@ -97,9 +97,9 @@ class GiteaHandler(BaseHookHandler): + pull_request['number'], + pull_request['title'], + pull_request['body']), +- 'revision': base['sha'], ++ 'revision': head['sha'], + 'when_timestamp': timestamp, +- 'branch': base['ref'], ++ 'branch': head['ref'], + 'revlink': pull_request['html_url'], + 'repository': base['repo']['ssh_url'], + 'project': repository['full_name'], +-- +2.44.1 + diff --git a/nix/master.nix b/nix/master.nix index 310d371..14b1ee6 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -403,17 +403,27 @@ in package = cfg.buildbotNixpkgs.buildbot.overrideAttrs (old: { patches = old.patches ++ [ ./0001-master-reporters-github-render-token-for-each-reques.patch ]; }); - pythonPackages = ps: [ - ps.requests - ps.treq - ps.psycopg2 - (ps.toPythonModule cfg.buildbotNixpkgs.buildbot-worker) - cfg.buildbotNixpkgs.buildbot-plugins.www - (cfg.buildbotNixpkgs.python3.pkgs.callPackage ../default.nix { }) - (cfg.buildbotNixpkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { - inherit (cfg.buildbotNixpkgs) buildbot; - }) - ]; + pythonPackages = + let + buildbot-gitea = (cfg.buildbotNixpkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { + inherit (cfg.buildbotNixpkgs) buildbot; + }).overrideAttrs (old: { + patches = old.patches ++ [ + ./0002-GiteaHandler-set-branch-to-the-PR-branch-not-the-bas.patch + ./0001-GiteaHandler-set-the-category-of-PR-changes-to-pull-.patch + ]; + }); + in + ps: [ + pkgs.nix + ps.requests + ps.treq + ps.psycopg2 + (ps.toPythonModule cfg.buildbotNixpkgs.buildbot-worker) + cfg.buildbotNixpkgs.buildbot-plugins.www + (cfg.buildbotNixpkgs.python3.pkgs.callPackage ../default.nix { }) + buildbot-gitea + ]; }; systemd.services.buildbot-master = {