Create gcroots on skipped builds too

Signed-off-by: magic_rb <richard@brezak.sk>

buildbot_nix/__init__.py

@@ -105,6 +105,18 @@ class BuildTrigger(Trigger):
             props.setProperty("status_name", f"nix-build .#checks.{attr}", source)
             props.setProperty("virtual_builder_tags", "", source)
 
+            drv_path = job.get("drvPath")
+            system = job.get("system")
+            out_path = job.get("outputs", {}).get("out")
+
+            props.setProperty("attr", attr, source)
+            props.setProperty("system", system, source)
+            props.setProperty("drv_path", drv_path, source)
+            props.setProperty("out_path", out_path, source)
+            props.setProperty("default_branch", self.project.default_branch, source)
+            # we use this to identify builds when running a retry
+            props.setProperty("build_uuid", str(uuid.uuid4()), source)
+
             if error is not None:
                 props.setProperty("error", error, source)
                 triggered_schedulers.append((self.skipped_builds_scheduler, props))
@@ -114,20 +126,9 @@ class BuildTrigger(Trigger):
                 triggered_schedulers.append((self.skipped_builds_scheduler, props))
                 continue
 
-            drv_path = job.get("drvPath")
-            system = job.get("system")
-            out_path = job.get("outputs", {}).get("out")
-
             build_props.setProperty(f"{attr}-out_path", out_path, source)
             build_props.setProperty(f"{attr}-drv_path", drv_path, source)
 
-            props.setProperty("attr", attr, source)
-            props.setProperty("system", system, source)
-            props.setProperty("drv_path", drv_path, source)
-            props.setProperty("out_path", out_path, source)
-            # we use this to identify builds when running a retry
-            props.setProperty("build_uuid", str(uuid.uuid4()), source)
-
             triggered_schedulers.append((self.builds_scheduler, props))
         return triggered_schedulers
 
@@ -442,6 +443,21 @@ class CachixConfig:
         return env
 
 
+@defer.inlineCallbacks
+def do_register_gcroot_if(s: steps.BuildStep) -> Generator[Any, object, Any]:
+    gc_root = yield util.Interpolate(
+        "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s"
+    ).getRenderingFor(s.getProperties())
+    out_path = yield util.Property("out_path").getRenderingFor(s.getProperties())
+    default_branch = yield util.Property("default_branch").getRenderingFor(
+        s.getProperties()
+    )
+
+    return s.getProperty("branch") == str(default_branch) and not (
+        Path(str(gc_root)).exists() and Path(str(gc_root)).readlink() == str(out_path)
+    )
+
+
 def nix_build_config(
     project: GitProject,
     worker_names: list[str],
@@ -492,19 +508,19 @@ def nix_build_config(
         )
 
     factory.addStep(
-        steps.ShellCommand(
+        Trigger(
             name="Register gcroot",
-            command=[
+            waitForFinish=True,
-                "nix-store",
+            schedulerNames=[
-                "--add-root",
+                f"{slugify_project_name(project.name)}-nix-register-gcroot"
-                # FIXME: cleanup old build attributes
-                util.Interpolate(
-                    "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s",
-                ),
-                "-r",
-                util.Property("out_path"),
             ],
-            doStepIf=lambda s: s.getProperty("branch") == project.default_branch,
+            haltOnFailure=True,
+            flunkOnFailure=True,
+            sourceStamps=[],
+            alwaysUseLatest=False,
+            updateSourceStamp=False,
+            doStepIf=do_register_gcroot_if,
+            copy_properties=["out_path", "attr"],
         ),
     )
     factory.addStep(
@@ -553,6 +569,24 @@ def nix_skipped_build_config(
             hideStepIf=lambda _, s: s.getProperty("error"),
         ),
     )
+
+    # if the change got pulled in from a PR, the roots haven't been created yet
+    factory.addStep(
+        Trigger(
+            name="Register gcroot",
+            waitForFinish=True,
+            schedulerNames=[
+                f"{slugify_project_name(project.name)}-nix-register-gcroot"
+            ],
+            haltOnFailure=True,
+            flunkOnFailure=True,
+            sourceStamps=[],
+            alwaysUseLatest=False,
+            updateSourceStamp=False,
+            doStepIf=do_register_gcroot_if,
+            copy_properties=["out_path", "attr"],
+        ),
+    )
     return util.BuilderConfig(
         name=f"{project.name}/nix-skipped-build",
         project=project.name,
@@ -563,6 +597,39 @@ def nix_skipped_build_config(
     )
 
 
+def nix_register_gcroot_config(
+    project: GitProject,
+    worker_names: list[str],
+) -> BuilderConfig:
+    factory = util.BuildFactory()
+
+    # if the change got pulled in from a PR, the roots haven't been created yet
+    factory.addStep(
+        steps.ShellCommand(
+            name="Register gcroot",
+            command=[
+                "nix-store",
+                "--add-root",
+                # FIXME: cleanup old build attributes
+                util.Interpolate(
+                    "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s",
+                ),
+                "-r",
+                util.Property("out_path"),
+            ],
+        ),
+    )
+
+    return util.BuilderConfig(
+        name=f"{project.name}/nix-register-gcroot",
+        project=project.name,
+        workernames=worker_names,
+        collapseRequests=False,
+        env={},
+        factory=factory,
+    )
+
+
 def config_for_project(
     config: dict[str, Any],
     project: GitProject,
@@ -615,6 +682,10 @@ def config_for_project(
                 name=f"{project.project_id}-nix-skipped-build",
                 builderNames=[f"{project.name}/nix-skipped-build"],
             ),
+            schedulers.Triggerable(
+                name=f"{project.project_id}-nix-register-gcroot",
+                builderNames=[f"{project.name}/nix-register-gcroot"],
+            ),
             # allow to manually trigger a nix-build
             schedulers.ForceScheduler(
                 name=f"{project.project_id}-force",
@@ -650,6 +721,7 @@ def config_for_project(
                 retries=build_retries,
             ),
             nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]),
+            nix_register_gcroot_config(project, worker_names),
         ],
     )
 

nix/0001-GiteaHandler-set-the-category-of-PR-changes-to-pull-.patch

@@ -0,0 +1,27 @@
+From 0cab84b1a8b27b097a980fb6a4de09001eec79dc Mon Sep 17 00:00:00 2001
+From: magic_rb <richard@brezak.sk>
+Date: Mon, 15 Jul 2024 15:58:50 +0200
+Subject: [PATCH] `GiteaHandler` set the category of PR changes to "pull",
+ match `GitHubHandler`
+
+Signed-off-by: magic_rb <richard@brezak.sk>
+---
+ buildbot_gitea/webhook.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/buildbot_gitea/webhook.py b/buildbot_gitea/webhook.py
+index 444f20e..468365c 100644
+--- a/buildbot_gitea/webhook.py
++++ b/buildbot_gitea/webhook.py
+@@ -103,7 +103,7 @@ class GiteaHandler(BaseHookHandler):
+             'revlink': pull_request['html_url'],
+             'repository': head['repo']['ssh_url'],
+             'project': repository['full_name'],
+-            'category': event_type,
++            'category': "pull",
+             'properties': {
+                 'event': event_type,
+                 'base_branch': base['ref'],
+-- 
+2.44.1
+

nix/0002-GiteaHandler-set-branch-to-the-PR-branch-not-the-bas.patch

@@ -0,0 +1,30 @@
+From 7be1b81102e71617ddc568f7957ccc1429f0513d Mon Sep 17 00:00:00 2001
+From: magic_rb <richard@brezak.sk>
+Date: Mon, 15 Jul 2024 15:44:15 +0200
+Subject: [PATCH 2/3] `GiteaHandler`, set `branch` to the PR branch not the
+ base
+
+Signed-off-by: magic_rb <richard@brezak.sk>
+---
+ buildbot_gitea/webhook.py | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/buildbot_gitea/webhook.py b/buildbot_gitea/webhook.py
+index a63cfad..17d4809 100644
+--- a/buildbot_gitea/webhook.py
++++ b/buildbot_gitea/webhook.py
+@@ -97,9 +97,9 @@ class GiteaHandler(BaseHookHandler):
+                 pull_request['number'],
+                 pull_request['title'],
+                 pull_request['body']),
+-            'revision': base['sha'],
++            'revision': head['sha'],
+             'when_timestamp': timestamp,
+-            'branch': base['ref'],
++            'branch': head['ref'],
+             'revlink': pull_request['html_url'],
+             'repository': base['repo']['ssh_url'],
+             'project': repository['full_name'],
+-- 
+2.44.1
+

nix/master.nix

@@ -403,17 +403,27 @@ in
         package = cfg.buildbotNixpkgs.buildbot.overrideAttrs (old: {
           patches = old.patches ++ [ ./0001-master-reporters-github-render-token-for-each-reques.patch ];
         });
-        pythonPackages = ps: [
+        pythonPackages =
-          ps.requests
+          let
-          ps.treq
+            buildbot-gitea = (cfg.buildbotNixpkgs.python3.pkgs.callPackage ./buildbot-gitea.nix {
-          ps.psycopg2
+              inherit (cfg.buildbotNixpkgs) buildbot;
-          (ps.toPythonModule cfg.buildbotNixpkgs.buildbot-worker)
+            }).overrideAttrs (old: {
-          cfg.buildbotNixpkgs.buildbot-plugins.www
+              patches = old.patches ++ [
-          (cfg.buildbotNixpkgs.python3.pkgs.callPackage ../default.nix { })
+                ./0002-GiteaHandler-set-branch-to-the-PR-branch-not-the-bas.patch
-          (cfg.buildbotNixpkgs.python3.pkgs.callPackage ./buildbot-gitea.nix {
+                ./0001-GiteaHandler-set-the-category-of-PR-changes-to-pull-.patch
-            inherit (cfg.buildbotNixpkgs) buildbot;
+              ];
-          })
+            });
-        ];
+          in
+          ps: [
+            pkgs.nix
+            ps.requests
+            ps.treq
+            ps.psycopg2
+            (ps.toPythonModule cfg.buildbotNixpkgs.buildbot-worker)
+            cfg.buildbotNixpkgs.buildbot-plugins.www
+            (cfg.buildbotNixpkgs.python3.pkgs.callPackage ../default.nix { })
+            buildbot-gitea
+          ];
       };
 
       systemd.services.buildbot-master = {