Merge pull request #229 from MagicRB/support-other-caches

Generalize Cachix support to `post_build_steps`

buildbot_nix/__init__.py

@@ -461,7 +461,7 @@ def do_register_gcroot_if(s: steps.BuildStep) -> Generator[Any, object, Any]:
 def nix_build_config(
     project: GitProject,
     worker_names: list[str],
-    cachix: CachixConfig | None = None,
+    post_build_steps: list[steps.BuildStep],
     outputs_path: Path | None = None,
     retries: int = 1,
 ) -> BuilderConfig:
@@ -493,19 +493,7 @@ def nix_build_config(
             haltOnFailure=True,
         ),
     )
-    if cachix:
+    factory.addSteps(post_build_steps)
-        factory.addStep(
-            steps.ShellCommand(
-                name="Upload cachix",
-                env=cachix.cachix_env(),
-                command=[
-                    "cachix",
-                    "push",
-                    cachix.name,
-                    util.Interpolate("result-%(prop:attr)s"),
-                ],
-            ),
-        )
 
     factory.addStep(
         Trigger(
@@ -638,7 +626,7 @@ def config_for_project(
     nix_eval_worker_count: int,
     nix_eval_max_memory_size: int,
     eval_lock: MasterLock,
-    cachix: CachixConfig | None = None,
+    post_build_steps: list[steps.BuildStep],
     outputs_path: Path | None = None,
     build_retries: int = 1,
 ) -> None:
@@ -716,9 +704,9 @@ def config_for_project(
             nix_build_config(
                 project,
                 worker_names,
-                cachix=cachix,
                 outputs_path=outputs_path,
                 retries=build_retries,
+                post_build_steps=post_build_steps,
             ),
             nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]),
             nix_register_gcroot_config(project, worker_names),
@@ -871,6 +859,7 @@ class NixConfigurator(ConfiguratorBase):
         nix_supported_systems: list[str],
         nix_eval_worker_count: int | None,
         nix_eval_max_memory_size: int,
+        post_build_steps: list[steps.BuildStep] | None = None,
         nix_workers_secret_name: str = "buildbot-nix-workers",  # noqa: S107
         cachix: CachixConfig | None = None,
         outputs_path: str | None = None,
@@ -880,6 +869,7 @@ class NixConfigurator(ConfiguratorBase):
         self.nix_eval_max_memory_size = nix_eval_max_memory_size
         self.nix_eval_worker_count = nix_eval_worker_count
         self.nix_supported_systems = nix_supported_systems
+        self.post_build_steps = post_build_steps or []
         self.auth_backend = auth_backend
         self.admins = admins
         self.github = github
@@ -928,6 +918,20 @@ class NixConfigurator(ConfiguratorBase):
 
         eval_lock = util.MasterLock("nix-eval")
 
+        if self.cachix is not None:
+            self.post_build_steps.append(
+                steps.ShellCommand(
+                    name="Upload cachix",
+                    env=self.cachix.cachix_env(),
+                    command=[
+                        "cachix",
+                        "push",
+                        self.cachix.name,
+                        util.Interpolate("result-%(prop:attr)s"),
+                    ],
+                )
+            )
+
         for project in projects:
             config_for_project(
                 config,
@@ -937,7 +941,7 @@ class NixConfigurator(ConfiguratorBase):
                 self.nix_eval_worker_count or multiprocessing.cpu_count(),
                 self.nix_eval_max_memory_size,
                 eval_lock,
-                self.cachix,
+                self.post_build_steps,
                 self.outputs_path,
                 self.build_retries,
             )

flake.nix

@@ -42,6 +42,11 @@
               };
             in
             examplesFor "x86_64-linux" // examplesFor "aarch64-linux";
+
+          lib = {
+            interpolate = value:
+              { _type = "interpolate"; inherit value; };
+          };
         };
         perSystem = { self', pkgs, system, ... }: {
           packages.default = pkgs.mkShell {

nix/master.nix

@@ -6,6 +6,25 @@
 let
   cfg = config.services.buildbot-nix.master;
   inherit (lib) mkRemovedOptionModule mkRenamedOptionModule;
+
+  interpolateType =
+    lib.mkOptionType {
+      name = "interpolate";
+
+      description = ''
+        A type represnting a Buildbot interpolation string, supports interpolations like `result-%(prop:attr)s`.
+      '';
+
+      check = x:
+        x ? "_type" && x._type == "interpolate" && x ? "value";
+    };
+
+  interpolateToString =
+    value:
+    if lib.isAttrs value && value ? "_type" && value._type == "interpolate" then
+      "util.Interpolate(${builtins.toJSON value.value})"
+    else
+      builtins.toJSON value;
 in
 {
   imports = [
@@ -80,6 +99,60 @@ in
         default = 1;
         description = "Number of times a build is retried";
       };
+
+      postBuildSteps = lib.mkOption {
+        default = [ ];
+        description = ''
+          A list of steps to execute after every successful build.
+        '';
+        type = lib.types.listOf (lib.types.submodule {
+          options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The name of the build step, will show up in Buildbot's UI.
+              '';
+            };
+
+            environment = lib.mkOption {
+              type = with lib.types; attrsOf (oneOf [ interpolateType str ]);
+              description = ''
+                Extra environment variables to add to the environment of this build step.
+                The base environment is the environment of the `buildbot-worker` service.
+
+                To access the properties of a build, use the `interpolate` function defined in
+                `inputs.buildbot-nix.lib.interpolate` like so `(interpolate "result-%(prop:attr)s")`.
+              '';
+              default = { };
+            };
+
+            command = lib.mkOption {
+              type = with lib.types; oneOf [ str (listOf (oneOf [ str interpolateType ])) ];
+              description = ''
+                The command to execute as part of the build step. Either a single string or
+                a list of strings. Be careful that neither variant is interpreted by a shell,
+                but is passed to `execve` verbatim. If you desire a shell, you must use
+                `writeShellScript` or similar functions.
+
+                To access the properties of a build, use the `interpolate` function defined in
+                `inputs.buildbot-nix.lib.interpolate` like so `(interpolate "result-%(prop:attr)s")`.
+              '';
+            };
+          };
+        });
+
+        example = lib.literalExpression ''
+          [
+            name = "upload-to-s3";
+            environment = {
+              S3_TOKEN = "%(secret:s3-token)";
+              S3_BUCKET = "bucket";
+            };
+            command = [ "nix" "copy" "%result%" ];
+          ]
+        '';
+      };
+
       cachix = {
         name = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -323,6 +396,10 @@ in
             CachixConfig,
             GiteaConfig,
           )
+          from buildbot.plugins import (
+            steps,
+            util,
+          )
           from buildbot_nix.github.auth._type import (
             AuthTypeLegacy,
             AuthTypeApp,
@@ -389,6 +466,23 @@ in
                 },
                 nix_supported_systems=${builtins.toJSON cfg.buildSystems},
                 outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
+                post_build_steps=[
+                  ${lib.concatMapStringsSep ",\n" ({ name, environment, command }: ''
+                    steps.ShellCommand(
+                      name=${builtins.toJSON name},
+                      env={
+                        ${lib.concatMapStringsSep ",\n" ({name, value}: ''
+                          ${name}: ${interpolateToString value}
+                        '') (lib.mapAttrsToList lib.nameValuePair environment)}
+                      },
+                      command=[
+                        ${lib.concatMapStringsSep ",\n" (value:
+                          interpolateToString value
+                        ) (if lib.isList command then command else [ command ])}
+                      ]
+                    )
+                  '') cfg.postBuildSteps}
+                ]
             )
           ''
         ];