From e99623b2f6c4c61a17b75d603a51c6a0222297e2 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Thu, 18 Jul 2024 17:39:36 +0200 Subject: [PATCH] Generalize Cachix support to `post_build_steps` Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 38 ++++++++-------- flake.nix | 5 +++ nix/master.nix | 94 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 17 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 6526b88..0b65cd1 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -445,7 +445,7 @@ class CachixConfig: def nix_build_config( project: GitProject, worker_names: list[str], - cachix: CachixConfig | None = None, + post_build_steps: list[steps.BuildStep], outputs_path: Path | None = None, retries: int = 1, ) -> BuilderConfig: @@ -477,19 +477,7 @@ def nix_build_config( haltOnFailure=True, ), ) - if cachix: - factory.addStep( - steps.ShellCommand( - name="Upload cachix", - env=cachix.cachix_env(), - command=[ - "cachix", - "push", - cachix.name, - util.Interpolate("result-%(prop:attr)s"), - ], - ), - ) + factory.addSteps(post_build_steps) factory.addStep( steps.ShellCommand( @@ -571,7 +559,7 @@ def config_for_project( nix_eval_worker_count: int, nix_eval_max_memory_size: int, eval_lock: MasterLock, - cachix: CachixConfig | None = None, + post_build_steps: list[steps.BuildStep], outputs_path: Path | None = None, build_retries: int = 1, ) -> None: @@ -645,9 +633,9 @@ def config_for_project( nix_build_config( project, worker_names, - cachix=cachix, outputs_path=outputs_path, retries=build_retries, + post_build_steps=post_build_steps, ), nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]), ], @@ -799,6 +787,7 @@ class NixConfigurator(ConfiguratorBase): nix_supported_systems: list[str], nix_eval_worker_count: int | None, nix_eval_max_memory_size: int, + post_build_steps: list[steps.BuildStep] | None = None, nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107 cachix: CachixConfig | None = None, outputs_path: str | None = None, @@ -808,6 +797,7 @@ class NixConfigurator(ConfiguratorBase): self.nix_eval_max_memory_size = nix_eval_max_memory_size self.nix_eval_worker_count = nix_eval_worker_count self.nix_supported_systems = nix_supported_systems + self.post_build_steps = post_build_steps or [] self.auth_backend = auth_backend self.admins = admins self.github = github @@ -856,6 +846,20 @@ class NixConfigurator(ConfiguratorBase): eval_lock = util.MasterLock("nix-eval") + if self.cachix is not None: + self.post_build_steps.append( + steps.ShellCommand( + name="Upload cachix", + env=self.cachix.cachix_env(), + command=[ + "cachix", + "push", + self.cachix.name, + util.Interpolate("result-%(prop:attr)s"), + ], + ) + ) + for project in projects: config_for_project( config, @@ -865,7 +869,7 @@ class NixConfigurator(ConfiguratorBase): self.nix_eval_worker_count or multiprocessing.cpu_count(), self.nix_eval_max_memory_size, eval_lock, - self.cachix, + self.post_build_steps, self.outputs_path, self.build_retries, ) diff --git a/flake.nix b/flake.nix index 0995490..9454a19 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,11 @@ }; in examplesFor "x86_64-linux" // examplesFor "aarch64-linux"; + + lib = { + interpolate = value: + { _type = "interpolate"; inherit value; }; + }; }; perSystem = { self', pkgs, system, ... }: { packages.default = pkgs.mkShell { diff --git a/nix/master.nix b/nix/master.nix index 310d371..f3946d3 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -6,6 +6,25 @@ let cfg = config.services.buildbot-nix.master; inherit (lib) mkRemovedOptionModule mkRenamedOptionModule; + + interpolateType = + lib.mkOptionType { + name = "interpolate"; + + description = '' + A type represnting a Buildbot interpolation string, supports interpolations like `result-%(prop:attr)s`. + ''; + + check = x: + x ? "_type" && x._type == "interpolate" && x ? "value"; + }; + + interpolateToString = + value: + if lib.isAttrs value && value ? "_type" && value._type == "interpolate" then + "util.Interpolate(${builtins.toJSON value.value})" + else + builtins.toJSON value; in { imports = [ @@ -80,6 +99,60 @@ in default = 1; description = "Number of times a build is retried"; }; + + postBuildSteps = lib.mkOption { + default = [ ]; + description = '' + A list of steps to execute after every successful build. + ''; + type = lib.types.listOf (lib.types.submodule { + options = { + name = lib.mkOption { + type = lib.types.str; + description = '' + The name of the build step, will show up in Buildbot's UI. + ''; + }; + + environment = lib.mkOption { + type = with lib.types; attrsOf (oneOf [ interpolateType str ]); + description = '' + Extra environment variables to add to the environment of this build step. + The base environment is the environment of the `buildbot-worker` service. + + To access the properties of a build, use the `interpolate` function defined in + `inputs.buildbot-nix.lib.interpolate` like so `(interpolate "result-%(prop:attr)s")`. + ''; + default = { }; + }; + + command = lib.mkOption { + type = with lib.types; oneOf [ str (listOf (oneOf [ str interpolateType ])) ]; + description = '' + The command to execute as part of the build step. Either a single string or + a list of strings. Be careful that neither variant is interpreted by a shell, + but is passed to `execve` verbatim. If you desire a shell, you must use + `writeShellScript` or similar functions. + + To access the properties of a build, use the `interpolate` function defined in + `inputs.buildbot-nix.lib.interpolate` like so `(interpolate "result-%(prop:attr)s")`. + ''; + }; + }; + }); + + example = lib.literalExpression '' + [ + name = "upload-to-s3"; + environment = { + S3_TOKEN = "xxxxxxx"; + S3_BUCKET = "bucket"; + }; + command = [ "nix" "copy" "%result%" ]; + ] + ''; + }; + cachix = { name = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -323,6 +396,10 @@ in CachixConfig, GiteaConfig, ) + from buildbot.plugins import ( + steps, + util, + ) from buildbot_nix.github.auth._type import ( AuthTypeLegacy, AuthTypeApp, @@ -389,6 +466,23 @@ in }, nix_supported_systems=${builtins.toJSON cfg.buildSystems}, outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath}, + post_build_steps=[ + ${lib.concatMapStringsSep ",\n" ({ name, environment, command }: '' + steps.ShellCommand( + name=${builtins.toJSON name}, + env={ + ${lib.concatMapStringsSep ",\n" ({name, value}: '' + ${name}: ${interpolateToString value} + '') (lib.mapAttrsToList lib.nameValuePair environment)} + }, + command=[ + ${lib.concatMapStringsSep ",\n" (value: + interpolateToString value + ) (if lib.isList command then command else [ command ])} + ] + ) + '') cfg.postBuildSteps} + ] ) '' ];