Vendor buildbot from the nixpkgs input (nixpkgs-unstable-small)

Signed-off-by: magic_rb <richard@brezak.sk>

README.md

@@ -25,6 +25,33 @@ separate machines. To support multiple architectures, configure them as
 For a practical NixOS example, see
 [this remote builder configuration](https://github.com/Mic92/dotfiles/blob/main/nixos/eve/modules/remote-builder.nix).
 
+## Using `buildbot` with NixOS 24.05 (stable release)
+
+The module applies custom patches that only apply to buildbot > 4.0.0. To use
+buildbot-nix with NixOS 24.05, you should therefore not override the nixpkgs
+input to your own stable version of buildbot-nix and leave it to the default
+instead that is set to nixos-unstable-small.
+
+So instead of using this in your flake
+
+```
+inputs = {
+  buildbot-nix.url = "github:nix-community/buildbot-nix";
+  buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
+};
+```
+
+Just use:
+
+```
+inputs = {
+  buildbot-nix.url = "github:nix-community/buildbot-nix";
+};
+```
+
+An alternative is to point nixpkgs to your own version of nixpkgs-unstable in
+case you are already using it elsewhere.
+
 ## Using Buildbot in Your Project
 
 Buildbot-nix automatically triggers builds for your project under these

flake.nix

@@ -20,8 +20,8 @@
         ] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
         systems = [ "x86_64-linux" ];
         flake = {
-          nixosModules.buildbot-master = ./nix/master.nix;
+          nixosModules.buildbot-master = import ./nix/master.nix inputs;
-          nixosModules.buildbot-worker = ./nix/worker.nix;
+          nixosModules.buildbot-worker = import ./nix/worker.nix inputs;
 
           nixosConfigurations =
             let

nix/master.nix

@@ -1,3 +1,4 @@
+{ nixpkgs, ... }:
 { config
 , pkgs
 , lib
@@ -8,6 +9,7 @@ let
   inherit (lib) mkRemovedOptionModule mkRenamedOptionModule;
 in
 {
+  _file = ./master.nix;
   imports = [
     (mkRenamedOptionModule
       [
@@ -57,8 +59,17 @@ in
   ];
 
   options = {
+    services.buildbot-nix.vendorBuildbot = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = ''
+        Whether to vendow `buildbot-master` from the `nixpkgs` input of the `buildbot-nix` flake.
+        `buildbot-nix` requires `buildbot-master` to be at least of version `4`.
+      '';
+    };
     services.buildbot-nix.master = {
       enable = lib.mkEnableOption "buildbot-master";
+      package = lib.mkPackageOption pkgs "buildbot" { };
       dbUrl = lib.mkOption {
         type = lib.types.str;
         default = "postgresql://@/buildbot";
@@ -263,210 +274,225 @@ in
       };
     };
   };
-  config = lib.mkIf cfg.enable {
+  config = lib.mkMerge [
-    # By default buildbot uses a normal user, which is not a good default, because
+    (lib.mkIf config.services.buildbot-nix.vendorBuildbot {
-    # we grant normal users potentially access to other resources. Also
+      nixpkgs.overlays = [
-    # we don't to be able to ssh into buildbot.
+        (final: prev:
-
+          {
-    users.users.buildbot = {
+            buildbotPackages = prev.python3.pkgs.callPackage "${nixpkgs}/pkgs/development/tools/continuous-integration/buildbot" { };
-      isNormalUser = lib.mkForce false;
+          })
-      isSystemUser = true;
-    };
-
-    assertions = [
-      {
-        assertion =
-          cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
-        message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
-      }
-      {
-        assertion =
-          cfg.authBackend != "github" || (cfg.github.oauthId != null && cfg.github.oauthSecretFile != null);
-        message = ''If config.services.buildbot-nix.master.authBackend is set to "github", then config.services.buildbot-nix.master.github.oauthId and config.services.buildbot-nix.master.github.oauthSecretFile have to be set.'';
-      }
-      {
-        assertion =
-          cfg.authBackend != "gitea" || (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null);
-        message = ''config.services.buildbot-nix.master.authBackend is set to "gitea", then config.services.buildbot-nix.master.gitea.oauthId and config.services.buildbot-nix.master.gitea.oauthSecretFile have to be set.'';
-      }
-    ];
-
-    services.buildbot-master = {
-      enable = true;
-
-      # disable example workers from nixpkgs
-      builders = [ ];
-      schedulers = [ ];
-      workers = [ ];
-
-      home = "/var/lib/buildbot";
-      extraImports = ''
-        from datetime import timedelta
-        from buildbot_nix import (
-          GithubConfig,
-          NixConfigurator,
-          CachixConfig,
-          GiteaConfig,
-        )
-        from buildbot_nix.github.auth._type import (
-          AuthTypeLegacy,
-          AuthTypeApp,
-        )
-      '';
-      configurators = [
-        ''
-          util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
-        ''
-        ''
-          NixConfigurator(
-              auth_backend=${builtins.toJSON cfg.authBackend},
-              github=${
-                if (!cfg.github.enable) then
-                  "None"
-                else
-                  "GithubConfig(
-                  oauth_id=${builtins.toJSON cfg.github.oauthId},
-                  topic=${builtins.toJSON cfg.github.topic},
-                  auth_type=${
-                          if cfg.github.authType ? "legacy" then
-                            ''AuthTypeLegacy()''
-                          else if cfg.github.authType ? "app" then
-                            ''
-                              AuthTypeApp(
-                                app_id=${toString cfg.github.authType.app.id},
-                              )
-                            ''
-                          else
-                            throw "One of AuthTypeApp or AuthTypeLegacy must be enabled"
-                        }
-              )"
-              },
-              gitea=${
-                if !cfg.gitea.enable then
-                  "None"
-                else
-                  "GiteaConfig(
-                  instance_url=${builtins.toJSON cfg.gitea.instanceUrl},
-                  oauth_id=${builtins.toJSON cfg.gitea.oauthId},
-                  topic=${builtins.toJSON cfg.gitea.topic},
-              )"
-              },
-              build_retries=${builtins.toJSON cfg.buildRetries},
-              cachix=${
-                if cfg.cachix.name == null then
-                  "None"
-                else
-                  "CachixConfig(
-                  name=${builtins.toJSON cfg.cachix.name},
-                  signing_key_secret_name=${
-                                      if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"
-                                    },
-                  auth_token_secret_name=${
-                                      if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"
-                                    },
-              )"
-              },
-              admins=${builtins.toJSON cfg.admins},
-              url=${builtins.toJSON config.services.buildbot-nix.master.webhookBaseUrl},
-              nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
-              nix_eval_worker_count=${
-                if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount
-              },
-              nix_supported_systems=${builtins.toJSON cfg.buildSystems},
-              outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
-          )
-        ''
       ];
-      buildbotUrl =
+    })
-        let
+    (lib.mkIf cfg.enable {
-          host = config.services.nginx.virtualHosts.${cfg.domain};
+      # By default buildbot uses a normal user, which is not a good default, because
-          hasSSL = host.forceSSL || host.addSSL || cfg.useHTTPS;
+      # we grant normal users potentially access to other resources. Also
-        in
+      # we don't to be able to ssh into buildbot.
-        "${if hasSSL then "https" else "http"}://${cfg.domain}/";
-      dbUrl = config.services.buildbot-nix.master.dbUrl;
-      package = pkgs.buildbot;
-      pythonPackages = ps: [
-        ps.requests
-        ps.treq
-        ps.psycopg2
-        (ps.toPythonModule pkgs.buildbot-worker)
-        pkgs.buildbot-plugins.www-react
-        (pkgs.python3.pkgs.callPackage ../default.nix { })
-        (pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = pkgs.buildbot; })
-      ];
-    };
 
-    systemd.services.buildbot-master = {
+      users.users.buildbot = {
-      after = [ "postgresql.service" ];
+        isNormalUser = lib.mkForce false;
-      path = [
+        isSystemUser = true;
-        pkgs.openssl
-      ];
-      serviceConfig = {
-        # in master.py we read secrets from $CREDENTIALS_DIRECTORY
-        LoadCredential =
-          [ "buildbot-nix-workers:${cfg.workersFile}" ]
-          ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
-          ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}"
-          ++ lib.optional
-            (
-              cfg.cachix.signingKeyFile != null
-            ) "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
-          ++ lib.optional
-            (
-              cfg.cachix.authTokenFile != null
-            ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
-          ++ lib.optionals (cfg.github.enable) ([
-            "github-webhook-secret:${cfg.github.webhookSecretFile}"
-          ]
-          ++ lib.optionals (cfg.github.authType ? "legacy") [
-            "github-token:${cfg.github.authType.legacy.tokenFile}"
-          ]
-          ++ lib.optionals (cfg.github.authType ? "app") [
-            "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}"
-          ])
-          ++ lib.optionals cfg.gitea.enable [
-            "gitea-token:${cfg.gitea.tokenFile}"
-            "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
-          ];
       };
-    };
 
-    services.postgresql = {
+      assertions = [
-      enable = true;
-      ensureDatabases = [ "buildbot" ];
-      ensureUsers = [
         {
-          name = "buildbot";
+          assertion =
-          ensureDBOwnership = true;
+            lib.versionAtLeast cfg.package.version "4.0.0";
+          message = "`buildbot-nix` requires `buildbot` 4.0.0 or greater to function";
+        }
+        {
+          assertion =
+            cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
+          message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
+        }
+        {
+          assertion =
+            cfg.authBackend != "github" || (cfg.github.oauthId != null && cfg.github.oauthSecretFile != null);
+          message = ''If config.services.buildbot-nix.master.authBackend is set to "github", then config.services.buildbot-nix.master.github.oauthId and config.services.buildbot-nix.master.github.oauthSecretFile have to be set.'';
+        }
+        {
+          assertion =
+            cfg.authBackend != "gitea" || (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null);
+          message = ''config.services.buildbot-nix.master.authBackend is set to "gitea", then config.services.buildbot-nix.master.gitea.oauthId and config.services.buildbot-nix.master.gitea.oauthSecretFile have to be set.'';
         }
       ];
-    };
 
-    services.nginx.enable = true;
+      services.buildbot-master = {
-    services.nginx.virtualHosts.${cfg.domain} = {
+        enable = true;
-      locations = {
-        "/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/";
-        "/sse" = {
-          proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse";
-          # proxy buffering will prevent sse to work
-          extraConfig = "proxy_buffering off;";
-        };
-        "/ws" = {
-          proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws";
-          proxyWebsockets = true;
-          # raise the proxy timeout for the websocket
-          extraConfig = "proxy_read_timeout 6000s;";
-        };
-      } // lib.optionalAttrs (cfg.outputsPath != null) { "/nix-outputs".root = cfg.outputsPath; };
-    };
 
-    systemd.tmpfiles.rules =
+        # disable example workers from nixpkgs
-      [
+        builders = [ ];
-        # delete legacy gcroot location, can be dropped after 2024-06-01
+        schedulers = [ ];
-        "R /var/lib/buildbot-worker/gcroot - - - - -"
+        workers = [ ];
-      ]
+
-      ++ lib.optional (cfg.outputsPath != null)
+        home = "/var/lib/buildbot";
-        # Allow buildbot-master to write to this directory
+        extraImports = ''
-        "d ${cfg.outputsPath} 0755 buildbot buildbot - -";
+          from datetime import timedelta
-  };
+          from buildbot_nix import (
+            GithubConfig,
+            NixConfigurator,
+            CachixConfig,
+            GiteaConfig,
+          )
+          from buildbot_nix.github.auth._type import (
+            AuthTypeLegacy,
+            AuthTypeApp,
+          )
+        '';
+        configurators = [
+          ''
+            util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
+          ''
+          ''
+            NixConfigurator(
+                auth_backend=${builtins.toJSON cfg.authBackend},
+                github=${
+                  if (!cfg.github.enable) then
+                    "None"
+                  else
+                    "GithubConfig(
+                    oauth_id=${builtins.toJSON cfg.github.oauthId},
+                    topic=${builtins.toJSON cfg.github.topic},
+                    auth_type=${
+                            if cfg.github.authType ? "legacy" then
+                              ''AuthTypeLegacy()''
+                            else if cfg.github.authType ? "app" then
+                              ''
+                                AuthTypeApp(
+                                  app_id=${toString cfg.github.authType.app.id},
+                                )
+                              ''
+                            else
+                              throw "One of AuthTypeApp or AuthTypeLegacy must be enabled"
+                          }
+                )"
+                },
+                gitea=${
+                  if !cfg.gitea.enable then
+                    "None"
+                  else
+                    "GiteaConfig(
+                    instance_url=${builtins.toJSON cfg.gitea.instanceUrl},
+                    oauth_id=${builtins.toJSON cfg.gitea.oauthId},
+                    topic=${builtins.toJSON cfg.gitea.topic},
+                )"
+                },
+                build_retries=${builtins.toJSON cfg.buildRetries},
+                cachix=${
+                  if cfg.cachix.name == null then
+                    "None"
+                  else
+                    "CachixConfig(
+                    name=${builtins.toJSON cfg.cachix.name},
+                    signing_key_secret_name=${
+                                        if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"
+                                      },
+                    auth_token_secret_name=${
+                                        if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"
+                                      },
+                )"
+                },
+                admins=${builtins.toJSON cfg.admins},
+                url=${builtins.toJSON config.services.buildbot-nix.master.webhookBaseUrl},
+                nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
+                nix_eval_worker_count=${
+                  if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount
+                },
+                nix_supported_systems=${builtins.toJSON cfg.buildSystems},
+                outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
+            )
+          ''
+        ];
+        buildbotUrl =
+          let
+            host = config.services.nginx.virtualHosts.${cfg.domain};
+            hasSSL = host.forceSSL || host.addSSL || cfg.useHTTPS;
+          in
+          "${if hasSSL then "https" else "http"}://${cfg.domain}/";
+        dbUrl = config.services.buildbot-nix.master.dbUrl;
+        package = cfg.package;
+        pythonPackages = ps: [
+          ps.requests
+          ps.treq
+          ps.psycopg2
+          (ps.toPythonModule pkgs.buildbot-worker)
+          pkgs.buildbot-plugins.www-react
+          (pkgs.python3.pkgs.callPackage ../default.nix { })
+          (pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = cfg.package; })
+        ];
+      };
+
+      systemd.services.buildbot-master = {
+        after = [ "postgresql.service" ];
+        path = [
+          pkgs.openssl
+        ];
+        serviceConfig = {
+          # in master.py we read secrets from $CREDENTIALS_DIRECTORY
+          LoadCredential =
+            [ "buildbot-nix-workers:${cfg.workersFile}" ]
+            ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
+            ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}"
+            ++ lib.optional
+              (
+                cfg.cachix.signingKeyFile != null
+              ) "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
+            ++ lib.optional
+              (
+                cfg.cachix.authTokenFile != null
+              ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
+            ++ lib.optionals (cfg.github.enable) ([
+              "github-webhook-secret:${cfg.github.webhookSecretFile}"
+            ]
+            ++ lib.optionals (cfg.github.authType ? "legacy") [
+              "github-token:${cfg.github.authType.legacy.tokenFile}"
+            ]
+            ++ lib.optionals (cfg.github.authType ? "app") [
+              "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}"
+            ])
+            ++ lib.optionals cfg.gitea.enable [
+              "gitea-token:${cfg.gitea.tokenFile}"
+              "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
+            ];
+        };
+      };
+
+      services.postgresql = {
+        enable = true;
+        ensureDatabases = [ "buildbot" ];
+        ensureUsers = [
+          {
+            name = "buildbot";
+            ensureDBOwnership = true;
+          }
+        ];
+      };
+
+      services.nginx.enable = true;
+      services.nginx.virtualHosts.${cfg.domain} = {
+        locations = {
+          "/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/";
+          "/sse" = {
+            proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse";
+            # proxy buffering will prevent sse to work
+            extraConfig = "proxy_buffering off;";
+          };
+          "/ws" = {
+            proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws";
+            proxyWebsockets = true;
+            # raise the proxy timeout for the websocket
+            extraConfig = "proxy_read_timeout 6000s;";
+          };
+        } // lib.optionalAttrs (cfg.outputsPath != null) { "/nix-outputs".root = cfg.outputsPath; };
+      };
+
+      systemd.tmpfiles.rules =
+        [
+          # delete legacy gcroot location, can be dropped after 2024-06-01
+          "R /var/lib/buildbot-worker/gcroot - - - - -"
+        ]
+        ++ lib.optional (cfg.outputsPath != null)
+          # Allow buildbot-master to write to this directory
+          "d ${cfg.outputsPath} 0755 buildbot buildbot - -";
+    })
+  ];
 }

nix/worker.nix

@@ -1,3 +1,4 @@
+{ ... }:
 { config
 , pkgs
 , lib
@@ -10,6 +11,7 @@ let
   python = cfg.package.pythonModule;
 in
 {
+  _file = ./worker.nix;
   options = {
     services.buildbot-nix.worker = {
       enable = lib.mkEnableOption "buildbot-worker";
@@ -86,7 +88,10 @@ in
 
         # Restart buildbot with a delay. This time way we can use buildbot to deploy itself.
         ExecReload = "+${config.systemd.package}/bin/systemd-run --on-active=60 ${config.systemd.package}/bin/systemctl restart buildbot-worker";
-        ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${../buildbot_nix}/worker.py";
+        ExecStart = lib.traceIf
+          (lib.versionOlder pkgs.buildbot-worker.version "4.0.0")
+          "`buildbot-nix` recommends `buildbot-worker` to be at least of version `4.0.0`"
+          "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${../buildbot_nix}/worker.py";
       };
     };
   };