Merge pull request #4 from Mic92/cleanups

wip: switch to configurators class

README.md

@@ -2,6 +2,7 @@
 
 A nixos module to make buildbot a proper Nix-CI.
 
-For an example checkout the [example](./examples/default.nix) and the module descriptions for [master](./nix/master.nix) and [worker](./nix/worker.nix).
+For an example checkout the [example](./examples/default.nix) and the module
+descriptions for [master](./nix/master.nix) and [worker](./nix/worker.nix).
 
 This project is still in early stage and many APIs might change over time.

buildbot_nix/buildbot_nix.py

@@ -3,18 +3,23 @@
 import json
 import multiprocessing
 import os
+import sys
 import uuid
 from collections import defaultdict
+from collections.abc import Generator
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Generator, List
+from typing import Any
 
-from buildbot.plugins import steps, util
+from buildbot.configurators import ConfiguratorBase
+from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
 from buildbot.process import buildstep, logobserver, remotecommand
 from buildbot.process.log import Log
-from buildbot.process.properties import Properties
+from buildbot.process.project import Project
+from buildbot.process.properties import Interpolate, Properties
 from buildbot.process.results import ALL_RESULTS, statusToString
 from buildbot.steps.trigger import Trigger
-from github_projects import GithubProject
+from github_projects import GithubProject, load_projects  # noqa: E402
 from twisted.internet import defer
 
 
@@ -348,7 +353,7 @@ def nix_eval_config(
     worker_names: list[str],
     github_token_secret: str,
     supported_systems: list[str],
-    automerge_users: List[str] = [],
+    automerge_users: list[str] = [],
     max_memory_size: int = 4096,
 ) -> util.BuilderConfig:
     """
@@ -527,3 +532,205 @@ def nix_build_config(
         env={},
         factory=factory,
     )
+
+
+def read_secret_file(secret_name: str) -> str:
+    directory = os.environ.get("CREDENTIALS_DIRECTORY")
+    if directory is None:
+        print("directory not set", file=sys.stderr)
+        sys.exit(1)
+    return Path(directory).joinpath(secret_name).read_text()
+
+
+@dataclass
+class GithubConfig:
+    oauth_id: str
+    admins: list[str]
+    buildbot_user: str
+    oauth_secret_name: str = "github-oauth-secret"
+    webhook_secret_name: str = "github-webhook-secret"
+    token_secret_name: str = "github-token"
+    project_cache_file: Path = Path("github-project-cache.json")
+
+    def token(self) -> str:
+        return read_secret_file(self.token_secret_name)
+
+
+def config_for_project(
+    config: dict[str, Any],
+    project: GithubProject,
+    credentials: str,
+    worker_names: list[str],
+    github: GithubConfig,
+    nix_supported_systems: list[str],
+    nix_eval_max_memory_size: int,
+) -> Project:
+    config["projects"].append(Project(project.name))
+    config["schedulers"].extend(
+        [
+            schedulers.SingleBranchScheduler(
+                name=f"default-branch-{project.id}",
+                change_filter=util.ChangeFilter(
+                    repository=project.url,
+                    filter_fn=lambda c: c.branch
+                    == c.properties.getProperty("github.repository.default_branch"),
+                ),
+                builderNames=[f"{project.name}/nix-eval"],
+            ),
+            # this is compatible with bors or github's merge queue
+            schedulers.SingleBranchScheduler(
+                name=f"merge-queue-{project.id}",
+                change_filter=util.ChangeFilter(
+                    repository=project.url,
+                    branch_re="(gh-readonly-queue/.*|staging|trying)",
+                ),
+                builderNames=[f"{project.name}/nix-eval"],
+            ),
+            # build all pull requests
+            schedulers.SingleBranchScheduler(
+                name=f"prs-{project.id}",
+                change_filter=util.ChangeFilter(
+                    repository=project.url, category="pull"
+                ),
+                builderNames=[f"{project.name}/nix-eval"],
+            ),
+            # this is triggered from `nix-eval`
+            schedulers.Triggerable(
+                name=f"{project.id}-nix-build",
+                builderNames=[f"{project.name}/nix-build"],
+            ),
+            # allow to manually trigger a nix-build
+            schedulers.ForceScheduler(
+                name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"]
+            ),
+            # allow to manually update flakes
+            schedulers.ForceScheduler(
+                name=f"{project.id}-update-flake",
+                builderNames=[f"{project.name}/update-flake"],
+                buttonName="Update flakes",
+            ),
+            # updates flakes once a weeek
+            schedulers.NightlyTriggerable(
+                name=f"{project.id}-update-flake-weekly",
+                builderNames=[f"{project.name}/update-flake"],
+                hour=3,
+                minute=0,
+                dayOfWeek=6,
+            ),
+        ]
+    )
+    has_cachix_auth_token = os.path.isfile(
+        os.path.join(credentials, "cachix-auth-token")
+    )
+    has_cachix_signing_key = os.path.isfile(
+        os.path.join(credentials, "cachix-signing-key")
+    )
+    config["builders"].extend(
+        [
+            # Since all workers run on the same machine, we only assign one of them to do the evaluation.
+            # This should prevent exessive memory usage.
+            nix_eval_config(
+                project,
+                [worker_names[0]],
+                github_token_secret=github.token_secret_name,
+                supported_systems=nix_supported_systems,
+                automerge_users=[github.buildbot_user],
+                max_memory_size=nix_eval_max_memory_size,
+            ),
+            nix_build_config(
+                project,
+                worker_names,
+                has_cachix_auth_token,
+                has_cachix_signing_key,
+            ),
+            nix_update_flake_config(
+                project,
+                worker_names,
+                github_token_secret=github.token_secret_name,
+                github_bot_user=github.buildbot_user,
+            ),
+        ]
+    )
+
+
+class NixConfigurator(ConfiguratorBase):
+    """Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
+
+    def __init__(
+        self,
+        # Shape of this file:
+        # [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
+        github: GithubConfig,
+        nix_supported_systems: list[str],
+        nix_eval_max_memory_size: int = 4096,
+        nix_workers_secret_name: str = "buildbot-nix-workers",
+    ) -> None:
+        super().__init__()
+        self.nix_workers_secret_name = nix_workers_secret_name
+        self.nix_eval_max_memory_size = nix_eval_max_memory_size
+        self.nix_supported_systems = nix_supported_systems
+        self.github = github
+        self.systemd_credentials_dir = os.environ["CREDENTIALS_DIRECTORY"]
+
+    def configure(self, config: dict[str, Any]) -> None:
+        projects = load_projects(self.github.token(), self.github.project_cache_file)
+        projects = [p for p in projects if "build-with-buildbot" in p.topics]
+        worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
+        worker_names = []
+        config["workers"] = config.get("workers", [])
+        for item in worker_config:
+            cores = item.get("cores", 0)
+            for i in range(cores):
+                worker_name = f"{item['name']}-{i}"
+                config["workers"].append(worker.Worker(worker_name, item["pass"]))
+                worker_names.append(worker_name)
+
+        for project in projects:
+            config_for_project(
+                config,
+                project,
+                self.systemd_credentials_dir,
+                worker_names,
+                self.github,
+                self.nix_supported_systems,
+                self.nix_eval_max_memory_size,
+            )
+        config["services"] = config.get("services", [])
+        config["services"].append(
+            reporters.GitHubStatusPush(
+                token=self.github.token(),
+                # Since we dynamically create build steps,
+                # we use `virtual_builder_name` in the webinterface
+                # so that we distinguish what has beeing build
+                context=Interpolate("buildbot/%(prop:status_name)s"),
+            )
+        )
+        systemd_secrets = secrets.SecretInAFile(
+            dirname=os.environ["CREDENTIALS_DIRECTORY"]
+        )
+        config["secretsProviders"] = config.get("secretsProviders", [])
+        config["secretsProviders"].append(systemd_secrets)
+        config["www"] = config.get("www", {})
+        config["www"]["avatar_methods"] = config["www"].get("avatar_methods", [])
+        config["www"]["avatar_methods"].append(util.AvatarGitHub())
+        config["www"]["auth"] = util.GitHubAuth(
+            self.github.oauth_id, read_secret_file(self.github.oauth_secret_name)
+        )
+        config["www"]["authz"] = util.Authz(
+            roleMatchers=[
+                util.RolesFromUsername(roles=["admin"], usernames=self.github.admins)
+            ],
+            allowRules=[
+                util.AnyEndpointMatcher(role="admin", defaultDeny=False),
+                util.AnyControlEndpointMatcher(role="admins"),
+            ],
+        )
+        config["www"]["change_hook_dialects"] = config["www"].get(
+            "change_hook_dialects", {}
+        )
+        config["www"]["change_hook_dialects"]["github"] = {
+            "secret": read_secret_file(self.github.webhook_secret_name),
+            "strict": True,
+            "token": self.github.token(),
+            "github_property_whitelist": "*",
+        }

buildbot_nix/master.py

@@ -1,161 +1,39 @@
 #!/usr/bin/env python3
 
-import json
 import os
 import sys
 from datetime import timedelta
 from pathlib import Path
 from typing import Any
 
-from buildbot.plugins import reporters, schedulers, secrets, util, worker
+from buildbot.plugins import schedulers, util
-from buildbot.process.project import Project
-from buildbot.process.properties import Interpolate
 
 # allow to import modules
 sys.path.append(str(Path(__file__).parent))
 
-from buildbot_nix import (  # noqa: E402
+from buildbot_nix import GithubConfig, NixConfigurator  # noqa: E402
-    nix_build_config,
-    nix_eval_config,
-    nix_update_flake_config,
-)
-from github_projects import GithubProject, load_projects  # noqa: E402
-
-
-def read_secret_file(secret_name: str) -> str:
-    directory = os.environ.get("CREDENTIALS_DIRECTORY")
-    if directory is None:
-        print("directory not set", file=sys.stderr)
-        sys.exit(1)
-    return Path(directory).joinpath(secret_name).read_text()
-
-
-GITHUB_OAUTH_ID = os.environ.get("GITHUB_OAUTH_ID")
-GITHUB_OAUTH_SECRET = read_secret_file("github-oauth-secret")
-GITHUB_ADMINS = os.environ.get("GITHUB_ADMINS", "").split(" ")
-GITHUB_TOKEN_SECRET_NAME = "github-token"
-GITHUB_TOKEN = read_secret_file(GITHUB_TOKEN_SECRET_NAME)
-GITHUB_WEBHOOK_SECRET = read_secret_file("github-webhook-secret")
-# Shape of this file:
-# [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
-BUILDBOT_NIX_WORKERS = read_secret_file("buildbot-nix-workers")
-BUILDBOT_URL = os.environ["BUILDBOT_URL"]
-BUILDBOT_GITHUB_USER = os.environ["BUILDBOT_GITHUB_USER"]
-NIX_SUPPORTED_SYSTEMS = os.environ["NIX_SUPPORTED_SYSTEMS"].split(" ")
-NIX_EVAL_MAX_MEMORY_SIZE = int(os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096"))
-BUILDBOT_DB_URL = os.environ.get("DB_URL", "sqlite:///state.sqlite")
-
-
-def config_for_project(
-    config: dict[str, Any],
-    project: GithubProject,
-    credentials: str,
-    worker_names: list[str],
-) -> Project:
-    config["projects"].append(Project(project.name))
-    config["schedulers"].extend(
-        [
-            schedulers.SingleBranchScheduler(
-                name=f"default-branch-{project.id}",
-                change_filter=util.ChangeFilter(
-                    repository=project.url,
-                    filter_fn=lambda c: c.branch
-                    == c.properties.getProperty("github.repository.default_branch"),
-                ),
-                builderNames=[f"{project.name}/nix-eval"],
-            ),
-            # this is compatible with bors or github's merge queue
-            schedulers.SingleBranchScheduler(
-                name=f"merge-queue-{project.id}",
-                change_filter=util.ChangeFilter(
-                    repository=project.url,
-                    branch_re="(gh-readonly-queue/.*|staging|trying)",
-                ),
-                builderNames=[f"{project.name}/nix-eval"],
-            ),
-            # build all pull requests
-            schedulers.SingleBranchScheduler(
-                name=f"prs-{project.id}",
-                change_filter=util.ChangeFilter(
-                    repository=project.url, category="pull"
-                ),
-                builderNames=[f"{project.name}/nix-eval"],
-            ),
-            # this is triggered from `nix-eval`
-            schedulers.Triggerable(
-                name=f"{project.id}-nix-build",
-                builderNames=[f"{project.name}/nix-build"],
-            ),
-            # allow to manually trigger a nix-build
-            schedulers.ForceScheduler(
-                name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"]
-            ),
-            # allow to manually update flakes
-            schedulers.ForceScheduler(
-                name=f"{project.id}-update-flake",
-                builderNames=[f"{project.name}/update-flake"],
-                buttonName="Update flakes",
-            ),
-            # updates flakes once a weeek
-            schedulers.NightlyTriggerable(
-                name=f"{project.id}-update-flake-weekly",
-                builderNames=[f"{project.name}/update-flake"],
-                hour=3,
-                minute=0,
-                dayOfWeek=6,
-            ),
-        ]
-    )
-    has_cachix_auth_token = os.path.isfile(
-        os.path.join(credentials, "cachix-auth-token")
-    )
-    has_cachix_signing_key = os.path.isfile(
-        os.path.join(credentials, "cachix-signing-key")
-    )
-    config["builders"].extend(
-        [
-            # Since all workers run on the same machine, we only assign one of them to do the evaluation.
-            # This should prevent exessive memory usage.
-            nix_eval_config(
-                project,
-                [worker_names[0]],
-                github_token_secret=GITHUB_TOKEN_SECRET_NAME,
-                supported_systems=NIX_SUPPORTED_SYSTEMS,
-                automerge_users=[BUILDBOT_GITHUB_USER],
-                max_memory_size=NIX_EVAL_MAX_MEMORY_SIZE,
-            ),
-            nix_build_config(
-                project,
-                worker_names,
-                has_cachix_auth_token,
-                has_cachix_signing_key,
-            ),
-            nix_update_flake_config(
-                project,
-                worker_names,
-                github_token_secret=GITHUB_TOKEN_SECRET_NAME,
-                github_bot_user=BUILDBOT_GITHUB_USER,
-            ),
-        ]
-    )
-
-
-PROJECT_CACHE_FILE = Path("github-project-cache.json")
 
 
 def build_config() -> dict[str, Any]:
-    projects = load_projects(GITHUB_TOKEN, PROJECT_CACHE_FILE)
-    projects = [p for p in projects if "build-with-buildbot" in p.topics]
-    import pprint
-
-    pprint.pprint(projects)
     c: dict[str, Any] = {}
     c["buildbotNetUsageData"] = None
     # configure a janitor which will delete all logs older than one month, and will run on sundays at noon
     c["configurators"] = [
-        util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
+        util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6),
+        NixConfigurator(
+            github=GithubConfig(
+                oauth_id=os.environ["GITHUB_OAUTH_ID"],
+                admins=os.environ.get("GITHUB_ADMINS", "").split(" "),
+                buildbot_user=os.environ["BUILDBOT_GITHUB_USER"],
+            ),
+            nix_eval_max_memory_size=int(
+                os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096")
+            ),
+            nix_supported_systems=os.environ.get("NIX_SUPPORTED_SYSTEMS", "auto").split(
+                " "
+            ),
+        ),
     ]
-    credentials = os.environ.get("CREDENTIALS_DIRECTORY", ".")
     c["schedulers"] = [
         schedulers.SingleBranchScheduler(
             name="nixpkgs",
@@ -171,62 +49,17 @@ def build_config() -> dict[str, Any]:
     c["builders"] = []
     c["projects"] = []
     c["workers"] = []
-
+    c["services"] = []
-    worker_config = json.loads(BUILDBOT_NIX_WORKERS)
-    worker_names = []
-    for item in worker_config:
-        cores = item.get("cores", 0)
-        for i in range(cores):
-            worker_name = f"{item['name']}-{i}"
-            c["workers"].append(worker.Worker(worker_name, item["pass"]))
-            worker_names.append(worker_name)
-
-    for project in projects:
-        config_for_project(c, project, credentials, worker_names)
-
-    c["services"] = [
-        reporters.GitHubStatusPush(
-            token=GITHUB_TOKEN,
-            # Since we dynamically create build steps,
-            # we use `virtual_builder_name` in the webinterface
-            # so that we distinguish what has beeing build
-            context=Interpolate("buildbot/%(prop:status_name)s"),
-        )
-    ]
-
-    systemd_secrets = secrets.SecretInAFile(dirname=credentials)
-    c["secretsProviders"] = [systemd_secrets]
-
     c["www"] = {
-        "avatar_methods": [util.AvatarGitHub()],
-        "port": int(os.environ.get("PORT", "1810")),
-        "auth": util.GitHubAuth(GITHUB_OAUTH_ID, GITHUB_OAUTH_SECRET),
-        "authz": util.Authz(
-            roleMatchers=[
-                util.RolesFromUsername(roles=["admin"], usernames=GITHUB_ADMINS)
-            ],
-            allowRules=[
-                util.AnyEndpointMatcher(role="admin", defaultDeny=False),
-                util.AnyControlEndpointMatcher(role="admins"),
-            ],
-        ),
         "plugins": dict(
             base_react={}, waterfall_view={}, console_view={}, grid_view={}
         ),
-        "change_hook_dialects": dict(
+        "port": int(os.environ.get("PORT", "1810")),
-            github={
-                "secret": GITHUB_WEBHOOK_SECRET,
-                "strict": True,
-                "token": GITHUB_TOKEN,
-                "github_property_whitelist": "*",
-            }
-        ),
     }
 
-    c["db"] = {"db_url": BUILDBOT_DB_URL}
+    c["db"] = {"db_url": os.environ.get("DB_URL", "sqlite:///state.sqlite")}
-
     c["protocols"] = {"pb": {"port": "tcp:9989:interface=\\:\\:"}}
-    c["buildbotURL"] = BUILDBOT_URL
+    c["buildbotURL"] = os.environ["BUILDBOT_URL"]
 
     return c
 

flake.lock

@@ -39,7 +39,28 @@
     "root": {
       "inputs": {
         "flake-parts": "flake-parts",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "treefmt-nix": "treefmt-nix"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1694528738,
+        "narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
       }
     }
   },

flake.nix

@@ -6,11 +6,16 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-parts.url = "github:hercules-ci/flake-parts";
     flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
+
+    # used for development
+    treefmt-nix.url = "github:numtide/treefmt-nix";
+    treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs = inputs@{ self, flake-parts, ... }:
     flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }:
       {
+        imports = inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
         systems = [ "x86_64-linux" "aarch64-linux" ];
         flake = {
           nixosModules.buildbot-master = ./nix/master.nix;

nix/master.nix

@@ -50,7 +50,8 @@ in
         };
         githubAdmins = lib.mkOption {
           type = lib.types.listOf lib.types.str;
-          description = "Users that are allowed to login to buildbot and do stuff";
+          default = [ ];
+          description = "Users that are allowed to login to buildbot, trigger builds and change settings";
         };
       };
       workersFile = lib.mkOption {

nix/treefmt/flake-module.nix

@@ -0,0 +1,29 @@
+{ inputs, ... }: {
+  imports = [
+    inputs.treefmt-nix.flakeModule
+  ];
+  perSystem = { config, pkgs, ... }: {
+    treefmt = {
+      projectRootFile = ".git/config";
+      programs.nixpkgs-fmt.enable = true;
+      programs.shellcheck.enable = true;
+      programs.deno.enable = true;
+      settings.formatter.shellcheck.options = [ "-s" "bash" ];
+
+      programs.mypy.enable = true;
+      programs.mypy.directories."." = { };
+      settings.formatter.python = {
+        command = "sh";
+        options = [
+          "-eucx"
+          ''
+            ${pkgs.ruff}/bin/ruff --fix "$@"
+            ${pkgs.python3.pkgs.black}/bin/black "$@"
+          ''
+          "--" # this argument is ignored by bash
+        ];
+        includes = [ "*.py" ];
+      };
+    };
+  };
+}

pyproject.toml

@@ -1,3 +1,10 @@
+[tool.ruff]
+target-version = "py311"
+line-length = 88
+
+select = ["E", "F", "I", "U"]
+ignore = [ "E501" ]
+
 [tool.mypy]
 python_version = "3.10"
 warn_redundant_casts = true