Utilize pydantic for serialization and deserialization

Signed-off-by: magic_rb <richard@brezak.sk>

buildbot_nix/__init__.py

@@ -5,7 +5,6 @@ import re
 import uuid
 from collections import defaultdict
 from collections.abc import Generator
-from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -33,13 +32,12 @@ from twisted.logger import Logger
 from .common import (
     slugify_project_name,
 )
-from .gitea_projects import GiteaBackend, GiteaConfig
+from .gitea_projects import GiteaBackend
 from .github_projects import (
     GithubBackend,
-    GithubConfig,
 )
+from .models import BuildbotNixConfig
 from .projects import GitBackend, GitProject
-from .secrets import read_secret_file
 
 SKIPPED_BUILDER_NAME = "skipped-builds"
 
@@ -428,21 +426,6 @@ def nix_eval_config(
     )
 
 
-@dataclass
-class CachixConfig:
-    name: str
-    signing_key_secret_name: str | None = None
-    auth_token_secret_name: str | None = None
-
-    def cachix_env(self) -> dict[str, str]:
-        env = {}
-        if self.signing_key_secret_name is not None:
-            env["CACHIX_SIGNING_KEY"] = util.Secret(self.signing_key_secret_name)
-        if self.auth_token_secret_name is not None:
-            env["CACHIX_AUTH_TOKEN"] = util.Secret(self.auth_token_secret_name)
-        return env
-
-
 @defer.inlineCallbacks
 def do_register_gcroot_if(s: steps.BuildStep) -> Generator[Any, object, Any]:
     gc_root = yield util.Interpolate(
@@ -847,53 +830,23 @@ class PeriodicWithStartup(schedulers.Periodic):
 class NixConfigurator(ConfiguratorBase):
     """Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
 
-    def __init__(
+    def __init__(self, config: BuildbotNixConfig) -> None:
-        self,
-        # Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
-        admins: list[str],
-        auth_backend: str,
-        build_retries: int,
-        github: GithubConfig | None,
-        gitea: GiteaConfig | None,
-        url: str,
-        nix_supported_systems: list[str],
-        nix_eval_worker_count: int | None,
-        nix_eval_max_memory_size: int,
-        post_build_steps: list[steps.BuildStep] | None = None,
-        nix_workers_secret_name: str = "buildbot-nix-workers",  # noqa: S107
-        cachix: CachixConfig | None = None,
-        outputs_path: str | None = None,
-    ) -> None:
         super().__init__()
-        self.nix_workers_secret_name = nix_workers_secret_name
+
-        self.nix_eval_max_memory_size = nix_eval_max_memory_size
+        self.config = config
-        self.nix_eval_worker_count = nix_eval_worker_count
-        self.nix_supported_systems = nix_supported_systems
-        self.post_build_steps = post_build_steps or []
-        self.auth_backend = auth_backend
-        self.admins = admins
-        self.github = github
-        self.gitea = gitea
-        self.url = url
-        self.cachix = cachix
-        self.build_retries = build_retries
-        if outputs_path is None:
-            self.outputs_path = None
-        else:
-            self.outputs_path = Path(outputs_path)
 
     def configure(self, config: dict[str, Any]) -> None:
         backends: dict[str, GitBackend] = {}
 
-        if self.github is not None:
+        if self.config.github is not None:
-            backends["github"] = GithubBackend(self.github, self.url)
+            backends["github"] = GithubBackend(self.config.github, self.config.url)
 
-        if self.gitea is not None:
+        if self.config.gitea is not None:
-            backends["gitea"] = GiteaBackend(self.gitea)
+            backends["gitea"] = GiteaBackend(self.config.gitea)
 
         auth: AuthBase | None = (
-            backends[self.auth_backend].create_auth()
+            backends[self.config.auth_backend].create_auth()
-            if self.auth_backend != "none"
+            if self.config.auth_backend != "none"
             else None
         )
 
@@ -902,7 +855,7 @@ class NixConfigurator(ConfiguratorBase):
         for backend in backends.values():
             projects += backend.load_projects()
 
-        worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
+        worker_config = json.loads(self.config.nix_workers_secret)
         worker_names = []
 
         config.setdefault("projects", [])
@@ -918,7 +871,7 @@ class NixConfigurator(ConfiguratorBase):
 
         eval_lock = util.MasterLock("nix-eval")
 
-        if self.cachix is not None:
+        if self.config.cachix is not None:
             self.post_build_steps.append(
                 steps.ShellCommand(
                     name="Upload cachix",
@@ -937,13 +890,13 @@ class NixConfigurator(ConfiguratorBase):
                 config,
                 project,
                 worker_names,
-                self.nix_supported_systems,
+                self.config.build_systems,
-                self.nix_eval_worker_count or multiprocessing.cpu_count(),
+                self.config.eval_worker_count or multiprocessing.cpu_count(),
-                self.nix_eval_max_memory_size,
+                self.config.eval_max_memory_size,
                 eval_lock,
-                self.post_build_steps,
+                [x.to_buildstep() for x in self.config.post_build_steps],
-                self.outputs_path,
+                self.config.outputs_path,
-                self.build_retries,
+                self.config.build_retries,
             )
 
         config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
@@ -998,7 +951,7 @@ class NixConfigurator(ConfiguratorBase):
                 config["www"]["auth"] = auth
 
             config["www"]["authz"] = setup_authz(
-                admins=self.admins,
+                admins=self.config.admins,
                 backends=list(backends.values()),
                 projects=projects,
             )

buildbot_nix/common.py

@@ -6,10 +6,12 @@ from abc import ABC, abstractmethod
 from collections.abc import Callable
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeVar
 
 if TYPE_CHECKING:
     from buildbot.process.log import StreamLog
+    from pydantic import BaseModel
+
 from collections.abc import Generator
 
 from buildbot.plugins import util
@@ -110,9 +112,12 @@ def atomic_write_file(file: Path, data: str) -> None:
             raise
 
 
+Y = TypeVar("Y")
+
+
 def filter_repos_by_topic(
-    topic: str | None, repos: list[Any], topics: Callable[[Any], list[str]]
+    topic: str | None, repos: list[Y], topics: Callable[[Y], list[str]]
-) -> list[Any]:
+) -> list[Y]:
     return list(
         filter(
             lambda repo: topic is None or topic in topics(repo),
@@ -154,3 +159,16 @@ class ThreadDeferredBuildStep(BuildStep, ABC):
             log: StreamLog = yield self.addLog("log")
             log.addStderr(f"Failed to reload project list: {self.error_msg}")
             return util.FAILURE
+
+
+_T = TypeVar("_T", bound="BaseModel")
+
+
+def model_validate_project_cache(cls: type[_T], project_cache_file: Path) -> list[_T]:
+    return [
+        cls.model_validate(data) for data in json.loads(project_cache_file.read_text())
+    ]
+
+
+def model_dump_project_cache(repos: list[_T]) -> str:
+    return json.dumps([repo.model_dump() for repo in repos])

buildbot_nix/gitea_projects.py

@@ -1,7 +1,5 @@
-import json
 import os
 import signal
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 from urllib.parse import urlparse
@@ -14,6 +12,7 @@ from buildbot.www.auth import AuthBase
 from buildbot.www.avatar import AvatarBase
 from buildbot_gitea.auth import GiteaAuth  # type: ignore[import]
 from buildbot_gitea.reporter import GiteaStatusPush  # type: ignore[import]
+from pydantic import BaseModel
 from twisted.logger import Logger
 from twisted.python import log
 
@@ -22,40 +21,37 @@ from .common import (
     atomic_write_file,
     filter_repos_by_topic,
     http_request,
+    model_dump_project_cache,
+    model_validate_project_cache,
     paginated_github_request,
     slugify_project_name,
 )
+from .models import GiteaConfig
 from .projects import GitBackend, GitProject
-from .secrets import read_secret_file
 
 tlog = Logger()
 
 
-@dataclass
+class RepoOwnerData(BaseModel):
-class GiteaConfig:
+    login: str
-    instance_url: str
-    oauth_id: str | None
 
-    oauth_secret_name: str = "gitea-oauth-secret"
-    token_secret_name: str = "gitea-token"
-    webhook_secret_name: str = "gitea-webhook-secret"
-    project_cache_file: Path = Path("gitea-project-cache.json")
-    topic: str | None = "build-with-buildbot"
 
-    def oauth_secret(self) -> str:
+class RepoData(BaseModel):
-        return read_secret_file(self.oauth_secret_name)
+    name: str
-
+    owner: RepoOwnerData
-    def token(self) -> str:
+    full_name: str
-        return read_secret_file(self.token_secret_name)
+    ssh_url: str
+    default_branch: str
+    topics: list[str]
 
 
 class GiteaProject(GitProject):
     config: GiteaConfig
     webhook_secret: str
-    data: dict[str, Any]
+    data: RepoData
 
     def __init__(
-        self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any]
+        self, config: GiteaConfig, webhook_secret: str, data: RepoData
     ) -> None:
         self.config = config
         self.webhook_secret = webhook_secret
@@ -63,7 +59,7 @@ class GiteaProject(GitProject):
 
     def get_project_url(self) -> str:
         url = urlparse(self.config.instance_url)
-        return f"{url.scheme}://git:%(secret:{self.config.token_secret_name})s@{url.hostname}/{self.name}"
+        return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.name}"
 
     @property
     def pretty_type(self) -> str:
@@ -75,33 +71,33 @@ class GiteaProject(GitProject):
 
     @property
     def repo(self) -> str:
-        return self.data["name"]
+        return self.data.name
 
     @property
     def owner(self) -> str:
-        return self.data["owner"]["login"]
+        return self.data.owner.login
 
     @property
     def name(self) -> str:
-        return self.data["full_name"]
+        return self.data.full_name
 
     @property
     def url(self) -> str:
         # not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34
-        return self.data["ssh_url"]
+        return self.data.ssh_url
 
     @property
     def project_id(self) -> str:
-        return slugify_project_name(self.data["full_name"])
+        return slugify_project_name(self.data.full_name)
 
     @property
     def default_branch(self) -> str:
-        return self.data["default_branch"]
+        return self.data.default_branch
 
     @property
     def topics(self) -> list[str]:
         # note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects`
-        return self.data["topics"]
+        return self.data.topics
 
     @property
     def belongs_to_org(self) -> bool:
@@ -111,11 +107,9 @@ class GiteaProject(GitProject):
 
 class GiteaBackend(GitBackend):
     config: GiteaConfig
-    webhook_secret: str
 
     def __init__(self, config: GiteaConfig) -> None:
         self.config = config
-        self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
 
     def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
         """Updates the flake an opens a PR for it."""
@@ -124,9 +118,7 @@ class GiteaBackend(GitBackend):
             ReloadGiteaProjects(self.config, self.config.project_cache_file),
         )
         factory.addStep(
-            CreateGiteaProjectHooks(
+            CreateGiteaProjectHooks(self.config),
-                self.config, self.config.project_cache_file, self.webhook_secret
-            ),
         )
         return util.BuilderConfig(
             name=self.reload_builder_name,
@@ -137,14 +129,14 @@ class GiteaBackend(GitBackend):
     def create_reporter(self) -> ReporterBase:
         return GiteaStatusPush(
             self.config.instance_url,
-            Interpolate(self.config.token()),
+            Interpolate(self.config.token),
             context=Interpolate("buildbot/%(prop:status_name)s"),
             context_pr=Interpolate("buildbot/%(prop:status_name)s"),
         )
 
     def create_change_hook(self) -> dict[str, Any]:
         return {
-            "secret": self.webhook_secret,
+            "secret": self.config.webhook_secret,
             # The "mergable" field is a bit buggy,
             # we already do the merge locally anyway.
             "onlyMergeablePullRequest": False,
@@ -158,29 +150,32 @@ class GiteaBackend(GitBackend):
         return GiteaAuth(
             self.config.instance_url,
             self.config.oauth_id,
-            self.config.oauth_secret(),
+            self.config.oauth_secret,
         )
 
     def load_projects(self) -> list["GitProject"]:
         if not self.config.project_cache_file.exists():
             return []
 
-        repos: list[dict[str, Any]] = filter_repos_by_topic(
+        repos: list[RepoData] = filter_repos_by_topic(
             self.config.topic,
             sorted(
-                json.loads(self.config.project_cache_file.read_text()),
+                model_validate_project_cache(RepoData, self.config.project_cache_file),
-                key=lambda x: x["full_name"],
+                key=lambda repo: repo.full_name,
             ),
-            lambda repo: repo["topics"],
+            lambda repo: repo.topics,
         )
-        repo_names: list[str] = [
+        repo_names: list[str] = [repo.owner.login + "/" + repo.name for repo in repos]
-            repo["owner"]["login"] + "/" + repo["name"] for repo in repos
-        ]
         tlog.info(
             f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
         )
 
-        return [GiteaProject(self.config, self.webhook_secret, repo) for repo in repos]
+        return [
+            GiteaProject(
+                self.config, self.config.webhook_secret, RepoData.model_validate(repo)
+            )
+            for repo in repos
+        ]
 
     def are_projects_cached(self) -> bool:
         return self.config.project_cache_file.exists()
@@ -245,30 +240,24 @@ class CreateGiteaProjectHooks(ThreadDeferredBuildStep):
     name = "create_gitea_project_hooks"
 
     config: GiteaConfig
-    project_cache_file: Path
-    webhook_secret: str
 
     def __init__(
         self,
         config: GiteaConfig,
-        project_cache_file: Path,
-        webhook_secret: str,
         **kwargs: Any,
     ) -> None:
         self.config = config
-        self.project_cache_file = project_cache_file
-        self.webhook_secret = webhook_secret
         super().__init__(**kwargs)
 
     def run_deferred(self) -> None:
-        repos = json.loads(self.project_cache_file.read_text())
+        repos = model_validate_project_cache(RepoData, self.config.project_cache_file)
 
         for repo in repos:
             create_repo_hook(
-                self.config.token(),
+                self.config.token,
-                self.webhook_secret,
+                self.config.webhook_secret,
-                repo["owner"]["login"],
+                repo.owner.login,
-                repo["name"],
+                repo.name,
                 self.config.instance_url,
             )
 
@@ -294,24 +283,24 @@ class ReloadGiteaProjects(ThreadDeferredBuildStep):
         super().__init__(**kwargs)
 
     def run_deferred(self) -> None:
-        repos = filter_repos_by_topic(
+        repos: list[RepoData] = filter_repos_by_topic(
             self.config.topic,
             refresh_projects(self.config, self.project_cache_file),
-            lambda repo: repo["topics"],
+            lambda repo: repo.topics,
         )
 
-        atomic_write_file(self.project_cache_file, json.dumps(repos))
+        atomic_write_file(self.project_cache_file, model_dump_project_cache(repos))
 
     def run_post(self) -> Any:
         return util.SUCCESS
 
 
-def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]:
+def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[RepoData]:
     repos = []
 
     for repo in paginated_github_request(
         f"{config.instance_url}/api/v1/user/repos?limit=100",
-        config.token(),
+        config.token,
     ):
         if not repo["permissions"]["admin"]:
             name = repo["full_name"]
@@ -323,10 +312,10 @@ def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]:
                 # Gitea doesn't include topics in the default repo listing, unlike GitHub
                 topics: list[str] = http_request(
                     f"{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics",
-                    headers={"Authorization": f"token {config.token()}"},
+                    headers={"Authorization": f"token {config.token}"},
                 ).json()["topics"]
                 repo["topics"] = topics
-                repos.append(repo)
+                repos.append(RepoData.model_validate(repo))
             except OSError:
                 pass
 

buildbot_nix/github/auth/__init__.py

@@ -1 +0,0 @@
-

buildbot_nix/github/auth/_type.py

@@ -1,23 +0,0 @@
-from dataclasses import dataclass
-from pathlib import Path
-
-
-@dataclass
-class AuthType:
-    pass
-
-
-@dataclass
-class AuthTypeLegacy(AuthType):
-    token_secret_name: str = "github-token"
-
-
-@dataclass
-class AuthTypeApp(AuthType):
-    app_id: int
-    app_secret_key_name: str = "github-app-secret-key"
-    app_installation_token_map_name: Path = Path(
-        "github-app-installation-token-map.json"
-    )
-    app_project_id_map_name: Path = Path("github-app-project-id-map-name.json")
-    app_jwt_token_name: Path = Path("github-app-jwt-token")

buildbot_nix/github/jwt_token.py

@@ -3,6 +3,7 @@ import json
 import os
 import subprocess
 from datetime import UTC, datetime, timedelta
+from pathlib import Path
 from typing import Any
 
 from .repo_token import RepoToken
@@ -10,7 +11,7 @@ from .repo_token import RepoToken
 
 class JWTToken(RepoToken):
     app_id: int
-    app_private_key: str
+    app_private_key_file: Path
     lifetime: timedelta
 
     expiration: datetime
@@ -19,20 +20,20 @@ class JWTToken(RepoToken):
     def __init__(
         self,
         app_id: int,
-        app_private_key: str,
+        app_private_key_file: Path,
         lifetime: timedelta = timedelta(minutes=10),
     ) -> None:
         self.app_id = app_id
-        self.app_private_key = app_private_key
+        self.app_private_key_file = app_private_key_file
         self.lifetime = lifetime
 
         self.token, self.expiration = JWTToken.generate_token(
-            self.app_id, self.app_private_key, lifetime
+            self.app_id, self.app_private_key_file, lifetime
         )
 
     @staticmethod
     def generate_token(
-        app_id: int, app_private_key: str, lifetime: timedelta
+        app_id: int, app_private_key_file: Path, lifetime: timedelta
     ) -> tuple[str, datetime]:
         def build_jwt_payload(
             app_id: int, lifetime: timedelta
@@ -48,9 +49,9 @@ class JWTToken(RepoToken):
             }
             return (jwt_payload, exp)
 
-        def rs256_sign(data: str, private_key: str) -> str:
+        def rs256_sign(data: str, private_key_file: Path) -> str:
             signature = subprocess.run(
-                ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key],
+                ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key_file],
                 input=data.encode("utf-8"),
                 stdout=subprocess.PIPE,
                 check=True,
@@ -65,7 +66,7 @@ class JWTToken(RepoToken):
         jwt_payload = json.dumps(jwt).encode("utf-8")
         json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8")
         encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}"
-        encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key)
+        encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key_file)
         return (f"{encoded_jwt_parts}.{encoded_mac}", expiration)
 
         # installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt)
@@ -75,7 +76,7 @@ class JWTToken(RepoToken):
     def get(self) -> str:
         if self.expiration - datetime.now(tz=UTC) < self.lifetime * 0.2:
             self.token, self.expiration = JWTToken.generate_token(
-                self.app_id, self.app_private_key, self.lifetime
+                self.app_id, self.app_private_key_file, self.lifetime
             )
 
         return self.token

buildbot_nix/github_projects.py

@@ -17,6 +17,7 @@ from buildbot.secrets.providers.base import SecretProviderBase
 from buildbot.www.auth import AuthBase
 from buildbot.www.avatar import AvatarBase, AvatarGitHub
 from buildbot.www.oauth2 import GitHubAuth
+from pydantic import BaseModel, ConfigDict, Field
 from twisted.logger import Logger
 from twisted.python import log
 
@@ -25,10 +26,11 @@ from .common import (
     atomic_write_file,
     filter_repos_by_topic,
     http_request,
+    model_dump_project_cache,
+    model_validate_project_cache,
     paginated_github_request,
     slugify_project_name,
 )
-from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy
 from .github.installation_token import InstallationToken
 from .github.jwt_token import JWTToken
 from .github.legacy_token import (
@@ -37,12 +39,33 @@ from .github.legacy_token import (
 from .github.repo_token import (
     RepoToken,
 )
+from .models import (
+    GitHubAppConfig,
+    GitHubConfig,
+    GitHubLegacyConfig,
+)
 from .projects import GitBackend, GitProject
-from .secrets import read_secret_file
 
 tlog = Logger()
 
 
+class RepoOwnerData(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    login: str
+    ttype: str = Field(alias="type")
+
+
+class RepoData(BaseModel):
+    name: str
+    owner: RepoOwnerData
+    full_name: str
+    html_url: str
+    default_branch: str
+    topics: list[str]
+    installation_id: int | None
+
+
 def get_installations(jwt_token: JWTToken) -> list[int]:
     installations = paginated_github_request(
         "https://api.github.com/app/installations?per_page=100", jwt_token.get()
@@ -80,7 +103,7 @@ class CreateGitHubInstallationHooks(ThreadDeferredBuildStep):
         super().__init__(**kwargs)
 
     def run_deferred(self) -> None:
-        repos = json.loads(self.project_cache_file.read_text())
+        repos = model_validate_project_cache(RepoData, self.project_cache_file)
         installation_token_map: dict[int, InstallationToken] = dict(
             starmap(
                 lambda k, v: (
@@ -94,11 +117,14 @@ class CreateGitHubInstallationHooks(ThreadDeferredBuildStep):
         )
 
         for repo in repos:
+            if repo.installation_id is None:
+                continue
+
             create_project_hook(
-                installation_token_map[repo["installation_id"]],
+                installation_token_map[repo.installation_id],
                 self.webhook_secret,
-                repo["owner"]["login"],
+                repo.owner.login,
-                repo["name"],
+                repo.name,
                 self.webhook_url,
             )
 
@@ -144,7 +170,7 @@ class ReloadGithubInstallations(ThreadDeferredBuildStep):
             get_installations(self.jwt_token),
         )
 
-        repos: list[Any] = []
+        repos: list[RepoData] = []
         project_id_map: dict[str, int] = {}
 
         repos = []
@@ -160,17 +186,17 @@ class ReloadGithubInstallations(ThreadDeferredBuildStep):
                     subkey="repositories",
                     require_admin=False,
                 ),
-                lambda repo: repo["topics"],
+                lambda repo: repo.topics,
             )
 
             for repo in new_repos:
-                repo["installation_id"] = k
+                repo.installation_id = k
 
-                project_id_map[repo["full_name"]] = k
+                project_id_map[repo.full_name] = k
 
             repos.extend(new_repos)
 
-        atomic_write_file(self.project_cache_file, json.dumps(repos))
+        atomic_write_file(self.project_cache_file, model_dump_project_cache(repos))
         atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
 
         tlog.info(
@@ -207,14 +233,14 @@ class CreateGitHubProjectHooks(ThreadDeferredBuildStep):
         super().__init__(**kwargs)
 
     def run_deferred(self) -> None:
-        repos = json.loads(self.project_cache_file.read_text())
+        repos = model_validate_project_cache(RepoData, self.project_cache_file)
 
         for repo in repos:
             create_project_hook(
                 self.token,
                 self.webhook_secret,
-                repo["owner"]["login"],
+                repo.owner.login,
-                repo["name"],
+                repo.name,
                 self.webhook_url,
             )
 
@@ -244,13 +270,13 @@ class ReloadGithubProjects(ThreadDeferredBuildStep):
         super().__init__(**kwargs)
 
     def run_deferred(self) -> None:
-        repos: list[Any] = filter_repos_by_topic(
+        repos: list[RepoData] = filter_repos_by_topic(
             self.topic,
             refresh_projects(self.token.get(), self.project_cache_file),
-            lambda repo: repo["topics"],
+            lambda repo: repo.topics,
         )
 
-        atomic_write_file(self.project_cache_file, json.dumps(repos))
+        atomic_write_file(self.project_cache_file, model_dump_project_cache(repos))
 
     def run_post(self) -> Any:
         return util.SUCCESS
@@ -262,7 +288,7 @@ class GithubAuthBackend(ABC):
         pass
 
     @abstractmethod
-    def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
+    def get_repo_token(self, repo: RepoData) -> RepoToken:
         pass
 
     @abstractmethod
@@ -285,18 +311,18 @@ class GithubAuthBackend(ABC):
 
 
 class GithubLegacyAuthBackend(GithubAuthBackend):
-    auth_type: AuthTypeLegacy
+    auth_type: GitHubLegacyConfig
 
     token: LegacyToken
 
-    def __init__(self, auth_type: AuthTypeLegacy) -> None:
+    def __init__(self, auth_type: GitHubLegacyConfig) -> None:
         self.auth_type = auth_type
-        self.token = LegacyToken(read_secret_file(auth_type.token_secret_name))
+        self.token = LegacyToken(auth_type.token)
 
     def get_general_token(self) -> RepoToken:
         return self.token
 
-    def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
+    def get_repo_token(self, repo: RepoData) -> RepoToken:
         return self.token
 
     def create_secret_providers(self) -> list[SecretProviderBase]:
@@ -351,24 +377,22 @@ class GitHubLegacySecretService(SecretProviderBase):
 
 
 class GithubAppAuthBackend(GithubAuthBackend):
-    auth_type: AuthTypeApp
+    auth_type: GitHubAppConfig
 
     jwt_token: JWTToken
     installation_tokens: dict[int, InstallationToken]
     project_id_map: dict[str, int]
 
-    def __init__(self, auth_type: AuthTypeApp) -> None:
+    def __init__(self, auth_type: GitHubAppConfig) -> None:
         self.auth_type = auth_type
-        self.jwt_token = JWTToken(
+        self.jwt_token = JWTToken(self.auth_type.id, self.auth_type.secret_key_file)
-            self.auth_type.app_id, self.auth_type.app_secret_key_name
-        )
         self.installation_tokens = GithubBackend.load_installations(
             self.jwt_token,
-            self.auth_type.app_installation_token_map_name,
+            self.auth_type.installation_token_map_file,
         )
-        if self.auth_type.app_project_id_map_name.exists():
+        if self.auth_type.project_id_map_file.exists():
             self.project_id_map = json.loads(
-                self.auth_type.app_project_id_map_name.read_text()
+                self.auth_type.project_id_map_file.read_text()
             )
         else:
             tlog.info(
@@ -379,9 +403,9 @@ class GithubAppAuthBackend(GithubAuthBackend):
     def get_general_token(self) -> RepoToken:
         return self.jwt_token
 
-    def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
+    def get_repo_token(self, repo: RepoData) -> RepoToken:
-        assert "installation_id" in repo, f"Missing installation_id in {repo}"
+        assert repo.installation_id is not None, f"Missing installation_id in {repo}"
-        return self.installation_tokens[repo["installation_id"]]
+        return self.installation_tokens[repo.installation_id]
 
     def create_secret_providers(self) -> list[SecretProviderBase]:
         return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)]
@@ -411,14 +435,14 @@ class GithubAppAuthBackend(GithubAuthBackend):
             ReloadGithubInstallations(
                 self.jwt_token,
                 project_cache_file,
-                self.auth_type.app_installation_token_map_name,
+                self.auth_type.installation_token_map_file,
-                self.auth_type.app_project_id_map_name,
+                self.auth_type.project_id_map_file,
                 topic,
             ),
             CreateGitHubInstallationHooks(
                 self.jwt_token,
                 project_cache_file,
-                self.auth_type.app_installation_token_map_name,
+                self.auth_type.installation_token_map_file,
                 webhook_secret=webhook_secret,
                 webhook_url=webhook_url,
                 topic=topic,
@@ -450,32 +474,22 @@ class GitHubAppSecretService(SecretProviderBase):
         return None
 
 
-@dataclass
-class GithubConfig:
-    oauth_id: str | None
-    auth_type: AuthType
-    oauth_secret_name: str = "github-oauth-secret"
-    webhook_secret_name: str = "github-webhook-secret"
-    project_cache_file: Path = Path("github-project-cache-v1.json")
-    topic: str | None = "build-with-buildbot"
-
-
 @dataclass
 class GithubBackend(GitBackend):
-    config: GithubConfig
+    config: GitHubConfig
     webhook_secret: str
     webhook_url: str
 
     auth_backend: GithubAuthBackend
 
-    def __init__(self, config: GithubConfig, webhook_url: str) -> None:
+    def __init__(self, config: GitHubConfig, webhook_url: str) -> None:
         self.config = config
-        self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
+        self.webhook_secret = self.config.webhook_secret
         self.webhook_url = webhook_url
 
-        if isinstance(self.config.auth_type, AuthTypeLegacy):
+        if isinstance(self.config.auth_type, GitHubLegacyConfig):
             self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
-        elif isinstance(self.config.auth_type, AuthTypeApp):
+        elif isinstance(self.config.auth_type, GitHubAppConfig):
             self.auth_backend = GithubAppAuthBackend(self.config.auth_type)
 
     @staticmethod
@@ -551,7 +565,7 @@ class GithubBackend(GitBackend):
         assert self.config.oauth_id is not None, "GitHub OAuth ID is required"
         return GitHubAuth(
             self.config.oauth_id,
-            read_secret_file(self.config.oauth_secret_name),
+            self.config.oauth_secret,
             apiVersion=4,
         )
 
@@ -562,30 +576,28 @@ class GithubBackend(GitBackend):
         if not self.config.project_cache_file.exists():
             return []
 
-        repos: list[dict[str, Any]] = filter_repos_by_topic(
+        repos: list[RepoData] = filter_repos_by_topic(
             self.config.topic,
             sorted(
-                json.loads(self.config.project_cache_file.read_text()),
+                model_validate_project_cache(RepoData, self.config.project_cache_file),
-                key=lambda x: x["full_name"],
+                key=lambda repo: repo.full_name,
             ),
-            lambda repo: repo["topics"],
+            lambda repo: repo.topics,
         )
 
         if isinstance(self.auth_backend, GithubAppAuthBackend):
             dropped_repos = list(
-                filter(lambda repo: "installation_id" not in repo, repos)
+                filter(lambda repo: repo.installation_id is None, repos)
             )
             if dropped_repos:
                 tlog.info(
                     "Dropped projects follow, refresh will follow after initialisation:"
                 )
                 for dropped_repo in dropped_repos:
-                    tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
+                    tlog.info(f"\tDropping repo {dropped_repo.full_name}")
-            repos = list(filter(lambda repo: "installation_id" in repo, repos))
+            repos = list(filter(lambda repo: repo.installation_id is not None, repos))
 
-        repo_names: list[str] = [
+        repo_names: list[str] = [repo.owner.login + "/" + repo.name for repo in repos]
-            repo["owner"]["login"] + "/" + repo["name"] for repo in repos
-        ]
 
         tlog.info(
             f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
@@ -595,7 +607,7 @@ class GithubBackend(GitBackend):
                 self.auth_backend.get_repo_token(repo),
                 self.config,
                 self.webhook_secret,
-                repo,
+                RepoData.model_validate(repo),
             )
             for repo in repos
         ]
@@ -605,14 +617,16 @@ class GithubBackend(GitBackend):
             return False
 
         if (
-            isinstance(self.config.auth_type, AuthTypeApp)
+            isinstance(self.config.auth_type, GitHubAppConfig)
-            and not self.config.auth_type.app_project_id_map_name.exists()
+            and not self.config.auth_type.project_id_map_file.exists()
         ):
             return False
 
         all_have_installation_id = True
-        for project in json.loads(self.config.project_cache_file.read_text()):
+        for project in model_validate_project_cache(
-            if "installation_id" not in project:
+            RepoData, self.config.project_cache_file
+        ):
+            if project.installation_id is not None:
                 all_have_installation_id = False
                 break
 
@@ -669,17 +683,17 @@ def create_project_hook(
 
 
 class GithubProject(GitProject):
-    config: GithubConfig
+    config: GitHubConfig
     webhook_secret: str
-    data: dict[str, Any]
+    data: RepoData
     token: RepoToken
 
     def __init__(
         self,
         token: RepoToken,
-        config: GithubConfig,
+        config: GitHubConfig,
         webhook_secret: str,
-        data: dict[str, Any],
+        data: RepoData,
     ) -> None:
         self.token = token
         self.config = config
@@ -699,35 +713,35 @@ class GithubProject(GitProject):
 
     @property
     def repo(self) -> str:
-        return self.data["name"]
+        return self.data.name
 
     @property
     def owner(self) -> str:
-        return self.data["owner"]["login"]
+        return self.data.owner.login
 
     @property
     def name(self) -> str:
-        return self.data["full_name"]
+        return self.data.full_name
 
     @property
     def url(self) -> str:
-        return self.data["html_url"]
+        return self.data.html_url
 
     @property
     def project_id(self) -> str:
-        return slugify_project_name(self.data["full_name"])
+        return slugify_project_name(self.data.full_name)
 
     @property
     def default_branch(self) -> str:
-        return self.data["default_branch"]
+        return self.data.default_branch
 
     @property
     def topics(self) -> list[str]:
-        return self.data["topics"]
+        return self.data.topics
 
     @property
     def belongs_to_org(self) -> bool:
-        return self.data["owner"]["type"] == "Organization"
+        return self.data.owner.ttype == "Organization"
 
 
 def refresh_projects(
@@ -738,7 +752,7 @@ def refresh_projects(
     api_endpoint: str = "/user/repos",
     subkey: None | str = None,
     require_admin: bool = True,
-) -> list[Any]:
+) -> list[RepoData]:
     if repos is None:
         repos = []
 
@@ -754,6 +768,7 @@ def refresh_projects(
                 f"skipping {name} because we do not have admin privileges, needed for hook management",
             )
         else:
-            repos.append(repo)
+            repo["installation_id"] = None
+            repos.append(RepoData.model_validate(repo))
 
     return repos

buildbot_nix/models.py

@@ -0,0 +1,182 @@
+from enum import Enum
+from pathlib import Path
+
+from pydantic import BaseModel, Field, field_serializer, field_validator, ConfigDict
+from buildbot.plugins import util, steps
+
+from .secrets import read_secret_file
+
+class InternalIssue(Exception):
+    pass
+
+
+def exclude_fields(fields: list[str]) -> dict[str, dict[str, bool]]:
+    return dict(map(lambda k: (k, {"exclude": True}), fields))
+
+class AuthBackendConfig(str, Enum):
+    github = "github"
+    gitea = "gitea"
+    none = "none"
+
+class CachixConfig(BaseModel):
+    name: str
+
+    signing_key_file: Path | None
+    auth_token_file: Path | None
+
+    @property
+    def signing_key(self) -> str:
+        if self.signing_key_file is None:
+            raise InternalIssue
+        return read_secret_file(self.signing_key_file)
+
+    @property
+    def auth_token(self) -> str:
+        if self.auth_token_file is None:
+            raise InternalIssue
+        return read_secret_file(self.auth_token_file)
+
+    # TODO why did the original implementation return an empty env if both files were missing?
+    @property
+    def environment(self) -> dict[str, str]:
+        environment = {}
+        environment["CACHIX_SIGNING_KEY"] = util.Secret(self.signing_key_file)
+        environment["CACHIX_AUTH_TOKEN"] = util.Secret(self.auth_token_file)
+        return environment
+
+    class Config:
+        fields = exclude_fields(["singing_key", "auth_token"])
+
+class GiteaConfig(BaseModel):
+    instance_url: str
+    topic: str | None
+
+    token_file: Path = Field(default = Path("gitea-token"))
+    webhook_secret_file: Path = Field(default = Path("gitea-webhook-secret"))
+    project_cache_file: Path = Field(default = Path("gitea-project-cache.json"))
+
+    oauth_id: str | None
+    oauth_secret_file: Path | None
+
+    @property
+    def token(self) -> str:
+        return read_secret_file(self.token_file)
+
+    @property
+    def webhook_secret(self) -> str:
+        return read_secret_file(self.webhook_secret_file)
+
+    @property
+    def oauth_secret(self) -> str:
+        if self.oauth_secret_file is None:
+            raise InternalIssue
+        return read_secret_file(self.oauth_secret_file)
+
+    class Config:
+        fields = exclude_fields(["token", "webhook_secret", "oauth_secret"])
+
+class GitHubLegacyConfig(BaseModel):
+    token_file: Path
+
+    @property
+    def token(self) -> str:
+        return read_secret_file(self.token_file)
+
+    class Config:
+        fields = exclude_fields(["token"])
+
+class GitHubAppConfig(BaseModel):
+    id: int
+
+    secret_key_file: Path
+    installation_token_map_file: Path = Field(default = Path(
+        "github-app-installation-token-map.json"
+    ))
+    project_id_map_file: Path = Field(default = Path(
+        "github-app-project-id-map-name.json"
+    ))
+    jwt_token_map: Path = Field(default = Path(
+        "github-app-jwt-token"
+    ))
+
+    @property
+    def secret_key(self) -> str:
+        return read_secret_file(self.secret_key_file)
+
+    class Config:
+        fields = exclude_fields(["secret_key"])
+
+class GitHubConfig(BaseModel):
+    auth_type: GitHubLegacyConfig | GitHubAppConfig
+    topic: str | None
+
+    project_cache_file: Path = Field(default = Path("github-project-cache-v1.json"))
+    webhook_secret_file: Path = Field(default = Path("github-webhook-secret"))
+
+    oauth_id: str | None
+    oauth_secret_file: Path | None
+
+    @property
+    def webhook_secret(self) -> str:
+        return read_secret_file(self.webhook_secret_file)
+
+    @property
+    def oauth_secret(self) -> str:
+        if self.oauth_secret_file is None:
+            raise InternalIssue
+        return read_secret_file(self.oauth_secret_file)
+
+# note that serialization isn't correct, as there is no way to *rename* the field `nix_type` to `_type`,
+# one must always specify `by_alias = True`, such as `model_dump(by_alias = True)`, relevant issue:
+# https://github.com/pydantic/pydantic/issues/8379
+class Interpolate(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    nix_type: str = Field(alias = "_type")
+    value: str
+
+
+class PostBuildStep(BaseModel):
+    name: str
+    environment: dict[str, str | Interpolate]
+    command: list[str | Interpolate]
+
+    def to_buildstep(self) -> steps.BuildStep:
+        def maybe_interpolate(value: str | Interpolate) -> str | util.Interpolate:
+            if isinstance(value, str):
+                return value
+            return util.Interpolate(value.value)
+
+        return steps.ShellCommand(
+            name = self.name,
+            env = {
+                k: maybe_interpolate(k) for k in self.environment
+            },
+            command = [
+                maybe_interpolate(x) for x in self.command
+            ]
+        )
+
+class BuildbotNixConfig(BaseModel):
+    db_url: str
+    auth_backend: AuthBackendConfig
+    build_retries: int
+    cachix: CachixConfig | None
+    gitea: GiteaConfig | None
+    github: GitHubConfig | None
+    admins: list[str]
+    workers_file: Path
+    build_systems: list[str]
+    eval_max_memory_size: int
+    eval_worker_count: int | None
+    nix_workers_secret_file: Path = Field(default = Path("buildbot-nix-workers"))
+    domain: str
+    webhook_base_url: str
+    use_https: bool
+    outputs_path: Path | None
+    url: str
+    post_build_steps: list[PostBuildStep]
+
+    @property
+    def nix_workers_secret(self) -> str:
+        return read_secret_file(self.nix_workers_secret_file)

buildbot_nix/secrets.py

@@ -3,9 +3,9 @@ import sys
 from pathlib import Path
 
 
-def read_secret_file(secret_name: str) -> str:
+def read_secret_file(secret_file: Path) -> str:
     directory = os.environ.get("CREDENTIALS_DIRECTORY")
     if directory is None:
         print("directory not set", file=sys.stderr)
         sys.exit(1)
-    return Path(directory).joinpath(secret_name).read_text().rstrip()
+    return Path(directory).joinpath(secret_file).read_text().rstrip()

nix/master.nix

@@ -154,21 +154,20 @@ in
       };
 
       cachix = {
+        enable = lib.mkEnableOption "Enable Cachix integration";
+
         name = lib.mkOption {
-          type = lib.types.nullOr lib.types.str;
+          type = lib.types.str;
-          default = null;
           description = "Cachix name";
         };
 
         signingKeyFile = lib.mkOption {
-          type = lib.types.nullOr lib.types.path;
+          type = lib.types.path;
-          default = null;
           description = "Cachix signing key";
         };
 
         authTokenFile = lib.mkOption {
-          type = lib.types.nullOr lib.types.str;
+          type = lib.types.str;
-          default = null;
           description = "Cachix auth token";
         };
       };
@@ -291,8 +290,8 @@ in
         description = "Systems that we will be build";
       };
       evalMaxMemorySize = lib.mkOption {
-        type = lib.types.str;
+        type = lib.types.int;
-        default = "2048";
+        default = 2048;
         description = ''
           Maximum memory size for nix-eval-jobs (in MiB) per
           worker. After the limit is reached, the worker is
@@ -364,19 +363,28 @@ in
         }
         {
           assertion =
-            cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
+            cfg.authBackend == "github" -> (cfg.github.oauthId != null && cfg.github.oauthSecretFile != null);
-          message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
-        }
-        {
-          assertion =
-            cfg.authBackend != "github" || (cfg.github.oauthId != null && cfg.github.oauthSecretFile != null);
           message = ''If config.services.buildbot-nix.master.authBackend is set to "github", then config.services.buildbot-nix.master.github.oauthId and config.services.buildbot-nix.master.github.oauthSecretFile have to be set.'';
         }
         {
           assertion =
-            cfg.authBackend != "gitea" || (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null);
+            cfg.authBackend == "gitea" -> (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null);
           message = ''config.services.buildbot-nix.master.authBackend is set to "gitea", then config.services.buildbot-nix.master.gitea.oauthId and config.services.buildbot-nix.master.gitea.oauthSecretFile have to be set.'';
         }
+        {
+          assertion =
+            cfg.authBackend == "github" -> cfg.github.enable;
+          message = ''
+            If `cfg.authBackend` is set to `"github"` the GitHub backend must be enabled with `cfg.github.enable`;
+          '';
+        }
+        {
+          assertion =
+            cfg.authBackend == "gitea" -> cfg.gitea.enable;
+          message = ''
+            If `cfg.authBackend` is set to `"gitea"` the GitHub backend must be enabled with `cfg.gitea.enable`;
+          '';
+        }
       ];
 
       services.buildbot-master = {
@@ -391,19 +399,11 @@ in
         extraImports = ''
           from datetime import timedelta
           from buildbot_nix import (
-            GithubConfig,
             NixConfigurator,
-            CachixConfig,
+            BuildbotNixConfig,
-            GiteaConfig,
-          )
-          from buildbot.plugins import (
-            steps,
-            util,
-          )
-          from buildbot_nix.github.auth._type import (
-            AuthTypeLegacy,
-            AuthTypeApp,
           )
+          from pathlib import Path
+          import json
         '';
         configurators = [
           ''
@@ -411,78 +411,65 @@ in
           ''
           ''
             NixConfigurator(
-                auth_backend=${builtins.toJSON cfg.authBackend},
+              BuildbotNixConfig.model_validate(json.loads(Path("${(pkgs.formats.json {}).generate "buildbot-nix-config.json" {
-                github=${
+                db_url = cfg.dbUrl;
-                  if (!cfg.github.enable) then
+                auth_backend = cfg.authBackend;
-                    "None"
+                build_retries = cfg.buildRetries;
-                  else
+                cachix = if !cfg.cachix.enable then
-                    "GithubConfig(
+                  null
-                    oauth_id=${builtins.toJSON cfg.github.oauthId},
+                         else
-                    topic=${builtins.toJSON cfg.github.topic},
+                           {
-                    auth_type=${
+                             name = cfg.cachix.name;
-                            if cfg.github.authType ? "legacy" then
+                             signing_key_file = cfg.cachix.signingKeyFile;
-                              ''AuthTypeLegacy()''
+                             auth_token_file = cfg.cachix.authTokenFile;
-                            else if cfg.github.authType ? "app" then
+                           };
-                              ''
+                gitea = if !cfg.gitea.enable then
-                                AuthTypeApp(
+                  null
-                                  app_id=${toString cfg.github.authType.app.id},
+                        else
-                                )
+                          {
-                              ''
+                            token_file = "gitea-token";
-                            else
+                            webhook_secret_file = "gitea-webhook-secret";
-                              throw "One of AuthTypeApp or AuthTypeLegacy must be enabled"
+                            project_cache_file = "gitea-project-cache.json";
-                          }
+                            oauth_secret_file = "gitea-oauth-secret";
-                )"
+                            instance_url = cfg.gitea.instanceUrl;
-                },
+                            oauth_id = cfg.gitea.oauthId;
-                gitea=${
+                            topic = cfg.gitea.topic;
-                  if !cfg.gitea.enable then
+                          };
-                    "None"
+                github = if !cfg.github.enable then
-                  else
+                  null
-                    "GiteaConfig(
+                         else {
-                    instance_url=${builtins.toJSON cfg.gitea.instanceUrl},
+                           auth_type = if (cfg.github.authType ? "legacy") then
-                    oauth_id=${builtins.toJSON cfg.gitea.oauthId},
+                             {
-                    topic=${builtins.toJSON cfg.gitea.topic},
+                               token_file = "github-token";
-                )"
+                             }
-                },
+                                       else if (cfg.github.authType ? "app") then
-                build_retries=${builtins.toJSON cfg.buildRetries},
+                                         {
-                cachix=${
+                                           id = cfg.github.authType.app.id;
-                  if cfg.cachix.name == null then
+                                           secret_key_file = cfg.github.authType.app.secretKeyFile;
-                    "None"
+                                           installation_token_map_file = "github-app-installation-token-map.json";
-                  else
+                                           project_id_map_file = "github-app-project-id-map-name.json";
-                    "CachixConfig(
+                                           jwt_token_map = "github-app-jwt-token";
-                    name=${builtins.toJSON cfg.cachix.name},
+                                         }
-                    signing_key_secret_name=${
+                                       else
-                                        if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"
+                                         throw "authType is neither \"legacy\" nor \"app\"";
-                                      },
+                           project_cache_file = "github-project-cache-v1.json";
-                    auth_token_secret_name=${
+                           webhook_secret_file = "github-webhook-secret";
-                                        if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"
+                           oauth_secret_file = "github-oauth-secret";
-                                      },
+                           oauth_id = cfg.github.oauthId;
-                )"
+                           topic = cfg.github.topic;
-                },
+                         };
-                admins=${builtins.toJSON cfg.admins},
+                admins = cfg.admins;
-                url=${builtins.toJSON config.services.buildbot-nix.master.webhookBaseUrl},
+                workers_file = cfg.workersFile;
-                nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
+                build_systems = cfg.buildSystems;
-                nix_eval_worker_count=${
+                eval_max_memory_size = cfg.evalMaxMemorySize;
-                  if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount
+                eval_worker_count = cfg.evalWorkerCount;
-                },
+                domain = cfg.domain;
-                nix_supported_systems=${builtins.toJSON cfg.buildSystems},
+                webhook_base_url = cfg.webhookBaseUrl;
-                outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
+                use_https = cfg.useHTTPS;
-                post_build_steps=[
+                outputs_path = cfg.outputsPath;
-                  ${lib.concatMapStringsSep ",\n" ({ name, environment, command }: ''
+                url = config.services.buildbot-nix.master.webhookBaseUrl;
-                    steps.ShellCommand(
+                post_build_steps = cfg.postBuildSteps;
-                      name=${builtins.toJSON name},
+              }}").read_text()))
-                      env={
-                        ${lib.concatMapStringsSep ",\n" ({name, value}: ''
-                          ${name}: ${interpolateToString value}
-                        '') (lib.mapAttrsToList lib.nameValuePair environment)}
-                      },
-                      command=[
-                        ${lib.concatMapStringsSep ",\n" (value:
-                          interpolateToString value
-                        ) (if lib.isList command then command else [ command ])}
-                      ]
-                    )
-                  '') cfg.postBuildSteps}
-                ]
             )
           ''
         ];
@@ -509,6 +496,7 @@ in
             });
           in
           ps: [
+            ps.pydantic
             pkgs.nix
             ps.requests
             ps.treq
@@ -529,25 +517,20 @@ in
           # in master.py we read secrets from $CREDENTIALS_DIRECTORY
           LoadCredential =
             [ "buildbot-nix-workers:${cfg.workersFile}" ]
-            ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
+            ++ lib.optionals cfg.github.enable ([
-            ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}"
-            ++ lib.optional
-              (
-                cfg.cachix.signingKeyFile != null
-              ) "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
-            ++ lib.optional
-              (
-                cfg.cachix.authTokenFile != null
-              ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
-            ++ lib.optionals (cfg.github.enable) ([
               "github-webhook-secret:${cfg.github.webhookSecretFile}"
             ]
-            ++ lib.optionals (cfg.github.authType ? "legacy") [
+            ++ lib.optional (cfg.github.authType ? "legacy")
               "github-token:${cfg.github.authType.legacy.tokenFile}"
-            ]
+            ++ lib.optional (cfg.github.authType ? "app")
-            ++ lib.optionals (cfg.github.authType ? "app") [
               "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}"
-            ])
+            )
+            ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
+            ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}"
+            ++ lib.optionals cfg.cachix.enable [
+              "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
+              "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
+            ]
             ++ lib.optionals cfg.gitea.enable [
               "gitea-token:${cfg.gitea.tokenFile}"
               "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"

pyproject.toml

@@ -27,7 +27,6 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" }
 packages = [
   "buildbot_nix",
   "buildbot_nix.github",
-  "buildbot_nix.github.auth",
   "buildbot_effects"
 ]