From 6e8e735628544db87fce8af9a458c09c2dd8690e Mon Sep 17 00:00:00 2001 From: magic_rb Date: Fri, 19 Jul 2024 22:20:42 +0200 Subject: [PATCH] Utilize `pydantic` for serialization and deserialization Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 87 +++-------- buildbot_nix/common.py | 24 ++- buildbot_nix/gitea_projects.py | 111 +++++++------- buildbot_nix/github/auth/__init__.py | 1 - buildbot_nix/github/auth/_type.py | 23 --- buildbot_nix/github/jwt_token.py | 19 +-- buildbot_nix/github_projects.py | 177 ++++++++++++----------- buildbot_nix/models.py | 182 +++++++++++++++++++++++ buildbot_nix/secrets.py | 4 +- nix/master.nix | 209 ++++++++++++--------------- pyproject.toml | 1 - 11 files changed, 477 insertions(+), 361 deletions(-) delete mode 100644 buildbot_nix/github/auth/__init__.py delete mode 100644 buildbot_nix/github/auth/_type.py create mode 100644 buildbot_nix/models.py diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 019cd4c..637a881 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -5,7 +5,6 @@ import re import uuid from collections import defaultdict from collections.abc import Generator -from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -33,13 +32,12 @@ from twisted.logger import Logger from .common import ( slugify_project_name, ) -from .gitea_projects import GiteaBackend, GiteaConfig +from .gitea_projects import GiteaBackend from .github_projects import ( GithubBackend, - GithubConfig, ) +from .models import BuildbotNixConfig from .projects import GitBackend, GitProject -from .secrets import read_secret_file SKIPPED_BUILDER_NAME = "skipped-builds" @@ -428,21 +426,6 @@ def nix_eval_config( ) -@dataclass -class CachixConfig: - name: str - signing_key_secret_name: str | None = None - auth_token_secret_name: str | None = None - - def cachix_env(self) -> dict[str, str]: - env = {} - if self.signing_key_secret_name is not None: - env["CACHIX_SIGNING_KEY"] = util.Secret(self.signing_key_secret_name) - if self.auth_token_secret_name is not None: - env["CACHIX_AUTH_TOKEN"] = util.Secret(self.auth_token_secret_name) - return env - - @defer.inlineCallbacks def do_register_gcroot_if(s: steps.BuildStep) -> Generator[Any, object, Any]: gc_root = yield util.Interpolate( @@ -847,53 +830,23 @@ class PeriodicWithStartup(schedulers.Periodic): class NixConfigurator(ConfiguratorBase): """Janitor is a configurator which create a Janitor Builder with all needed Janitor steps""" - def __init__( - self, - # Shape of this file: [ { "name": "", "pass": "", "cores": "" } ] - admins: list[str], - auth_backend: str, - build_retries: int, - github: GithubConfig | None, - gitea: GiteaConfig | None, - url: str, - nix_supported_systems: list[str], - nix_eval_worker_count: int | None, - nix_eval_max_memory_size: int, - post_build_steps: list[steps.BuildStep] | None = None, - nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107 - cachix: CachixConfig | None = None, - outputs_path: str | None = None, - ) -> None: + def __init__(self, config: BuildbotNixConfig) -> None: super().__init__() - self.nix_workers_secret_name = nix_workers_secret_name - self.nix_eval_max_memory_size = nix_eval_max_memory_size - self.nix_eval_worker_count = nix_eval_worker_count - self.nix_supported_systems = nix_supported_systems - self.post_build_steps = post_build_steps or [] - self.auth_backend = auth_backend - self.admins = admins - self.github = github - self.gitea = gitea - self.url = url - self.cachix = cachix - self.build_retries = build_retries - if outputs_path is None: - self.outputs_path = None - else: - self.outputs_path = Path(outputs_path) + + self.config = config def configure(self, config: dict[str, Any]) -> None: backends: dict[str, GitBackend] = {} - if self.github is not None: - backends["github"] = GithubBackend(self.github, self.url) + if self.config.github is not None: + backends["github"] = GithubBackend(self.config.github, self.config.url) - if self.gitea is not None: - backends["gitea"] = GiteaBackend(self.gitea) + if self.config.gitea is not None: + backends["gitea"] = GiteaBackend(self.config.gitea) auth: AuthBase | None = ( - backends[self.auth_backend].create_auth() - if self.auth_backend != "none" + backends[self.config.auth_backend].create_auth() + if self.config.auth_backend != "none" else None ) @@ -902,7 +855,7 @@ class NixConfigurator(ConfiguratorBase): for backend in backends.values(): projects += backend.load_projects() - worker_config = json.loads(read_secret_file(self.nix_workers_secret_name)) + worker_config = json.loads(self.config.nix_workers_secret) worker_names = [] config.setdefault("projects", []) @@ -918,7 +871,7 @@ class NixConfigurator(ConfiguratorBase): eval_lock = util.MasterLock("nix-eval") - if self.cachix is not None: + if self.config.cachix is not None: self.post_build_steps.append( steps.ShellCommand( name="Upload cachix", @@ -937,13 +890,13 @@ class NixConfigurator(ConfiguratorBase): config, project, worker_names, - self.nix_supported_systems, - self.nix_eval_worker_count or multiprocessing.cpu_count(), - self.nix_eval_max_memory_size, + self.config.build_systems, + self.config.eval_worker_count or multiprocessing.cpu_count(), + self.config.eval_max_memory_size, eval_lock, - self.post_build_steps, - self.outputs_path, - self.build_retries, + [x.to_buildstep() for x in self.config.post_build_steps], + self.config.outputs_path, + self.config.build_retries, ) config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME)) @@ -998,7 +951,7 @@ class NixConfigurator(ConfiguratorBase): config["www"]["auth"] = auth config["www"]["authz"] = setup_authz( - admins=self.admins, + admins=self.config.admins, backends=list(backends.values()), projects=projects, ) diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py index 2b9f733..2b0c101 100644 --- a/buildbot_nix/common.py +++ b/buildbot_nix/common.py @@ -6,10 +6,12 @@ from abc import ABC, abstractmethod from collections.abc import Callable from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeVar if TYPE_CHECKING: from buildbot.process.log import StreamLog + from pydantic import BaseModel + from collections.abc import Generator from buildbot.plugins import util @@ -110,9 +112,12 @@ def atomic_write_file(file: Path, data: str) -> None: raise +Y = TypeVar("Y") + + def filter_repos_by_topic( - topic: str | None, repos: list[Any], topics: Callable[[Any], list[str]] -) -> list[Any]: + topic: str | None, repos: list[Y], topics: Callable[[Y], list[str]] +) -> list[Y]: return list( filter( lambda repo: topic is None or topic in topics(repo), @@ -154,3 +159,16 @@ class ThreadDeferredBuildStep(BuildStep, ABC): log: StreamLog = yield self.addLog("log") log.addStderr(f"Failed to reload project list: {self.error_msg}") return util.FAILURE + + +_T = TypeVar("_T", bound="BaseModel") + + +def model_validate_project_cache(cls: type[_T], project_cache_file: Path) -> list[_T]: + return [ + cls.model_validate(data) for data in json.loads(project_cache_file.read_text()) + ] + + +def model_dump_project_cache(repos: list[_T]) -> str: + return json.dumps([repo.model_dump() for repo in repos]) diff --git a/buildbot_nix/gitea_projects.py b/buildbot_nix/gitea_projects.py index f7d72de..9eef8eb 100644 --- a/buildbot_nix/gitea_projects.py +++ b/buildbot_nix/gitea_projects.py @@ -1,7 +1,5 @@ -import json import os import signal -from dataclasses import dataclass from pathlib import Path from typing import Any from urllib.parse import urlparse @@ -14,6 +12,7 @@ from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase from buildbot_gitea.auth import GiteaAuth # type: ignore[import] from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import] +from pydantic import BaseModel from twisted.logger import Logger from twisted.python import log @@ -22,40 +21,37 @@ from .common import ( atomic_write_file, filter_repos_by_topic, http_request, + model_dump_project_cache, + model_validate_project_cache, paginated_github_request, slugify_project_name, ) +from .models import GiteaConfig from .projects import GitBackend, GitProject -from .secrets import read_secret_file tlog = Logger() -@dataclass -class GiteaConfig: - instance_url: str - oauth_id: str | None +class RepoOwnerData(BaseModel): + login: str - oauth_secret_name: str = "gitea-oauth-secret" - token_secret_name: str = "gitea-token" - webhook_secret_name: str = "gitea-webhook-secret" - project_cache_file: Path = Path("gitea-project-cache.json") - topic: str | None = "build-with-buildbot" - def oauth_secret(self) -> str: - return read_secret_file(self.oauth_secret_name) - - def token(self) -> str: - return read_secret_file(self.token_secret_name) +class RepoData(BaseModel): + name: str + owner: RepoOwnerData + full_name: str + ssh_url: str + default_branch: str + topics: list[str] class GiteaProject(GitProject): config: GiteaConfig webhook_secret: str - data: dict[str, Any] + data: RepoData def __init__( - self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any] + self, config: GiteaConfig, webhook_secret: str, data: RepoData ) -> None: self.config = config self.webhook_secret = webhook_secret @@ -63,7 +59,7 @@ class GiteaProject(GitProject): def get_project_url(self) -> str: url = urlparse(self.config.instance_url) - return f"{url.scheme}://git:%(secret:{self.config.token_secret_name})s@{url.hostname}/{self.name}" + return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.name}" @property def pretty_type(self) -> str: @@ -75,33 +71,33 @@ class GiteaProject(GitProject): @property def repo(self) -> str: - return self.data["name"] + return self.data.name @property def owner(self) -> str: - return self.data["owner"]["login"] + return self.data.owner.login @property def name(self) -> str: - return self.data["full_name"] + return self.data.full_name @property def url(self) -> str: # not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34 - return self.data["ssh_url"] + return self.data.ssh_url @property def project_id(self) -> str: - return slugify_project_name(self.data["full_name"]) + return slugify_project_name(self.data.full_name) @property def default_branch(self) -> str: - return self.data["default_branch"] + return self.data.default_branch @property def topics(self) -> list[str]: # note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects` - return self.data["topics"] + return self.data.topics @property def belongs_to_org(self) -> bool: @@ -111,11 +107,9 @@ class GiteaProject(GitProject): class GiteaBackend(GitBackend): config: GiteaConfig - webhook_secret: str def __init__(self, config: GiteaConfig) -> None: self.config = config - self.webhook_secret = read_secret_file(self.config.webhook_secret_name) def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" @@ -124,9 +118,7 @@ class GiteaBackend(GitBackend): ReloadGiteaProjects(self.config, self.config.project_cache_file), ) factory.addStep( - CreateGiteaProjectHooks( - self.config, self.config.project_cache_file, self.webhook_secret - ), + CreateGiteaProjectHooks(self.config), ) return util.BuilderConfig( name=self.reload_builder_name, @@ -137,14 +129,14 @@ class GiteaBackend(GitBackend): def create_reporter(self) -> ReporterBase: return GiteaStatusPush( self.config.instance_url, - Interpolate(self.config.token()), + Interpolate(self.config.token), context=Interpolate("buildbot/%(prop:status_name)s"), context_pr=Interpolate("buildbot/%(prop:status_name)s"), ) def create_change_hook(self) -> dict[str, Any]: return { - "secret": self.webhook_secret, + "secret": self.config.webhook_secret, # The "mergable" field is a bit buggy, # we already do the merge locally anyway. "onlyMergeablePullRequest": False, @@ -158,29 +150,32 @@ class GiteaBackend(GitBackend): return GiteaAuth( self.config.instance_url, self.config.oauth_id, - self.config.oauth_secret(), + self.config.oauth_secret, ) def load_projects(self) -> list["GitProject"]: if not self.config.project_cache_file.exists(): return [] - repos: list[dict[str, Any]] = filter_repos_by_topic( + repos: list[RepoData] = filter_repos_by_topic( self.config.topic, sorted( - json.loads(self.config.project_cache_file.read_text()), - key=lambda x: x["full_name"], + model_validate_project_cache(RepoData, self.config.project_cache_file), + key=lambda repo: repo.full_name, ), - lambda repo: repo["topics"], + lambda repo: repo.topics, ) - repo_names: list[str] = [ - repo["owner"]["login"] + "/" + repo["name"] for repo in repos - ] + repo_names: list[str] = [repo.owner.login + "/" + repo.name for repo in repos] tlog.info( f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]" ) - return [GiteaProject(self.config, self.webhook_secret, repo) for repo in repos] + return [ + GiteaProject( + self.config, self.config.webhook_secret, RepoData.model_validate(repo) + ) + for repo in repos + ] def are_projects_cached(self) -> bool: return self.config.project_cache_file.exists() @@ -245,30 +240,24 @@ class CreateGiteaProjectHooks(ThreadDeferredBuildStep): name = "create_gitea_project_hooks" config: GiteaConfig - project_cache_file: Path - webhook_secret: str def __init__( self, config: GiteaConfig, - project_cache_file: Path, - webhook_secret: str, **kwargs: Any, ) -> None: self.config = config - self.project_cache_file = project_cache_file - self.webhook_secret = webhook_secret super().__init__(**kwargs) def run_deferred(self) -> None: - repos = json.loads(self.project_cache_file.read_text()) + repos = model_validate_project_cache(RepoData, self.config.project_cache_file) for repo in repos: create_repo_hook( - self.config.token(), - self.webhook_secret, - repo["owner"]["login"], - repo["name"], + self.config.token, + self.config.webhook_secret, + repo.owner.login, + repo.name, self.config.instance_url, ) @@ -294,24 +283,24 @@ class ReloadGiteaProjects(ThreadDeferredBuildStep): super().__init__(**kwargs) def run_deferred(self) -> None: - repos = filter_repos_by_topic( + repos: list[RepoData] = filter_repos_by_topic( self.config.topic, refresh_projects(self.config, self.project_cache_file), - lambda repo: repo["topics"], + lambda repo: repo.topics, ) - atomic_write_file(self.project_cache_file, json.dumps(repos)) + atomic_write_file(self.project_cache_file, model_dump_project_cache(repos)) def run_post(self) -> Any: return util.SUCCESS -def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]: +def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[RepoData]: repos = [] for repo in paginated_github_request( f"{config.instance_url}/api/v1/user/repos?limit=100", - config.token(), + config.token, ): if not repo["permissions"]["admin"]: name = repo["full_name"] @@ -323,10 +312,10 @@ def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]: # Gitea doesn't include topics in the default repo listing, unlike GitHub topics: list[str] = http_request( f"{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics", - headers={"Authorization": f"token {config.token()}"}, + headers={"Authorization": f"token {config.token}"}, ).json()["topics"] repo["topics"] = topics - repos.append(repo) + repos.append(RepoData.model_validate(repo)) except OSError: pass diff --git a/buildbot_nix/github/auth/__init__.py b/buildbot_nix/github/auth/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/buildbot_nix/github/auth/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/buildbot_nix/github/auth/_type.py b/buildbot_nix/github/auth/_type.py deleted file mode 100644 index 8c4c01b..0000000 --- a/buildbot_nix/github/auth/_type.py +++ /dev/null @@ -1,23 +0,0 @@ -from dataclasses import dataclass -from pathlib import Path - - -@dataclass -class AuthType: - pass - - -@dataclass -class AuthTypeLegacy(AuthType): - token_secret_name: str = "github-token" - - -@dataclass -class AuthTypeApp(AuthType): - app_id: int - app_secret_key_name: str = "github-app-secret-key" - app_installation_token_map_name: Path = Path( - "github-app-installation-token-map.json" - ) - app_project_id_map_name: Path = Path("github-app-project-id-map-name.json") - app_jwt_token_name: Path = Path("github-app-jwt-token") diff --git a/buildbot_nix/github/jwt_token.py b/buildbot_nix/github/jwt_token.py index 0d78171..a0f9ad4 100644 --- a/buildbot_nix/github/jwt_token.py +++ b/buildbot_nix/github/jwt_token.py @@ -3,6 +3,7 @@ import json import os import subprocess from datetime import UTC, datetime, timedelta +from pathlib import Path from typing import Any from .repo_token import RepoToken @@ -10,7 +11,7 @@ from .repo_token import RepoToken class JWTToken(RepoToken): app_id: int - app_private_key: str + app_private_key_file: Path lifetime: timedelta expiration: datetime @@ -19,20 +20,20 @@ class JWTToken(RepoToken): def __init__( self, app_id: int, - app_private_key: str, + app_private_key_file: Path, lifetime: timedelta = timedelta(minutes=10), ) -> None: self.app_id = app_id - self.app_private_key = app_private_key + self.app_private_key_file = app_private_key_file self.lifetime = lifetime self.token, self.expiration = JWTToken.generate_token( - self.app_id, self.app_private_key, lifetime + self.app_id, self.app_private_key_file, lifetime ) @staticmethod def generate_token( - app_id: int, app_private_key: str, lifetime: timedelta + app_id: int, app_private_key_file: Path, lifetime: timedelta ) -> tuple[str, datetime]: def build_jwt_payload( app_id: int, lifetime: timedelta @@ -48,9 +49,9 @@ class JWTToken(RepoToken): } return (jwt_payload, exp) - def rs256_sign(data: str, private_key: str) -> str: + def rs256_sign(data: str, private_key_file: Path) -> str: signature = subprocess.run( - ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key], + ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key_file], input=data.encode("utf-8"), stdout=subprocess.PIPE, check=True, @@ -65,7 +66,7 @@ class JWTToken(RepoToken): jwt_payload = json.dumps(jwt).encode("utf-8") json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8") encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}" - encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key) + encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key_file) return (f"{encoded_jwt_parts}.{encoded_mac}", expiration) # installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt) @@ -75,7 +76,7 @@ class JWTToken(RepoToken): def get(self) -> str: if self.expiration - datetime.now(tz=UTC) < self.lifetime * 0.2: self.token, self.expiration = JWTToken.generate_token( - self.app_id, self.app_private_key, self.lifetime + self.app_id, self.app_private_key_file, self.lifetime ) return self.token diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 1d567cd..12957e2 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -17,6 +17,7 @@ from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth +from pydantic import BaseModel, ConfigDict, Field from twisted.logger import Logger from twisted.python import log @@ -25,10 +26,11 @@ from .common import ( atomic_write_file, filter_repos_by_topic, http_request, + model_dump_project_cache, + model_validate_project_cache, paginated_github_request, slugify_project_name, ) -from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy from .github.installation_token import InstallationToken from .github.jwt_token import JWTToken from .github.legacy_token import ( @@ -37,12 +39,33 @@ from .github.legacy_token import ( from .github.repo_token import ( RepoToken, ) +from .models import ( + GitHubAppConfig, + GitHubConfig, + GitHubLegacyConfig, +) from .projects import GitBackend, GitProject -from .secrets import read_secret_file tlog = Logger() +class RepoOwnerData(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + login: str + ttype: str = Field(alias="type") + + +class RepoData(BaseModel): + name: str + owner: RepoOwnerData + full_name: str + html_url: str + default_branch: str + topics: list[str] + installation_id: int | None + + def get_installations(jwt_token: JWTToken) -> list[int]: installations = paginated_github_request( "https://api.github.com/app/installations?per_page=100", jwt_token.get() @@ -80,7 +103,7 @@ class CreateGitHubInstallationHooks(ThreadDeferredBuildStep): super().__init__(**kwargs) def run_deferred(self) -> None: - repos = json.loads(self.project_cache_file.read_text()) + repos = model_validate_project_cache(RepoData, self.project_cache_file) installation_token_map: dict[int, InstallationToken] = dict( starmap( lambda k, v: ( @@ -94,11 +117,14 @@ class CreateGitHubInstallationHooks(ThreadDeferredBuildStep): ) for repo in repos: + if repo.installation_id is None: + continue + create_project_hook( - installation_token_map[repo["installation_id"]], + installation_token_map[repo.installation_id], self.webhook_secret, - repo["owner"]["login"], - repo["name"], + repo.owner.login, + repo.name, self.webhook_url, ) @@ -144,7 +170,7 @@ class ReloadGithubInstallations(ThreadDeferredBuildStep): get_installations(self.jwt_token), ) - repos: list[Any] = [] + repos: list[RepoData] = [] project_id_map: dict[str, int] = {} repos = [] @@ -160,17 +186,17 @@ class ReloadGithubInstallations(ThreadDeferredBuildStep): subkey="repositories", require_admin=False, ), - lambda repo: repo["topics"], + lambda repo: repo.topics, ) for repo in new_repos: - repo["installation_id"] = k + repo.installation_id = k - project_id_map[repo["full_name"]] = k + project_id_map[repo.full_name] = k repos.extend(new_repos) - atomic_write_file(self.project_cache_file, json.dumps(repos)) + atomic_write_file(self.project_cache_file, model_dump_project_cache(repos)) atomic_write_file(self.project_id_map_name, json.dumps(project_id_map)) tlog.info( @@ -207,14 +233,14 @@ class CreateGitHubProjectHooks(ThreadDeferredBuildStep): super().__init__(**kwargs) def run_deferred(self) -> None: - repos = json.loads(self.project_cache_file.read_text()) + repos = model_validate_project_cache(RepoData, self.project_cache_file) for repo in repos: create_project_hook( self.token, self.webhook_secret, - repo["owner"]["login"], - repo["name"], + repo.owner.login, + repo.name, self.webhook_url, ) @@ -244,13 +270,13 @@ class ReloadGithubProjects(ThreadDeferredBuildStep): super().__init__(**kwargs) def run_deferred(self) -> None: - repos: list[Any] = filter_repos_by_topic( + repos: list[RepoData] = filter_repos_by_topic( self.topic, refresh_projects(self.token.get(), self.project_cache_file), - lambda repo: repo["topics"], + lambda repo: repo.topics, ) - atomic_write_file(self.project_cache_file, json.dumps(repos)) + atomic_write_file(self.project_cache_file, model_dump_project_cache(repos)) def run_post(self) -> Any: return util.SUCCESS @@ -262,7 +288,7 @@ class GithubAuthBackend(ABC): pass @abstractmethod - def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + def get_repo_token(self, repo: RepoData) -> RepoToken: pass @abstractmethod @@ -285,18 +311,18 @@ class GithubAuthBackend(ABC): class GithubLegacyAuthBackend(GithubAuthBackend): - auth_type: AuthTypeLegacy + auth_type: GitHubLegacyConfig token: LegacyToken - def __init__(self, auth_type: AuthTypeLegacy) -> None: + def __init__(self, auth_type: GitHubLegacyConfig) -> None: self.auth_type = auth_type - self.token = LegacyToken(read_secret_file(auth_type.token_secret_name)) + self.token = LegacyToken(auth_type.token) def get_general_token(self) -> RepoToken: return self.token - def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + def get_repo_token(self, repo: RepoData) -> RepoToken: return self.token def create_secret_providers(self) -> list[SecretProviderBase]: @@ -351,24 +377,22 @@ class GitHubLegacySecretService(SecretProviderBase): class GithubAppAuthBackend(GithubAuthBackend): - auth_type: AuthTypeApp + auth_type: GitHubAppConfig jwt_token: JWTToken installation_tokens: dict[int, InstallationToken] project_id_map: dict[str, int] - def __init__(self, auth_type: AuthTypeApp) -> None: + def __init__(self, auth_type: GitHubAppConfig) -> None: self.auth_type = auth_type - self.jwt_token = JWTToken( - self.auth_type.app_id, self.auth_type.app_secret_key_name - ) + self.jwt_token = JWTToken(self.auth_type.id, self.auth_type.secret_key_file) self.installation_tokens = GithubBackend.load_installations( self.jwt_token, - self.auth_type.app_installation_token_map_name, + self.auth_type.installation_token_map_file, ) - if self.auth_type.app_project_id_map_name.exists(): + if self.auth_type.project_id_map_file.exists(): self.project_id_map = json.loads( - self.auth_type.app_project_id_map_name.read_text() + self.auth_type.project_id_map_file.read_text() ) else: tlog.info( @@ -379,9 +403,9 @@ class GithubAppAuthBackend(GithubAuthBackend): def get_general_token(self) -> RepoToken: return self.jwt_token - def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: - assert "installation_id" in repo, f"Missing installation_id in {repo}" - return self.installation_tokens[repo["installation_id"]] + def get_repo_token(self, repo: RepoData) -> RepoToken: + assert repo.installation_id is not None, f"Missing installation_id in {repo}" + return self.installation_tokens[repo.installation_id] def create_secret_providers(self) -> list[SecretProviderBase]: return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)] @@ -411,14 +435,14 @@ class GithubAppAuthBackend(GithubAuthBackend): ReloadGithubInstallations( self.jwt_token, project_cache_file, - self.auth_type.app_installation_token_map_name, - self.auth_type.app_project_id_map_name, + self.auth_type.installation_token_map_file, + self.auth_type.project_id_map_file, topic, ), CreateGitHubInstallationHooks( self.jwt_token, project_cache_file, - self.auth_type.app_installation_token_map_name, + self.auth_type.installation_token_map_file, webhook_secret=webhook_secret, webhook_url=webhook_url, topic=topic, @@ -450,32 +474,22 @@ class GitHubAppSecretService(SecretProviderBase): return None -@dataclass -class GithubConfig: - oauth_id: str | None - auth_type: AuthType - oauth_secret_name: str = "github-oauth-secret" - webhook_secret_name: str = "github-webhook-secret" - project_cache_file: Path = Path("github-project-cache-v1.json") - topic: str | None = "build-with-buildbot" - - @dataclass class GithubBackend(GitBackend): - config: GithubConfig + config: GitHubConfig webhook_secret: str webhook_url: str auth_backend: GithubAuthBackend - def __init__(self, config: GithubConfig, webhook_url: str) -> None: + def __init__(self, config: GitHubConfig, webhook_url: str) -> None: self.config = config - self.webhook_secret = read_secret_file(self.config.webhook_secret_name) + self.webhook_secret = self.config.webhook_secret self.webhook_url = webhook_url - if isinstance(self.config.auth_type, AuthTypeLegacy): + if isinstance(self.config.auth_type, GitHubLegacyConfig): self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type) - elif isinstance(self.config.auth_type, AuthTypeApp): + elif isinstance(self.config.auth_type, GitHubAppConfig): self.auth_backend = GithubAppAuthBackend(self.config.auth_type) @staticmethod @@ -551,7 +565,7 @@ class GithubBackend(GitBackend): assert self.config.oauth_id is not None, "GitHub OAuth ID is required" return GitHubAuth( self.config.oauth_id, - read_secret_file(self.config.oauth_secret_name), + self.config.oauth_secret, apiVersion=4, ) @@ -562,30 +576,28 @@ class GithubBackend(GitBackend): if not self.config.project_cache_file.exists(): return [] - repos: list[dict[str, Any]] = filter_repos_by_topic( + repos: list[RepoData] = filter_repos_by_topic( self.config.topic, sorted( - json.loads(self.config.project_cache_file.read_text()), - key=lambda x: x["full_name"], + model_validate_project_cache(RepoData, self.config.project_cache_file), + key=lambda repo: repo.full_name, ), - lambda repo: repo["topics"], + lambda repo: repo.topics, ) if isinstance(self.auth_backend, GithubAppAuthBackend): dropped_repos = list( - filter(lambda repo: "installation_id" not in repo, repos) + filter(lambda repo: repo.installation_id is None, repos) ) if dropped_repos: tlog.info( "Dropped projects follow, refresh will follow after initialisation:" ) for dropped_repo in dropped_repos: - tlog.info(f"\tDropping repo {dropped_repo['full_name']}") - repos = list(filter(lambda repo: "installation_id" in repo, repos)) + tlog.info(f"\tDropping repo {dropped_repo.full_name}") + repos = list(filter(lambda repo: repo.installation_id is not None, repos)) - repo_names: list[str] = [ - repo["owner"]["login"] + "/" + repo["name"] for repo in repos - ] + repo_names: list[str] = [repo.owner.login + "/" + repo.name for repo in repos] tlog.info( f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]" @@ -595,7 +607,7 @@ class GithubBackend(GitBackend): self.auth_backend.get_repo_token(repo), self.config, self.webhook_secret, - repo, + RepoData.model_validate(repo), ) for repo in repos ] @@ -605,14 +617,16 @@ class GithubBackend(GitBackend): return False if ( - isinstance(self.config.auth_type, AuthTypeApp) - and not self.config.auth_type.app_project_id_map_name.exists() + isinstance(self.config.auth_type, GitHubAppConfig) + and not self.config.auth_type.project_id_map_file.exists() ): return False all_have_installation_id = True - for project in json.loads(self.config.project_cache_file.read_text()): - if "installation_id" not in project: + for project in model_validate_project_cache( + RepoData, self.config.project_cache_file + ): + if project.installation_id is not None: all_have_installation_id = False break @@ -669,17 +683,17 @@ def create_project_hook( class GithubProject(GitProject): - config: GithubConfig + config: GitHubConfig webhook_secret: str - data: dict[str, Any] + data: RepoData token: RepoToken def __init__( self, token: RepoToken, - config: GithubConfig, + config: GitHubConfig, webhook_secret: str, - data: dict[str, Any], + data: RepoData, ) -> None: self.token = token self.config = config @@ -699,35 +713,35 @@ class GithubProject(GitProject): @property def repo(self) -> str: - return self.data["name"] + return self.data.name @property def owner(self) -> str: - return self.data["owner"]["login"] + return self.data.owner.login @property def name(self) -> str: - return self.data["full_name"] + return self.data.full_name @property def url(self) -> str: - return self.data["html_url"] + return self.data.html_url @property def project_id(self) -> str: - return slugify_project_name(self.data["full_name"]) + return slugify_project_name(self.data.full_name) @property def default_branch(self) -> str: - return self.data["default_branch"] + return self.data.default_branch @property def topics(self) -> list[str]: - return self.data["topics"] + return self.data.topics @property def belongs_to_org(self) -> bool: - return self.data["owner"]["type"] == "Organization" + return self.data.owner.ttype == "Organization" def refresh_projects( @@ -738,7 +752,7 @@ def refresh_projects( api_endpoint: str = "/user/repos", subkey: None | str = None, require_admin: bool = True, -) -> list[Any]: +) -> list[RepoData]: if repos is None: repos = [] @@ -754,6 +768,7 @@ def refresh_projects( f"skipping {name} because we do not have admin privileges, needed for hook management", ) else: - repos.append(repo) + repo["installation_id"] = None + repos.append(RepoData.model_validate(repo)) return repos diff --git a/buildbot_nix/models.py b/buildbot_nix/models.py new file mode 100644 index 0000000..6198506 --- /dev/null +++ b/buildbot_nix/models.py @@ -0,0 +1,182 @@ +from enum import Enum +from pathlib import Path + +from pydantic import BaseModel, Field, field_serializer, field_validator, ConfigDict +from buildbot.plugins import util, steps + +from .secrets import read_secret_file + +class InternalIssue(Exception): + pass + + +def exclude_fields(fields: list[str]) -> dict[str, dict[str, bool]]: + return dict(map(lambda k: (k, {"exclude": True}), fields)) + +class AuthBackendConfig(str, Enum): + github = "github" + gitea = "gitea" + none = "none" + +class CachixConfig(BaseModel): + name: str + + signing_key_file: Path | None + auth_token_file: Path | None + + @property + def signing_key(self) -> str: + if self.signing_key_file is None: + raise InternalIssue + return read_secret_file(self.signing_key_file) + + @property + def auth_token(self) -> str: + if self.auth_token_file is None: + raise InternalIssue + return read_secret_file(self.auth_token_file) + + # TODO why did the original implementation return an empty env if both files were missing? + @property + def environment(self) -> dict[str, str]: + environment = {} + environment["CACHIX_SIGNING_KEY"] = util.Secret(self.signing_key_file) + environment["CACHIX_AUTH_TOKEN"] = util.Secret(self.auth_token_file) + return environment + + class Config: + fields = exclude_fields(["singing_key", "auth_token"]) + +class GiteaConfig(BaseModel): + instance_url: str + topic: str | None + + token_file: Path = Field(default = Path("gitea-token")) + webhook_secret_file: Path = Field(default = Path("gitea-webhook-secret")) + project_cache_file: Path = Field(default = Path("gitea-project-cache.json")) + + oauth_id: str | None + oauth_secret_file: Path | None + + @property + def token(self) -> str: + return read_secret_file(self.token_file) + + @property + def webhook_secret(self) -> str: + return read_secret_file(self.webhook_secret_file) + + @property + def oauth_secret(self) -> str: + if self.oauth_secret_file is None: + raise InternalIssue + return read_secret_file(self.oauth_secret_file) + + class Config: + fields = exclude_fields(["token", "webhook_secret", "oauth_secret"]) + +class GitHubLegacyConfig(BaseModel): + token_file: Path + + @property + def token(self) -> str: + return read_secret_file(self.token_file) + + class Config: + fields = exclude_fields(["token"]) + +class GitHubAppConfig(BaseModel): + id: int + + secret_key_file: Path + installation_token_map_file: Path = Field(default = Path( + "github-app-installation-token-map.json" + )) + project_id_map_file: Path = Field(default = Path( + "github-app-project-id-map-name.json" + )) + jwt_token_map: Path = Field(default = Path( + "github-app-jwt-token" + )) + + @property + def secret_key(self) -> str: + return read_secret_file(self.secret_key_file) + + class Config: + fields = exclude_fields(["secret_key"]) + +class GitHubConfig(BaseModel): + auth_type: GitHubLegacyConfig | GitHubAppConfig + topic: str | None + + project_cache_file: Path = Field(default = Path("github-project-cache-v1.json")) + webhook_secret_file: Path = Field(default = Path("github-webhook-secret")) + + oauth_id: str | None + oauth_secret_file: Path | None + + @property + def webhook_secret(self) -> str: + return read_secret_file(self.webhook_secret_file) + + @property + def oauth_secret(self) -> str: + if self.oauth_secret_file is None: + raise InternalIssue + return read_secret_file(self.oauth_secret_file) + +# note that serialization isn't correct, as there is no way to *rename* the field `nix_type` to `_type`, +# one must always specify `by_alias = True`, such as `model_dump(by_alias = True)`, relevant issue: +# https://github.com/pydantic/pydantic/issues/8379 +class Interpolate(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + nix_type: str = Field(alias = "_type") + value: str + + +class PostBuildStep(BaseModel): + name: str + environment: dict[str, str | Interpolate] + command: list[str | Interpolate] + + def to_buildstep(self) -> steps.BuildStep: + def maybe_interpolate(value: str | Interpolate) -> str | util.Interpolate: + if isinstance(value, str): + return value + return util.Interpolate(value.value) + + return steps.ShellCommand( + name = self.name, + env = { + k: maybe_interpolate(k) for k in self.environment + }, + command = [ + maybe_interpolate(x) for x in self.command + ] + ) + +class BuildbotNixConfig(BaseModel): + db_url: str + auth_backend: AuthBackendConfig + build_retries: int + cachix: CachixConfig | None + gitea: GiteaConfig | None + github: GitHubConfig | None + admins: list[str] + workers_file: Path + build_systems: list[str] + eval_max_memory_size: int + eval_worker_count: int | None + nix_workers_secret_file: Path = Field(default = Path("buildbot-nix-workers")) + domain: str + webhook_base_url: str + use_https: bool + outputs_path: Path | None + url: str + post_build_steps: list[PostBuildStep] + + @property + def nix_workers_secret(self) -> str: + return read_secret_file(self.nix_workers_secret_file) diff --git a/buildbot_nix/secrets.py b/buildbot_nix/secrets.py index 44a3e00..1311e5c 100644 --- a/buildbot_nix/secrets.py +++ b/buildbot_nix/secrets.py @@ -3,9 +3,9 @@ import sys from pathlib import Path -def read_secret_file(secret_name: str) -> str: +def read_secret_file(secret_file: Path) -> str: directory = os.environ.get("CREDENTIALS_DIRECTORY") if directory is None: print("directory not set", file=sys.stderr) sys.exit(1) - return Path(directory).joinpath(secret_name).read_text().rstrip() + return Path(directory).joinpath(secret_file).read_text().rstrip() diff --git a/nix/master.nix b/nix/master.nix index b5a8504..9a4d520 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -154,21 +154,20 @@ in }; cachix = { + enable = lib.mkEnableOption "Enable Cachix integration"; + name = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; + type = lib.types.str; description = "Cachix name"; }; signingKeyFile = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; + type = lib.types.path; description = "Cachix signing key"; }; authTokenFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; + type = lib.types.str; description = "Cachix auth token"; }; }; @@ -291,8 +290,8 @@ in description = "Systems that we will be build"; }; evalMaxMemorySize = lib.mkOption { - type = lib.types.str; - default = "2048"; + type = lib.types.int; + default = 2048; description = '' Maximum memory size for nix-eval-jobs (in MiB) per worker. After the limit is reached, the worker is @@ -364,19 +363,28 @@ in } { assertion = - cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null; - message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set"; - } - { - assertion = - cfg.authBackend != "github" || (cfg.github.oauthId != null && cfg.github.oauthSecretFile != null); + cfg.authBackend == "github" -> (cfg.github.oauthId != null && cfg.github.oauthSecretFile != null); message = ''If config.services.buildbot-nix.master.authBackend is set to "github", then config.services.buildbot-nix.master.github.oauthId and config.services.buildbot-nix.master.github.oauthSecretFile have to be set.''; } { assertion = - cfg.authBackend != "gitea" || (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null); + cfg.authBackend == "gitea" -> (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null); message = ''config.services.buildbot-nix.master.authBackend is set to "gitea", then config.services.buildbot-nix.master.gitea.oauthId and config.services.buildbot-nix.master.gitea.oauthSecretFile have to be set.''; } + { + assertion = + cfg.authBackend == "github" -> cfg.github.enable; + message = '' + If `cfg.authBackend` is set to `"github"` the GitHub backend must be enabled with `cfg.github.enable`; + ''; + } + { + assertion = + cfg.authBackend == "gitea" -> cfg.gitea.enable; + message = '' + If `cfg.authBackend` is set to `"gitea"` the GitHub backend must be enabled with `cfg.gitea.enable`; + ''; + } ]; services.buildbot-master = { @@ -391,19 +399,11 @@ in extraImports = '' from datetime import timedelta from buildbot_nix import ( - GithubConfig, NixConfigurator, - CachixConfig, - GiteaConfig, - ) - from buildbot.plugins import ( - steps, - util, - ) - from buildbot_nix.github.auth._type import ( - AuthTypeLegacy, - AuthTypeApp, + BuildbotNixConfig, ) + from pathlib import Path + import json ''; configurators = [ '' @@ -411,78 +411,65 @@ in '' '' NixConfigurator( - auth_backend=${builtins.toJSON cfg.authBackend}, - github=${ - if (!cfg.github.enable) then - "None" - else - "GithubConfig( - oauth_id=${builtins.toJSON cfg.github.oauthId}, - topic=${builtins.toJSON cfg.github.topic}, - auth_type=${ - if cfg.github.authType ? "legacy" then - ''AuthTypeLegacy()'' - else if cfg.github.authType ? "app" then - '' - AuthTypeApp( - app_id=${toString cfg.github.authType.app.id}, - ) - '' - else - throw "One of AuthTypeApp or AuthTypeLegacy must be enabled" - } - )" - }, - gitea=${ - if !cfg.gitea.enable then - "None" - else - "GiteaConfig( - instance_url=${builtins.toJSON cfg.gitea.instanceUrl}, - oauth_id=${builtins.toJSON cfg.gitea.oauthId}, - topic=${builtins.toJSON cfg.gitea.topic}, - )" - }, - build_retries=${builtins.toJSON cfg.buildRetries}, - cachix=${ - if cfg.cachix.name == null then - "None" - else - "CachixConfig( - name=${builtins.toJSON cfg.cachix.name}, - signing_key_secret_name=${ - if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None" - }, - auth_token_secret_name=${ - if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None" - }, - )" - }, - admins=${builtins.toJSON cfg.admins}, - url=${builtins.toJSON config.services.buildbot-nix.master.webhookBaseUrl}, - nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize}, - nix_eval_worker_count=${ - if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount - }, - nix_supported_systems=${builtins.toJSON cfg.buildSystems}, - outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath}, - post_build_steps=[ - ${lib.concatMapStringsSep ",\n" ({ name, environment, command }: '' - steps.ShellCommand( - name=${builtins.toJSON name}, - env={ - ${lib.concatMapStringsSep ",\n" ({name, value}: '' - ${name}: ${interpolateToString value} - '') (lib.mapAttrsToList lib.nameValuePair environment)} - }, - command=[ - ${lib.concatMapStringsSep ",\n" (value: - interpolateToString value - ) (if lib.isList command then command else [ command ])} - ] - ) - '') cfg.postBuildSteps} - ] + BuildbotNixConfig.model_validate(json.loads(Path("${(pkgs.formats.json {}).generate "buildbot-nix-config.json" { + db_url = cfg.dbUrl; + auth_backend = cfg.authBackend; + build_retries = cfg.buildRetries; + cachix = if !cfg.cachix.enable then + null + else + { + name = cfg.cachix.name; + signing_key_file = cfg.cachix.signingKeyFile; + auth_token_file = cfg.cachix.authTokenFile; + }; + gitea = if !cfg.gitea.enable then + null + else + { + token_file = "gitea-token"; + webhook_secret_file = "gitea-webhook-secret"; + project_cache_file = "gitea-project-cache.json"; + oauth_secret_file = "gitea-oauth-secret"; + instance_url = cfg.gitea.instanceUrl; + oauth_id = cfg.gitea.oauthId; + topic = cfg.gitea.topic; + }; + github = if !cfg.github.enable then + null + else { + auth_type = if (cfg.github.authType ? "legacy") then + { + token_file = "github-token"; + } + else if (cfg.github.authType ? "app") then + { + id = cfg.github.authType.app.id; + secret_key_file = cfg.github.authType.app.secretKeyFile; + installation_token_map_file = "github-app-installation-token-map.json"; + project_id_map_file = "github-app-project-id-map-name.json"; + jwt_token_map = "github-app-jwt-token"; + } + else + throw "authType is neither \"legacy\" nor \"app\""; + project_cache_file = "github-project-cache-v1.json"; + webhook_secret_file = "github-webhook-secret"; + oauth_secret_file = "github-oauth-secret"; + oauth_id = cfg.github.oauthId; + topic = cfg.github.topic; + }; + admins = cfg.admins; + workers_file = cfg.workersFile; + build_systems = cfg.buildSystems; + eval_max_memory_size = cfg.evalMaxMemorySize; + eval_worker_count = cfg.evalWorkerCount; + domain = cfg.domain; + webhook_base_url = cfg.webhookBaseUrl; + use_https = cfg.useHTTPS; + outputs_path = cfg.outputsPath; + url = config.services.buildbot-nix.master.webhookBaseUrl; + post_build_steps = cfg.postBuildSteps; + }}").read_text())) ) '' ]; @@ -509,6 +496,7 @@ in }); in ps: [ + ps.pydantic pkgs.nix ps.requests ps.treq @@ -529,25 +517,20 @@ in # in master.py we read secrets from $CREDENTIALS_DIRECTORY LoadCredential = [ "buildbot-nix-workers:${cfg.workersFile}" ] - ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}" - ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}" - ++ lib.optional - ( - cfg.cachix.signingKeyFile != null - ) "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}" - ++ lib.optional - ( - cfg.cachix.authTokenFile != null - ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" - ++ lib.optionals (cfg.github.enable) ([ + ++ lib.optionals cfg.github.enable ([ "github-webhook-secret:${cfg.github.webhookSecretFile}" ] - ++ lib.optionals (cfg.github.authType ? "legacy") [ + ++ lib.optional (cfg.github.authType ? "legacy") "github-token:${cfg.github.authType.legacy.tokenFile}" - ] - ++ lib.optionals (cfg.github.authType ? "app") [ + ++ lib.optional (cfg.github.authType ? "app") "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}" - ]) + ) + ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}" + ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}" + ++ lib.optionals cfg.cachix.enable [ + "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}" + "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" + ] ++ lib.optionals cfg.gitea.enable [ "gitea-token:${cfg.gitea.tokenFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" diff --git a/pyproject.toml b/pyproject.toml index c0b6710..ceec14f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" } packages = [ "buildbot_nix", "buildbot_nix.github", - "buildbot_nix.github.auth", "buildbot_effects" ]