From 067f3e0fc1c12ddc8f00edca27c452f14197c3ae Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 18 May 2024 21:36:22 +0200 Subject: [PATCH] Add GitHub App support Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 2 + buildbot_nix/common.py | 23 +- buildbot_nix/github/__init__.py | 1 + buildbot_nix/github/auth/__init__.py | 1 + buildbot_nix/github/auth/_type.py | 23 + buildbot_nix/github/installation_token.py | 104 +++++ buildbot_nix/github/jwt_token.py | 84 ++++ buildbot_nix/github/legacy_token.py | 14 + buildbot_nix/github/repo_token.py | 11 + buildbot_nix/github_projects.py | 511 ++++++++++++++++++++-- buildbot_nix/projects.py | 4 + nix/master.nix | 82 +++- 12 files changed, 813 insertions(+), 47 deletions(-) create mode 100644 buildbot_nix/github/__init__.py create mode 100644 buildbot_nix/github/auth/__init__.py create mode 100644 buildbot_nix/github/auth/_type.py create mode 100644 buildbot_nix/github/installation_token.py create mode 100644 buildbot_nix/github/jwt_token.py create mode 100644 buildbot_nix/github/legacy_token.py create mode 100644 buildbot_nix/github/repo_token.py diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index e16fdea..abd18dc 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -872,6 +872,8 @@ class NixConfigurator(ConfiguratorBase): ], ) config["services"].append(backend.create_reporter()) + config.setdefault("secretProviders", []) + config["secretsProviders"].extend(backend.create_secret_providers()) systemd_secrets = SecretInAFile( dirname=os.environ["CREDENTIALS_DIRECTORY"], diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py index 6dcd4f3..b4d3941 100644 --- a/buildbot_nix/common.py +++ b/buildbot_nix/common.py @@ -2,6 +2,8 @@ import contextlib import http.client import json import urllib.request +from pathlib import Path +from tempfile import NamedTemporaryFile from typing import Any @@ -9,7 +11,9 @@ def slugify_project_name(name: str) -> str: return name.replace(".", "-").replace("/", "-") -def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: +def paginated_github_request( + url: str, token: str, subkey: None | str = None +) -> list[dict[str, Any]]: next_url: str | None = url items = [] while next_url: @@ -29,7 +33,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: link_parts = link.split(";") if link_parts[1].strip() == 'rel="next"': next_url = link_parts[0][1:-1] - items += res.json() + if subkey is not None: + items += res.json()[subkey] + else: + items += res.json() return items @@ -78,3 +85,15 @@ def http_request( msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" raise HttpError(msg) from e return HttpResponse(resp) + + +def atomic_write_file(file: Path, data: str) -> None: + with NamedTemporaryFile("w", delete=False, dir=file.parent) as f: + path = Path(f.name) + try: + f.write(data) + f.flush() + path.rename(file) + except OSError: + path.unlink() + raise diff --git a/buildbot_nix/github/__init__.py b/buildbot_nix/github/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot_nix/github/__init__.py @@ -0,0 +1 @@ + diff --git a/buildbot_nix/github/auth/__init__.py b/buildbot_nix/github/auth/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot_nix/github/auth/__init__.py @@ -0,0 +1 @@ + diff --git a/buildbot_nix/github/auth/_type.py b/buildbot_nix/github/auth/_type.py new file mode 100644 index 0000000..abbff13 --- /dev/null +++ b/buildbot_nix/github/auth/_type.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class AuthType: + pass + + +@dataclass +class AuthTypeLegacy(AuthType): + token_secret_name: str = "github-token" + + +@dataclass +class AuthTypeApp(AuthType): + app_id: int + app_secret_key_name: str = "github-app-secret-key" + app_installation_token_map_name: Path = Path( + "github-app-installation-token-map.json" + ) + app_project_id_map_name: Path = Path("github-app-project-id-map-name") + app_jwt_token_name: Path = Path("github-app-jwt-token") diff --git a/buildbot_nix/github/installation_token.py b/buildbot_nix/github/installation_token.py new file mode 100644 index 0000000..a5affce --- /dev/null +++ b/buildbot_nix/github/installation_token.py @@ -0,0 +1,104 @@ +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from buildbot_nix.common import ( + HttpResponse, + atomic_write_file, + http_request, +) + +from .jwt_token import JWTToken +from .repo_token import RepoToken + + +class InstallationToken(RepoToken): + GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60) + + jwt_token: JWTToken + installation_id: int + + token: str + expiration: datetime + installations_token_map_name: Path + + @staticmethod + def _create_installation_access_token( + jwt_token: JWTToken, installation_id: int + ) -> HttpResponse: + return http_request( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + data={}, + headers={"Authorization": f"Bearer {jwt_token.get()}"}, + method="POST", + ) + + @staticmethod + def _generate_token( + jwt_token: JWTToken, installation_id: int + ) -> tuple[str, datetime]: + token = InstallationToken._create_installation_access_token( + jwt_token, installation_id + ).json()["token"] + expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME + + return token, expiration + + def __init__( + self, + jwt_token: JWTToken, + installation_id: int, + installations_token_map_name: Path, + installation_token: None | tuple[str, datetime] = None, + ) -> None: + self.jwt_token = jwt_token + self.installation_id = installation_id + self.installations_token_map_name = installations_token_map_name + + if installation_token is None: + self.token, self.expiration = InstallationToken._generate_token( + self.jwt_token, self.installation_id + ) + self._save() + else: + self.token, self.expiration = installation_token + + def get(self) -> str: + self.verify() + return self.token + + def get_as_secret(self) -> str: + return f"%(secret:github-token-{self.installation_id})" + + def verify(self) -> None: + if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8: + self.token, self.expiration = InstallationToken._generate_token( + self.jwt_token, self.installation_id + ) + self._save() + + def _save(self) -> None: + # of format: + # { + # 123: { + # expiration: , + # token: "token" + # } + # } + installations_token_map: dict[int, Any] + if self.installations_token_map_name.exists(): + installations_token_map = json.loads( + self.installations_token_map_name.read_text() + ) + else: + installations_token_map = {} + + installations_token_map[self.installation_id] = { + "expiration": self.expiration.isoformat(), + "token": self.token, + } + + atomic_write_file( + self.installations_token_map_name, json.dumps(installations_token_map) + ) diff --git a/buildbot_nix/github/jwt_token.py b/buildbot_nix/github/jwt_token.py new file mode 100644 index 0000000..79fe56b --- /dev/null +++ b/buildbot_nix/github/jwt_token.py @@ -0,0 +1,84 @@ +import base64 +import json +import os +import subprocess +from datetime import UTC, datetime, timedelta +from typing import Any + +from .repo_token import RepoToken + + +class JWTToken(RepoToken): + app_id: int + app_private_key: str + lifetime: timedelta + + expiration: datetime + token: str + + def __init__( + self, + app_id: int, + app_private_key: str, + lifetime: timedelta = timedelta(minutes=10), + ) -> None: + self.app_id = app_id + self.app_private_key = app_private_key + self.lifetime = lifetime + + self.token, self.expiration = JWTToken.generate_token( + self.app_id, self.app_private_key, lifetime + ) + + @staticmethod + def generate_token( + app_id: int, app_private_key: str, lifetime: timedelta + ) -> tuple[str, datetime]: + def build_jwt_payload( + app_id: int, lifetime: timedelta + ) -> tuple[dict[str, Any], datetime]: + jwt_iat_drift: timedelta = timedelta(seconds=60) + now: datetime = datetime.now(tz=UTC) + iat: datetime = now - jwt_iat_drift + exp: datetime = iat + lifetime + jwt_payload = { + "iat": int(iat.timestamp()), + "exp": int(exp.timestamp()), + "iss": str(app_id), + } + return (jwt_payload, exp) + + def rs256_sign(data: str, private_key: str) -> str: + signature = subprocess.run( + ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key], + input=data.encode("utf-8"), + stdout=subprocess.PIPE, + check=True, + cwd=os.environ.get("CREDENTIALS_DIRECTORY"), + ).stdout + return base64url(signature) + + def base64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8") + + jwt, expiration = build_jwt_payload(app_id, lifetime) + jwt_payload = json.dumps(jwt).encode("utf-8") + json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8") + encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}" + encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key) + return (f"{encoded_jwt_parts}.{encoded_mac}", expiration) + + # installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt) + + # return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations)) + + def get(self) -> str: + if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8: + self.token, self.expiration = JWTToken.generate_token( + self.app_id, self.app_private_key, self.lifetime + ) + + return self.token + + def get_as_secret(self) -> str: + return "%(secret:github-jwt-token)" diff --git a/buildbot_nix/github/legacy_token.py b/buildbot_nix/github/legacy_token.py new file mode 100644 index 0000000..3aff388 --- /dev/null +++ b/buildbot_nix/github/legacy_token.py @@ -0,0 +1,14 @@ +from .repo_token import RepoToken + + +class LegacyToken(RepoToken): + token: str + + def __init__(self, token: str) -> None: + self.token = token + + def get(self) -> str: + return self.token + + def get_as_secret(self) -> str: + return "%(secret:github-token)" diff --git a/buildbot_nix/github/repo_token.py b/buildbot_nix/github/repo_token.py new file mode 100644 index 0000000..398702a --- /dev/null +++ b/buildbot_nix/github/repo_token.py @@ -0,0 +1,11 @@ +from abc import abstractmethod + + +class RepoToken: + @abstractmethod + def get(self) -> str: + pass + + @abstractmethod + def get_as_secret(self) -> str: + pass diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 6f40319..289d7c4 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,10 +1,13 @@ import json import os import signal -from collections.abc import Generator +from abc import ABC, abstractmethod +from collections.abc import Callable, Generator from dataclasses import dataclass +from datetime import ( + datetime, +) from pathlib import Path -from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Any from buildbot.config.builder import BuilderConfig @@ -13,10 +16,12 @@ from buildbot.process.buildstep import BuildStep from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.reporters.github import GitHubStatusPush +from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth from twisted.internet import defer, threads +from twisted.logger import Logger from twisted.python import log from twisted.python.failure import Failure @@ -24,24 +29,96 @@ if TYPE_CHECKING: from buildbot.process.log import StreamLog from .common import ( + atomic_write_file, http_request, paginated_github_request, slugify_project_name, ) +from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy +from .github.installation_token import InstallationToken +from .github.jwt_token import JWTToken +from .github.legacy_token import ( + LegacyToken, +) +from .github.repo_token import ( + RepoToken, +) from .projects import GitBackend, GitProject from .secrets import read_secret_file +tlog = Logger() -class ReloadGithubProjects(BuildStep): + +def get_installations(jwt_token: JWTToken) -> list[int]: + installations = paginated_github_request( + "https://api.github.com/app/installations?per_page=100", jwt_token.get() + ) + + return [installation["id"] for installation in installations] + + +class ReloadGithubInstallations(BuildStep): name = "reload_github_projects" - def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: - self.token = token + jwt_token: JWTToken + project_cache_file: Path + installation_token_map_name: Path + project_id_map_name: Path + + def __init__( + self, + jwt_token: JWTToken, + project_cache_file: Path, + installation_token_map_name: Path, + project_id_map_name: Path, + **kwargs: Any, + ) -> None: + self.jwt_token = jwt_token + self.installation_token_map_name = installation_token_map_name + self.project_id_map_name = project_id_map_name self.project_cache_file = project_cache_file super().__init__(**kwargs) def reload_projects(self) -> None: - refresh_projects(self.token, self.project_cache_file) + installation_token_map = GithubBackend.create_missing_installations( + self.jwt_token, + self.installation_token_map_name, + GithubBackend.load_installations( + self.jwt_token, + self.installation_token_map_name, + ), + get_installations(self.jwt_token), + ) + + repos: list[Any] = [] + project_id_map: dict[str, int] = {} + + repos = [] + + for k, v in installation_token_map.items(): + new_repos = refresh_projects( + v.get(), + self.project_cache_file, + clear=True, + api_endpoint="/installation/repositories", + subkey="repositories", + require_admin=False, + ) + + for repo in new_repos: + repo["installation_id"] = k + + repos.extend(new_repos) + + for repo in new_repos: + project_id_map[repo["full_name"]] = k + + atomic_write_file(self.project_cache_file, json.dumps(repos)) + atomic_write_file(self.project_id_map_name, json.dumps(project_id_map)) + + tlog.info( + f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token." + ) @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: @@ -65,36 +142,378 @@ class ReloadGithubProjects(BuildStep): return util.FAILURE +class ReloadGithubProjects(BuildStep): + name = "reload_github_projects" + + def __init__( + self, token: RepoToken, project_cache_file: Path, **kwargs: Any + ) -> None: + self.token = token + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + repos: list[Any] = [] + + if self.project_cache_file.exists(): + repos = json.loads(self.project_cache_file.read_text()) + + refresh_projects(self.token.get(), self.project_cache_file) + + atomic_write_file(self.project_cache_file, json.dumps(repos)) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + log: StreamLog = yield self.addLog("log") + log.addStderr(f"Failed to reload project list: {self.error_msg}") + return util.FAILURE + + +class GitHubAppStatusPush(GitHubStatusPush): + token_source: Callable[[int], RepoToken] + project_id_source: Callable[[str], int] + saved_args: dict[str, Any] + saved_kwargs: dict[str, Any] + + def checkConfig( + self, + token_source: Callable[[int], RepoToken], + project_id_source: Callable[[str], int], + context: Any = None, + baseURL: Any = None, + verbose: Any = False, + debug: Any = None, + verify: Any = None, + generators: Any = None, + **kwargs: dict[str, Any], + ) -> Any: + if generators is None: + generators = self._create_default_generators() + + if "token" in kwargs: + del kwargs["token"] + super().checkConfig( + token="", + context=context, + baseURL=baseURL, + verbose=verbose, + debug=debug, + verify=verify, + generators=generators, + **kwargs, + ) + + def reconfigService( + self, + token_source: Callable[[int], RepoToken], + project_id_source: Callable[[str], int], + context: Any = None, + baseURL: Any = None, + verbose: Any = False, + debug: Any = None, + verify: Any = None, + generators: Any = None, + **kwargs: dict[str, Any], + ) -> Any: + if "saved_args" not in self or self.saved_args is None: + self.saved_args = {} + self.token_source = token_source + self.project_id_source = project_id_source + self.saved_kwargs = kwargs + self.saved_args["context"] = context + self.saved_args["baseURL"] = baseURL + self.saved_args["verbose"] = verbose + self.saved_args["debug"] = debug + self.saved_args["verify"] = verify + self.saved_args["generators"] = generators + + if generators is None: + generators = self._create_default_generators() + + if "token" in kwargs: + del kwargs["token"] + super().reconfigService( + token="", + context=context, + baseURL=baseURL, + verbose=verbose, + debug=debug, + verify=verify, + generators=generators, + **kwargs, + ) + + def sendMessage(self, reports: Any) -> Any: + build = reports[0]["builds"][0] + sourcestamps = build["buildset"].get("sourcestamps") + if not sourcestamps: + return None + + for sourcestamp in sourcestamps: + build["buildset"]["sourcestamps"] = [sourcestamp] + + token: str + + if "project" in sourcestamp and sourcestamp["project"] != "": + token = self.token_source( + self.project_id_source(sourcestamp["project"]) + ).get() + else: + token = "" + + super().reconfigService( + token, + context=self.saved_args["context"], + baseURL=self.saved_args["baseURL"], + verbose=self.saved_args["verbose"], + debug=self.saved_args["debug"], + verify=self.saved_args["verify"], + generators=self.saved_args["generators"], + **self.saved_kwargs, + ) + + return super().sendMessage(reports) + + return None + + +class GithubAuthBackend(ABC): + @abstractmethod + def get_general_token(self) -> RepoToken: + pass + + @abstractmethod + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + pass + + @abstractmethod + def create_secret_providers(self) -> list[SecretProviderBase]: + pass + + @abstractmethod + def create_reporter(self) -> ReporterBase: + pass + + @abstractmethod + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + pass + + +class GithubLegacyAuthBackend(GithubAuthBackend): + auth_type: AuthTypeLegacy + + token: LegacyToken + + def __init__(self, auth_type: AuthTypeLegacy) -> None: + self.auth_type = auth_type + self.token = LegacyToken(read_secret_file(auth_type.token_secret_name)) + + def get_general_token(self) -> RepoToken: + return self.token + + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + return self.token + + def create_secret_providers(self) -> list[SecretProviderBase]: + return [GitHubLagacySecretService(self.token)] + + def create_reporter(self) -> ReporterBase: + return GitHubStatusPush( + token=self.token.get(), + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + return ReloadGithubProjects( + token=self.token, project_cache_file=project_cache_file + ) + + +class GitHubLagacySecretService(SecretProviderBase): + name = "GitHubLegacySecretService" + token: LegacyToken + + def reconfigService(self, token: LegacyToken) -> None: + self.token = token + + def get(self, entry: str) -> str | None: + """ + get the value from the file identified by 'entry' + """ + if entry.startswith("github-token"): + return self.token.get() + return None + + +class GithubAppAuthBackend(GithubAuthBackend): + auth_type: AuthTypeApp + + jwt_token: JWTToken + installation_tokens: dict[int, InstallationToken] + project_id_map: dict[str, int] + + def __init__(self, auth_type: AuthTypeApp) -> None: + self.auth_type = auth_type + self.jwt_token = JWTToken( + self.auth_type.app_id, self.auth_type.app_secret_key_name + ) + self.installation_tokens = GithubBackend.load_installations( + self.jwt_token, + self.auth_type.app_installation_token_map_name, + ) + if self.auth_type.app_project_id_map_name.exists(): + self.project_id_map = json.loads( + self.auth_type.app_project_id_map_name.read_text() + ) + else: + self.project_id_map = {} + + def get_general_token(self) -> RepoToken: + return self.jwt_token + + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + assert "installation_id" in repo + return self.installation_tokens[repo["installation_id"]] + + def create_secret_providers(self) -> list[SecretProviderBase]: + return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)] + + def create_reporter(self) -> ReporterBase: + return GitHubAppStatusPush( + token_source=lambda iid: self.installation_tokens[iid], + project_id_source=lambda project: self.project_id_map[project], + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + return ReloadGithubInstallations( + self.jwt_token, + project_cache_file, + self.auth_type.app_installation_token_map_name, + self.auth_type.app_project_id_map_name, + ) + + +class GitHubAppSecretService(SecretProviderBase): + name = "GitHubAppSecretService" + installation_tokens: dict[int, InstallationToken] + jwt_token: JWTToken + + def reconfigService( + self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken + ) -> None: + self.installation_tokens = installation_tokens + self.jwt_token = jwt_token + + def get(self, entry: str) -> str | None: + """ + get the value from the file identified by 'entry' + """ + if entry.startswith("github-token-"): + return self.installation_tokens[ + int(entry.removeprefix("github-token-")) + ].get() + if entry == "github-jwt-token": + return self.jwt_token.get() + return None + + @dataclass class GithubConfig: oauth_id: str | None - + auth_type: AuthType # TODO unused buildbot_user: str oauth_secret_name: str = "github-oauth-secret" - token_secret_name: str = "github-token" webhook_secret_name: str = "github-webhook-secret" project_cache_file: Path = Path("github-project-cache.json") topic: str | None = "build-with-buildbot" - def token(self) -> str: - return read_secret_file(self.token_secret_name) - @dataclass class GithubBackend(GitBackend): config: GithubConfig webhook_secret: str + auth_backend: GithubAuthBackend + def __init__(self, config: GithubConfig) -> None: self.config = config self.webhook_secret = read_secret_file(self.config.webhook_secret_name) + if isinstance(self.config.auth_type, AuthTypeLegacy): + self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type) + elif isinstance(self.config.auth_type, AuthTypeApp): + self.auth_backend = GithubAppAuthBackend(self.config.auth_type) + + @staticmethod + def load_installations( + jwt_token: JWTToken, installations_token_map_name: Path + ) -> dict[int, InstallationToken]: + initial_installations_map: dict[str, Any] + if installations_token_map_name.exists(): + initial_installations_map = json.loads( + installations_token_map_name.read_text() + ) + else: + initial_installations_map = {} + + installations_map: dict[int, InstallationToken] = {} + + for iid, installation in initial_installations_map.items(): + token: str = installation["token"] + expiration: datetime = datetime.fromisoformat(installation["expiration"]) + installations_map[int(iid)] = InstallationToken( + jwt_token, + int(iid), + installations_token_map_name, + installation_token=(token, expiration), + ) + + return installations_map + + @staticmethod + def create_missing_installations( + jwt_token: JWTToken, + installations_token_map_name: Path, + installations_map: dict[int, InstallationToken], + installations: list[int], + ) -> dict[int, InstallationToken]: + for installation in set(installations) - installations_map.keys(): + installations_map[installation] = InstallationToken( + jwt_token, + installation, + installations_token_map_name, + ) + + return installations_map + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" factory = util.BuildFactory() factory.addStep( - ReloadGithubProjects(self.config.token(), self.config.project_cache_file), + self.auth_backend.create_reload_builder_step(self.config.project_cache_file) ) return util.BuilderConfig( name=self.reload_builder_name, @@ -103,24 +522,18 @@ class GithubBackend(GitBackend): ) def create_reporter(self) -> ReporterBase: - return GitHubStatusPush( - token=self.config.token(), - # Since we dynamically create build steps, - # we use `virtual_builder_name` in the webinterface - # so that we distinguish what has beeing build - context=Interpolate("buildbot/%(prop:status_name)s"), - ) + return self.auth_backend.create_reporter() def create_change_hook(self) -> dict[str, Any]: return { "secret": self.webhook_secret, "strict": True, - "token": self.config.token(), + "token": self.auth_backend.get_general_token().get(), "github_property_whitelist": ["github.base.sha", "github.head.sha"], } def create_avatar_method(self) -> AvatarBase | None: - return AvatarGitHub(token=self.config.token()) + return AvatarGitHub(token=self.auth_backend.get_general_token().get()) def create_auth(self) -> AuthBase: assert self.config.oauth_id is not None, "GitHub OAuth ID is required" @@ -130,6 +543,9 @@ class GithubBackend(GitBackend): apiVersion=4, ) + def create_secret_providers(self) -> list[SecretProviderBase]: + return self.auth_backend.create_secret_providers() + def load_projects(self) -> list["GitProject"]: if not self.config.project_cache_file.exists(): return [] @@ -138,12 +554,18 @@ class GithubBackend(GitBackend): json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"], ) + tlog.info(f"Loading {len(repos)} cached repositories.") return list( filter( lambda project: self.config.topic is not None and self.config.topic in project.topics, [ - GithubProject(self.config, self.webhook_secret, repo) + GithubProject( + self.auth_backend.get_repo_token(repo), + self.config, + self.webhook_secret, + repo, + ) for repo in repos ], ) @@ -171,10 +593,18 @@ class GithubBackend(GitBackend): class GithubProject(GitProject): config: GithubConfig + webhook_secret: str + data: dict[str, Any] + token: RepoToken def __init__( - self, config: GithubConfig, webhook_secret: str, data: dict[str, Any] + self, + token: RepoToken, + config: GithubConfig, + webhook_secret: str, + data: dict[str, Any], ) -> None: + self.token = token self.config = config self.webhook_secret = webhook_secret self.data = data @@ -187,7 +617,7 @@ class GithubProject(GitProject): ) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", - self.config.token(), + self.token.get(), ) config = dict( url=webhook_url + "change_hook/github", @@ -199,7 +629,7 @@ class GithubProject(GitProject): name="web", active=True, events=["push", "pull_request"], config=config ) headers = { - "Authorization": f"Bearer {self.config.token()}", + "Authorization": f"Bearer {self.token.get()}", "Accept": "application/vnd.github+json", "Content-Type": "application/json", "X-GitHub-Api-Version": "2022-11-28", @@ -217,7 +647,7 @@ class GithubProject(GitProject): ) def get_project_url(self) -> str: - return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}" + return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}" @property def pretty_type(self) -> str: @@ -260,14 +690,25 @@ class GithubProject(GitProject): return self.data["owner"]["type"] == "Organization" -def refresh_projects(github_token: str, repo_cache_file: Path) -> None: - repos = [] +def refresh_projects( + github_token: str, + repo_cache_file: Path, + repos: list[Any] | None = None, + clear: bool = True, + api_endpoint: str = "/user/repos", + subkey: None | str = None, + require_admin: bool = True, +) -> list[Any]: + if repos is None: + repos = [] for repo in paginated_github_request( - "https://api.github.com/user/repos?per_page=100", + f"https://api.github.com{api_endpoint}?per_page=100", github_token, + subkey=subkey, ): - if not repo["permissions"]["admin"]: + # TODO actually check for this properly + if not repo["permissions"]["admin"] and require_admin: name = repo["full_name"] log.msg( f"skipping {name} because we do not have admin privileges, needed for hook management", @@ -275,12 +716,4 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: else: repos.append(repo) - with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: - path = Path(f.name) - try: - f.write(json.dumps(repos)) - f.flush() - path.rename(repo_cache_file) - except OSError: - path.unlink() - raise + return repos diff --git a/buildbot_nix/projects.py b/buildbot_nix/projects.py index 947bc1f..23ef1db 100644 --- a/buildbot_nix/projects.py +++ b/buildbot_nix/projects.py @@ -3,6 +3,7 @@ from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.reporters.base import ReporterBase +from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase @@ -28,6 +29,9 @@ class GitBackend(ABC): def create_auth(self) -> AuthBase: pass + def create_secret_providers(self) -> list[SecretProviderBase]: + return [] + @abstractmethod def load_projects(self) -> list["GitProject"]: pass diff --git a/nix/master.nix b/nix/master.nix index e4881e9..329a34d 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -24,6 +24,24 @@ in "admins" ] ) + (mkRenamedOptionModule + [ + "services" + "buildbot-nix" + "master" + "github" + "tokenFile" + ] + [ + "services" + "buildbot-nix" + "master" + "github" + "authType" + "legacy" + "tokenFile" + ] + ) ]; options = { @@ -106,10 +124,33 @@ in default = cfg.authBackend == "github"; }; - tokenFile = lib.mkOption { - type = lib.types.path; - description = "Github token file"; + authType = { + legacy = { + enable = lib.mkEnableOption ""; + tokenFile = lib.mkOption { + type = lib.types.path; + description = "Github token file"; + }; + }; + + app = { + enable = lib.mkEnableOption ""; + id = lib.mkOption { + type = lib.types.int; + description = '' + GitHub app ID. + ''; + }; + + secretKeyFile = lib.mkOption { + type = lib.types.str; + description = '' + GitHub app secret key file location. + ''; + }; + }; }; + webhookSecretFile = lib.mkOption { type = lib.types.path; description = "Github webhook secret file"; @@ -243,7 +284,16 @@ in home = "/var/lib/buildbot"; extraImports = '' from datetime import timedelta - from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig + from buildbot_nix import ( + GithubConfig, + NixConfigurator, + CachixConfig, + GiteaConfig, + ) + from buildbot_nix.github.auth._type import ( + AuthTypeLegacy, + AuthTypeApp, + ) ''; configurators = [ '' @@ -260,6 +310,18 @@ in oauth_id=${builtins.toJSON cfg.github.oauthId}, buildbot_user=${builtins.toJSON cfg.github.user}, topic=${builtins.toJSON cfg.github.topic}, + auth_type=${ + if cfg.github.authType.legacy.enable then + ''AuthTypeLegacy()'' + else if cfg.github.authType.app.enable then + '' + AuthTypeApp( + app_id=${toString cfg.github.authType.app.id}, + ) + '' + else + throw "Universe broke" + } )" }, gitea=${ @@ -323,6 +385,9 @@ in systemd.services.buildbot-master = { after = [ "postgresql.service" ]; + path = [ + pkgs.openssl + ]; serviceConfig = { # in master.py we read secrets from $CREDENTIALS_DIRECTORY LoadCredential = @@ -337,10 +402,15 @@ in ( cfg.cachix.authTokenFile != null ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" - ++ lib.optionals (cfg.github.enable) [ - "github-token:${cfg.github.tokenFile}" + ++ lib.optionals (cfg.github.enable) ([ "github-webhook-secret:${cfg.github.webhookSecretFile}" ] + ++ lib.optionals (cfg.github.authType.legacy.enable) [ + "github-token:${cfg.github.authType.legacy.tokenFile}" + ] + ++ lib.optionals (cfg.github.authType.app.enable) [ + "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}" + ]) ++ lib.optionals cfg.gitea.enable [ "gitea-token:${cfg.gitea.tokenFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"