From 067f3e0fc1c12ddc8f00edca27c452f14197c3ae Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 18 May 2024 21:36:22 +0200 Subject: [PATCH 01/13] Add GitHub App support Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 2 + buildbot_nix/common.py | 23 +- buildbot_nix/github/__init__.py | 1 + buildbot_nix/github/auth/__init__.py | 1 + buildbot_nix/github/auth/_type.py | 23 + buildbot_nix/github/installation_token.py | 104 +++++ buildbot_nix/github/jwt_token.py | 84 ++++ buildbot_nix/github/legacy_token.py | 14 + buildbot_nix/github/repo_token.py | 11 + buildbot_nix/github_projects.py | 511 ++++++++++++++++++++-- buildbot_nix/projects.py | 4 + nix/master.nix | 82 +++- 12 files changed, 813 insertions(+), 47 deletions(-) create mode 100644 buildbot_nix/github/__init__.py create mode 100644 buildbot_nix/github/auth/__init__.py create mode 100644 buildbot_nix/github/auth/_type.py create mode 100644 buildbot_nix/github/installation_token.py create mode 100644 buildbot_nix/github/jwt_token.py create mode 100644 buildbot_nix/github/legacy_token.py create mode 100644 buildbot_nix/github/repo_token.py diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index e16fdea..abd18dc 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -872,6 +872,8 @@ class NixConfigurator(ConfiguratorBase): ], ) config["services"].append(backend.create_reporter()) + config.setdefault("secretProviders", []) + config["secretsProviders"].extend(backend.create_secret_providers()) systemd_secrets = SecretInAFile( dirname=os.environ["CREDENTIALS_DIRECTORY"], diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py index 6dcd4f3..b4d3941 100644 --- a/buildbot_nix/common.py +++ b/buildbot_nix/common.py @@ -2,6 +2,8 @@ import contextlib import http.client import json import urllib.request +from pathlib import Path +from tempfile import NamedTemporaryFile from typing import Any @@ -9,7 +11,9 @@ def slugify_project_name(name: str) -> str: return name.replace(".", "-").replace("/", "-") -def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: +def paginated_github_request( + url: str, token: str, subkey: None | str = None +) -> list[dict[str, Any]]: next_url: str | None = url items = [] while next_url: @@ -29,7 +33,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: link_parts = link.split(";") if link_parts[1].strip() == 'rel="next"': next_url = link_parts[0][1:-1] - items += res.json() + if subkey is not None: + items += res.json()[subkey] + else: + items += res.json() return items @@ -78,3 +85,15 @@ def http_request( msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" raise HttpError(msg) from e return HttpResponse(resp) + + +def atomic_write_file(file: Path, data: str) -> None: + with NamedTemporaryFile("w", delete=False, dir=file.parent) as f: + path = Path(f.name) + try: + f.write(data) + f.flush() + path.rename(file) + except OSError: + path.unlink() + raise diff --git a/buildbot_nix/github/__init__.py b/buildbot_nix/github/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot_nix/github/__init__.py @@ -0,0 +1 @@ + diff --git a/buildbot_nix/github/auth/__init__.py b/buildbot_nix/github/auth/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot_nix/github/auth/__init__.py @@ -0,0 +1 @@ + diff --git a/buildbot_nix/github/auth/_type.py b/buildbot_nix/github/auth/_type.py new file mode 100644 index 0000000..abbff13 --- /dev/null +++ b/buildbot_nix/github/auth/_type.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class AuthType: + pass + + +@dataclass +class AuthTypeLegacy(AuthType): + token_secret_name: str = "github-token" + + +@dataclass +class AuthTypeApp(AuthType): + app_id: int + app_secret_key_name: str = "github-app-secret-key" + app_installation_token_map_name: Path = Path( + "github-app-installation-token-map.json" + ) + app_project_id_map_name: Path = Path("github-app-project-id-map-name") + app_jwt_token_name: Path = Path("github-app-jwt-token") diff --git a/buildbot_nix/github/installation_token.py b/buildbot_nix/github/installation_token.py new file mode 100644 index 0000000..a5affce --- /dev/null +++ b/buildbot_nix/github/installation_token.py @@ -0,0 +1,104 @@ +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from buildbot_nix.common import ( + HttpResponse, + atomic_write_file, + http_request, +) + +from .jwt_token import JWTToken +from .repo_token import RepoToken + + +class InstallationToken(RepoToken): + GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60) + + jwt_token: JWTToken + installation_id: int + + token: str + expiration: datetime + installations_token_map_name: Path + + @staticmethod + def _create_installation_access_token( + jwt_token: JWTToken, installation_id: int + ) -> HttpResponse: + return http_request( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + data={}, + headers={"Authorization": f"Bearer {jwt_token.get()}"}, + method="POST", + ) + + @staticmethod + def _generate_token( + jwt_token: JWTToken, installation_id: int + ) -> tuple[str, datetime]: + token = InstallationToken._create_installation_access_token( + jwt_token, installation_id + ).json()["token"] + expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME + + return token, expiration + + def __init__( + self, + jwt_token: JWTToken, + installation_id: int, + installations_token_map_name: Path, + installation_token: None | tuple[str, datetime] = None, + ) -> None: + self.jwt_token = jwt_token + self.installation_id = installation_id + self.installations_token_map_name = installations_token_map_name + + if installation_token is None: + self.token, self.expiration = InstallationToken._generate_token( + self.jwt_token, self.installation_id + ) + self._save() + else: + self.token, self.expiration = installation_token + + def get(self) -> str: + self.verify() + return self.token + + def get_as_secret(self) -> str: + return f"%(secret:github-token-{self.installation_id})" + + def verify(self) -> None: + if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8: + self.token, self.expiration = InstallationToken._generate_token( + self.jwt_token, self.installation_id + ) + self._save() + + def _save(self) -> None: + # of format: + # { + # 123: { + # expiration: , + # token: "token" + # } + # } + installations_token_map: dict[int, Any] + if self.installations_token_map_name.exists(): + installations_token_map = json.loads( + self.installations_token_map_name.read_text() + ) + else: + installations_token_map = {} + + installations_token_map[self.installation_id] = { + "expiration": self.expiration.isoformat(), + "token": self.token, + } + + atomic_write_file( + self.installations_token_map_name, json.dumps(installations_token_map) + ) diff --git a/buildbot_nix/github/jwt_token.py b/buildbot_nix/github/jwt_token.py new file mode 100644 index 0000000..79fe56b --- /dev/null +++ b/buildbot_nix/github/jwt_token.py @@ -0,0 +1,84 @@ +import base64 +import json +import os +import subprocess +from datetime import UTC, datetime, timedelta +from typing import Any + +from .repo_token import RepoToken + + +class JWTToken(RepoToken): + app_id: int + app_private_key: str + lifetime: timedelta + + expiration: datetime + token: str + + def __init__( + self, + app_id: int, + app_private_key: str, + lifetime: timedelta = timedelta(minutes=10), + ) -> None: + self.app_id = app_id + self.app_private_key = app_private_key + self.lifetime = lifetime + + self.token, self.expiration = JWTToken.generate_token( + self.app_id, self.app_private_key, lifetime + ) + + @staticmethod + def generate_token( + app_id: int, app_private_key: str, lifetime: timedelta + ) -> tuple[str, datetime]: + def build_jwt_payload( + app_id: int, lifetime: timedelta + ) -> tuple[dict[str, Any], datetime]: + jwt_iat_drift: timedelta = timedelta(seconds=60) + now: datetime = datetime.now(tz=UTC) + iat: datetime = now - jwt_iat_drift + exp: datetime = iat + lifetime + jwt_payload = { + "iat": int(iat.timestamp()), + "exp": int(exp.timestamp()), + "iss": str(app_id), + } + return (jwt_payload, exp) + + def rs256_sign(data: str, private_key: str) -> str: + signature = subprocess.run( + ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key], + input=data.encode("utf-8"), + stdout=subprocess.PIPE, + check=True, + cwd=os.environ.get("CREDENTIALS_DIRECTORY"), + ).stdout + return base64url(signature) + + def base64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8") + + jwt, expiration = build_jwt_payload(app_id, lifetime) + jwt_payload = json.dumps(jwt).encode("utf-8") + json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8") + encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}" + encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key) + return (f"{encoded_jwt_parts}.{encoded_mac}", expiration) + + # installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt) + + # return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations)) + + def get(self) -> str: + if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8: + self.token, self.expiration = JWTToken.generate_token( + self.app_id, self.app_private_key, self.lifetime + ) + + return self.token + + def get_as_secret(self) -> str: + return "%(secret:github-jwt-token)" diff --git a/buildbot_nix/github/legacy_token.py b/buildbot_nix/github/legacy_token.py new file mode 100644 index 0000000..3aff388 --- /dev/null +++ b/buildbot_nix/github/legacy_token.py @@ -0,0 +1,14 @@ +from .repo_token import RepoToken + + +class LegacyToken(RepoToken): + token: str + + def __init__(self, token: str) -> None: + self.token = token + + def get(self) -> str: + return self.token + + def get_as_secret(self) -> str: + return "%(secret:github-token)" diff --git a/buildbot_nix/github/repo_token.py b/buildbot_nix/github/repo_token.py new file mode 100644 index 0000000..398702a --- /dev/null +++ b/buildbot_nix/github/repo_token.py @@ -0,0 +1,11 @@ +from abc import abstractmethod + + +class RepoToken: + @abstractmethod + def get(self) -> str: + pass + + @abstractmethod + def get_as_secret(self) -> str: + pass diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 6f40319..289d7c4 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,10 +1,13 @@ import json import os import signal -from collections.abc import Generator +from abc import ABC, abstractmethod +from collections.abc import Callable, Generator from dataclasses import dataclass +from datetime import ( + datetime, +) from pathlib import Path -from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Any from buildbot.config.builder import BuilderConfig @@ -13,10 +16,12 @@ from buildbot.process.buildstep import BuildStep from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.reporters.github import GitHubStatusPush +from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth from twisted.internet import defer, threads +from twisted.logger import Logger from twisted.python import log from twisted.python.failure import Failure @@ -24,24 +29,96 @@ if TYPE_CHECKING: from buildbot.process.log import StreamLog from .common import ( + atomic_write_file, http_request, paginated_github_request, slugify_project_name, ) +from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy +from .github.installation_token import InstallationToken +from .github.jwt_token import JWTToken +from .github.legacy_token import ( + LegacyToken, +) +from .github.repo_token import ( + RepoToken, +) from .projects import GitBackend, GitProject from .secrets import read_secret_file +tlog = Logger() -class ReloadGithubProjects(BuildStep): + +def get_installations(jwt_token: JWTToken) -> list[int]: + installations = paginated_github_request( + "https://api.github.com/app/installations?per_page=100", jwt_token.get() + ) + + return [installation["id"] for installation in installations] + + +class ReloadGithubInstallations(BuildStep): name = "reload_github_projects" - def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: - self.token = token + jwt_token: JWTToken + project_cache_file: Path + installation_token_map_name: Path + project_id_map_name: Path + + def __init__( + self, + jwt_token: JWTToken, + project_cache_file: Path, + installation_token_map_name: Path, + project_id_map_name: Path, + **kwargs: Any, + ) -> None: + self.jwt_token = jwt_token + self.installation_token_map_name = installation_token_map_name + self.project_id_map_name = project_id_map_name self.project_cache_file = project_cache_file super().__init__(**kwargs) def reload_projects(self) -> None: - refresh_projects(self.token, self.project_cache_file) + installation_token_map = GithubBackend.create_missing_installations( + self.jwt_token, + self.installation_token_map_name, + GithubBackend.load_installations( + self.jwt_token, + self.installation_token_map_name, + ), + get_installations(self.jwt_token), + ) + + repos: list[Any] = [] + project_id_map: dict[str, int] = {} + + repos = [] + + for k, v in installation_token_map.items(): + new_repos = refresh_projects( + v.get(), + self.project_cache_file, + clear=True, + api_endpoint="/installation/repositories", + subkey="repositories", + require_admin=False, + ) + + for repo in new_repos: + repo["installation_id"] = k + + repos.extend(new_repos) + + for repo in new_repos: + project_id_map[repo["full_name"]] = k + + atomic_write_file(self.project_cache_file, json.dumps(repos)) + atomic_write_file(self.project_id_map_name, json.dumps(project_id_map)) + + tlog.info( + f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token." + ) @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: @@ -65,36 +142,378 @@ class ReloadGithubProjects(BuildStep): return util.FAILURE +class ReloadGithubProjects(BuildStep): + name = "reload_github_projects" + + def __init__( + self, token: RepoToken, project_cache_file: Path, **kwargs: Any + ) -> None: + self.token = token + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + repos: list[Any] = [] + + if self.project_cache_file.exists(): + repos = json.loads(self.project_cache_file.read_text()) + + refresh_projects(self.token.get(), self.project_cache_file) + + atomic_write_file(self.project_cache_file, json.dumps(repos)) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + log: StreamLog = yield self.addLog("log") + log.addStderr(f"Failed to reload project list: {self.error_msg}") + return util.FAILURE + + +class GitHubAppStatusPush(GitHubStatusPush): + token_source: Callable[[int], RepoToken] + project_id_source: Callable[[str], int] + saved_args: dict[str, Any] + saved_kwargs: dict[str, Any] + + def checkConfig( + self, + token_source: Callable[[int], RepoToken], + project_id_source: Callable[[str], int], + context: Any = None, + baseURL: Any = None, + verbose: Any = False, + debug: Any = None, + verify: Any = None, + generators: Any = None, + **kwargs: dict[str, Any], + ) -> Any: + if generators is None: + generators = self._create_default_generators() + + if "token" in kwargs: + del kwargs["token"] + super().checkConfig( + token="", + context=context, + baseURL=baseURL, + verbose=verbose, + debug=debug, + verify=verify, + generators=generators, + **kwargs, + ) + + def reconfigService( + self, + token_source: Callable[[int], RepoToken], + project_id_source: Callable[[str], int], + context: Any = None, + baseURL: Any = None, + verbose: Any = False, + debug: Any = None, + verify: Any = None, + generators: Any = None, + **kwargs: dict[str, Any], + ) -> Any: + if "saved_args" not in self or self.saved_args is None: + self.saved_args = {} + self.token_source = token_source + self.project_id_source = project_id_source + self.saved_kwargs = kwargs + self.saved_args["context"] = context + self.saved_args["baseURL"] = baseURL + self.saved_args["verbose"] = verbose + self.saved_args["debug"] = debug + self.saved_args["verify"] = verify + self.saved_args["generators"] = generators + + if generators is None: + generators = self._create_default_generators() + + if "token" in kwargs: + del kwargs["token"] + super().reconfigService( + token="", + context=context, + baseURL=baseURL, + verbose=verbose, + debug=debug, + verify=verify, + generators=generators, + **kwargs, + ) + + def sendMessage(self, reports: Any) -> Any: + build = reports[0]["builds"][0] + sourcestamps = build["buildset"].get("sourcestamps") + if not sourcestamps: + return None + + for sourcestamp in sourcestamps: + build["buildset"]["sourcestamps"] = [sourcestamp] + + token: str + + if "project" in sourcestamp and sourcestamp["project"] != "": + token = self.token_source( + self.project_id_source(sourcestamp["project"]) + ).get() + else: + token = "" + + super().reconfigService( + token, + context=self.saved_args["context"], + baseURL=self.saved_args["baseURL"], + verbose=self.saved_args["verbose"], + debug=self.saved_args["debug"], + verify=self.saved_args["verify"], + generators=self.saved_args["generators"], + **self.saved_kwargs, + ) + + return super().sendMessage(reports) + + return None + + +class GithubAuthBackend(ABC): + @abstractmethod + def get_general_token(self) -> RepoToken: + pass + + @abstractmethod + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + pass + + @abstractmethod + def create_secret_providers(self) -> list[SecretProviderBase]: + pass + + @abstractmethod + def create_reporter(self) -> ReporterBase: + pass + + @abstractmethod + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + pass + + +class GithubLegacyAuthBackend(GithubAuthBackend): + auth_type: AuthTypeLegacy + + token: LegacyToken + + def __init__(self, auth_type: AuthTypeLegacy) -> None: + self.auth_type = auth_type + self.token = LegacyToken(read_secret_file(auth_type.token_secret_name)) + + def get_general_token(self) -> RepoToken: + return self.token + + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + return self.token + + def create_secret_providers(self) -> list[SecretProviderBase]: + return [GitHubLagacySecretService(self.token)] + + def create_reporter(self) -> ReporterBase: + return GitHubStatusPush( + token=self.token.get(), + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + return ReloadGithubProjects( + token=self.token, project_cache_file=project_cache_file + ) + + +class GitHubLagacySecretService(SecretProviderBase): + name = "GitHubLegacySecretService" + token: LegacyToken + + def reconfigService(self, token: LegacyToken) -> None: + self.token = token + + def get(self, entry: str) -> str | None: + """ + get the value from the file identified by 'entry' + """ + if entry.startswith("github-token"): + return self.token.get() + return None + + +class GithubAppAuthBackend(GithubAuthBackend): + auth_type: AuthTypeApp + + jwt_token: JWTToken + installation_tokens: dict[int, InstallationToken] + project_id_map: dict[str, int] + + def __init__(self, auth_type: AuthTypeApp) -> None: + self.auth_type = auth_type + self.jwt_token = JWTToken( + self.auth_type.app_id, self.auth_type.app_secret_key_name + ) + self.installation_tokens = GithubBackend.load_installations( + self.jwt_token, + self.auth_type.app_installation_token_map_name, + ) + if self.auth_type.app_project_id_map_name.exists(): + self.project_id_map = json.loads( + self.auth_type.app_project_id_map_name.read_text() + ) + else: + self.project_id_map = {} + + def get_general_token(self) -> RepoToken: + return self.jwt_token + + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + assert "installation_id" in repo + return self.installation_tokens[repo["installation_id"]] + + def create_secret_providers(self) -> list[SecretProviderBase]: + return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)] + + def create_reporter(self) -> ReporterBase: + return GitHubAppStatusPush( + token_source=lambda iid: self.installation_tokens[iid], + project_id_source=lambda project: self.project_id_map[project], + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + return ReloadGithubInstallations( + self.jwt_token, + project_cache_file, + self.auth_type.app_installation_token_map_name, + self.auth_type.app_project_id_map_name, + ) + + +class GitHubAppSecretService(SecretProviderBase): + name = "GitHubAppSecretService" + installation_tokens: dict[int, InstallationToken] + jwt_token: JWTToken + + def reconfigService( + self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken + ) -> None: + self.installation_tokens = installation_tokens + self.jwt_token = jwt_token + + def get(self, entry: str) -> str | None: + """ + get the value from the file identified by 'entry' + """ + if entry.startswith("github-token-"): + return self.installation_tokens[ + int(entry.removeprefix("github-token-")) + ].get() + if entry == "github-jwt-token": + return self.jwt_token.get() + return None + + @dataclass class GithubConfig: oauth_id: str | None - + auth_type: AuthType # TODO unused buildbot_user: str oauth_secret_name: str = "github-oauth-secret" - token_secret_name: str = "github-token" webhook_secret_name: str = "github-webhook-secret" project_cache_file: Path = Path("github-project-cache.json") topic: str | None = "build-with-buildbot" - def token(self) -> str: - return read_secret_file(self.token_secret_name) - @dataclass class GithubBackend(GitBackend): config: GithubConfig webhook_secret: str + auth_backend: GithubAuthBackend + def __init__(self, config: GithubConfig) -> None: self.config = config self.webhook_secret = read_secret_file(self.config.webhook_secret_name) + if isinstance(self.config.auth_type, AuthTypeLegacy): + self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type) + elif isinstance(self.config.auth_type, AuthTypeApp): + self.auth_backend = GithubAppAuthBackend(self.config.auth_type) + + @staticmethod + def load_installations( + jwt_token: JWTToken, installations_token_map_name: Path + ) -> dict[int, InstallationToken]: + initial_installations_map: dict[str, Any] + if installations_token_map_name.exists(): + initial_installations_map = json.loads( + installations_token_map_name.read_text() + ) + else: + initial_installations_map = {} + + installations_map: dict[int, InstallationToken] = {} + + for iid, installation in initial_installations_map.items(): + token: str = installation["token"] + expiration: datetime = datetime.fromisoformat(installation["expiration"]) + installations_map[int(iid)] = InstallationToken( + jwt_token, + int(iid), + installations_token_map_name, + installation_token=(token, expiration), + ) + + return installations_map + + @staticmethod + def create_missing_installations( + jwt_token: JWTToken, + installations_token_map_name: Path, + installations_map: dict[int, InstallationToken], + installations: list[int], + ) -> dict[int, InstallationToken]: + for installation in set(installations) - installations_map.keys(): + installations_map[installation] = InstallationToken( + jwt_token, + installation, + installations_token_map_name, + ) + + return installations_map + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" factory = util.BuildFactory() factory.addStep( - ReloadGithubProjects(self.config.token(), self.config.project_cache_file), + self.auth_backend.create_reload_builder_step(self.config.project_cache_file) ) return util.BuilderConfig( name=self.reload_builder_name, @@ -103,24 +522,18 @@ class GithubBackend(GitBackend): ) def create_reporter(self) -> ReporterBase: - return GitHubStatusPush( - token=self.config.token(), - # Since we dynamically create build steps, - # we use `virtual_builder_name` in the webinterface - # so that we distinguish what has beeing build - context=Interpolate("buildbot/%(prop:status_name)s"), - ) + return self.auth_backend.create_reporter() def create_change_hook(self) -> dict[str, Any]: return { "secret": self.webhook_secret, "strict": True, - "token": self.config.token(), + "token": self.auth_backend.get_general_token().get(), "github_property_whitelist": ["github.base.sha", "github.head.sha"], } def create_avatar_method(self) -> AvatarBase | None: - return AvatarGitHub(token=self.config.token()) + return AvatarGitHub(token=self.auth_backend.get_general_token().get()) def create_auth(self) -> AuthBase: assert self.config.oauth_id is not None, "GitHub OAuth ID is required" @@ -130,6 +543,9 @@ class GithubBackend(GitBackend): apiVersion=4, ) + def create_secret_providers(self) -> list[SecretProviderBase]: + return self.auth_backend.create_secret_providers() + def load_projects(self) -> list["GitProject"]: if not self.config.project_cache_file.exists(): return [] @@ -138,12 +554,18 @@ class GithubBackend(GitBackend): json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"], ) + tlog.info(f"Loading {len(repos)} cached repositories.") return list( filter( lambda project: self.config.topic is not None and self.config.topic in project.topics, [ - GithubProject(self.config, self.webhook_secret, repo) + GithubProject( + self.auth_backend.get_repo_token(repo), + self.config, + self.webhook_secret, + repo, + ) for repo in repos ], ) @@ -171,10 +593,18 @@ class GithubBackend(GitBackend): class GithubProject(GitProject): config: GithubConfig + webhook_secret: str + data: dict[str, Any] + token: RepoToken def __init__( - self, config: GithubConfig, webhook_secret: str, data: dict[str, Any] + self, + token: RepoToken, + config: GithubConfig, + webhook_secret: str, + data: dict[str, Any], ) -> None: + self.token = token self.config = config self.webhook_secret = webhook_secret self.data = data @@ -187,7 +617,7 @@ class GithubProject(GitProject): ) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", - self.config.token(), + self.token.get(), ) config = dict( url=webhook_url + "change_hook/github", @@ -199,7 +629,7 @@ class GithubProject(GitProject): name="web", active=True, events=["push", "pull_request"], config=config ) headers = { - "Authorization": f"Bearer {self.config.token()}", + "Authorization": f"Bearer {self.token.get()}", "Accept": "application/vnd.github+json", "Content-Type": "application/json", "X-GitHub-Api-Version": "2022-11-28", @@ -217,7 +647,7 @@ class GithubProject(GitProject): ) def get_project_url(self) -> str: - return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}" + return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}" @property def pretty_type(self) -> str: @@ -260,14 +690,25 @@ class GithubProject(GitProject): return self.data["owner"]["type"] == "Organization" -def refresh_projects(github_token: str, repo_cache_file: Path) -> None: - repos = [] +def refresh_projects( + github_token: str, + repo_cache_file: Path, + repos: list[Any] | None = None, + clear: bool = True, + api_endpoint: str = "/user/repos", + subkey: None | str = None, + require_admin: bool = True, +) -> list[Any]: + if repos is None: + repos = [] for repo in paginated_github_request( - "https://api.github.com/user/repos?per_page=100", + f"https://api.github.com{api_endpoint}?per_page=100", github_token, + subkey=subkey, ): - if not repo["permissions"]["admin"]: + # TODO actually check for this properly + if not repo["permissions"]["admin"] and require_admin: name = repo["full_name"] log.msg( f"skipping {name} because we do not have admin privileges, needed for hook management", @@ -275,12 +716,4 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: else: repos.append(repo) - with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: - path = Path(f.name) - try: - f.write(json.dumps(repos)) - f.flush() - path.rename(repo_cache_file) - except OSError: - path.unlink() - raise + return repos diff --git a/buildbot_nix/projects.py b/buildbot_nix/projects.py index 947bc1f..23ef1db 100644 --- a/buildbot_nix/projects.py +++ b/buildbot_nix/projects.py @@ -3,6 +3,7 @@ from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.reporters.base import ReporterBase +from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase @@ -28,6 +29,9 @@ class GitBackend(ABC): def create_auth(self) -> AuthBase: pass + def create_secret_providers(self) -> list[SecretProviderBase]: + return [] + @abstractmethod def load_projects(self) -> list["GitProject"]: pass diff --git a/nix/master.nix b/nix/master.nix index e4881e9..329a34d 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -24,6 +24,24 @@ in "admins" ] ) + (mkRenamedOptionModule + [ + "services" + "buildbot-nix" + "master" + "github" + "tokenFile" + ] + [ + "services" + "buildbot-nix" + "master" + "github" + "authType" + "legacy" + "tokenFile" + ] + ) ]; options = { @@ -106,10 +124,33 @@ in default = cfg.authBackend == "github"; }; - tokenFile = lib.mkOption { - type = lib.types.path; - description = "Github token file"; + authType = { + legacy = { + enable = lib.mkEnableOption ""; + tokenFile = lib.mkOption { + type = lib.types.path; + description = "Github token file"; + }; + }; + + app = { + enable = lib.mkEnableOption ""; + id = lib.mkOption { + type = lib.types.int; + description = '' + GitHub app ID. + ''; + }; + + secretKeyFile = lib.mkOption { + type = lib.types.str; + description = '' + GitHub app secret key file location. + ''; + }; + }; }; + webhookSecretFile = lib.mkOption { type = lib.types.path; description = "Github webhook secret file"; @@ -243,7 +284,16 @@ in home = "/var/lib/buildbot"; extraImports = '' from datetime import timedelta - from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig + from buildbot_nix import ( + GithubConfig, + NixConfigurator, + CachixConfig, + GiteaConfig, + ) + from buildbot_nix.github.auth._type import ( + AuthTypeLegacy, + AuthTypeApp, + ) ''; configurators = [ '' @@ -260,6 +310,18 @@ in oauth_id=${builtins.toJSON cfg.github.oauthId}, buildbot_user=${builtins.toJSON cfg.github.user}, topic=${builtins.toJSON cfg.github.topic}, + auth_type=${ + if cfg.github.authType.legacy.enable then + ''AuthTypeLegacy()'' + else if cfg.github.authType.app.enable then + '' + AuthTypeApp( + app_id=${toString cfg.github.authType.app.id}, + ) + '' + else + throw "Universe broke" + } )" }, gitea=${ @@ -323,6 +385,9 @@ in systemd.services.buildbot-master = { after = [ "postgresql.service" ]; + path = [ + pkgs.openssl + ]; serviceConfig = { # in master.py we read secrets from $CREDENTIALS_DIRECTORY LoadCredential = @@ -337,10 +402,15 @@ in ( cfg.cachix.authTokenFile != null ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" - ++ lib.optionals (cfg.github.enable) [ - "github-token:${cfg.github.tokenFile}" + ++ lib.optionals (cfg.github.enable) ([ "github-webhook-secret:${cfg.github.webhookSecretFile}" ] + ++ lib.optionals (cfg.github.authType.legacy.enable) [ + "github-token:${cfg.github.authType.legacy.tokenFile}" + ] + ++ lib.optionals (cfg.github.authType.app.enable) [ + "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}" + ]) ++ lib.optionals cfg.gitea.enable [ "gitea-token:${cfg.gitea.tokenFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" From d586b9155bd4d19f9ec871292056f1436cf77128 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Thu, 23 May 2024 10:37:02 +0200 Subject: [PATCH 02/13] Fix examples and checks Signed-off-by: magic_rb --- examples/master.nix | 7 +++++-- nix/checks/master.nix | 5 ++++- nix/master.nix | 2 +- pyproject.toml | 13 +++++++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/examples/master.nix b/examples/master.nix index 4677af0..de5cc4f 100644 --- a/examples/master.nix +++ b/examples/master.nix @@ -21,8 +21,11 @@ github = { # Github user used as a CI identity user = "mic92-buildbot"; - # Github token of the same user - tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store + authType.legacy = { + enable = true; + # Github token of the same user + tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store + }; # A random secret used to verify incoming webhooks from GitHub # buildbot-nix will set up a webhook for each project in the organization webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store diff --git a/nix/checks/master.nix b/nix/checks/master.nix index d79d31e..9c06c2c 100644 --- a/nix/checks/master.nix +++ b/nix/checks/master.nix @@ -16,7 +16,10 @@ ''; admins = [ "Mic92" ]; github = { - tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; + authType.legacy = { + enable = true; + tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; + }; webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff"; oauthId = "aaaaaaaaaaaaaaaaaaaa"; diff --git a/nix/master.nix b/nix/master.nix index 329a34d..73d2602 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -320,7 +320,7 @@ in ) '' else - throw "Universe broke" + throw "One of AuthTypeApp or AuthTypeLegacy must be enabled" } )" }, diff --git a/pyproject.toml b/pyproject.toml index 17d67e6..c0b6710 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" } [tool.setuptools] packages = [ "buildbot_nix", + "buildbot_nix.github", + "buildbot_nix.github.auth", "buildbot_effects" ] @@ -82,6 +84,17 @@ ignore = [ # not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html "G010", # Logging statement uses `warn` instead of `warning` + + # gives falls positives and isn't hard to check munually + "ERA001" +] + +[tool.ruff.lint.pep8-naming] +ignore-names = [ + "checkConfig", + "baseURL", + "reconfigService", + "sendMessage", ] [tool.mypy] From 5dc05ea3fefa6afb3d00d7411ef401040a58c167 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sun, 2 Jun 2024 16:12:09 +0200 Subject: [PATCH 03/13] Fix GitHub installation tokens not getting updated in the cache file Signed-off-by: magic_rb --- buildbot_nix/github/installation_token.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/buildbot_nix/github/installation_token.py b/buildbot_nix/github/installation_token.py index a5affce..6714246 100644 --- a/buildbot_nix/github/installation_token.py +++ b/buildbot_nix/github/installation_token.py @@ -86,7 +86,7 @@ class InstallationToken(RepoToken): # token: "token" # } # } - installations_token_map: dict[int, Any] + installations_token_map: dict[str, Any] if self.installations_token_map_name.exists(): installations_token_map = json.loads( self.installations_token_map_name.read_text() @@ -94,10 +94,14 @@ class InstallationToken(RepoToken): else: installations_token_map = {} - installations_token_map[self.installation_id] = { - "expiration": self.expiration.isoformat(), - "token": self.token, - } + installations_token_map.update( + { + str(self.installation_id): { + "expiration": self.expiration.isoformat(), + "token": self.token, + } + } + ) atomic_write_file( self.installations_token_map_name, json.dumps(installations_token_map) From 475fbf3952e8124c2dd11c6da9abd87741ccc4e1 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sun, 2 Jun 2024 16:13:14 +0200 Subject: [PATCH 04/13] Fix GitHub avatars for GitHub Apps Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 14 ++++++++------ buildbot_nix/github_projects.py | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index abd18dc..2b1379f 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -889,13 +889,15 @@ class NixConfigurator(ConfiguratorBase): backend.create_change_hook() ) - if "auth" not in config["www"]: - config["www"].setdefault("avatar_methods", []) + config["www"].setdefault("avatar_methods", []) - for backend in backends.values(): - avatar_method = backend.create_avatar_method() - if avatar_method is not None: - config["www"]["avatar_methods"].append(avatar_method) + for backend in backends.values(): + avatar_method = backend.create_avatar_method() + print(avatar_method) + if avatar_method is not None: + config["www"]["avatar_methods"].append(avatar_method) + + if "auth" not in config["www"]: # TODO one cannot have multiple auth backends... if auth is not None: config["www"]["auth"] = auth diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 289d7c4..2901b14 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -533,7 +533,22 @@ class GithubBackend(GitBackend): } def create_avatar_method(self) -> AvatarBase | None: - return AvatarGitHub(token=self.auth_backend.get_general_token().get()) + avatar = AvatarGitHub(token=self.auth_backend.get_general_token().get()) + + # TODO: not a proper fix, the /users/{username} endpoint is per installation, but I'm not sure + # how to tell which installation token to use, unless there is a way to build a huge map of + # username -> token, or we just try each one in order + def _get_avatar_by_username(self: Any, username: Any) -> Any: + return f"https://github.com/{username}.png" + + import types + + avatar._get_avatar_by_username = types.MethodType( # noqa: SLF001 + _get_avatar_by_username, + avatar, + ) + + return avatar def create_auth(self) -> AuthBase: assert self.config.oauth_id is not None, "GitHub OAuth ID is required" From 4f6d08a33d09a6ae5e51129ba3b0d273b33989c6 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sun, 2 Jun 2024 16:13:38 +0200 Subject: [PATCH 05/13] Improve Nix code and docs Signed-off-by: magic_rb --- README.md | 22 ++++++++++++++--- examples/master.nix | 5 +++- nix/checks/master.nix | 1 - nix/master.nix | 56 ++++++++++++++++++++++++------------------- 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 79ebc66..a8e30df 100644 --- a/README.md +++ b/README.md @@ -66,16 +66,32 @@ We have the following two roles: ### Integration with GitHub -To integrate with GitHub: +#### GitHub App + +To integrate with GitHub using app authentication: + +1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user + authentication on the Buildbot dashboard. +2. **GitHub App private key**: Get the app private key and app ID from GitHub, + configure using the buildbot-nix NixOS module. +3. **Install App**: Install the for an organization or specific user. +4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to + changes (new repositories or installations) automatically, it is therefore + necessary to manually trigger a reload or wait for the next periodic reload. + +#### Legacy Token Auth + +To integrate with GitHub using legacy token authentication: 1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo` permissions. For GitHub organizations, it's advisable to create a separate GitHub user for managing repository webhooks. -#### Optional when using GitHub login +### Optional when using GitHub login 1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user - authentication on the Buildbot dashboard. + authentication on the Buildbot dashboard. (can be the same as for GitHub App + auth) 2. **OAuth Credentials**: After installing the app, generate OAuth credentials and configure them in the buildbot-nix NixOS module. Set the callback url to `https:///auth/login`. diff --git a/examples/master.nix b/examples/master.nix index de5cc4f..8d9e366 100644 --- a/examples/master.nix +++ b/examples/master.nix @@ -22,10 +22,13 @@ # Github user used as a CI identity user = "mic92-buildbot"; authType.legacy = { - enable = true; # Github token of the same user tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store }; + # authType.app = { + # id = "00000000000000000"; # FIXME: replace with App ID obtained from GitHub + # secretKeyFile = pkgs.writeText "app-secret.key" "00000000000000000000"; # FIXME: replace with App secret key obtained from GitHub + # }; # A random secret used to verify incoming webhooks from GitHub # buildbot-nix will set up a webhook for each project in the organization webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store diff --git a/nix/checks/master.nix b/nix/checks/master.nix index 9c06c2c..8b43e88 100644 --- a/nix/checks/master.nix +++ b/nix/checks/master.nix @@ -17,7 +17,6 @@ admins = [ "Mic92" ]; github = { authType.legacy = { - enable = true; tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; }; webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; diff --git a/nix/master.nix b/nix/master.nix index 73d2602..5ad12e8 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -124,29 +124,35 @@ in default = cfg.authBackend == "github"; }; - authType = { - legacy = { - enable = lib.mkEnableOption ""; - tokenFile = lib.mkOption { - type = lib.types.path; - description = "Github token file"; - }; - }; - - app = { - enable = lib.mkEnableOption ""; - id = lib.mkOption { - type = lib.types.int; - description = '' - GitHub app ID. - ''; + authType = lib.mkOption { + type = lib.types.attrTag { + legacy = lib.mkOption { + description = "GitHub legacy auth backend"; + type = lib.types.submodule { + options.tokenFile = lib.mkOption { + type = lib.types.path; + description = "Github token file"; + }; + }; }; - secretKeyFile = lib.mkOption { - type = lib.types.str; - description = '' - GitHub app secret key file location. - ''; + app = lib.mkOption { + description = "GitHub legacy auth backend"; + type = lib.types.submodule { + options.id = lib.mkOption { + type = lib.types.int; + description = '' + GitHub app ID. + ''; + }; + + options.secretKeyFile = lib.mkOption { + type = lib.types.str; + description = '' + GitHub app secret key file location. + ''; + }; + }; }; }; }; @@ -311,9 +317,9 @@ in buildbot_user=${builtins.toJSON cfg.github.user}, topic=${builtins.toJSON cfg.github.topic}, auth_type=${ - if cfg.github.authType.legacy.enable then + if cfg.github.authType ? "legacy" then ''AuthTypeLegacy()'' - else if cfg.github.authType.app.enable then + else if cfg.github.authType ? "app" then '' AuthTypeApp( app_id=${toString cfg.github.authType.app.id}, @@ -405,10 +411,10 @@ in ++ lib.optionals (cfg.github.enable) ([ "github-webhook-secret:${cfg.github.webhookSecretFile}" ] - ++ lib.optionals (cfg.github.authType.legacy.enable) [ + ++ lib.optionals (cfg.github.authType ? "legacy") [ "github-token:${cfg.github.authType.legacy.tokenFile}" ] - ++ lib.optionals (cfg.github.authType.app.enable) [ + ++ lib.optionals (cfg.github.authType ? "app") [ "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}" ]) ++ lib.optionals cfg.gitea.enable [ From 8664657737caf496f94fea08f0c3bc85f1aa880c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 7 Jun 2024 10:08:12 +0200 Subject: [PATCH 06/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8e30df..820486e 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ To integrate with GitHub using app authentication: authentication on the Buildbot dashboard. 2. **GitHub App private key**: Get the app private key and app ID from GitHub, configure using the buildbot-nix NixOS module. -3. **Install App**: Install the for an organization or specific user. +3. **Install App**: Install the app for an organization or specific user. 4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to changes (new repositories or installations) automatically, it is therefore necessary to manually trigger a reload or wait for the next periodic reload. From 43576044630ddffeb31a03ef958d275cb19718ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 7 Jun 2024 16:50:03 +0200 Subject: [PATCH 07/13] add assert message to get_repo_token --- buildbot_nix/github_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 2901b14..01eada6 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -390,7 +390,7 @@ class GithubAppAuthBackend(GithubAuthBackend): return self.jwt_token def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: - assert "installation_id" in repo + assert "installation_id" in repo, f"Missing installation_id in {repo}" return self.installation_tokens[repo["installation_id"]] def create_secret_providers(self) -> list[SecretProviderBase]: From 30a49d4e88a80ca918f47dc82d7c62a410d7c540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 7 Jun 2024 16:55:36 +0200 Subject: [PATCH 08/13] fix typo in class name --- buildbot_nix/github_projects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 01eada6..b58d40f 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -330,7 +330,7 @@ class GithubLegacyAuthBackend(GithubAuthBackend): return self.token def create_secret_providers(self) -> list[SecretProviderBase]: - return [GitHubLagacySecretService(self.token)] + return [GitHubLegacySecretService(self.token)] def create_reporter(self) -> ReporterBase: return GitHubStatusPush( @@ -347,7 +347,7 @@ class GithubLegacyAuthBackend(GithubAuthBackend): ) -class GitHubLagacySecretService(SecretProviderBase): +class GitHubLegacySecretService(SecretProviderBase): name = "GitHubLegacySecretService" token: LegacyToken From 9f04067a6a6373be537869cbea7b343d27d06284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 12 Jun 2024 07:58:34 +0200 Subject: [PATCH 09/13] bust cache after adding installation_id --- buildbot_nix/github_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index b58d40f..6418500 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -447,7 +447,7 @@ class GithubConfig: buildbot_user: str oauth_secret_name: str = "github-oauth-secret" webhook_secret_name: str = "github-webhook-secret" - project_cache_file: Path = Path("github-project-cache.json") + project_cache_file: Path = Path("github-project-cache-v1.json") topic: str | None = "build-with-buildbot" From 6a9394bed5fbdf822402c0ea109b2ac1447804b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 12 Jun 2024 09:05:36 +0200 Subject: [PATCH 10/13] document permissions required for github app --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 820486e..fad8ac7 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,11 @@ We have the following two roles: To integrate with GitHub using app authentication: 1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user - authentication on the Buildbot dashboard. + authentication on the Buildbot dashboard. Enable the following permissions: + - Contents: Read-only + - Metadata: Read-only + - Commit statuses: Read and write + - Webhooks: Read and write 2. **GitHub App private key**: Get the app private key and app ID from GitHub, configure using the buildbot-nix NixOS module. 3. **Install App**: Install the app for an organization or specific user. From eeb21e9dddc9e457f657c2506191651fcdb5d0ec Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 12 Jun 2024 17:19:11 +0200 Subject: [PATCH 11/13] Streamline `Legacy` <-> `App` GitHub auth backend migrations Signed-off-by: magic_rb --- buildbot_nix/github_projects.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 6418500..1055084 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -153,12 +153,9 @@ class ReloadGithubProjects(BuildStep): super().__init__(**kwargs) def reload_projects(self) -> None: - repos: list[Any] = [] + repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file) - if self.project_cache_file.exists(): - repos = json.loads(self.project_cache_file.read_text()) - - refresh_projects(self.token.get(), self.project_cache_file) + log.msg(repos) atomic_write_file(self.project_cache_file, json.dumps(repos)) @@ -569,6 +566,19 @@ class GithubBackend(GitBackend): json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"], ) + + if isinstance(self.auth_backend, GithubAppAuthBackend): + dropped_repos = list( + filter(lambda repo: not "installation_id" in repo, repos) + ) + if dropped_repos: + tlog.info( + "Dropped projects follow, refresh will follow after initialisation:" + ) + for dropped_repo in dropped_repos: + tlog.info(f"\tDropping repo {dropped_repo['full_name']}") + repos = list(filter(lambda repo: "installation_id" in repo, repos)) + tlog.info(f"Loading {len(repos)} cached repositories.") return list( filter( @@ -587,7 +597,16 @@ class GithubBackend(GitBackend): ) def are_projects_cached(self) -> bool: - return self.config.project_cache_file.exists() + if not self.config.project_cache_file.exists(): + return False + + all_have_installation_id = True + for project in json.loads(self.config.project_cache_file.read_text()): + if not "installation_id" in project: + all_have_installation_id = False + break + + return all_have_installation_id @property def type(self) -> str: From 467b9164ded2eef77ddd94dd455193541b8a7f94 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 12 Jun 2024 17:36:08 +0200 Subject: [PATCH 12/13] Fix `app_project_id_map_name` in GitHub App backend, not having a `.json` extension Signed-off-by: magic_rb --- buildbot_nix/github/auth/_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_nix/github/auth/_type.py b/buildbot_nix/github/auth/_type.py index abbff13..8c4c01b 100644 --- a/buildbot_nix/github/auth/_type.py +++ b/buildbot_nix/github/auth/_type.py @@ -19,5 +19,5 @@ class AuthTypeApp(AuthType): app_installation_token_map_name: Path = Path( "github-app-installation-token-map.json" ) - app_project_id_map_name: Path = Path("github-app-project-id-map-name") + app_project_id_map_name: Path = Path("github-app-project-id-map-name.json") app_jwt_token_name: Path = Path("github-app-jwt-token") From 0ac5dcb33d894dd16a590ccaf72a10ab4c4c98f6 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 12 Jun 2024 17:36:43 +0200 Subject: [PATCH 13/13] Fix GitHub App backend failing if `project-id-map` disappears Signed-off-by: magic_rb --- buildbot_nix/github_projects.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 1055084..6a2a9d5 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -381,6 +381,9 @@ class GithubAppAuthBackend(GithubAuthBackend): self.auth_type.app_project_id_map_name.read_text() ) else: + tlog.info( + "~project-id-map~ is not present, GitHub project reload will follow." + ) self.project_id_map = {} def get_general_token(self) -> RepoToken: @@ -600,6 +603,12 @@ class GithubBackend(GitBackend): if not self.config.project_cache_file.exists(): return False + if ( + isinstance(self.config.auth_type, AuthTypeApp) + and not self.config.auth_type.app_project_id_map_name.exists() + ): + return False + all_have_installation_id = True for project in json.loads(self.config.project_cache_file.read_text()): if not "installation_id" in project: