diff --git a/README.md b/README.md index cf4a668..234e7f9 100644 --- a/README.md +++ b/README.md @@ -66,16 +66,36 @@ We have the following two roles: ### Integration with GitHub -To integrate with GitHub: +#### GitHub App + +To integrate with GitHub using app authentication: + +1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user + authentication on the Buildbot dashboard. Enable the following permissions: + - Contents: Read-only + - Metadata: Read-only + - Commit statuses: Read and write + - Webhooks: Read and write +2. **GitHub App private key**: Get the app private key and app ID from GitHub, + configure using the buildbot-nix NixOS module. +3. **Install App**: Install the app for an organization or specific user. +4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to + changes (new repositories or installations) automatically, it is therefore + necessary to manually trigger a reload or wait for the next periodic reload. + +#### Legacy Token Auth + +To integrate with GitHub using legacy token authentication: 1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo` permissions. For GitHub organizations, it's advisable to create a separate GitHub user for managing repository webhooks. -#### Optional when using GitHub login +### Optional when using GitHub login 1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user - authentication on the Buildbot dashboard. + authentication on the Buildbot dashboard. (can be the same as for GitHub App + auth) 2. **OAuth Credentials**: After installing the app, generate OAuth credentials and configure them in the buildbot-nix NixOS module. Set the callback url to `https:///auth/login`. diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index e16fdea..2b1379f 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -872,6 +872,8 @@ class NixConfigurator(ConfiguratorBase): ], ) config["services"].append(backend.create_reporter()) + config.setdefault("secretProviders", []) + config["secretsProviders"].extend(backend.create_secret_providers()) systemd_secrets = SecretInAFile( dirname=os.environ["CREDENTIALS_DIRECTORY"], @@ -887,13 +889,15 @@ class NixConfigurator(ConfiguratorBase): backend.create_change_hook() ) - if "auth" not in config["www"]: - config["www"].setdefault("avatar_methods", []) + config["www"].setdefault("avatar_methods", []) - for backend in backends.values(): - avatar_method = backend.create_avatar_method() - if avatar_method is not None: - config["www"]["avatar_methods"].append(avatar_method) + for backend in backends.values(): + avatar_method = backend.create_avatar_method() + print(avatar_method) + if avatar_method is not None: + config["www"]["avatar_methods"].append(avatar_method) + + if "auth" not in config["www"]: # TODO one cannot have multiple auth backends... if auth is not None: config["www"]["auth"] = auth diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py index 6dcd4f3..b4d3941 100644 --- a/buildbot_nix/common.py +++ b/buildbot_nix/common.py @@ -2,6 +2,8 @@ import contextlib import http.client import json import urllib.request +from pathlib import Path +from tempfile import NamedTemporaryFile from typing import Any @@ -9,7 +11,9 @@ def slugify_project_name(name: str) -> str: return name.replace(".", "-").replace("/", "-") -def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: +def paginated_github_request( + url: str, token: str, subkey: None | str = None +) -> list[dict[str, Any]]: next_url: str | None = url items = [] while next_url: @@ -29,7 +33,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: link_parts = link.split(";") if link_parts[1].strip() == 'rel="next"': next_url = link_parts[0][1:-1] - items += res.json() + if subkey is not None: + items += res.json()[subkey] + else: + items += res.json() return items @@ -78,3 +85,15 @@ def http_request( msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" raise HttpError(msg) from e return HttpResponse(resp) + + +def atomic_write_file(file: Path, data: str) -> None: + with NamedTemporaryFile("w", delete=False, dir=file.parent) as f: + path = Path(f.name) + try: + f.write(data) + f.flush() + path.rename(file) + except OSError: + path.unlink() + raise diff --git a/buildbot_nix/github/__init__.py b/buildbot_nix/github/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot_nix/github/__init__.py @@ -0,0 +1 @@ + diff --git a/buildbot_nix/github/auth/__init__.py b/buildbot_nix/github/auth/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot_nix/github/auth/__init__.py @@ -0,0 +1 @@ + diff --git a/buildbot_nix/github/auth/_type.py b/buildbot_nix/github/auth/_type.py new file mode 100644 index 0000000..8c4c01b --- /dev/null +++ b/buildbot_nix/github/auth/_type.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class AuthType: + pass + + +@dataclass +class AuthTypeLegacy(AuthType): + token_secret_name: str = "github-token" + + +@dataclass +class AuthTypeApp(AuthType): + app_id: int + app_secret_key_name: str = "github-app-secret-key" + app_installation_token_map_name: Path = Path( + "github-app-installation-token-map.json" + ) + app_project_id_map_name: Path = Path("github-app-project-id-map-name.json") + app_jwt_token_name: Path = Path("github-app-jwt-token") diff --git a/buildbot_nix/github/installation_token.py b/buildbot_nix/github/installation_token.py new file mode 100644 index 0000000..6714246 --- /dev/null +++ b/buildbot_nix/github/installation_token.py @@ -0,0 +1,108 @@ +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from buildbot_nix.common import ( + HttpResponse, + atomic_write_file, + http_request, +) + +from .jwt_token import JWTToken +from .repo_token import RepoToken + + +class InstallationToken(RepoToken): + GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60) + + jwt_token: JWTToken + installation_id: int + + token: str + expiration: datetime + installations_token_map_name: Path + + @staticmethod + def _create_installation_access_token( + jwt_token: JWTToken, installation_id: int + ) -> HttpResponse: + return http_request( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + data={}, + headers={"Authorization": f"Bearer {jwt_token.get()}"}, + method="POST", + ) + + @staticmethod + def _generate_token( + jwt_token: JWTToken, installation_id: int + ) -> tuple[str, datetime]: + token = InstallationToken._create_installation_access_token( + jwt_token, installation_id + ).json()["token"] + expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME + + return token, expiration + + def __init__( + self, + jwt_token: JWTToken, + installation_id: int, + installations_token_map_name: Path, + installation_token: None | tuple[str, datetime] = None, + ) -> None: + self.jwt_token = jwt_token + self.installation_id = installation_id + self.installations_token_map_name = installations_token_map_name + + if installation_token is None: + self.token, self.expiration = InstallationToken._generate_token( + self.jwt_token, self.installation_id + ) + self._save() + else: + self.token, self.expiration = installation_token + + def get(self) -> str: + self.verify() + return self.token + + def get_as_secret(self) -> str: + return f"%(secret:github-token-{self.installation_id})" + + def verify(self) -> None: + if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8: + self.token, self.expiration = InstallationToken._generate_token( + self.jwt_token, self.installation_id + ) + self._save() + + def _save(self) -> None: + # of format: + # { + # 123: { + # expiration: , + # token: "token" + # } + # } + installations_token_map: dict[str, Any] + if self.installations_token_map_name.exists(): + installations_token_map = json.loads( + self.installations_token_map_name.read_text() + ) + else: + installations_token_map = {} + + installations_token_map.update( + { + str(self.installation_id): { + "expiration": self.expiration.isoformat(), + "token": self.token, + } + } + ) + + atomic_write_file( + self.installations_token_map_name, json.dumps(installations_token_map) + ) diff --git a/buildbot_nix/github/jwt_token.py b/buildbot_nix/github/jwt_token.py new file mode 100644 index 0000000..79fe56b --- /dev/null +++ b/buildbot_nix/github/jwt_token.py @@ -0,0 +1,84 @@ +import base64 +import json +import os +import subprocess +from datetime import UTC, datetime, timedelta +from typing import Any + +from .repo_token import RepoToken + + +class JWTToken(RepoToken): + app_id: int + app_private_key: str + lifetime: timedelta + + expiration: datetime + token: str + + def __init__( + self, + app_id: int, + app_private_key: str, + lifetime: timedelta = timedelta(minutes=10), + ) -> None: + self.app_id = app_id + self.app_private_key = app_private_key + self.lifetime = lifetime + + self.token, self.expiration = JWTToken.generate_token( + self.app_id, self.app_private_key, lifetime + ) + + @staticmethod + def generate_token( + app_id: int, app_private_key: str, lifetime: timedelta + ) -> tuple[str, datetime]: + def build_jwt_payload( + app_id: int, lifetime: timedelta + ) -> tuple[dict[str, Any], datetime]: + jwt_iat_drift: timedelta = timedelta(seconds=60) + now: datetime = datetime.now(tz=UTC) + iat: datetime = now - jwt_iat_drift + exp: datetime = iat + lifetime + jwt_payload = { + "iat": int(iat.timestamp()), + "exp": int(exp.timestamp()), + "iss": str(app_id), + } + return (jwt_payload, exp) + + def rs256_sign(data: str, private_key: str) -> str: + signature = subprocess.run( + ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key], + input=data.encode("utf-8"), + stdout=subprocess.PIPE, + check=True, + cwd=os.environ.get("CREDENTIALS_DIRECTORY"), + ).stdout + return base64url(signature) + + def base64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8") + + jwt, expiration = build_jwt_payload(app_id, lifetime) + jwt_payload = json.dumps(jwt).encode("utf-8") + json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8") + encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}" + encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key) + return (f"{encoded_jwt_parts}.{encoded_mac}", expiration) + + # installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt) + + # return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations)) + + def get(self) -> str: + if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8: + self.token, self.expiration = JWTToken.generate_token( + self.app_id, self.app_private_key, self.lifetime + ) + + return self.token + + def get_as_secret(self) -> str: + return "%(secret:github-jwt-token)" diff --git a/buildbot_nix/github/legacy_token.py b/buildbot_nix/github/legacy_token.py new file mode 100644 index 0000000..3aff388 --- /dev/null +++ b/buildbot_nix/github/legacy_token.py @@ -0,0 +1,14 @@ +from .repo_token import RepoToken + + +class LegacyToken(RepoToken): + token: str + + def __init__(self, token: str) -> None: + self.token = token + + def get(self) -> str: + return self.token + + def get_as_secret(self) -> str: + return "%(secret:github-token)" diff --git a/buildbot_nix/github/repo_token.py b/buildbot_nix/github/repo_token.py new file mode 100644 index 0000000..398702a --- /dev/null +++ b/buildbot_nix/github/repo_token.py @@ -0,0 +1,11 @@ +from abc import abstractmethod + + +class RepoToken: + @abstractmethod + def get(self) -> str: + pass + + @abstractmethod + def get_as_secret(self) -> str: + pass diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 6f40319..6a2a9d5 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,10 +1,13 @@ import json import os import signal -from collections.abc import Generator +from abc import ABC, abstractmethod +from collections.abc import Callable, Generator from dataclasses import dataclass +from datetime import ( + datetime, +) from pathlib import Path -from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Any from buildbot.config.builder import BuilderConfig @@ -13,10 +16,12 @@ from buildbot.process.buildstep import BuildStep from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.reporters.github import GitHubStatusPush +from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth from twisted.internet import defer, threads +from twisted.logger import Logger from twisted.python import log from twisted.python.failure import Failure @@ -24,24 +29,96 @@ if TYPE_CHECKING: from buildbot.process.log import StreamLog from .common import ( + atomic_write_file, http_request, paginated_github_request, slugify_project_name, ) +from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy +from .github.installation_token import InstallationToken +from .github.jwt_token import JWTToken +from .github.legacy_token import ( + LegacyToken, +) +from .github.repo_token import ( + RepoToken, +) from .projects import GitBackend, GitProject from .secrets import read_secret_file +tlog = Logger() -class ReloadGithubProjects(BuildStep): + +def get_installations(jwt_token: JWTToken) -> list[int]: + installations = paginated_github_request( + "https://api.github.com/app/installations?per_page=100", jwt_token.get() + ) + + return [installation["id"] for installation in installations] + + +class ReloadGithubInstallations(BuildStep): name = "reload_github_projects" - def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: - self.token = token + jwt_token: JWTToken + project_cache_file: Path + installation_token_map_name: Path + project_id_map_name: Path + + def __init__( + self, + jwt_token: JWTToken, + project_cache_file: Path, + installation_token_map_name: Path, + project_id_map_name: Path, + **kwargs: Any, + ) -> None: + self.jwt_token = jwt_token + self.installation_token_map_name = installation_token_map_name + self.project_id_map_name = project_id_map_name self.project_cache_file = project_cache_file super().__init__(**kwargs) def reload_projects(self) -> None: - refresh_projects(self.token, self.project_cache_file) + installation_token_map = GithubBackend.create_missing_installations( + self.jwt_token, + self.installation_token_map_name, + GithubBackend.load_installations( + self.jwt_token, + self.installation_token_map_name, + ), + get_installations(self.jwt_token), + ) + + repos: list[Any] = [] + project_id_map: dict[str, int] = {} + + repos = [] + + for k, v in installation_token_map.items(): + new_repos = refresh_projects( + v.get(), + self.project_cache_file, + clear=True, + api_endpoint="/installation/repositories", + subkey="repositories", + require_admin=False, + ) + + for repo in new_repos: + repo["installation_id"] = k + + repos.extend(new_repos) + + for repo in new_repos: + project_id_map[repo["full_name"]] = k + + atomic_write_file(self.project_cache_file, json.dumps(repos)) + atomic_write_file(self.project_id_map_name, json.dumps(project_id_map)) + + tlog.info( + f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token." + ) @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: @@ -65,36 +142,378 @@ class ReloadGithubProjects(BuildStep): return util.FAILURE +class ReloadGithubProjects(BuildStep): + name = "reload_github_projects" + + def __init__( + self, token: RepoToken, project_cache_file: Path, **kwargs: Any + ) -> None: + self.token = token + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file) + + log.msg(repos) + + atomic_write_file(self.project_cache_file, json.dumps(repos)) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + log: StreamLog = yield self.addLog("log") + log.addStderr(f"Failed to reload project list: {self.error_msg}") + return util.FAILURE + + +class GitHubAppStatusPush(GitHubStatusPush): + token_source: Callable[[int], RepoToken] + project_id_source: Callable[[str], int] + saved_args: dict[str, Any] + saved_kwargs: dict[str, Any] + + def checkConfig( + self, + token_source: Callable[[int], RepoToken], + project_id_source: Callable[[str], int], + context: Any = None, + baseURL: Any = None, + verbose: Any = False, + debug: Any = None, + verify: Any = None, + generators: Any = None, + **kwargs: dict[str, Any], + ) -> Any: + if generators is None: + generators = self._create_default_generators() + + if "token" in kwargs: + del kwargs["token"] + super().checkConfig( + token="", + context=context, + baseURL=baseURL, + verbose=verbose, + debug=debug, + verify=verify, + generators=generators, + **kwargs, + ) + + def reconfigService( + self, + token_source: Callable[[int], RepoToken], + project_id_source: Callable[[str], int], + context: Any = None, + baseURL: Any = None, + verbose: Any = False, + debug: Any = None, + verify: Any = None, + generators: Any = None, + **kwargs: dict[str, Any], + ) -> Any: + if "saved_args" not in self or self.saved_args is None: + self.saved_args = {} + self.token_source = token_source + self.project_id_source = project_id_source + self.saved_kwargs = kwargs + self.saved_args["context"] = context + self.saved_args["baseURL"] = baseURL + self.saved_args["verbose"] = verbose + self.saved_args["debug"] = debug + self.saved_args["verify"] = verify + self.saved_args["generators"] = generators + + if generators is None: + generators = self._create_default_generators() + + if "token" in kwargs: + del kwargs["token"] + super().reconfigService( + token="", + context=context, + baseURL=baseURL, + verbose=verbose, + debug=debug, + verify=verify, + generators=generators, + **kwargs, + ) + + def sendMessage(self, reports: Any) -> Any: + build = reports[0]["builds"][0] + sourcestamps = build["buildset"].get("sourcestamps") + if not sourcestamps: + return None + + for sourcestamp in sourcestamps: + build["buildset"]["sourcestamps"] = [sourcestamp] + + token: str + + if "project" in sourcestamp and sourcestamp["project"] != "": + token = self.token_source( + self.project_id_source(sourcestamp["project"]) + ).get() + else: + token = "" + + super().reconfigService( + token, + context=self.saved_args["context"], + baseURL=self.saved_args["baseURL"], + verbose=self.saved_args["verbose"], + debug=self.saved_args["debug"], + verify=self.saved_args["verify"], + generators=self.saved_args["generators"], + **self.saved_kwargs, + ) + + return super().sendMessage(reports) + + return None + + +class GithubAuthBackend(ABC): + @abstractmethod + def get_general_token(self) -> RepoToken: + pass + + @abstractmethod + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + pass + + @abstractmethod + def create_secret_providers(self) -> list[SecretProviderBase]: + pass + + @abstractmethod + def create_reporter(self) -> ReporterBase: + pass + + @abstractmethod + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + pass + + +class GithubLegacyAuthBackend(GithubAuthBackend): + auth_type: AuthTypeLegacy + + token: LegacyToken + + def __init__(self, auth_type: AuthTypeLegacy) -> None: + self.auth_type = auth_type + self.token = LegacyToken(read_secret_file(auth_type.token_secret_name)) + + def get_general_token(self) -> RepoToken: + return self.token + + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + return self.token + + def create_secret_providers(self) -> list[SecretProviderBase]: + return [GitHubLegacySecretService(self.token)] + + def create_reporter(self) -> ReporterBase: + return GitHubStatusPush( + token=self.token.get(), + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + return ReloadGithubProjects( + token=self.token, project_cache_file=project_cache_file + ) + + +class GitHubLegacySecretService(SecretProviderBase): + name = "GitHubLegacySecretService" + token: LegacyToken + + def reconfigService(self, token: LegacyToken) -> None: + self.token = token + + def get(self, entry: str) -> str | None: + """ + get the value from the file identified by 'entry' + """ + if entry.startswith("github-token"): + return self.token.get() + return None + + +class GithubAppAuthBackend(GithubAuthBackend): + auth_type: AuthTypeApp + + jwt_token: JWTToken + installation_tokens: dict[int, InstallationToken] + project_id_map: dict[str, int] + + def __init__(self, auth_type: AuthTypeApp) -> None: + self.auth_type = auth_type + self.jwt_token = JWTToken( + self.auth_type.app_id, self.auth_type.app_secret_key_name + ) + self.installation_tokens = GithubBackend.load_installations( + self.jwt_token, + self.auth_type.app_installation_token_map_name, + ) + if self.auth_type.app_project_id_map_name.exists(): + self.project_id_map = json.loads( + self.auth_type.app_project_id_map_name.read_text() + ) + else: + tlog.info( + "~project-id-map~ is not present, GitHub project reload will follow." + ) + self.project_id_map = {} + + def get_general_token(self) -> RepoToken: + return self.jwt_token + + def get_repo_token(self, repo: dict[str, Any]) -> RepoToken: + assert "installation_id" in repo, f"Missing installation_id in {repo}" + return self.installation_tokens[repo["installation_id"]] + + def create_secret_providers(self) -> list[SecretProviderBase]: + return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)] + + def create_reporter(self) -> ReporterBase: + return GitHubAppStatusPush( + token_source=lambda iid: self.installation_tokens[iid], + project_id_source=lambda project: self.project_id_map[project], + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + return ReloadGithubInstallations( + self.jwt_token, + project_cache_file, + self.auth_type.app_installation_token_map_name, + self.auth_type.app_project_id_map_name, + ) + + +class GitHubAppSecretService(SecretProviderBase): + name = "GitHubAppSecretService" + installation_tokens: dict[int, InstallationToken] + jwt_token: JWTToken + + def reconfigService( + self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken + ) -> None: + self.installation_tokens = installation_tokens + self.jwt_token = jwt_token + + def get(self, entry: str) -> str | None: + """ + get the value from the file identified by 'entry' + """ + if entry.startswith("github-token-"): + return self.installation_tokens[ + int(entry.removeprefix("github-token-")) + ].get() + if entry == "github-jwt-token": + return self.jwt_token.get() + return None + + @dataclass class GithubConfig: oauth_id: str | None - + auth_type: AuthType # TODO unused buildbot_user: str oauth_secret_name: str = "github-oauth-secret" - token_secret_name: str = "github-token" webhook_secret_name: str = "github-webhook-secret" - project_cache_file: Path = Path("github-project-cache.json") + project_cache_file: Path = Path("github-project-cache-v1.json") topic: str | None = "build-with-buildbot" - def token(self) -> str: - return read_secret_file(self.token_secret_name) - @dataclass class GithubBackend(GitBackend): config: GithubConfig webhook_secret: str + auth_backend: GithubAuthBackend + def __init__(self, config: GithubConfig) -> None: self.config = config self.webhook_secret = read_secret_file(self.config.webhook_secret_name) + if isinstance(self.config.auth_type, AuthTypeLegacy): + self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type) + elif isinstance(self.config.auth_type, AuthTypeApp): + self.auth_backend = GithubAppAuthBackend(self.config.auth_type) + + @staticmethod + def load_installations( + jwt_token: JWTToken, installations_token_map_name: Path + ) -> dict[int, InstallationToken]: + initial_installations_map: dict[str, Any] + if installations_token_map_name.exists(): + initial_installations_map = json.loads( + installations_token_map_name.read_text() + ) + else: + initial_installations_map = {} + + installations_map: dict[int, InstallationToken] = {} + + for iid, installation in initial_installations_map.items(): + token: str = installation["token"] + expiration: datetime = datetime.fromisoformat(installation["expiration"]) + installations_map[int(iid)] = InstallationToken( + jwt_token, + int(iid), + installations_token_map_name, + installation_token=(token, expiration), + ) + + return installations_map + + @staticmethod + def create_missing_installations( + jwt_token: JWTToken, + installations_token_map_name: Path, + installations_map: dict[int, InstallationToken], + installations: list[int], + ) -> dict[int, InstallationToken]: + for installation in set(installations) - installations_map.keys(): + installations_map[installation] = InstallationToken( + jwt_token, + installation, + installations_token_map_name, + ) + + return installations_map + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" factory = util.BuildFactory() factory.addStep( - ReloadGithubProjects(self.config.token(), self.config.project_cache_file), + self.auth_backend.create_reload_builder_step(self.config.project_cache_file) ) return util.BuilderConfig( name=self.reload_builder_name, @@ -103,24 +522,33 @@ class GithubBackend(GitBackend): ) def create_reporter(self) -> ReporterBase: - return GitHubStatusPush( - token=self.config.token(), - # Since we dynamically create build steps, - # we use `virtual_builder_name` in the webinterface - # so that we distinguish what has beeing build - context=Interpolate("buildbot/%(prop:status_name)s"), - ) + return self.auth_backend.create_reporter() def create_change_hook(self) -> dict[str, Any]: return { "secret": self.webhook_secret, "strict": True, - "token": self.config.token(), + "token": self.auth_backend.get_general_token().get(), "github_property_whitelist": ["github.base.sha", "github.head.sha"], } def create_avatar_method(self) -> AvatarBase | None: - return AvatarGitHub(token=self.config.token()) + avatar = AvatarGitHub(token=self.auth_backend.get_general_token().get()) + + # TODO: not a proper fix, the /users/{username} endpoint is per installation, but I'm not sure + # how to tell which installation token to use, unless there is a way to build a huge map of + # username -> token, or we just try each one in order + def _get_avatar_by_username(self: Any, username: Any) -> Any: + return f"https://github.com/{username}.png" + + import types + + avatar._get_avatar_by_username = types.MethodType( # noqa: SLF001 + _get_avatar_by_username, + avatar, + ) + + return avatar def create_auth(self) -> AuthBase: assert self.config.oauth_id is not None, "GitHub OAuth ID is required" @@ -130,6 +558,9 @@ class GithubBackend(GitBackend): apiVersion=4, ) + def create_secret_providers(self) -> list[SecretProviderBase]: + return self.auth_backend.create_secret_providers() + def load_projects(self) -> list["GitProject"]: if not self.config.project_cache_file.exists(): return [] @@ -138,19 +569,53 @@ class GithubBackend(GitBackend): json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"], ) + + if isinstance(self.auth_backend, GithubAppAuthBackend): + dropped_repos = list( + filter(lambda repo: not "installation_id" in repo, repos) + ) + if dropped_repos: + tlog.info( + "Dropped projects follow, refresh will follow after initialisation:" + ) + for dropped_repo in dropped_repos: + tlog.info(f"\tDropping repo {dropped_repo['full_name']}") + repos = list(filter(lambda repo: "installation_id" in repo, repos)) + + tlog.info(f"Loading {len(repos)} cached repositories.") return list( filter( lambda project: self.config.topic is not None and self.config.topic in project.topics, [ - GithubProject(self.config, self.webhook_secret, repo) + GithubProject( + self.auth_backend.get_repo_token(repo), + self.config, + self.webhook_secret, + repo, + ) for repo in repos ], ) ) def are_projects_cached(self) -> bool: - return self.config.project_cache_file.exists() + if not self.config.project_cache_file.exists(): + return False + + if ( + isinstance(self.config.auth_type, AuthTypeApp) + and not self.config.auth_type.app_project_id_map_name.exists() + ): + return False + + all_have_installation_id = True + for project in json.loads(self.config.project_cache_file.read_text()): + if not "installation_id" in project: + all_have_installation_id = False + break + + return all_have_installation_id @property def type(self) -> str: @@ -171,10 +636,18 @@ class GithubBackend(GitBackend): class GithubProject(GitProject): config: GithubConfig + webhook_secret: str + data: dict[str, Any] + token: RepoToken def __init__( - self, config: GithubConfig, webhook_secret: str, data: dict[str, Any] + self, + token: RepoToken, + config: GithubConfig, + webhook_secret: str, + data: dict[str, Any], ) -> None: + self.token = token self.config = config self.webhook_secret = webhook_secret self.data = data @@ -187,7 +660,7 @@ class GithubProject(GitProject): ) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", - self.config.token(), + self.token.get(), ) config = dict( url=webhook_url + "change_hook/github", @@ -199,7 +672,7 @@ class GithubProject(GitProject): name="web", active=True, events=["push", "pull_request"], config=config ) headers = { - "Authorization": f"Bearer {self.config.token()}", + "Authorization": f"Bearer {self.token.get()}", "Accept": "application/vnd.github+json", "Content-Type": "application/json", "X-GitHub-Api-Version": "2022-11-28", @@ -217,7 +690,7 @@ class GithubProject(GitProject): ) def get_project_url(self) -> str: - return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}" + return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}" @property def pretty_type(self) -> str: @@ -260,14 +733,25 @@ class GithubProject(GitProject): return self.data["owner"]["type"] == "Organization" -def refresh_projects(github_token: str, repo_cache_file: Path) -> None: - repos = [] +def refresh_projects( + github_token: str, + repo_cache_file: Path, + repos: list[Any] | None = None, + clear: bool = True, + api_endpoint: str = "/user/repos", + subkey: None | str = None, + require_admin: bool = True, +) -> list[Any]: + if repos is None: + repos = [] for repo in paginated_github_request( - "https://api.github.com/user/repos?per_page=100", + f"https://api.github.com{api_endpoint}?per_page=100", github_token, + subkey=subkey, ): - if not repo["permissions"]["admin"]: + # TODO actually check for this properly + if not repo["permissions"]["admin"] and require_admin: name = repo["full_name"] log.msg( f"skipping {name} because we do not have admin privileges, needed for hook management", @@ -275,12 +759,4 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: else: repos.append(repo) - with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: - path = Path(f.name) - try: - f.write(json.dumps(repos)) - f.flush() - path.rename(repo_cache_file) - except OSError: - path.unlink() - raise + return repos diff --git a/buildbot_nix/projects.py b/buildbot_nix/projects.py index 947bc1f..23ef1db 100644 --- a/buildbot_nix/projects.py +++ b/buildbot_nix/projects.py @@ -3,6 +3,7 @@ from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.reporters.base import ReporterBase +from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase @@ -28,6 +29,9 @@ class GitBackend(ABC): def create_auth(self) -> AuthBase: pass + def create_secret_providers(self) -> list[SecretProviderBase]: + return [] + @abstractmethod def load_projects(self) -> list["GitProject"]: pass diff --git a/examples/master.nix b/examples/master.nix index 4677af0..8d9e366 100644 --- a/examples/master.nix +++ b/examples/master.nix @@ -21,8 +21,14 @@ github = { # Github user used as a CI identity user = "mic92-buildbot"; - # Github token of the same user - tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store + authType.legacy = { + # Github token of the same user + tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store + }; + # authType.app = { + # id = "00000000000000000"; # FIXME: replace with App ID obtained from GitHub + # secretKeyFile = pkgs.writeText "app-secret.key" "00000000000000000000"; # FIXME: replace with App secret key obtained from GitHub + # }; # A random secret used to verify incoming webhooks from GitHub # buildbot-nix will set up a webhook for each project in the organization webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store diff --git a/nix/checks/master.nix b/nix/checks/master.nix index d79d31e..8b43e88 100644 --- a/nix/checks/master.nix +++ b/nix/checks/master.nix @@ -16,7 +16,9 @@ ''; admins = [ "Mic92" ]; github = { - tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; + authType.legacy = { + tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; + }; webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff"; oauthId = "aaaaaaaaaaaaaaaaaaaa"; diff --git a/nix/master.nix b/nix/master.nix index e4881e9..5ad12e8 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -24,6 +24,24 @@ in "admins" ] ) + (mkRenamedOptionModule + [ + "services" + "buildbot-nix" + "master" + "github" + "tokenFile" + ] + [ + "services" + "buildbot-nix" + "master" + "github" + "authType" + "legacy" + "tokenFile" + ] + ) ]; options = { @@ -106,10 +124,39 @@ in default = cfg.authBackend == "github"; }; - tokenFile = lib.mkOption { - type = lib.types.path; - description = "Github token file"; + authType = lib.mkOption { + type = lib.types.attrTag { + legacy = lib.mkOption { + description = "GitHub legacy auth backend"; + type = lib.types.submodule { + options.tokenFile = lib.mkOption { + type = lib.types.path; + description = "Github token file"; + }; + }; + }; + + app = lib.mkOption { + description = "GitHub legacy auth backend"; + type = lib.types.submodule { + options.id = lib.mkOption { + type = lib.types.int; + description = '' + GitHub app ID. + ''; + }; + + options.secretKeyFile = lib.mkOption { + type = lib.types.str; + description = '' + GitHub app secret key file location. + ''; + }; + }; + }; + }; }; + webhookSecretFile = lib.mkOption { type = lib.types.path; description = "Github webhook secret file"; @@ -243,7 +290,16 @@ in home = "/var/lib/buildbot"; extraImports = '' from datetime import timedelta - from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig + from buildbot_nix import ( + GithubConfig, + NixConfigurator, + CachixConfig, + GiteaConfig, + ) + from buildbot_nix.github.auth._type import ( + AuthTypeLegacy, + AuthTypeApp, + ) ''; configurators = [ '' @@ -260,6 +316,18 @@ in oauth_id=${builtins.toJSON cfg.github.oauthId}, buildbot_user=${builtins.toJSON cfg.github.user}, topic=${builtins.toJSON cfg.github.topic}, + auth_type=${ + if cfg.github.authType ? "legacy" then + ''AuthTypeLegacy()'' + else if cfg.github.authType ? "app" then + '' + AuthTypeApp( + app_id=${toString cfg.github.authType.app.id}, + ) + '' + else + throw "One of AuthTypeApp or AuthTypeLegacy must be enabled" + } )" }, gitea=${ @@ -323,6 +391,9 @@ in systemd.services.buildbot-master = { after = [ "postgresql.service" ]; + path = [ + pkgs.openssl + ]; serviceConfig = { # in master.py we read secrets from $CREDENTIALS_DIRECTORY LoadCredential = @@ -337,10 +408,15 @@ in ( cfg.cachix.authTokenFile != null ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" - ++ lib.optionals (cfg.github.enable) [ - "github-token:${cfg.github.tokenFile}" + ++ lib.optionals (cfg.github.enable) ([ "github-webhook-secret:${cfg.github.webhookSecretFile}" ] + ++ lib.optionals (cfg.github.authType ? "legacy") [ + "github-token:${cfg.github.authType.legacy.tokenFile}" + ] + ++ lib.optionals (cfg.github.authType ? "app") [ + "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}" + ]) ++ lib.optionals cfg.gitea.enable [ "gitea-token:${cfg.gitea.tokenFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" diff --git a/pyproject.toml b/pyproject.toml index 17d67e6..c0b6710 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" } [tool.setuptools] packages = [ "buildbot_nix", + "buildbot_nix.github", + "buildbot_nix.github.auth", "buildbot_effects" ] @@ -82,6 +84,17 @@ ignore = [ # not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html "G010", # Logging statement uses `warn` instead of `warning` + + # gives falls positives and isn't hard to check munually + "ERA001" +] + +[tool.ruff.lint.pep8-naming] +ignore-names = [ + "checkConfig", + "baseURL", + "reconfigService", + "sendMessage", ] [tool.mypy]