Merge pull request #156 from MagicRB/github_app

Add GitHub App support

README.md

@@ -66,16 +66,36 @@ We have the following two roles:
 
 ### Integration with GitHub
 
-To integrate with GitHub:
+#### GitHub App
+
+To integrate with GitHub using app authentication:
+
+1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
+   authentication on the Buildbot dashboard. Enable the following permissions:
+   - Contents: Read-only
+   - Metadata: Read-only
+   - Commit statuses: Read and write
+   - Webhooks: Read and write
+2. **GitHub App private key**: Get the app private key and app ID from GitHub,
+   configure using the buildbot-nix NixOS module.
+3. **Install App**: Install the app for an organization or specific user.
+4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to
+   changes (new repositories or installations) automatically, it is therefore
+   necessary to manually trigger a reload or wait for the next periodic reload.
+
+#### Legacy Token Auth
+
+To integrate with GitHub using legacy token authentication:
 
 1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo`
    permissions. For GitHub organizations, it's advisable to create a separate
    GitHub user for managing repository webhooks.
 
-#### Optional when using GitHub login
+### Optional when using GitHub login
 
 1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
-   authentication on the Buildbot dashboard.
+   authentication on the Buildbot dashboard. (can be the same as for GitHub App
+   auth)
 2. **OAuth Credentials**: After installing the app, generate OAuth credentials
    and configure them in the buildbot-nix NixOS module. Set the callback url to
    `https://<your-domain>/auth/login`.

buildbot_nix/__init__.py

@@ -872,6 +872,8 @@ class NixConfigurator(ConfiguratorBase):
                 ],
             )
             config["services"].append(backend.create_reporter())
+            config.setdefault("secretProviders", [])
+            config["secretsProviders"].extend(backend.create_secret_providers())
 
         systemd_secrets = SecretInAFile(
             dirname=os.environ["CREDENTIALS_DIRECTORY"],
@@ -887,13 +889,15 @@ class NixConfigurator(ConfiguratorBase):
                 backend.create_change_hook()
             )
 
-        if "auth" not in config["www"]:
-            config["www"].setdefault("avatar_methods", [])
+        config["www"].setdefault("avatar_methods", [])
 
-            for backend in backends.values():
-                avatar_method = backend.create_avatar_method()
-                if avatar_method is not None:
-                    config["www"]["avatar_methods"].append(avatar_method)
+        for backend in backends.values():
+            avatar_method = backend.create_avatar_method()
+            print(avatar_method)
+            if avatar_method is not None:
+                config["www"]["avatar_methods"].append(avatar_method)
+
+        if "auth" not in config["www"]:
             # TODO one cannot have multiple auth backends...
             if auth is not None:
                 config["www"]["auth"] = auth

buildbot_nix/common.py

@@ -2,6 +2,8 @@ import contextlib
 import http.client
 import json
 import urllib.request
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Any
 
 
@@ -9,7 +11,9 @@ def slugify_project_name(name: str) -> str:
     return name.replace(".", "-").replace("/", "-")
 
 
-def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
+def paginated_github_request(
+    url: str, token: str, subkey: None | str = None
+) -> list[dict[str, Any]]:
     next_url: str | None = url
     items = []
     while next_url:
@@ -29,7 +33,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
                 link_parts = link.split(";")
                 if link_parts[1].strip() == 'rel="next"':
                     next_url = link_parts[0][1:-1]
-        items += res.json()
+        if subkey is not None:
+            items += res.json()[subkey]
+        else:
+            items += res.json()
     return items
 
 
@@ -78,3 +85,15 @@ def http_request(
         msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
         raise HttpError(msg) from e
     return HttpResponse(resp)
+
+
+def atomic_write_file(file: Path, data: str) -> None:
+    with NamedTemporaryFile("w", delete=False, dir=file.parent) as f:
+        path = Path(f.name)
+        try:
+            f.write(data)
+            f.flush()
+            path.rename(file)
+        except OSError:
+            path.unlink()
+            raise

buildbot_nix/github/__init__.py

@@ -0,0 +1 @@
+

buildbot_nix/github/auth/__init__.py

@@ -0,0 +1 @@
+

buildbot_nix/github/auth/_type.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class AuthType:
+    pass
+
+
+@dataclass
+class AuthTypeLegacy(AuthType):
+    token_secret_name: str = "github-token"
+
+
+@dataclass
+class AuthTypeApp(AuthType):
+    app_id: int
+    app_secret_key_name: str = "github-app-secret-key"
+    app_installation_token_map_name: Path = Path(
+        "github-app-installation-token-map.json"
+    )
+    app_project_id_map_name: Path = Path("github-app-project-id-map-name.json")
+    app_jwt_token_name: Path = Path("github-app-jwt-token")

buildbot_nix/github/installation_token.py

@@ -0,0 +1,108 @@
+import json
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from buildbot_nix.common import (
+    HttpResponse,
+    atomic_write_file,
+    http_request,
+)
+
+from .jwt_token import JWTToken
+from .repo_token import RepoToken
+
+
+class InstallationToken(RepoToken):
+    GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
+
+    jwt_token: JWTToken
+    installation_id: int
+
+    token: str
+    expiration: datetime
+    installations_token_map_name: Path
+
+    @staticmethod
+    def _create_installation_access_token(
+        jwt_token: JWTToken, installation_id: int
+    ) -> HttpResponse:
+        return http_request(
+            f"https://api.github.com/app/installations/{installation_id}/access_tokens",
+            data={},
+            headers={"Authorization": f"Bearer {jwt_token.get()}"},
+            method="POST",
+        )
+
+    @staticmethod
+    def _generate_token(
+        jwt_token: JWTToken, installation_id: int
+    ) -> tuple[str, datetime]:
+        token = InstallationToken._create_installation_access_token(
+            jwt_token, installation_id
+        ).json()["token"]
+        expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME
+
+        return token, expiration
+
+    def __init__(
+        self,
+        jwt_token: JWTToken,
+        installation_id: int,
+        installations_token_map_name: Path,
+        installation_token: None | tuple[str, datetime] = None,
+    ) -> None:
+        self.jwt_token = jwt_token
+        self.installation_id = installation_id
+        self.installations_token_map_name = installations_token_map_name
+
+        if installation_token is None:
+            self.token, self.expiration = InstallationToken._generate_token(
+                self.jwt_token, self.installation_id
+            )
+            self._save()
+        else:
+            self.token, self.expiration = installation_token
+
+    def get(self) -> str:
+        self.verify()
+        return self.token
+
+    def get_as_secret(self) -> str:
+        return f"%(secret:github-token-{self.installation_id})"
+
+    def verify(self) -> None:
+        if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8:
+            self.token, self.expiration = InstallationToken._generate_token(
+                self.jwt_token, self.installation_id
+            )
+            self._save()
+
+    def _save(self) -> None:
+        # of format:
+        # {
+        #   123: {
+        #     expiration: <datetime>,
+        #     token: "token"
+        #   }
+        # }
+        installations_token_map: dict[str, Any]
+        if self.installations_token_map_name.exists():
+            installations_token_map = json.loads(
+                self.installations_token_map_name.read_text()
+            )
+        else:
+            installations_token_map = {}
+
+        installations_token_map.update(
+            {
+                str(self.installation_id): {
+                    "expiration": self.expiration.isoformat(),
+                    "token": self.token,
+                }
+            }
+        )
+
+        atomic_write_file(
+            self.installations_token_map_name, json.dumps(installations_token_map)
+        )

buildbot_nix/github/jwt_token.py

@@ -0,0 +1,84 @@
+import base64
+import json
+import os
+import subprocess
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from .repo_token import RepoToken
+
+
+class JWTToken(RepoToken):
+    app_id: int
+    app_private_key: str
+    lifetime: timedelta
+
+    expiration: datetime
+    token: str
+
+    def __init__(
+        self,
+        app_id: int,
+        app_private_key: str,
+        lifetime: timedelta = timedelta(minutes=10),
+    ) -> None:
+        self.app_id = app_id
+        self.app_private_key = app_private_key
+        self.lifetime = lifetime
+
+        self.token, self.expiration = JWTToken.generate_token(
+            self.app_id, self.app_private_key, lifetime
+        )
+
+    @staticmethod
+    def generate_token(
+        app_id: int, app_private_key: str, lifetime: timedelta
+    ) -> tuple[str, datetime]:
+        def build_jwt_payload(
+            app_id: int, lifetime: timedelta
+        ) -> tuple[dict[str, Any], datetime]:
+            jwt_iat_drift: timedelta = timedelta(seconds=60)
+            now: datetime = datetime.now(tz=UTC)
+            iat: datetime = now - jwt_iat_drift
+            exp: datetime = iat + lifetime
+            jwt_payload = {
+                "iat": int(iat.timestamp()),
+                "exp": int(exp.timestamp()),
+                "iss": str(app_id),
+            }
+            return (jwt_payload, exp)
+
+        def rs256_sign(data: str, private_key: str) -> str:
+            signature = subprocess.run(
+                ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key],
+                input=data.encode("utf-8"),
+                stdout=subprocess.PIPE,
+                check=True,
+                cwd=os.environ.get("CREDENTIALS_DIRECTORY"),
+            ).stdout
+            return base64url(signature)
+
+        def base64url(data: bytes) -> str:
+            return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8")
+
+        jwt, expiration = build_jwt_payload(app_id, lifetime)
+        jwt_payload = json.dumps(jwt).encode("utf-8")
+        json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8")
+        encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}"
+        encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key)
+        return (f"{encoded_jwt_parts}.{encoded_mac}", expiration)
+
+        # installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt)
+
+        # return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations))
+
+    def get(self) -> str:
+        if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8:
+            self.token, self.expiration = JWTToken.generate_token(
+                self.app_id, self.app_private_key, self.lifetime
+            )
+
+        return self.token
+
+    def get_as_secret(self) -> str:
+        return "%(secret:github-jwt-token)"

buildbot_nix/github/legacy_token.py

@@ -0,0 +1,14 @@
+from .repo_token import RepoToken
+
+
+class LegacyToken(RepoToken):
+    token: str
+
+    def __init__(self, token: str) -> None:
+        self.token = token
+
+    def get(self) -> str:
+        return self.token
+
+    def get_as_secret(self) -> str:
+        return "%(secret:github-token)"

buildbot_nix/github/repo_token.py

@@ -0,0 +1,11 @@
+from abc import abstractmethod
+
+
+class RepoToken:
+    @abstractmethod
+    def get(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_as_secret(self) -> str:
+        pass

buildbot_nix/github_projects.py

@@ -1,10 +1,13 @@
 import json
 import os
 import signal
-from collections.abc import Generator
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Generator
 from dataclasses import dataclass
+from datetime import (
+    datetime,
+)
 from pathlib import Path
-from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any
 
 from buildbot.config.builder import BuilderConfig
@@ -13,10 +16,12 @@ from buildbot.process.buildstep import BuildStep
 from buildbot.process.properties import Interpolate
 from buildbot.reporters.base import ReporterBase
 from buildbot.reporters.github import GitHubStatusPush
+from buildbot.secrets.providers.base import SecretProviderBase
 from buildbot.www.auth import AuthBase
 from buildbot.www.avatar import AvatarBase, AvatarGitHub
 from buildbot.www.oauth2 import GitHubAuth
 from twisted.internet import defer, threads
+from twisted.logger import Logger
 from twisted.python import log
 from twisted.python.failure import Failure
 
@@ -24,24 +29,96 @@ if TYPE_CHECKING:
     from buildbot.process.log import StreamLog
 
 from .common import (
+    atomic_write_file,
     http_request,
     paginated_github_request,
     slugify_project_name,
 )
+from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy
+from .github.installation_token import InstallationToken
+from .github.jwt_token import JWTToken
+from .github.legacy_token import (
+    LegacyToken,
+)
+from .github.repo_token import (
+    RepoToken,
+)
 from .projects import GitBackend, GitProject
 from .secrets import read_secret_file
 
+tlog = Logger()
 
-class ReloadGithubProjects(BuildStep):
+
+def get_installations(jwt_token: JWTToken) -> list[int]:
+    installations = paginated_github_request(
+        "https://api.github.com/app/installations?per_page=100", jwt_token.get()
+    )
+
+    return [installation["id"] for installation in installations]
+
+
+class ReloadGithubInstallations(BuildStep):
     name = "reload_github_projects"
 
-    def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
-        self.token = token
+    jwt_token: JWTToken
+    project_cache_file: Path
+    installation_token_map_name: Path
+    project_id_map_name: Path
+
+    def __init__(
+        self,
+        jwt_token: JWTToken,
+        project_cache_file: Path,
+        installation_token_map_name: Path,
+        project_id_map_name: Path,
+        **kwargs: Any,
+    ) -> None:
+        self.jwt_token = jwt_token
+        self.installation_token_map_name = installation_token_map_name
+        self.project_id_map_name = project_id_map_name
         self.project_cache_file = project_cache_file
         super().__init__(**kwargs)
 
     def reload_projects(self) -> None:
-        refresh_projects(self.token, self.project_cache_file)
+        installation_token_map = GithubBackend.create_missing_installations(
+            self.jwt_token,
+            self.installation_token_map_name,
+            GithubBackend.load_installations(
+                self.jwt_token,
+                self.installation_token_map_name,
+            ),
+            get_installations(self.jwt_token),
+        )
+
+        repos: list[Any] = []
+        project_id_map: dict[str, int] = {}
+
+        repos = []
+
+        for k, v in installation_token_map.items():
+            new_repos = refresh_projects(
+                v.get(),
+                self.project_cache_file,
+                clear=True,
+                api_endpoint="/installation/repositories",
+                subkey="repositories",
+                require_admin=False,
+            )
+
+            for repo in new_repos:
+                repo["installation_id"] = k
+
+            repos.extend(new_repos)
+
+            for repo in new_repos:
+                project_id_map[repo["full_name"]] = k
+
+        atomic_write_file(self.project_cache_file, json.dumps(repos))
+        atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
+
+        tlog.info(
+            f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token."
+        )
 
     @defer.inlineCallbacks
     def run(self) -> Generator[Any, object, Any]:
@@ -65,36 +142,378 @@ class ReloadGithubProjects(BuildStep):
             return util.FAILURE
 
 
+class ReloadGithubProjects(BuildStep):
+    name = "reload_github_projects"
+
+    def __init__(
+        self, token: RepoToken, project_cache_file: Path, **kwargs: Any
+    ) -> None:
+        self.token = token
+        self.project_cache_file = project_cache_file
+        super().__init__(**kwargs)
+
+    def reload_projects(self) -> None:
+        repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file)
+
+        log.msg(repos)
+
+        atomic_write_file(self.project_cache_file, json.dumps(repos))
+
+    @defer.inlineCallbacks
+    def run(self) -> Generator[Any, object, Any]:
+        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
+
+        self.error_msg = ""
+
+        def error_cb(failure: Failure) -> int:
+            self.error_msg += failure.getTraceback()
+            return util.FAILURE
+
+        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
+        res = yield d
+        if res == util.SUCCESS:
+            # reload the buildbot config
+            os.kill(os.getpid(), signal.SIGHUP)
+            return util.SUCCESS
+        else:
+            log: StreamLog = yield self.addLog("log")
+            log.addStderr(f"Failed to reload project list: {self.error_msg}")
+            return util.FAILURE
+
+
+class GitHubAppStatusPush(GitHubStatusPush):
+    token_source: Callable[[int], RepoToken]
+    project_id_source: Callable[[str], int]
+    saved_args: dict[str, Any]
+    saved_kwargs: dict[str, Any]
+
+    def checkConfig(
+        self,
+        token_source: Callable[[int], RepoToken],
+        project_id_source: Callable[[str], int],
+        context: Any = None,
+        baseURL: Any = None,
+        verbose: Any = False,
+        debug: Any = None,
+        verify: Any = None,
+        generators: Any = None,
+        **kwargs: dict[str, Any],
+    ) -> Any:
+        if generators is None:
+            generators = self._create_default_generators()
+
+        if "token" in kwargs:
+            del kwargs["token"]
+        super().checkConfig(
+            token="",
+            context=context,
+            baseURL=baseURL,
+            verbose=verbose,
+            debug=debug,
+            verify=verify,
+            generators=generators,
+            **kwargs,
+        )
+
+    def reconfigService(
+        self,
+        token_source: Callable[[int], RepoToken],
+        project_id_source: Callable[[str], int],
+        context: Any = None,
+        baseURL: Any = None,
+        verbose: Any = False,
+        debug: Any = None,
+        verify: Any = None,
+        generators: Any = None,
+        **kwargs: dict[str, Any],
+    ) -> Any:
+        if "saved_args" not in self or self.saved_args is None:
+            self.saved_args = {}
+        self.token_source = token_source
+        self.project_id_source = project_id_source
+        self.saved_kwargs = kwargs
+        self.saved_args["context"] = context
+        self.saved_args["baseURL"] = baseURL
+        self.saved_args["verbose"] = verbose
+        self.saved_args["debug"] = debug
+        self.saved_args["verify"] = verify
+        self.saved_args["generators"] = generators
+
+        if generators is None:
+            generators = self._create_default_generators()
+
+        if "token" in kwargs:
+            del kwargs["token"]
+        super().reconfigService(
+            token="",
+            context=context,
+            baseURL=baseURL,
+            verbose=verbose,
+            debug=debug,
+            verify=verify,
+            generators=generators,
+            **kwargs,
+        )
+
+    def sendMessage(self, reports: Any) -> Any:
+        build = reports[0]["builds"][0]
+        sourcestamps = build["buildset"].get("sourcestamps")
+        if not sourcestamps:
+            return None
+
+        for sourcestamp in sourcestamps:
+            build["buildset"]["sourcestamps"] = [sourcestamp]
+
+            token: str
+
+            if "project" in sourcestamp and sourcestamp["project"] != "":
+                token = self.token_source(
+                    self.project_id_source(sourcestamp["project"])
+                ).get()
+            else:
+                token = ""
+
+            super().reconfigService(
+                token,
+                context=self.saved_args["context"],
+                baseURL=self.saved_args["baseURL"],
+                verbose=self.saved_args["verbose"],
+                debug=self.saved_args["debug"],
+                verify=self.saved_args["verify"],
+                generators=self.saved_args["generators"],
+                **self.saved_kwargs,
+            )
+
+            return super().sendMessage(reports)
+
+        return None
+
+
+class GithubAuthBackend(ABC):
+    @abstractmethod
+    def get_general_token(self) -> RepoToken:
+        pass
+
+    @abstractmethod
+    def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
+        pass
+
+    @abstractmethod
+    def create_secret_providers(self) -> list[SecretProviderBase]:
+        pass
+
+    @abstractmethod
+    def create_reporter(self) -> ReporterBase:
+        pass
+
+    @abstractmethod
+    def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
+        pass
+
+
+class GithubLegacyAuthBackend(GithubAuthBackend):
+    auth_type: AuthTypeLegacy
+
+    token: LegacyToken
+
+    def __init__(self, auth_type: AuthTypeLegacy) -> None:
+        self.auth_type = auth_type
+        self.token = LegacyToken(read_secret_file(auth_type.token_secret_name))
+
+    def get_general_token(self) -> RepoToken:
+        return self.token
+
+    def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
+        return self.token
+
+    def create_secret_providers(self) -> list[SecretProviderBase]:
+        return [GitHubLegacySecretService(self.token)]
+
+    def create_reporter(self) -> ReporterBase:
+        return GitHubStatusPush(
+            token=self.token.get(),
+            # Since we dynamically create build steps,
+            # we use `virtual_builder_name` in the webinterface
+            # so that we distinguish what has beeing build
+            context=Interpolate("buildbot/%(prop:status_name)s"),
+        )
+
+    def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
+        return ReloadGithubProjects(
+            token=self.token, project_cache_file=project_cache_file
+        )
+
+
+class GitHubLegacySecretService(SecretProviderBase):
+    name = "GitHubLegacySecretService"
+    token: LegacyToken
+
+    def reconfigService(self, token: LegacyToken) -> None:
+        self.token = token
+
+    def get(self, entry: str) -> str | None:
+        """
+        get the value from the file identified by 'entry'
+        """
+        if entry.startswith("github-token"):
+            return self.token.get()
+        return None
+
+
+class GithubAppAuthBackend(GithubAuthBackend):
+    auth_type: AuthTypeApp
+
+    jwt_token: JWTToken
+    installation_tokens: dict[int, InstallationToken]
+    project_id_map: dict[str, int]
+
+    def __init__(self, auth_type: AuthTypeApp) -> None:
+        self.auth_type = auth_type
+        self.jwt_token = JWTToken(
+            self.auth_type.app_id, self.auth_type.app_secret_key_name
+        )
+        self.installation_tokens = GithubBackend.load_installations(
+            self.jwt_token,
+            self.auth_type.app_installation_token_map_name,
+        )
+        if self.auth_type.app_project_id_map_name.exists():
+            self.project_id_map = json.loads(
+                self.auth_type.app_project_id_map_name.read_text()
+            )
+        else:
+            tlog.info(
+                "~project-id-map~ is not present, GitHub project reload will follow."
+            )
+            self.project_id_map = {}
+
+    def get_general_token(self) -> RepoToken:
+        return self.jwt_token
+
+    def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
+        assert "installation_id" in repo, f"Missing installation_id in {repo}"
+        return self.installation_tokens[repo["installation_id"]]
+
+    def create_secret_providers(self) -> list[SecretProviderBase]:
+        return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)]
+
+    def create_reporter(self) -> ReporterBase:
+        return GitHubAppStatusPush(
+            token_source=lambda iid: self.installation_tokens[iid],
+            project_id_source=lambda project: self.project_id_map[project],
+            # Since we dynamically create build steps,
+            # we use `virtual_builder_name` in the webinterface
+            # so that we distinguish what has beeing build
+            context=Interpolate("buildbot/%(prop:status_name)s"),
+        )
+
+    def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
+        return ReloadGithubInstallations(
+            self.jwt_token,
+            project_cache_file,
+            self.auth_type.app_installation_token_map_name,
+            self.auth_type.app_project_id_map_name,
+        )
+
+
+class GitHubAppSecretService(SecretProviderBase):
+    name = "GitHubAppSecretService"
+    installation_tokens: dict[int, InstallationToken]
+    jwt_token: JWTToken
+
+    def reconfigService(
+        self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken
+    ) -> None:
+        self.installation_tokens = installation_tokens
+        self.jwt_token = jwt_token
+
+    def get(self, entry: str) -> str | None:
+        """
+        get the value from the file identified by 'entry'
+        """
+        if entry.startswith("github-token-"):
+            return self.installation_tokens[
+                int(entry.removeprefix("github-token-"))
+            ].get()
+        if entry == "github-jwt-token":
+            return self.jwt_token.get()
+        return None
+
+
 @dataclass
 class GithubConfig:
     oauth_id: str | None
-
+    auth_type: AuthType
     # TODO unused
     buildbot_user: str
     oauth_secret_name: str = "github-oauth-secret"
-    token_secret_name: str = "github-token"
     webhook_secret_name: str = "github-webhook-secret"
-    project_cache_file: Path = Path("github-project-cache.json")
+    project_cache_file: Path = Path("github-project-cache-v1.json")
     topic: str | None = "build-with-buildbot"
 
-    def token(self) -> str:
-        return read_secret_file(self.token_secret_name)
-
 
 @dataclass
 class GithubBackend(GitBackend):
     config: GithubConfig
     webhook_secret: str
 
+    auth_backend: GithubAuthBackend
+
     def __init__(self, config: GithubConfig) -> None:
         self.config = config
         self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
 
+        if isinstance(self.config.auth_type, AuthTypeLegacy):
+            self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
+        elif isinstance(self.config.auth_type, AuthTypeApp):
+            self.auth_backend = GithubAppAuthBackend(self.config.auth_type)
+
+    @staticmethod
+    def load_installations(
+        jwt_token: JWTToken, installations_token_map_name: Path
+    ) -> dict[int, InstallationToken]:
+        initial_installations_map: dict[str, Any]
+        if installations_token_map_name.exists():
+            initial_installations_map = json.loads(
+                installations_token_map_name.read_text()
+            )
+        else:
+            initial_installations_map = {}
+
+        installations_map: dict[int, InstallationToken] = {}
+
+        for iid, installation in initial_installations_map.items():
+            token: str = installation["token"]
+            expiration: datetime = datetime.fromisoformat(installation["expiration"])
+            installations_map[int(iid)] = InstallationToken(
+                jwt_token,
+                int(iid),
+                installations_token_map_name,
+                installation_token=(token, expiration),
+            )
+
+        return installations_map
+
+    @staticmethod
+    def create_missing_installations(
+        jwt_token: JWTToken,
+        installations_token_map_name: Path,
+        installations_map: dict[int, InstallationToken],
+        installations: list[int],
+    ) -> dict[int, InstallationToken]:
+        for installation in set(installations) - installations_map.keys():
+            installations_map[installation] = InstallationToken(
+                jwt_token,
+                installation,
+                installations_token_map_name,
+            )
+
+        return installations_map
+
     def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
         """Updates the flake an opens a PR for it."""
         factory = util.BuildFactory()
         factory.addStep(
-            ReloadGithubProjects(self.config.token(), self.config.project_cache_file),
+            self.auth_backend.create_reload_builder_step(self.config.project_cache_file)
         )
         return util.BuilderConfig(
             name=self.reload_builder_name,
@@ -103,24 +522,33 @@ class GithubBackend(GitBackend):
         )
 
     def create_reporter(self) -> ReporterBase:
-        return GitHubStatusPush(
-            token=self.config.token(),
-            # Since we dynamically create build steps,
-            # we use `virtual_builder_name` in the webinterface
-            # so that we distinguish what has beeing build
-            context=Interpolate("buildbot/%(prop:status_name)s"),
-        )
+        return self.auth_backend.create_reporter()
 
     def create_change_hook(self) -> dict[str, Any]:
         return {
             "secret": self.webhook_secret,
             "strict": True,
-            "token": self.config.token(),
+            "token": self.auth_backend.get_general_token().get(),
             "github_property_whitelist": ["github.base.sha", "github.head.sha"],
         }
 
     def create_avatar_method(self) -> AvatarBase | None:
-        return AvatarGitHub(token=self.config.token())
+        avatar = AvatarGitHub(token=self.auth_backend.get_general_token().get())
+
+        # TODO: not a proper fix, the /users/{username} endpoint is per installation, but I'm not sure
+        # how to tell which installation token to use, unless there is a way to build a huge map of
+        # username -> token, or we just try each one in order
+        def _get_avatar_by_username(self: Any, username: Any) -> Any:
+            return f"https://github.com/{username}.png"
+
+        import types
+
+        avatar._get_avatar_by_username = types.MethodType(  # noqa: SLF001
+            _get_avatar_by_username,
+            avatar,
+        )
+
+        return avatar
 
     def create_auth(self) -> AuthBase:
         assert self.config.oauth_id is not None, "GitHub OAuth ID is required"
@@ -130,6 +558,9 @@ class GithubBackend(GitBackend):
             apiVersion=4,
         )
 
+    def create_secret_providers(self) -> list[SecretProviderBase]:
+        return self.auth_backend.create_secret_providers()
+
     def load_projects(self) -> list["GitProject"]:
         if not self.config.project_cache_file.exists():
             return []
@@ -138,19 +569,53 @@ class GithubBackend(GitBackend):
             json.loads(self.config.project_cache_file.read_text()),
             key=lambda x: x["full_name"],
         )
+
+        if isinstance(self.auth_backend, GithubAppAuthBackend):
+            dropped_repos = list(
+                filter(lambda repo: not "installation_id" in repo, repos)
+            )
+            if dropped_repos:
+                tlog.info(
+                    "Dropped projects follow, refresh will follow after initialisation:"
+                )
+                for dropped_repo in dropped_repos:
+                    tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
+            repos = list(filter(lambda repo: "installation_id" in repo, repos))
+
+        tlog.info(f"Loading {len(repos)} cached repositories.")
         return list(
             filter(
                 lambda project: self.config.topic is not None
                 and self.config.topic in project.topics,
                 [
-                    GithubProject(self.config, self.webhook_secret, repo)
+                    GithubProject(
+                        self.auth_backend.get_repo_token(repo),
+                        self.config,
+                        self.webhook_secret,
+                        repo,
+                    )
                     for repo in repos
                 ],
             )
         )
 
     def are_projects_cached(self) -> bool:
-        return self.config.project_cache_file.exists()
+        if not self.config.project_cache_file.exists():
+            return False
+
+        if (
+            isinstance(self.config.auth_type, AuthTypeApp)
+            and not self.config.auth_type.app_project_id_map_name.exists()
+        ):
+            return False
+
+        all_have_installation_id = True
+        for project in json.loads(self.config.project_cache_file.read_text()):
+            if not "installation_id" in project:
+                all_have_installation_id = False
+                break
+
+        return all_have_installation_id
 
     @property
     def type(self) -> str:
@@ -171,10 +636,18 @@ class GithubBackend(GitBackend):
 
 class GithubProject(GitProject):
     config: GithubConfig
+    webhook_secret: str
+    data: dict[str, Any]
+    token: RepoToken
 
     def __init__(
-        self, config: GithubConfig, webhook_secret: str, data: dict[str, Any]
+        self,
+        token: RepoToken,
+        config: GithubConfig,
+        webhook_secret: str,
+        data: dict[str, Any],
     ) -> None:
+        self.token = token
         self.config = config
         self.webhook_secret = webhook_secret
         self.data = data
@@ -187,7 +660,7 @@ class GithubProject(GitProject):
     ) -> None:
         hooks = paginated_github_request(
             f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
-            self.config.token(),
+            self.token.get(),
         )
         config = dict(
             url=webhook_url + "change_hook/github",
@@ -199,7 +672,7 @@ class GithubProject(GitProject):
             name="web", active=True, events=["push", "pull_request"], config=config
         )
         headers = {
-            "Authorization": f"Bearer {self.config.token()}",
+            "Authorization": f"Bearer {self.token.get()}",
             "Accept": "application/vnd.github+json",
             "Content-Type": "application/json",
             "X-GitHub-Api-Version": "2022-11-28",
@@ -217,7 +690,7 @@ class GithubProject(GitProject):
         )
 
     def get_project_url(self) -> str:
-        return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}"
+        return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}"
 
     @property
     def pretty_type(self) -> str:
@@ -260,14 +733,25 @@ class GithubProject(GitProject):
         return self.data["owner"]["type"] == "Organization"
 
 
-def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
-    repos = []
+def refresh_projects(
+    github_token: str,
+    repo_cache_file: Path,
+    repos: list[Any] | None = None,
+    clear: bool = True,
+    api_endpoint: str = "/user/repos",
+    subkey: None | str = None,
+    require_admin: bool = True,
+) -> list[Any]:
+    if repos is None:
+        repos = []
 
     for repo in paginated_github_request(
-        "https://api.github.com/user/repos?per_page=100",
+        f"https://api.github.com{api_endpoint}?per_page=100",
         github_token,
+        subkey=subkey,
     ):
-        if not repo["permissions"]["admin"]:
+        # TODO actually check for this properly
+        if not repo["permissions"]["admin"] and require_admin:
             name = repo["full_name"]
             log.msg(
                 f"skipping {name} because we do not have admin privileges, needed for hook management",
@@ -275,12 +759,4 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
         else:
             repos.append(repo)
 
-    with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
-        path = Path(f.name)
-        try:
-            f.write(json.dumps(repos))
-            f.flush()
-            path.rename(repo_cache_file)
-        except OSError:
-            path.unlink()
-            raise
+    return repos

buildbot_nix/projects.py

@@ -3,6 +3,7 @@ from typing import Any
 
 from buildbot.config.builder import BuilderConfig
 from buildbot.reporters.base import ReporterBase
+from buildbot.secrets.providers.base import SecretProviderBase
 from buildbot.www.auth import AuthBase
 from buildbot.www.avatar import AvatarBase
 
@@ -28,6 +29,9 @@ class GitBackend(ABC):
     def create_auth(self) -> AuthBase:
         pass
 
+    def create_secret_providers(self) -> list[SecretProviderBase]:
+        return []
+
     @abstractmethod
     def load_projects(self) -> list["GitProject"]:
         pass

examples/master.nix

@@ -21,8 +21,14 @@
     github = {
       # Github user used as a CI identity
       user = "mic92-buildbot";
-      # Github token of the same user
-      tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
+      authType.legacy = {
+        # Github token of the same user
+        tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
+      };
+      # authType.app = {
+      #   id = "00000000000000000"; # FIXME: replace with App ID obtained from GitHub
+      #   secretKeyFile = pkgs.writeText "app-secret.key" "00000000000000000000"; # FIXME: replace with App secret key obtained from GitHub
+      # };
       # A random secret used to verify incoming webhooks from GitHub
       # buildbot-nix will set up a webhook for each project in the organization
       webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store

nix/checks/master.nix

@@ -16,7 +16,9 @@
         '';
         admins = [ "Mic92" ];
         github = {
-          tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
+          authType.legacy = {
+            tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
+          };
           webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
           oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
           oauthId = "aaaaaaaaaaaaaaaaaaaa";

nix/master.nix

@@ -24,6 +24,24 @@ in
         "admins"
       ]
     )
+    (mkRenamedOptionModule
+      [
+        "services"
+        "buildbot-nix"
+        "master"
+        "github"
+        "tokenFile"
+      ]
+      [
+        "services"
+        "buildbot-nix"
+        "master"
+        "github"
+        "authType"
+        "legacy"
+        "tokenFile"
+      ]
+    )
   ];
 
   options = {
@@ -106,10 +124,39 @@ in
           default = cfg.authBackend == "github";
         };
 
-        tokenFile = lib.mkOption {
-          type = lib.types.path;
-          description = "Github token file";
+        authType = lib.mkOption {
+          type = lib.types.attrTag {
+            legacy = lib.mkOption {
+              description = "GitHub legacy auth backend";
+              type = lib.types.submodule {
+                options.tokenFile = lib.mkOption {
+                  type = lib.types.path;
+                  description = "Github token file";
+                };
+              };
+            };
+
+            app = lib.mkOption {
+              description = "GitHub legacy auth backend";
+              type = lib.types.submodule {
+                options.id = lib.mkOption {
+                  type = lib.types.int;
+                  description = ''
+                    GitHub app ID.
+                  '';
+                };
+
+                options.secretKeyFile = lib.mkOption {
+                  type = lib.types.str;
+                  description = ''
+                    GitHub app secret key file location.
+                  '';
+                };
+              };
+            };
+          };
         };
+
         webhookSecretFile = lib.mkOption {
           type = lib.types.path;
           description = "Github webhook secret file";
@@ -243,7 +290,16 @@ in
       home = "/var/lib/buildbot";
       extraImports = ''
         from datetime import timedelta
-        from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig
+        from buildbot_nix import (
+          GithubConfig,
+          NixConfigurator,
+          CachixConfig,
+          GiteaConfig,
+        )
+        from buildbot_nix.github.auth._type import (
+          AuthTypeLegacy,
+          AuthTypeApp,
+        )
       '';
       configurators = [
         ''
@@ -260,6 +316,18 @@ in
                   oauth_id=${builtins.toJSON cfg.github.oauthId},
                   buildbot_user=${builtins.toJSON cfg.github.user},
                   topic=${builtins.toJSON cfg.github.topic},
+                  auth_type=${
+                          if cfg.github.authType ? "legacy" then
+                            ''AuthTypeLegacy()''
+                          else if cfg.github.authType ? "app" then
+                            ''
+                              AuthTypeApp(
+                                app_id=${toString cfg.github.authType.app.id},
+                              )
+                            ''
+                          else
+                            throw "One of AuthTypeApp or AuthTypeLegacy must be enabled"
+                        }
               )"
               },
               gitea=${
@@ -323,6 +391,9 @@ in
 
     systemd.services.buildbot-master = {
       after = [ "postgresql.service" ];
+      path = [
+        pkgs.openssl
+      ];
       serviceConfig = {
         # in master.py we read secrets from $CREDENTIALS_DIRECTORY
         LoadCredential =
@@ -337,10 +408,15 @@ in
             (
               cfg.cachix.authTokenFile != null
             ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
-          ++ lib.optionals (cfg.github.enable) [
-            "github-token:${cfg.github.tokenFile}"
+          ++ lib.optionals (cfg.github.enable) ([
             "github-webhook-secret:${cfg.github.webhookSecretFile}"
           ]
+          ++ lib.optionals (cfg.github.authType ? "legacy") [
+            "github-token:${cfg.github.authType.legacy.tokenFile}"
+          ]
+          ++ lib.optionals (cfg.github.authType ? "app") [
+            "github-app-secret-key:${cfg.github.authType.app.secretKeyFile}"
+          ])
           ++ lib.optionals cfg.gitea.enable [
             "gitea-token:${cfg.gitea.tokenFile}"
             "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"

pyproject.toml

@@ -26,6 +26,8 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" }
 [tool.setuptools]
 packages = [
   "buildbot_nix",
+  "buildbot_nix.github",
+  "buildbot_nix.github.auth",
   "buildbot_effects"
 ]
 
@@ -82,6 +84,17 @@ ignore = [
 
   # not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html
   "G010",  # Logging statement uses `warn` instead of `warning`
+
+  # gives falls positives and isn't hard to check munually
+  "ERA001"
+]
+
+[tool.ruff.lint.pep8-naming]
+ignore-names = [
+  "checkConfig",
+  "baseURL",
+  "reconfigService",
+  "sendMessage",
 ]
 
 [tool.mypy]