Merge pull request #156 from MagicRB/github_app

Add GitHub App support
This commit is contained in:
Richard Brežák 2024-06-12 15:40:19 +00:00 committed by GitHub
commit 7e327b88b6
Failed to generate hash of commit
16 changed files with 923 additions and 61 deletions

View file

@ -66,16 +66,36 @@ We have the following two roles:
### Integration with GitHub ### Integration with GitHub
To integrate with GitHub: #### GitHub App
To integrate with GitHub using app authentication:
1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
authentication on the Buildbot dashboard. Enable the following permissions:
- Contents: Read-only
- Metadata: Read-only
- Commit statuses: Read and write
- Webhooks: Read and write
2. **GitHub App private key**: Get the app private key and app ID from GitHub,
configure using the buildbot-nix NixOS module.
3. **Install App**: Install the app for an organization or specific user.
4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to
changes (new repositories or installations) automatically, it is therefore
necessary to manually trigger a reload or wait for the next periodic reload.
#### Legacy Token Auth
To integrate with GitHub using legacy token authentication:
1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo` 1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo`
permissions. For GitHub organizations, it's advisable to create a separate permissions. For GitHub organizations, it's advisable to create a separate
GitHub user for managing repository webhooks. GitHub user for managing repository webhooks.
#### Optional when using GitHub login ### Optional when using GitHub login
1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user 1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
authentication on the Buildbot dashboard. authentication on the Buildbot dashboard. (can be the same as for GitHub App
auth)
2. **OAuth Credentials**: After installing the app, generate OAuth credentials 2. **OAuth Credentials**: After installing the app, generate OAuth credentials
and configure them in the buildbot-nix NixOS module. Set the callback url to and configure them in the buildbot-nix NixOS module. Set the callback url to
`https://<your-domain>/auth/login`. `https://<your-domain>/auth/login`.

View file

@ -872,6 +872,8 @@ class NixConfigurator(ConfiguratorBase):
], ],
) )
config["services"].append(backend.create_reporter()) config["services"].append(backend.create_reporter())
config.setdefault("secretProviders", [])
config["secretsProviders"].extend(backend.create_secret_providers())
systemd_secrets = SecretInAFile( systemd_secrets = SecretInAFile(
dirname=os.environ["CREDENTIALS_DIRECTORY"], dirname=os.environ["CREDENTIALS_DIRECTORY"],
@ -887,13 +889,15 @@ class NixConfigurator(ConfiguratorBase):
backend.create_change_hook() backend.create_change_hook()
) )
if "auth" not in config["www"]:
config["www"].setdefault("avatar_methods", []) config["www"].setdefault("avatar_methods", [])
for backend in backends.values(): for backend in backends.values():
avatar_method = backend.create_avatar_method() avatar_method = backend.create_avatar_method()
print(avatar_method)
if avatar_method is not None: if avatar_method is not None:
config["www"]["avatar_methods"].append(avatar_method) config["www"]["avatar_methods"].append(avatar_method)
if "auth" not in config["www"]:
# TODO one cannot have multiple auth backends... # TODO one cannot have multiple auth backends...
if auth is not None: if auth is not None:
config["www"]["auth"] = auth config["www"]["auth"] = auth

View file

@ -2,6 +2,8 @@ import contextlib
import http.client import http.client
import json import json
import urllib.request import urllib.request
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any from typing import Any
@ -9,7 +11,9 @@ def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-") return name.replace(".", "-").replace("/", "-")
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: def paginated_github_request(
url: str, token: str, subkey: None | str = None
) -> list[dict[str, Any]]:
next_url: str | None = url next_url: str | None = url
items = [] items = []
while next_url: while next_url:
@ -29,6 +33,9 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
link_parts = link.split(";") link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"': if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1] next_url = link_parts[0][1:-1]
if subkey is not None:
items += res.json()[subkey]
else:
items += res.json() items += res.json()
return items return items
@ -78,3 +85,15 @@ def http_request(
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise HttpError(msg) from e raise HttpError(msg) from e
return HttpResponse(resp) return HttpResponse(resp)
def atomic_write_file(file: Path, data: str) -> None:
with NamedTemporaryFile("w", delete=False, dir=file.parent) as f:
path = Path(f.name)
try:
f.write(data)
f.flush()
path.rename(file)
except OSError:
path.unlink()
raise

View file

@ -0,0 +1 @@

View file

@ -0,0 +1 @@

View file

@ -0,0 +1,23 @@
from dataclasses import dataclass
from pathlib import Path
@dataclass
class AuthType:
pass
@dataclass
class AuthTypeLegacy(AuthType):
token_secret_name: str = "github-token"
@dataclass
class AuthTypeApp(AuthType):
app_id: int
app_secret_key_name: str = "github-app-secret-key"
app_installation_token_map_name: Path = Path(
"github-app-installation-token-map.json"
)
app_project_id_map_name: Path = Path("github-app-project-id-map-name.json")
app_jwt_token_name: Path = Path("github-app-jwt-token")

View file

@ -0,0 +1,108 @@
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from buildbot_nix.common import (
HttpResponse,
atomic_write_file,
http_request,
)
from .jwt_token import JWTToken
from .repo_token import RepoToken
class InstallationToken(RepoToken):
GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
jwt_token: JWTToken
installation_id: int
token: str
expiration: datetime
installations_token_map_name: Path
@staticmethod
def _create_installation_access_token(
jwt_token: JWTToken, installation_id: int
) -> HttpResponse:
return http_request(
f"https://api.github.com/app/installations/{installation_id}/access_tokens",
data={},
headers={"Authorization": f"Bearer {jwt_token.get()}"},
method="POST",
)
@staticmethod
def _generate_token(
jwt_token: JWTToken, installation_id: int
) -> tuple[str, datetime]:
token = InstallationToken._create_installation_access_token(
jwt_token, installation_id
).json()["token"]
expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME
return token, expiration
def __init__(
self,
jwt_token: JWTToken,
installation_id: int,
installations_token_map_name: Path,
installation_token: None | tuple[str, datetime] = None,
) -> None:
self.jwt_token = jwt_token
self.installation_id = installation_id
self.installations_token_map_name = installations_token_map_name
if installation_token is None:
self.token, self.expiration = InstallationToken._generate_token(
self.jwt_token, self.installation_id
)
self._save()
else:
self.token, self.expiration = installation_token
def get(self) -> str:
self.verify()
return self.token
def get_as_secret(self) -> str:
return f"%(secret:github-token-{self.installation_id})"
def verify(self) -> None:
if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8:
self.token, self.expiration = InstallationToken._generate_token(
self.jwt_token, self.installation_id
)
self._save()
def _save(self) -> None:
# of format:
# {
# 123: {
# expiration: <datetime>,
# token: "token"
# }
# }
installations_token_map: dict[str, Any]
if self.installations_token_map_name.exists():
installations_token_map = json.loads(
self.installations_token_map_name.read_text()
)
else:
installations_token_map = {}
installations_token_map.update(
{
str(self.installation_id): {
"expiration": self.expiration.isoformat(),
"token": self.token,
}
}
)
atomic_write_file(
self.installations_token_map_name, json.dumps(installations_token_map)
)

View file

@ -0,0 +1,84 @@
import base64
import json
import os
import subprocess
from datetime import UTC, datetime, timedelta
from typing import Any
from .repo_token import RepoToken
class JWTToken(RepoToken):
app_id: int
app_private_key: str
lifetime: timedelta
expiration: datetime
token: str
def __init__(
self,
app_id: int,
app_private_key: str,
lifetime: timedelta = timedelta(minutes=10),
) -> None:
self.app_id = app_id
self.app_private_key = app_private_key
self.lifetime = lifetime
self.token, self.expiration = JWTToken.generate_token(
self.app_id, self.app_private_key, lifetime
)
@staticmethod
def generate_token(
app_id: int, app_private_key: str, lifetime: timedelta
) -> tuple[str, datetime]:
def build_jwt_payload(
app_id: int, lifetime: timedelta
) -> tuple[dict[str, Any], datetime]:
jwt_iat_drift: timedelta = timedelta(seconds=60)
now: datetime = datetime.now(tz=UTC)
iat: datetime = now - jwt_iat_drift
exp: datetime = iat + lifetime
jwt_payload = {
"iat": int(iat.timestamp()),
"exp": int(exp.timestamp()),
"iss": str(app_id),
}
return (jwt_payload, exp)
def rs256_sign(data: str, private_key: str) -> str:
signature = subprocess.run(
["openssl", "dgst", "-binary", "-sha256", "-sign", private_key],
input=data.encode("utf-8"),
stdout=subprocess.PIPE,
check=True,
cwd=os.environ.get("CREDENTIALS_DIRECTORY"),
).stdout
return base64url(signature)
def base64url(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8")
jwt, expiration = build_jwt_payload(app_id, lifetime)
jwt_payload = json.dumps(jwt).encode("utf-8")
json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8")
encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}"
encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key)
return (f"{encoded_jwt_parts}.{encoded_mac}", expiration)
# installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt)
# return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations))
def get(self) -> str:
if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8:
self.token, self.expiration = JWTToken.generate_token(
self.app_id, self.app_private_key, self.lifetime
)
return self.token
def get_as_secret(self) -> str:
return "%(secret:github-jwt-token)"

View file

@ -0,0 +1,14 @@
from .repo_token import RepoToken
class LegacyToken(RepoToken):
token: str
def __init__(self, token: str) -> None:
self.token = token
def get(self) -> str:
return self.token
def get_as_secret(self) -> str:
return "%(secret:github-token)"

View file

@ -0,0 +1,11 @@
from abc import abstractmethod
class RepoToken:
@abstractmethod
def get(self) -> str:
pass
@abstractmethod
def get_as_secret(self) -> str:
pass

View file

@ -1,10 +1,13 @@
import json import json
import os import os
import signal import signal
from collections.abc import Generator from abc import ABC, abstractmethod
from collections.abc import Callable, Generator
from dataclasses import dataclass from dataclasses import dataclass
from datetime import (
datetime,
)
from pathlib import Path from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from buildbot.config.builder import BuilderConfig from buildbot.config.builder import BuilderConfig
@ -13,10 +16,12 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase from buildbot.reporters.base import ReporterBase
from buildbot.reporters.github import GitHubStatusPush from buildbot.reporters.github import GitHubStatusPush
from buildbot.secrets.providers.base import SecretProviderBase
from buildbot.www.auth import AuthBase from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.oauth2 import GitHubAuth from buildbot.www.oauth2 import GitHubAuth
from twisted.internet import defer, threads from twisted.internet import defer, threads
from twisted.logger import Logger
from twisted.python import log from twisted.python import log
from twisted.python.failure import Failure from twisted.python.failure import Failure
@ -24,24 +29,96 @@ if TYPE_CHECKING:
from buildbot.process.log import StreamLog from buildbot.process.log import StreamLog
from .common import ( from .common import (
atomic_write_file,
http_request, http_request,
paginated_github_request, paginated_github_request,
slugify_project_name, slugify_project_name,
) )
from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy
from .github.installation_token import InstallationToken
from .github.jwt_token import JWTToken
from .github.legacy_token import (
LegacyToken,
)
from .github.repo_token import (
RepoToken,
)
from .projects import GitBackend, GitProject from .projects import GitBackend, GitProject
from .secrets import read_secret_file from .secrets import read_secret_file
tlog = Logger()
class ReloadGithubProjects(BuildStep):
def get_installations(jwt_token: JWTToken) -> list[int]:
installations = paginated_github_request(
"https://api.github.com/app/installations?per_page=100", jwt_token.get()
)
return [installation["id"] for installation in installations]
class ReloadGithubInstallations(BuildStep):
name = "reload_github_projects" name = "reload_github_projects"
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: jwt_token: JWTToken
self.token = token project_cache_file: Path
installation_token_map_name: Path
project_id_map_name: Path
def __init__(
self,
jwt_token: JWTToken,
project_cache_file: Path,
installation_token_map_name: Path,
project_id_map_name: Path,
**kwargs: Any,
) -> None:
self.jwt_token = jwt_token
self.installation_token_map_name = installation_token_map_name
self.project_id_map_name = project_id_map_name
self.project_cache_file = project_cache_file self.project_cache_file = project_cache_file
super().__init__(**kwargs) super().__init__(**kwargs)
def reload_projects(self) -> None: def reload_projects(self) -> None:
refresh_projects(self.token, self.project_cache_file) installation_token_map = GithubBackend.create_missing_installations(
self.jwt_token,
self.installation_token_map_name,
GithubBackend.load_installations(
self.jwt_token,
self.installation_token_map_name,
),
get_installations(self.jwt_token),
)
repos: list[Any] = []
project_id_map: dict[str, int] = {}
repos = []
for k, v in installation_token_map.items():
new_repos = refresh_projects(
v.get(),
self.project_cache_file,
clear=True,
api_endpoint="/installation/repositories",
subkey="repositories",
require_admin=False,
)
for repo in new_repos:
repo["installation_id"] = k
repos.extend(new_repos)
for repo in new_repos:
project_id_map[repo["full_name"]] = k
atomic_write_file(self.project_cache_file, json.dumps(repos))
atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
tlog.info(
f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token."
)
@defer.inlineCallbacks @defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]: def run(self) -> Generator[Any, object, Any]:
@ -65,36 +142,378 @@ class ReloadGithubProjects(BuildStep):
return util.FAILURE return util.FAILURE
class ReloadGithubProjects(BuildStep):
name = "reload_github_projects"
def __init__(
self, token: RepoToken, project_cache_file: Path, **kwargs: Any
) -> None:
self.token = token
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
def reload_projects(self) -> None:
repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file)
log.msg(repos)
atomic_write_file(self.project_cache_file, json.dumps(repos))
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
class GitHubAppStatusPush(GitHubStatusPush):
token_source: Callable[[int], RepoToken]
project_id_source: Callable[[str], int]
saved_args: dict[str, Any]
saved_kwargs: dict[str, Any]
def checkConfig(
self,
token_source: Callable[[int], RepoToken],
project_id_source: Callable[[str], int],
context: Any = None,
baseURL: Any = None,
verbose: Any = False,
debug: Any = None,
verify: Any = None,
generators: Any = None,
**kwargs: dict[str, Any],
) -> Any:
if generators is None:
generators = self._create_default_generators()
if "token" in kwargs:
del kwargs["token"]
super().checkConfig(
token="",
context=context,
baseURL=baseURL,
verbose=verbose,
debug=debug,
verify=verify,
generators=generators,
**kwargs,
)
def reconfigService(
self,
token_source: Callable[[int], RepoToken],
project_id_source: Callable[[str], int],
context: Any = None,
baseURL: Any = None,
verbose: Any = False,
debug: Any = None,
verify: Any = None,
generators: Any = None,
**kwargs: dict[str, Any],
) -> Any:
if "saved_args" not in self or self.saved_args is None:
self.saved_args = {}
self.token_source = token_source
self.project_id_source = project_id_source
self.saved_kwargs = kwargs
self.saved_args["context"] = context
self.saved_args["baseURL"] = baseURL
self.saved_args["verbose"] = verbose
self.saved_args["debug"] = debug
self.saved_args["verify"] = verify
self.saved_args["generators"] = generators
if generators is None:
generators = self._create_default_generators()
if "token" in kwargs:
del kwargs["token"]
super().reconfigService(
token="",
context=context,
baseURL=baseURL,
verbose=verbose,
debug=debug,
verify=verify,
generators=generators,
**kwargs,
)
def sendMessage(self, reports: Any) -> Any:
build = reports[0]["builds"][0]
sourcestamps = build["buildset"].get("sourcestamps")
if not sourcestamps:
return None
for sourcestamp in sourcestamps:
build["buildset"]["sourcestamps"] = [sourcestamp]
token: str
if "project" in sourcestamp and sourcestamp["project"] != "":
token = self.token_source(
self.project_id_source(sourcestamp["project"])
).get()
else:
token = ""
super().reconfigService(
token,
context=self.saved_args["context"],
baseURL=self.saved_args["baseURL"],
verbose=self.saved_args["verbose"],
debug=self.saved_args["debug"],
verify=self.saved_args["verify"],
generators=self.saved_args["generators"],
**self.saved_kwargs,
)
return super().sendMessage(reports)
return None
class GithubAuthBackend(ABC):
@abstractmethod
def get_general_token(self) -> RepoToken:
pass
@abstractmethod
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
pass
@abstractmethod
def create_secret_providers(self) -> list[SecretProviderBase]:
pass
@abstractmethod
def create_reporter(self) -> ReporterBase:
pass
@abstractmethod
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
pass
class GithubLegacyAuthBackend(GithubAuthBackend):
auth_type: AuthTypeLegacy
token: LegacyToken
def __init__(self, auth_type: AuthTypeLegacy) -> None:
self.auth_type = auth_type
self.token = LegacyToken(read_secret_file(auth_type.token_secret_name))
def get_general_token(self) -> RepoToken:
return self.token
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
return self.token
def create_secret_providers(self) -> list[SecretProviderBase]:
return [GitHubLegacySecretService(self.token)]
def create_reporter(self) -> ReporterBase:
return GitHubStatusPush(
token=self.token.get(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
return ReloadGithubProjects(
token=self.token, project_cache_file=project_cache_file
)
class GitHubLegacySecretService(SecretProviderBase):
name = "GitHubLegacySecretService"
token: LegacyToken
def reconfigService(self, token: LegacyToken) -> None:
self.token = token
def get(self, entry: str) -> str | None:
"""
get the value from the file identified by 'entry'
"""
if entry.startswith("github-token"):
return self.token.get()
return None
class GithubAppAuthBackend(GithubAuthBackend):
auth_type: AuthTypeApp
jwt_token: JWTToken
installation_tokens: dict[int, InstallationToken]
project_id_map: dict[str, int]
def __init__(self, auth_type: AuthTypeApp) -> None:
self.auth_type = auth_type
self.jwt_token = JWTToken(
self.auth_type.app_id, self.auth_type.app_secret_key_name
)
self.installation_tokens = GithubBackend.load_installations(
self.jwt_token,
self.auth_type.app_installation_token_map_name,
)
if self.auth_type.app_project_id_map_name.exists():
self.project_id_map = json.loads(
self.auth_type.app_project_id_map_name.read_text()
)
else:
tlog.info(
"~project-id-map~ is not present, GitHub project reload will follow."
)
self.project_id_map = {}
def get_general_token(self) -> RepoToken:
return self.jwt_token
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
assert "installation_id" in repo, f"Missing installation_id in {repo}"
return self.installation_tokens[repo["installation_id"]]
def create_secret_providers(self) -> list[SecretProviderBase]:
return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)]
def create_reporter(self) -> ReporterBase:
return GitHubAppStatusPush(
token_source=lambda iid: self.installation_tokens[iid],
project_id_source=lambda project: self.project_id_map[project],
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
return ReloadGithubInstallations(
self.jwt_token,
project_cache_file,
self.auth_type.app_installation_token_map_name,
self.auth_type.app_project_id_map_name,
)
class GitHubAppSecretService(SecretProviderBase):
name = "GitHubAppSecretService"
installation_tokens: dict[int, InstallationToken]
jwt_token: JWTToken
def reconfigService(
self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken
) -> None:
self.installation_tokens = installation_tokens
self.jwt_token = jwt_token
def get(self, entry: str) -> str | None:
"""
get the value from the file identified by 'entry'
"""
if entry.startswith("github-token-"):
return self.installation_tokens[
int(entry.removeprefix("github-token-"))
].get()
if entry == "github-jwt-token":
return self.jwt_token.get()
return None
@dataclass @dataclass
class GithubConfig: class GithubConfig:
oauth_id: str | None oauth_id: str | None
auth_type: AuthType
# TODO unused # TODO unused
buildbot_user: str buildbot_user: str
oauth_secret_name: str = "github-oauth-secret" oauth_secret_name: str = "github-oauth-secret"
token_secret_name: str = "github-token"
webhook_secret_name: str = "github-webhook-secret" webhook_secret_name: str = "github-webhook-secret"
project_cache_file: Path = Path("github-project-cache.json") project_cache_file: Path = Path("github-project-cache-v1.json")
topic: str | None = "build-with-buildbot" topic: str | None = "build-with-buildbot"
def token(self) -> str:
return read_secret_file(self.token_secret_name)
@dataclass @dataclass
class GithubBackend(GitBackend): class GithubBackend(GitBackend):
config: GithubConfig config: GithubConfig
webhook_secret: str webhook_secret: str
auth_backend: GithubAuthBackend
def __init__(self, config: GithubConfig) -> None: def __init__(self, config: GithubConfig) -> None:
self.config = config self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name) self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
if isinstance(self.config.auth_type, AuthTypeLegacy):
self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
elif isinstance(self.config.auth_type, AuthTypeApp):
self.auth_backend = GithubAppAuthBackend(self.config.auth_type)
@staticmethod
def load_installations(
jwt_token: JWTToken, installations_token_map_name: Path
) -> dict[int, InstallationToken]:
initial_installations_map: dict[str, Any]
if installations_token_map_name.exists():
initial_installations_map = json.loads(
installations_token_map_name.read_text()
)
else:
initial_installations_map = {}
installations_map: dict[int, InstallationToken] = {}
for iid, installation in initial_installations_map.items():
token: str = installation["token"]
expiration: datetime = datetime.fromisoformat(installation["expiration"])
installations_map[int(iid)] = InstallationToken(
jwt_token,
int(iid),
installations_token_map_name,
installation_token=(token, expiration),
)
return installations_map
@staticmethod
def create_missing_installations(
jwt_token: JWTToken,
installations_token_map_name: Path,
installations_map: dict[int, InstallationToken],
installations: list[int],
) -> dict[int, InstallationToken]:
for installation in set(installations) - installations_map.keys():
installations_map[installation] = InstallationToken(
jwt_token,
installation,
installations_token_map_name,
)
return installations_map
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it.""" """Updates the flake an opens a PR for it."""
factory = util.BuildFactory() factory = util.BuildFactory()
factory.addStep( factory.addStep(
ReloadGithubProjects(self.config.token(), self.config.project_cache_file), self.auth_backend.create_reload_builder_step(self.config.project_cache_file)
) )
return util.BuilderConfig( return util.BuilderConfig(
name=self.reload_builder_name, name=self.reload_builder_name,
@ -103,24 +522,33 @@ class GithubBackend(GitBackend):
) )
def create_reporter(self) -> ReporterBase: def create_reporter(self) -> ReporterBase:
return GitHubStatusPush( return self.auth_backend.create_reporter()
token=self.config.token(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_change_hook(self) -> dict[str, Any]: def create_change_hook(self) -> dict[str, Any]:
return { return {
"secret": self.webhook_secret, "secret": self.webhook_secret,
"strict": True, "strict": True,
"token": self.config.token(), "token": self.auth_backend.get_general_token().get(),
"github_property_whitelist": ["github.base.sha", "github.head.sha"], "github_property_whitelist": ["github.base.sha", "github.head.sha"],
} }
def create_avatar_method(self) -> AvatarBase | None: def create_avatar_method(self) -> AvatarBase | None:
return AvatarGitHub(token=self.config.token()) avatar = AvatarGitHub(token=self.auth_backend.get_general_token().get())
# TODO: not a proper fix, the /users/{username} endpoint is per installation, but I'm not sure
# how to tell which installation token to use, unless there is a way to build a huge map of
# username -> token, or we just try each one in order
def _get_avatar_by_username(self: Any, username: Any) -> Any:
return f"https://github.com/{username}.png"
import types
avatar._get_avatar_by_username = types.MethodType( # noqa: SLF001
_get_avatar_by_username,
avatar,
)
return avatar
def create_auth(self) -> AuthBase: def create_auth(self) -> AuthBase:
assert self.config.oauth_id is not None, "GitHub OAuth ID is required" assert self.config.oauth_id is not None, "GitHub OAuth ID is required"
@ -130,6 +558,9 @@ class GithubBackend(GitBackend):
apiVersion=4, apiVersion=4,
) )
def create_secret_providers(self) -> list[SecretProviderBase]:
return self.auth_backend.create_secret_providers()
def load_projects(self) -> list["GitProject"]: def load_projects(self) -> list["GitProject"]:
if not self.config.project_cache_file.exists(): if not self.config.project_cache_file.exists():
return [] return []
@ -138,19 +569,53 @@ class GithubBackend(GitBackend):
json.loads(self.config.project_cache_file.read_text()), json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"], key=lambda x: x["full_name"],
) )
if isinstance(self.auth_backend, GithubAppAuthBackend):
dropped_repos = list(
filter(lambda repo: not "installation_id" in repo, repos)
)
if dropped_repos:
tlog.info(
"Dropped projects follow, refresh will follow after initialisation:"
)
for dropped_repo in dropped_repos:
tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
repos = list(filter(lambda repo: "installation_id" in repo, repos))
tlog.info(f"Loading {len(repos)} cached repositories.")
return list( return list(
filter( filter(
lambda project: self.config.topic is not None lambda project: self.config.topic is not None
and self.config.topic in project.topics, and self.config.topic in project.topics,
[ [
GithubProject(self.config, self.webhook_secret, repo) GithubProject(
self.auth_backend.get_repo_token(repo),
self.config,
self.webhook_secret,
repo,
)
for repo in repos for repo in repos
], ],
) )
) )
def are_projects_cached(self) -> bool: def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists() if not self.config.project_cache_file.exists():
return False
if (
isinstance(self.config.auth_type, AuthTypeApp)
and not self.config.auth_type.app_project_id_map_name.exists()
):
return False
all_have_installation_id = True
for project in json.loads(self.config.project_cache_file.read_text()):
if not "installation_id" in project:
all_have_installation_id = False
break
return all_have_installation_id
@property @property
def type(self) -> str: def type(self) -> str:
@ -171,10 +636,18 @@ class GithubBackend(GitBackend):
class GithubProject(GitProject): class GithubProject(GitProject):
config: GithubConfig config: GithubConfig
webhook_secret: str
data: dict[str, Any]
token: RepoToken
def __init__( def __init__(
self, config: GithubConfig, webhook_secret: str, data: dict[str, Any] self,
token: RepoToken,
config: GithubConfig,
webhook_secret: str,
data: dict[str, Any],
) -> None: ) -> None:
self.token = token
self.config = config self.config = config
self.webhook_secret = webhook_secret self.webhook_secret = webhook_secret
self.data = data self.data = data
@ -187,7 +660,7 @@ class GithubProject(GitProject):
) -> None: ) -> None:
hooks = paginated_github_request( hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
self.config.token(), self.token.get(),
) )
config = dict( config = dict(
url=webhook_url + "change_hook/github", url=webhook_url + "change_hook/github",
@ -199,7 +672,7 @@ class GithubProject(GitProject):
name="web", active=True, events=["push", "pull_request"], config=config name="web", active=True, events=["push", "pull_request"], config=config
) )
headers = { headers = {
"Authorization": f"Bearer {self.config.token()}", "Authorization": f"Bearer {self.token.get()}",
"Accept": "application/vnd.github+json", "Accept": "application/vnd.github+json",
"Content-Type": "application/json", "Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28", "X-GitHub-Api-Version": "2022-11-28",
@ -217,7 +690,7 @@ class GithubProject(GitProject):
) )
def get_project_url(self) -> str: def get_project_url(self) -> str:
return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}" return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}"
@property @property
def pretty_type(self) -> str: def pretty_type(self) -> str:
@ -260,14 +733,25 @@ class GithubProject(GitProject):
return self.data["owner"]["type"] == "Organization" return self.data["owner"]["type"] == "Organization"
def refresh_projects(github_token: str, repo_cache_file: Path) -> None: def refresh_projects(
github_token: str,
repo_cache_file: Path,
repos: list[Any] | None = None,
clear: bool = True,
api_endpoint: str = "/user/repos",
subkey: None | str = None,
require_admin: bool = True,
) -> list[Any]:
if repos is None:
repos = [] repos = []
for repo in paginated_github_request( for repo in paginated_github_request(
"https://api.github.com/user/repos?per_page=100", f"https://api.github.com{api_endpoint}?per_page=100",
github_token, github_token,
subkey=subkey,
): ):
if not repo["permissions"]["admin"]: # TODO actually check for this properly
if not repo["permissions"]["admin"] and require_admin:
name = repo["full_name"] name = repo["full_name"]
log.msg( log.msg(
f"skipping {name} because we do not have admin privileges, needed for hook management", f"skipping {name} because we do not have admin privileges, needed for hook management",
@ -275,12 +759,4 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
else: else:
repos.append(repo) repos.append(repo)
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: return repos
path = Path(f.name)
try:
f.write(json.dumps(repos))
f.flush()
path.rename(repo_cache_file)
except OSError:
path.unlink()
raise

View file

@ -3,6 +3,7 @@ from typing import Any
from buildbot.config.builder import BuilderConfig from buildbot.config.builder import BuilderConfig
from buildbot.reporters.base import ReporterBase from buildbot.reporters.base import ReporterBase
from buildbot.secrets.providers.base import SecretProviderBase
from buildbot.www.auth import AuthBase from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase from buildbot.www.avatar import AvatarBase
@ -28,6 +29,9 @@ class GitBackend(ABC):
def create_auth(self) -> AuthBase: def create_auth(self) -> AuthBase:
pass pass
def create_secret_providers(self) -> list[SecretProviderBase]:
return []
@abstractmethod @abstractmethod
def load_projects(self) -> list["GitProject"]: def load_projects(self) -> list["GitProject"]:
pass pass

View file

@ -21,8 +21,14 @@
github = { github = {
# Github user used as a CI identity # Github user used as a CI identity
user = "mic92-buildbot"; user = "mic92-buildbot";
authType.legacy = {
# Github token of the same user # Github token of the same user
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
};
# authType.app = {
# id = "00000000000000000"; # FIXME: replace with App ID obtained from GitHub
# secretKeyFile = pkgs.writeText "app-secret.key" "00000000000000000000"; # FIXME: replace with App secret key obtained from GitHub
# };
# A random secret used to verify incoming webhooks from GitHub # A random secret used to verify incoming webhooks from GitHub
# buildbot-nix will set up a webhook for each project in the organization # buildbot-nix will set up a webhook for each project in the organization
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store

View file

@ -16,7 +16,9 @@
''; '';
admins = [ "Mic92" ]; admins = [ "Mic92" ];
github = { github = {
authType.legacy = {
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
};
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff"; oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
oauthId = "aaaaaaaaaaaaaaaaaaaa"; oauthId = "aaaaaaaaaaaaaaaaaaaa";

View file

@ -24,6 +24,24 @@ in
"admins" "admins"
] ]
) )
(mkRenamedOptionModule
[
"services"
"buildbot-nix"
"master"
"github"
"tokenFile"
]
[
"services"
"buildbot-nix"
"master"
"github"
"authType"
"legacy"
"tokenFile"
]
)
]; ];
options = { options = {
@ -106,10 +124,39 @@ in
default = cfg.authBackend == "github"; default = cfg.authBackend == "github";
}; };
tokenFile = lib.mkOption { authType = lib.mkOption {
type = lib.types.attrTag {
legacy = lib.mkOption {
description = "GitHub legacy auth backend";
type = lib.types.submodule {
options.tokenFile = lib.mkOption {
type = lib.types.path; type = lib.types.path;
description = "Github token file"; description = "Github token file";
}; };
};
};
app = lib.mkOption {
description = "GitHub legacy auth backend";
type = lib.types.submodule {
options.id = lib.mkOption {
type = lib.types.int;
description = ''
GitHub app ID.
'';
};
options.secretKeyFile = lib.mkOption {
type = lib.types.str;
description = ''
GitHub app secret key file location.
'';
};
};
};
};
};
webhookSecretFile = lib.mkOption { webhookSecretFile = lib.mkOption {
type = lib.types.path; type = lib.types.path;
description = "Github webhook secret file"; description = "Github webhook secret file";
@ -243,7 +290,16 @@ in
home = "/var/lib/buildbot"; home = "/var/lib/buildbot";
extraImports = '' extraImports = ''
from datetime import timedelta from datetime import timedelta
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig from buildbot_nix import (
GithubConfig,
NixConfigurator,
CachixConfig,
GiteaConfig,
)
from buildbot_nix.github.auth._type import (
AuthTypeLegacy,
AuthTypeApp,
)
''; '';
configurators = [ configurators = [
'' ''
@ -260,6 +316,18 @@ in
oauth_id=${builtins.toJSON cfg.github.oauthId}, oauth_id=${builtins.toJSON cfg.github.oauthId},
buildbot_user=${builtins.toJSON cfg.github.user}, buildbot_user=${builtins.toJSON cfg.github.user},
topic=${builtins.toJSON cfg.github.topic}, topic=${builtins.toJSON cfg.github.topic},
auth_type=${
if cfg.github.authType ? "legacy" then
''AuthTypeLegacy()''
else if cfg.github.authType ? "app" then
''
AuthTypeApp(
app_id=${toString cfg.github.authType.app.id},
)
''
else
throw "One of AuthTypeApp or AuthTypeLegacy must be enabled"
}
)" )"
}, },
gitea=${ gitea=${
@ -323,6 +391,9 @@ in
systemd.services.buildbot-master = { systemd.services.buildbot-master = {
after = [ "postgresql.service" ]; after = [ "postgresql.service" ];
path = [
pkgs.openssl
];
serviceConfig = { serviceConfig = {
# in master.py we read secrets from $CREDENTIALS_DIRECTORY # in master.py we read secrets from $CREDENTIALS_DIRECTORY
LoadCredential = LoadCredential =
@ -337,10 +408,15 @@ in
( (
cfg.cachix.authTokenFile != null cfg.cachix.authTokenFile != null
) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" ) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
++ lib.optionals (cfg.github.enable) [ ++ lib.optionals (cfg.github.enable) ([
"github-token:${cfg.github.tokenFile}"
"github-webhook-secret:${cfg.github.webhookSecretFile}" "github-webhook-secret:${cfg.github.webhookSecretFile}"
] ]
++ lib.optionals (cfg.github.authType ? "legacy") [
"github-token:${cfg.github.authType.legacy.tokenFile}"
]
++ lib.optionals (cfg.github.authType ? "app") [
"github-app-secret-key:${cfg.github.authType.app.secretKeyFile}"
])
++ lib.optionals cfg.gitea.enable [ ++ lib.optionals cfg.gitea.enable [
"gitea-token:${cfg.gitea.tokenFile}" "gitea-token:${cfg.gitea.tokenFile}"
"gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"

View file

@ -26,6 +26,8 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" }
[tool.setuptools] [tool.setuptools]
packages = [ packages = [
"buildbot_nix", "buildbot_nix",
"buildbot_nix.github",
"buildbot_nix.github.auth",
"buildbot_effects" "buildbot_effects"
] ]
@ -82,6 +84,17 @@ ignore = [
# not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html # not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html
"G010", # Logging statement uses `warn` instead of `warning` "G010", # Logging statement uses `warn` instead of `warning`
# gives falls positives and isn't hard to check munually
"ERA001"
]
[tool.ruff.lint.pep8-naming]
ignore-names = [
"checkConfig",
"baseURL",
"reconfigService",
"sendMessage",
] ]
[tool.mypy] [tool.mypy]