Merge pull request #156 from MagicRB/github_app

Add GitHub App support
This commit is contained in:
Richard Brežák 2024-06-12 15:40:19 +00:00 committed by GitHub
commit 7e327b88b6
Failed to generate hash of commit
16 changed files with 923 additions and 61 deletions

View file

@ -66,16 +66,36 @@ We have the following two roles:
### Integration with GitHub
To integrate with GitHub:
#### GitHub App
To integrate with GitHub using app authentication:
1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
authentication on the Buildbot dashboard. Enable the following permissions:
- Contents: Read-only
- Metadata: Read-only
- Commit statuses: Read and write
- Webhooks: Read and write
2. **GitHub App private key**: Get the app private key and app ID from GitHub,
configure using the buildbot-nix NixOS module.
3. **Install App**: Install the app for an organization or specific user.
4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to
changes (new repositories or installations) automatically, it is therefore
necessary to manually trigger a reload or wait for the next periodic reload.
#### Legacy Token Auth
To integrate with GitHub using legacy token authentication:
1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo`
permissions. For GitHub organizations, it's advisable to create a separate
GitHub user for managing repository webhooks.
#### Optional when using GitHub login
### Optional when using GitHub login
1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
authentication on the Buildbot dashboard.
authentication on the Buildbot dashboard. (can be the same as for GitHub App
auth)
2. **OAuth Credentials**: After installing the app, generate OAuth credentials
and configure them in the buildbot-nix NixOS module. Set the callback url to
`https://<your-domain>/auth/login`.

View file

@ -872,6 +872,8 @@ class NixConfigurator(ConfiguratorBase):
],
)
config["services"].append(backend.create_reporter())
config.setdefault("secretProviders", [])
config["secretsProviders"].extend(backend.create_secret_providers())
systemd_secrets = SecretInAFile(
dirname=os.environ["CREDENTIALS_DIRECTORY"],
@ -887,13 +889,15 @@ class NixConfigurator(ConfiguratorBase):
backend.create_change_hook()
)
if "auth" not in config["www"]:
config["www"].setdefault("avatar_methods", [])
for backend in backends.values():
avatar_method = backend.create_avatar_method()
print(avatar_method)
if avatar_method is not None:
config["www"]["avatar_methods"].append(avatar_method)
if "auth" not in config["www"]:
# TODO one cannot have multiple auth backends...
if auth is not None:
config["www"]["auth"] = auth

View file

@ -2,6 +2,8 @@ import contextlib
import http.client
import json
import urllib.request
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
@ -9,7 +11,9 @@ def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
def paginated_github_request(
url: str, token: str, subkey: None | str = None
) -> list[dict[str, Any]]:
next_url: str | None = url
items = []
while next_url:
@ -29,6 +33,9 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1]
if subkey is not None:
items += res.json()[subkey]
else:
items += res.json()
return items
@ -78,3 +85,15 @@ def http_request(
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise HttpError(msg) from e
return HttpResponse(resp)
def atomic_write_file(file: Path, data: str) -> None:
with NamedTemporaryFile("w", delete=False, dir=file.parent) as f:
path = Path(f.name)
try:
f.write(data)
f.flush()
path.rename(file)
except OSError:
path.unlink()
raise

View file

@ -0,0 +1 @@

View file

@ -0,0 +1 @@

View file

@ -0,0 +1,23 @@
from dataclasses import dataclass
from pathlib import Path
@dataclass
class AuthType:
pass
@dataclass
class AuthTypeLegacy(AuthType):
token_secret_name: str = "github-token"
@dataclass
class AuthTypeApp(AuthType):
app_id: int
app_secret_key_name: str = "github-app-secret-key"
app_installation_token_map_name: Path = Path(
"github-app-installation-token-map.json"
)
app_project_id_map_name: Path = Path("github-app-project-id-map-name.json")
app_jwt_token_name: Path = Path("github-app-jwt-token")

View file

@ -0,0 +1,108 @@
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from buildbot_nix.common import (
HttpResponse,
atomic_write_file,
http_request,
)
from .jwt_token import JWTToken
from .repo_token import RepoToken
class InstallationToken(RepoToken):
GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
jwt_token: JWTToken
installation_id: int
token: str
expiration: datetime
installations_token_map_name: Path
@staticmethod
def _create_installation_access_token(
jwt_token: JWTToken, installation_id: int
) -> HttpResponse:
return http_request(
f"https://api.github.com/app/installations/{installation_id}/access_tokens",
data={},
headers={"Authorization": f"Bearer {jwt_token.get()}"},
method="POST",
)
@staticmethod
def _generate_token(
jwt_token: JWTToken, installation_id: int
) -> tuple[str, datetime]:
token = InstallationToken._create_installation_access_token(
jwt_token, installation_id
).json()["token"]
expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME
return token, expiration
def __init__(
self,
jwt_token: JWTToken,
installation_id: int,
installations_token_map_name: Path,
installation_token: None | tuple[str, datetime] = None,
) -> None:
self.jwt_token = jwt_token
self.installation_id = installation_id
self.installations_token_map_name = installations_token_map_name
if installation_token is None:
self.token, self.expiration = InstallationToken._generate_token(
self.jwt_token, self.installation_id
)
self._save()
else:
self.token, self.expiration = installation_token
def get(self) -> str:
self.verify()
return self.token
def get_as_secret(self) -> str:
return f"%(secret:github-token-{self.installation_id})"
def verify(self) -> None:
if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8:
self.token, self.expiration = InstallationToken._generate_token(
self.jwt_token, self.installation_id
)
self._save()
def _save(self) -> None:
# of format:
# {
# 123: {
# expiration: <datetime>,
# token: "token"
# }
# }
installations_token_map: dict[str, Any]
if self.installations_token_map_name.exists():
installations_token_map = json.loads(
self.installations_token_map_name.read_text()
)
else:
installations_token_map = {}
installations_token_map.update(
{
str(self.installation_id): {
"expiration": self.expiration.isoformat(),
"token": self.token,
}
}
)
atomic_write_file(
self.installations_token_map_name, json.dumps(installations_token_map)
)

View file

@ -0,0 +1,84 @@
import base64
import json
import os
import subprocess
from datetime import UTC, datetime, timedelta
from typing import Any
from .repo_token import RepoToken
class JWTToken(RepoToken):
app_id: int
app_private_key: str
lifetime: timedelta
expiration: datetime
token: str
def __init__(
self,
app_id: int,
app_private_key: str,
lifetime: timedelta = timedelta(minutes=10),
) -> None:
self.app_id = app_id
self.app_private_key = app_private_key
self.lifetime = lifetime
self.token, self.expiration = JWTToken.generate_token(
self.app_id, self.app_private_key, lifetime
)
@staticmethod
def generate_token(
app_id: int, app_private_key: str, lifetime: timedelta
) -> tuple[str, datetime]:
def build_jwt_payload(
app_id: int, lifetime: timedelta
) -> tuple[dict[str, Any], datetime]:
jwt_iat_drift: timedelta = timedelta(seconds=60)
now: datetime = datetime.now(tz=UTC)
iat: datetime = now - jwt_iat_drift
exp: datetime = iat + lifetime
jwt_payload = {
"iat": int(iat.timestamp()),
"exp": int(exp.timestamp()),
"iss": str(app_id),
}
return (jwt_payload, exp)
def rs256_sign(data: str, private_key: str) -> str:
signature = subprocess.run(
["openssl", "dgst", "-binary", "-sha256", "-sign", private_key],
input=data.encode("utf-8"),
stdout=subprocess.PIPE,
check=True,
cwd=os.environ.get("CREDENTIALS_DIRECTORY"),
).stdout
return base64url(signature)
def base64url(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8")
jwt, expiration = build_jwt_payload(app_id, lifetime)
jwt_payload = json.dumps(jwt).encode("utf-8")
json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8")
encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}"
encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key)
return (f"{encoded_jwt_parts}.{encoded_mac}", expiration)
# installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt)
# return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations))
def get(self) -> str:
if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8:
self.token, self.expiration = JWTToken.generate_token(
self.app_id, self.app_private_key, self.lifetime
)
return self.token
def get_as_secret(self) -> str:
return "%(secret:github-jwt-token)"

View file

@ -0,0 +1,14 @@
from .repo_token import RepoToken
class LegacyToken(RepoToken):
token: str
def __init__(self, token: str) -> None:
self.token = token
def get(self) -> str:
return self.token
def get_as_secret(self) -> str:
return "%(secret:github-token)"

View file

@ -0,0 +1,11 @@
from abc import abstractmethod
class RepoToken:
@abstractmethod
def get(self) -> str:
pass
@abstractmethod
def get_as_secret(self) -> str:
pass

View file

@ -1,10 +1,13 @@
import json
import os
import signal
from collections.abc import Generator
from abc import ABC, abstractmethod
from collections.abc import Callable, Generator
from dataclasses import dataclass
from datetime import (
datetime,
)
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any
from buildbot.config.builder import BuilderConfig
@ -13,10 +16,12 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase
from buildbot.reporters.github import GitHubStatusPush
from buildbot.secrets.providers.base import SecretProviderBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.oauth2 import GitHubAuth
from twisted.internet import defer, threads
from twisted.logger import Logger
from twisted.python import log
from twisted.python.failure import Failure
@ -24,24 +29,96 @@ if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from .common import (
atomic_write_file,
http_request,
paginated_github_request,
slugify_project_name,
)
from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy
from .github.installation_token import InstallationToken
from .github.jwt_token import JWTToken
from .github.legacy_token import (
LegacyToken,
)
from .github.repo_token import (
RepoToken,
)
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
tlog = Logger()
class ReloadGithubProjects(BuildStep):
def get_installations(jwt_token: JWTToken) -> list[int]:
installations = paginated_github_request(
"https://api.github.com/app/installations?per_page=100", jwt_token.get()
)
return [installation["id"] for installation in installations]
class ReloadGithubInstallations(BuildStep):
name = "reload_github_projects"
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
self.token = token
jwt_token: JWTToken
project_cache_file: Path
installation_token_map_name: Path
project_id_map_name: Path
def __init__(
self,
jwt_token: JWTToken,
project_cache_file: Path,
installation_token_map_name: Path,
project_id_map_name: Path,
**kwargs: Any,
) -> None:
self.jwt_token = jwt_token
self.installation_token_map_name = installation_token_map_name
self.project_id_map_name = project_id_map_name
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
def reload_projects(self) -> None:
refresh_projects(self.token, self.project_cache_file)
installation_token_map = GithubBackend.create_missing_installations(
self.jwt_token,
self.installation_token_map_name,
GithubBackend.load_installations(
self.jwt_token,
self.installation_token_map_name,
),
get_installations(self.jwt_token),
)
repos: list[Any] = []
project_id_map: dict[str, int] = {}
repos = []
for k, v in installation_token_map.items():
new_repos = refresh_projects(
v.get(),
self.project_cache_file,
clear=True,
api_endpoint="/installation/repositories",
subkey="repositories",
require_admin=False,
)
for repo in new_repos:
repo["installation_id"] = k
repos.extend(new_repos)
for repo in new_repos:
project_id_map[repo["full_name"]] = k
atomic_write_file(self.project_cache_file, json.dumps(repos))
atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
tlog.info(
f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token."
)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
@ -65,36 +142,378 @@ class ReloadGithubProjects(BuildStep):
return util.FAILURE
class ReloadGithubProjects(BuildStep):
name = "reload_github_projects"
def __init__(
self, token: RepoToken, project_cache_file: Path, **kwargs: Any
) -> None:
self.token = token
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
def reload_projects(self) -> None:
repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file)
log.msg(repos)
atomic_write_file(self.project_cache_file, json.dumps(repos))
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
class GitHubAppStatusPush(GitHubStatusPush):
token_source: Callable[[int], RepoToken]
project_id_source: Callable[[str], int]
saved_args: dict[str, Any]
saved_kwargs: dict[str, Any]
def checkConfig(
self,
token_source: Callable[[int], RepoToken],
project_id_source: Callable[[str], int],
context: Any = None,
baseURL: Any = None,
verbose: Any = False,
debug: Any = None,
verify: Any = None,
generators: Any = None,
**kwargs: dict[str, Any],
) -> Any:
if generators is None:
generators = self._create_default_generators()
if "token" in kwargs:
del kwargs["token"]
super().checkConfig(
token="",
context=context,
baseURL=baseURL,
verbose=verbose,
debug=debug,
verify=verify,
generators=generators,
**kwargs,
)
def reconfigService(
self,
token_source: Callable[[int], RepoToken],
project_id_source: Callable[[str], int],
context: Any = None,
baseURL: Any = None,
verbose: Any = False,
debug: Any = None,
verify: Any = None,
generators: Any = None,
**kwargs: dict[str, Any],
) -> Any:
if "saved_args" not in self or self.saved_args is None:
self.saved_args = {}
self.token_source = token_source
self.project_id_source = project_id_source
self.saved_kwargs = kwargs
self.saved_args["context"] = context
self.saved_args["baseURL"] = baseURL
self.saved_args["verbose"] = verbose
self.saved_args["debug"] = debug
self.saved_args["verify"] = verify
self.saved_args["generators"] = generators
if generators is None:
generators = self._create_default_generators()
if "token" in kwargs:
del kwargs["token"]
super().reconfigService(
token="",
context=context,
baseURL=baseURL,
verbose=verbose,
debug=debug,
verify=verify,
generators=generators,
**kwargs,
)
def sendMessage(self, reports: Any) -> Any:
build = reports[0]["builds"][0]
sourcestamps = build["buildset"].get("sourcestamps")
if not sourcestamps:
return None
for sourcestamp in sourcestamps:
build["buildset"]["sourcestamps"] = [sourcestamp]
token: str
if "project" in sourcestamp and sourcestamp["project"] != "":
token = self.token_source(
self.project_id_source(sourcestamp["project"])
).get()
else:
token = ""
super().reconfigService(
token,
context=self.saved_args["context"],
baseURL=self.saved_args["baseURL"],
verbose=self.saved_args["verbose"],
debug=self.saved_args["debug"],
verify=self.saved_args["verify"],
generators=self.saved_args["generators"],
**self.saved_kwargs,
)
return super().sendMessage(reports)
return None
class GithubAuthBackend(ABC):
@abstractmethod
def get_general_token(self) -> RepoToken:
pass
@abstractmethod
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
pass
@abstractmethod
def create_secret_providers(self) -> list[SecretProviderBase]:
pass
@abstractmethod
def create_reporter(self) -> ReporterBase:
pass
@abstractmethod
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
pass
class GithubLegacyAuthBackend(GithubAuthBackend):
auth_type: AuthTypeLegacy
token: LegacyToken
def __init__(self, auth_type: AuthTypeLegacy) -> None:
self.auth_type = auth_type
self.token = LegacyToken(read_secret_file(auth_type.token_secret_name))
def get_general_token(self) -> RepoToken:
return self.token
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
return self.token
def create_secret_providers(self) -> list[SecretProviderBase]:
return [GitHubLegacySecretService(self.token)]
def create_reporter(self) -> ReporterBase:
return GitHubStatusPush(
token=self.token.get(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
return ReloadGithubProjects(
token=self.token, project_cache_file=project_cache_file
)
class GitHubLegacySecretService(SecretProviderBase):
name = "GitHubLegacySecretService"
token: LegacyToken
def reconfigService(self, token: LegacyToken) -> None:
self.token = token
def get(self, entry: str) -> str | None:
"""
get the value from the file identified by 'entry'
"""
if entry.startswith("github-token"):
return self.token.get()
return None
class GithubAppAuthBackend(GithubAuthBackend):
auth_type: AuthTypeApp
jwt_token: JWTToken
installation_tokens: dict[int, InstallationToken]
project_id_map: dict[str, int]
def __init__(self, auth_type: AuthTypeApp) -> None:
self.auth_type = auth_type
self.jwt_token = JWTToken(
self.auth_type.app_id, self.auth_type.app_secret_key_name
)
self.installation_tokens = GithubBackend.load_installations(
self.jwt_token,
self.auth_type.app_installation_token_map_name,
)
if self.auth_type.app_project_id_map_name.exists():
self.project_id_map = json.loads(
self.auth_type.app_project_id_map_name.read_text()
)
else:
tlog.info(
"~project-id-map~ is not present, GitHub project reload will follow."
)
self.project_id_map = {}
def get_general_token(self) -> RepoToken:
return self.jwt_token
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
assert "installation_id" in repo, f"Missing installation_id in {repo}"
return self.installation_tokens[repo["installation_id"]]
def create_secret_providers(self) -> list[SecretProviderBase]:
return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)]
def create_reporter(self) -> ReporterBase:
return GitHubAppStatusPush(
token_source=lambda iid: self.installation_tokens[iid],
project_id_source=lambda project: self.project_id_map[project],
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
return ReloadGithubInstallations(
self.jwt_token,
project_cache_file,
self.auth_type.app_installation_token_map_name,
self.auth_type.app_project_id_map_name,
)
class GitHubAppSecretService(SecretProviderBase):
name = "GitHubAppSecretService"
installation_tokens: dict[int, InstallationToken]
jwt_token: JWTToken
def reconfigService(
self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken
) -> None:
self.installation_tokens = installation_tokens
self.jwt_token = jwt_token
def get(self, entry: str) -> str | None:
"""
get the value from the file identified by 'entry'
"""
if entry.startswith("github-token-"):
return self.installation_tokens[
int(entry.removeprefix("github-token-"))
].get()
if entry == "github-jwt-token":
return self.jwt_token.get()
return None
@dataclass
class GithubConfig:
oauth_id: str | None
auth_type: AuthType
# TODO unused
buildbot_user: str
oauth_secret_name: str = "github-oauth-secret"
token_secret_name: str = "github-token"
webhook_secret_name: str = "github-webhook-secret"
project_cache_file: Path = Path("github-project-cache.json")
project_cache_file: Path = Path("github-project-cache-v1.json")
topic: str | None = "build-with-buildbot"
def token(self) -> str:
return read_secret_file(self.token_secret_name)
@dataclass
class GithubBackend(GitBackend):
config: GithubConfig
webhook_secret: str
auth_backend: GithubAuthBackend
def __init__(self, config: GithubConfig) -> None:
self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
if isinstance(self.config.auth_type, AuthTypeLegacy):
self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
elif isinstance(self.config.auth_type, AuthTypeApp):
self.auth_backend = GithubAppAuthBackend(self.config.auth_type)
@staticmethod
def load_installations(
jwt_token: JWTToken, installations_token_map_name: Path
) -> dict[int, InstallationToken]:
initial_installations_map: dict[str, Any]
if installations_token_map_name.exists():
initial_installations_map = json.loads(
installations_token_map_name.read_text()
)
else:
initial_installations_map = {}
installations_map: dict[int, InstallationToken] = {}
for iid, installation in initial_installations_map.items():
token: str = installation["token"]
expiration: datetime = datetime.fromisoformat(installation["expiration"])
installations_map[int(iid)] = InstallationToken(
jwt_token,
int(iid),
installations_token_map_name,
installation_token=(token, expiration),
)
return installations_map
@staticmethod
def create_missing_installations(
jwt_token: JWTToken,
installations_token_map_name: Path,
installations_map: dict[int, InstallationToken],
installations: list[int],
) -> dict[int, InstallationToken]:
for installation in set(installations) - installations_map.keys():
installations_map[installation] = InstallationToken(
jwt_token,
installation,
installations_token_map_name,
)
return installations_map
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it."""
factory = util.BuildFactory()
factory.addStep(
ReloadGithubProjects(self.config.token(), self.config.project_cache_file),
self.auth_backend.create_reload_builder_step(self.config.project_cache_file)
)
return util.BuilderConfig(
name=self.reload_builder_name,
@ -103,24 +522,33 @@ class GithubBackend(GitBackend):
)
def create_reporter(self) -> ReporterBase:
return GitHubStatusPush(
token=self.config.token(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
return self.auth_backend.create_reporter()
def create_change_hook(self) -> dict[str, Any]:
return {
"secret": self.webhook_secret,
"strict": True,
"token": self.config.token(),
"token": self.auth_backend.get_general_token().get(),
"github_property_whitelist": ["github.base.sha", "github.head.sha"],
}
def create_avatar_method(self) -> AvatarBase | None:
return AvatarGitHub(token=self.config.token())
avatar = AvatarGitHub(token=self.auth_backend.get_general_token().get())
# TODO: not a proper fix, the /users/{username} endpoint is per installation, but I'm not sure
# how to tell which installation token to use, unless there is a way to build a huge map of
# username -> token, or we just try each one in order
def _get_avatar_by_username(self: Any, username: Any) -> Any:
return f"https://github.com/{username}.png"
import types
avatar._get_avatar_by_username = types.MethodType( # noqa: SLF001
_get_avatar_by_username,
avatar,
)
return avatar
def create_auth(self) -> AuthBase:
assert self.config.oauth_id is not None, "GitHub OAuth ID is required"
@ -130,6 +558,9 @@ class GithubBackend(GitBackend):
apiVersion=4,
)
def create_secret_providers(self) -> list[SecretProviderBase]:
return self.auth_backend.create_secret_providers()
def load_projects(self) -> list["GitProject"]:
if not self.config.project_cache_file.exists():
return []
@ -138,19 +569,53 @@ class GithubBackend(GitBackend):
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
)
if isinstance(self.auth_backend, GithubAppAuthBackend):
dropped_repos = list(
filter(lambda repo: not "installation_id" in repo, repos)
)
if dropped_repos:
tlog.info(
"Dropped projects follow, refresh will follow after initialisation:"
)
for dropped_repo in dropped_repos:
tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
repos = list(filter(lambda repo: "installation_id" in repo, repos))
tlog.info(f"Loading {len(repos)} cached repositories.")
return list(
filter(
lambda project: self.config.topic is not None
and self.config.topic in project.topics,
[
GithubProject(self.config, self.webhook_secret, repo)
GithubProject(
self.auth_backend.get_repo_token(repo),
self.config,
self.webhook_secret,
repo,
)
for repo in repos
],
)
)
def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists()
if not self.config.project_cache_file.exists():
return False
if (
isinstance(self.config.auth_type, AuthTypeApp)
and not self.config.auth_type.app_project_id_map_name.exists()
):
return False
all_have_installation_id = True
for project in json.loads(self.config.project_cache_file.read_text()):
if not "installation_id" in project:
all_have_installation_id = False
break
return all_have_installation_id
@property
def type(self) -> str:
@ -171,10 +636,18 @@ class GithubBackend(GitBackend):
class GithubProject(GitProject):
config: GithubConfig
webhook_secret: str
data: dict[str, Any]
token: RepoToken
def __init__(
self, config: GithubConfig, webhook_secret: str, data: dict[str, Any]
self,
token: RepoToken,
config: GithubConfig,
webhook_secret: str,
data: dict[str, Any],
) -> None:
self.token = token
self.config = config
self.webhook_secret = webhook_secret
self.data = data
@ -187,7 +660,7 @@ class GithubProject(GitProject):
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
self.config.token(),
self.token.get(),
)
config = dict(
url=webhook_url + "change_hook/github",
@ -199,7 +672,7 @@ class GithubProject(GitProject):
name="web", active=True, events=["push", "pull_request"], config=config
)
headers = {
"Authorization": f"Bearer {self.config.token()}",
"Authorization": f"Bearer {self.token.get()}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
@ -217,7 +690,7 @@ class GithubProject(GitProject):
)
def get_project_url(self) -> str:
return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}"
return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}"
@property
def pretty_type(self) -> str:
@ -260,14 +733,25 @@ class GithubProject(GitProject):
return self.data["owner"]["type"] == "Organization"
def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
def refresh_projects(
github_token: str,
repo_cache_file: Path,
repos: list[Any] | None = None,
clear: bool = True,
api_endpoint: str = "/user/repos",
subkey: None | str = None,
require_admin: bool = True,
) -> list[Any]:
if repos is None:
repos = []
for repo in paginated_github_request(
"https://api.github.com/user/repos?per_page=100",
f"https://api.github.com{api_endpoint}?per_page=100",
github_token,
subkey=subkey,
):
if not repo["permissions"]["admin"]:
# TODO actually check for this properly
if not repo["permissions"]["admin"] and require_admin:
name = repo["full_name"]
log.msg(
f"skipping {name} because we do not have admin privileges, needed for hook management",
@ -275,12 +759,4 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
else:
repos.append(repo)
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
path = Path(f.name)
try:
f.write(json.dumps(repos))
f.flush()
path.rename(repo_cache_file)
except OSError:
path.unlink()
raise
return repos

View file

@ -3,6 +3,7 @@ from typing import Any
from buildbot.config.builder import BuilderConfig
from buildbot.reporters.base import ReporterBase
from buildbot.secrets.providers.base import SecretProviderBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase
@ -28,6 +29,9 @@ class GitBackend(ABC):
def create_auth(self) -> AuthBase:
pass
def create_secret_providers(self) -> list[SecretProviderBase]:
return []
@abstractmethod
def load_projects(self) -> list["GitProject"]:
pass

View file

@ -21,8 +21,14 @@
github = {
# Github user used as a CI identity
user = "mic92-buildbot";
authType.legacy = {
# Github token of the same user
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
};
# authType.app = {
# id = "00000000000000000"; # FIXME: replace with App ID obtained from GitHub
# secretKeyFile = pkgs.writeText "app-secret.key" "00000000000000000000"; # FIXME: replace with App secret key obtained from GitHub
# };
# A random secret used to verify incoming webhooks from GitHub
# buildbot-nix will set up a webhook for each project in the organization
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store

View file

@ -16,7 +16,9 @@
'';
admins = [ "Mic92" ];
github = {
authType.legacy = {
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
};
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
oauthId = "aaaaaaaaaaaaaaaaaaaa";

View file

@ -24,6 +24,24 @@ in
"admins"
]
)
(mkRenamedOptionModule
[
"services"
"buildbot-nix"
"master"
"github"
"tokenFile"
]
[
"services"
"buildbot-nix"
"master"
"github"
"authType"
"legacy"
"tokenFile"
]
)
];
options = {
@ -106,10 +124,39 @@ in
default = cfg.authBackend == "github";
};
tokenFile = lib.mkOption {
authType = lib.mkOption {
type = lib.types.attrTag {
legacy = lib.mkOption {
description = "GitHub legacy auth backend";
type = lib.types.submodule {
options.tokenFile = lib.mkOption {
type = lib.types.path;
description = "Github token file";
};
};
};
app = lib.mkOption {
description = "GitHub legacy auth backend";
type = lib.types.submodule {
options.id = lib.mkOption {
type = lib.types.int;
description = ''
GitHub app ID.
'';
};
options.secretKeyFile = lib.mkOption {
type = lib.types.str;
description = ''
GitHub app secret key file location.
'';
};
};
};
};
};
webhookSecretFile = lib.mkOption {
type = lib.types.path;
description = "Github webhook secret file";
@ -243,7 +290,16 @@ in
home = "/var/lib/buildbot";
extraImports = ''
from datetime import timedelta
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig
from buildbot_nix import (
GithubConfig,
NixConfigurator,
CachixConfig,
GiteaConfig,
)
from buildbot_nix.github.auth._type import (
AuthTypeLegacy,
AuthTypeApp,
)
'';
configurators = [
''
@ -260,6 +316,18 @@ in
oauth_id=${builtins.toJSON cfg.github.oauthId},
buildbot_user=${builtins.toJSON cfg.github.user},
topic=${builtins.toJSON cfg.github.topic},
auth_type=${
if cfg.github.authType ? "legacy" then
''AuthTypeLegacy()''
else if cfg.github.authType ? "app" then
''
AuthTypeApp(
app_id=${toString cfg.github.authType.app.id},
)
''
else
throw "One of AuthTypeApp or AuthTypeLegacy must be enabled"
}
)"
},
gitea=${
@ -323,6 +391,9 @@ in
systemd.services.buildbot-master = {
after = [ "postgresql.service" ];
path = [
pkgs.openssl
];
serviceConfig = {
# in master.py we read secrets from $CREDENTIALS_DIRECTORY
LoadCredential =
@ -337,10 +408,15 @@ in
(
cfg.cachix.authTokenFile != null
) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
++ lib.optionals (cfg.github.enable) [
"github-token:${cfg.github.tokenFile}"
++ lib.optionals (cfg.github.enable) ([
"github-webhook-secret:${cfg.github.webhookSecretFile}"
]
++ lib.optionals (cfg.github.authType ? "legacy") [
"github-token:${cfg.github.authType.legacy.tokenFile}"
]
++ lib.optionals (cfg.github.authType ? "app") [
"github-app-secret-key:${cfg.github.authType.app.secretKeyFile}"
])
++ lib.optionals cfg.gitea.enable [
"gitea-token:${cfg.gitea.tokenFile}"
"gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"

View file

@ -26,6 +26,8 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" }
[tool.setuptools]
packages = [
"buildbot_nix",
"buildbot_nix.github",
"buildbot_nix.github.auth",
"buildbot_effects"
]
@ -82,6 +84,17 @@ ignore = [
# not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html
"G010", # Logging statement uses `warn` instead of `warning`
# gives falls positives and isn't hard to check munually
"ERA001"
]
[tool.ruff.lint.pep8-naming]
ignore-names = [
"checkConfig",
"baseURL",
"reconfigService",
"sendMessage",
]
[tool.mypy]