From 3f01a961479910c036925c51939de5150bfb93dc Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 27 Apr 2024 17:06:17 +0200 Subject: [PATCH] Add Gitea backend Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 139 +++++++-------- buildbot_nix/common.py | 80 +++++++++ buildbot_nix/gitea_projects.py | 300 ++++++++++++++++++++++++++++++++ buildbot_nix/github_projects.py | 203 +++++++-------------- buildbot_nix/projects.py | 17 +- buildbot_nix/secrets.py | 4 +- buildbot_nix/util.py | 80 +++++++++ examples/master.nix | 6 +- nix/buildbot-gitea.nix | 27 +++ nix/checks/master.nix | 2 +- nix/master.nix | 85 +++++++-- 11 files changed, 704 insertions(+), 239 deletions(-) create mode 100644 buildbot_nix/common.py create mode 100644 buildbot_nix/gitea_projects.py create mode 100644 buildbot_nix/util.py create mode 100644 nix/buildbot-gitea.nix diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 81e58be..c093adb 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -2,56 +2,44 @@ import json import multiprocessing import os import re -import signal -import sys import uuid from collections import defaultdict from collections.abc import Generator from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable -import typing +from typing import TYPE_CHECKING, Any -from buildbot.process.log import StreamLog +from buildbot.config.builder import BuilderConfig from buildbot.configurators import ConfiguratorBase from buildbot.interfaces import WorkerSetupError -from buildbot.plugins import reporters, schedulers, steps, util, worker +from buildbot.locks import MasterLock +from buildbot.plugins import schedulers, steps, util, worker from buildbot.process import buildstep, logobserver, remotecommand from buildbot.process.project import Project -from buildbot.process.properties import Interpolate, Properties +from buildbot.process.properties import Properties from buildbot.process.results import ALL_RESULTS, statusToString -from buildbot.steps.trigger import Trigger -from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match -from buildbot.www.authz import Authz from buildbot.secrets.providers.file import SecretInAFile -from buildbot.locks import MasterLock -from buildbot.config.builder import BuilderConfig +from buildbot.steps.trigger import Trigger +from buildbot.www.authz import Authz +from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match if TYPE_CHECKING: - from buildbot.process.log import Log + from buildbot.process.log import StreamLog + from buildbot.www.auth import AuthBase -from twisted.internet import defer, threads +from twisted.internet import defer from twisted.logger import Logger -from twisted.python.failure import Failure -from .gitea_projects import ( - GiteaConfig +from .common import ( + slugify_project_name, ) - +from .gitea_projects import GiteaBackend, GiteaConfig from .github_projects import ( GithubBackend, GithubConfig, - slugify_project_name, -) - -from .projects import ( - GitProject, - GitBackend -) - -from .secrets import ( - read_secret_file ) +from .projects import GitBackend, GitProject +from .secrets import read_secret_file SKIPPED_BUILDER_NAME = "skipped-builds" @@ -98,8 +86,7 @@ class BuildTrigger(Trigger): return props def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802 - # TODO when is this None? - build_props = self.build.getProperties() if self.build is not None else Properties() + build_props = self.build.getProperties() repo_name = self.project.name project_id = slugify_project_name(repo_name) source = f"nix-eval-{project_id}" @@ -166,7 +153,9 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): project: GitProject - def __init__(self, project: GitProject, supported_systems: list[str], **kwargs: Any) -> None: + def __init__( + self, project: GitProject, supported_systems: list[str], **kwargs: Any + ) -> None: kwargs = self.setupShellMixin(kwargs) super().__init__(**kwargs) self.project = project @@ -177,9 +166,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: # run nix-eval-jobs --flake .#checks to generate the dict of stages - cmd_: object = yield self.makeRemoteShellCommand() - # TODO why doesn't type information pass through yield again? - cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_) + cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() yield self.runCommand(cmd) # if the command passes extract the list of stages @@ -245,9 +232,7 @@ class EvalErrorStep(steps.BuildStep): error = self.getProperty("error") attr = self.getProperty("attr") # show eval error - # TODO why doesn't type information pass through yield again? - error_log_: object = yield self.addLog("nix_error") - error_log: StreamLog = typing.cast(StreamLog, error_log_); + error_log: StreamLog = yield self.addLog("nix_error") error_log.addStderr(f"{attr} failed to evaluate:\n{error}") return util.FAILURE @@ -262,9 +247,7 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: # run `nix build` - # TODO why doesn't type information pass through yield again? - cmd_: object = yield self.makeRemoteShellCommand() - cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_) + cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() yield self.runCommand(cmd) res = cmd.results() @@ -300,11 +283,10 @@ class UpdateBuildOutput(steps.BuildStep): (self.path / attr).write_text(out_path) return util.SUCCESS + # GitHub somtimes fires the PR webhook before it has computed the merge commit # This is a workaround to fetch the merge commit and checkout the PR branch in CI class GitLocalPrMerge(steps.Git): - stdio_log: StreamLog - @defer.inlineCallbacks def run_vc( self, @@ -322,9 +304,7 @@ class GitLocalPrMerge(steps.Git): return res # The code below is a modified version of Git.run_vc - # TODO why doesn't type information pass through yield again? - stdio_log_: object = yield self.addLogForRemoteCommands("stdio") - self.stdio_log = typing.cast(StreamLog, stdio_log_) + self.stdio_log: StreamLog = yield self.addLogForRemoteCommands("stdio") self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n") git_installed = yield self.checkFeatureSupport() @@ -517,8 +497,7 @@ def nix_build_config( "-r", util.Property("out_path"), ], - doStepIf=lambda s: s.getProperty("branch") - == project.default_branch, + doStepIf=lambda s: s.getProperty("branch") == project.default_branch, ), ) factory.addStep( @@ -576,6 +555,7 @@ def nix_skipped_build_config( factory=factory, ) + def config_for_project( config: dict[str, Any], project: GitProject, @@ -664,8 +644,8 @@ def config_for_project( ], ) + def normalize_virtual_builder_name(name: str) -> str: - # TODO this code is a mystery to me if re.match(r"^[^:]+:", name) is not None: # rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build match = re.match(r"[^:]:(?P[^/]+)/(?P[^#]+)#.+", name) @@ -736,7 +716,9 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase): return self.check_builder(epobject, epdict, "buildrequest") -def setup_authz(backends: list[GitBackend], projects: list[GitProject], admins: list[str]) -> Authz: +def setup_authz( + backends: list[GitBackend], projects: list[GitProject], admins: list[str] +) -> Authz: allow_rules = [] allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict( lambda: {backend.reload_builder_name for backend in backends}, @@ -785,7 +767,9 @@ class NixConfigurator(ConfiguratorBase): def __init__( self, # Shape of this file: [ { "name": "", "pass": "", "cores": "" } ] - github: GithubConfig, + admins: list[str], + auth_backend: str, + github: GithubConfig | None, gitea: GiteaConfig | None, url: str, nix_supported_systems: list[str], @@ -800,7 +784,10 @@ class NixConfigurator(ConfiguratorBase): self.nix_eval_max_memory_size = nix_eval_max_memory_size self.nix_eval_worker_count = nix_eval_worker_count self.nix_supported_systems = nix_supported_systems + self.auth_backend = auth_backend + self.admins = admins self.github = github + self.gitea = gitea self.url = url self.cachix = cachix if outputs_path is None: @@ -809,15 +796,23 @@ class NixConfigurator(ConfiguratorBase): self.outputs_path = Path(outputs_path) def configure(self, config: dict[str, Any]) -> None: - backends: list[GitBackend] = [] + backends: dict[str, GitBackend] = {} - github_backend: GitBackend = GithubBackend(self.github) if self.github is not None: - backends.append(github_backend) + backends["github"] = GithubBackend(self.github) + + if self.gitea is not None: + backends["gitea"] = GiteaBackend(self.gitea) + + auth: AuthBase | None = ( + backends[self.auth_backend].create_auth() + if self.auth_backend != "none" + else None + ) projects: list[GitProject] = [] - for backend in backends: + for backend in backends.values(): projects += backend.load_projects() worker_config = json.loads(read_secret_file(self.nix_workers_secret_name)) @@ -834,17 +829,10 @@ class NixConfigurator(ConfiguratorBase): config["workers"].append(worker.Worker(worker_name, item["pass"])) worker_names.append(worker_name) - # TODO pull out into global config - webhook_secret = read_secret_file(self.github.webhook_secret_name) eval_lock = util.MasterLock("nix-eval") for project in projects: - project.create_project_hook( - project.owner, - project.repo, - self.url, - webhook_secret, - ) + project.create_project_hook(project.owner, project.repo, self.url) config_for_project( config, project, @@ -859,11 +847,9 @@ class NixConfigurator(ConfiguratorBase): config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME)) - for backend in backends: + for backend in backends.values(): # Reload backend projects - config["builders"].append( - backend.create_reload_builder([worker_names[0]]) - ) + config["builders"].append(backend.create_reload_builder([worker_names[0]])) config["schedulers"].extend( [ schedulers.ForceScheduler( @@ -891,21 +877,24 @@ class NixConfigurator(ConfiguratorBase): config["www"]["plugins"].update(dict(base_react={})) config["www"].setdefault("change_hook_dialects", {}) - for backend in backends: - config["www"]["change_hook_dialects"][backend.change_hook_name] = \ - backend.create_change_hook(webhook_secret) + for backend in backends.values(): + config["www"]["change_hook_dialects"][backend.change_hook_name] = ( + backend.create_change_hook() + ) if "auth" not in config["www"]: config["www"].setdefault("avatar_methods", []) - for backend in backends: - config["www"]["avatar_methods"].append(backend.create_avatar_method()) + for backend in backends.values(): + avatar_method = backend.create_avatar_method() + if avatar_method is not None: + config["www"]["avatar_methods"].append(avatar_method) # TODO one cannot have multiple auth backends... - config["www"]["auth"] = backends[0].create_auth() + if auth is not None: + config["www"]["auth"] = auth config["www"]["authz"] = setup_authz( - # TODO pull out into global config - admins=self.github.admins, - backends=backends, + admins=self.admins, + backends=list(backends.values()), projects=projects, ) diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py new file mode 100644 index 0000000..6dcd4f3 --- /dev/null +++ b/buildbot_nix/common.py @@ -0,0 +1,80 @@ +import contextlib +import http.client +import json +import urllib.request +from typing import Any + + +def slugify_project_name(name: str) -> str: + return name.replace(".", "-").replace("/", "-") + + +def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: + next_url: str | None = url + items = [] + while next_url: + try: + res = http_request( + next_url, + headers={"Authorization": f"Bearer {token}"}, + ) + except OSError as e: + msg = f"failed to fetch {next_url}: {e}" + raise HttpError(msg) from e + next_url = None + link = res.headers()["Link"] + if link is not None: + links = link.split(", ") + for link in links: # pagination + link_parts = link.split(";") + if link_parts[1].strip() == 'rel="next"': + next_url = link_parts[0][1:-1] + items += res.json() + return items + + +class HttpResponse: + def __init__(self, raw: http.client.HTTPResponse) -> None: + self.raw = raw + + def json(self) -> Any: + return json.load(self.raw) + + def headers(self) -> http.client.HTTPMessage: + return self.raw.headers + + +class HttpError(Exception): + pass + + +def http_request( + url: str, + method: str = "GET", + headers: dict[str, str] | None = None, + data: dict[str, Any] | None = None, +) -> HttpResponse: + body = None + if data: + body = json.dumps(data).encode("ascii") + if headers is None: + headers = {} + headers = headers.copy() + headers["User-Agent"] = "buildbot-nix" + + if not url.startswith("https:"): + msg = "url must be https: {url}" + raise HttpError(msg) + + req = urllib.request.Request( # noqa: S310 + url, headers=headers, method=method, data=body + ) + try: + resp = urllib.request.urlopen(req) # noqa: S310 + except urllib.request.HTTPError as e: + resp_body = "" + with contextlib.suppress(OSError, UnicodeDecodeError): + resp_body = e.fp.read().decode("utf-8", "replace") + msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" + raise HttpError(msg) from e + return HttpResponse(resp) diff --git a/buildbot_nix/gitea_projects.py b/buildbot_nix/gitea_projects.py new file mode 100644 index 0000000..a52841e --- /dev/null +++ b/buildbot_nix/gitea_projects.py @@ -0,0 +1,300 @@ +import json +import os +import signal +from collections.abc import Generator +from dataclasses import dataclass +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Any + +from buildbot.config.builder import BuilderConfig +from buildbot.plugins import util +from buildbot.process.buildstep import BuildStep +from buildbot.process.properties import Interpolate +from buildbot.reporters.base import ReporterBase +from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase +from buildbot_gitea.auth import GiteaAuth # type: ignore[import] +from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import] +from twisted.internet import defer, threads +from twisted.python import log +from twisted.python.failure import Failure + +from .common import ( + http_request, + paginated_github_request, + slugify_project_name, +) +from .projects import GitBackend, GitProject +from .secrets import read_secret_file + + +@dataclass +class GiteaConfig: + instance_url: str + oauth_id: str + admins: list[str] + + oauth_secret_name: str = "gitea-oauth-secret" + token_secret_name: str = "gitea-token" + webhook_secret_name: str = "gitea-webhook-secret" + project_cache_file: Path = Path("gitea-project-cache.json") + topic: str | None = "build-with-buildbot" + + def oauth_secret(self) -> str: + return read_secret_file(self.oauth_secret_name) + + def token(self) -> str: + return read_secret_file(self.token_secret_name) + + +class GiteaProject(GitProject): + config: GiteaConfig + webhook_secret: str + data: dict[str, Any] + + def __init__( + self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any] + ) -> None: + self.config = config + self.webhook_secret = webhook_secret + self.data = data + + def create_project_hook( + self, + owner: str, + repo: str, + webhook_url: str, + ) -> None: + hooks = paginated_github_request( + f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100", + self.config.token(), + ) + config = dict( + url=webhook_url + "change_hook/gitea", + content_type="json", + insecure_ssl="0", + secret=self.webhook_secret, + ) + data = dict( + name="web", + active=True, + events=["push", "pull_request"], + config=config, + type="gitea", + ) + headers = { + "Authorization": f"token {self.config.token()}", + "Accept": "application/json", + "Content-Type": "application/json", + } + for hook in hooks: + if hook["config"]["url"] == webhook_url + "change_hook/gitea": + log.msg(f"hook for {owner}/{repo} already exists") + return + + http_request( + f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks", + method="POST", + headers=headers, + data=data, + ) + + def get_project_url(self) -> str: + return f"https://git:%(secret:{self.config.token_secret_name})s@{self.config.instance_url}/{self.name}" + + @property + def pretty_type(self) -> str: + return "Gitea" + + @property + def type(self) -> str: + return "gitea" + + @property + def repo(self) -> str: + return self.data["name"] + + @property + def owner(self) -> str: + return self.data["owner"]["login"] + + @property + def name(self) -> str: + return self.data["full_name"] + + @property + def url(self) -> str: + # not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34 + return self.data["ssh_url"] + + @property + def project_id(self) -> str: + return slugify_project_name(self.data["full_name"]) + + @property + def default_branch(self) -> str: + return self.data["default_branch"] + + @property + def topics(self) -> list[str]: + # note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects` + return self.data["topics"] + + @property + def belongs_to_org(self) -> bool: + # TODO Gitea doesn't include this information + return False # self.data["owner"]["type"] == "Organization" + + +class GiteaBackend(GitBackend): + config: GiteaConfig + + def __init__(self, config: GiteaConfig) -> None: + self.config = config + self.webhook_secret = read_secret_file(self.config.webhook_secret_name) + + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: + """Updates the flake an opens a PR for it.""" + factory = util.BuildFactory() + factory.addStep( + ReloadGiteaProjects(self.config, self.config.project_cache_file), + ) + return util.BuilderConfig( + name=self.reload_builder_name, + workernames=worker_names, + factory=factory, + ) + + def create_reporter(self) -> ReporterBase: + return GiteaStatusPush( + "https://" + self.config.instance_url, + Interpolate(self.config.token()), + context=Interpolate("buildbot/%(prop:status_name)s"), + context_pr=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_change_hook(self) -> dict[str, Any]: + return { + "secret": self.webhook_secret, + } + + def create_avatar_method(self) -> AvatarBase | None: + return None + + def create_auth(self) -> AuthBase: + return GiteaAuth( + "https://" + self.config.instance_url, + self.config.oauth_id, + self.config.oauth_secret(), + ) + + def load_projects(self) -> list["GitProject"]: + if not self.config.project_cache_file.exists(): + return [] + + repos: list[dict[str, Any]] = sorted( + json.loads(self.config.project_cache_file.read_text()), + key=lambda x: x["full_name"], + ) + return list( + filter( + lambda project: self.config.topic is not None + and self.config.topic in project.topics, + [ + GiteaProject(self.config, self.webhook_secret, repo) + for repo in repos + ], + ) + ) + + def are_projects_cached(self) -> bool: + return self.config.project_cache_file.exists() + + @property + def type(self) -> str: + return "gitea" + + @property + def pretty_type(self) -> str: + return "Gitea" + + @property + def reload_builder_name(self) -> str: + return "reload-gitea-projects" + + @property + def change_hook_name(self) -> str: + return "gitea" + + +class ReloadGiteaProjects(BuildStep): + name = "reload_gitea_projects" + config: GiteaConfig + + def __init__( + self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any + ) -> None: + self.config = config + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + refresh_projects(self.config, self.project_cache_file) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + yield self.addLog("log").addStderr( + f"Failed to reload project list: {self.error_msg}" + ) + return util.FAILURE + + +def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None: + repos = [] + + for repo in paginated_github_request( + f"https://{config.instance_url}/api/v1/user/repos?limit=100", + config.token(), + ): + if not repo["permissions"]["admin"]: + name = repo["full_name"] + log.msg( + f"skipping {name} because we do not have admin privileges, needed for hook management", + ) + else: + try: + # Gitea doesn't include topics in the default repo listing, unlike GitHub + topics: list[str] = http_request( + f"https://{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics", + headers={"Authorization": f"token {config.token}"}, + ).json()["topics"] + repo["topics"] = topics + repos.append(repo) + except OSError: + pass + + with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: + path = Path(f.name) + try: + f.write(json.dumps(repos)) + f.flush() + path.rename(repo_cache_file) + except OSError: + path.unlink() + raise diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 3a62b3b..5323bbf 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,41 +1,33 @@ -import contextlib -import http.client import json -import urllib.request -import signal import os +import signal from collections.abc import Generator - -import typing +from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Any -from dataclasses import dataclass +from typing import Any -from twisted.python import log -from twisted.internet import defer, threads -from twisted.python.failure import Failure - -if TYPE_CHECKING: - from buildbot.process.log import StreamLog - -from buildbot.process.properties import Interpolate from buildbot.config.builder import BuilderConfig +from buildbot.plugins import util from buildbot.process.buildstep import BuildStep +from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.reporters.github import GitHubStatusPush -from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth -from buildbot.plugins import util +from twisted.internet import defer, threads +from twisted.python import log +from twisted.python.failure import Failure -from .projects import ( - GitProject, - GitBackend -) -from .secrets import ( - read_secret_file +from .common import ( + http_request, + paginated_github_request, + slugify_project_name, ) +from .projects import GitBackend, GitProject +from .secrets import read_secret_file + class ReloadGithubProjects(BuildStep): name = "reload_github_projects" @@ -65,43 +57,42 @@ class ReloadGithubProjects(BuildStep): os.kill(os.getpid(), signal.SIGHUP) return util.SUCCESS else: - log: object = yield self.addLog("log") - # TODO this assumes that log is of type StreamLog and not something else - typing.cast(StreamLog, log).addStderr(f"Failed to reload project list: {self.error_msg}") + yield self.addLog("log").addStderr( + f"Failed to reload project list: {self.error_msg}" + ) return util.FAILURE + @dataclass class GithubConfig: oauth_id: str - admins: list[str] + # TODO unused buildbot_user: str oauth_secret_name: str = "github-oauth-secret" - webhook_secret_name: str = "github-webhook-secret" token_secret_name: str = "github-token" + webhook_secret_name: str = "github-webhook-secret" project_cache_file: Path = Path("github-project-cache.json") topic: str | None = "build-with-buildbot" def token(self) -> str: return read_secret_file(self.token_secret_name) + @dataclass class GithubBackend(GitBackend): config: GithubConfig + webhook_secret: str - def __init__(self, config: GithubConfig): + def __init__(self, config: GithubConfig) -> None: self.config = config + self.webhook_secret = read_secret_file(self.config.webhook_secret_name) - def create_reload_builder( - self, - worker_names: list[str] - ) -> BuilderConfig: + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" factory = util.BuildFactory() factory.addStep( - ReloadGithubProjects( - self.config.token(), self.config.project_cache_file - ), + ReloadGithubProjects(self.config.token(), self.config.project_cache_file), ) return util.BuilderConfig( name=self.reload_builder_name, @@ -110,46 +101,50 @@ class GithubBackend(GitBackend): ) def create_reporter(self) -> ReporterBase: - return \ - GitHubStatusPush( - token=self.config.token(), - # Since we dynamically create build steps, - # we use `virtual_builder_name` in the webinterface - # so that we distinguish what has beeing build - context=Interpolate("buildbot/%(prop:status_name)s"), - ) + return GitHubStatusPush( + token=self.config.token(), + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) - def create_change_hook(self, webhook_secret: str) -> dict[str, Any]: + def create_change_hook(self) -> dict[str, Any]: return { - "secret": webhook_secret, + "secret": self.webhook_secret, "strict": True, - "token": self.config.token, + "token": self.config.token(), "github_property_whitelist": "*", } - def create_avatar_method(self) -> AvatarBase: + def create_avatar_method(self) -> AvatarBase | None: return AvatarGitHub(token=self.config.token()) def create_auth(self) -> AuthBase: - return \ - GitHubAuth( - self.config.oauth_id, - read_secret_file(self.config.oauth_secret_name), - apiVersion=4, - ) + return GitHubAuth( + self.config.oauth_id, + read_secret_file(self.config.oauth_secret_name), + apiVersion=4, + ) def load_projects(self) -> list["GitProject"]: if not self.config.project_cache_file.exists(): return [] repos: list[dict[str, Any]] = sorted( - json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"] + json.loads(self.config.project_cache_file.read_text()), + key=lambda x: x["full_name"], + ) + return list( + filter( + lambda project: self.config.topic is not None + and self.config.topic in project.topics, + [ + GithubProject(self.config, self.webhook_secret, repo) + for repo in repos + ], + ) ) - return \ - list(filter(\ - lambda project: self.config.topic != None and self.config.topic in project.topics, \ - [GithubProject(self.config, repo) for repo in repos] \ - )) def are_projects_cached(self) -> bool: return self.config.project_cache_file.exists() @@ -170,11 +165,15 @@ class GithubBackend(GitBackend): def change_hook_name(self) -> str: return "github" + class GithubProject(GitProject): config: GithubConfig - def __init__(self, config: GithubConfig, data: dict[str, Any]) -> None: + def __init__( + self, config: GithubConfig, webhook_secret: str, data: dict[str, Any] + ) -> None: self.config = config + self.webhook_secret = webhook_secret self.data = data def create_project_hook( @@ -182,7 +181,6 @@ class GithubProject(GitProject): owner: str, repo: str, webhook_url: str, - webhook_secret: str, ) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", @@ -192,9 +190,11 @@ class GithubProject(GitProject): url=webhook_url + "change_hook/github", content_type="json", insecure_ssl="0", - secret=webhook_secret, + secret=self.webhook_secret, + ) + data = dict( + name="web", active=True, events=["push", "pull_request"], config=config ) - data = dict(name="web", active=True, events=["push", "pull_request"], config=config) headers = { "Authorization": f"Bearer {self.config.token()}", "Accept": "application/vnd.github+json", @@ -256,79 +256,6 @@ class GithubProject(GitProject): def belongs_to_org(self) -> bool: return self.data["owner"]["type"] == "Organization" -class GithubError(Exception): - pass - - -class HttpResponse: - def __init__(self, raw: http.client.HTTPResponse) -> None: - self.raw = raw - - def json(self) -> Any: - return json.load(self.raw) - - def headers(self) -> http.client.HTTPMessage: - return self.raw.headers - - -def http_request( - url: str, - method: str = "GET", - headers: dict[str, str] | None = None, - data: dict[str, Any] | None = None, -) -> HttpResponse: - body = None - if data: - body = json.dumps(data).encode("ascii") - if headers is None: - headers = {} - headers = headers.copy() - headers["User-Agent"] = "buildbot-nix" - - if not url.startswith("https:"): - msg = "url must be https: {url}" - raise GithubError(msg) - - req = urllib.request.Request( # noqa: S310 - url, headers=headers, method=method, data=body - ) - try: - resp = urllib.request.urlopen(req) # noqa: S310 - except urllib.request.HTTPError as e: - resp_body = "" - with contextlib.suppress(OSError, UnicodeDecodeError): - resp_body = e.fp.read().decode("utf-8", "replace") - msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" - raise GithubError(msg) from e - return HttpResponse(resp) - - -def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: - next_url: str | None = url - items = [] - while next_url: - try: - res = http_request( - next_url, - headers={"Authorization": f"Bearer {token}"}, - ) - except OSError as e: - msg = f"failed to fetch {next_url}: {e}" - raise GithubError(msg) from e - next_url = None - link = res.headers()["Link"] - if link is not None: - links = link.split(", ") - for link in links: # pagination - link_parts = link.split(";") - if link_parts[1].strip() == 'rel="next"': - next_url = link_parts[0][1:-1] - items += res.json() - return items - - -def slugify_project_name(name: str) -> str: - return name.replace(".", "-").replace("/", "-") def refresh_projects(github_token: str, repo_cache_file: Path) -> None: repos = [] diff --git a/buildbot_nix/projects.py b/buildbot_nix/projects.py index 27e4b87..947bc1f 100644 --- a/buildbot_nix/projects.py +++ b/buildbot_nix/projects.py @@ -3,15 +3,13 @@ from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.reporters.base import ReporterBase -from buildbot.www.avatar import AvatarBase from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase + class GitBackend(ABC): @abstractmethod - def create_reload_builder( - self, - worker_names: list[str] - ) -> BuilderConfig: + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: pass @abstractmethod @@ -19,11 +17,11 @@ class GitBackend(ABC): pass @abstractmethod - def create_change_hook(self, webhook_secret: str) -> dict[str, Any]: + def create_change_hook(self) -> dict[str, Any]: pass @abstractmethod - def create_avatar_method(self) -> AvatarBase: + def create_avatar_method(self) -> AvatarBase | None: pass @abstractmethod @@ -58,6 +56,7 @@ class GitBackend(ABC): def change_hook_name(self) -> str: pass + class GitProject(ABC): @abstractmethod def create_project_hook( @@ -65,12 +64,11 @@ class GitProject(ABC): owner: str, repo: str, webhook_url: str, - webhook_secret: str, ) -> None: pass @abstractmethod - def get_project_url() -> str: + def get_project_url(self) -> str: pass @property @@ -113,7 +111,6 @@ class GitProject(ABC): def default_branch(self) -> str: pass - @property @abstractmethod def topics(self) -> list[str]: diff --git a/buildbot_nix/secrets.py b/buildbot_nix/secrets.py index 0f2ed69..44a3e00 100644 --- a/buildbot_nix/secrets.py +++ b/buildbot_nix/secrets.py @@ -1,6 +1,8 @@ -import os, sys +import os +import sys from pathlib import Path + def read_secret_file(secret_name: str) -> str: directory = os.environ.get("CREDENTIALS_DIRECTORY") if directory is None: diff --git a/buildbot_nix/util.py b/buildbot_nix/util.py new file mode 100644 index 0000000..6dcd4f3 --- /dev/null +++ b/buildbot_nix/util.py @@ -0,0 +1,80 @@ +import contextlib +import http.client +import json +import urllib.request +from typing import Any + + +def slugify_project_name(name: str) -> str: + return name.replace(".", "-").replace("/", "-") + + +def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: + next_url: str | None = url + items = [] + while next_url: + try: + res = http_request( + next_url, + headers={"Authorization": f"Bearer {token}"}, + ) + except OSError as e: + msg = f"failed to fetch {next_url}: {e}" + raise HttpError(msg) from e + next_url = None + link = res.headers()["Link"] + if link is not None: + links = link.split(", ") + for link in links: # pagination + link_parts = link.split(";") + if link_parts[1].strip() == 'rel="next"': + next_url = link_parts[0][1:-1] + items += res.json() + return items + + +class HttpResponse: + def __init__(self, raw: http.client.HTTPResponse) -> None: + self.raw = raw + + def json(self) -> Any: + return json.load(self.raw) + + def headers(self) -> http.client.HTTPMessage: + return self.raw.headers + + +class HttpError(Exception): + pass + + +def http_request( + url: str, + method: str = "GET", + headers: dict[str, str] | None = None, + data: dict[str, Any] | None = None, +) -> HttpResponse: + body = None + if data: + body = json.dumps(data).encode("ascii") + if headers is None: + headers = {} + headers = headers.copy() + headers["User-Agent"] = "buildbot-nix" + + if not url.startswith("https:"): + msg = "url must be https: {url}" + raise HttpError(msg) + + req = urllib.request.Request( # noqa: S310 + url, headers=headers, method=method, data=body + ) + try: + resp = urllib.request.urlopen(req) # noqa: S310 + except urllib.request.HTTPError as e: + resp_body = "" + with contextlib.suppress(OSError, UnicodeDecodeError): + resp_body = e.fp.read().decode("utf-8", "replace") + msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" + raise HttpError(msg) from e + return HttpResponse(resp) diff --git a/examples/master.nix b/examples/master.nix index 4ced75f..97cc2a5 100644 --- a/examples/master.nix +++ b/examples/master.nix @@ -15,6 +15,9 @@ { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 } ] ''; + # Users in this list will be able to reload the project list. + # All other user in the organization will be able to restart builds or evaluations. + admins = [ "Mic92" ]; github = { # Github user used as a CI identity user = "mic92-buildbot"; @@ -27,9 +30,6 @@ # After creating the app, press "Generate a new client secret" and fill in the client ID and secret below oauthId = "aaaaaaaaaaaaaaaaaaaa"; oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff"; - # Users in this list will be able to reload the project list. - # All other user in the organization will be able to restart builds or evaluations. - admins = [ "Mic92" ]; # All github projects with this topic will be added to buildbot. # One can trigger a project scan by visiting the Builds -> Builders page and looking for the "reload-github-project" builder. # This builder has a "Update Github Projects" button that everyone in the github organization can use. diff --git a/nix/buildbot-gitea.nix b/nix/buildbot-gitea.nix new file mode 100644 index 0000000..acaf92d --- /dev/null +++ b/nix/buildbot-gitea.nix @@ -0,0 +1,27 @@ +{ buildPythonPackage +, fetchPypi +, lib + +, pip +, buildbot +, requests +}: +buildPythonPackage (lib.fix (self: { + pname = "buildbot-gitea"; + version = "1.8.0"; + + nativeBuildInputs = [ + + ]; + + propagatedBuildInputs = [ + pip + buildbot + requests + ]; + + src = fetchPypi { + inherit (self) pname version; + hash = "sha256-zYcILPp42QuQyfEIzmYKV9vWf47sBAQI8FOKJlZ60yA="; + }; +})) diff --git a/nix/checks/master.nix b/nix/checks/master.nix index 0534b2d..d79d31e 100644 --- a/nix/checks/master.nix +++ b/nix/checks/master.nix @@ -14,13 +14,13 @@ { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 } ] ''; + admins = [ "Mic92" ]; github = { tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff"; oauthId = "aaaaaaaaaaaaaaaaaaaa"; user = "mic92-buildbot"; - admins = [ "Mic92" ]; }; }; }; diff --git a/nix/master.nix b/nix/master.nix index 8a8b64f..b88f048 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -5,8 +5,18 @@ }: let cfg = config.services.buildbot-nix.master; + inherit + (lib) + mkRenamedOptionModule + ; in { + imports = [ + (mkRenamedOptionModule + [ "services" "buildbot-nix" "master" "github" "admins" ] + [ "services" "buildbot-nix" "master" "admins" ]) + ]; + options = { services.buildbot-nix.master = { enable = lib.mkEnableOption "buildbot-master"; @@ -15,6 +25,13 @@ in default = "postgresql://@/buildbot"; description = "Postgresql database url"; }; + authBackend = lib.mkOption { + type = lib.types.enum [ "github" "gitea" "none" ]; + default = "github"; + description = '' + Which OAuth2 backend to use. + ''; + }; cachix = { name = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -34,7 +51,42 @@ in description = "Cachix auth token"; }; }; + gitea = { + enable = lib.mkEnableOption "Enable Gitea integration"; + + tokenFile = lib.mkOption { + type = lib.types.path; + description = "Gitea token file"; + }; + webhookSecretFile = lib.mkOption { + type = lib.types.path; + description = "Github webhook secret file"; + }; + oauthSecretFile = lib.mkOption { + type = lib.types.path; + description = "Gitea oauth secret file"; + }; + + instanceURL = lib.mkOption { + type = lib.types.str; + description = "Gitea instance URL"; + }; + oauthId = lib.mkOption { + type = lib.types.str; + description = "Gitea oauth id. Used for the login button"; + }; + topic = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "build-with-buildbot"; + description = '' + Projects that have this topic will be built by buildbot. + If null, all projects that the buildbot Gitea user has access to, are built. + ''; + }; + }; github = { + disable = lib.mkEnableOption "Disable GitHub integration"; + tokenFile = lib.mkOption { type = lib.types.path; description = "Github token file"; @@ -62,11 +114,6 @@ in type = lib.types.str; description = "Github user that is used for the buildbot"; }; - admins = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - description = "Users that are allowed to login to buildbot, trigger builds and change settings"; - }; topic = lib.mkOption { type = lib.types.nullOr lib.types.str; default = "build-with-buildbot"; @@ -76,6 +123,11 @@ in ''; }; }; + admins = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Users that are allowed to login to buildbot, trigger builds and change settings"; + }; workersFile = lib.mkOption { type = lib.types.path; description = "File containing a list of nix workers"; @@ -144,7 +196,7 @@ in home = "/var/lib/buildbot"; extraImports = '' from datetime import timedelta - from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig + from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig ''; configurators = [ '' @@ -152,18 +204,23 @@ in '' '' NixConfigurator( - github=GithubConfig( + auth_backend=${builtins.toJSON cfg.authBackend}, + github=${if cfg.github.disable then "None" else "GithubConfig( oauth_id=${builtins.toJSON cfg.github.oauthId}, - admins=${builtins.toJSON cfg.github.admins}, buildbot_user=${builtins.toJSON cfg.github.user}, topic=${builtins.toJSON cfg.github.topic}, - ), - gitea=None, + )"}, + gitea=${if !cfg.gitea.enable then "None" else "GiteaConfig( + instance_url=${builtins.toJSON cfg.gitea.instanceURL}, + oauth_id=${builtins.toJSON cfg.gitea.oauthId}, + topic=${builtins.toJSON cfg.gitea.topic}, + )"}, cachix=${if cfg.cachix.name == null then "None" else "CachixConfig( name=${builtins.toJSON cfg.cachix.name}, signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"}, auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"}, )"}, + admins=${builtins.toJSON cfg.admins}, url=${builtins.toJSON config.services.buildbot-master.buildbotUrl}, nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize}, nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount}, @@ -190,6 +247,7 @@ in (ps.toPythonModule pkgs.buildbot-worker) pkgs.buildbot-plugins.www-react (pkgs.python3.pkgs.callPackage ../default.nix { }) + (pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = pkgs.buildbot; }) ]; }; @@ -206,7 +264,12 @@ in ++ lib.optional (cfg.cachix.signingKeyFile != null) "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}" ++ lib.optional (cfg.cachix.authTokenFile != null) - "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"; + "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" + ++ lib.optionals cfg.gitea.enable [ + "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}" + "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" + "gitea-token:${cfg.gitea.tokenFile}" + ]; # Needed because it tries to reach out to github on boot. # FIXME: if github is not available, we shouldn't fail buildbot, instead it should just try later again in the background