From 1605d2d3c2feb2ff5901ac8061d1262724e65326 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Mon, 22 Apr 2024 15:58:38 +0200 Subject: [PATCH 1/2] Remove assumption of GitHub being the only forge Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 311 ++++++++++++----------------- buildbot_nix/github_projects.py | 334 ++++++++++++++++++++++++-------- buildbot_nix/projects.py | 125 ++++++++++++ buildbot_nix/secrets.py | 9 + nix/master.nix | 1 + 5 files changed, 510 insertions(+), 270 deletions(-) create mode 100644 buildbot_nix/projects.py create mode 100644 buildbot_nix/secrets.py diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index c60d8ca..81e58be 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -9,17 +9,23 @@ from collections import defaultdict from collections.abc import Generator from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Callable +import typing +from buildbot.process.log import StreamLog from buildbot.configurators import ConfiguratorBase from buildbot.interfaces import WorkerSetupError -from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker +from buildbot.plugins import reporters, schedulers, steps, util, worker from buildbot.process import buildstep, logobserver, remotecommand from buildbot.process.project import Project from buildbot.process.properties import Interpolate, Properties from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.steps.trigger import Trigger from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match +from buildbot.www.authz import Authz +from buildbot.secrets.providers.file import SecretInAFile +from buildbot.locks import MasterLock +from buildbot.config.builder import BuilderConfig if TYPE_CHECKING: from buildbot.process.log import Log @@ -28,14 +34,25 @@ from twisted.internet import defer, threads from twisted.logger import Logger from twisted.python.failure import Failure +from .gitea_projects import ( + GiteaConfig +) + from .github_projects import ( - GithubProject, - create_project_hook, - load_projects, - refresh_projects, + GithubBackend, + GithubConfig, slugify_project_name, ) +from .projects import ( + GitProject, + GitBackend +) + +from .secrets import ( + read_secret_file +) + SKIPPED_BUILDER_NAME = "skipped-builds" log = Logger() @@ -48,8 +65,11 @@ class BuildbotNixError(Exception): class BuildTrigger(Trigger): """Dynamic trigger that creates a build for every attribute.""" + project: GitProject + def __init__( self, + project: GitProject, builds_scheduler: str, skipped_builds_scheduler: str, jobs: list[dict[str, Any]], @@ -57,6 +77,7 @@ class BuildTrigger(Trigger): ) -> None: if "name" not in kwargs: kwargs["name"] = "trigger" + self.project = project self.jobs = jobs self.config = None self.builds_scheduler = builds_scheduler @@ -77,11 +98,9 @@ class BuildTrigger(Trigger): return props def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802 - build_props = self.build.getProperties() - repo_name = build_props.getProperty( - "github.base.repo.full_name", - build_props.getProperty("github.repository.full_name"), - ) + # TODO when is this None? + build_props = self.build.getProperties() if self.build is not None else Properties() + repo_name = self.project.name project_id = slugify_project_name(repo_name) source = f"nix-eval-{project_id}" @@ -145,9 +164,12 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): every attribute. """ - def __init__(self, supported_systems: list[str], **kwargs: Any) -> None: + project: GitProject + + def __init__(self, project: GitProject, supported_systems: list[str], **kwargs: Any) -> None: kwargs = self.setupShellMixin(kwargs) super().__init__(**kwargs) + self.project = project self.observer = logobserver.BufferLogObserver() self.addLogObserver("stdio", self.observer) self.supported_systems = supported_systems @@ -155,7 +177,9 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: # run nix-eval-jobs --flake .#checks to generate the dict of stages - cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() + cmd_: object = yield self.makeRemoteShellCommand() + # TODO why doesn't type information pass through yield again? + cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_) yield self.runCommand(cmd) # if the command passes extract the list of stages @@ -172,11 +196,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): msg = f"Failed to parse line: {line}" raise BuildbotNixError(msg) from e jobs.append(job) - build_props = self.build.getProperties() - repo_name = build_props.getProperty( - "github.base.repo.full_name", - build_props.getProperty("github.repository.full_name"), - ) + repo_name = self.project.name project_id = slugify_project_name(repo_name) filtered_jobs = [] for job in jobs: @@ -187,6 +207,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): self.build.addStepsAfterCurrentStep( [ BuildTrigger( + self.project, builds_scheduler=f"{project_id}-nix-build", skipped_builds_scheduler=f"{project_id}-nix-skipped-build", name="build flake", @@ -224,7 +245,9 @@ class EvalErrorStep(steps.BuildStep): error = self.getProperty("error") attr = self.getProperty("attr") # show eval error - error_log: Log = yield self.addLog("nix_error") + # TODO why doesn't type information pass through yield again? + error_log_: object = yield self.addLog("nix_error") + error_log: StreamLog = typing.cast(StreamLog, error_log_); error_log.addStderr(f"{attr} failed to evaluate:\n{error}") return util.FAILURE @@ -239,7 +262,9 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: # run `nix build` - cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() + # TODO why doesn't type information pass through yield again? + cmd_: object = yield self.makeRemoteShellCommand() + cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_) yield self.runCommand(cmd) res = cmd.results() @@ -256,15 +281,16 @@ class UpdateBuildOutput(steps.BuildStep): on the target machine. """ - def __init__(self, path: Path, **kwargs: Any) -> None: + project: GitProject + + def __init__(self, project: GitProject, path: Path, **kwargs: Any) -> None: super().__init__(**kwargs) + self.project = project self.path = path def run(self) -> Generator[Any, object, Any]: props = self.build.getProperties() - if props.getProperty("branch") != props.getProperty( - "github.repository.default_branch", - ): + if props.getProperty("branch") != self.project.default_branch: return util.SKIPPED attr = Path(props.getProperty("attr")).name @@ -274,62 +300,11 @@ class UpdateBuildOutput(steps.BuildStep): (self.path / attr).write_text(out_path) return util.SUCCESS - -class ReloadGithubProjects(steps.BuildStep): - name = "reload_github_projects" - - def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: - self.token = token - self.project_cache_file = project_cache_file - super().__init__(**kwargs) - - def reload_projects(self) -> None: - refresh_projects(self.token, self.project_cache_file) - - @defer.inlineCallbacks - def run(self) -> Generator[Any, object, Any]: - d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] - - self.error_msg = "" - - def error_cb(failure: Failure) -> int: - self.error_msg += failure.getTraceback() - return util.FAILURE - - d.addCallbacks(lambda _: util.SUCCESS, error_cb) - res = yield d - if res == util.SUCCESS: - # reload the buildbot config - os.kill(os.getpid(), signal.SIGHUP) - return util.SUCCESS - else: - log: Log = yield self.addLog("log") - log.addStderr(f"Failed to reload project list: {self.error_msg}") - return util.FAILURE - - -def reload_github_projects( - worker_names: list[str], - github_token_secret: str, - project_cache_file: Path, -) -> util.BuilderConfig: - """Updates the flake an opens a PR for it.""" - factory = util.BuildFactory() - factory.addStep( - ReloadGithubProjects( - github_token_secret, project_cache_file=project_cache_file - ), - ) - return util.BuilderConfig( - name="reload-github-projects", - workernames=worker_names, - factory=factory, - ) - - # GitHub somtimes fires the PR webhook before it has computed the merge commit # This is a workaround to fetch the merge commit and checkout the PR branch in CI class GitLocalPrMerge(steps.Git): + stdio_log: StreamLog + @defer.inlineCallbacks def run_vc( self, @@ -347,8 +322,9 @@ class GitLocalPrMerge(steps.Git): return res # The code below is a modified version of Git.run_vc - self.stdio_log: Log = yield self.addLogForRemoteCommands("stdio") - + # TODO why doesn't type information pass through yield again? + stdio_log_: object = yield self.addLogForRemoteCommands("stdio") + self.stdio_log = typing.cast(StreamLog, stdio_log_) self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n") git_installed = yield self.checkFeatureSupport() @@ -392,22 +368,20 @@ class GitLocalPrMerge(steps.Git): def nix_eval_config( - project: GithubProject, + project: GitProject, worker_names: list[str], - github_token_secret: str, + git_url: str, supported_systems: list[str], - eval_lock: util.MasterLock, + eval_lock: MasterLock, worker_count: int, max_memory_size: int, -) -> util.BuilderConfig: +) -> BuilderConfig: """Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel. For each evaluated attribute a new build pipeline is started. """ factory = util.BuildFactory() # check out the source - url_with_secret = util.Interpolate( - f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s", - ) + url_with_secret = util.Interpolate(git_url) factory.addStep( GitLocalPrMerge( repourl=url_with_secret, @@ -422,6 +396,7 @@ def nix_eval_config( factory.addStep( NixEvalCommand( + project=project, env={}, name="evaluate flake", supported_systems=supported_systems, @@ -482,11 +457,11 @@ class CachixConfig: def nix_build_config( - project: GithubProject, + project: GitProject, worker_names: list[str], cachix: CachixConfig | None = None, outputs_path: Path | None = None, -) -> util.BuilderConfig: +) -> BuilderConfig: """Builds one nix flake attribute.""" factory = util.BuildFactory() factory.addStep( @@ -543,7 +518,7 @@ def nix_build_config( util.Property("out_path"), ], doStepIf=lambda s: s.getProperty("branch") - == s.getProperty("github.repository.default_branch"), + == project.default_branch, ), ) factory.addStep( @@ -555,6 +530,7 @@ def nix_build_config( if outputs_path is not None: factory.addStep( UpdateBuildOutput( + project=project, name="Update build output", path=outputs_path, ), @@ -570,9 +546,9 @@ def nix_build_config( def nix_skipped_build_config( - project: GithubProject, + project: GitProject, worker_names: list[str], -) -> util.BuilderConfig: +) -> BuilderConfig: """Dummy builder that is triggered when a build is skipped.""" factory = util.BuildFactory() factory.addStep( @@ -600,43 +576,17 @@ def nix_skipped_build_config( factory=factory, ) - -def read_secret_file(secret_name: str) -> str: - directory = os.environ.get("CREDENTIALS_DIRECTORY") - if directory is None: - print("directory not set", file=sys.stderr) - sys.exit(1) - return Path(directory).joinpath(secret_name).read_text().rstrip() - - -@dataclass -class GithubConfig: - oauth_id: str - admins: list[str] - - buildbot_user: str - oauth_secret_name: str = "github-oauth-secret" - webhook_secret_name: str = "github-webhook-secret" - token_secret_name: str = "github-token" - project_cache_file: Path = Path("github-project-cache.json") - topic: str | None = "build-with-buildbot" - - def token(self) -> str: - return read_secret_file(self.token_secret_name) - - def config_for_project( config: dict[str, Any], - project: GithubProject, + project: GitProject, worker_names: list[str], - github: GithubConfig, nix_supported_systems: list[str], nix_eval_worker_count: int, nix_eval_max_memory_size: int, - eval_lock: util.MasterLock, + eval_lock: MasterLock, cachix: CachixConfig | None = None, outputs_path: Path | None = None, -) -> Project: +) -> None: config["projects"].append(Project(project.name)) config["schedulers"].extend( [ @@ -644,8 +594,7 @@ def config_for_project( name=f"{project.project_id}-default-branch", change_filter=util.ChangeFilter( repository=project.url, - filter_fn=lambda c: c.branch - == c.properties.getProperty("github.repository.default_branch"), + filter_fn=lambda c: c.branch == project.default_branch, ), builderNames=[f"{project.name}/nix-eval"], treeStableTimer=5, @@ -685,7 +634,7 @@ def config_for_project( properties=[ util.StringParameter( name="project", - label="Name of the GitHub repository.", + label=f"Name of the {project.pretty_type} repository.", default=project.name, ), ], @@ -699,7 +648,7 @@ def config_for_project( nix_eval_config( project, worker_names, - github_token_secret=github.token_secret_name, + git_url=project.get_project_url(), supported_systems=nix_supported_systems, worker_count=nix_eval_worker_count, max_memory_size=nix_eval_max_memory_size, @@ -715,11 +664,11 @@ def config_for_project( ], ) - def normalize_virtual_builder_name(name: str) -> str: - if name.startswith("github:"): + # TODO this code is a mystery to me + if re.match(r"^[^:]+:", name) is not None: # rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build - match = re.match(r"github:(?P[^/]+)/(?P[^#]+)#.+", name) + match = re.match(r"[^:]:(?P[^/]+)/(?P[^#]+)#.+", name) if match: return f"{match['owner']}/{match['repo']}/nix-build" @@ -787,10 +736,10 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase): return self.check_builder(epobject, epdict, "buildrequest") -def setup_authz(projects: list[GithubProject], admins: list[str]) -> util.Authz: +def setup_authz(backends: list[GitBackend], projects: list[GitProject], admins: list[str]) -> Authz: allow_rules = [] allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict( - lambda: {"reload-github-projects"}, + lambda: {backend.reload_builder_name for backend in backends}, ) for project in projects: @@ -837,6 +786,7 @@ class NixConfigurator(ConfiguratorBase): self, # Shape of this file: [ { "name": "", "pass": "", "cores": "" } ] github: GithubConfig, + gitea: GiteaConfig | None, url: str, nix_supported_systems: list[str], nix_eval_worker_count: int | None, @@ -859,9 +809,17 @@ class NixConfigurator(ConfiguratorBase): self.outputs_path = Path(outputs_path) def configure(self, config: dict[str, Any]) -> None: - projects = load_projects(self.github.token(), self.github.project_cache_file) - if self.github.topic is not None: - projects = [p for p in projects if self.github.topic in p.topics] + backends: list[GitBackend] = [] + + github_backend: GitBackend = GithubBackend(self.github) + if self.github is not None: + backends.append(github_backend) + + projects: list[GitProject] = [] + + for backend in backends: + projects += backend.load_projects() + worker_config = json.loads(read_secret_file(self.nix_workers_secret_name)) worker_names = [] @@ -876,22 +834,21 @@ class NixConfigurator(ConfiguratorBase): config["workers"].append(worker.Worker(worker_name, item["pass"])) worker_names.append(worker_name) + # TODO pull out into global config webhook_secret = read_secret_file(self.github.webhook_secret_name) eval_lock = util.MasterLock("nix-eval") for project in projects: - create_project_hook( + project.create_project_hook( project.owner, project.repo, - self.github.token(), - self.url + "change_hook/github", + self.url, webhook_secret, ) config_for_project( config, project, worker_names, - self.github, self.nix_supported_systems, self.nix_eval_worker_count or multiprocessing.cpu_count(), self.nix_eval_max_memory_size, @@ -900,42 +857,32 @@ class NixConfigurator(ConfiguratorBase): self.outputs_path, ) - # Reload github projects - config["builders"].append( - reload_github_projects( - [worker_names[0]], - self.github.token(), - self.github.project_cache_file, - ), - ) config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME)) - config["schedulers"].extend( - [ - schedulers.ForceScheduler( - name="reload-github-projects", - builderNames=["reload-github-projects"], - buttonName="Update projects", - ), - # project list twice a day and on startup - PeriodicWithStartup( - name="reload-github-projects-bidaily", - builderNames=["reload-github-projects"], - periodicBuildTimer=12 * 60 * 60, - run_on_startup=not self.github.project_cache_file.exists(), - ), - ], - ) - config["services"].append( - reporters.GitHubStatusPush( - token=self.github.token(), - # Since we dynamically create build steps, - # we use `virtual_builder_name` in the webinterface - # so that we distinguish what has beeing build - context=Interpolate("buildbot/%(prop:status_name)s"), - ), - ) - systemd_secrets = secrets.SecretInAFile( + for backend in backends: + # Reload backend projects + config["builders"].append( + backend.create_reload_builder([worker_names[0]]) + ) + config["schedulers"].extend( + [ + schedulers.ForceScheduler( + name=f"reload-{backend.type}-projects", + builderNames=[backend.reload_builder_name], + buttonName="Update projects", + ), + # project list twice a day and on startup + PeriodicWithStartup( + name=f"reload-{backend.type}-projects-bidaily", + builderNames=[backend.reload_builder_name], + periodicBuildTimer=12 * 60 * 60, + run_on_startup=not backend.are_projects_cached(), + ), + ], + ) + config["services"].append(backend.create_reporter()) + + systemd_secrets = SecretInAFile( dirname=os.environ["CREDENTIALS_DIRECTORY"], ) config["secretsProviders"].append(systemd_secrets) @@ -944,25 +891,21 @@ class NixConfigurator(ConfiguratorBase): config["www"]["plugins"].update(dict(base_react={})) config["www"].setdefault("change_hook_dialects", {}) - config["www"]["change_hook_dialects"]["github"] = { - "secret": webhook_secret, - "strict": True, - "token": self.github.token(), - "github_property_whitelist": "*", - } + for backend in backends: + config["www"]["change_hook_dialects"][backend.change_hook_name] = \ + backend.create_change_hook(webhook_secret) if "auth" not in config["www"]: config["www"].setdefault("avatar_methods", []) - config["www"]["avatar_methods"].append( - util.AvatarGitHub(token=self.github.token()), - ) - config["www"]["auth"] = util.GitHubAuth( - self.github.oauth_id, - read_secret_file(self.github.oauth_secret_name), - apiVersion=4, - ) + + for backend in backends: + config["www"]["avatar_methods"].append(backend.create_avatar_method()) + # TODO one cannot have multiple auth backends... + config["www"]["auth"] = backends[0].create_auth() config["www"]["authz"] = setup_authz( + # TODO pull out into global config admins=self.github.admins, + backends=backends, projects=projects, ) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 67ae791..3a62b3b 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -2,12 +2,259 @@ import contextlib import http.client import json import urllib.request +import signal +import os +from collections.abc import Generator + +import typing from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any +from typing import TYPE_CHECKING, Any +from dataclasses import dataclass from twisted.python import log +from twisted.internet import defer, threads +from twisted.python.failure import Failure +if TYPE_CHECKING: + from buildbot.process.log import StreamLog + +from buildbot.process.properties import Interpolate +from buildbot.config.builder import BuilderConfig +from buildbot.process.buildstep import BuildStep +from buildbot.reporters.base import ReporterBase +from buildbot.reporters.github import GitHubStatusPush +from buildbot.www.avatar import AvatarBase, AvatarGitHub +from buildbot.www.auth import AuthBase +from buildbot.www.oauth2 import GitHubAuth +from buildbot.plugins import util + +from .projects import ( + GitProject, + GitBackend +) +from .secrets import ( + read_secret_file +) + +class ReloadGithubProjects(BuildStep): + name = "reload_github_projects" + + def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: + self.token = token + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + refresh_projects(self.token, self.project_cache_file) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + log: object = yield self.addLog("log") + # TODO this assumes that log is of type StreamLog and not something else + typing.cast(StreamLog, log).addStderr(f"Failed to reload project list: {self.error_msg}") + return util.FAILURE + +@dataclass +class GithubConfig: + oauth_id: str + admins: list[str] + + buildbot_user: str + oauth_secret_name: str = "github-oauth-secret" + webhook_secret_name: str = "github-webhook-secret" + token_secret_name: str = "github-token" + project_cache_file: Path = Path("github-project-cache.json") + topic: str | None = "build-with-buildbot" + + def token(self) -> str: + return read_secret_file(self.token_secret_name) + +@dataclass +class GithubBackend(GitBackend): + config: GithubConfig + + def __init__(self, config: GithubConfig): + self.config = config + + def create_reload_builder( + self, + worker_names: list[str] + ) -> BuilderConfig: + """Updates the flake an opens a PR for it.""" + factory = util.BuildFactory() + factory.addStep( + ReloadGithubProjects( + self.config.token(), self.config.project_cache_file + ), + ) + return util.BuilderConfig( + name=self.reload_builder_name, + workernames=worker_names, + factory=factory, + ) + + def create_reporter(self) -> ReporterBase: + return \ + GitHubStatusPush( + token=self.config.token(), + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_change_hook(self, webhook_secret: str) -> dict[str, Any]: + return { + "secret": webhook_secret, + "strict": True, + "token": self.config.token, + "github_property_whitelist": "*", + } + + def create_avatar_method(self) -> AvatarBase: + return AvatarGitHub(token=self.config.token()) + + def create_auth(self) -> AuthBase: + return \ + GitHubAuth( + self.config.oauth_id, + read_secret_file(self.config.oauth_secret_name), + apiVersion=4, + ) + + def load_projects(self) -> list["GitProject"]: + if not self.config.project_cache_file.exists(): + return [] + + repos: list[dict[str, Any]] = sorted( + json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"] + ) + return \ + list(filter(\ + lambda project: self.config.topic != None and self.config.topic in project.topics, \ + [GithubProject(self.config, repo) for repo in repos] \ + )) + + def are_projects_cached(self) -> bool: + return self.config.project_cache_file.exists() + + @property + def type(self) -> str: + return "github" + + @property + def pretty_type(self) -> str: + return "GitHub" + + @property + def reload_builder_name(self) -> str: + return "reload-github-projects" + + @property + def change_hook_name(self) -> str: + return "github" + +class GithubProject(GitProject): + config: GithubConfig + + def __init__(self, config: GithubConfig, data: dict[str, Any]) -> None: + self.config = config + self.data = data + + def create_project_hook( + self, + owner: str, + repo: str, + webhook_url: str, + webhook_secret: str, + ) -> None: + hooks = paginated_github_request( + f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", + self.config.token(), + ) + config = dict( + url=webhook_url + "change_hook/github", + content_type="json", + insecure_ssl="0", + secret=webhook_secret, + ) + data = dict(name="web", active=True, events=["push", "pull_request"], config=config) + headers = { + "Authorization": f"Bearer {self.config.token()}", + "Accept": "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + } + for hook in hooks: + if hook["config"]["url"] == webhook_url + "change_hook/github": + log.msg(f"hook for {owner}/{repo} already exists") + return + + http_request( + f"https://api.github.com/repos/{owner}/{repo}/hooks", + method="POST", + headers=headers, + data=data, + ) + + def get_project_url(self) -> str: + return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}" + + @property + def pretty_type(self) -> str: + return "GitHub" + + @property + def type(self) -> str: + return "github" + + @property + def repo(self) -> str: + return self.data["name"] + + @property + def owner(self) -> str: + return self.data["owner"]["login"] + + @property + def name(self) -> str: + return self.data["full_name"] + + @property + def url(self) -> str: + return self.data["html_url"] + + @property + def project_id(self) -> str: + return slugify_project_name(self.data["full_name"]) + + @property + def default_branch(self) -> str: + return self.data["default_branch"] + + @property + def topics(self) -> list[str]: + return self.data["topics"] + + @property + def belongs_to_org(self) -> bool: + return self.data["owner"]["type"] == "Organization" class GithubError(Exception): pass @@ -83,81 +330,6 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: def slugify_project_name(name: str) -> str: return name.replace(".", "-").replace("/", "-") - -class GithubProject: - def __init__(self, data: dict[str, Any]) -> None: - self.data = data - - @property - def repo(self) -> str: - return self.data["name"] - - @property - def owner(self) -> str: - return self.data["owner"]["login"] - - @property - def name(self) -> str: - return self.data["full_name"] - - @property - def url(self) -> str: - return self.data["html_url"] - - @property - def project_id(self) -> str: - return slugify_project_name(self.data["full_name"]) - - @property - def default_branch(self) -> str: - return self.data["default_branch"] - - @property - def topics(self) -> list[str]: - return self.data["topics"] - - @property - def belongs_to_org(self) -> bool: - return self.data["owner"]["type"] == "Organization" - - -def create_project_hook( - owner: str, - repo: str, - token: str, - webhook_url: str, - webhook_secret: str, -) -> None: - hooks = paginated_github_request( - f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", - token, - ) - config = dict( - url=webhook_url, - content_type="json", - insecure_ssl="0", - secret=webhook_secret, - ) - data = dict(name="web", active=True, events=["push", "pull_request"], config=config) - headers = { - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json", - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - } - for hook in hooks: - if hook["config"]["url"] == webhook_url: - log.msg(f"hook for {owner}/{repo} already exists") - return - - http_request( - f"https://api.github.com/repos/{owner}/{repo}/hooks", - method="POST", - headers=headers, - data=data, - ) - - def refresh_projects(github_token: str, repo_cache_file: Path) -> None: repos = [] @@ -182,13 +354,3 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: except OSError: path.unlink() raise - - -def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]: - if not repo_cache_file.exists(): - return [] - - repos: list[dict[str, Any]] = sorted( - json.loads(repo_cache_file.read_text()), key=lambda x: x["full_name"] - ) - return [GithubProject(repo) for repo in repos] diff --git a/buildbot_nix/projects.py b/buildbot_nix/projects.py new file mode 100644 index 0000000..27e4b87 --- /dev/null +++ b/buildbot_nix/projects.py @@ -0,0 +1,125 @@ +from abc import ABC, abstractmethod +from typing import Any + +from buildbot.config.builder import BuilderConfig +from buildbot.reporters.base import ReporterBase +from buildbot.www.avatar import AvatarBase +from buildbot.www.auth import AuthBase + +class GitBackend(ABC): + @abstractmethod + def create_reload_builder( + self, + worker_names: list[str] + ) -> BuilderConfig: + pass + + @abstractmethod + def create_reporter(self) -> ReporterBase: + pass + + @abstractmethod + def create_change_hook(self, webhook_secret: str) -> dict[str, Any]: + pass + + @abstractmethod + def create_avatar_method(self) -> AvatarBase: + pass + + @abstractmethod + def create_auth(self) -> AuthBase: + pass + + @abstractmethod + def load_projects(self) -> list["GitProject"]: + pass + + @abstractmethod + def are_projects_cached(self) -> bool: + pass + + @property + @abstractmethod + def pretty_type(self) -> str: + pass + + @property + @abstractmethod + def type(self) -> str: + pass + + @property + @abstractmethod + def reload_builder_name(self) -> str: + pass + + @property + @abstractmethod + def change_hook_name(self) -> str: + pass + +class GitProject(ABC): + @abstractmethod + def create_project_hook( + self, + owner: str, + repo: str, + webhook_url: str, + webhook_secret: str, + ) -> None: + pass + + @abstractmethod + def get_project_url() -> str: + pass + + @property + @abstractmethod + def pretty_type(self) -> str: + pass + + @property + @abstractmethod + def type(self) -> str: + pass + + @property + @abstractmethod + def repo(self) -> str: + pass + + @property + @abstractmethod + def owner(self) -> str: + pass + + @property + @abstractmethod + def name(self) -> str: + pass + + @property + @abstractmethod + def url(self) -> str: + pass + + @property + @abstractmethod + def project_id(self) -> str: + pass + + @property + @abstractmethod + def default_branch(self) -> str: + pass + + + @property + @abstractmethod + def topics(self) -> list[str]: + pass + + @property + @abstractmethod + def belongs_to_org(self) -> bool: + pass diff --git a/buildbot_nix/secrets.py b/buildbot_nix/secrets.py new file mode 100644 index 0000000..0f2ed69 --- /dev/null +++ b/buildbot_nix/secrets.py @@ -0,0 +1,9 @@ +import os, sys +from pathlib import Path + +def read_secret_file(secret_name: str) -> str: + directory = os.environ.get("CREDENTIALS_DIRECTORY") + if directory is None: + print("directory not set", file=sys.stderr) + sys.exit(1) + return Path(directory).joinpath(secret_name).read_text().rstrip() diff --git a/nix/master.nix b/nix/master.nix index 32bfea1..8a8b64f 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -158,6 +158,7 @@ in buildbot_user=${builtins.toJSON cfg.github.user}, topic=${builtins.toJSON cfg.github.topic}, ), + gitea=None, cachix=${if cfg.cachix.name == null then "None" else "CachixConfig( name=${builtins.toJSON cfg.cachix.name}, signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"}, From 3f01a961479910c036925c51939de5150bfb93dc Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 27 Apr 2024 17:06:17 +0200 Subject: [PATCH 2/2] Add Gitea backend Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 139 +++++++-------- buildbot_nix/common.py | 80 +++++++++ buildbot_nix/gitea_projects.py | 300 ++++++++++++++++++++++++++++++++ buildbot_nix/github_projects.py | 203 +++++++-------------- buildbot_nix/projects.py | 17 +- buildbot_nix/secrets.py | 4 +- buildbot_nix/util.py | 80 +++++++++ examples/master.nix | 6 +- nix/buildbot-gitea.nix | 27 +++ nix/checks/master.nix | 2 +- nix/master.nix | 85 +++++++-- 11 files changed, 704 insertions(+), 239 deletions(-) create mode 100644 buildbot_nix/common.py create mode 100644 buildbot_nix/gitea_projects.py create mode 100644 buildbot_nix/util.py create mode 100644 nix/buildbot-gitea.nix diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 81e58be..c093adb 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -2,56 +2,44 @@ import json import multiprocessing import os import re -import signal -import sys import uuid from collections import defaultdict from collections.abc import Generator from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable -import typing +from typing import TYPE_CHECKING, Any -from buildbot.process.log import StreamLog +from buildbot.config.builder import BuilderConfig from buildbot.configurators import ConfiguratorBase from buildbot.interfaces import WorkerSetupError -from buildbot.plugins import reporters, schedulers, steps, util, worker +from buildbot.locks import MasterLock +from buildbot.plugins import schedulers, steps, util, worker from buildbot.process import buildstep, logobserver, remotecommand from buildbot.process.project import Project -from buildbot.process.properties import Interpolate, Properties +from buildbot.process.properties import Properties from buildbot.process.results import ALL_RESULTS, statusToString -from buildbot.steps.trigger import Trigger -from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match -from buildbot.www.authz import Authz from buildbot.secrets.providers.file import SecretInAFile -from buildbot.locks import MasterLock -from buildbot.config.builder import BuilderConfig +from buildbot.steps.trigger import Trigger +from buildbot.www.authz import Authz +from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match if TYPE_CHECKING: - from buildbot.process.log import Log + from buildbot.process.log import StreamLog + from buildbot.www.auth import AuthBase -from twisted.internet import defer, threads +from twisted.internet import defer from twisted.logger import Logger -from twisted.python.failure import Failure -from .gitea_projects import ( - GiteaConfig +from .common import ( + slugify_project_name, ) - +from .gitea_projects import GiteaBackend, GiteaConfig from .github_projects import ( GithubBackend, GithubConfig, - slugify_project_name, -) - -from .projects import ( - GitProject, - GitBackend -) - -from .secrets import ( - read_secret_file ) +from .projects import GitBackend, GitProject +from .secrets import read_secret_file SKIPPED_BUILDER_NAME = "skipped-builds" @@ -98,8 +86,7 @@ class BuildTrigger(Trigger): return props def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802 - # TODO when is this None? - build_props = self.build.getProperties() if self.build is not None else Properties() + build_props = self.build.getProperties() repo_name = self.project.name project_id = slugify_project_name(repo_name) source = f"nix-eval-{project_id}" @@ -166,7 +153,9 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): project: GitProject - def __init__(self, project: GitProject, supported_systems: list[str], **kwargs: Any) -> None: + def __init__( + self, project: GitProject, supported_systems: list[str], **kwargs: Any + ) -> None: kwargs = self.setupShellMixin(kwargs) super().__init__(**kwargs) self.project = project @@ -177,9 +166,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: # run nix-eval-jobs --flake .#checks to generate the dict of stages - cmd_: object = yield self.makeRemoteShellCommand() - # TODO why doesn't type information pass through yield again? - cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_) + cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() yield self.runCommand(cmd) # if the command passes extract the list of stages @@ -245,9 +232,7 @@ class EvalErrorStep(steps.BuildStep): error = self.getProperty("error") attr = self.getProperty("attr") # show eval error - # TODO why doesn't type information pass through yield again? - error_log_: object = yield self.addLog("nix_error") - error_log: StreamLog = typing.cast(StreamLog, error_log_); + error_log: StreamLog = yield self.addLog("nix_error") error_log.addStderr(f"{attr} failed to evaluate:\n{error}") return util.FAILURE @@ -262,9 +247,7 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: # run `nix build` - # TODO why doesn't type information pass through yield again? - cmd_: object = yield self.makeRemoteShellCommand() - cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_) + cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() yield self.runCommand(cmd) res = cmd.results() @@ -300,11 +283,10 @@ class UpdateBuildOutput(steps.BuildStep): (self.path / attr).write_text(out_path) return util.SUCCESS + # GitHub somtimes fires the PR webhook before it has computed the merge commit # This is a workaround to fetch the merge commit and checkout the PR branch in CI class GitLocalPrMerge(steps.Git): - stdio_log: StreamLog - @defer.inlineCallbacks def run_vc( self, @@ -322,9 +304,7 @@ class GitLocalPrMerge(steps.Git): return res # The code below is a modified version of Git.run_vc - # TODO why doesn't type information pass through yield again? - stdio_log_: object = yield self.addLogForRemoteCommands("stdio") - self.stdio_log = typing.cast(StreamLog, stdio_log_) + self.stdio_log: StreamLog = yield self.addLogForRemoteCommands("stdio") self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n") git_installed = yield self.checkFeatureSupport() @@ -517,8 +497,7 @@ def nix_build_config( "-r", util.Property("out_path"), ], - doStepIf=lambda s: s.getProperty("branch") - == project.default_branch, + doStepIf=lambda s: s.getProperty("branch") == project.default_branch, ), ) factory.addStep( @@ -576,6 +555,7 @@ def nix_skipped_build_config( factory=factory, ) + def config_for_project( config: dict[str, Any], project: GitProject, @@ -664,8 +644,8 @@ def config_for_project( ], ) + def normalize_virtual_builder_name(name: str) -> str: - # TODO this code is a mystery to me if re.match(r"^[^:]+:", name) is not None: # rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build match = re.match(r"[^:]:(?P[^/]+)/(?P[^#]+)#.+", name) @@ -736,7 +716,9 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase): return self.check_builder(epobject, epdict, "buildrequest") -def setup_authz(backends: list[GitBackend], projects: list[GitProject], admins: list[str]) -> Authz: +def setup_authz( + backends: list[GitBackend], projects: list[GitProject], admins: list[str] +) -> Authz: allow_rules = [] allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict( lambda: {backend.reload_builder_name for backend in backends}, @@ -785,7 +767,9 @@ class NixConfigurator(ConfiguratorBase): def __init__( self, # Shape of this file: [ { "name": "", "pass": "", "cores": "" } ] - github: GithubConfig, + admins: list[str], + auth_backend: str, + github: GithubConfig | None, gitea: GiteaConfig | None, url: str, nix_supported_systems: list[str], @@ -800,7 +784,10 @@ class NixConfigurator(ConfiguratorBase): self.nix_eval_max_memory_size = nix_eval_max_memory_size self.nix_eval_worker_count = nix_eval_worker_count self.nix_supported_systems = nix_supported_systems + self.auth_backend = auth_backend + self.admins = admins self.github = github + self.gitea = gitea self.url = url self.cachix = cachix if outputs_path is None: @@ -809,15 +796,23 @@ class NixConfigurator(ConfiguratorBase): self.outputs_path = Path(outputs_path) def configure(self, config: dict[str, Any]) -> None: - backends: list[GitBackend] = [] + backends: dict[str, GitBackend] = {} - github_backend: GitBackend = GithubBackend(self.github) if self.github is not None: - backends.append(github_backend) + backends["github"] = GithubBackend(self.github) + + if self.gitea is not None: + backends["gitea"] = GiteaBackend(self.gitea) + + auth: AuthBase | None = ( + backends[self.auth_backend].create_auth() + if self.auth_backend != "none" + else None + ) projects: list[GitProject] = [] - for backend in backends: + for backend in backends.values(): projects += backend.load_projects() worker_config = json.loads(read_secret_file(self.nix_workers_secret_name)) @@ -834,17 +829,10 @@ class NixConfigurator(ConfiguratorBase): config["workers"].append(worker.Worker(worker_name, item["pass"])) worker_names.append(worker_name) - # TODO pull out into global config - webhook_secret = read_secret_file(self.github.webhook_secret_name) eval_lock = util.MasterLock("nix-eval") for project in projects: - project.create_project_hook( - project.owner, - project.repo, - self.url, - webhook_secret, - ) + project.create_project_hook(project.owner, project.repo, self.url) config_for_project( config, project, @@ -859,11 +847,9 @@ class NixConfigurator(ConfiguratorBase): config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME)) - for backend in backends: + for backend in backends.values(): # Reload backend projects - config["builders"].append( - backend.create_reload_builder([worker_names[0]]) - ) + config["builders"].append(backend.create_reload_builder([worker_names[0]])) config["schedulers"].extend( [ schedulers.ForceScheduler( @@ -891,21 +877,24 @@ class NixConfigurator(ConfiguratorBase): config["www"]["plugins"].update(dict(base_react={})) config["www"].setdefault("change_hook_dialects", {}) - for backend in backends: - config["www"]["change_hook_dialects"][backend.change_hook_name] = \ - backend.create_change_hook(webhook_secret) + for backend in backends.values(): + config["www"]["change_hook_dialects"][backend.change_hook_name] = ( + backend.create_change_hook() + ) if "auth" not in config["www"]: config["www"].setdefault("avatar_methods", []) - for backend in backends: - config["www"]["avatar_methods"].append(backend.create_avatar_method()) + for backend in backends.values(): + avatar_method = backend.create_avatar_method() + if avatar_method is not None: + config["www"]["avatar_methods"].append(avatar_method) # TODO one cannot have multiple auth backends... - config["www"]["auth"] = backends[0].create_auth() + if auth is not None: + config["www"]["auth"] = auth config["www"]["authz"] = setup_authz( - # TODO pull out into global config - admins=self.github.admins, - backends=backends, + admins=self.admins, + backends=list(backends.values()), projects=projects, ) diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py new file mode 100644 index 0000000..6dcd4f3 --- /dev/null +++ b/buildbot_nix/common.py @@ -0,0 +1,80 @@ +import contextlib +import http.client +import json +import urllib.request +from typing import Any + + +def slugify_project_name(name: str) -> str: + return name.replace(".", "-").replace("/", "-") + + +def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: + next_url: str | None = url + items = [] + while next_url: + try: + res = http_request( + next_url, + headers={"Authorization": f"Bearer {token}"}, + ) + except OSError as e: + msg = f"failed to fetch {next_url}: {e}" + raise HttpError(msg) from e + next_url = None + link = res.headers()["Link"] + if link is not None: + links = link.split(", ") + for link in links: # pagination + link_parts = link.split(";") + if link_parts[1].strip() == 'rel="next"': + next_url = link_parts[0][1:-1] + items += res.json() + return items + + +class HttpResponse: + def __init__(self, raw: http.client.HTTPResponse) -> None: + self.raw = raw + + def json(self) -> Any: + return json.load(self.raw) + + def headers(self) -> http.client.HTTPMessage: + return self.raw.headers + + +class HttpError(Exception): + pass + + +def http_request( + url: str, + method: str = "GET", + headers: dict[str, str] | None = None, + data: dict[str, Any] | None = None, +) -> HttpResponse: + body = None + if data: + body = json.dumps(data).encode("ascii") + if headers is None: + headers = {} + headers = headers.copy() + headers["User-Agent"] = "buildbot-nix" + + if not url.startswith("https:"): + msg = "url must be https: {url}" + raise HttpError(msg) + + req = urllib.request.Request( # noqa: S310 + url, headers=headers, method=method, data=body + ) + try: + resp = urllib.request.urlopen(req) # noqa: S310 + except urllib.request.HTTPError as e: + resp_body = "" + with contextlib.suppress(OSError, UnicodeDecodeError): + resp_body = e.fp.read().decode("utf-8", "replace") + msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" + raise HttpError(msg) from e + return HttpResponse(resp) diff --git a/buildbot_nix/gitea_projects.py b/buildbot_nix/gitea_projects.py new file mode 100644 index 0000000..a52841e --- /dev/null +++ b/buildbot_nix/gitea_projects.py @@ -0,0 +1,300 @@ +import json +import os +import signal +from collections.abc import Generator +from dataclasses import dataclass +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Any + +from buildbot.config.builder import BuilderConfig +from buildbot.plugins import util +from buildbot.process.buildstep import BuildStep +from buildbot.process.properties import Interpolate +from buildbot.reporters.base import ReporterBase +from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase +from buildbot_gitea.auth import GiteaAuth # type: ignore[import] +from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import] +from twisted.internet import defer, threads +from twisted.python import log +from twisted.python.failure import Failure + +from .common import ( + http_request, + paginated_github_request, + slugify_project_name, +) +from .projects import GitBackend, GitProject +from .secrets import read_secret_file + + +@dataclass +class GiteaConfig: + instance_url: str + oauth_id: str + admins: list[str] + + oauth_secret_name: str = "gitea-oauth-secret" + token_secret_name: str = "gitea-token" + webhook_secret_name: str = "gitea-webhook-secret" + project_cache_file: Path = Path("gitea-project-cache.json") + topic: str | None = "build-with-buildbot" + + def oauth_secret(self) -> str: + return read_secret_file(self.oauth_secret_name) + + def token(self) -> str: + return read_secret_file(self.token_secret_name) + + +class GiteaProject(GitProject): + config: GiteaConfig + webhook_secret: str + data: dict[str, Any] + + def __init__( + self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any] + ) -> None: + self.config = config + self.webhook_secret = webhook_secret + self.data = data + + def create_project_hook( + self, + owner: str, + repo: str, + webhook_url: str, + ) -> None: + hooks = paginated_github_request( + f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100", + self.config.token(), + ) + config = dict( + url=webhook_url + "change_hook/gitea", + content_type="json", + insecure_ssl="0", + secret=self.webhook_secret, + ) + data = dict( + name="web", + active=True, + events=["push", "pull_request"], + config=config, + type="gitea", + ) + headers = { + "Authorization": f"token {self.config.token()}", + "Accept": "application/json", + "Content-Type": "application/json", + } + for hook in hooks: + if hook["config"]["url"] == webhook_url + "change_hook/gitea": + log.msg(f"hook for {owner}/{repo} already exists") + return + + http_request( + f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks", + method="POST", + headers=headers, + data=data, + ) + + def get_project_url(self) -> str: + return f"https://git:%(secret:{self.config.token_secret_name})s@{self.config.instance_url}/{self.name}" + + @property + def pretty_type(self) -> str: + return "Gitea" + + @property + def type(self) -> str: + return "gitea" + + @property + def repo(self) -> str: + return self.data["name"] + + @property + def owner(self) -> str: + return self.data["owner"]["login"] + + @property + def name(self) -> str: + return self.data["full_name"] + + @property + def url(self) -> str: + # not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34 + return self.data["ssh_url"] + + @property + def project_id(self) -> str: + return slugify_project_name(self.data["full_name"]) + + @property + def default_branch(self) -> str: + return self.data["default_branch"] + + @property + def topics(self) -> list[str]: + # note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects` + return self.data["topics"] + + @property + def belongs_to_org(self) -> bool: + # TODO Gitea doesn't include this information + return False # self.data["owner"]["type"] == "Organization" + + +class GiteaBackend(GitBackend): + config: GiteaConfig + + def __init__(self, config: GiteaConfig) -> None: + self.config = config + self.webhook_secret = read_secret_file(self.config.webhook_secret_name) + + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: + """Updates the flake an opens a PR for it.""" + factory = util.BuildFactory() + factory.addStep( + ReloadGiteaProjects(self.config, self.config.project_cache_file), + ) + return util.BuilderConfig( + name=self.reload_builder_name, + workernames=worker_names, + factory=factory, + ) + + def create_reporter(self) -> ReporterBase: + return GiteaStatusPush( + "https://" + self.config.instance_url, + Interpolate(self.config.token()), + context=Interpolate("buildbot/%(prop:status_name)s"), + context_pr=Interpolate("buildbot/%(prop:status_name)s"), + ) + + def create_change_hook(self) -> dict[str, Any]: + return { + "secret": self.webhook_secret, + } + + def create_avatar_method(self) -> AvatarBase | None: + return None + + def create_auth(self) -> AuthBase: + return GiteaAuth( + "https://" + self.config.instance_url, + self.config.oauth_id, + self.config.oauth_secret(), + ) + + def load_projects(self) -> list["GitProject"]: + if not self.config.project_cache_file.exists(): + return [] + + repos: list[dict[str, Any]] = sorted( + json.loads(self.config.project_cache_file.read_text()), + key=lambda x: x["full_name"], + ) + return list( + filter( + lambda project: self.config.topic is not None + and self.config.topic in project.topics, + [ + GiteaProject(self.config, self.webhook_secret, repo) + for repo in repos + ], + ) + ) + + def are_projects_cached(self) -> bool: + return self.config.project_cache_file.exists() + + @property + def type(self) -> str: + return "gitea" + + @property + def pretty_type(self) -> str: + return "Gitea" + + @property + def reload_builder_name(self) -> str: + return "reload-gitea-projects" + + @property + def change_hook_name(self) -> str: + return "gitea" + + +class ReloadGiteaProjects(BuildStep): + name = "reload_gitea_projects" + config: GiteaConfig + + def __init__( + self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any + ) -> None: + self.config = config + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + refresh_projects(self.config, self.project_cache_file) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + yield self.addLog("log").addStderr( + f"Failed to reload project list: {self.error_msg}" + ) + return util.FAILURE + + +def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None: + repos = [] + + for repo in paginated_github_request( + f"https://{config.instance_url}/api/v1/user/repos?limit=100", + config.token(), + ): + if not repo["permissions"]["admin"]: + name = repo["full_name"] + log.msg( + f"skipping {name} because we do not have admin privileges, needed for hook management", + ) + else: + try: + # Gitea doesn't include topics in the default repo listing, unlike GitHub + topics: list[str] = http_request( + f"https://{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics", + headers={"Authorization": f"token {config.token}"}, + ).json()["topics"] + repo["topics"] = topics + repos.append(repo) + except OSError: + pass + + with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: + path = Path(f.name) + try: + f.write(json.dumps(repos)) + f.flush() + path.rename(repo_cache_file) + except OSError: + path.unlink() + raise diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 3a62b3b..5323bbf 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,41 +1,33 @@ -import contextlib -import http.client import json -import urllib.request -import signal import os +import signal from collections.abc import Generator - -import typing +from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Any -from dataclasses import dataclass +from typing import Any -from twisted.python import log -from twisted.internet import defer, threads -from twisted.python.failure import Failure - -if TYPE_CHECKING: - from buildbot.process.log import StreamLog - -from buildbot.process.properties import Interpolate from buildbot.config.builder import BuilderConfig +from buildbot.plugins import util from buildbot.process.buildstep import BuildStep +from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.reporters.github import GitHubStatusPush -from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth -from buildbot.plugins import util +from twisted.internet import defer, threads +from twisted.python import log +from twisted.python.failure import Failure -from .projects import ( - GitProject, - GitBackend -) -from .secrets import ( - read_secret_file +from .common import ( + http_request, + paginated_github_request, + slugify_project_name, ) +from .projects import GitBackend, GitProject +from .secrets import read_secret_file + class ReloadGithubProjects(BuildStep): name = "reload_github_projects" @@ -65,43 +57,42 @@ class ReloadGithubProjects(BuildStep): os.kill(os.getpid(), signal.SIGHUP) return util.SUCCESS else: - log: object = yield self.addLog("log") - # TODO this assumes that log is of type StreamLog and not something else - typing.cast(StreamLog, log).addStderr(f"Failed to reload project list: {self.error_msg}") + yield self.addLog("log").addStderr( + f"Failed to reload project list: {self.error_msg}" + ) return util.FAILURE + @dataclass class GithubConfig: oauth_id: str - admins: list[str] + # TODO unused buildbot_user: str oauth_secret_name: str = "github-oauth-secret" - webhook_secret_name: str = "github-webhook-secret" token_secret_name: str = "github-token" + webhook_secret_name: str = "github-webhook-secret" project_cache_file: Path = Path("github-project-cache.json") topic: str | None = "build-with-buildbot" def token(self) -> str: return read_secret_file(self.token_secret_name) + @dataclass class GithubBackend(GitBackend): config: GithubConfig + webhook_secret: str - def __init__(self, config: GithubConfig): + def __init__(self, config: GithubConfig) -> None: self.config = config + self.webhook_secret = read_secret_file(self.config.webhook_secret_name) - def create_reload_builder( - self, - worker_names: list[str] - ) -> BuilderConfig: + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" factory = util.BuildFactory() factory.addStep( - ReloadGithubProjects( - self.config.token(), self.config.project_cache_file - ), + ReloadGithubProjects(self.config.token(), self.config.project_cache_file), ) return util.BuilderConfig( name=self.reload_builder_name, @@ -110,46 +101,50 @@ class GithubBackend(GitBackend): ) def create_reporter(self) -> ReporterBase: - return \ - GitHubStatusPush( - token=self.config.token(), - # Since we dynamically create build steps, - # we use `virtual_builder_name` in the webinterface - # so that we distinguish what has beeing build - context=Interpolate("buildbot/%(prop:status_name)s"), - ) + return GitHubStatusPush( + token=self.config.token(), + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) - def create_change_hook(self, webhook_secret: str) -> dict[str, Any]: + def create_change_hook(self) -> dict[str, Any]: return { - "secret": webhook_secret, + "secret": self.webhook_secret, "strict": True, - "token": self.config.token, + "token": self.config.token(), "github_property_whitelist": "*", } - def create_avatar_method(self) -> AvatarBase: + def create_avatar_method(self) -> AvatarBase | None: return AvatarGitHub(token=self.config.token()) def create_auth(self) -> AuthBase: - return \ - GitHubAuth( - self.config.oauth_id, - read_secret_file(self.config.oauth_secret_name), - apiVersion=4, - ) + return GitHubAuth( + self.config.oauth_id, + read_secret_file(self.config.oauth_secret_name), + apiVersion=4, + ) def load_projects(self) -> list["GitProject"]: if not self.config.project_cache_file.exists(): return [] repos: list[dict[str, Any]] = sorted( - json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"] + json.loads(self.config.project_cache_file.read_text()), + key=lambda x: x["full_name"], + ) + return list( + filter( + lambda project: self.config.topic is not None + and self.config.topic in project.topics, + [ + GithubProject(self.config, self.webhook_secret, repo) + for repo in repos + ], + ) ) - return \ - list(filter(\ - lambda project: self.config.topic != None and self.config.topic in project.topics, \ - [GithubProject(self.config, repo) for repo in repos] \ - )) def are_projects_cached(self) -> bool: return self.config.project_cache_file.exists() @@ -170,11 +165,15 @@ class GithubBackend(GitBackend): def change_hook_name(self) -> str: return "github" + class GithubProject(GitProject): config: GithubConfig - def __init__(self, config: GithubConfig, data: dict[str, Any]) -> None: + def __init__( + self, config: GithubConfig, webhook_secret: str, data: dict[str, Any] + ) -> None: self.config = config + self.webhook_secret = webhook_secret self.data = data def create_project_hook( @@ -182,7 +181,6 @@ class GithubProject(GitProject): owner: str, repo: str, webhook_url: str, - webhook_secret: str, ) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", @@ -192,9 +190,11 @@ class GithubProject(GitProject): url=webhook_url + "change_hook/github", content_type="json", insecure_ssl="0", - secret=webhook_secret, + secret=self.webhook_secret, + ) + data = dict( + name="web", active=True, events=["push", "pull_request"], config=config ) - data = dict(name="web", active=True, events=["push", "pull_request"], config=config) headers = { "Authorization": f"Bearer {self.config.token()}", "Accept": "application/vnd.github+json", @@ -256,79 +256,6 @@ class GithubProject(GitProject): def belongs_to_org(self) -> bool: return self.data["owner"]["type"] == "Organization" -class GithubError(Exception): - pass - - -class HttpResponse: - def __init__(self, raw: http.client.HTTPResponse) -> None: - self.raw = raw - - def json(self) -> Any: - return json.load(self.raw) - - def headers(self) -> http.client.HTTPMessage: - return self.raw.headers - - -def http_request( - url: str, - method: str = "GET", - headers: dict[str, str] | None = None, - data: dict[str, Any] | None = None, -) -> HttpResponse: - body = None - if data: - body = json.dumps(data).encode("ascii") - if headers is None: - headers = {} - headers = headers.copy() - headers["User-Agent"] = "buildbot-nix" - - if not url.startswith("https:"): - msg = "url must be https: {url}" - raise GithubError(msg) - - req = urllib.request.Request( # noqa: S310 - url, headers=headers, method=method, data=body - ) - try: - resp = urllib.request.urlopen(req) # noqa: S310 - except urllib.request.HTTPError as e: - resp_body = "" - with contextlib.suppress(OSError, UnicodeDecodeError): - resp_body = e.fp.read().decode("utf-8", "replace") - msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" - raise GithubError(msg) from e - return HttpResponse(resp) - - -def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: - next_url: str | None = url - items = [] - while next_url: - try: - res = http_request( - next_url, - headers={"Authorization": f"Bearer {token}"}, - ) - except OSError as e: - msg = f"failed to fetch {next_url}: {e}" - raise GithubError(msg) from e - next_url = None - link = res.headers()["Link"] - if link is not None: - links = link.split(", ") - for link in links: # pagination - link_parts = link.split(";") - if link_parts[1].strip() == 'rel="next"': - next_url = link_parts[0][1:-1] - items += res.json() - return items - - -def slugify_project_name(name: str) -> str: - return name.replace(".", "-").replace("/", "-") def refresh_projects(github_token: str, repo_cache_file: Path) -> None: repos = [] diff --git a/buildbot_nix/projects.py b/buildbot_nix/projects.py index 27e4b87..947bc1f 100644 --- a/buildbot_nix/projects.py +++ b/buildbot_nix/projects.py @@ -3,15 +3,13 @@ from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.reporters.base import ReporterBase -from buildbot.www.avatar import AvatarBase from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase + class GitBackend(ABC): @abstractmethod - def create_reload_builder( - self, - worker_names: list[str] - ) -> BuilderConfig: + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: pass @abstractmethod @@ -19,11 +17,11 @@ class GitBackend(ABC): pass @abstractmethod - def create_change_hook(self, webhook_secret: str) -> dict[str, Any]: + def create_change_hook(self) -> dict[str, Any]: pass @abstractmethod - def create_avatar_method(self) -> AvatarBase: + def create_avatar_method(self) -> AvatarBase | None: pass @abstractmethod @@ -58,6 +56,7 @@ class GitBackend(ABC): def change_hook_name(self) -> str: pass + class GitProject(ABC): @abstractmethod def create_project_hook( @@ -65,12 +64,11 @@ class GitProject(ABC): owner: str, repo: str, webhook_url: str, - webhook_secret: str, ) -> None: pass @abstractmethod - def get_project_url() -> str: + def get_project_url(self) -> str: pass @property @@ -113,7 +111,6 @@ class GitProject(ABC): def default_branch(self) -> str: pass - @property @abstractmethod def topics(self) -> list[str]: diff --git a/buildbot_nix/secrets.py b/buildbot_nix/secrets.py index 0f2ed69..44a3e00 100644 --- a/buildbot_nix/secrets.py +++ b/buildbot_nix/secrets.py @@ -1,6 +1,8 @@ -import os, sys +import os +import sys from pathlib import Path + def read_secret_file(secret_name: str) -> str: directory = os.environ.get("CREDENTIALS_DIRECTORY") if directory is None: diff --git a/buildbot_nix/util.py b/buildbot_nix/util.py new file mode 100644 index 0000000..6dcd4f3 --- /dev/null +++ b/buildbot_nix/util.py @@ -0,0 +1,80 @@ +import contextlib +import http.client +import json +import urllib.request +from typing import Any + + +def slugify_project_name(name: str) -> str: + return name.replace(".", "-").replace("/", "-") + + +def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: + next_url: str | None = url + items = [] + while next_url: + try: + res = http_request( + next_url, + headers={"Authorization": f"Bearer {token}"}, + ) + except OSError as e: + msg = f"failed to fetch {next_url}: {e}" + raise HttpError(msg) from e + next_url = None + link = res.headers()["Link"] + if link is not None: + links = link.split(", ") + for link in links: # pagination + link_parts = link.split(";") + if link_parts[1].strip() == 'rel="next"': + next_url = link_parts[0][1:-1] + items += res.json() + return items + + +class HttpResponse: + def __init__(self, raw: http.client.HTTPResponse) -> None: + self.raw = raw + + def json(self) -> Any: + return json.load(self.raw) + + def headers(self) -> http.client.HTTPMessage: + return self.raw.headers + + +class HttpError(Exception): + pass + + +def http_request( + url: str, + method: str = "GET", + headers: dict[str, str] | None = None, + data: dict[str, Any] | None = None, +) -> HttpResponse: + body = None + if data: + body = json.dumps(data).encode("ascii") + if headers is None: + headers = {} + headers = headers.copy() + headers["User-Agent"] = "buildbot-nix" + + if not url.startswith("https:"): + msg = "url must be https: {url}" + raise HttpError(msg) + + req = urllib.request.Request( # noqa: S310 + url, headers=headers, method=method, data=body + ) + try: + resp = urllib.request.urlopen(req) # noqa: S310 + except urllib.request.HTTPError as e: + resp_body = "" + with contextlib.suppress(OSError, UnicodeDecodeError): + resp_body = e.fp.read().decode("utf-8", "replace") + msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" + raise HttpError(msg) from e + return HttpResponse(resp) diff --git a/examples/master.nix b/examples/master.nix index 4ced75f..97cc2a5 100644 --- a/examples/master.nix +++ b/examples/master.nix @@ -15,6 +15,9 @@ { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 } ] ''; + # Users in this list will be able to reload the project list. + # All other user in the organization will be able to restart builds or evaluations. + admins = [ "Mic92" ]; github = { # Github user used as a CI identity user = "mic92-buildbot"; @@ -27,9 +30,6 @@ # After creating the app, press "Generate a new client secret" and fill in the client ID and secret below oauthId = "aaaaaaaaaaaaaaaaaaaa"; oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff"; - # Users in this list will be able to reload the project list. - # All other user in the organization will be able to restart builds or evaluations. - admins = [ "Mic92" ]; # All github projects with this topic will be added to buildbot. # One can trigger a project scan by visiting the Builds -> Builders page and looking for the "reload-github-project" builder. # This builder has a "Update Github Projects" button that everyone in the github organization can use. diff --git a/nix/buildbot-gitea.nix b/nix/buildbot-gitea.nix new file mode 100644 index 0000000..acaf92d --- /dev/null +++ b/nix/buildbot-gitea.nix @@ -0,0 +1,27 @@ +{ buildPythonPackage +, fetchPypi +, lib + +, pip +, buildbot +, requests +}: +buildPythonPackage (lib.fix (self: { + pname = "buildbot-gitea"; + version = "1.8.0"; + + nativeBuildInputs = [ + + ]; + + propagatedBuildInputs = [ + pip + buildbot + requests + ]; + + src = fetchPypi { + inherit (self) pname version; + hash = "sha256-zYcILPp42QuQyfEIzmYKV9vWf47sBAQI8FOKJlZ60yA="; + }; +})) diff --git a/nix/checks/master.nix b/nix/checks/master.nix index 0534b2d..d79d31e 100644 --- a/nix/checks/master.nix +++ b/nix/checks/master.nix @@ -14,13 +14,13 @@ { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 } ] ''; + admins = [ "Mic92" ]; github = { tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff"; oauthId = "aaaaaaaaaaaaaaaaaaaa"; user = "mic92-buildbot"; - admins = [ "Mic92" ]; }; }; }; diff --git a/nix/master.nix b/nix/master.nix index 8a8b64f..b88f048 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -5,8 +5,18 @@ }: let cfg = config.services.buildbot-nix.master; + inherit + (lib) + mkRenamedOptionModule + ; in { + imports = [ + (mkRenamedOptionModule + [ "services" "buildbot-nix" "master" "github" "admins" ] + [ "services" "buildbot-nix" "master" "admins" ]) + ]; + options = { services.buildbot-nix.master = { enable = lib.mkEnableOption "buildbot-master"; @@ -15,6 +25,13 @@ in default = "postgresql://@/buildbot"; description = "Postgresql database url"; }; + authBackend = lib.mkOption { + type = lib.types.enum [ "github" "gitea" "none" ]; + default = "github"; + description = '' + Which OAuth2 backend to use. + ''; + }; cachix = { name = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -34,7 +51,42 @@ in description = "Cachix auth token"; }; }; + gitea = { + enable = lib.mkEnableOption "Enable Gitea integration"; + + tokenFile = lib.mkOption { + type = lib.types.path; + description = "Gitea token file"; + }; + webhookSecretFile = lib.mkOption { + type = lib.types.path; + description = "Github webhook secret file"; + }; + oauthSecretFile = lib.mkOption { + type = lib.types.path; + description = "Gitea oauth secret file"; + }; + + instanceURL = lib.mkOption { + type = lib.types.str; + description = "Gitea instance URL"; + }; + oauthId = lib.mkOption { + type = lib.types.str; + description = "Gitea oauth id. Used for the login button"; + }; + topic = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "build-with-buildbot"; + description = '' + Projects that have this topic will be built by buildbot. + If null, all projects that the buildbot Gitea user has access to, are built. + ''; + }; + }; github = { + disable = lib.mkEnableOption "Disable GitHub integration"; + tokenFile = lib.mkOption { type = lib.types.path; description = "Github token file"; @@ -62,11 +114,6 @@ in type = lib.types.str; description = "Github user that is used for the buildbot"; }; - admins = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - description = "Users that are allowed to login to buildbot, trigger builds and change settings"; - }; topic = lib.mkOption { type = lib.types.nullOr lib.types.str; default = "build-with-buildbot"; @@ -76,6 +123,11 @@ in ''; }; }; + admins = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Users that are allowed to login to buildbot, trigger builds and change settings"; + }; workersFile = lib.mkOption { type = lib.types.path; description = "File containing a list of nix workers"; @@ -144,7 +196,7 @@ in home = "/var/lib/buildbot"; extraImports = '' from datetime import timedelta - from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig + from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig ''; configurators = [ '' @@ -152,18 +204,23 @@ in '' '' NixConfigurator( - github=GithubConfig( + auth_backend=${builtins.toJSON cfg.authBackend}, + github=${if cfg.github.disable then "None" else "GithubConfig( oauth_id=${builtins.toJSON cfg.github.oauthId}, - admins=${builtins.toJSON cfg.github.admins}, buildbot_user=${builtins.toJSON cfg.github.user}, topic=${builtins.toJSON cfg.github.topic}, - ), - gitea=None, + )"}, + gitea=${if !cfg.gitea.enable then "None" else "GiteaConfig( + instance_url=${builtins.toJSON cfg.gitea.instanceURL}, + oauth_id=${builtins.toJSON cfg.gitea.oauthId}, + topic=${builtins.toJSON cfg.gitea.topic}, + )"}, cachix=${if cfg.cachix.name == null then "None" else "CachixConfig( name=${builtins.toJSON cfg.cachix.name}, signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"}, auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"}, )"}, + admins=${builtins.toJSON cfg.admins}, url=${builtins.toJSON config.services.buildbot-master.buildbotUrl}, nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize}, nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount}, @@ -190,6 +247,7 @@ in (ps.toPythonModule pkgs.buildbot-worker) pkgs.buildbot-plugins.www-react (pkgs.python3.pkgs.callPackage ../default.nix { }) + (pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = pkgs.buildbot; }) ]; }; @@ -206,7 +264,12 @@ in ++ lib.optional (cfg.cachix.signingKeyFile != null) "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}" ++ lib.optional (cfg.cachix.authTokenFile != null) - "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"; + "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}" + ++ lib.optionals cfg.gitea.enable [ + "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}" + "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" + "gitea-token:${cfg.gitea.tokenFile}" + ]; # Needed because it tries to reach out to github on boot. # FIXME: if github is not available, we shouldn't fail buildbot, instead it should just try later again in the background