Merge pull request #124 from MagicRB/remove_github_assumption

Add support for Gitea

buildbot_nix/__init__.py

@@ -2,8 +2,6 @@ import json
 import multiprocessing
 import os
 import re
-import signal
-import sys
 import uuid
 from collections import defaultdict
 from collections.abc import Generator
@@ -11,30 +9,37 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
+from buildbot.config.builder import BuilderConfig
 from buildbot.configurators import ConfiguratorBase
 from buildbot.interfaces import WorkerSetupError
-from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
+from buildbot.locks import MasterLock
+from buildbot.plugins import schedulers, steps, util, worker
 from buildbot.process import buildstep, logobserver, remotecommand
 from buildbot.process.project import Project
-from buildbot.process.properties import Interpolate, Properties
+from buildbot.process.properties import Properties
 from buildbot.process.results import ALL_RESULTS, statusToString
+from buildbot.secrets.providers.file import SecretInAFile
 from buildbot.steps.trigger import Trigger
+from buildbot.www.authz import Authz
 from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
 
 if TYPE_CHECKING:
-    from buildbot.process.log import Log
+    from buildbot.process.log import StreamLog
+    from buildbot.www.auth import AuthBase
 
-from twisted.internet import defer, threads
+from twisted.internet import defer
 from twisted.logger import Logger
-from twisted.python.failure import Failure
 
-from .github_projects import (
+from .common import (
-    GithubProject,
-    create_project_hook,
-    load_projects,
-    refresh_projects,
     slugify_project_name,
 )
+from .gitea_projects import GiteaBackend, GiteaConfig
+from .github_projects import (
+    GithubBackend,
+    GithubConfig,
+)
+from .projects import GitBackend, GitProject
+from .secrets import read_secret_file
 
 SKIPPED_BUILDER_NAME = "skipped-builds"
 
@@ -48,8 +53,11 @@ class BuildbotNixError(Exception):
 class BuildTrigger(Trigger):
     """Dynamic trigger that creates a build for every attribute."""
 
+    project: GitProject
+
     def __init__(
         self,
+        project: GitProject,
         builds_scheduler: str,
         skipped_builds_scheduler: str,
         jobs: list[dict[str, Any]],
@@ -57,6 +65,7 @@ class BuildTrigger(Trigger):
     ) -> None:
         if "name" not in kwargs:
             kwargs["name"] = "trigger"
+        self.project = project
         self.jobs = jobs
         self.config = None
         self.builds_scheduler = builds_scheduler
@@ -78,10 +87,7 @@ class BuildTrigger(Trigger):
 
     def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]:  # noqa: N802
         build_props = self.build.getProperties()
-        repo_name = build_props.getProperty(
+        repo_name = self.project.name
-            "github.base.repo.full_name",
-            build_props.getProperty("github.repository.full_name"),
-        )
         project_id = slugify_project_name(repo_name)
         source = f"nix-eval-{project_id}"
 
@@ -145,9 +151,14 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
     every attribute.
     """
 
-    def __init__(self, supported_systems: list[str], **kwargs: Any) -> None:
+    project: GitProject
+
+    def __init__(
+        self, project: GitProject, supported_systems: list[str], **kwargs: Any
+    ) -> None:
         kwargs = self.setupShellMixin(kwargs)
         super().__init__(**kwargs)
+        self.project = project
         self.observer = logobserver.BufferLogObserver()
         self.addLogObserver("stdio", self.observer)
         self.supported_systems = supported_systems
@@ -172,11 +183,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
                         msg = f"Failed to parse line: {line}"
                         raise BuildbotNixError(msg) from e
                     jobs.append(job)
-            build_props = self.build.getProperties()
+            repo_name = self.project.name
-            repo_name = build_props.getProperty(
-                "github.base.repo.full_name",
-                build_props.getProperty("github.repository.full_name"),
-            )
             project_id = slugify_project_name(repo_name)
             filtered_jobs = []
             for job in jobs:
@@ -187,6 +194,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
             self.build.addStepsAfterCurrentStep(
                 [
                     BuildTrigger(
+                        self.project,
                         builds_scheduler=f"{project_id}-nix-build",
                         skipped_builds_scheduler=f"{project_id}-nix-skipped-build",
                         name="build flake",
@@ -224,7 +232,7 @@ class EvalErrorStep(steps.BuildStep):
         error = self.getProperty("error")
         attr = self.getProperty("attr")
         # show eval error
-        error_log: Log = yield self.addLog("nix_error")
+        error_log: StreamLog = yield self.addLog("nix_error")
         error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
         return util.FAILURE
 
@@ -256,15 +264,16 @@ class UpdateBuildOutput(steps.BuildStep):
     on the target machine.
     """
 
-    def __init__(self, path: Path, **kwargs: Any) -> None:
+    project: GitProject
+
+    def __init__(self, project: GitProject, path: Path, **kwargs: Any) -> None:
         super().__init__(**kwargs)
+        self.project = project
         self.path = path
 
     def run(self) -> Generator[Any, object, Any]:
         props = self.build.getProperties()
-        if props.getProperty("branch") != props.getProperty(
+        if props.getProperty("branch") != self.project.default_branch:
-            "github.repository.default_branch",
-        ):
             return util.SKIPPED
 
         attr = Path(props.getProperty("attr")).name
@@ -275,58 +284,6 @@ class UpdateBuildOutput(steps.BuildStep):
         return util.SUCCESS
 
 
-class ReloadGithubProjects(steps.BuildStep):
-    name = "reload_github_projects"
-
-    def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
-        self.token = token
-        self.project_cache_file = project_cache_file
-        super().__init__(**kwargs)
-
-    def reload_projects(self) -> None:
-        refresh_projects(self.token, self.project_cache_file)
-
-    @defer.inlineCallbacks
-    def run(self) -> Generator[Any, object, Any]:
-        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
-
-        self.error_msg = ""
-
-        def error_cb(failure: Failure) -> int:
-            self.error_msg += failure.getTraceback()
-            return util.FAILURE
-
-        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
-        res = yield d
-        if res == util.SUCCESS:
-            # reload the buildbot config
-            os.kill(os.getpid(), signal.SIGHUP)
-            return util.SUCCESS
-        else:
-            log: Log = yield self.addLog("log")
-            log.addStderr(f"Failed to reload project list: {self.error_msg}")
-            return util.FAILURE
-
-
-def reload_github_projects(
-    worker_names: list[str],
-    github_token_secret: str,
-    project_cache_file: Path,
-) -> util.BuilderConfig:
-    """Updates the flake an opens a PR for it."""
-    factory = util.BuildFactory()
-    factory.addStep(
-        ReloadGithubProjects(
-            github_token_secret, project_cache_file=project_cache_file
-        ),
-    )
-    return util.BuilderConfig(
-        name="reload-github-projects",
-        workernames=worker_names,
-        factory=factory,
-    )
-
-
 # GitHub somtimes fires the PR webhook before it has computed the merge commit
 # This is a workaround to fetch the merge commit and checkout the PR branch in CI
 class GitLocalPrMerge(steps.Git):
@@ -347,8 +304,7 @@ class GitLocalPrMerge(steps.Git):
             return res
 
         # The code below is a modified version of Git.run_vc
-        self.stdio_log: Log = yield self.addLogForRemoteCommands("stdio")
+        self.stdio_log: StreamLog = yield self.addLogForRemoteCommands("stdio")
-
         self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n")
 
         git_installed = yield self.checkFeatureSupport()
@@ -392,22 +348,20 @@ class GitLocalPrMerge(steps.Git):
 
 
 def nix_eval_config(
-    project: GithubProject,
+    project: GitProject,
     worker_names: list[str],
-    github_token_secret: str,
+    git_url: str,
     supported_systems: list[str],
-    eval_lock: util.MasterLock,
+    eval_lock: MasterLock,
     worker_count: int,
     max_memory_size: int,
-) -> util.BuilderConfig:
+) -> BuilderConfig:
     """Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
     For each evaluated attribute a new build pipeline is started.
     """
     factory = util.BuildFactory()
     # check out the source
-    url_with_secret = util.Interpolate(
+    url_with_secret = util.Interpolate(git_url)
-        f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s",
-    )
     factory.addStep(
         GitLocalPrMerge(
             repourl=url_with_secret,
@@ -422,6 +376,7 @@ def nix_eval_config(
 
     factory.addStep(
         NixEvalCommand(
+            project=project,
             env={},
             name="evaluate flake",
             supported_systems=supported_systems,
@@ -482,11 +437,11 @@ class CachixConfig:
 
 
 def nix_build_config(
-    project: GithubProject,
+    project: GitProject,
     worker_names: list[str],
     cachix: CachixConfig | None = None,
     outputs_path: Path | None = None,
-) -> util.BuilderConfig:
+) -> BuilderConfig:
     """Builds one nix flake attribute."""
     factory = util.BuildFactory()
     factory.addStep(
@@ -542,8 +497,7 @@ def nix_build_config(
                 "-r",
                 util.Property("out_path"),
             ],
-            doStepIf=lambda s: s.getProperty("branch")
+            doStepIf=lambda s: s.getProperty("branch") == project.default_branch,
-            == s.getProperty("github.repository.default_branch"),
         ),
     )
     factory.addStep(
@@ -555,6 +509,7 @@ def nix_build_config(
     if outputs_path is not None:
         factory.addStep(
             UpdateBuildOutput(
+                project=project,
                 name="Update build output",
                 path=outputs_path,
             ),
@@ -570,9 +525,9 @@ def nix_build_config(
 
 
 def nix_skipped_build_config(
-    project: GithubProject,
+    project: GitProject,
     worker_names: list[str],
-) -> util.BuilderConfig:
+) -> BuilderConfig:
     """Dummy builder that is triggered when a build is skipped."""
     factory = util.BuildFactory()
     factory.addStep(
@@ -601,42 +556,17 @@ def nix_skipped_build_config(
     )
 
 
-def read_secret_file(secret_name: str) -> str:
-    directory = os.environ.get("CREDENTIALS_DIRECTORY")
-    if directory is None:
-        print("directory not set", file=sys.stderr)
-        sys.exit(1)
-    return Path(directory).joinpath(secret_name).read_text().rstrip()
-
-
-@dataclass
-class GithubConfig:
-    oauth_id: str
-    admins: list[str]
-
-    buildbot_user: str
-    oauth_secret_name: str = "github-oauth-secret"
-    webhook_secret_name: str = "github-webhook-secret"
-    token_secret_name: str = "github-token"
-    project_cache_file: Path = Path("github-project-cache.json")
-    topic: str | None = "build-with-buildbot"
-
-    def token(self) -> str:
-        return read_secret_file(self.token_secret_name)
-
-
 def config_for_project(
     config: dict[str, Any],
-    project: GithubProject,
+    project: GitProject,
     worker_names: list[str],
-    github: GithubConfig,
     nix_supported_systems: list[str],
     nix_eval_worker_count: int,
     nix_eval_max_memory_size: int,
-    eval_lock: util.MasterLock,
+    eval_lock: MasterLock,
     cachix: CachixConfig | None = None,
     outputs_path: Path | None = None,
-) -> Project:
+) -> None:
     config["projects"].append(Project(project.name))
     config["schedulers"].extend(
         [
@@ -644,8 +574,7 @@ def config_for_project(
                 name=f"{project.project_id}-default-branch",
                 change_filter=util.ChangeFilter(
                     repository=project.url,
-                    filter_fn=lambda c: c.branch
+                    filter_fn=lambda c: c.branch == project.default_branch,
-                    == c.properties.getProperty("github.repository.default_branch"),
                 ),
                 builderNames=[f"{project.name}/nix-eval"],
                 treeStableTimer=5,
@@ -685,7 +614,7 @@ def config_for_project(
                 properties=[
                     util.StringParameter(
                         name="project",
-                        label="Name of the GitHub repository.",
+                        label=f"Name of the {project.pretty_type} repository.",
                         default=project.name,
                     ),
                 ],
@@ -699,7 +628,7 @@ def config_for_project(
             nix_eval_config(
                 project,
                 worker_names,
-                github_token_secret=github.token_secret_name,
+                git_url=project.get_project_url(),
                 supported_systems=nix_supported_systems,
                 worker_count=nix_eval_worker_count,
                 max_memory_size=nix_eval_max_memory_size,
@@ -717,9 +646,9 @@ def config_for_project(
 
 
 def normalize_virtual_builder_name(name: str) -> str:
-    if name.startswith("github:"):
+    if re.match(r"^[^:]+:", name) is not None:
         # rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build
-        match = re.match(r"github:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
+        match = re.match(r"[^:]:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
         if match:
             return f"{match['owner']}/{match['repo']}/nix-build"
 
@@ -787,10 +716,12 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase):
         return self.check_builder(epobject, epdict, "buildrequest")
 
 
-def setup_authz(projects: list[GithubProject], admins: list[str]) -> util.Authz:
+def setup_authz(
+    backends: list[GitBackend], projects: list[GitProject], admins: list[str]
+) -> Authz:
     allow_rules = []
     allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict(
-        lambda: {"reload-github-projects"},
+        lambda: {backend.reload_builder_name for backend in backends},
     )
 
     for project in projects:
@@ -836,7 +767,10 @@ class NixConfigurator(ConfiguratorBase):
     def __init__(
         self,
         # Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
-        github: GithubConfig,
+        admins: list[str],
+        auth_backend: str,
+        github: GithubConfig | None,
+        gitea: GiteaConfig | None,
         url: str,
         nix_supported_systems: list[str],
         nix_eval_worker_count: int | None,
@@ -850,7 +784,10 @@ class NixConfigurator(ConfiguratorBase):
         self.nix_eval_max_memory_size = nix_eval_max_memory_size
         self.nix_eval_worker_count = nix_eval_worker_count
         self.nix_supported_systems = nix_supported_systems
+        self.auth_backend = auth_backend
+        self.admins = admins
         self.github = github
+        self.gitea = gitea
         self.url = url
         self.cachix = cachix
         if outputs_path is None:
@@ -859,9 +796,25 @@ class NixConfigurator(ConfiguratorBase):
             self.outputs_path = Path(outputs_path)
 
     def configure(self, config: dict[str, Any]) -> None:
-        projects = load_projects(self.github.token(), self.github.project_cache_file)
+        backends: dict[str, GitBackend] = {}
-        if self.github.topic is not None:
+
-            projects = [p for p in projects if self.github.topic in p.topics]
+        if self.github is not None:
+            backends["github"] = GithubBackend(self.github)
+
+        if self.gitea is not None:
+            backends["gitea"] = GiteaBackend(self.gitea)
+
+        auth: AuthBase | None = (
+            backends[self.auth_backend].create_auth()
+            if self.auth_backend != "none"
+            else None
+        )
+
+        projects: list[GitProject] = []
+
+        for backend in backends.values():
+            projects += backend.load_projects()
+
         worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
         worker_names = []
 
@@ -876,22 +829,14 @@ class NixConfigurator(ConfiguratorBase):
                 config["workers"].append(worker.Worker(worker_name, item["pass"]))
                 worker_names.append(worker_name)
 
-        webhook_secret = read_secret_file(self.github.webhook_secret_name)
         eval_lock = util.MasterLock("nix-eval")
 
         for project in projects:
-            create_project_hook(
+            project.create_project_hook(project.owner, project.repo, self.url)
-                project.owner,
-                project.repo,
-                self.github.token(),
-                self.url + "change_hook/github",
-                webhook_secret,
-            )
             config_for_project(
                 config,
                 project,
                 worker_names,
-                self.github,
                 self.nix_supported_systems,
                 self.nix_eval_worker_count or multiprocessing.cpu_count(),
                 self.nix_eval_max_memory_size,
@@ -900,42 +845,30 @@ class NixConfigurator(ConfiguratorBase):
                 self.outputs_path,
             )
 
-        # Reload github projects
-        config["builders"].append(
-            reload_github_projects(
-                [worker_names[0]],
-                self.github.token(),
-                self.github.project_cache_file,
-            ),
-        )
         config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
+
+        for backend in backends.values():
+            # Reload backend projects
+            config["builders"].append(backend.create_reload_builder([worker_names[0]]))
             config["schedulers"].extend(
                 [
                     schedulers.ForceScheduler(
-                    name="reload-github-projects",
+                        name=f"reload-{backend.type}-projects",
-                    builderNames=["reload-github-projects"],
+                        builderNames=[backend.reload_builder_name],
                         buttonName="Update projects",
                     ),
                     # project list twice a day and on startup
                     PeriodicWithStartup(
-                    name="reload-github-projects-bidaily",
+                        name=f"reload-{backend.type}-projects-bidaily",
-                    builderNames=["reload-github-projects"],
+                        builderNames=[backend.reload_builder_name],
                         periodicBuildTimer=12 * 60 * 60,
-                    run_on_startup=not self.github.project_cache_file.exists(),
+                        run_on_startup=not backend.are_projects_cached(),
                     ),
                 ],
             )
-        config["services"].append(
+            config["services"].append(backend.create_reporter())
-            reporters.GitHubStatusPush(
-                token=self.github.token(),
-                # Since we dynamically create build steps,
-                # we use `virtual_builder_name` in the webinterface
-                # so that we distinguish what has beeing build
-                context=Interpolate("buildbot/%(prop:status_name)s"),
-            ),
-        )
 
-        systemd_secrets = secrets.SecretInAFile(
+        systemd_secrets = SecretInAFile(
             dirname=os.environ["CREDENTIALS_DIRECTORY"],
         )
         config["secretsProviders"].append(systemd_secrets)
@@ -944,25 +877,24 @@ class NixConfigurator(ConfiguratorBase):
         config["www"]["plugins"].update(dict(base_react={}))
 
         config["www"].setdefault("change_hook_dialects", {})
-        config["www"]["change_hook_dialects"]["github"] = {
+        for backend in backends.values():
-            "secret": webhook_secret,
+            config["www"]["change_hook_dialects"][backend.change_hook_name] = (
-            "strict": True,
+                backend.create_change_hook()
-            "token": self.github.token(),
+            )
-            "github_property_whitelist": "*",
-        }
 
         if "auth" not in config["www"]:
             config["www"].setdefault("avatar_methods", [])
-            config["www"]["avatar_methods"].append(
+
-                util.AvatarGitHub(token=self.github.token()),
+            for backend in backends.values():
-            )
+                avatar_method = backend.create_avatar_method()
-            config["www"]["auth"] = util.GitHubAuth(
+                if avatar_method is not None:
-                self.github.oauth_id,
+                    config["www"]["avatar_methods"].append(avatar_method)
-                read_secret_file(self.github.oauth_secret_name),
+            # TODO one cannot have multiple auth backends...
-                apiVersion=4,
+            if auth is not None:
-            )
+                config["www"]["auth"] = auth
 
             config["www"]["authz"] = setup_authz(
-                admins=self.github.admins,
+                admins=self.admins,
+                backends=list(backends.values()),
                 projects=projects,
             )

buildbot_nix/common.py

@@ -0,0 +1,80 @@
+import contextlib
+import http.client
+import json
+import urllib.request
+from typing import Any
+
+
+def slugify_project_name(name: str) -> str:
+    return name.replace(".", "-").replace("/", "-")
+
+
+def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
+    next_url: str | None = url
+    items = []
+    while next_url:
+        try:
+            res = http_request(
+                next_url,
+                headers={"Authorization": f"Bearer {token}"},
+            )
+        except OSError as e:
+            msg = f"failed to fetch {next_url}: {e}"
+            raise HttpError(msg) from e
+        next_url = None
+        link = res.headers()["Link"]
+        if link is not None:
+            links = link.split(", ")
+            for link in links:  # pagination
+                link_parts = link.split(";")
+                if link_parts[1].strip() == 'rel="next"':
+                    next_url = link_parts[0][1:-1]
+        items += res.json()
+    return items
+
+
+class HttpResponse:
+    def __init__(self, raw: http.client.HTTPResponse) -> None:
+        self.raw = raw
+
+    def json(self) -> Any:
+        return json.load(self.raw)
+
+    def headers(self) -> http.client.HTTPMessage:
+        return self.raw.headers
+
+
+class HttpError(Exception):
+    pass
+
+
+def http_request(
+    url: str,
+    method: str = "GET",
+    headers: dict[str, str] | None = None,
+    data: dict[str, Any] | None = None,
+) -> HttpResponse:
+    body = None
+    if data:
+        body = json.dumps(data).encode("ascii")
+    if headers is None:
+        headers = {}
+    headers = headers.copy()
+    headers["User-Agent"] = "buildbot-nix"
+
+    if not url.startswith("https:"):
+        msg = "url must be https: {url}"
+        raise HttpError(msg)
+
+    req = urllib.request.Request(  # noqa: S310
+        url, headers=headers, method=method, data=body
+    )
+    try:
+        resp = urllib.request.urlopen(req)  # noqa: S310
+    except urllib.request.HTTPError as e:
+        resp_body = ""
+        with contextlib.suppress(OSError, UnicodeDecodeError):
+            resp_body = e.fp.read().decode("utf-8", "replace")
+        msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
+        raise HttpError(msg) from e
+    return HttpResponse(resp)

buildbot_nix/gitea_projects.py

@@ -0,0 +1,300 @@
+import json
+import os
+import signal
+from collections.abc import Generator
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Any
+
+from buildbot.config.builder import BuilderConfig
+from buildbot.plugins import util
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Interpolate
+from buildbot.reporters.base import ReporterBase
+from buildbot.www.auth import AuthBase
+from buildbot.www.avatar import AvatarBase
+from buildbot_gitea.auth import GiteaAuth  # type: ignore[import]
+from buildbot_gitea.reporter import GiteaStatusPush  # type: ignore[import]
+from twisted.internet import defer, threads
+from twisted.python import log
+from twisted.python.failure import Failure
+
+from .common import (
+    http_request,
+    paginated_github_request,
+    slugify_project_name,
+)
+from .projects import GitBackend, GitProject
+from .secrets import read_secret_file
+
+
+@dataclass
+class GiteaConfig:
+    instance_url: str
+    oauth_id: str
+    admins: list[str]
+
+    oauth_secret_name: str = "gitea-oauth-secret"
+    token_secret_name: str = "gitea-token"
+    webhook_secret_name: str = "gitea-webhook-secret"
+    project_cache_file: Path = Path("gitea-project-cache.json")
+    topic: str | None = "build-with-buildbot"
+
+    def oauth_secret(self) -> str:
+        return read_secret_file(self.oauth_secret_name)
+
+    def token(self) -> str:
+        return read_secret_file(self.token_secret_name)
+
+
+class GiteaProject(GitProject):
+    config: GiteaConfig
+    webhook_secret: str
+    data: dict[str, Any]
+
+    def __init__(
+        self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any]
+    ) -> None:
+        self.config = config
+        self.webhook_secret = webhook_secret
+        self.data = data
+
+    def create_project_hook(
+        self,
+        owner: str,
+        repo: str,
+        webhook_url: str,
+    ) -> None:
+        hooks = paginated_github_request(
+            f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
+            self.config.token(),
+        )
+        config = dict(
+            url=webhook_url + "change_hook/gitea",
+            content_type="json",
+            insecure_ssl="0",
+            secret=self.webhook_secret,
+        )
+        data = dict(
+            name="web",
+            active=True,
+            events=["push", "pull_request"],
+            config=config,
+            type="gitea",
+        )
+        headers = {
+            "Authorization": f"token {self.config.token()}",
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        for hook in hooks:
+            if hook["config"]["url"] == webhook_url + "change_hook/gitea":
+                log.msg(f"hook for {owner}/{repo} already exists")
+                return
+
+        http_request(
+            f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks",
+            method="POST",
+            headers=headers,
+            data=data,
+        )
+
+    def get_project_url(self) -> str:
+        return f"https://git:%(secret:{self.config.token_secret_name})s@{self.config.instance_url}/{self.name}"
+
+    @property
+    def pretty_type(self) -> str:
+        return "Gitea"
+
+    @property
+    def type(self) -> str:
+        return "gitea"
+
+    @property
+    def repo(self) -> str:
+        return self.data["name"]
+
+    @property
+    def owner(self) -> str:
+        return self.data["owner"]["login"]
+
+    @property
+    def name(self) -> str:
+        return self.data["full_name"]
+
+    @property
+    def url(self) -> str:
+        # not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34
+        return self.data["ssh_url"]
+
+    @property
+    def project_id(self) -> str:
+        return slugify_project_name(self.data["full_name"])
+
+    @property
+    def default_branch(self) -> str:
+        return self.data["default_branch"]
+
+    @property
+    def topics(self) -> list[str]:
+        # note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects`
+        return self.data["topics"]
+
+    @property
+    def belongs_to_org(self) -> bool:
+        # TODO Gitea doesn't include this information
+        return False  # self.data["owner"]["type"] == "Organization"
+
+
+class GiteaBackend(GitBackend):
+    config: GiteaConfig
+
+    def __init__(self, config: GiteaConfig) -> None:
+        self.config = config
+        self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
+
+    def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
+        """Updates the flake an opens a PR for it."""
+        factory = util.BuildFactory()
+        factory.addStep(
+            ReloadGiteaProjects(self.config, self.config.project_cache_file),
+        )
+        return util.BuilderConfig(
+            name=self.reload_builder_name,
+            workernames=worker_names,
+            factory=factory,
+        )
+
+    def create_reporter(self) -> ReporterBase:
+        return GiteaStatusPush(
+            "https://" + self.config.instance_url,
+            Interpolate(self.config.token()),
+            context=Interpolate("buildbot/%(prop:status_name)s"),
+            context_pr=Interpolate("buildbot/%(prop:status_name)s"),
+        )
+
+    def create_change_hook(self) -> dict[str, Any]:
+        return {
+            "secret": self.webhook_secret,
+        }
+
+    def create_avatar_method(self) -> AvatarBase | None:
+        return None
+
+    def create_auth(self) -> AuthBase:
+        return GiteaAuth(
+            "https://" + self.config.instance_url,
+            self.config.oauth_id,
+            self.config.oauth_secret(),
+        )
+
+    def load_projects(self) -> list["GitProject"]:
+        if not self.config.project_cache_file.exists():
+            return []
+
+        repos: list[dict[str, Any]] = sorted(
+            json.loads(self.config.project_cache_file.read_text()),
+            key=lambda x: x["full_name"],
+        )
+        return list(
+            filter(
+                lambda project: self.config.topic is not None
+                and self.config.topic in project.topics,
+                [
+                    GiteaProject(self.config, self.webhook_secret, repo)
+                    for repo in repos
+                ],
+            )
+        )
+
+    def are_projects_cached(self) -> bool:
+        return self.config.project_cache_file.exists()
+
+    @property
+    def type(self) -> str:
+        return "gitea"
+
+    @property
+    def pretty_type(self) -> str:
+        return "Gitea"
+
+    @property
+    def reload_builder_name(self) -> str:
+        return "reload-gitea-projects"
+
+    @property
+    def change_hook_name(self) -> str:
+        return "gitea"
+
+
+class ReloadGiteaProjects(BuildStep):
+    name = "reload_gitea_projects"
+    config: GiteaConfig
+
+    def __init__(
+        self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any
+    ) -> None:
+        self.config = config
+        self.project_cache_file = project_cache_file
+        super().__init__(**kwargs)
+
+    def reload_projects(self) -> None:
+        refresh_projects(self.config, self.project_cache_file)
+
+    @defer.inlineCallbacks
+    def run(self) -> Generator[Any, object, Any]:
+        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
+
+        self.error_msg = ""
+
+        def error_cb(failure: Failure) -> int:
+            self.error_msg += failure.getTraceback()
+            return util.FAILURE
+
+        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
+        res = yield d
+        if res == util.SUCCESS:
+            # reload the buildbot config
+            os.kill(os.getpid(), signal.SIGHUP)
+            return util.SUCCESS
+        else:
+            yield self.addLog("log").addStderr(
+                f"Failed to reload project list: {self.error_msg}"
+            )
+            return util.FAILURE
+
+
+def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
+    repos = []
+
+    for repo in paginated_github_request(
+        f"https://{config.instance_url}/api/v1/user/repos?limit=100",
+        config.token(),
+    ):
+        if not repo["permissions"]["admin"]:
+            name = repo["full_name"]
+            log.msg(
+                f"skipping {name} because we do not have admin privileges, needed for hook management",
+            )
+        else:
+            try:
+                # Gitea doesn't include topics in the default repo listing, unlike GitHub
+                topics: list[str] = http_request(
+                    f"https://{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics",
+                    headers={"Authorization": f"token {config.token}"},
+                ).json()["topics"]
+                repo["topics"] = topics
+                repos.append(repo)
+            except OSError:
+                pass
+
+    with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
+        path = Path(f.name)
+        try:
+            f.write(json.dumps(repos))
+            f.flush()
+            path.rename(repo_cache_file)
+        except OSError:
+            path.unlink()
+            raise

buildbot_nix/github_projects.py

@@ -1,93 +1,229 @@
-import contextlib
-import http.client
 import json
-import urllib.request
+import os
+import signal
+from collections.abc import Generator
+from dataclasses import dataclass
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Any
 
+from buildbot.config.builder import BuilderConfig
+from buildbot.plugins import util
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Interpolate
+from buildbot.reporters.base import ReporterBase
+from buildbot.reporters.github import GitHubStatusPush
+from buildbot.www.auth import AuthBase
+from buildbot.www.avatar import AvatarBase, AvatarGitHub
+from buildbot.www.oauth2 import GitHubAuth
+from twisted.internet import defer, threads
 from twisted.python import log
+from twisted.python.failure import Failure
+
+from .common import (
+    http_request,
+    paginated_github_request,
+    slugify_project_name,
+)
+from .projects import GitBackend, GitProject
+from .secrets import read_secret_file
 
 
-class GithubError(Exception):
+class ReloadGithubProjects(BuildStep):
-    pass
+    name = "reload_github_projects"
 
+    def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
+        self.token = token
+        self.project_cache_file = project_cache_file
+        super().__init__(**kwargs)
 
-class HttpResponse:
+    def reload_projects(self) -> None:
-    def __init__(self, raw: http.client.HTTPResponse) -> None:
+        refresh_projects(self.token, self.project_cache_file)
-        self.raw = raw
 
-    def json(self) -> Any:
+    @defer.inlineCallbacks
-        return json.load(self.raw)
+    def run(self) -> Generator[Any, object, Any]:
+        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
 
-    def headers(self) -> http.client.HTTPMessage:
+        self.error_msg = ""
-        return self.raw.headers
 
+        def error_cb(failure: Failure) -> int:
+            self.error_msg += failure.getTraceback()
+            return util.FAILURE
 
-def http_request(
+        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
-    url: str,
+        res = yield d
-    method: str = "GET",
+        if res == util.SUCCESS:
-    headers: dict[str, str] | None = None,
+            # reload the buildbot config
-    data: dict[str, Any] | None = None,
+            os.kill(os.getpid(), signal.SIGHUP)
-) -> HttpResponse:
+            return util.SUCCESS
-    body = None
+        else:
-    if data:
+            yield self.addLog("log").addStderr(
-        body = json.dumps(data).encode("ascii")
+                f"Failed to reload project list: {self.error_msg}"
-    if headers is None:
-        headers = {}
-    headers = headers.copy()
-    headers["User-Agent"] = "buildbot-nix"
-
-    if not url.startswith("https:"):
-        msg = "url must be https: {url}"
-        raise GithubError(msg)
-
-    req = urllib.request.Request(  # noqa: S310
-        url, headers=headers, method=method, data=body
             )
-    try:
+            return util.FAILURE
-        resp = urllib.request.urlopen(req)  # noqa: S310
-    except urllib.request.HTTPError as e:
-        resp_body = ""
-        with contextlib.suppress(OSError, UnicodeDecodeError):
-            resp_body = e.fp.read().decode("utf-8", "replace")
-        msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
-        raise GithubError(msg) from e
-    return HttpResponse(resp)
 
 
-def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
+@dataclass
-    next_url: str | None = url
+class GithubConfig:
-    items = []
+    oauth_id: str
-    while next_url:
+
-        try:
+    # TODO unused
-            res = http_request(
+    buildbot_user: str
-                next_url,
+    oauth_secret_name: str = "github-oauth-secret"
-                headers={"Authorization": f"Bearer {token}"},
+    token_secret_name: str = "github-token"
+    webhook_secret_name: str = "github-webhook-secret"
+    project_cache_file: Path = Path("github-project-cache.json")
+    topic: str | None = "build-with-buildbot"
+
+    def token(self) -> str:
+        return read_secret_file(self.token_secret_name)
+
+
+@dataclass
+class GithubBackend(GitBackend):
+    config: GithubConfig
+    webhook_secret: str
+
+    def __init__(self, config: GithubConfig) -> None:
+        self.config = config
+        self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
+
+    def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
+        """Updates the flake an opens a PR for it."""
+        factory = util.BuildFactory()
+        factory.addStep(
+            ReloadGithubProjects(self.config.token(), self.config.project_cache_file),
         )
-        except OSError as e:
+        return util.BuilderConfig(
-            msg = f"failed to fetch {next_url}: {e}"
+            name=self.reload_builder_name,
-            raise GithubError(msg) from e
+            workernames=worker_names,
-        next_url = None
+            factory=factory,
-        link = res.headers()["Link"]
+        )
-        if link is not None:
+
-            links = link.split(", ")
+    def create_reporter(self) -> ReporterBase:
-            for link in links:  # pagination
+        return GitHubStatusPush(
-                link_parts = link.split(";")
+            token=self.config.token(),
-                if link_parts[1].strip() == 'rel="next"':
+            # Since we dynamically create build steps,
-                    next_url = link_parts[0][1:-1]
+            # we use `virtual_builder_name` in the webinterface
-        items += res.json()
+            # so that we distinguish what has beeing build
-    return items
+            context=Interpolate("buildbot/%(prop:status_name)s"),
+        )
+
+    def create_change_hook(self) -> dict[str, Any]:
+        return {
+            "secret": self.webhook_secret,
+            "strict": True,
+            "token": self.config.token(),
+            "github_property_whitelist": "*",
+        }
+
+    def create_avatar_method(self) -> AvatarBase | None:
+        return AvatarGitHub(token=self.config.token())
+
+    def create_auth(self) -> AuthBase:
+        return GitHubAuth(
+            self.config.oauth_id,
+            read_secret_file(self.config.oauth_secret_name),
+            apiVersion=4,
+        )
+
+    def load_projects(self) -> list["GitProject"]:
+        if not self.config.project_cache_file.exists():
+            return []
+
+        repos: list[dict[str, Any]] = sorted(
+            json.loads(self.config.project_cache_file.read_text()),
+            key=lambda x: x["full_name"],
+        )
+        return list(
+            filter(
+                lambda project: self.config.topic is not None
+                and self.config.topic in project.topics,
+                [
+                    GithubProject(self.config, self.webhook_secret, repo)
+                    for repo in repos
+                ],
+            )
+        )
+
+    def are_projects_cached(self) -> bool:
+        return self.config.project_cache_file.exists()
+
+    @property
+    def type(self) -> str:
+        return "github"
+
+    @property
+    def pretty_type(self) -> str:
+        return "GitHub"
+
+    @property
+    def reload_builder_name(self) -> str:
+        return "reload-github-projects"
+
+    @property
+    def change_hook_name(self) -> str:
+        return "github"
 
 
-def slugify_project_name(name: str) -> str:
+class GithubProject(GitProject):
-    return name.replace(".", "-").replace("/", "-")
+    config: GithubConfig
 
-
+    def __init__(
-class GithubProject:
+        self, config: GithubConfig, webhook_secret: str, data: dict[str, Any]
-    def __init__(self, data: dict[str, Any]) -> None:
+    ) -> None:
+        self.config = config
+        self.webhook_secret = webhook_secret
         self.data = data
 
+    def create_project_hook(
+        self,
+        owner: str,
+        repo: str,
+        webhook_url: str,
+    ) -> None:
+        hooks = paginated_github_request(
+            f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
+            self.config.token(),
+        )
+        config = dict(
+            url=webhook_url + "change_hook/github",
+            content_type="json",
+            insecure_ssl="0",
+            secret=self.webhook_secret,
+        )
+        data = dict(
+            name="web", active=True, events=["push", "pull_request"], config=config
+        )
+        headers = {
+            "Authorization": f"Bearer {self.config.token()}",
+            "Accept": "application/vnd.github+json",
+            "Content-Type": "application/json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+        for hook in hooks:
+            if hook["config"]["url"] == webhook_url + "change_hook/github":
+                log.msg(f"hook for {owner}/{repo} already exists")
+                return
+
+        http_request(
+            f"https://api.github.com/repos/{owner}/{repo}/hooks",
+            method="POST",
+            headers=headers,
+            data=data,
+        )
+
+    def get_project_url(self) -> str:
+        return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}"
+
+    @property
+    def pretty_type(self) -> str:
+        return "GitHub"
+
+    @property
+    def type(self) -> str:
+        return "github"
+
     @property
     def repo(self) -> str:
         return self.data["name"]
@@ -121,43 +257,6 @@ class GithubProject:
         return self.data["owner"]["type"] == "Organization"
 
 
-def create_project_hook(
-    owner: str,
-    repo: str,
-    token: str,
-    webhook_url: str,
-    webhook_secret: str,
-) -> None:
-    hooks = paginated_github_request(
-        f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
-        token,
-    )
-    config = dict(
-        url=webhook_url,
-        content_type="json",
-        insecure_ssl="0",
-        secret=webhook_secret,
-    )
-    data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github+json",
-        "Content-Type": "application/json",
-        "X-GitHub-Api-Version": "2022-11-28",
-    }
-    for hook in hooks:
-        if hook["config"]["url"] == webhook_url:
-            log.msg(f"hook for {owner}/{repo} already exists")
-            return
-
-    http_request(
-        f"https://api.github.com/repos/{owner}/{repo}/hooks",
-        method="POST",
-        headers=headers,
-        data=data,
-    )
-
-
 def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
     repos = []
 
@@ -182,13 +281,3 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
         except OSError:
             path.unlink()
             raise
-
-
-def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]:
-    if not repo_cache_file.exists():
-        return []
-
-    repos: list[dict[str, Any]] = sorted(
-        json.loads(repo_cache_file.read_text()), key=lambda x: x["full_name"]
-    )
-    return [GithubProject(repo) for repo in repos]

buildbot_nix/projects.py

@@ -0,0 +1,122 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from buildbot.config.builder import BuilderConfig
+from buildbot.reporters.base import ReporterBase
+from buildbot.www.auth import AuthBase
+from buildbot.www.avatar import AvatarBase
+
+
+class GitBackend(ABC):
+    @abstractmethod
+    def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
+        pass
+
+    @abstractmethod
+    def create_reporter(self) -> ReporterBase:
+        pass
+
+    @abstractmethod
+    def create_change_hook(self) -> dict[str, Any]:
+        pass
+
+    @abstractmethod
+    def create_avatar_method(self) -> AvatarBase | None:
+        pass
+
+    @abstractmethod
+    def create_auth(self) -> AuthBase:
+        pass
+
+    @abstractmethod
+    def load_projects(self) -> list["GitProject"]:
+        pass
+
+    @abstractmethod
+    def are_projects_cached(self) -> bool:
+        pass
+
+    @property
+    @abstractmethod
+    def pretty_type(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def type(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def reload_builder_name(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def change_hook_name(self) -> str:
+        pass
+
+
+class GitProject(ABC):
+    @abstractmethod
+    def create_project_hook(
+        self,
+        owner: str,
+        repo: str,
+        webhook_url: str,
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def get_project_url(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def pretty_type(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def type(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def repo(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def owner(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def url(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def project_id(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def default_branch(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def topics(self) -> list[str]:
+        pass
+
+    @property
+    @abstractmethod
+    def belongs_to_org(self) -> bool:
+        pass

buildbot_nix/secrets.py

@@ -0,0 +1,11 @@
+import os
+import sys
+from pathlib import Path
+
+
+def read_secret_file(secret_name: str) -> str:
+    directory = os.environ.get("CREDENTIALS_DIRECTORY")
+    if directory is None:
+        print("directory not set", file=sys.stderr)
+        sys.exit(1)
+    return Path(directory).joinpath(secret_name).read_text().rstrip()

buildbot_nix/util.py

@@ -0,0 +1,80 @@
+import contextlib
+import http.client
+import json
+import urllib.request
+from typing import Any
+
+
+def slugify_project_name(name: str) -> str:
+    return name.replace(".", "-").replace("/", "-")
+
+
+def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
+    next_url: str | None = url
+    items = []
+    while next_url:
+        try:
+            res = http_request(
+                next_url,
+                headers={"Authorization": f"Bearer {token}"},
+            )
+        except OSError as e:
+            msg = f"failed to fetch {next_url}: {e}"
+            raise HttpError(msg) from e
+        next_url = None
+        link = res.headers()["Link"]
+        if link is not None:
+            links = link.split(", ")
+            for link in links:  # pagination
+                link_parts = link.split(";")
+                if link_parts[1].strip() == 'rel="next"':
+                    next_url = link_parts[0][1:-1]
+        items += res.json()
+    return items
+
+
+class HttpResponse:
+    def __init__(self, raw: http.client.HTTPResponse) -> None:
+        self.raw = raw
+
+    def json(self) -> Any:
+        return json.load(self.raw)
+
+    def headers(self) -> http.client.HTTPMessage:
+        return self.raw.headers
+
+
+class HttpError(Exception):
+    pass
+
+
+def http_request(
+    url: str,
+    method: str = "GET",
+    headers: dict[str, str] | None = None,
+    data: dict[str, Any] | None = None,
+) -> HttpResponse:
+    body = None
+    if data:
+        body = json.dumps(data).encode("ascii")
+    if headers is None:
+        headers = {}
+    headers = headers.copy()
+    headers["User-Agent"] = "buildbot-nix"
+
+    if not url.startswith("https:"):
+        msg = "url must be https: {url}"
+        raise HttpError(msg)
+
+    req = urllib.request.Request(  # noqa: S310
+        url, headers=headers, method=method, data=body
+    )
+    try:
+        resp = urllib.request.urlopen(req)  # noqa: S310
+    except urllib.request.HTTPError as e:
+        resp_body = ""
+        with contextlib.suppress(OSError, UnicodeDecodeError):
+            resp_body = e.fp.read().decode("utf-8", "replace")
+        msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
+        raise HttpError(msg) from e
+    return HttpResponse(resp)

examples/master.nix

@@ -15,6 +15,9 @@
         { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
       ]
     '';
+    # Users in this list will be able to reload the project list.
+    # All other user in the organization will be able to restart builds or evaluations.
+    admins = [ "Mic92" ];
     github = {
       # Github user used as a CI identity
       user = "mic92-buildbot";
@@ -27,9 +30,6 @@
       # After creating the app, press "Generate a new client secret" and fill in the client ID and secret below
       oauthId = "aaaaaaaaaaaaaaaaaaaa";
       oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
-      # Users in this list will be able to reload the project list.
-      # All other user in the organization will be able to restart builds or evaluations.
-      admins = [ "Mic92" ];
       # All github projects with this topic will be added to buildbot.
       # One can trigger a project scan by visiting the Builds -> Builders page and looking for the "reload-github-project" builder.
       # This builder has a "Update Github Projects" button that everyone in the github organization can use.

nix/buildbot-gitea.nix

@@ -0,0 +1,27 @@
+{ buildPythonPackage
+, fetchPypi
+, lib
+
+, pip
+, buildbot
+, requests
+}:
+buildPythonPackage (lib.fix (self: {
+  pname = "buildbot-gitea";
+  version = "1.8.0";
+
+  nativeBuildInputs = [
+
+  ];
+
+  propagatedBuildInputs = [
+    pip
+    buildbot
+    requests
+  ];
+
+  src = fetchPypi {
+    inherit (self) pname version;
+    hash = "sha256-zYcILPp42QuQyfEIzmYKV9vWf47sBAQI8FOKJlZ60yA=";
+  };
+}))

nix/checks/master.nix

@@ -14,13 +14,13 @@
             { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
           ]
         '';
+        admins = [ "Mic92" ];
         github = {
           tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
           webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
           oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
           oauthId = "aaaaaaaaaaaaaaaaaaaa";
           user = "mic92-buildbot";
-          admins = [ "Mic92" ];
         };
       };
     };

nix/master.nix

@@ -5,8 +5,18 @@
 }:
 let
   cfg = config.services.buildbot-nix.master;
+  inherit
+    (lib)
+    mkRenamedOptionModule
+    ;
 in
 {
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "buildbot-nix" "master" "github" "admins" ]
+      [ "services" "buildbot-nix" "master" "admins" ])
+  ];
+
   options = {
     services.buildbot-nix.master = {
       enable = lib.mkEnableOption "buildbot-master";
@@ -15,6 +25,13 @@ in
         default = "postgresql://@/buildbot";
         description = "Postgresql database url";
       };
+      authBackend = lib.mkOption {
+        type = lib.types.enum [ "github" "gitea" "none" ];
+        default = "github";
+        description = ''
+          Which OAuth2 backend to use.
+        '';
+      };
       cachix = {
         name = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -34,7 +51,42 @@ in
           description = "Cachix auth token";
         };
       };
+      gitea = {
+        enable = lib.mkEnableOption "Enable Gitea integration";
+
+        tokenFile = lib.mkOption {
+          type = lib.types.path;
+          description = "Gitea token file";
+        };
+        webhookSecretFile = lib.mkOption {
+          type = lib.types.path;
+          description = "Github webhook secret file";
+        };
+        oauthSecretFile = lib.mkOption {
+          type = lib.types.path;
+          description = "Gitea oauth secret file";
+        };
+
+        instanceURL = lib.mkOption {
+          type = lib.types.str;
+          description = "Gitea instance URL";
+        };
+        oauthId = lib.mkOption {
+          type = lib.types.str;
+          description = "Gitea oauth id. Used for the login button";
+        };
+        topic = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = "build-with-buildbot";
+          description = ''
+            Projects that have this topic will be built by buildbot.
+            If null, all projects that the buildbot Gitea user has access to, are built.
+          '';
+        };
+      };
       github = {
+        disable = lib.mkEnableOption "Disable GitHub integration";
+
         tokenFile = lib.mkOption {
           type = lib.types.path;
           description = "Github token file";
@@ -62,11 +114,6 @@ in
           type = lib.types.str;
           description = "Github user that is used for the buildbot";
         };
-        admins = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          default = [ ];
-          description = "Users that are allowed to login to buildbot, trigger builds and change settings";
-        };
         topic = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
           default = "build-with-buildbot";
@@ -76,6 +123,11 @@ in
           '';
         };
       };
+      admins = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = "Users that are allowed to login to buildbot, trigger builds and change settings";
+      };
       workersFile = lib.mkOption {
         type = lib.types.path;
         description = "File containing a list of nix workers";
@@ -144,7 +196,7 @@ in
       home = "/var/lib/buildbot";
       extraImports = ''
         from datetime import timedelta
-        from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig
+        from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig
       '';
       configurators = [
         ''
@@ -152,17 +204,23 @@ in
         ''
         ''
           NixConfigurator(
-              github=GithubConfig(
+              auth_backend=${builtins.toJSON cfg.authBackend},
+              github=${if cfg.github.disable then "None" else "GithubConfig(
                   oauth_id=${builtins.toJSON cfg.github.oauthId},
-                  admins=${builtins.toJSON cfg.github.admins},
                   buildbot_user=${builtins.toJSON cfg.github.user},
                   topic=${builtins.toJSON cfg.github.topic},
-              ),
+              )"},
+              gitea=${if !cfg.gitea.enable then "None" else "GiteaConfig(
+                  instance_url=${builtins.toJSON cfg.gitea.instanceURL},
+                  oauth_id=${builtins.toJSON cfg.gitea.oauthId},
+                  topic=${builtins.toJSON cfg.gitea.topic},
+              )"},
               cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
                   name=${builtins.toJSON cfg.cachix.name},
                   signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
                   auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"},
               )"},
+              admins=${builtins.toJSON cfg.admins},
               url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
               nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
               nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
@@ -189,6 +247,7 @@ in
         (ps.toPythonModule pkgs.buildbot-worker)
         pkgs.buildbot-plugins.www-react
         (pkgs.python3.pkgs.callPackage ../default.nix { })
+        (pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = pkgs.buildbot; })
       ];
     };
 
@@ -205,7 +264,12 @@ in
         ++ lib.optional (cfg.cachix.signingKeyFile != null)
           "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
         ++ lib.optional (cfg.cachix.authTokenFile != null)
-          "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
+          "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
+        ++ lib.optionals cfg.gitea.enable [
+          "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
+          "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
+          "gitea-token:${cfg.gitea.tokenFile}"
+        ];
 
         # Needed because it tries to reach out to github on boot.
         # FIXME: if github is not available, we shouldn't fail buildbot, instead it should just try later again in the background