Add Gitea backend

Signed-off-by: magic_rb <richard@brezak.sk>

buildbot_nix/__init__.py

@@ -2,56 +2,44 @@ import json
 import multiprocessing
 import os
 import re
-import signal
-import sys
 import uuid
 from collections import defaultdict
 from collections.abc import Generator
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable
+from typing import TYPE_CHECKING, Any
-import typing
 
-from buildbot.process.log import StreamLog
+from buildbot.config.builder import BuilderConfig
 from buildbot.configurators import ConfiguratorBase
 from buildbot.interfaces import WorkerSetupError
-from buildbot.plugins import reporters, schedulers, steps, util, worker
+from buildbot.locks import MasterLock
+from buildbot.plugins import schedulers, steps, util, worker
 from buildbot.process import buildstep, logobserver, remotecommand
 from buildbot.process.project import Project
-from buildbot.process.properties import Interpolate, Properties
+from buildbot.process.properties import Properties
 from buildbot.process.results import ALL_RESULTS, statusToString
-from buildbot.steps.trigger import Trigger
-from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
-from buildbot.www.authz import Authz
 from buildbot.secrets.providers.file import SecretInAFile
-from buildbot.locks import MasterLock
+from buildbot.steps.trigger import Trigger
-from buildbot.config.builder import BuilderConfig
+from buildbot.www.authz import Authz
+from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
 
 if TYPE_CHECKING:
-    from buildbot.process.log import Log
+    from buildbot.process.log import StreamLog
+    from buildbot.www.auth import AuthBase
 
-from twisted.internet import defer, threads
+from twisted.internet import defer
 from twisted.logger import Logger
-from twisted.python.failure import Failure
 
-from .gitea_projects import (
+from .common import (
-    GiteaConfig
+    slugify_project_name,
 )
-
+from .gitea_projects import GiteaBackend, GiteaConfig
 from .github_projects import (
     GithubBackend,
     GithubConfig,
-    slugify_project_name,
-)
-
-from .projects import (
-    GitProject,
-    GitBackend
-)
-
-from .secrets import (
-    read_secret_file
 )
+from .projects import GitBackend, GitProject
+from .secrets import read_secret_file
 
 SKIPPED_BUILDER_NAME = "skipped-builds"
 
@@ -98,8 +86,7 @@ class BuildTrigger(Trigger):
         return props
 
     def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]:  # noqa: N802
-        # TODO when is this None?
+        build_props = self.build.getProperties()
-        build_props = self.build.getProperties() if self.build is not None else Properties()
         repo_name = self.project.name
         project_id = slugify_project_name(repo_name)
         source = f"nix-eval-{project_id}"
@@ -166,7 +153,9 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
 
     project: GitProject
 
-    def __init__(self, project: GitProject, supported_systems: list[str], **kwargs: Any) -> None:
+    def __init__(
+        self, project: GitProject, supported_systems: list[str], **kwargs: Any
+    ) -> None:
         kwargs = self.setupShellMixin(kwargs)
         super().__init__(**kwargs)
         self.project = project
@@ -177,9 +166,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
     @defer.inlineCallbacks
     def run(self) -> Generator[Any, object, Any]:
         # run nix-eval-jobs --flake .#checks to generate the dict of stages
-        cmd_: object = yield self.makeRemoteShellCommand()
+        cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
-        # TODO why doesn't type information pass through yield again?
-        cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_)
         yield self.runCommand(cmd)
 
         # if the command passes extract the list of stages
@@ -245,9 +232,7 @@ class EvalErrorStep(steps.BuildStep):
         error = self.getProperty("error")
         attr = self.getProperty("attr")
         # show eval error
-        # TODO why doesn't type information pass through yield again?
+        error_log: StreamLog = yield self.addLog("nix_error")
-        error_log_: object  = yield self.addLog("nix_error")
-        error_log: StreamLog = typing.cast(StreamLog, error_log_);
         error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
         return util.FAILURE
 
@@ -262,9 +247,7 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
     @defer.inlineCallbacks
     def run(self) -> Generator[Any, object, Any]:
         # run `nix build`
-        # TODO why doesn't type information pass through yield again?
+        cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
-        cmd_: object = yield self.makeRemoteShellCommand()
-        cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_)
         yield self.runCommand(cmd)
 
         res = cmd.results()
@@ -300,11 +283,10 @@ class UpdateBuildOutput(steps.BuildStep):
         (self.path / attr).write_text(out_path)
         return util.SUCCESS
 
+
 # GitHub somtimes fires the PR webhook before it has computed the merge commit
 # This is a workaround to fetch the merge commit and checkout the PR branch in CI
 class GitLocalPrMerge(steps.Git):
-    stdio_log: StreamLog
-
     @defer.inlineCallbacks
     def run_vc(
         self,
@@ -322,9 +304,7 @@ class GitLocalPrMerge(steps.Git):
             return res
 
         # The code below is a modified version of Git.run_vc
-        # TODO why doesn't type information pass through yield again?
+        self.stdio_log: StreamLog = yield self.addLogForRemoteCommands("stdio")
-        stdio_log_: object = yield self.addLogForRemoteCommands("stdio")
-        self.stdio_log = typing.cast(StreamLog, stdio_log_)
         self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n")
 
         git_installed = yield self.checkFeatureSupport()
@@ -517,8 +497,7 @@ def nix_build_config(
                 "-r",
                 util.Property("out_path"),
             ],
-            doStepIf=lambda s: s.getProperty("branch")
+            doStepIf=lambda s: s.getProperty("branch") == project.default_branch,
-            == project.default_branch,
         ),
     )
     factory.addStep(
@@ -576,6 +555,7 @@ def nix_skipped_build_config(
         factory=factory,
     )
 
+
 def config_for_project(
     config: dict[str, Any],
     project: GitProject,
@@ -664,8 +644,8 @@ def config_for_project(
         ],
     )
 
+
 def normalize_virtual_builder_name(name: str) -> str:
-    # TODO this code is a mystery to me
     if re.match(r"^[^:]+:", name) is not None:
         # rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build
         match = re.match(r"[^:]:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
@@ -736,7 +716,9 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase):
         return self.check_builder(epobject, epdict, "buildrequest")
 
 
-def setup_authz(backends: list[GitBackend], projects: list[GitProject], admins: list[str]) -> Authz:
+def setup_authz(
+    backends: list[GitBackend], projects: list[GitProject], admins: list[str]
+) -> Authz:
     allow_rules = []
     allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict(
         lambda: {backend.reload_builder_name for backend in backends},
@@ -785,7 +767,9 @@ class NixConfigurator(ConfiguratorBase):
     def __init__(
         self,
         # Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
-        github: GithubConfig,
+        admins: list[str],
+        auth_backend: str,
+        github: GithubConfig | None,
         gitea: GiteaConfig | None,
         url: str,
         nix_supported_systems: list[str],
@@ -800,7 +784,10 @@ class NixConfigurator(ConfiguratorBase):
         self.nix_eval_max_memory_size = nix_eval_max_memory_size
         self.nix_eval_worker_count = nix_eval_worker_count
         self.nix_supported_systems = nix_supported_systems
+        self.auth_backend = auth_backend
+        self.admins = admins
         self.github = github
+        self.gitea = gitea
         self.url = url
         self.cachix = cachix
         if outputs_path is None:
@@ -809,15 +796,23 @@ class NixConfigurator(ConfiguratorBase):
             self.outputs_path = Path(outputs_path)
 
     def configure(self, config: dict[str, Any]) -> None:
-        backends: list[GitBackend] = []
+        backends: dict[str, GitBackend] = {}
 
-        github_backend: GitBackend = GithubBackend(self.github)
         if self.github is not None:
-            backends.append(github_backend)
+            backends["github"] = GithubBackend(self.github)
+
+        if self.gitea is not None:
+            backends["gitea"] = GiteaBackend(self.gitea)
+
+        auth: AuthBase | None = (
+            backends[self.auth_backend].create_auth()
+            if self.auth_backend != "none"
+            else None
+        )
 
         projects: list[GitProject] = []
 
-        for backend in backends:
+        for backend in backends.values():
             projects += backend.load_projects()
 
         worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
@@ -834,17 +829,10 @@ class NixConfigurator(ConfiguratorBase):
                 config["workers"].append(worker.Worker(worker_name, item["pass"]))
                 worker_names.append(worker_name)
 
-        # TODO pull out into global config
-        webhook_secret = read_secret_file(self.github.webhook_secret_name)
         eval_lock = util.MasterLock("nix-eval")
 
         for project in projects:
-            project.create_project_hook(
+            project.create_project_hook(project.owner, project.repo, self.url)
-                project.owner,
-                project.repo,
-                self.url,
-                webhook_secret,
-            )
             config_for_project(
                 config,
                 project,
@@ -859,11 +847,9 @@ class NixConfigurator(ConfiguratorBase):
 
         config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
 
-        for backend in backends:
+        for backend in backends.values():
             # Reload backend projects
-            config["builders"].append(
+            config["builders"].append(backend.create_reload_builder([worker_names[0]]))
-              backend.create_reload_builder([worker_names[0]])
-            )
             config["schedulers"].extend(
                 [
                     schedulers.ForceScheduler(
@@ -891,21 +877,24 @@ class NixConfigurator(ConfiguratorBase):
         config["www"]["plugins"].update(dict(base_react={}))
 
         config["www"].setdefault("change_hook_dialects", {})
-        for backend in backends:
+        for backend in backends.values():
-            config["www"]["change_hook_dialects"][backend.change_hook_name] = \
+            config["www"]["change_hook_dialects"][backend.change_hook_name] = (
-                backend.create_change_hook(webhook_secret)
+                backend.create_change_hook()
+            )
 
         if "auth" not in config["www"]:
             config["www"].setdefault("avatar_methods", [])
 
-            for backend in backends:
+            for backend in backends.values():
-                config["www"]["avatar_methods"].append(backend.create_avatar_method())
+                avatar_method = backend.create_avatar_method()
+                if avatar_method is not None:
+                    config["www"]["avatar_methods"].append(avatar_method)
             # TODO one cannot have multiple auth backends...
-            config["www"]["auth"] = backends[0].create_auth()
+            if auth is not None:
+                config["www"]["auth"] = auth
 
             config["www"]["authz"] = setup_authz(
-                # TODO pull out into global config
+                admins=self.admins,
-                admins=self.github.admins,
+                backends=list(backends.values()),
-                backends=backends,
                 projects=projects,
             )

buildbot_nix/common.py

@@ -0,0 +1,80 @@
+import contextlib
+import http.client
+import json
+import urllib.request
+from typing import Any
+
+
+def slugify_project_name(name: str) -> str:
+    return name.replace(".", "-").replace("/", "-")
+
+
+def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
+    next_url: str | None = url
+    items = []
+    while next_url:
+        try:
+            res = http_request(
+                next_url,
+                headers={"Authorization": f"Bearer {token}"},
+            )
+        except OSError as e:
+            msg = f"failed to fetch {next_url}: {e}"
+            raise HttpError(msg) from e
+        next_url = None
+        link = res.headers()["Link"]
+        if link is not None:
+            links = link.split(", ")
+            for link in links:  # pagination
+                link_parts = link.split(";")
+                if link_parts[1].strip() == 'rel="next"':
+                    next_url = link_parts[0][1:-1]
+        items += res.json()
+    return items
+
+
+class HttpResponse:
+    def __init__(self, raw: http.client.HTTPResponse) -> None:
+        self.raw = raw
+
+    def json(self) -> Any:
+        return json.load(self.raw)
+
+    def headers(self) -> http.client.HTTPMessage:
+        return self.raw.headers
+
+
+class HttpError(Exception):
+    pass
+
+
+def http_request(
+    url: str,
+    method: str = "GET",
+    headers: dict[str, str] | None = None,
+    data: dict[str, Any] | None = None,
+) -> HttpResponse:
+    body = None
+    if data:
+        body = json.dumps(data).encode("ascii")
+    if headers is None:
+        headers = {}
+    headers = headers.copy()
+    headers["User-Agent"] = "buildbot-nix"
+
+    if not url.startswith("https:"):
+        msg = "url must be https: {url}"
+        raise HttpError(msg)
+
+    req = urllib.request.Request(  # noqa: S310
+        url, headers=headers, method=method, data=body
+    )
+    try:
+        resp = urllib.request.urlopen(req)  # noqa: S310
+    except urllib.request.HTTPError as e:
+        resp_body = ""
+        with contextlib.suppress(OSError, UnicodeDecodeError):
+            resp_body = e.fp.read().decode("utf-8", "replace")
+        msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
+        raise HttpError(msg) from e
+    return HttpResponse(resp)

buildbot_nix/gitea_projects.py

@@ -0,0 +1,300 @@
+import json
+import os
+import signal
+from collections.abc import Generator
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Any
+
+from buildbot.config.builder import BuilderConfig
+from buildbot.plugins import util
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Interpolate
+from buildbot.reporters.base import ReporterBase
+from buildbot.www.auth import AuthBase
+from buildbot.www.avatar import AvatarBase
+from buildbot_gitea.auth import GiteaAuth  # type: ignore[import]
+from buildbot_gitea.reporter import GiteaStatusPush  # type: ignore[import]
+from twisted.internet import defer, threads
+from twisted.python import log
+from twisted.python.failure import Failure
+
+from .common import (
+    http_request,
+    paginated_github_request,
+    slugify_project_name,
+)
+from .projects import GitBackend, GitProject
+from .secrets import read_secret_file
+
+
+@dataclass
+class GiteaConfig:
+    instance_url: str
+    oauth_id: str
+    admins: list[str]
+
+    oauth_secret_name: str = "gitea-oauth-secret"
+    token_secret_name: str = "gitea-token"
+    webhook_secret_name: str = "gitea-webhook-secret"
+    project_cache_file: Path = Path("gitea-project-cache.json")
+    topic: str | None = "build-with-buildbot"
+
+    def oauth_secret(self) -> str:
+        return read_secret_file(self.oauth_secret_name)
+
+    def token(self) -> str:
+        return read_secret_file(self.token_secret_name)
+
+
+class GiteaProject(GitProject):
+    config: GiteaConfig
+    webhook_secret: str
+    data: dict[str, Any]
+
+    def __init__(
+        self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any]
+    ) -> None:
+        self.config = config
+        self.webhook_secret = webhook_secret
+        self.data = data
+
+    def create_project_hook(
+        self,
+        owner: str,
+        repo: str,
+        webhook_url: str,
+    ) -> None:
+        hooks = paginated_github_request(
+            f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
+            self.config.token(),
+        )
+        config = dict(
+            url=webhook_url + "change_hook/gitea",
+            content_type="json",
+            insecure_ssl="0",
+            secret=self.webhook_secret,
+        )
+        data = dict(
+            name="web",
+            active=True,
+            events=["push", "pull_request"],
+            config=config,
+            type="gitea",
+        )
+        headers = {
+            "Authorization": f"token {self.config.token()}",
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        for hook in hooks:
+            if hook["config"]["url"] == webhook_url + "change_hook/gitea":
+                log.msg(f"hook for {owner}/{repo} already exists")
+                return
+
+        http_request(
+            f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks",
+            method="POST",
+            headers=headers,
+            data=data,
+        )
+
+    def get_project_url(self) -> str:
+        return f"https://git:%(secret:{self.config.token_secret_name})s@{self.config.instance_url}/{self.name}"
+
+    @property
+    def pretty_type(self) -> str:
+        return "Gitea"
+
+    @property
+    def type(self) -> str:
+        return "gitea"
+
+    @property
+    def repo(self) -> str:
+        return self.data["name"]
+
+    @property
+    def owner(self) -> str:
+        return self.data["owner"]["login"]
+
+    @property
+    def name(self) -> str:
+        return self.data["full_name"]
+
+    @property
+    def url(self) -> str:
+        # not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34
+        return self.data["ssh_url"]
+
+    @property
+    def project_id(self) -> str:
+        return slugify_project_name(self.data["full_name"])
+
+    @property
+    def default_branch(self) -> str:
+        return self.data["default_branch"]
+
+    @property
+    def topics(self) -> list[str]:
+        # note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects`
+        return self.data["topics"]
+
+    @property
+    def belongs_to_org(self) -> bool:
+        # TODO Gitea doesn't include this information
+        return False  # self.data["owner"]["type"] == "Organization"
+
+
+class GiteaBackend(GitBackend):
+    config: GiteaConfig
+
+    def __init__(self, config: GiteaConfig) -> None:
+        self.config = config
+        self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
+
+    def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
+        """Updates the flake an opens a PR for it."""
+        factory = util.BuildFactory()
+        factory.addStep(
+            ReloadGiteaProjects(self.config, self.config.project_cache_file),
+        )
+        return util.BuilderConfig(
+            name=self.reload_builder_name,
+            workernames=worker_names,
+            factory=factory,
+        )
+
+    def create_reporter(self) -> ReporterBase:
+        return GiteaStatusPush(
+            "https://" + self.config.instance_url,
+            Interpolate(self.config.token()),
+            context=Interpolate("buildbot/%(prop:status_name)s"),
+            context_pr=Interpolate("buildbot/%(prop:status_name)s"),
+        )
+
+    def create_change_hook(self) -> dict[str, Any]:
+        return {
+            "secret": self.webhook_secret,
+        }
+
+    def create_avatar_method(self) -> AvatarBase | None:
+        return None
+
+    def create_auth(self) -> AuthBase:
+        return GiteaAuth(
+            "https://" + self.config.instance_url,
+            self.config.oauth_id,
+            self.config.oauth_secret(),
+        )
+
+    def load_projects(self) -> list["GitProject"]:
+        if not self.config.project_cache_file.exists():
+            return []
+
+        repos: list[dict[str, Any]] = sorted(
+            json.loads(self.config.project_cache_file.read_text()),
+            key=lambda x: x["full_name"],
+        )
+        return list(
+            filter(
+                lambda project: self.config.topic is not None
+                and self.config.topic in project.topics,
+                [
+                    GiteaProject(self.config, self.webhook_secret, repo)
+                    for repo in repos
+                ],
+            )
+        )
+
+    def are_projects_cached(self) -> bool:
+        return self.config.project_cache_file.exists()
+
+    @property
+    def type(self) -> str:
+        return "gitea"
+
+    @property
+    def pretty_type(self) -> str:
+        return "Gitea"
+
+    @property
+    def reload_builder_name(self) -> str:
+        return "reload-gitea-projects"
+
+    @property
+    def change_hook_name(self) -> str:
+        return "gitea"
+
+
+class ReloadGiteaProjects(BuildStep):
+    name = "reload_gitea_projects"
+    config: GiteaConfig
+
+    def __init__(
+        self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any
+    ) -> None:
+        self.config = config
+        self.project_cache_file = project_cache_file
+        super().__init__(**kwargs)
+
+    def reload_projects(self) -> None:
+        refresh_projects(self.config, self.project_cache_file)
+
+    @defer.inlineCallbacks
+    def run(self) -> Generator[Any, object, Any]:
+        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
+
+        self.error_msg = ""
+
+        def error_cb(failure: Failure) -> int:
+            self.error_msg += failure.getTraceback()
+            return util.FAILURE
+
+        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
+        res = yield d
+        if res == util.SUCCESS:
+            # reload the buildbot config
+            os.kill(os.getpid(), signal.SIGHUP)
+            return util.SUCCESS
+        else:
+            yield self.addLog("log").addStderr(
+                f"Failed to reload project list: {self.error_msg}"
+            )
+            return util.FAILURE
+
+
+def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
+    repos = []
+
+    for repo in paginated_github_request(
+        f"https://{config.instance_url}/api/v1/user/repos?limit=100",
+        config.token(),
+    ):
+        if not repo["permissions"]["admin"]:
+            name = repo["full_name"]
+            log.msg(
+                f"skipping {name} because we do not have admin privileges, needed for hook management",
+            )
+        else:
+            try:
+                # Gitea doesn't include topics in the default repo listing, unlike GitHub
+                topics: list[str] = http_request(
+                    f"https://{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics",
+                    headers={"Authorization": f"token {config.token}"},
+                ).json()["topics"]
+                repo["topics"] = topics
+                repos.append(repo)
+            except OSError:
+                pass
+
+    with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
+        path = Path(f.name)
+        try:
+            f.write(json.dumps(repos))
+            f.flush()
+            path.rename(repo_cache_file)
+        except OSError:
+            path.unlink()
+            raise

buildbot_nix/github_projects.py

@@ -1,41 +1,33 @@
-import contextlib
-import http.client
 import json
-import urllib.request
-import signal
 import os
+import signal
 from collections.abc import Generator
-
+from dataclasses import dataclass
-import typing
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, Any
+from typing import Any
-from dataclasses import dataclass
 
-from twisted.python import log
-from twisted.internet import defer, threads
-from twisted.python.failure import Failure
-
-if TYPE_CHECKING:
-    from buildbot.process.log import StreamLog
-
-from buildbot.process.properties import Interpolate
 from buildbot.config.builder import BuilderConfig
+from buildbot.plugins import util
 from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Interpolate
 from buildbot.reporters.base import ReporterBase
 from buildbot.reporters.github import GitHubStatusPush
-from buildbot.www.avatar import AvatarBase, AvatarGitHub
 from buildbot.www.auth import AuthBase
+from buildbot.www.avatar import AvatarBase, AvatarGitHub
 from buildbot.www.oauth2 import GitHubAuth
-from buildbot.plugins import util
+from twisted.internet import defer, threads
+from twisted.python import log
+from twisted.python.failure import Failure
 
-from .projects import (
+from .common import (
-    GitProject,
+    http_request,
-    GitBackend
+    paginated_github_request,
-)
+    slugify_project_name,
-from .secrets import (
-    read_secret_file
 )
+from .projects import GitBackend, GitProject
+from .secrets import read_secret_file
+
 
 class ReloadGithubProjects(BuildStep):
     name = "reload_github_projects"
@@ -65,43 +57,42 @@ class ReloadGithubProjects(BuildStep):
             os.kill(os.getpid(), signal.SIGHUP)
             return util.SUCCESS
         else:
-            log: object = yield self.addLog("log")
+            yield self.addLog("log").addStderr(
-            # TODO this assumes that log is of type StreamLog and not something else
+                f"Failed to reload project list: {self.error_msg}"
-            typing.cast(StreamLog, log).addStderr(f"Failed to reload project list: {self.error_msg}")
+            )
             return util.FAILURE
 
+
 @dataclass
 class GithubConfig:
     oauth_id: str
-    admins: list[str]
 
+    # TODO unused
     buildbot_user: str
     oauth_secret_name: str = "github-oauth-secret"
-    webhook_secret_name: str = "github-webhook-secret"
     token_secret_name: str = "github-token"
+    webhook_secret_name: str = "github-webhook-secret"
     project_cache_file: Path = Path("github-project-cache.json")
     topic: str | None = "build-with-buildbot"
 
     def token(self) -> str:
         return read_secret_file(self.token_secret_name)
 
+
 @dataclass
 class GithubBackend(GitBackend):
     config: GithubConfig
+    webhook_secret: str
 
-    def __init__(self, config: GithubConfig):
+    def __init__(self, config: GithubConfig) -> None:
         self.config = config
+        self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
 
-    def create_reload_builder(
+    def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
-            self,
-            worker_names: list[str]
-    ) -> BuilderConfig:
         """Updates the flake an opens a PR for it."""
         factory = util.BuildFactory()
         factory.addStep(
-            ReloadGithubProjects(
+            ReloadGithubProjects(self.config.token(), self.config.project_cache_file),
-                self.config.token(), self.config.project_cache_file
-            ),
         )
         return util.BuilderConfig(
             name=self.reload_builder_name,
@@ -110,46 +101,50 @@ class GithubBackend(GitBackend):
         )
 
     def create_reporter(self) -> ReporterBase:
-        return \
+        return GitHubStatusPush(
-          GitHubStatusPush(
+            token=self.config.token(),
-              token=self.config.token(),
+            # Since we dynamically create build steps,
-              # Since we dynamically create build steps,
+            # we use `virtual_builder_name` in the webinterface
-              # we use `virtual_builder_name` in the webinterface
+            # so that we distinguish what has beeing build
-              # so that we distinguish what has beeing build
+            context=Interpolate("buildbot/%(prop:status_name)s"),
-              context=Interpolate("buildbot/%(prop:status_name)s"),
+        )
-          )
 
-    def create_change_hook(self, webhook_secret: str) -> dict[str, Any]:
+    def create_change_hook(self) -> dict[str, Any]:
         return {
-            "secret": webhook_secret,
+            "secret": self.webhook_secret,
             "strict": True,
-            "token": self.config.token,
+            "token": self.config.token(),
             "github_property_whitelist": "*",
         }
 
-    def create_avatar_method(self) -> AvatarBase:
+    def create_avatar_method(self) -> AvatarBase | None:
         return AvatarGitHub(token=self.config.token())
 
     def create_auth(self) -> AuthBase:
-        return \
+        return GitHubAuth(
-            GitHubAuth(
+            self.config.oauth_id,
-                self.config.oauth_id,
+            read_secret_file(self.config.oauth_secret_name),
-                read_secret_file(self.config.oauth_secret_name),
+            apiVersion=4,
-                apiVersion=4,
+        )
-            )
 
     def load_projects(self) -> list["GitProject"]:
         if not self.config.project_cache_file.exists():
             return []
 
         repos: list[dict[str, Any]] = sorted(
-            json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"]
+            json.loads(self.config.project_cache_file.read_text()),
+            key=lambda x: x["full_name"],
+        )
+        return list(
+            filter(
+                lambda project: self.config.topic is not None
+                and self.config.topic in project.topics,
+                [
+                    GithubProject(self.config, self.webhook_secret, repo)
+                    for repo in repos
+                ],
+            )
         )
-        return \
-          list(filter(\
-            lambda project: self.config.topic != None and self.config.topic in project.topics, \
-            [GithubProject(self.config, repo) for repo in repos] \
-          ))
 
     def are_projects_cached(self) -> bool:
         return self.config.project_cache_file.exists()
@@ -170,11 +165,15 @@ class GithubBackend(GitBackend):
     def change_hook_name(self) -> str:
         return "github"
 
+
 class GithubProject(GitProject):
     config: GithubConfig
 
-    def __init__(self, config: GithubConfig, data: dict[str, Any]) -> None:
+    def __init__(
+        self, config: GithubConfig, webhook_secret: str, data: dict[str, Any]
+    ) -> None:
         self.config = config
+        self.webhook_secret = webhook_secret
         self.data = data
 
     def create_project_hook(
@@ -182,7 +181,6 @@ class GithubProject(GitProject):
         owner: str,
         repo: str,
         webhook_url: str,
-        webhook_secret: str,
     ) -> None:
         hooks = paginated_github_request(
             f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
@@ -192,9 +190,11 @@ class GithubProject(GitProject):
             url=webhook_url + "change_hook/github",
             content_type="json",
             insecure_ssl="0",
-            secret=webhook_secret,
+            secret=self.webhook_secret,
+        )
+        data = dict(
+            name="web", active=True, events=["push", "pull_request"], config=config
         )
-        data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
         headers = {
             "Authorization": f"Bearer {self.config.token()}",
             "Accept": "application/vnd.github+json",
@@ -256,79 +256,6 @@ class GithubProject(GitProject):
     def belongs_to_org(self) -> bool:
         return self.data["owner"]["type"] == "Organization"
 
-class GithubError(Exception):
-    pass
-
-
-class HttpResponse:
-    def __init__(self, raw: http.client.HTTPResponse) -> None:
-        self.raw = raw
-
-    def json(self) -> Any:
-        return json.load(self.raw)
-
-    def headers(self) -> http.client.HTTPMessage:
-        return self.raw.headers
-
-
-def http_request(
-    url: str,
-    method: str = "GET",
-    headers: dict[str, str] | None = None,
-    data: dict[str, Any] | None = None,
-) -> HttpResponse:
-    body = None
-    if data:
-        body = json.dumps(data).encode("ascii")
-    if headers is None:
-        headers = {}
-    headers = headers.copy()
-    headers["User-Agent"] = "buildbot-nix"
-
-    if not url.startswith("https:"):
-        msg = "url must be https: {url}"
-        raise GithubError(msg)
-
-    req = urllib.request.Request(  # noqa: S310
-        url, headers=headers, method=method, data=body
-    )
-    try:
-        resp = urllib.request.urlopen(req)  # noqa: S310
-    except urllib.request.HTTPError as e:
-        resp_body = ""
-        with contextlib.suppress(OSError, UnicodeDecodeError):
-            resp_body = e.fp.read().decode("utf-8", "replace")
-        msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
-        raise GithubError(msg) from e
-    return HttpResponse(resp)
-
-
-def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
-    next_url: str | None = url
-    items = []
-    while next_url:
-        try:
-            res = http_request(
-                next_url,
-                headers={"Authorization": f"Bearer {token}"},
-            )
-        except OSError as e:
-            msg = f"failed to fetch {next_url}: {e}"
-            raise GithubError(msg) from e
-        next_url = None
-        link = res.headers()["Link"]
-        if link is not None:
-            links = link.split(", ")
-            for link in links:  # pagination
-                link_parts = link.split(";")
-                if link_parts[1].strip() == 'rel="next"':
-                    next_url = link_parts[0][1:-1]
-        items += res.json()
-    return items
-
-
-def slugify_project_name(name: str) -> str:
-    return name.replace(".", "-").replace("/", "-")
 
 def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
     repos = []

buildbot_nix/projects.py

@@ -3,15 +3,13 @@ from typing import Any
 
 from buildbot.config.builder import BuilderConfig
 from buildbot.reporters.base import ReporterBase
-from buildbot.www.avatar import AvatarBase
 from buildbot.www.auth import AuthBase
+from buildbot.www.avatar import AvatarBase
+
 
 class GitBackend(ABC):
     @abstractmethod
-    def create_reload_builder(
+    def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
-            self,
-            worker_names: list[str]
-    ) -> BuilderConfig:
         pass
 
     @abstractmethod
@@ -19,11 +17,11 @@ class GitBackend(ABC):
         pass
 
     @abstractmethod
-    def create_change_hook(self, webhook_secret: str) -> dict[str, Any]:
+    def create_change_hook(self) -> dict[str, Any]:
         pass
 
     @abstractmethod
-    def create_avatar_method(self) -> AvatarBase:
+    def create_avatar_method(self) -> AvatarBase | None:
         pass
 
     @abstractmethod
@@ -58,6 +56,7 @@ class GitBackend(ABC):
     def change_hook_name(self) -> str:
         pass
 
+
 class GitProject(ABC):
     @abstractmethod
     def create_project_hook(
@@ -65,12 +64,11 @@ class GitProject(ABC):
         owner: str,
         repo: str,
         webhook_url: str,
-        webhook_secret: str,
     ) -> None:
         pass
 
     @abstractmethod
-    def get_project_url() -> str:
+    def get_project_url(self) -> str:
         pass
 
     @property
@@ -113,7 +111,6 @@ class GitProject(ABC):
     def default_branch(self) -> str:
         pass
 
-
     @property
     @abstractmethod
     def topics(self) -> list[str]:

buildbot_nix/secrets.py

@@ -1,6 +1,8 @@
-import os, sys
+import os
+import sys
 from pathlib import Path
 
+
 def read_secret_file(secret_name: str) -> str:
     directory = os.environ.get("CREDENTIALS_DIRECTORY")
     if directory is None:

buildbot_nix/util.py

@@ -0,0 +1,80 @@
+import contextlib
+import http.client
+import json
+import urllib.request
+from typing import Any
+
+
+def slugify_project_name(name: str) -> str:
+    return name.replace(".", "-").replace("/", "-")
+
+
+def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
+    next_url: str | None = url
+    items = []
+    while next_url:
+        try:
+            res = http_request(
+                next_url,
+                headers={"Authorization": f"Bearer {token}"},
+            )
+        except OSError as e:
+            msg = f"failed to fetch {next_url}: {e}"
+            raise HttpError(msg) from e
+        next_url = None
+        link = res.headers()["Link"]
+        if link is not None:
+            links = link.split(", ")
+            for link in links:  # pagination
+                link_parts = link.split(";")
+                if link_parts[1].strip() == 'rel="next"':
+                    next_url = link_parts[0][1:-1]
+        items += res.json()
+    return items
+
+
+class HttpResponse:
+    def __init__(self, raw: http.client.HTTPResponse) -> None:
+        self.raw = raw
+
+    def json(self) -> Any:
+        return json.load(self.raw)
+
+    def headers(self) -> http.client.HTTPMessage:
+        return self.raw.headers
+
+
+class HttpError(Exception):
+    pass
+
+
+def http_request(
+    url: str,
+    method: str = "GET",
+    headers: dict[str, str] | None = None,
+    data: dict[str, Any] | None = None,
+) -> HttpResponse:
+    body = None
+    if data:
+        body = json.dumps(data).encode("ascii")
+    if headers is None:
+        headers = {}
+    headers = headers.copy()
+    headers["User-Agent"] = "buildbot-nix"
+
+    if not url.startswith("https:"):
+        msg = "url must be https: {url}"
+        raise HttpError(msg)
+
+    req = urllib.request.Request(  # noqa: S310
+        url, headers=headers, method=method, data=body
+    )
+    try:
+        resp = urllib.request.urlopen(req)  # noqa: S310
+    except urllib.request.HTTPError as e:
+        resp_body = ""
+        with contextlib.suppress(OSError, UnicodeDecodeError):
+            resp_body = e.fp.read().decode("utf-8", "replace")
+        msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
+        raise HttpError(msg) from e
+    return HttpResponse(resp)

examples/master.nix

@@ -15,6 +15,9 @@
         { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
       ]
     '';
+    # Users in this list will be able to reload the project list.
+    # All other user in the organization will be able to restart builds or evaluations.
+    admins = [ "Mic92" ];
     github = {
       # Github user used as a CI identity
       user = "mic92-buildbot";
@@ -27,9 +30,6 @@
       # After creating the app, press "Generate a new client secret" and fill in the client ID and secret below
       oauthId = "aaaaaaaaaaaaaaaaaaaa";
       oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
-      # Users in this list will be able to reload the project list.
-      # All other user in the organization will be able to restart builds or evaluations.
-      admins = [ "Mic92" ];
       # All github projects with this topic will be added to buildbot.
       # One can trigger a project scan by visiting the Builds -> Builders page and looking for the "reload-github-project" builder.
       # This builder has a "Update Github Projects" button that everyone in the github organization can use.

nix/buildbot-gitea.nix

@@ -0,0 +1,27 @@
+{ buildPythonPackage
+, fetchPypi
+, lib
+
+, pip
+, buildbot
+, requests
+}:
+buildPythonPackage (lib.fix (self: {
+  pname = "buildbot-gitea";
+  version = "1.8.0";
+
+  nativeBuildInputs = [
+
+  ];
+
+  propagatedBuildInputs = [
+    pip
+    buildbot
+    requests
+  ];
+
+  src = fetchPypi {
+    inherit (self) pname version;
+    hash = "sha256-zYcILPp42QuQyfEIzmYKV9vWf47sBAQI8FOKJlZ60yA=";
+  };
+}))

nix/checks/master.nix

@@ -14,13 +14,13 @@
             { "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
           ]
         '';
+        admins = [ "Mic92" ];
         github = {
           tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
           webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
           oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
           oauthId = "aaaaaaaaaaaaaaaaaaaa";
           user = "mic92-buildbot";
-          admins = [ "Mic92" ];
         };
       };
     };

nix/master.nix

@@ -5,8 +5,18 @@
 }:
 let
   cfg = config.services.buildbot-nix.master;
+  inherit
+    (lib)
+    mkRenamedOptionModule
+    ;
 in
 {
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "buildbot-nix" "master" "github" "admins" ]
+      [ "services" "buildbot-nix" "master" "admins" ])
+  ];
+
   options = {
     services.buildbot-nix.master = {
       enable = lib.mkEnableOption "buildbot-master";
@@ -15,6 +25,13 @@ in
         default = "postgresql://@/buildbot";
         description = "Postgresql database url";
       };
+      authBackend = lib.mkOption {
+        type = lib.types.enum [ "github" "gitea" "none" ];
+        default = "github";
+        description = ''
+          Which OAuth2 backend to use.
+        '';
+      };
       cachix = {
         name = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -34,7 +51,42 @@ in
           description = "Cachix auth token";
         };
       };
+      gitea = {
+        enable = lib.mkEnableOption "Enable Gitea integration";
+
+        tokenFile = lib.mkOption {
+          type = lib.types.path;
+          description = "Gitea token file";
+        };
+        webhookSecretFile = lib.mkOption {
+          type = lib.types.path;
+          description = "Github webhook secret file";
+        };
+        oauthSecretFile = lib.mkOption {
+          type = lib.types.path;
+          description = "Gitea oauth secret file";
+        };
+
+        instanceURL = lib.mkOption {
+          type = lib.types.str;
+          description = "Gitea instance URL";
+        };
+        oauthId = lib.mkOption {
+          type = lib.types.str;
+          description = "Gitea oauth id. Used for the login button";
+        };
+        topic = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = "build-with-buildbot";
+          description = ''
+            Projects that have this topic will be built by buildbot.
+            If null, all projects that the buildbot Gitea user has access to, are built.
+          '';
+        };
+      };
       github = {
+        disable = lib.mkEnableOption "Disable GitHub integration";
+
         tokenFile = lib.mkOption {
           type = lib.types.path;
           description = "Github token file";
@@ -62,11 +114,6 @@ in
           type = lib.types.str;
           description = "Github user that is used for the buildbot";
         };
-        admins = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          default = [ ];
-          description = "Users that are allowed to login to buildbot, trigger builds and change settings";
-        };
         topic = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
           default = "build-with-buildbot";
@@ -76,6 +123,11 @@ in
           '';
         };
       };
+      admins = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = "Users that are allowed to login to buildbot, trigger builds and change settings";
+      };
       workersFile = lib.mkOption {
         type = lib.types.path;
         description = "File containing a list of nix workers";
@@ -144,7 +196,7 @@ in
       home = "/var/lib/buildbot";
       extraImports = ''
         from datetime import timedelta
-        from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig
+        from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig
       '';
       configurators = [
         ''
@@ -152,18 +204,23 @@ in
         ''
         ''
           NixConfigurator(
-              github=GithubConfig(
+              auth_backend=${builtins.toJSON cfg.authBackend},
+              github=${if cfg.github.disable then "None" else "GithubConfig(
                   oauth_id=${builtins.toJSON cfg.github.oauthId},
-                  admins=${builtins.toJSON cfg.github.admins},
                   buildbot_user=${builtins.toJSON cfg.github.user},
                   topic=${builtins.toJSON cfg.github.topic},
-              ),
+              )"},
-              gitea=None,
+              gitea=${if !cfg.gitea.enable then "None" else "GiteaConfig(
+                  instance_url=${builtins.toJSON cfg.gitea.instanceURL},
+                  oauth_id=${builtins.toJSON cfg.gitea.oauthId},
+                  topic=${builtins.toJSON cfg.gitea.topic},
+              )"},
               cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
                   name=${builtins.toJSON cfg.cachix.name},
                   signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
                   auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"},
               )"},
+              admins=${builtins.toJSON cfg.admins},
               url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
               nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
               nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
@@ -190,6 +247,7 @@ in
         (ps.toPythonModule pkgs.buildbot-worker)
         pkgs.buildbot-plugins.www-react
         (pkgs.python3.pkgs.callPackage ../default.nix { })
+        (pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = pkgs.buildbot; })
       ];
     };
 
@@ -206,7 +264,12 @@ in
         ++ lib.optional (cfg.cachix.signingKeyFile != null)
           "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
         ++ lib.optional (cfg.cachix.authTokenFile != null)
-          "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
+          "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
+        ++ lib.optionals cfg.gitea.enable [
+          "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
+          "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
+          "gitea-token:${cfg.gitea.tokenFile}"
+        ];
 
         # Needed because it tries to reach out to github on boot.
         # FIXME: if github is not available, we shouldn't fail buildbot, instead it should just try later again in the background