auxolotl/buildbot-nix
26
1
Fork 0
Code Issues Pull requests 1 Actions Packages Projects Releases Wiki Activity

Add Gitea backend

Browse source 
Signed-off-by: magic_rb <richard@brezak.sk>
This commit is contained in:
magic_rb 2024-04-27 17:06:17 +02:00 committed by Jörg Thalheim
parent 1605d2d3c2
commit 3f01a96147
11 changed files with 704 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
buildbot_nix/__init__.py
View file

@ -2,56 +2,44 @@ import json
import multiprocessing
import os
import re
import signal
import sys
import uuid
from collections import defaultdict
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable
import typing
from typing import TYPE_CHECKING, Any
from buildbot.process.log import StreamLog
from buildbot.config.builder import BuilderConfig
from buildbot.configurators import ConfiguratorBase
from buildbot.interfaces import WorkerSetupError
from buildbot.plugins import reporters, schedulers, steps, util, worker
from buildbot.locks import MasterLock
from buildbot.plugins import schedulers, steps, util, worker
from buildbot.process import buildstep, logobserver, remotecommand
from buildbot.process.project import Project
from buildbot.process.properties import Interpolate, Properties
from buildbot.process.properties import Properties
from buildbot.process.results import ALL_RESULTS, statusToString
from buildbot.steps.trigger import Trigger
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
from buildbot.www.authz import Authz
from buildbot.secrets.providers.file import SecretInAFile
from buildbot.locks import MasterLock
from buildbot.config.builder import BuilderConfig
from buildbot.steps.trigger import Trigger
from buildbot.www.authz import Authz
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
if TYPE_CHECKING:
from buildbot.process.log import Log
from buildbot.process.log import StreamLog
from buildbot.www.auth import AuthBase
from twisted.internet import defer, threads
from twisted.internet import defer
from twisted.logger import Logger
from twisted.python.failure import Failure
from .gitea_projects import (
GiteaConfig
from .common import (
slugify_project_name,
)
from .gitea_projects import GiteaBackend, GiteaConfig
from .github_projects import (
GithubBackend,
GithubConfig,
slugify_project_name,
)
from .projects import (
GitProject,
GitBackend
)
from .secrets import (
read_secret_file
)
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
SKIPPED_BUILDER_NAME = "skipped-builds"
@ -98,8 +86,7 @@ class BuildTrigger(Trigger):
return props
def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802
# TODO when is this None?
build_props = self.build.getProperties() if self.build is not None else Properties()
build_props = self.build.getProperties()
repo_name = self.project.name
project_id = slugify_project_name(repo_name)
source = f"nix-eval-{project_id}"
@ -166,7 +153,9 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
project: GitProject
def __init__(self, project: GitProject, supported_systems: list[str], **kwargs: Any) -> None:
def __init__(
self, project: GitProject, supported_systems: list[str], **kwargs: Any
) -> None:
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.project = project
@ -177,9 +166,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
# run nix-eval-jobs --flake .#checks to generate the dict of stages
cmd_: object = yield self.makeRemoteShellCommand()
# TODO why doesn't type information pass through yield again?
cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_)
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
@ -245,9 +232,7 @@ class EvalErrorStep(steps.BuildStep):
error = self.getProperty("error")
attr = self.getProperty("attr")
# show eval error
# TODO why doesn't type information pass through yield again?
error_log_: object = yield self.addLog("nix_error")
error_log: StreamLog = typing.cast(StreamLog, error_log_);
error_log: StreamLog = yield self.addLog("nix_error")
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
return util.FAILURE
@ -262,9 +247,7 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
# run `nix build`
# TODO why doesn't type information pass through yield again?
cmd_: object = yield self.makeRemoteShellCommand()
cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_)
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
res = cmd.results()
@ -300,11 +283,10 @@ class UpdateBuildOutput(steps.BuildStep):
(self.path / attr).write_text(out_path)
return util.SUCCESS
# GitHub somtimes fires the PR webhook before it has computed the merge commit
# This is a workaround to fetch the merge commit and checkout the PR branch in CI
class GitLocalPrMerge(steps.Git):
stdio_log: StreamLog
@defer.inlineCallbacks
def run_vc(
self,
@ -322,9 +304,7 @@ class GitLocalPrMerge(steps.Git):
return res
# The code below is a modified version of Git.run_vc
# TODO why doesn't type information pass through yield again?
stdio_log_: object = yield self.addLogForRemoteCommands("stdio")
self.stdio_log = typing.cast(StreamLog, stdio_log_)
self.stdio_log: StreamLog = yield self.addLogForRemoteCommands("stdio")
self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n")
git_installed = yield self.checkFeatureSupport()
@ -517,8 +497,7 @@ def nix_build_config(
"-r",
util.Property("out_path"),
],
doStepIf=lambda s: s.getProperty("branch")
== project.default_branch,
doStepIf=lambda s: s.getProperty("branch") == project.default_branch,
),
)
factory.addStep(
@ -576,6 +555,7 @@ def nix_skipped_build_config(
factory=factory,
)
def config_for_project(
config: dict[str, Any],
project: GitProject,
@ -664,8 +644,8 @@ def config_for_project(
],
)
def normalize_virtual_builder_name(name: str) -> str:
# TODO this code is a mystery to me
if re.match(r"^[^:]+:", name) is not None:
# rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build
match = re.match(r"[^:]:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
@ -736,7 +716,9 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase):
return self.check_builder(epobject, epdict, "buildrequest")
def setup_authz(backends: list[GitBackend], projects: list[GitProject], admins: list[str]) -> Authz:
def setup_authz(
backends: list[GitBackend], projects: list[GitProject], admins: list[str]
) -> Authz:
allow_rules = []
allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict(
lambda: {backend.reload_builder_name for backend in backends},
@ -785,7 +767,9 @@ class NixConfigurator(ConfiguratorBase):
def __init__(
self,
# Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
github: GithubConfig,
admins: list[str],
auth_backend: str,
github: GithubConfig | None,
gitea: GiteaConfig | None,
url: str,
nix_supported_systems: list[str],
@ -800,7 +784,10 @@ class NixConfigurator(ConfiguratorBase):
self.nix_eval_max_memory_size = nix_eval_max_memory_size
self.nix_eval_worker_count = nix_eval_worker_count
self.nix_supported_systems = nix_supported_systems
self.auth_backend = auth_backend
self.admins = admins
self.github = github
self.gitea = gitea
self.url = url
self.cachix = cachix
if outputs_path is None:
@ -809,15 +796,23 @@ class NixConfigurator(ConfiguratorBase):
self.outputs_path = Path(outputs_path)
def configure(self, config: dict[str, Any]) -> None:
backends: list[GitBackend] = []
backends: dict[str, GitBackend] = {}
github_backend: GitBackend = GithubBackend(self.github)
if self.github is not None:
backends.append(github_backend)
backends["github"] = GithubBackend(self.github)
if self.gitea is not None:
backends["gitea"] = GiteaBackend(self.gitea)
auth: AuthBase | None = (
backends[self.auth_backend].create_auth()
if self.auth_backend != "none"
else None
)
projects: list[GitProject] = []
for backend in backends:
for backend in backends.values():
projects += backend.load_projects()
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
@ -834,17 +829,10 @@ class NixConfigurator(ConfiguratorBase):
config["workers"].append(worker.Worker(worker_name, item["pass"]))
worker_names.append(worker_name)
# TODO pull out into global config
webhook_secret = read_secret_file(self.github.webhook_secret_name)
eval_lock = util.MasterLock("nix-eval")
for project in projects:
project.create_project_hook(
project.owner,
project.repo,
self.url,
webhook_secret,
)
project.create_project_hook(project.owner, project.repo, self.url)
config_for_project(
config,
project,
@ -859,11 +847,9 @@ class NixConfigurator(ConfiguratorBase):
config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
for backend in backends:
for backend in backends.values():
# Reload backend projects
config["builders"].append(
backend.create_reload_builder([worker_names[0]])
)
config["builders"].append(backend.create_reload_builder([worker_names[0]]))
config["schedulers"].extend(
[
schedulers.ForceScheduler(
@ -891,21 +877,24 @@ class NixConfigurator(ConfiguratorBase):
config["www"]["plugins"].update(dict(base_react={}))
config["www"].setdefault("change_hook_dialects", {})
for backend in backends:
config["www"]["change_hook_dialects"][backend.change_hook_name] = \
backend.create_change_hook(webhook_secret)
for backend in backends.values():
config["www"]["change_hook_dialects"][backend.change_hook_name] = (
backend.create_change_hook()
)
if "auth" not in config["www"]:
config["www"].setdefault("avatar_methods", [])
for backend in backends:
config["www"]["avatar_methods"].append(backend.create_avatar_method())
for backend in backends.values():
avatar_method = backend.create_avatar_method()
if avatar_method is not None:
config["www"]["avatar_methods"].append(avatar_method)
# TODO one cannot have multiple auth backends...
config["www"]["auth"] = backends[0].create_auth()
if auth is not None:
config["www"]["auth"] = auth
config["www"]["authz"] = setup_authz(
# TODO pull out into global config
admins=self.github.admins,
backends=backends,
admins=self.admins,
backends=list(backends.values()),
projects=projects,
)

80
buildbot_nix/common.py Normal file
View file

@ -0,0 +1,80 @@
import contextlib
import http.client
import json
import urllib.request
from typing import Any
def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
next_url: str | None = url
items = []
while next_url:
try:
res = http_request(
next_url,
headers={"Authorization": f"Bearer {token}"},
)
except OSError as e:
msg = f"failed to fetch {next_url}: {e}"
raise HttpError(msg) from e
next_url = None
link = res.headers()["Link"]
if link is not None:
links = link.split(", ")
for link in links: # pagination
link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1]
items += res.json()
return items
class HttpResponse:
def __init__(self, raw: http.client.HTTPResponse) -> None:
self.raw = raw
def json(self) -> Any:
return json.load(self.raw)
def headers(self) -> http.client.HTTPMessage:
return self.raw.headers
class HttpError(Exception):
pass
def http_request(
url: str,
method: str = "GET",
headers: dict[str, str] | None = None,
data: dict[str, Any] | None = None,
) -> HttpResponse:
body = None
if data:
body = json.dumps(data).encode("ascii")
if headers is None:
headers = {}
headers = headers.copy()
headers["User-Agent"] = "buildbot-nix"
if not url.startswith("https:"):
msg = "url must be https: {url}"
raise HttpError(msg)
req = urllib.request.Request( # noqa: S310
url, headers=headers, method=method, data=body
)
try:
resp = urllib.request.urlopen(req) # noqa: S310
except urllib.request.HTTPError as e:
resp_body = ""
with contextlib.suppress(OSError, UnicodeDecodeError):
resp_body = e.fp.read().decode("utf-8", "replace")
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise HttpError(msg) from e
return HttpResponse(resp)

300
buildbot_nix/gitea_projects.py Normal file
View file

@ -0,0 +1,300 @@
import json
import os
import signal
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase
from buildbot_gitea.auth import GiteaAuth # type: ignore[import]
from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import]
from twisted.internet import defer, threads
from twisted.python import log
from twisted.python.failure import Failure
from .common import (
http_request,
paginated_github_request,
slugify_project_name,
)
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
@dataclass
class GiteaConfig:
instance_url: str
oauth_id: str
admins: list[str]
oauth_secret_name: str = "gitea-oauth-secret"
token_secret_name: str = "gitea-token"
webhook_secret_name: str = "gitea-webhook-secret"
project_cache_file: Path = Path("gitea-project-cache.json")
topic: str | None = "build-with-buildbot"
def oauth_secret(self) -> str:
return read_secret_file(self.oauth_secret_name)
def token(self) -> str:
return read_secret_file(self.token_secret_name)
class GiteaProject(GitProject):
config: GiteaConfig
webhook_secret: str
data: dict[str, Any]
def __init__(
self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any]
) -> None:
self.config = config
self.webhook_secret = webhook_secret
self.data = data
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
hooks = paginated_github_request(
f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
self.config.token(),
)
config = dict(
url=webhook_url + "change_hook/gitea",
content_type="json",
insecure_ssl="0",
secret=self.webhook_secret,
)
data = dict(
name="web",
active=True,
events=["push", "pull_request"],
config=config,
type="gitea",
)
headers = {
"Authorization": f"token {self.config.token()}",
"Accept": "application/json",
"Content-Type": "application/json",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/gitea":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def get_project_url(self) -> str:
return f"https://git:%(secret:{self.config.token_secret_name})s@{self.config.instance_url}/{self.name}"
@property
def pretty_type(self) -> str:
return "Gitea"
@property
def type(self) -> str:
return "gitea"
@property
def repo(self) -> str:
return self.data["name"]
@property
def owner(self) -> str:
return self.data["owner"]["login"]
@property
def name(self) -> str:
return self.data["full_name"]
@property
def url(self) -> str:
# not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34
return self.data["ssh_url"]
@property
def project_id(self) -> str:
return slugify_project_name(self.data["full_name"])
@property
def default_branch(self) -> str:
return self.data["default_branch"]
@property
def topics(self) -> list[str]:
# note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects`
return self.data["topics"]
@property
def belongs_to_org(self) -> bool:
# TODO Gitea doesn't include this information
return False # self.data["owner"]["type"] == "Organization"
class GiteaBackend(GitBackend):
config: GiteaConfig
def __init__(self, config: GiteaConfig) -> None:
self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it."""
factory = util.BuildFactory()
factory.addStep(
ReloadGiteaProjects(self.config, self.config.project_cache_file),
)
return util.BuilderConfig(
name=self.reload_builder_name,
workernames=worker_names,
factory=factory,
)
def create_reporter(self) -> ReporterBase:
return GiteaStatusPush(
"https://" + self.config.instance_url,
Interpolate(self.config.token()),
context=Interpolate("buildbot/%(prop:status_name)s"),
context_pr=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_change_hook(self) -> dict[str, Any]:
return {
"secret": self.webhook_secret,
}
def create_avatar_method(self) -> AvatarBase | None:
return None
def create_auth(self) -> AuthBase:
return GiteaAuth(
"https://" + self.config.instance_url,
self.config.oauth_id,
self.config.oauth_secret(),
)
def load_projects(self) -> list["GitProject"]:
if not self.config.project_cache_file.exists():
return []
repos: list[dict[str, Any]] = sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
)
return list(
filter(
lambda project: self.config.topic is not None
and self.config.topic in project.topics,
[
GiteaProject(self.config, self.webhook_secret, repo)
for repo in repos
],
)
)
def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists()
@property
def type(self) -> str:
return "gitea"
@property
def pretty_type(self) -> str:
return "Gitea"
@property
def reload_builder_name(self) -> str:
return "reload-gitea-projects"
@property
def change_hook_name(self) -> str:
return "gitea"
class ReloadGiteaProjects(BuildStep):
name = "reload_gitea_projects"
config: GiteaConfig
def __init__(
self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any
) -> None:
self.config = config
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
def reload_projects(self) -> None:
refresh_projects(self.config, self.project_cache_file)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
yield self.addLog("log").addStderr(
f"Failed to reload project list: {self.error_msg}"
)
return util.FAILURE
def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
repos = []
for repo in paginated_github_request(
f"https://{config.instance_url}/api/v1/user/repos?limit=100",
config.token(),
):
if not repo["permissions"]["admin"]:
name = repo["full_name"]
log.msg(
f"skipping {name} because we do not have admin privileges, needed for hook management",
)
else:
try:
# Gitea doesn't include topics in the default repo listing, unlike GitHub
topics: list[str] = http_request(
f"https://{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics",
headers={"Authorization": f"token {config.token}"},
).json()["topics"]
repo["topics"] = topics
repos.append(repo)
except OSError:
pass
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
path = Path(f.name)
try:
f.write(json.dumps(repos))
f.flush()
path.rename(repo_cache_file)
except OSError:
path.unlink()
raise

183
buildbot_nix/github_projects.py
View file

@ -1,41 +1,33 @@
import contextlib
import http.client
import json
import urllib.request
import signal
import os
import signal
from collections.abc import Generator
import typing
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any
from dataclasses import dataclass
from typing import Any
from twisted.python import log
from twisted.internet import defer, threads
from twisted.python.failure import Failure
if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from buildbot.process.properties import Interpolate
from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase
from buildbot.reporters.github import GitHubStatusPush
from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.oauth2 import GitHubAuth
from buildbot.plugins import util
from twisted.internet import defer, threads
from twisted.python import log
from twisted.python.failure import Failure
from .projects import (
GitProject,
GitBackend
)
from .secrets import (
read_secret_file
from .common import (
http_request,
paginated_github_request,
slugify_project_name,
)
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
class ReloadGithubProjects(BuildStep):
name = "reload_github_projects"
@ -65,43 +57,42 @@ class ReloadGithubProjects(BuildStep):
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: object = yield self.addLog("log")
# TODO this assumes that log is of type StreamLog and not something else
typing.cast(StreamLog, log).addStderr(f"Failed to reload project list: {self.error_msg}")
yield self.addLog("log").addStderr(
f"Failed to reload project list: {self.error_msg}"
)
return util.FAILURE
@dataclass
class GithubConfig:
oauth_id: str
admins: list[str]
# TODO unused
buildbot_user: str
oauth_secret_name: str = "github-oauth-secret"
webhook_secret_name: str = "github-webhook-secret"
token_secret_name: str = "github-token"
webhook_secret_name: str = "github-webhook-secret"
project_cache_file: Path = Path("github-project-cache.json")
topic: str | None = "build-with-buildbot"
def token(self) -> str:
return read_secret_file(self.token_secret_name)
@dataclass
class GithubBackend(GitBackend):
config: GithubConfig
webhook_secret: str
def __init__(self, config: GithubConfig):
def __init__(self, config: GithubConfig) -> None:
self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
def create_reload_builder(
self,
worker_names: list[str]
) -> BuilderConfig:
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it."""
factory = util.BuildFactory()
factory.addStep(
ReloadGithubProjects(
self.config.token(), self.config.project_cache_file
),
ReloadGithubProjects(self.config.token(), self.config.project_cache_file),
)
return util.BuilderConfig(
name=self.reload_builder_name,
@ -110,8 +101,7 @@ class GithubBackend(GitBackend):
)
def create_reporter(self) -> ReporterBase:
return \
GitHubStatusPush(
return GitHubStatusPush(
token=self.config.token(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
@ -119,20 +109,19 @@ class GithubBackend(GitBackend):
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_change_hook(self, webhook_secret: str) -> dict[str, Any]:
def create_change_hook(self) -> dict[str, Any]:
return {
"secret": webhook_secret,
"secret": self.webhook_secret,
"strict": True,
"token": self.config.token,
"token": self.config.token(),
"github_property_whitelist": "*",
}
def create_avatar_method(self) -> AvatarBase:
def create_avatar_method(self) -> AvatarBase | None:
return AvatarGitHub(token=self.config.token())
def create_auth(self) -> AuthBase:
return \
GitHubAuth(
return GitHubAuth(
self.config.oauth_id,
read_secret_file(self.config.oauth_secret_name),
apiVersion=4,
@ -143,13 +132,19 @@ class GithubBackend(GitBackend):
return []
repos: list[dict[str, Any]] = sorted(
json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"]
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
)
return list(
filter(
lambda project: self.config.topic is not None
and self.config.topic in project.topics,
[
GithubProject(self.config, self.webhook_secret, repo)
for repo in repos
],
)
)
return \
list(filter(\
lambda project: self.config.topic != None and self.config.topic in project.topics, \
[GithubProject(self.config, repo) for repo in repos] \
))
def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists()
@ -170,11 +165,15 @@ class GithubBackend(GitBackend):
def change_hook_name(self) -> str:
return "github"
class GithubProject(GitProject):
config: GithubConfig
def __init__(self, config: GithubConfig, data: dict[str, Any]) -> None:
def __init__(
self, config: GithubConfig, webhook_secret: str, data: dict[str, Any]
) -> None:
self.config = config
self.webhook_secret = webhook_secret
self.data = data
def create_project_hook(
@ -182,7 +181,6 @@ class GithubProject(GitProject):
owner: str,
repo: str,
webhook_url: str,
webhook_secret: str,
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
@ -192,9 +190,11 @@ class GithubProject(GitProject):
url=webhook_url + "change_hook/github",
content_type="json",
insecure_ssl="0",
secret=webhook_secret,
secret=self.webhook_secret,
)
data = dict(
name="web", active=True, events=["push", "pull_request"], config=config
)
data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
headers = {
"Authorization": f"Bearer {self.config.token()}",
"Accept": "application/vnd.github+json",
@ -256,79 +256,6 @@ class GithubProject(GitProject):
def belongs_to_org(self) -> bool:
return self.data["owner"]["type"] == "Organization"
class GithubError(Exception):
pass
class HttpResponse:
def __init__(self, raw: http.client.HTTPResponse) -> None:
self.raw = raw
def json(self) -> Any:
return json.load(self.raw)
def headers(self) -> http.client.HTTPMessage:
return self.raw.headers
def http_request(
url: str,
method: str = "GET",
headers: dict[str, str] | None = None,
data: dict[str, Any] | None = None,
) -> HttpResponse:
body = None
if data:
body = json.dumps(data).encode("ascii")
if headers is None:
headers = {}
headers = headers.copy()
headers["User-Agent"] = "buildbot-nix"
if not url.startswith("https:"):
msg = "url must be https: {url}"
raise GithubError(msg)
req = urllib.request.Request( # noqa: S310
url, headers=headers, method=method, data=body
)
try:
resp = urllib.request.urlopen(req) # noqa: S310
except urllib.request.HTTPError as e:
resp_body = ""
with contextlib.suppress(OSError, UnicodeDecodeError):
resp_body = e.fp.read().decode("utf-8", "replace")
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise GithubError(msg) from e
return HttpResponse(resp)
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
next_url: str | None = url
items = []
while next_url:
try:
res = http_request(
next_url,
headers={"Authorization": f"Bearer {token}"},
)
except OSError as e:
msg = f"failed to fetch {next_url}: {e}"
raise GithubError(msg) from e
next_url = None
link = res.headers()["Link"]
if link is not None:
links = link.split(", ")
for link in links: # pagination
link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1]
items += res.json()
return items
def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
repos = []

17
buildbot_nix/projects.py
View file

@ -3,15 +3,13 @@ from typing import Any
from buildbot.config.builder import BuilderConfig
from buildbot.reporters.base import ReporterBase
from buildbot.www.avatar import AvatarBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase
class GitBackend(ABC):
@abstractmethod
def create_reload_builder(
self,
worker_names: list[str]
) -> BuilderConfig:
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
pass
@abstractmethod
@ -19,11 +17,11 @@ class GitBackend(ABC):
pass
@abstractmethod
def create_change_hook(self, webhook_secret: str) -> dict[str, Any]:
def create_change_hook(self) -> dict[str, Any]:
pass
@abstractmethod
def create_avatar_method(self) -> AvatarBase:
def create_avatar_method(self) -> AvatarBase | None:
pass
@abstractmethod
@ -58,6 +56,7 @@ class GitBackend(ABC):
def change_hook_name(self) -> str:
pass
class GitProject(ABC):
@abstractmethod
def create_project_hook(
@ -65,12 +64,11 @@ class GitProject(ABC):
owner: str,
repo: str,
webhook_url: str,
webhook_secret: str,
) -> None:
pass
@abstractmethod
def get_project_url() -> str:
def get_project_url(self) -> str:
pass
@property
@ -113,7 +111,6 @@ class GitProject(ABC):
def default_branch(self) -> str:
pass
@property
@abstractmethod
def topics(self) -> list[str]:

4
buildbot_nix/secrets.py
View file

@ -1,6 +1,8 @@
import os, sys
import os
import sys
from pathlib import Path
def read_secret_file(secret_name: str) -> str:
directory = os.environ.get("CREDENTIALS_DIRECTORY")
if directory is None:

80
buildbot_nix/util.py Normal file
View file

@ -0,0 +1,80 @@
import contextlib
import http.client
import json
import urllib.request
from typing import Any
def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
next_url: str | None = url
items = []
while next_url:
try:
res = http_request(
next_url,
headers={"Authorization": f"Bearer {token}"},
)
except OSError as e:
msg = f"failed to fetch {next_url}: {e}"
raise HttpError(msg) from e
next_url = None
link = res.headers()["Link"]
if link is not None:
links = link.split(", ")
for link in links: # pagination
link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1]
items += res.json()
return items
class HttpResponse:
def __init__(self, raw: http.client.HTTPResponse) -> None:
self.raw = raw
def json(self) -> Any:
return json.load(self.raw)
def headers(self) -> http.client.HTTPMessage:
return self.raw.headers
class HttpError(Exception):
pass
def http_request(
url: str,
method: str = "GET",
headers: dict[str, str] | None = None,
data: dict[str, Any] | None = None,
) -> HttpResponse:
body = None
if data:
body = json.dumps(data).encode("ascii")
if headers is None:
headers = {}
headers = headers.copy()
headers["User-Agent"] = "buildbot-nix"
if not url.startswith("https:"):
msg = "url must be https: {url}"
raise HttpError(msg)
req = urllib.request.Request( # noqa: S310
url, headers=headers, method=method, data=body
)
try:
resp = urllib.request.urlopen(req) # noqa: S310
except urllib.request.HTTPError as e:
resp_body = ""
with contextlib.suppress(OSError, UnicodeDecodeError):
resp_body = e.fp.read().decode("utf-8", "replace")
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise HttpError(msg) from e
return HttpResponse(resp)

6
examples/master.nix
View file

@ -15,6 +15,9 @@
{ "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
]
'';
# Users in this list will be able to reload the project list.
# All other user in the organization will be able to restart builds or evaluations.
admins = [ "Mic92" ];
github = {
# Github user used as a CI identity
user = "mic92-buildbot";
@ -27,9 +30,6 @@
# After creating the app, press "Generate a new client secret" and fill in the client ID and secret below
oauthId = "aaaaaaaaaaaaaaaaaaaa";
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
# Users in this list will be able to reload the project list.
# All other user in the organization will be able to restart builds or evaluations.
admins = [ "Mic92" ];
# All github projects with this topic will be added to buildbot.
# One can trigger a project scan by visiting the Builds -> Builders page and looking for the "reload-github-project" builder.
# This builder has a "Update Github Projects" button that everyone in the github organization can use.

27
nix/buildbot-gitea.nix Normal file
View file

@ -0,0 +1,27 @@
{ buildPythonPackage
, fetchPypi
, lib
, pip
, buildbot
, requests
}:
buildPythonPackage (lib.fix (self: {
pname = "buildbot-gitea";
version = "1.8.0";
nativeBuildInputs = [
];
propagatedBuildInputs = [
pip
buildbot
requests
];
src = fetchPypi {
inherit (self) pname version;
hash = "sha256-zYcILPp42QuQyfEIzmYKV9vWf47sBAQI8FOKJlZ60yA=";
};
}))

2
nix/checks/master.nix
View file

@ -14,13 +14,13 @@
{ "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
]
'';
admins = [ "Mic92" ];
github = {
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
oauthId = "aaaaaaaaaaaaaaaaaaaa";
user = "mic92-buildbot";
admins = [ "Mic92" ];
};
};
};

85
nix/master.nix
View file

@ -5,8 +5,18 @@
}:
let
cfg = config.services.buildbot-nix.master;
inherit
(lib)
mkRenamedOptionModule
;
in
{
imports = [
(mkRenamedOptionModule
[ "services" "buildbot-nix" "master" "github" "admins" ]
[ "services" "buildbot-nix" "master" "admins" ])
];
options = {
services.buildbot-nix.master = {
enable = lib.mkEnableOption "buildbot-master";
@ -15,6 +25,13 @@ in
default = "postgresql://@/buildbot";
description = "Postgresql database url";
};
authBackend = lib.mkOption {
type = lib.types.enum [ "github" "gitea" "none" ];
default = "github";
description = ''
Which OAuth2 backend to use.
'';
};
cachix = {
name = lib.mkOption {
type = lib.types.nullOr lib.types.str;
@ -34,7 +51,42 @@ in
description = "Cachix auth token";
};
};
gitea = {
enable = lib.mkEnableOption "Enable Gitea integration";
tokenFile = lib.mkOption {
type = lib.types.path;
description = "Gitea token file";
};
webhookSecretFile = lib.mkOption {
type = lib.types.path;
description = "Github webhook secret file";
};
oauthSecretFile = lib.mkOption {
type = lib.types.path;
description = "Gitea oauth secret file";
};
instanceURL = lib.mkOption {
type = lib.types.str;
description = "Gitea instance URL";
};
oauthId = lib.mkOption {
type = lib.types.str;
description = "Gitea oauth id. Used for the login button";
};
topic = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "build-with-buildbot";
description = ''
Projects that have this topic will be built by buildbot.
If null, all projects that the buildbot Gitea user has access to, are built.
'';
};
};
github = {
disable = lib.mkEnableOption "Disable GitHub integration";
tokenFile = lib.mkOption {
type = lib.types.path;
description = "Github token file";
@ -62,11 +114,6 @@ in
type = lib.types.str;
description = "Github user that is used for the buildbot";
};
admins = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Users that are allowed to login to buildbot, trigger builds and change settings";
};
topic = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "build-with-buildbot";
@ -76,6 +123,11 @@ in
'';
};
};
admins = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Users that are allowed to login to buildbot, trigger builds and change settings";
};
workersFile = lib.mkOption {
type = lib.types.path;
description = "File containing a list of nix workers";
@ -144,7 +196,7 @@ in
home = "/var/lib/buildbot";
extraImports = ''
from datetime import timedelta
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig
'';
configurators = [
''
@ -152,18 +204,23 @@ in
''
''
NixConfigurator(
github=GithubConfig(
auth_backend=${builtins.toJSON cfg.authBackend},
github=${if cfg.github.disable then "None" else "GithubConfig(
oauth_id=${builtins.toJSON cfg.github.oauthId},
admins=${builtins.toJSON cfg.github.admins},
buildbot_user=${builtins.toJSON cfg.github.user},
topic=${builtins.toJSON cfg.github.topic},
),
gitea=None,
)"},
gitea=${if !cfg.gitea.enable then "None" else "GiteaConfig(
instance_url=${builtins.toJSON cfg.gitea.instanceURL},
oauth_id=${builtins.toJSON cfg.gitea.oauthId},
topic=${builtins.toJSON cfg.gitea.topic},
)"},
cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
name=${builtins.toJSON cfg.cachix.name},
signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"},
)"},
admins=${builtins.toJSON cfg.admins},
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
@ -190,6 +247,7 @@ in
(ps.toPythonModule pkgs.buildbot-worker)
pkgs.buildbot-plugins.www-react
(pkgs.python3.pkgs.callPackage ../default.nix { })
(pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = pkgs.buildbot; })
];
};
@ -206,7 +264,12 @@ in
++ lib.optional (cfg.cachix.signingKeyFile != null)
"cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
++ lib.optional (cfg.cachix.authTokenFile != null)
"cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
"cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
++ lib.optionals cfg.gitea.enable [
"gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
"gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
"gitea-token:${cfg.gitea.tokenFile}"
];
# Needed because it tries to reach out to github on boot.
# FIXME: if github is not available, we shouldn't fail buildbot, instead it should just try later again in the background