Merge pull request #124 from MagicRB/remove_github_assumption

Add support for Gitea
This commit is contained in:
Jörg Thalheim 2024-04-30 11:35:30 +02:00 committed by GitHub
commit 508ceb8856
Failed to generate hash of commit
11 changed files with 1027 additions and 322 deletions

View file

@ -2,8 +2,6 @@ import json
import multiprocessing
import os
import re
import signal
import sys
import uuid
from collections import defaultdict
from collections.abc import Generator
@ -11,30 +9,37 @@ from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any
from buildbot.config.builder import BuilderConfig
from buildbot.configurators import ConfiguratorBase
from buildbot.interfaces import WorkerSetupError
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
from buildbot.locks import MasterLock
from buildbot.plugins import schedulers, steps, util, worker
from buildbot.process import buildstep, logobserver, remotecommand
from buildbot.process.project import Project
from buildbot.process.properties import Interpolate, Properties
from buildbot.process.properties import Properties
from buildbot.process.results import ALL_RESULTS, statusToString
from buildbot.secrets.providers.file import SecretInAFile
from buildbot.steps.trigger import Trigger
from buildbot.www.authz import Authz
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
if TYPE_CHECKING:
from buildbot.process.log import Log
from buildbot.process.log import StreamLog
from buildbot.www.auth import AuthBase
from twisted.internet import defer, threads
from twisted.internet import defer
from twisted.logger import Logger
from twisted.python.failure import Failure
from .github_projects import (
GithubProject,
create_project_hook,
load_projects,
refresh_projects,
from .common import (
slugify_project_name,
)
from .gitea_projects import GiteaBackend, GiteaConfig
from .github_projects import (
GithubBackend,
GithubConfig,
)
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
SKIPPED_BUILDER_NAME = "skipped-builds"
@ -48,8 +53,11 @@ class BuildbotNixError(Exception):
class BuildTrigger(Trigger):
"""Dynamic trigger that creates a build for every attribute."""
project: GitProject
def __init__(
self,
project: GitProject,
builds_scheduler: str,
skipped_builds_scheduler: str,
jobs: list[dict[str, Any]],
@ -57,6 +65,7 @@ class BuildTrigger(Trigger):
) -> None:
if "name" not in kwargs:
kwargs["name"] = "trigger"
self.project = project
self.jobs = jobs
self.config = None
self.builds_scheduler = builds_scheduler
@ -78,10 +87,7 @@ class BuildTrigger(Trigger):
def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802
build_props = self.build.getProperties()
repo_name = build_props.getProperty(
"github.base.repo.full_name",
build_props.getProperty("github.repository.full_name"),
)
repo_name = self.project.name
project_id = slugify_project_name(repo_name)
source = f"nix-eval-{project_id}"
@ -145,9 +151,14 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
every attribute.
"""
def __init__(self, supported_systems: list[str], **kwargs: Any) -> None:
project: GitProject
def __init__(
self, project: GitProject, supported_systems: list[str], **kwargs: Any
) -> None:
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.project = project
self.observer = logobserver.BufferLogObserver()
self.addLogObserver("stdio", self.observer)
self.supported_systems = supported_systems
@ -172,11 +183,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
msg = f"Failed to parse line: {line}"
raise BuildbotNixError(msg) from e
jobs.append(job)
build_props = self.build.getProperties()
repo_name = build_props.getProperty(
"github.base.repo.full_name",
build_props.getProperty("github.repository.full_name"),
)
repo_name = self.project.name
project_id = slugify_project_name(repo_name)
filtered_jobs = []
for job in jobs:
@ -187,6 +194,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
self.build.addStepsAfterCurrentStep(
[
BuildTrigger(
self.project,
builds_scheduler=f"{project_id}-nix-build",
skipped_builds_scheduler=f"{project_id}-nix-skipped-build",
name="build flake",
@ -224,7 +232,7 @@ class EvalErrorStep(steps.BuildStep):
error = self.getProperty("error")
attr = self.getProperty("attr")
# show eval error
error_log: Log = yield self.addLog("nix_error")
error_log: StreamLog = yield self.addLog("nix_error")
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
return util.FAILURE
@ -256,15 +264,16 @@ class UpdateBuildOutput(steps.BuildStep):
on the target machine.
"""
def __init__(self, path: Path, **kwargs: Any) -> None:
project: GitProject
def __init__(self, project: GitProject, path: Path, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.project = project
self.path = path
def run(self) -> Generator[Any, object, Any]:
props = self.build.getProperties()
if props.getProperty("branch") != props.getProperty(
"github.repository.default_branch",
):
if props.getProperty("branch") != self.project.default_branch:
return util.SKIPPED
attr = Path(props.getProperty("attr")).name
@ -275,58 +284,6 @@ class UpdateBuildOutput(steps.BuildStep):
return util.SUCCESS
class ReloadGithubProjects(steps.BuildStep):
name = "reload_github_projects"
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
self.token = token
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
def reload_projects(self) -> None:
refresh_projects(self.token, self.project_cache_file)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: Log = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
def reload_github_projects(
worker_names: list[str],
github_token_secret: str,
project_cache_file: Path,
) -> util.BuilderConfig:
"""Updates the flake an opens a PR for it."""
factory = util.BuildFactory()
factory.addStep(
ReloadGithubProjects(
github_token_secret, project_cache_file=project_cache_file
),
)
return util.BuilderConfig(
name="reload-github-projects",
workernames=worker_names,
factory=factory,
)
# GitHub somtimes fires the PR webhook before it has computed the merge commit
# This is a workaround to fetch the merge commit and checkout the PR branch in CI
class GitLocalPrMerge(steps.Git):
@ -347,8 +304,7 @@ class GitLocalPrMerge(steps.Git):
return res
# The code below is a modified version of Git.run_vc
self.stdio_log: Log = yield self.addLogForRemoteCommands("stdio")
self.stdio_log: StreamLog = yield self.addLogForRemoteCommands("stdio")
self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n")
git_installed = yield self.checkFeatureSupport()
@ -392,22 +348,20 @@ class GitLocalPrMerge(steps.Git):
def nix_eval_config(
project: GithubProject,
project: GitProject,
worker_names: list[str],
github_token_secret: str,
git_url: str,
supported_systems: list[str],
eval_lock: util.MasterLock,
eval_lock: MasterLock,
worker_count: int,
max_memory_size: int,
) -> util.BuilderConfig:
) -> BuilderConfig:
"""Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
For each evaluated attribute a new build pipeline is started.
"""
factory = util.BuildFactory()
# check out the source
url_with_secret = util.Interpolate(
f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s",
)
url_with_secret = util.Interpolate(git_url)
factory.addStep(
GitLocalPrMerge(
repourl=url_with_secret,
@ -422,6 +376,7 @@ def nix_eval_config(
factory.addStep(
NixEvalCommand(
project=project,
env={},
name="evaluate flake",
supported_systems=supported_systems,
@ -482,11 +437,11 @@ class CachixConfig:
def nix_build_config(
project: GithubProject,
project: GitProject,
worker_names: list[str],
cachix: CachixConfig | None = None,
outputs_path: Path | None = None,
) -> util.BuilderConfig:
) -> BuilderConfig:
"""Builds one nix flake attribute."""
factory = util.BuildFactory()
factory.addStep(
@ -542,8 +497,7 @@ def nix_build_config(
"-r",
util.Property("out_path"),
],
doStepIf=lambda s: s.getProperty("branch")
== s.getProperty("github.repository.default_branch"),
doStepIf=lambda s: s.getProperty("branch") == project.default_branch,
),
)
factory.addStep(
@ -555,6 +509,7 @@ def nix_build_config(
if outputs_path is not None:
factory.addStep(
UpdateBuildOutput(
project=project,
name="Update build output",
path=outputs_path,
),
@ -570,9 +525,9 @@ def nix_build_config(
def nix_skipped_build_config(
project: GithubProject,
project: GitProject,
worker_names: list[str],
) -> util.BuilderConfig:
) -> BuilderConfig:
"""Dummy builder that is triggered when a build is skipped."""
factory = util.BuildFactory()
factory.addStep(
@ -601,42 +556,17 @@ def nix_skipped_build_config(
)
def read_secret_file(secret_name: str) -> str:
directory = os.environ.get("CREDENTIALS_DIRECTORY")
if directory is None:
print("directory not set", file=sys.stderr)
sys.exit(1)
return Path(directory).joinpath(secret_name).read_text().rstrip()
@dataclass
class GithubConfig:
oauth_id: str
admins: list[str]
buildbot_user: str
oauth_secret_name: str = "github-oauth-secret"
webhook_secret_name: str = "github-webhook-secret"
token_secret_name: str = "github-token"
project_cache_file: Path = Path("github-project-cache.json")
topic: str | None = "build-with-buildbot"
def token(self) -> str:
return read_secret_file(self.token_secret_name)
def config_for_project(
config: dict[str, Any],
project: GithubProject,
project: GitProject,
worker_names: list[str],
github: GithubConfig,
nix_supported_systems: list[str],
nix_eval_worker_count: int,
nix_eval_max_memory_size: int,
eval_lock: util.MasterLock,
eval_lock: MasterLock,
cachix: CachixConfig | None = None,
outputs_path: Path | None = None,
) -> Project:
) -> None:
config["projects"].append(Project(project.name))
config["schedulers"].extend(
[
@ -644,8 +574,7 @@ def config_for_project(
name=f"{project.project_id}-default-branch",
change_filter=util.ChangeFilter(
repository=project.url,
filter_fn=lambda c: c.branch
== c.properties.getProperty("github.repository.default_branch"),
filter_fn=lambda c: c.branch == project.default_branch,
),
builderNames=[f"{project.name}/nix-eval"],
treeStableTimer=5,
@ -685,7 +614,7 @@ def config_for_project(
properties=[
util.StringParameter(
name="project",
label="Name of the GitHub repository.",
label=f"Name of the {project.pretty_type} repository.",
default=project.name,
),
],
@ -699,7 +628,7 @@ def config_for_project(
nix_eval_config(
project,
worker_names,
github_token_secret=github.token_secret_name,
git_url=project.get_project_url(),
supported_systems=nix_supported_systems,
worker_count=nix_eval_worker_count,
max_memory_size=nix_eval_max_memory_size,
@ -717,9 +646,9 @@ def config_for_project(
def normalize_virtual_builder_name(name: str) -> str:
if name.startswith("github:"):
if re.match(r"^[^:]+:", name) is not None:
# rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build
match = re.match(r"github:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
match = re.match(r"[^:]:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
if match:
return f"{match['owner']}/{match['repo']}/nix-build"
@ -787,10 +716,12 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase):
return self.check_builder(epobject, epdict, "buildrequest")
def setup_authz(projects: list[GithubProject], admins: list[str]) -> util.Authz:
def setup_authz(
backends: list[GitBackend], projects: list[GitProject], admins: list[str]
) -> Authz:
allow_rules = []
allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict(
lambda: {"reload-github-projects"},
lambda: {backend.reload_builder_name for backend in backends},
)
for project in projects:
@ -836,7 +767,10 @@ class NixConfigurator(ConfiguratorBase):
def __init__(
self,
# Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
github: GithubConfig,
admins: list[str],
auth_backend: str,
github: GithubConfig | None,
gitea: GiteaConfig | None,
url: str,
nix_supported_systems: list[str],
nix_eval_worker_count: int | None,
@ -850,7 +784,10 @@ class NixConfigurator(ConfiguratorBase):
self.nix_eval_max_memory_size = nix_eval_max_memory_size
self.nix_eval_worker_count = nix_eval_worker_count
self.nix_supported_systems = nix_supported_systems
self.auth_backend = auth_backend
self.admins = admins
self.github = github
self.gitea = gitea
self.url = url
self.cachix = cachix
if outputs_path is None:
@ -859,9 +796,25 @@ class NixConfigurator(ConfiguratorBase):
self.outputs_path = Path(outputs_path)
def configure(self, config: dict[str, Any]) -> None:
projects = load_projects(self.github.token(), self.github.project_cache_file)
if self.github.topic is not None:
projects = [p for p in projects if self.github.topic in p.topics]
backends: dict[str, GitBackend] = {}
if self.github is not None:
backends["github"] = GithubBackend(self.github)
if self.gitea is not None:
backends["gitea"] = GiteaBackend(self.gitea)
auth: AuthBase | None = (
backends[self.auth_backend].create_auth()
if self.auth_backend != "none"
else None
)
projects: list[GitProject] = []
for backend in backends.values():
projects += backend.load_projects()
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
worker_names = []
@ -876,22 +829,14 @@ class NixConfigurator(ConfiguratorBase):
config["workers"].append(worker.Worker(worker_name, item["pass"]))
worker_names.append(worker_name)
webhook_secret = read_secret_file(self.github.webhook_secret_name)
eval_lock = util.MasterLock("nix-eval")
for project in projects:
create_project_hook(
project.owner,
project.repo,
self.github.token(),
self.url + "change_hook/github",
webhook_secret,
)
project.create_project_hook(project.owner, project.repo, self.url)
config_for_project(
config,
project,
worker_names,
self.github,
self.nix_supported_systems,
self.nix_eval_worker_count or multiprocessing.cpu_count(),
self.nix_eval_max_memory_size,
@ -900,42 +845,30 @@ class NixConfigurator(ConfiguratorBase):
self.outputs_path,
)
# Reload github projects
config["builders"].append(
reload_github_projects(
[worker_names[0]],
self.github.token(),
self.github.project_cache_file,
),
)
config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
config["schedulers"].extend(
[
schedulers.ForceScheduler(
name="reload-github-projects",
builderNames=["reload-github-projects"],
buttonName="Update projects",
),
# project list twice a day and on startup
PeriodicWithStartup(
name="reload-github-projects-bidaily",
builderNames=["reload-github-projects"],
periodicBuildTimer=12 * 60 * 60,
run_on_startup=not self.github.project_cache_file.exists(),
),
],
)
config["services"].append(
reporters.GitHubStatusPush(
token=self.github.token(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
),
)
systemd_secrets = secrets.SecretInAFile(
for backend in backends.values():
# Reload backend projects
config["builders"].append(backend.create_reload_builder([worker_names[0]]))
config["schedulers"].extend(
[
schedulers.ForceScheduler(
name=f"reload-{backend.type}-projects",
builderNames=[backend.reload_builder_name],
buttonName="Update projects",
),
# project list twice a day and on startup
PeriodicWithStartup(
name=f"reload-{backend.type}-projects-bidaily",
builderNames=[backend.reload_builder_name],
periodicBuildTimer=12 * 60 * 60,
run_on_startup=not backend.are_projects_cached(),
),
],
)
config["services"].append(backend.create_reporter())
systemd_secrets = SecretInAFile(
dirname=os.environ["CREDENTIALS_DIRECTORY"],
)
config["secretsProviders"].append(systemd_secrets)
@ -944,25 +877,24 @@ class NixConfigurator(ConfiguratorBase):
config["www"]["plugins"].update(dict(base_react={}))
config["www"].setdefault("change_hook_dialects", {})
config["www"]["change_hook_dialects"]["github"] = {
"secret": webhook_secret,
"strict": True,
"token": self.github.token(),
"github_property_whitelist": "*",
}
for backend in backends.values():
config["www"]["change_hook_dialects"][backend.change_hook_name] = (
backend.create_change_hook()
)
if "auth" not in config["www"]:
config["www"].setdefault("avatar_methods", [])
config["www"]["avatar_methods"].append(
util.AvatarGitHub(token=self.github.token()),
)
config["www"]["auth"] = util.GitHubAuth(
self.github.oauth_id,
read_secret_file(self.github.oauth_secret_name),
apiVersion=4,
)
for backend in backends.values():
avatar_method = backend.create_avatar_method()
if avatar_method is not None:
config["www"]["avatar_methods"].append(avatar_method)
# TODO one cannot have multiple auth backends...
if auth is not None:
config["www"]["auth"] = auth
config["www"]["authz"] = setup_authz(
admins=self.github.admins,
admins=self.admins,
backends=list(backends.values()),
projects=projects,
)

80
buildbot_nix/common.py Normal file
View file

@ -0,0 +1,80 @@
import contextlib
import http.client
import json
import urllib.request
from typing import Any
def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
next_url: str | None = url
items = []
while next_url:
try:
res = http_request(
next_url,
headers={"Authorization": f"Bearer {token}"},
)
except OSError as e:
msg = f"failed to fetch {next_url}: {e}"
raise HttpError(msg) from e
next_url = None
link = res.headers()["Link"]
if link is not None:
links = link.split(", ")
for link in links: # pagination
link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1]
items += res.json()
return items
class HttpResponse:
def __init__(self, raw: http.client.HTTPResponse) -> None:
self.raw = raw
def json(self) -> Any:
return json.load(self.raw)
def headers(self) -> http.client.HTTPMessage:
return self.raw.headers
class HttpError(Exception):
pass
def http_request(
url: str,
method: str = "GET",
headers: dict[str, str] | None = None,
data: dict[str, Any] | None = None,
) -> HttpResponse:
body = None
if data:
body = json.dumps(data).encode("ascii")
if headers is None:
headers = {}
headers = headers.copy()
headers["User-Agent"] = "buildbot-nix"
if not url.startswith("https:"):
msg = "url must be https: {url}"
raise HttpError(msg)
req = urllib.request.Request( # noqa: S310
url, headers=headers, method=method, data=body
)
try:
resp = urllib.request.urlopen(req) # noqa: S310
except urllib.request.HTTPError as e:
resp_body = ""
with contextlib.suppress(OSError, UnicodeDecodeError):
resp_body = e.fp.read().decode("utf-8", "replace")
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise HttpError(msg) from e
return HttpResponse(resp)

View file

@ -0,0 +1,300 @@
import json
import os
import signal
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase
from buildbot_gitea.auth import GiteaAuth # type: ignore[import]
from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import]
from twisted.internet import defer, threads
from twisted.python import log
from twisted.python.failure import Failure
from .common import (
http_request,
paginated_github_request,
slugify_project_name,
)
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
@dataclass
class GiteaConfig:
instance_url: str
oauth_id: str
admins: list[str]
oauth_secret_name: str = "gitea-oauth-secret"
token_secret_name: str = "gitea-token"
webhook_secret_name: str = "gitea-webhook-secret"
project_cache_file: Path = Path("gitea-project-cache.json")
topic: str | None = "build-with-buildbot"
def oauth_secret(self) -> str:
return read_secret_file(self.oauth_secret_name)
def token(self) -> str:
return read_secret_file(self.token_secret_name)
class GiteaProject(GitProject):
config: GiteaConfig
webhook_secret: str
data: dict[str, Any]
def __init__(
self, config: GiteaConfig, webhook_secret: str, data: dict[str, Any]
) -> None:
self.config = config
self.webhook_secret = webhook_secret
self.data = data
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
hooks = paginated_github_request(
f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
self.config.token(),
)
config = dict(
url=webhook_url + "change_hook/gitea",
content_type="json",
insecure_ssl="0",
secret=self.webhook_secret,
)
data = dict(
name="web",
active=True,
events=["push", "pull_request"],
config=config,
type="gitea",
)
headers = {
"Authorization": f"token {self.config.token()}",
"Accept": "application/json",
"Content-Type": "application/json",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/gitea":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def get_project_url(self) -> str:
return f"https://git:%(secret:{self.config.token_secret_name})s@{self.config.instance_url}/{self.name}"
@property
def pretty_type(self) -> str:
return "Gitea"
@property
def type(self) -> str:
return "gitea"
@property
def repo(self) -> str:
return self.data["name"]
@property
def owner(self) -> str:
return self.data["owner"]["login"]
@property
def name(self) -> str:
return self.data["full_name"]
@property
def url(self) -> str:
# not `html_url` because https://github.com/lab132/buildbot-gitea/blob/f569a2294ea8501ef3bcc5d5b8c777dfdbf26dcc/buildbot_gitea/webhook.py#L34
return self.data["ssh_url"]
@property
def project_id(self) -> str:
return slugify_project_name(self.data["full_name"])
@property
def default_branch(self) -> str:
return self.data["default_branch"]
@property
def topics(self) -> list[str]:
# note that Gitea doesn't by default put this data here, we splice it in, in `refresh_projects`
return self.data["topics"]
@property
def belongs_to_org(self) -> bool:
# TODO Gitea doesn't include this information
return False # self.data["owner"]["type"] == "Organization"
class GiteaBackend(GitBackend):
config: GiteaConfig
def __init__(self, config: GiteaConfig) -> None:
self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it."""
factory = util.BuildFactory()
factory.addStep(
ReloadGiteaProjects(self.config, self.config.project_cache_file),
)
return util.BuilderConfig(
name=self.reload_builder_name,
workernames=worker_names,
factory=factory,
)
def create_reporter(self) -> ReporterBase:
return GiteaStatusPush(
"https://" + self.config.instance_url,
Interpolate(self.config.token()),
context=Interpolate("buildbot/%(prop:status_name)s"),
context_pr=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_change_hook(self) -> dict[str, Any]:
return {
"secret": self.webhook_secret,
}
def create_avatar_method(self) -> AvatarBase | None:
return None
def create_auth(self) -> AuthBase:
return GiteaAuth(
"https://" + self.config.instance_url,
self.config.oauth_id,
self.config.oauth_secret(),
)
def load_projects(self) -> list["GitProject"]:
if not self.config.project_cache_file.exists():
return []
repos: list[dict[str, Any]] = sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
)
return list(
filter(
lambda project: self.config.topic is not None
and self.config.topic in project.topics,
[
GiteaProject(self.config, self.webhook_secret, repo)
for repo in repos
],
)
)
def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists()
@property
def type(self) -> str:
return "gitea"
@property
def pretty_type(self) -> str:
return "Gitea"
@property
def reload_builder_name(self) -> str:
return "reload-gitea-projects"
@property
def change_hook_name(self) -> str:
return "gitea"
class ReloadGiteaProjects(BuildStep):
name = "reload_gitea_projects"
config: GiteaConfig
def __init__(
self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any
) -> None:
self.config = config
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
def reload_projects(self) -> None:
refresh_projects(self.config, self.project_cache_file)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
yield self.addLog("log").addStderr(
f"Failed to reload project list: {self.error_msg}"
)
return util.FAILURE
def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
repos = []
for repo in paginated_github_request(
f"https://{config.instance_url}/api/v1/user/repos?limit=100",
config.token(),
):
if not repo["permissions"]["admin"]:
name = repo["full_name"]
log.msg(
f"skipping {name} because we do not have admin privileges, needed for hook management",
)
else:
try:
# Gitea doesn't include topics in the default repo listing, unlike GitHub
topics: list[str] = http_request(
f"https://{config.instance_url}/api/v1/repos/{repo['owner']['login']}/{repo['name']}/topics",
headers={"Authorization": f"token {config.token}"},
).json()["topics"]
repo["topics"] = topics
repos.append(repo)
except OSError:
pass
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
path = Path(f.name)
try:
f.write(json.dumps(repos))
f.flush()
path.rename(repo_cache_file)
except OSError:
path.unlink()
raise

View file

@ -1,93 +1,229 @@
import contextlib
import http.client
import json
import urllib.request
import os
import signal
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase
from buildbot.reporters.github import GitHubStatusPush
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.oauth2 import GitHubAuth
from twisted.internet import defer, threads
from twisted.python import log
from twisted.python.failure import Failure
from .common import (
http_request,
paginated_github_request,
slugify_project_name,
)
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
class GithubError(Exception):
pass
class ReloadGithubProjects(BuildStep):
name = "reload_github_projects"
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
self.token = token
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
class HttpResponse:
def __init__(self, raw: http.client.HTTPResponse) -> None:
self.raw = raw
def reload_projects(self) -> None:
refresh_projects(self.token, self.project_cache_file)
def json(self) -> Any:
return json.load(self.raw)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
def headers(self) -> http.client.HTTPMessage:
return self.raw.headers
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
def http_request(
url: str,
method: str = "GET",
headers: dict[str, str] | None = None,
data: dict[str, Any] | None = None,
) -> HttpResponse:
body = None
if data:
body = json.dumps(data).encode("ascii")
if headers is None:
headers = {}
headers = headers.copy()
headers["User-Agent"] = "buildbot-nix"
if not url.startswith("https:"):
msg = "url must be https: {url}"
raise GithubError(msg)
req = urllib.request.Request( # noqa: S310
url, headers=headers, method=method, data=body
)
try:
resp = urllib.request.urlopen(req) # noqa: S310
except urllib.request.HTTPError as e:
resp_body = ""
with contextlib.suppress(OSError, UnicodeDecodeError):
resp_body = e.fp.read().decode("utf-8", "replace")
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise GithubError(msg) from e
return HttpResponse(resp)
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
next_url: str | None = url
items = []
while next_url:
try:
res = http_request(
next_url,
headers={"Authorization": f"Bearer {token}"},
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
yield self.addLog("log").addStderr(
f"Failed to reload project list: {self.error_msg}"
)
except OSError as e:
msg = f"failed to fetch {next_url}: {e}"
raise GithubError(msg) from e
next_url = None
link = res.headers()["Link"]
if link is not None:
links = link.split(", ")
for link in links: # pagination
link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1]
items += res.json()
return items
return util.FAILURE
def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
@dataclass
class GithubConfig:
oauth_id: str
# TODO unused
buildbot_user: str
oauth_secret_name: str = "github-oauth-secret"
token_secret_name: str = "github-token"
webhook_secret_name: str = "github-webhook-secret"
project_cache_file: Path = Path("github-project-cache.json")
topic: str | None = "build-with-buildbot"
def token(self) -> str:
return read_secret_file(self.token_secret_name)
class GithubProject:
def __init__(self, data: dict[str, Any]) -> None:
@dataclass
class GithubBackend(GitBackend):
config: GithubConfig
webhook_secret: str
def __init__(self, config: GithubConfig) -> None:
self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it."""
factory = util.BuildFactory()
factory.addStep(
ReloadGithubProjects(self.config.token(), self.config.project_cache_file),
)
return util.BuilderConfig(
name=self.reload_builder_name,
workernames=worker_names,
factory=factory,
)
def create_reporter(self) -> ReporterBase:
return GitHubStatusPush(
token=self.config.token(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_change_hook(self) -> dict[str, Any]:
return {
"secret": self.webhook_secret,
"strict": True,
"token": self.config.token(),
"github_property_whitelist": "*",
}
def create_avatar_method(self) -> AvatarBase | None:
return AvatarGitHub(token=self.config.token())
def create_auth(self) -> AuthBase:
return GitHubAuth(
self.config.oauth_id,
read_secret_file(self.config.oauth_secret_name),
apiVersion=4,
)
def load_projects(self) -> list["GitProject"]:
if not self.config.project_cache_file.exists():
return []
repos: list[dict[str, Any]] = sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
)
return list(
filter(
lambda project: self.config.topic is not None
and self.config.topic in project.topics,
[
GithubProject(self.config, self.webhook_secret, repo)
for repo in repos
],
)
)
def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists()
@property
def type(self) -> str:
return "github"
@property
def pretty_type(self) -> str:
return "GitHub"
@property
def reload_builder_name(self) -> str:
return "reload-github-projects"
@property
def change_hook_name(self) -> str:
return "github"
class GithubProject(GitProject):
config: GithubConfig
def __init__(
self, config: GithubConfig, webhook_secret: str, data: dict[str, Any]
) -> None:
self.config = config
self.webhook_secret = webhook_secret
self.data = data
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
self.config.token(),
)
config = dict(
url=webhook_url + "change_hook/github",
content_type="json",
insecure_ssl="0",
secret=self.webhook_secret,
)
data = dict(
name="web", active=True, events=["push", "pull_request"], config=config
)
headers = {
"Authorization": f"Bearer {self.config.token()}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/github":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def get_project_url(self) -> str:
return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}"
@property
def pretty_type(self) -> str:
return "GitHub"
@property
def type(self) -> str:
return "github"
@property
def repo(self) -> str:
return self.data["name"]
@ -121,43 +257,6 @@ class GithubProject:
return self.data["owner"]["type"] == "Organization"
def create_project_hook(
owner: str,
repo: str,
token: str,
webhook_url: str,
webhook_secret: str,
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
token,
)
config = dict(
url=webhook_url,
content_type="json",
insecure_ssl="0",
secret=webhook_secret,
)
data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
headers = {
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url:
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
repos = []
@ -182,13 +281,3 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
except OSError:
path.unlink()
raise
def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]:
if not repo_cache_file.exists():
return []
repos: list[dict[str, Any]] = sorted(
json.loads(repo_cache_file.read_text()), key=lambda x: x["full_name"]
)
return [GithubProject(repo) for repo in repos]

122
buildbot_nix/projects.py Normal file
View file

@ -0,0 +1,122 @@
from abc import ABC, abstractmethod
from typing import Any
from buildbot.config.builder import BuilderConfig
from buildbot.reporters.base import ReporterBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase
class GitBackend(ABC):
@abstractmethod
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
pass
@abstractmethod
def create_reporter(self) -> ReporterBase:
pass
@abstractmethod
def create_change_hook(self) -> dict[str, Any]:
pass
@abstractmethod
def create_avatar_method(self) -> AvatarBase | None:
pass
@abstractmethod
def create_auth(self) -> AuthBase:
pass
@abstractmethod
def load_projects(self) -> list["GitProject"]:
pass
@abstractmethod
def are_projects_cached(self) -> bool:
pass
@property
@abstractmethod
def pretty_type(self) -> str:
pass
@property
@abstractmethod
def type(self) -> str:
pass
@property
@abstractmethod
def reload_builder_name(self) -> str:
pass
@property
@abstractmethod
def change_hook_name(self) -> str:
pass
class GitProject(ABC):
@abstractmethod
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
pass
@abstractmethod
def get_project_url(self) -> str:
pass
@property
@abstractmethod
def pretty_type(self) -> str:
pass
@property
@abstractmethod
def type(self) -> str:
pass
@property
@abstractmethod
def repo(self) -> str:
pass
@property
@abstractmethod
def owner(self) -> str:
pass
@property
@abstractmethod
def name(self) -> str:
pass
@property
@abstractmethod
def url(self) -> str:
pass
@property
@abstractmethod
def project_id(self) -> str:
pass
@property
@abstractmethod
def default_branch(self) -> str:
pass
@property
@abstractmethod
def topics(self) -> list[str]:
pass
@property
@abstractmethod
def belongs_to_org(self) -> bool:
pass

11
buildbot_nix/secrets.py Normal file
View file

@ -0,0 +1,11 @@
import os
import sys
from pathlib import Path
def read_secret_file(secret_name: str) -> str:
directory = os.environ.get("CREDENTIALS_DIRECTORY")
if directory is None:
print("directory not set", file=sys.stderr)
sys.exit(1)
return Path(directory).joinpath(secret_name).read_text().rstrip()

80
buildbot_nix/util.py Normal file
View file

@ -0,0 +1,80 @@
import contextlib
import http.client
import json
import urllib.request
from typing import Any
def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
next_url: str | None = url
items = []
while next_url:
try:
res = http_request(
next_url,
headers={"Authorization": f"Bearer {token}"},
)
except OSError as e:
msg = f"failed to fetch {next_url}: {e}"
raise HttpError(msg) from e
next_url = None
link = res.headers()["Link"]
if link is not None:
links = link.split(", ")
for link in links: # pagination
link_parts = link.split(";")
if link_parts[1].strip() == 'rel="next"':
next_url = link_parts[0][1:-1]
items += res.json()
return items
class HttpResponse:
def __init__(self, raw: http.client.HTTPResponse) -> None:
self.raw = raw
def json(self) -> Any:
return json.load(self.raw)
def headers(self) -> http.client.HTTPMessage:
return self.raw.headers
class HttpError(Exception):
pass
def http_request(
url: str,
method: str = "GET",
headers: dict[str, str] | None = None,
data: dict[str, Any] | None = None,
) -> HttpResponse:
body = None
if data:
body = json.dumps(data).encode("ascii")
if headers is None:
headers = {}
headers = headers.copy()
headers["User-Agent"] = "buildbot-nix"
if not url.startswith("https:"):
msg = "url must be https: {url}"
raise HttpError(msg)
req = urllib.request.Request( # noqa: S310
url, headers=headers, method=method, data=body
)
try:
resp = urllib.request.urlopen(req) # noqa: S310
except urllib.request.HTTPError as e:
resp_body = ""
with contextlib.suppress(OSError, UnicodeDecodeError):
resp_body = e.fp.read().decode("utf-8", "replace")
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
raise HttpError(msg) from e
return HttpResponse(resp)

View file

@ -15,6 +15,9 @@
{ "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
]
'';
# Users in this list will be able to reload the project list.
# All other user in the organization will be able to restart builds or evaluations.
admins = [ "Mic92" ];
github = {
# Github user used as a CI identity
user = "mic92-buildbot";
@ -27,9 +30,6 @@
# After creating the app, press "Generate a new client secret" and fill in the client ID and secret below
oauthId = "aaaaaaaaaaaaaaaaaaaa";
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
# Users in this list will be able to reload the project list.
# All other user in the organization will be able to restart builds or evaluations.
admins = [ "Mic92" ];
# All github projects with this topic will be added to buildbot.
# One can trigger a project scan by visiting the Builds -> Builders page and looking for the "reload-github-project" builder.
# This builder has a "Update Github Projects" button that everyone in the github organization can use.

27
nix/buildbot-gitea.nix Normal file
View file

@ -0,0 +1,27 @@
{ buildPythonPackage
, fetchPypi
, lib
, pip
, buildbot
, requests
}:
buildPythonPackage (lib.fix (self: {
pname = "buildbot-gitea";
version = "1.8.0";
nativeBuildInputs = [
];
propagatedBuildInputs = [
pip
buildbot
requests
];
src = fetchPypi {
inherit (self) pname version;
hash = "sha256-zYcILPp42QuQyfEIzmYKV9vWf47sBAQI8FOKJlZ60yA=";
};
}))

View file

@ -14,13 +14,13 @@
{ "name": "eve", "pass": "XXXXXXXXXXXXXXXXXXXX", "cores": 16 }
]
'';
admins = [ "Mic92" ];
github = {
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
oauthId = "aaaaaaaaaaaaaaaaaaaa";
user = "mic92-buildbot";
admins = [ "Mic92" ];
};
};
};

View file

@ -5,8 +5,18 @@
}:
let
cfg = config.services.buildbot-nix.master;
inherit
(lib)
mkRenamedOptionModule
;
in
{
imports = [
(mkRenamedOptionModule
[ "services" "buildbot-nix" "master" "github" "admins" ]
[ "services" "buildbot-nix" "master" "admins" ])
];
options = {
services.buildbot-nix.master = {
enable = lib.mkEnableOption "buildbot-master";
@ -15,6 +25,13 @@ in
default = "postgresql://@/buildbot";
description = "Postgresql database url";
};
authBackend = lib.mkOption {
type = lib.types.enum [ "github" "gitea" "none" ];
default = "github";
description = ''
Which OAuth2 backend to use.
'';
};
cachix = {
name = lib.mkOption {
type = lib.types.nullOr lib.types.str;
@ -34,7 +51,42 @@ in
description = "Cachix auth token";
};
};
gitea = {
enable = lib.mkEnableOption "Enable Gitea integration";
tokenFile = lib.mkOption {
type = lib.types.path;
description = "Gitea token file";
};
webhookSecretFile = lib.mkOption {
type = lib.types.path;
description = "Github webhook secret file";
};
oauthSecretFile = lib.mkOption {
type = lib.types.path;
description = "Gitea oauth secret file";
};
instanceURL = lib.mkOption {
type = lib.types.str;
description = "Gitea instance URL";
};
oauthId = lib.mkOption {
type = lib.types.str;
description = "Gitea oauth id. Used for the login button";
};
topic = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "build-with-buildbot";
description = ''
Projects that have this topic will be built by buildbot.
If null, all projects that the buildbot Gitea user has access to, are built.
'';
};
};
github = {
disable = lib.mkEnableOption "Disable GitHub integration";
tokenFile = lib.mkOption {
type = lib.types.path;
description = "Github token file";
@ -62,11 +114,6 @@ in
type = lib.types.str;
description = "Github user that is used for the buildbot";
};
admins = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Users that are allowed to login to buildbot, trigger builds and change settings";
};
topic = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "build-with-buildbot";
@ -76,6 +123,11 @@ in
'';
};
};
admins = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Users that are allowed to login to buildbot, trigger builds and change settings";
};
workersFile = lib.mkOption {
type = lib.types.path;
description = "File containing a list of nix workers";
@ -144,7 +196,7 @@ in
home = "/var/lib/buildbot";
extraImports = ''
from datetime import timedelta
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig
'';
configurators = [
''
@ -152,17 +204,23 @@ in
''
''
NixConfigurator(
github=GithubConfig(
auth_backend=${builtins.toJSON cfg.authBackend},
github=${if cfg.github.disable then "None" else "GithubConfig(
oauth_id=${builtins.toJSON cfg.github.oauthId},
admins=${builtins.toJSON cfg.github.admins},
buildbot_user=${builtins.toJSON cfg.github.user},
topic=${builtins.toJSON cfg.github.topic},
),
)"},
gitea=${if !cfg.gitea.enable then "None" else "GiteaConfig(
instance_url=${builtins.toJSON cfg.gitea.instanceURL},
oauth_id=${builtins.toJSON cfg.gitea.oauthId},
topic=${builtins.toJSON cfg.gitea.topic},
)"},
cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
name=${builtins.toJSON cfg.cachix.name},
signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"},
)"},
admins=${builtins.toJSON cfg.admins},
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
@ -189,6 +247,7 @@ in
(ps.toPythonModule pkgs.buildbot-worker)
pkgs.buildbot-plugins.www-react
(pkgs.python3.pkgs.callPackage ../default.nix { })
(pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix { buildbot = pkgs.buildbot; })
];
};
@ -205,7 +264,12 @@ in
++ lib.optional (cfg.cachix.signingKeyFile != null)
"cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
++ lib.optional (cfg.cachix.authTokenFile != null)
"cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
"cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
++ lib.optionals cfg.gitea.enable [
"gitea-oauth-secret:${cfg.gitea.oauthSecretFile}"
"gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
"gitea-token:${cfg.gitea.tokenFile}"
];
# Needed because it tries to reach out to github on boot.
# FIXME: if github is not available, we shouldn't fail buildbot, instead it should just try later again in the background