Remove assumption of GitHub being the only forge
Signed-off-by: magic_rb <richard@brezak.sk>
This commit is contained in:
magic_rb
Jörg Thalheim
parent
35a1162d84
commit
1605d2d3c2
|
|
@ -9,17 +9,23 @@ from collections import defaultdict
|
|||
from collections.abc import Generator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
import typing
|
||||
|
||||
from buildbot.process.log import StreamLog
|
||||
from buildbot.configurators import ConfiguratorBase
|
||||
from buildbot.interfaces import WorkerSetupError
|
||||
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
|
||||
from buildbot.plugins import reporters, schedulers, steps, util, worker
|
||||
from buildbot.process import buildstep, logobserver, remotecommand
|
||||
from buildbot.process.project import Project
|
||||
from buildbot.process.properties import Interpolate, Properties
|
||||
from buildbot.process.results import ALL_RESULTS, statusToString
|
||||
from buildbot.steps.trigger import Trigger
|
||||
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
|
||||
from buildbot.www.authz import Authz
|
||||
from buildbot.secrets.providers.file import SecretInAFile
|
||||
from buildbot.locks import MasterLock
|
||||
from buildbot.config.builder import BuilderConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from buildbot.process.log import Log
|
||||
|
|
@ -28,14 +34,25 @@ from twisted.internet import defer, threads
|
|||
from twisted.logger import Logger
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from .gitea_projects import (
|
||||
GiteaConfig
|
||||
)
|
||||
|
||||
from .github_projects import (
|
||||
GithubProject,
|
||||
create_project_hook,
|
||||
load_projects,
|
||||
refresh_projects,
|
||||
GithubBackend,
|
||||
GithubConfig,
|
||||
slugify_project_name,
|
||||
)
|
||||
|
||||
from .projects import (
|
||||
GitProject,
|
||||
GitBackend
|
||||
)
|
||||
|
||||
from .secrets import (
|
||||
read_secret_file
|
||||
)
|
||||
|
||||
SKIPPED_BUILDER_NAME = "skipped-builds"
|
||||
|
||||
log = Logger()
|
||||
|
|
@ -48,8 +65,11 @@ class BuildbotNixError(Exception):
|
|||
class BuildTrigger(Trigger):
|
||||
"""Dynamic trigger that creates a build for every attribute."""
|
||||
|
||||
project: GitProject
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project: GitProject,
|
||||
builds_scheduler: str,
|
||||
skipped_builds_scheduler: str,
|
||||
jobs: list[dict[str, Any]],
|
||||
|
|
@ -57,6 +77,7 @@ class BuildTrigger(Trigger):
|
|||
) -> None:
|
||||
if "name" not in kwargs:
|
||||
kwargs["name"] = "trigger"
|
||||
self.project = project
|
||||
self.jobs = jobs
|
||||
self.config = None
|
||||
self.builds_scheduler = builds_scheduler
|
||||
|
|
@ -77,11 +98,9 @@ class BuildTrigger(Trigger):
|
|||
return props
|
||||
|
||||
def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802
|
||||
build_props = self.build.getProperties()
|
||||
repo_name = build_props.getProperty(
|
||||
"github.base.repo.full_name",
|
||||
build_props.getProperty("github.repository.full_name"),
|
||||
)
|
||||
# TODO when is this None?
|
||||
build_props = self.build.getProperties() if self.build is not None else Properties()
|
||||
repo_name = self.project.name
|
||||
project_id = slugify_project_name(repo_name)
|
||||
source = f"nix-eval-{project_id}"
|
||||
|
||||
|
|
@ -145,9 +164,12 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
|||
every attribute.
|
||||
"""
|
||||
|
||||
def __init__(self, supported_systems: list[str], **kwargs: Any) -> None:
|
||||
project: GitProject
|
||||
|
||||
def __init__(self, project: GitProject, supported_systems: list[str], **kwargs: Any) -> None:
|
||||
kwargs = self.setupShellMixin(kwargs)
|
||||
super().__init__(**kwargs)
|
||||
self.project = project
|
||||
self.observer = logobserver.BufferLogObserver()
|
||||
self.addLogObserver("stdio", self.observer)
|
||||
self.supported_systems = supported_systems
|
||||
|
|
@ -155,7 +177,9 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
|||
@defer.inlineCallbacks
|
||||
def run(self) -> Generator[Any, object, Any]:
|
||||
# run nix-eval-jobs --flake .#checks to generate the dict of stages
|
||||
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
|
||||
cmd_: object = yield self.makeRemoteShellCommand()
|
||||
# TODO why doesn't type information pass through yield again?
|
||||
cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_)
|
||||
yield self.runCommand(cmd)
|
||||
|
||||
# if the command passes extract the list of stages
|
||||
|
|
@ -172,11 +196,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
|||
msg = f"Failed to parse line: {line}"
|
||||
raise BuildbotNixError(msg) from e
|
||||
jobs.append(job)
|
||||
build_props = self.build.getProperties()
|
||||
repo_name = build_props.getProperty(
|
||||
"github.base.repo.full_name",
|
||||
build_props.getProperty("github.repository.full_name"),
|
||||
)
|
||||
repo_name = self.project.name
|
||||
project_id = slugify_project_name(repo_name)
|
||||
filtered_jobs = []
|
||||
for job in jobs:
|
||||
|
|
@ -187,6 +207,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
|||
self.build.addStepsAfterCurrentStep(
|
||||
[
|
||||
BuildTrigger(
|
||||
self.project,
|
||||
builds_scheduler=f"{project_id}-nix-build",
|
||||
skipped_builds_scheduler=f"{project_id}-nix-skipped-build",
|
||||
name="build flake",
|
||||
|
|
@ -224,7 +245,9 @@ class EvalErrorStep(steps.BuildStep):
|
|||
error = self.getProperty("error")
|
||||
attr = self.getProperty("attr")
|
||||
# show eval error
|
||||
error_log: Log = yield self.addLog("nix_error")
|
||||
# TODO why doesn't type information pass through yield again?
|
||||
error_log_: object = yield self.addLog("nix_error")
|
||||
error_log: StreamLog = typing.cast(StreamLog, error_log_);
|
||||
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
|
||||
return util.FAILURE
|
||||
|
||||
|
|
@ -239,7 +262,9 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
|||
@defer.inlineCallbacks
|
||||
def run(self) -> Generator[Any, object, Any]:
|
||||
# run `nix build`
|
||||
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
|
||||
# TODO why doesn't type information pass through yield again?
|
||||
cmd_: object = yield self.makeRemoteShellCommand()
|
||||
cmd: remotecommand.RemoteCommand = typing.cast(remotecommand.RemoteCommand, cmd_)
|
||||
yield self.runCommand(cmd)
|
||||
|
||||
res = cmd.results()
|
||||
|
|
@ -256,15 +281,16 @@ class UpdateBuildOutput(steps.BuildStep):
|
|||
on the target machine.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path, **kwargs: Any) -> None:
|
||||
project: GitProject
|
||||
|
||||
def __init__(self, project: GitProject, path: Path, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.project = project
|
||||
self.path = path
|
||||
|
||||
def run(self) -> Generator[Any, object, Any]:
|
||||
props = self.build.getProperties()
|
||||
if props.getProperty("branch") != props.getProperty(
|
||||
"github.repository.default_branch",
|
||||
):
|
||||
if props.getProperty("branch") != self.project.default_branch:
|
||||
return util.SKIPPED
|
||||
|
||||
attr = Path(props.getProperty("attr")).name
|
||||
|
|
@ -274,62 +300,11 @@ class UpdateBuildOutput(steps.BuildStep):
|
|||
(self.path / attr).write_text(out_path)
|
||||
return util.SUCCESS
|
||||
|
||||
|
||||
class ReloadGithubProjects(steps.BuildStep):
|
||||
name = "reload_github_projects"
|
||||
|
||||
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
|
||||
self.token = token
|
||||
self.project_cache_file = project_cache_file
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def reload_projects(self) -> None:
|
||||
refresh_projects(self.token, self.project_cache_file)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(self) -> Generator[Any, object, Any]:
|
||||
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
|
||||
|
||||
self.error_msg = ""
|
||||
|
||||
def error_cb(failure: Failure) -> int:
|
||||
self.error_msg += failure.getTraceback()
|
||||
return util.FAILURE
|
||||
|
||||
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
|
||||
res = yield d
|
||||
if res == util.SUCCESS:
|
||||
# reload the buildbot config
|
||||
os.kill(os.getpid(), signal.SIGHUP)
|
||||
return util.SUCCESS
|
||||
else:
|
||||
log: Log = yield self.addLog("log")
|
||||
log.addStderr(f"Failed to reload project list: {self.error_msg}")
|
||||
return util.FAILURE
|
||||
|
||||
|
||||
def reload_github_projects(
|
||||
worker_names: list[str],
|
||||
github_token_secret: str,
|
||||
project_cache_file: Path,
|
||||
) -> util.BuilderConfig:
|
||||
"""Updates the flake an opens a PR for it."""
|
||||
factory = util.BuildFactory()
|
||||
factory.addStep(
|
||||
ReloadGithubProjects(
|
||||
github_token_secret, project_cache_file=project_cache_file
|
||||
),
|
||||
)
|
||||
return util.BuilderConfig(
|
||||
name="reload-github-projects",
|
||||
workernames=worker_names,
|
||||
factory=factory,
|
||||
)
|
||||
|
||||
|
||||
# GitHub somtimes fires the PR webhook before it has computed the merge commit
|
||||
# This is a workaround to fetch the merge commit and checkout the PR branch in CI
|
||||
class GitLocalPrMerge(steps.Git):
|
||||
stdio_log: StreamLog
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_vc(
|
||||
self,
|
||||
|
|
@ -347,8 +322,9 @@ class GitLocalPrMerge(steps.Git):
|
|||
return res
|
||||
|
||||
# The code below is a modified version of Git.run_vc
|
||||
self.stdio_log: Log = yield self.addLogForRemoteCommands("stdio")
|
||||
|
||||
# TODO why doesn't type information pass through yield again?
|
||||
stdio_log_: object = yield self.addLogForRemoteCommands("stdio")
|
||||
self.stdio_log = typing.cast(StreamLog, stdio_log_)
|
||||
self.stdio_log.addStdout(f"Merging {merge_base} into {pr_head}\n")
|
||||
|
||||
git_installed = yield self.checkFeatureSupport()
|
||||
|
|
@ -392,22 +368,20 @@ class GitLocalPrMerge(steps.Git):
|
|||
|
||||
|
||||
def nix_eval_config(
|
||||
project: GithubProject,
|
||||
project: GitProject,
|
||||
worker_names: list[str],
|
||||
github_token_secret: str,
|
||||
git_url: str,
|
||||
supported_systems: list[str],
|
||||
eval_lock: util.MasterLock,
|
||||
eval_lock: MasterLock,
|
||||
worker_count: int,
|
||||
max_memory_size: int,
|
||||
) -> util.BuilderConfig:
|
||||
) -> BuilderConfig:
|
||||
"""Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
|
||||
For each evaluated attribute a new build pipeline is started.
|
||||
"""
|
||||
factory = util.BuildFactory()
|
||||
# check out the source
|
||||
url_with_secret = util.Interpolate(
|
||||
f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s",
|
||||
)
|
||||
url_with_secret = util.Interpolate(git_url)
|
||||
factory.addStep(
|
||||
GitLocalPrMerge(
|
||||
repourl=url_with_secret,
|
||||
|
|
@ -422,6 +396,7 @@ def nix_eval_config(
|
|||
|
||||
factory.addStep(
|
||||
NixEvalCommand(
|
||||
project=project,
|
||||
env={},
|
||||
name="evaluate flake",
|
||||
supported_systems=supported_systems,
|
||||
|
|
@ -482,11 +457,11 @@ class CachixConfig:
|
|||
|
||||
|
||||
def nix_build_config(
|
||||
project: GithubProject,
|
||||
project: GitProject,
|
||||
worker_names: list[str],
|
||||
cachix: CachixConfig | None = None,
|
||||
outputs_path: Path | None = None,
|
||||
) -> util.BuilderConfig:
|
||||
) -> BuilderConfig:
|
||||
"""Builds one nix flake attribute."""
|
||||
factory = util.BuildFactory()
|
||||
factory.addStep(
|
||||
|
|
@ -543,7 +518,7 @@ def nix_build_config(
|
|||
util.Property("out_path"),
|
||||
],
|
||||
doStepIf=lambda s: s.getProperty("branch")
|
||||
== s.getProperty("github.repository.default_branch"),
|
||||
== project.default_branch,
|
||||
),
|
||||
)
|
||||
factory.addStep(
|
||||
|
|
@ -555,6 +530,7 @@ def nix_build_config(
|
|||
if outputs_path is not None:
|
||||
factory.addStep(
|
||||
UpdateBuildOutput(
|
||||
project=project,
|
||||
name="Update build output",
|
||||
path=outputs_path,
|
||||
),
|
||||
|
|
@ -570,9 +546,9 @@ def nix_build_config(
|
|||
|
||||
|
||||
def nix_skipped_build_config(
|
||||
project: GithubProject,
|
||||
project: GitProject,
|
||||
worker_names: list[str],
|
||||
) -> util.BuilderConfig:
|
||||
) -> BuilderConfig:
|
||||
"""Dummy builder that is triggered when a build is skipped."""
|
||||
factory = util.BuildFactory()
|
||||
factory.addStep(
|
||||
|
|
@ -600,43 +576,17 @@ def nix_skipped_build_config(
|
|||
factory=factory,
|
||||
)
|
||||
|
||||
|
||||
def read_secret_file(secret_name: str) -> str:
|
||||
directory = os.environ.get("CREDENTIALS_DIRECTORY")
|
||||
if directory is None:
|
||||
print("directory not set", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return Path(directory).joinpath(secret_name).read_text().rstrip()
|
||||
|
||||
|
||||
@dataclass
|
||||
class GithubConfig:
|
||||
oauth_id: str
|
||||
admins: list[str]
|
||||
|
||||
buildbot_user: str
|
||||
oauth_secret_name: str = "github-oauth-secret"
|
||||
webhook_secret_name: str = "github-webhook-secret"
|
||||
token_secret_name: str = "github-token"
|
||||
project_cache_file: Path = Path("github-project-cache.json")
|
||||
topic: str | None = "build-with-buildbot"
|
||||
|
||||
def token(self) -> str:
|
||||
return read_secret_file(self.token_secret_name)
|
||||
|
||||
|
||||
def config_for_project(
|
||||
config: dict[str, Any],
|
||||
project: GithubProject,
|
||||
project: GitProject,
|
||||
worker_names: list[str],
|
||||
github: GithubConfig,
|
||||
nix_supported_systems: list[str],
|
||||
nix_eval_worker_count: int,
|
||||
nix_eval_max_memory_size: int,
|
||||
eval_lock: util.MasterLock,
|
||||
eval_lock: MasterLock,
|
||||
cachix: CachixConfig | None = None,
|
||||
outputs_path: Path | None = None,
|
||||
) -> Project:
|
||||
) -> None:
|
||||
config["projects"].append(Project(project.name))
|
||||
config["schedulers"].extend(
|
||||
[
|
||||
|
|
@ -644,8 +594,7 @@ def config_for_project(
|
|||
name=f"{project.project_id}-default-branch",
|
||||
change_filter=util.ChangeFilter(
|
||||
repository=project.url,
|
||||
filter_fn=lambda c: c.branch
|
||||
== c.properties.getProperty("github.repository.default_branch"),
|
||||
filter_fn=lambda c: c.branch == project.default_branch,
|
||||
),
|
||||
builderNames=[f"{project.name}/nix-eval"],
|
||||
treeStableTimer=5,
|
||||
|
|
@ -685,7 +634,7 @@ def config_for_project(
|
|||
properties=[
|
||||
util.StringParameter(
|
||||
name="project",
|
||||
label="Name of the GitHub repository.",
|
||||
label=f"Name of the {project.pretty_type} repository.",
|
||||
default=project.name,
|
||||
),
|
||||
],
|
||||
|
|
@ -699,7 +648,7 @@ def config_for_project(
|
|||
nix_eval_config(
|
||||
project,
|
||||
worker_names,
|
||||
github_token_secret=github.token_secret_name,
|
||||
git_url=project.get_project_url(),
|
||||
supported_systems=nix_supported_systems,
|
||||
worker_count=nix_eval_worker_count,
|
||||
max_memory_size=nix_eval_max_memory_size,
|
||||
|
|
@ -715,11 +664,11 @@ def config_for_project(
|
|||
],
|
||||
)
|
||||
|
||||
|
||||
def normalize_virtual_builder_name(name: str) -> str:
|
||||
if name.startswith("github:"):
|
||||
# TODO this code is a mystery to me
|
||||
if re.match(r"^[^:]+:", name) is not None:
|
||||
# rewrites github:nix-community/srvos#checks.aarch64-linux.nixos-stable-example-hardware-hetzner-online-intel -> nix-community/srvos/nix-build
|
||||
match = re.match(r"github:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
|
||||
match = re.match(r"[^:]:(?P<owner>[^/]+)/(?P<repo>[^#]+)#.+", name)
|
||||
if match:
|
||||
return f"{match['owner']}/{match['repo']}/nix-build"
|
||||
|
||||
|
|
@ -787,10 +736,10 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase):
|
|||
return self.check_builder(epobject, epdict, "buildrequest")
|
||||
|
||||
|
||||
def setup_authz(projects: list[GithubProject], admins: list[str]) -> util.Authz:
|
||||
def setup_authz(backends: list[GitBackend], projects: list[GitProject], admins: list[str]) -> Authz:
|
||||
allow_rules = []
|
||||
allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict(
|
||||
lambda: {"reload-github-projects"},
|
||||
lambda: {backend.reload_builder_name for backend in backends},
|
||||
)
|
||||
|
||||
for project in projects:
|
||||
|
|
@ -837,6 +786,7 @@ class NixConfigurator(ConfiguratorBase):
|
|||
self,
|
||||
# Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
|
||||
github: GithubConfig,
|
||||
gitea: GiteaConfig | None,
|
||||
url: str,
|
||||
nix_supported_systems: list[str],
|
||||
nix_eval_worker_count: int | None,
|
||||
|
|
@ -859,9 +809,17 @@ class NixConfigurator(ConfiguratorBase):
|
|||
self.outputs_path = Path(outputs_path)
|
||||
|
||||
def configure(self, config: dict[str, Any]) -> None:
|
||||
projects = load_projects(self.github.token(), self.github.project_cache_file)
|
||||
if self.github.topic is not None:
|
||||
projects = [p for p in projects if self.github.topic in p.topics]
|
||||
backends: list[GitBackend] = []
|
||||
|
||||
github_backend: GitBackend = GithubBackend(self.github)
|
||||
if self.github is not None:
|
||||
backends.append(github_backend)
|
||||
|
||||
projects: list[GitProject] = []
|
||||
|
||||
for backend in backends:
|
||||
projects += backend.load_projects()
|
||||
|
||||
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
|
||||
worker_names = []
|
||||
|
||||
|
|
@ -876,22 +834,21 @@ class NixConfigurator(ConfiguratorBase):
|
|||
config["workers"].append(worker.Worker(worker_name, item["pass"]))
|
||||
worker_names.append(worker_name)
|
||||
|
||||
# TODO pull out into global config
|
||||
webhook_secret = read_secret_file(self.github.webhook_secret_name)
|
||||
eval_lock = util.MasterLock("nix-eval")
|
||||
|
||||
for project in projects:
|
||||
create_project_hook(
|
||||
project.create_project_hook(
|
||||
project.owner,
|
||||
project.repo,
|
||||
self.github.token(),
|
||||
self.url + "change_hook/github",
|
||||
self.url,
|
||||
webhook_secret,
|
||||
)
|
||||
config_for_project(
|
||||
config,
|
||||
project,
|
||||
worker_names,
|
||||
self.github,
|
||||
self.nix_supported_systems,
|
||||
self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
||||
self.nix_eval_max_memory_size,
|
||||
|
|
@ -900,42 +857,32 @@ class NixConfigurator(ConfiguratorBase):
|
|||
self.outputs_path,
|
||||
)
|
||||
|
||||
# Reload github projects
|
||||
config["builders"].append(
|
||||
reload_github_projects(
|
||||
[worker_names[0]],
|
||||
self.github.token(),
|
||||
self.github.project_cache_file,
|
||||
),
|
||||
)
|
||||
config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
|
||||
config["schedulers"].extend(
|
||||
[
|
||||
schedulers.ForceScheduler(
|
||||
name="reload-github-projects",
|
||||
builderNames=["reload-github-projects"],
|
||||
buttonName="Update projects",
|
||||
),
|
||||
# project list twice a day and on startup
|
||||
PeriodicWithStartup(
|
||||
name="reload-github-projects-bidaily",
|
||||
builderNames=["reload-github-projects"],
|
||||
periodicBuildTimer=12 * 60 * 60,
|
||||
run_on_startup=not self.github.project_cache_file.exists(),
|
||||
),
|
||||
],
|
||||
)
|
||||
config["services"].append(
|
||||
reporters.GitHubStatusPush(
|
||||
token=self.github.token(),
|
||||
# Since we dynamically create build steps,
|
||||
# we use `virtual_builder_name` in the webinterface
|
||||
# so that we distinguish what has beeing build
|
||||
context=Interpolate("buildbot/%(prop:status_name)s"),
|
||||
),
|
||||
)
|
||||
|
||||
systemd_secrets = secrets.SecretInAFile(
|
||||
for backend in backends:
|
||||
# Reload backend projects
|
||||
config["builders"].append(
|
||||
backend.create_reload_builder([worker_names[0]])
|
||||
)
|
||||
config["schedulers"].extend(
|
||||
[
|
||||
schedulers.ForceScheduler(
|
||||
name=f"reload-{backend.type}-projects",
|
||||
builderNames=[backend.reload_builder_name],
|
||||
buttonName="Update projects",
|
||||
),
|
||||
# project list twice a day and on startup
|
||||
PeriodicWithStartup(
|
||||
name=f"reload-{backend.type}-projects-bidaily",
|
||||
builderNames=[backend.reload_builder_name],
|
||||
periodicBuildTimer=12 * 60 * 60,
|
||||
run_on_startup=not backend.are_projects_cached(),
|
||||
),
|
||||
],
|
||||
)
|
||||
config["services"].append(backend.create_reporter())
|
||||
|
||||
systemd_secrets = SecretInAFile(
|
||||
dirname=os.environ["CREDENTIALS_DIRECTORY"],
|
||||
)
|
||||
config["secretsProviders"].append(systemd_secrets)
|
||||
|
|
@ -944,25 +891,21 @@ class NixConfigurator(ConfiguratorBase):
|
|||
config["www"]["plugins"].update(dict(base_react={}))
|
||||
|
||||
config["www"].setdefault("change_hook_dialects", {})
|
||||
config["www"]["change_hook_dialects"]["github"] = {
|
||||
"secret": webhook_secret,
|
||||
"strict": True,
|
||||
"token": self.github.token(),
|
||||
"github_property_whitelist": "*",
|
||||
}
|
||||
for backend in backends:
|
||||
config["www"]["change_hook_dialects"][backend.change_hook_name] = \
|
||||
backend.create_change_hook(webhook_secret)
|
||||
|
||||
if "auth" not in config["www"]:
|
||||
config["www"].setdefault("avatar_methods", [])
|
||||
config["www"]["avatar_methods"].append(
|
||||
util.AvatarGitHub(token=self.github.token()),
|
||||
)
|
||||
config["www"]["auth"] = util.GitHubAuth(
|
||||
self.github.oauth_id,
|
||||
read_secret_file(self.github.oauth_secret_name),
|
||||
apiVersion=4,
|
||||
)
|
||||
|
||||
for backend in backends:
|
||||
config["www"]["avatar_methods"].append(backend.create_avatar_method())
|
||||
# TODO one cannot have multiple auth backends...
|
||||
config["www"]["auth"] = backends[0].create_auth()
|
||||
|
||||
config["www"]["authz"] = setup_authz(
|
||||
# TODO pull out into global config
|
||||
admins=self.github.admins,
|
||||
backends=backends,
|
||||
projects=projects,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,12 +2,259 @@ import contextlib
|
|||
import http.client
|
||||
import json
|
||||
import urllib.request
|
||||
import signal
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
|
||||
import typing
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
from twisted.python import log
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from buildbot.process.log import StreamLog
|
||||
|
||||
from buildbot.process.properties import Interpolate
|
||||
from buildbot.config.builder import BuilderConfig
|
||||
from buildbot.process.buildstep import BuildStep
|
||||
from buildbot.reporters.base import ReporterBase
|
||||
from buildbot.reporters.github import GitHubStatusPush
|
||||
from buildbot.www.avatar import AvatarBase, AvatarGitHub
|
||||
from buildbot.www.auth import AuthBase
|
||||
from buildbot.www.oauth2 import GitHubAuth
|
||||
from buildbot.plugins import util
|
||||
|
||||
from .projects import (
|
||||
GitProject,
|
||||
GitBackend
|
||||
)
|
||||
from .secrets import (
|
||||
read_secret_file
|
||||
)
|
||||
|
||||
class ReloadGithubProjects(BuildStep):
|
||||
name = "reload_github_projects"
|
||||
|
||||
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
|
||||
self.token = token
|
||||
self.project_cache_file = project_cache_file
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def reload_projects(self) -> None:
|
||||
refresh_projects(self.token, self.project_cache_file)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(self) -> Generator[Any, object, Any]:
|
||||
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
|
||||
|
||||
self.error_msg = ""
|
||||
|
||||
def error_cb(failure: Failure) -> int:
|
||||
self.error_msg += failure.getTraceback()
|
||||
return util.FAILURE
|
||||
|
||||
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
|
||||
res = yield d
|
||||
if res == util.SUCCESS:
|
||||
# reload the buildbot config
|
||||
os.kill(os.getpid(), signal.SIGHUP)
|
||||
return util.SUCCESS
|
||||
else:
|
||||
log: object = yield self.addLog("log")
|
||||
# TODO this assumes that log is of type StreamLog and not something else
|
||||
typing.cast(StreamLog, log).addStderr(f"Failed to reload project list: {self.error_msg}")
|
||||
return util.FAILURE
|
||||
|
||||
@dataclass
|
||||
class GithubConfig:
|
||||
oauth_id: str
|
||||
admins: list[str]
|
||||
|
||||
buildbot_user: str
|
||||
oauth_secret_name: str = "github-oauth-secret"
|
||||
webhook_secret_name: str = "github-webhook-secret"
|
||||
token_secret_name: str = "github-token"
|
||||
project_cache_file: Path = Path("github-project-cache.json")
|
||||
topic: str | None = "build-with-buildbot"
|
||||
|
||||
def token(self) -> str:
|
||||
return read_secret_file(self.token_secret_name)
|
||||
|
||||
@dataclass
|
||||
class GithubBackend(GitBackend):
|
||||
config: GithubConfig
|
||||
|
||||
def __init__(self, config: GithubConfig):
|
||||
self.config = config
|
||||
|
||||
def create_reload_builder(
|
||||
self,
|
||||
worker_names: list[str]
|
||||
) -> BuilderConfig:
|
||||
"""Updates the flake an opens a PR for it."""
|
||||
factory = util.BuildFactory()
|
||||
factory.addStep(
|
||||
ReloadGithubProjects(
|
||||
self.config.token(), self.config.project_cache_file
|
||||
),
|
||||
)
|
||||
return util.BuilderConfig(
|
||||
name=self.reload_builder_name,
|
||||
workernames=worker_names,
|
||||
factory=factory,
|
||||
)
|
||||
|
||||
def create_reporter(self) -> ReporterBase:
|
||||
return \
|
||||
GitHubStatusPush(
|
||||
token=self.config.token(),
|
||||
# Since we dynamically create build steps,
|
||||
# we use `virtual_builder_name` in the webinterface
|
||||
# so that we distinguish what has beeing build
|
||||
context=Interpolate("buildbot/%(prop:status_name)s"),
|
||||
)
|
||||
|
||||
def create_change_hook(self, webhook_secret: str) -> dict[str, Any]:
|
||||
return {
|
||||
"secret": webhook_secret,
|
||||
"strict": True,
|
||||
"token": self.config.token,
|
||||
"github_property_whitelist": "*",
|
||||
}
|
||||
|
||||
def create_avatar_method(self) -> AvatarBase:
|
||||
return AvatarGitHub(token=self.config.token())
|
||||
|
||||
def create_auth(self) -> AuthBase:
|
||||
return \
|
||||
GitHubAuth(
|
||||
self.config.oauth_id,
|
||||
read_secret_file(self.config.oauth_secret_name),
|
||||
apiVersion=4,
|
||||
)
|
||||
|
||||
def load_projects(self) -> list["GitProject"]:
|
||||
if not self.config.project_cache_file.exists():
|
||||
return []
|
||||
|
||||
repos: list[dict[str, Any]] = sorted(
|
||||
json.loads(self.config.project_cache_file.read_text()), key=lambda x: x["full_name"]
|
||||
)
|
||||
return \
|
||||
list(filter(\
|
||||
lambda project: self.config.topic != None and self.config.topic in project.topics, \
|
||||
[GithubProject(self.config, repo) for repo in repos] \
|
||||
))
|
||||
|
||||
def are_projects_cached(self) -> bool:
|
||||
return self.config.project_cache_file.exists()
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return "github"
|
||||
|
||||
@property
|
||||
def pretty_type(self) -> str:
|
||||
return "GitHub"
|
||||
|
||||
@property
|
||||
def reload_builder_name(self) -> str:
|
||||
return "reload-github-projects"
|
||||
|
||||
@property
|
||||
def change_hook_name(self) -> str:
|
||||
return "github"
|
||||
|
||||
class GithubProject(GitProject):
|
||||
config: GithubConfig
|
||||
|
||||
def __init__(self, config: GithubConfig, data: dict[str, Any]) -> None:
|
||||
self.config = config
|
||||
self.data = data
|
||||
|
||||
def create_project_hook(
|
||||
self,
|
||||
owner: str,
|
||||
repo: str,
|
||||
webhook_url: str,
|
||||
webhook_secret: str,
|
||||
) -> None:
|
||||
hooks = paginated_github_request(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
|
||||
self.config.token(),
|
||||
)
|
||||
config = dict(
|
||||
url=webhook_url + "change_hook/github",
|
||||
content_type="json",
|
||||
insecure_ssl="0",
|
||||
secret=webhook_secret,
|
||||
)
|
||||
data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.config.token()}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Content-Type": "application/json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
for hook in hooks:
|
||||
if hook["config"]["url"] == webhook_url + "change_hook/github":
|
||||
log.msg(f"hook for {owner}/{repo} already exists")
|
||||
return
|
||||
|
||||
http_request(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/hooks",
|
||||
method="POST",
|
||||
headers=headers,
|
||||
data=data,
|
||||
)
|
||||
|
||||
def get_project_url(self) -> str:
|
||||
return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}"
|
||||
|
||||
@property
|
||||
def pretty_type(self) -> str:
|
||||
return "GitHub"
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return "github"
|
||||
|
||||
@property
|
||||
def repo(self) -> str:
|
||||
return self.data["name"]
|
||||
|
||||
@property
|
||||
def owner(self) -> str:
|
||||
return self.data["owner"]["login"]
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.data["full_name"]
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.data["html_url"]
|
||||
|
||||
@property
|
||||
def project_id(self) -> str:
|
||||
return slugify_project_name(self.data["full_name"])
|
||||
|
||||
@property
|
||||
def default_branch(self) -> str:
|
||||
return self.data["default_branch"]
|
||||
|
||||
@property
|
||||
def topics(self) -> list[str]:
|
||||
return self.data["topics"]
|
||||
|
||||
@property
|
||||
def belongs_to_org(self) -> bool:
|
||||
return self.data["owner"]["type"] == "Organization"
|
||||
|
||||
class GithubError(Exception):
|
||||
pass
|
||||
|
|
@ -83,81 +330,6 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
|
|||
def slugify_project_name(name: str) -> str:
|
||||
return name.replace(".", "-").replace("/", "-")
|
||||
|
||||
|
||||
class GithubProject:
|
||||
def __init__(self, data: dict[str, Any]) -> None:
|
||||
self.data = data
|
||||
|
||||
@property
|
||||
def repo(self) -> str:
|
||||
return self.data["name"]
|
||||
|
||||
@property
|
||||
def owner(self) -> str:
|
||||
return self.data["owner"]["login"]
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.data["full_name"]
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.data["html_url"]
|
||||
|
||||
@property
|
||||
def project_id(self) -> str:
|
||||
return slugify_project_name(self.data["full_name"])
|
||||
|
||||
@property
|
||||
def default_branch(self) -> str:
|
||||
return self.data["default_branch"]
|
||||
|
||||
@property
|
||||
def topics(self) -> list[str]:
|
||||
return self.data["topics"]
|
||||
|
||||
@property
|
||||
def belongs_to_org(self) -> bool:
|
||||
return self.data["owner"]["type"] == "Organization"
|
||||
|
||||
|
||||
def create_project_hook(
|
||||
owner: str,
|
||||
repo: str,
|
||||
token: str,
|
||||
webhook_url: str,
|
||||
webhook_secret: str,
|
||||
) -> None:
|
||||
hooks = paginated_github_request(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
|
||||
token,
|
||||
)
|
||||
config = dict(
|
||||
url=webhook_url,
|
||||
content_type="json",
|
||||
insecure_ssl="0",
|
||||
secret=webhook_secret,
|
||||
)
|
||||
data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Content-Type": "application/json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
for hook in hooks:
|
||||
if hook["config"]["url"] == webhook_url:
|
||||
log.msg(f"hook for {owner}/{repo} already exists")
|
||||
return
|
||||
|
||||
http_request(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/hooks",
|
||||
method="POST",
|
||||
headers=headers,
|
||||
data=data,
|
||||
)
|
||||
|
||||
|
||||
def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
|
||||
repos = []
|
||||
|
||||
|
|
@ -182,13 +354,3 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
|
|||
except OSError:
|
||||
path.unlink()
|
||||
raise
|
||||
|
||||
|
||||
def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]:
|
||||
if not repo_cache_file.exists():
|
||||
return []
|
||||
|
||||
repos: list[dict[str, Any]] = sorted(
|
||||
json.loads(repo_cache_file.read_text()), key=lambda x: x["full_name"]
|
||||
)
|
||||
return [GithubProject(repo) for repo in repos]
|
||||
|
|
|
|||
125
buildbot_nix/projects.py
Normal file
125
buildbot_nix/projects.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
from buildbot.config.builder import BuilderConfig
|
||||
from buildbot.reporters.base import ReporterBase
|
||||
from buildbot.www.avatar import AvatarBase
|
||||
from buildbot.www.auth import AuthBase
|
||||
|
||||
class GitBackend(ABC):
|
||||
@abstractmethod
|
||||
def create_reload_builder(
|
||||
self,
|
||||
worker_names: list[str]
|
||||
) -> BuilderConfig:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def create_reporter(self) -> ReporterBase:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def create_change_hook(self, webhook_secret: str) -> dict[str, Any]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def create_avatar_method(self) -> AvatarBase:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def create_auth(self) -> AuthBase:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load_projects(self) -> list["GitProject"]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def are_projects_cached(self) -> bool:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def pretty_type(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def type(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def reload_builder_name(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def change_hook_name(self) -> str:
|
||||
pass
|
||||
|
||||
class GitProject(ABC):
|
||||
@abstractmethod
|
||||
def create_project_hook(
|
||||
self,
|
||||
owner: str,
|
||||
repo: str,
|
||||
webhook_url: str,
|
||||
webhook_secret: str,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_project_url() -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def pretty_type(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def type(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def repo(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def owner(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def url(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def project_id(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def default_branch(self) -> str:
|
||||
pass
|
||||
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def topics(self) -> list[str]:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def belongs_to_org(self) -> bool:
|
||||
pass
|
||||
9
buildbot_nix/secrets.py
Normal file
9
buildbot_nix/secrets.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import os, sys
|
||||
from pathlib import Path
|
||||
|
||||
def read_secret_file(secret_name: str) -> str:
|
||||
directory = os.environ.get("CREDENTIALS_DIRECTORY")
|
||||
if directory is None:
|
||||
print("directory not set", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return Path(directory).joinpath(secret_name).read_text().rstrip()
|
||||
|
|
@ -158,6 +158,7 @@ in
|
|||
buildbot_user=${builtins.toJSON cfg.github.user},
|
||||
topic=${builtins.toJSON cfg.github.topic},
|
||||
),
|
||||
gitea=None,
|
||||
cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
|
||||
name=${builtins.toJSON cfg.cachix.name},
|
||||
signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
|
||||
|
|
|
|||
Loading…
Reference in a new issue