auxolotl/buildbot-nix
26
1
Fork 0
Code Issues Pull requests 1 Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #4 from Mic92/cleanups

Browse source 
wip: switch to configurators class
This commit is contained in:
Jörg Thalheim 2023-09-18 17:08:46 +02:00 committed by GitHub
parent c09566cfba 2488ef4424
commit cb356fc006
Failed to generate hash of commit
8 changed files with 299 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -2,6 +2,7 @@
A nixos module to make buildbot a proper Nix-CI.
For an example checkout the [example](./examples/default.nix) and the module descriptions for [master](./nix/master.nix) and [worker](./nix/worker.nix).
For an example checkout the [example](./examples/default.nix) and the module
descriptions for [master](./nix/master.nix) and [worker](./nix/worker.nix).
This project is still in early stage and many APIs might change over time.

217
buildbot_nix/buildbot_nix.py
View file

@ -3,18 +3,23 @@
import json
import multiprocessing
import os
import sys
import uuid
from collections import defaultdict
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Generator, List
from typing import Any
from buildbot.plugins import steps, util
from buildbot.configurators import ConfiguratorBase
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
from buildbot.process import buildstep, logobserver, remotecommand
from buildbot.process.log import Log
from buildbot.process.properties import Properties
from buildbot.process.project import Project
from buildbot.process.properties import Interpolate, Properties
from buildbot.process.results import ALL_RESULTS, statusToString
from buildbot.steps.trigger import Trigger
from github_projects import GithubProject
from github_projects import GithubProject, load_projects # noqa: E402
from twisted.internet import defer
@ -348,7 +353,7 @@ def nix_eval_config(
worker_names: list[str],
github_token_secret: str,
supported_systems: list[str],
automerge_users: List[str] = [],
automerge_users: list[str] = [],
max_memory_size: int = 4096,
) -> util.BuilderConfig:
"""
@ -527,3 +532,205 @@ def nix_build_config(
env={},
factory=factory,
)
def read_secret_file(secret_name: str) -> str:
directory = os.environ.get("CREDENTIALS_DIRECTORY")
if directory is None:
print("directory not set", file=sys.stderr)
sys.exit(1)
return Path(directory).joinpath(secret_name).read_text()
@dataclass
class GithubConfig:
oauth_id: str
admins: list[str]
buildbot_user: str
oauth_secret_name: str = "github-oauth-secret"
webhook_secret_name: str = "github-webhook-secret"
token_secret_name: str = "github-token"
project_cache_file: Path = Path("github-project-cache.json")
def token(self) -> str:
return read_secret_file(self.token_secret_name)
def config_for_project(
config: dict[str, Any],
project: GithubProject,
credentials: str,
worker_names: list[str],
github: GithubConfig,
nix_supported_systems: list[str],
nix_eval_max_memory_size: int,
) -> Project:
config["projects"].append(Project(project.name))
config["schedulers"].extend(
[
schedulers.SingleBranchScheduler(
name=f"default-branch-{project.id}",
change_filter=util.ChangeFilter(
repository=project.url,
filter_fn=lambda c: c.branch
== c.properties.getProperty("github.repository.default_branch"),
),
builderNames=[f"{project.name}/nix-eval"],
),
# this is compatible with bors or github's merge queue
schedulers.SingleBranchScheduler(
name=f"merge-queue-{project.id}",
change_filter=util.ChangeFilter(
repository=project.url,
branch_re="(gh-readonly-queue/.*|staging|trying)",
),
builderNames=[f"{project.name}/nix-eval"],
),
# build all pull requests
schedulers.SingleBranchScheduler(
name=f"prs-{project.id}",
change_filter=util.ChangeFilter(
repository=project.url, category="pull"
),
builderNames=[f"{project.name}/nix-eval"],
),
# this is triggered from `nix-eval`
schedulers.Triggerable(
name=f"{project.id}-nix-build",
builderNames=[f"{project.name}/nix-build"],
),
# allow to manually trigger a nix-build
schedulers.ForceScheduler(
name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"]
),
# allow to manually update flakes
schedulers.ForceScheduler(
name=f"{project.id}-update-flake",
builderNames=[f"{project.name}/update-flake"],
buttonName="Update flakes",
),
# updates flakes once a weeek
schedulers.NightlyTriggerable(
name=f"{project.id}-update-flake-weekly",
builderNames=[f"{project.name}/update-flake"],
hour=3,
minute=0,
dayOfWeek=6,
),
]
)
has_cachix_auth_token = os.path.isfile(
os.path.join(credentials, "cachix-auth-token")
)
has_cachix_signing_key = os.path.isfile(
os.path.join(credentials, "cachix-signing-key")
)
config["builders"].extend(
[
# Since all workers run on the same machine, we only assign one of them to do the evaluation.
# This should prevent exessive memory usage.
nix_eval_config(
project,
[worker_names[0]],
github_token_secret=github.token_secret_name,
supported_systems=nix_supported_systems,
automerge_users=[github.buildbot_user],
max_memory_size=nix_eval_max_memory_size,
),
nix_build_config(
project,
worker_names,
has_cachix_auth_token,
has_cachix_signing_key,
),
nix_update_flake_config(
project,
worker_names,
github_token_secret=github.token_secret_name,
github_bot_user=github.buildbot_user,
),
]
)
class NixConfigurator(ConfiguratorBase):
"""Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
def __init__(
self,
# Shape of this file:
# [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
github: GithubConfig,
nix_supported_systems: list[str],
nix_eval_max_memory_size: int = 4096,
nix_workers_secret_name: str = "buildbot-nix-workers",
) -> None:
super().__init__()
self.nix_workers_secret_name = nix_workers_secret_name
self.nix_eval_max_memory_size = nix_eval_max_memory_size
self.nix_supported_systems = nix_supported_systems
self.github = github
self.systemd_credentials_dir = os.environ["CREDENTIALS_DIRECTORY"]
def configure(self, config: dict[str, Any]) -> None:
projects = load_projects(self.github.token(), self.github.project_cache_file)
projects = [p for p in projects if "build-with-buildbot" in p.topics]
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
worker_names = []
config["workers"] = config.get("workers", [])
for item in worker_config:
cores = item.get("cores", 0)
for i in range(cores):
worker_name = f"{item['name']}-{i}"
config["workers"].append(worker.Worker(worker_name, item["pass"]))
worker_names.append(worker_name)
for project in projects:
config_for_project(
config,
project,
self.systemd_credentials_dir,
worker_names,
self.github,
self.nix_supported_systems,
self.nix_eval_max_memory_size,
)
config["services"] = config.get("services", [])
config["services"].append(
reporters.GitHubStatusPush(
token=self.github.token(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
)
systemd_secrets = secrets.SecretInAFile(
dirname=os.environ["CREDENTIALS_DIRECTORY"]
)
config["secretsProviders"] = config.get("secretsProviders", [])
config["secretsProviders"].append(systemd_secrets)
config["www"] = config.get("www", {})
config["www"]["avatar_methods"] = config["www"].get("avatar_methods", [])
config["www"]["avatar_methods"].append(util.AvatarGitHub())
config["www"]["auth"] = util.GitHubAuth(
self.github.oauth_id, read_secret_file(self.github.oauth_secret_name)
)
config["www"]["authz"] = util.Authz(
roleMatchers=[
util.RolesFromUsername(roles=["admin"], usernames=self.github.admins)
],
allowRules=[
util.AnyEndpointMatcher(role="admin", defaultDeny=False),
util.AnyControlEndpointMatcher(role="admins"),
],
)
config["www"]["change_hook_dialects"] = config["www"].get(
"change_hook_dialects", {}
)
config["www"]["change_hook_dialects"]["github"] = {
"secret": read_secret_file(self.github.webhook_secret_name),
"strict": True,
"token": self.github.token(),
"github_property_whitelist": "*",
}

207
buildbot_nix/master.py
View file

@ -1,161 +1,39 @@
#!/usr/bin/env python3
import json
import os
import sys
from datetime import timedelta
from pathlib import Path
from typing import Any
from buildbot.plugins import reporters, schedulers, secrets, util, worker
from buildbot.process.project import Project
from buildbot.process.properties import Interpolate
from buildbot.plugins import schedulers, util
# allow to import modules
sys.path.append(str(Path(__file__).parent))
from buildbot_nix import ( # noqa: E402
nix_build_config,
nix_eval_config,
nix_update_flake_config,
)
from github_projects import GithubProject, load_projects # noqa: E402
def read_secret_file(secret_name: str) -> str:
directory = os.environ.get("CREDENTIALS_DIRECTORY")
if directory is None:
print("directory not set", file=sys.stderr)
sys.exit(1)
return Path(directory).joinpath(secret_name).read_text()
GITHUB_OAUTH_ID = os.environ.get("GITHUB_OAUTH_ID")
GITHUB_OAUTH_SECRET = read_secret_file("github-oauth-secret")
GITHUB_ADMINS = os.environ.get("GITHUB_ADMINS", "").split(" ")
GITHUB_TOKEN_SECRET_NAME = "github-token"
GITHUB_TOKEN = read_secret_file(GITHUB_TOKEN_SECRET_NAME)
GITHUB_WEBHOOK_SECRET = read_secret_file("github-webhook-secret")
# Shape of this file:
# [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
BUILDBOT_NIX_WORKERS = read_secret_file("buildbot-nix-workers")
BUILDBOT_URL = os.environ["BUILDBOT_URL"]
BUILDBOT_GITHUB_USER = os.environ["BUILDBOT_GITHUB_USER"]
NIX_SUPPORTED_SYSTEMS = os.environ["NIX_SUPPORTED_SYSTEMS"].split(" ")
NIX_EVAL_MAX_MEMORY_SIZE = int(os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096"))
BUILDBOT_DB_URL = os.environ.get("DB_URL", "sqlite:///state.sqlite")
def config_for_project(
config: dict[str, Any],
project: GithubProject,
credentials: str,
worker_names: list[str],
) -> Project:
config["projects"].append(Project(project.name))
config["schedulers"].extend(
[
schedulers.SingleBranchScheduler(
name=f"default-branch-{project.id}",
change_filter=util.ChangeFilter(
repository=project.url,
filter_fn=lambda c: c.branch
== c.properties.getProperty("github.repository.default_branch"),
),
builderNames=[f"{project.name}/nix-eval"],
),
# this is compatible with bors or github's merge queue
schedulers.SingleBranchScheduler(
name=f"merge-queue-{project.id}",
change_filter=util.ChangeFilter(
repository=project.url,
branch_re="(gh-readonly-queue/.*|staging|trying)",
),
builderNames=[f"{project.name}/nix-eval"],
),
# build all pull requests
schedulers.SingleBranchScheduler(
name=f"prs-{project.id}",
change_filter=util.ChangeFilter(
repository=project.url, category="pull"
),
builderNames=[f"{project.name}/nix-eval"],
),
# this is triggered from `nix-eval`
schedulers.Triggerable(
name=f"{project.id}-nix-build",
builderNames=[f"{project.name}/nix-build"],
),
# allow to manually trigger a nix-build
schedulers.ForceScheduler(
name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"]
),
# allow to manually update flakes
schedulers.ForceScheduler(
name=f"{project.id}-update-flake",
builderNames=[f"{project.name}/update-flake"],
buttonName="Update flakes",
),
# updates flakes once a weeek
schedulers.NightlyTriggerable(
name=f"{project.id}-update-flake-weekly",
builderNames=[f"{project.name}/update-flake"],
hour=3,
minute=0,
dayOfWeek=6,
),
]
)
has_cachix_auth_token = os.path.isfile(
os.path.join(credentials, "cachix-auth-token")
)
has_cachix_signing_key = os.path.isfile(
os.path.join(credentials, "cachix-signing-key")
)
config["builders"].extend(
[
# Since all workers run on the same machine, we only assign one of them to do the evaluation.
# This should prevent exessive memory usage.
nix_eval_config(
project,
[worker_names[0]],
github_token_secret=GITHUB_TOKEN_SECRET_NAME,
supported_systems=NIX_SUPPORTED_SYSTEMS,
automerge_users=[BUILDBOT_GITHUB_USER],
max_memory_size=NIX_EVAL_MAX_MEMORY_SIZE,
),
nix_build_config(
project,
worker_names,
has_cachix_auth_token,
has_cachix_signing_key,
),
nix_update_flake_config(
project,
worker_names,
github_token_secret=GITHUB_TOKEN_SECRET_NAME,
github_bot_user=BUILDBOT_GITHUB_USER,
),
]
)
PROJECT_CACHE_FILE = Path("github-project-cache.json")
from buildbot_nix import GithubConfig, NixConfigurator # noqa: E402
def build_config() -> dict[str, Any]:
projects = load_projects(GITHUB_TOKEN, PROJECT_CACHE_FILE)
projects = [p for p in projects if "build-with-buildbot" in p.topics]
import pprint
pprint.pprint(projects)
c: dict[str, Any] = {}
c["buildbotNetUsageData"] = None
# configure a janitor which will delete all logs older than one month, and will run on sundays at noon
c["configurators"] = [
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6),
NixConfigurator(
github=GithubConfig(
oauth_id=os.environ["GITHUB_OAUTH_ID"],
admins=os.environ.get("GITHUB_ADMINS", "").split(" "),
buildbot_user=os.environ["BUILDBOT_GITHUB_USER"],
),
nix_eval_max_memory_size=int(
os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096")
),
nix_supported_systems=os.environ.get("NIX_SUPPORTED_SYSTEMS", "auto").split(
" "
),
),
]
credentials = os.environ.get("CREDENTIALS_DIRECTORY", ".")
c["schedulers"] = [
schedulers.SingleBranchScheduler(
name="nixpkgs",
@ -171,62 +49,17 @@ def build_config() -> dict[str, Any]:
c["builders"] = []
c["projects"] = []
c["workers"] = []
worker_config = json.loads(BUILDBOT_NIX_WORKERS)
worker_names = []
for item in worker_config:
cores = item.get("cores", 0)
for i in range(cores):
worker_name = f"{item['name']}-{i}"
c["workers"].append(worker.Worker(worker_name, item["pass"]))
worker_names.append(worker_name)
for project in projects:
config_for_project(c, project, credentials, worker_names)
c["services"] = [
reporters.GitHubStatusPush(
token=GITHUB_TOKEN,
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
)
]
systemd_secrets = secrets.SecretInAFile(dirname=credentials)
c["secretsProviders"] = [systemd_secrets]
c["services"] = []
c["www"] = {
"avatar_methods": [util.AvatarGitHub()],
"port": int(os.environ.get("PORT", "1810")),
"auth": util.GitHubAuth(GITHUB_OAUTH_ID, GITHUB_OAUTH_SECRET),
"authz": util.Authz(
roleMatchers=[
util.RolesFromUsername(roles=["admin"], usernames=GITHUB_ADMINS)
],
allowRules=[
util.AnyEndpointMatcher(role="admin", defaultDeny=False),
util.AnyControlEndpointMatcher(role="admins"),
],
),
"plugins": dict(
base_react={}, waterfall_view={}, console_view={}, grid_view={}
),
"change_hook_dialects": dict(
github={
"secret": GITHUB_WEBHOOK_SECRET,
"strict": True,
"token": GITHUB_TOKEN,
"github_property_whitelist": "*",
}
),
"port": int(os.environ.get("PORT", "1810")),
}
c["db"] = {"db_url": BUILDBOT_DB_URL}
c["db"] = {"db_url": os.environ.get("DB_URL", "sqlite:///state.sqlite")}
c["protocols"] = {"pb": {"port": "tcp:9989:interface=\\:\\:"}}
c["buildbotURL"] = BUILDBOT_URL
c["buildbotURL"] = os.environ["BUILDBOT_URL"]
return c

23
flake.lock
View file

@ -39,7 +39,28 @@
"root": {
"inputs": {
"flake-parts": "flake-parts",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs",
"treefmt-nix": "treefmt-nix"
}
},
"treefmt-nix": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1694528738,
"narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
}
},

5
flake.nix
View file

@ -6,11 +6,16 @@
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
flake-parts.url = "github:hercules-ci/flake-parts";
flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
# used for development
treefmt-nix.url = "github:numtide/treefmt-nix";
treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
};
outputs = inputs@{ self, flake-parts, ... }:
flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }:
{
imports = inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
systems = [ "x86_64-linux" "aarch64-linux" ];
flake = {
nixosModules.buildbot-master = ./nix/master.nix;

3
nix/master.nix
View file

@ -50,7 +50,8 @@ in
};
githubAdmins = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Users that are allowed to login to buildbot and do stuff";
default = [ ];
description = "Users that are allowed to login to buildbot, trigger builds and change settings";
};
};
workersFile = lib.mkOption {

29
nix/treefmt/flake-module.nix Normal file
View file

@ -0,0 +1,29 @@
{ inputs, ... }: {
imports = [
inputs.treefmt-nix.flakeModule
];
perSystem = { config, pkgs, ... }: {
treefmt = {
projectRootFile = ".git/config";
programs.nixpkgs-fmt.enable = true;
programs.shellcheck.enable = true;
programs.deno.enable = true;
settings.formatter.shellcheck.options = [ "-s" "bash" ];
programs.mypy.enable = true;
programs.mypy.directories."." = { };
settings.formatter.python = {
command = "sh";
options = [
"-eucx"
''
${pkgs.ruff}/bin/ruff --fix "$@"
${pkgs.python3.pkgs.black}/bin/black "$@"
''
"--" # this argument is ignored by bash
];
includes = [ "*.py" ];
};
};
};
}

7
pyproject.toml
View file

@ -1,3 +1,10 @@
[tool.ruff]
target-version = "py311"
line-length = 88
select = ["E", "F", "I", "U"]
ignore = [ "E501" ]
[tool.mypy]
python_version = "3.10"
warn_redundant_casts = true