From 05a918572674170335bd15d30fd1e0454b27586e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Sep 2023 15:14:45 +0200 Subject: [PATCH 1/3] add treefmt-nix configuration --- README.md | 3 ++- flake.lock | 23 ++++++++++++++++++++++- nix/treefmt/flake-module.nix | 29 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 nix/treefmt/flake-module.nix diff --git a/README.md b/README.md index 9b9c6cc..52324aa 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ A nixos module to make buildbot a proper Nix-CI. -For an example checkout the [example](./examples/default.nix) and the module descriptions for [master](./nix/master.nix) and [worker](./nix/worker.nix). +For an example checkout the [example](./examples/default.nix) and the module +descriptions for [master](./nix/master.nix) and [worker](./nix/worker.nix). This project is still in early stage and many APIs might change over time. diff --git a/flake.lock b/flake.lock index 83e2df4..4b5fe22 100644 --- a/flake.lock +++ b/flake.lock @@ -39,7 +39,28 @@ "root": { "inputs": { "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "treefmt-nix": "treefmt-nix" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1694528738, + "narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" } } }, diff --git a/nix/treefmt/flake-module.nix b/nix/treefmt/flake-module.nix new file mode 100644 index 0000000..9f2c05e --- /dev/null +++ b/nix/treefmt/flake-module.nix @@ -0,0 +1,29 @@ +{ inputs, ... }: { + imports = [ + inputs.treefmt-nix.flakeModule + ]; + perSystem = { config, pkgs, ... }: { + treefmt = { + projectRootFile = ".git/config"; + programs.nixpkgs-fmt.enable = true; + programs.shellcheck.enable = true; + programs.deno.enable = true; + settings.formatter.shellcheck.options = [ "-s" "bash" ]; + + programs.mypy.enable = true; + programs.mypy.directories."." = { }; + settings.formatter.python = { + command = "sh"; + options = [ + "-eucx" + '' + ${pkgs.ruff}/bin/ruff --fix "$@" + ${pkgs.python3.pkgs.black}/bin/black "$@" + '' + "--" # this argument is ignored by bash + ]; + includes = [ "*.py" ]; + }; + }; + }; +} From 5915bf4ffa58119fc9d36f33d60507c145f22785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Sep 2023 15:41:34 +0200 Subject: [PATCH 2/3] ruff: also lint upgrade rules --- buildbot_nix/buildbot_nix.py | 4 ++-- pyproject.toml | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/buildbot_nix/buildbot_nix.py b/buildbot_nix/buildbot_nix.py index 3ed6955..7852b1b 100644 --- a/buildbot_nix/buildbot_nix.py +++ b/buildbot_nix/buildbot_nix.py @@ -6,7 +6,7 @@ import os import uuid from collections import defaultdict from pathlib import Path -from typing import Any, Generator, List +from typing import Any from buildbot.plugins import steps, util from buildbot.process import buildstep, logobserver, remotecommand @@ -348,7 +348,7 @@ def nix_eval_config( worker_names: list[str], github_token_secret: str, supported_systems: list[str], - automerge_users: List[str] = [], + automerge_users: list[str] = [], max_memory_size: int = 4096, ) -> util.BuilderConfig: """ diff --git a/pyproject.toml b/pyproject.toml index 8545e68..40a2122 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,10 @@ +[tool.ruff] +target-version = "py311" +line-length = 88 + +select = ["E", "F", "I", "U"] +ignore = [ "E501" ] + [tool.mypy] python_version = "3.10" warn_redundant_casts = true From 2488ef4424918c28fb2b2ff4749a2327c5af33e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 17 Sep 2023 22:14:56 +0200 Subject: [PATCH 3/3] switch to configurators class --- buildbot_nix/buildbot_nix.py | 213 ++++++++++++++++++++++++++++++++++- buildbot_nix/master.py | 207 ++++------------------------------ flake.nix | 5 + nix/master.nix | 3 +- 4 files changed, 237 insertions(+), 191 deletions(-) diff --git a/buildbot_nix/buildbot_nix.py b/buildbot_nix/buildbot_nix.py index 7852b1b..8d10873 100644 --- a/buildbot_nix/buildbot_nix.py +++ b/buildbot_nix/buildbot_nix.py @@ -3,18 +3,23 @@ import json import multiprocessing import os +import sys import uuid from collections import defaultdict +from collections.abc import Generator +from dataclasses import dataclass from pathlib import Path from typing import Any -from buildbot.plugins import steps, util +from buildbot.configurators import ConfiguratorBase +from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker from buildbot.process import buildstep, logobserver, remotecommand from buildbot.process.log import Log -from buildbot.process.properties import Properties +from buildbot.process.project import Project +from buildbot.process.properties import Interpolate, Properties from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.steps.trigger import Trigger -from github_projects import GithubProject +from github_projects import GithubProject, load_projects # noqa: E402 from twisted.internet import defer @@ -527,3 +532,205 @@ def nix_build_config( env={}, factory=factory, ) + + +def read_secret_file(secret_name: str) -> str: + directory = os.environ.get("CREDENTIALS_DIRECTORY") + if directory is None: + print("directory not set", file=sys.stderr) + sys.exit(1) + return Path(directory).joinpath(secret_name).read_text() + + +@dataclass +class GithubConfig: + oauth_id: str + admins: list[str] + buildbot_user: str + oauth_secret_name: str = "github-oauth-secret" + webhook_secret_name: str = "github-webhook-secret" + token_secret_name: str = "github-token" + project_cache_file: Path = Path("github-project-cache.json") + + def token(self) -> str: + return read_secret_file(self.token_secret_name) + + +def config_for_project( + config: dict[str, Any], + project: GithubProject, + credentials: str, + worker_names: list[str], + github: GithubConfig, + nix_supported_systems: list[str], + nix_eval_max_memory_size: int, +) -> Project: + config["projects"].append(Project(project.name)) + config["schedulers"].extend( + [ + schedulers.SingleBranchScheduler( + name=f"default-branch-{project.id}", + change_filter=util.ChangeFilter( + repository=project.url, + filter_fn=lambda c: c.branch + == c.properties.getProperty("github.repository.default_branch"), + ), + builderNames=[f"{project.name}/nix-eval"], + ), + # this is compatible with bors or github's merge queue + schedulers.SingleBranchScheduler( + name=f"merge-queue-{project.id}", + change_filter=util.ChangeFilter( + repository=project.url, + branch_re="(gh-readonly-queue/.*|staging|trying)", + ), + builderNames=[f"{project.name}/nix-eval"], + ), + # build all pull requests + schedulers.SingleBranchScheduler( + name=f"prs-{project.id}", + change_filter=util.ChangeFilter( + repository=project.url, category="pull" + ), + builderNames=[f"{project.name}/nix-eval"], + ), + # this is triggered from `nix-eval` + schedulers.Triggerable( + name=f"{project.id}-nix-build", + builderNames=[f"{project.name}/nix-build"], + ), + # allow to manually trigger a nix-build + schedulers.ForceScheduler( + name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"] + ), + # allow to manually update flakes + schedulers.ForceScheduler( + name=f"{project.id}-update-flake", + builderNames=[f"{project.name}/update-flake"], + buttonName="Update flakes", + ), + # updates flakes once a weeek + schedulers.NightlyTriggerable( + name=f"{project.id}-update-flake-weekly", + builderNames=[f"{project.name}/update-flake"], + hour=3, + minute=0, + dayOfWeek=6, + ), + ] + ) + has_cachix_auth_token = os.path.isfile( + os.path.join(credentials, "cachix-auth-token") + ) + has_cachix_signing_key = os.path.isfile( + os.path.join(credentials, "cachix-signing-key") + ) + config["builders"].extend( + [ + # Since all workers run on the same machine, we only assign one of them to do the evaluation. + # This should prevent exessive memory usage. + nix_eval_config( + project, + [worker_names[0]], + github_token_secret=github.token_secret_name, + supported_systems=nix_supported_systems, + automerge_users=[github.buildbot_user], + max_memory_size=nix_eval_max_memory_size, + ), + nix_build_config( + project, + worker_names, + has_cachix_auth_token, + has_cachix_signing_key, + ), + nix_update_flake_config( + project, + worker_names, + github_token_secret=github.token_secret_name, + github_bot_user=github.buildbot_user, + ), + ] + ) + + +class NixConfigurator(ConfiguratorBase): + """Janitor is a configurator which create a Janitor Builder with all needed Janitor steps""" + + def __init__( + self, + # Shape of this file: + # [ { "name": "", "pass": "", "cores": "" } ] + github: GithubConfig, + nix_supported_systems: list[str], + nix_eval_max_memory_size: int = 4096, + nix_workers_secret_name: str = "buildbot-nix-workers", + ) -> None: + super().__init__() + self.nix_workers_secret_name = nix_workers_secret_name + self.nix_eval_max_memory_size = nix_eval_max_memory_size + self.nix_supported_systems = nix_supported_systems + self.github = github + self.systemd_credentials_dir = os.environ["CREDENTIALS_DIRECTORY"] + + def configure(self, config: dict[str, Any]) -> None: + projects = load_projects(self.github.token(), self.github.project_cache_file) + projects = [p for p in projects if "build-with-buildbot" in p.topics] + worker_config = json.loads(read_secret_file(self.nix_workers_secret_name)) + worker_names = [] + config["workers"] = config.get("workers", []) + for item in worker_config: + cores = item.get("cores", 0) + for i in range(cores): + worker_name = f"{item['name']}-{i}" + config["workers"].append(worker.Worker(worker_name, item["pass"])) + worker_names.append(worker_name) + + for project in projects: + config_for_project( + config, + project, + self.systemd_credentials_dir, + worker_names, + self.github, + self.nix_supported_systems, + self.nix_eval_max_memory_size, + ) + config["services"] = config.get("services", []) + config["services"].append( + reporters.GitHubStatusPush( + token=self.github.token(), + # Since we dynamically create build steps, + # we use `virtual_builder_name` in the webinterface + # so that we distinguish what has beeing build + context=Interpolate("buildbot/%(prop:status_name)s"), + ) + ) + systemd_secrets = secrets.SecretInAFile( + dirname=os.environ["CREDENTIALS_DIRECTORY"] + ) + config["secretsProviders"] = config.get("secretsProviders", []) + config["secretsProviders"].append(systemd_secrets) + config["www"] = config.get("www", {}) + config["www"]["avatar_methods"] = config["www"].get("avatar_methods", []) + config["www"]["avatar_methods"].append(util.AvatarGitHub()) + config["www"]["auth"] = util.GitHubAuth( + self.github.oauth_id, read_secret_file(self.github.oauth_secret_name) + ) + config["www"]["authz"] = util.Authz( + roleMatchers=[ + util.RolesFromUsername(roles=["admin"], usernames=self.github.admins) + ], + allowRules=[ + util.AnyEndpointMatcher(role="admin", defaultDeny=False), + util.AnyControlEndpointMatcher(role="admins"), + ], + ) + config["www"]["change_hook_dialects"] = config["www"].get( + "change_hook_dialects", {} + ) + config["www"]["change_hook_dialects"]["github"] = { + "secret": read_secret_file(self.github.webhook_secret_name), + "strict": True, + "token": self.github.token(), + "github_property_whitelist": "*", + } diff --git a/buildbot_nix/master.py b/buildbot_nix/master.py index 4525e7e..46c270b 100644 --- a/buildbot_nix/master.py +++ b/buildbot_nix/master.py @@ -1,161 +1,39 @@ #!/usr/bin/env python3 -import json import os import sys from datetime import timedelta from pathlib import Path from typing import Any -from buildbot.plugins import reporters, schedulers, secrets, util, worker -from buildbot.process.project import Project -from buildbot.process.properties import Interpolate +from buildbot.plugins import schedulers, util # allow to import modules sys.path.append(str(Path(__file__).parent)) -from buildbot_nix import ( # noqa: E402 - nix_build_config, - nix_eval_config, - nix_update_flake_config, -) -from github_projects import GithubProject, load_projects # noqa: E402 - - -def read_secret_file(secret_name: str) -> str: - directory = os.environ.get("CREDENTIALS_DIRECTORY") - if directory is None: - print("directory not set", file=sys.stderr) - sys.exit(1) - return Path(directory).joinpath(secret_name).read_text() - - -GITHUB_OAUTH_ID = os.environ.get("GITHUB_OAUTH_ID") -GITHUB_OAUTH_SECRET = read_secret_file("github-oauth-secret") -GITHUB_ADMINS = os.environ.get("GITHUB_ADMINS", "").split(" ") -GITHUB_TOKEN_SECRET_NAME = "github-token" -GITHUB_TOKEN = read_secret_file(GITHUB_TOKEN_SECRET_NAME) -GITHUB_WEBHOOK_SECRET = read_secret_file("github-webhook-secret") -# Shape of this file: -# [ { "name": "", "pass": "", "cores": "" } ] -BUILDBOT_NIX_WORKERS = read_secret_file("buildbot-nix-workers") -BUILDBOT_URL = os.environ["BUILDBOT_URL"] -BUILDBOT_GITHUB_USER = os.environ["BUILDBOT_GITHUB_USER"] -NIX_SUPPORTED_SYSTEMS = os.environ["NIX_SUPPORTED_SYSTEMS"].split(" ") -NIX_EVAL_MAX_MEMORY_SIZE = int(os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096")) -BUILDBOT_DB_URL = os.environ.get("DB_URL", "sqlite:///state.sqlite") - - -def config_for_project( - config: dict[str, Any], - project: GithubProject, - credentials: str, - worker_names: list[str], -) -> Project: - config["projects"].append(Project(project.name)) - config["schedulers"].extend( - [ - schedulers.SingleBranchScheduler( - name=f"default-branch-{project.id}", - change_filter=util.ChangeFilter( - repository=project.url, - filter_fn=lambda c: c.branch - == c.properties.getProperty("github.repository.default_branch"), - ), - builderNames=[f"{project.name}/nix-eval"], - ), - # this is compatible with bors or github's merge queue - schedulers.SingleBranchScheduler( - name=f"merge-queue-{project.id}", - change_filter=util.ChangeFilter( - repository=project.url, - branch_re="(gh-readonly-queue/.*|staging|trying)", - ), - builderNames=[f"{project.name}/nix-eval"], - ), - # build all pull requests - schedulers.SingleBranchScheduler( - name=f"prs-{project.id}", - change_filter=util.ChangeFilter( - repository=project.url, category="pull" - ), - builderNames=[f"{project.name}/nix-eval"], - ), - # this is triggered from `nix-eval` - schedulers.Triggerable( - name=f"{project.id}-nix-build", - builderNames=[f"{project.name}/nix-build"], - ), - # allow to manually trigger a nix-build - schedulers.ForceScheduler( - name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"] - ), - # allow to manually update flakes - schedulers.ForceScheduler( - name=f"{project.id}-update-flake", - builderNames=[f"{project.name}/update-flake"], - buttonName="Update flakes", - ), - # updates flakes once a weeek - schedulers.NightlyTriggerable( - name=f"{project.id}-update-flake-weekly", - builderNames=[f"{project.name}/update-flake"], - hour=3, - minute=0, - dayOfWeek=6, - ), - ] - ) - has_cachix_auth_token = os.path.isfile( - os.path.join(credentials, "cachix-auth-token") - ) - has_cachix_signing_key = os.path.isfile( - os.path.join(credentials, "cachix-signing-key") - ) - config["builders"].extend( - [ - # Since all workers run on the same machine, we only assign one of them to do the evaluation. - # This should prevent exessive memory usage. - nix_eval_config( - project, - [worker_names[0]], - github_token_secret=GITHUB_TOKEN_SECRET_NAME, - supported_systems=NIX_SUPPORTED_SYSTEMS, - automerge_users=[BUILDBOT_GITHUB_USER], - max_memory_size=NIX_EVAL_MAX_MEMORY_SIZE, - ), - nix_build_config( - project, - worker_names, - has_cachix_auth_token, - has_cachix_signing_key, - ), - nix_update_flake_config( - project, - worker_names, - github_token_secret=GITHUB_TOKEN_SECRET_NAME, - github_bot_user=BUILDBOT_GITHUB_USER, - ), - ] - ) - - -PROJECT_CACHE_FILE = Path("github-project-cache.json") +from buildbot_nix import GithubConfig, NixConfigurator # noqa: E402 def build_config() -> dict[str, Any]: - projects = load_projects(GITHUB_TOKEN, PROJECT_CACHE_FILE) - projects = [p for p in projects if "build-with-buildbot" in p.topics] - import pprint - - pprint.pprint(projects) c: dict[str, Any] = {} c["buildbotNetUsageData"] = None # configure a janitor which will delete all logs older than one month, and will run on sundays at noon c["configurators"] = [ - util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6) + util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6), + NixConfigurator( + github=GithubConfig( + oauth_id=os.environ["GITHUB_OAUTH_ID"], + admins=os.environ.get("GITHUB_ADMINS", "").split(" "), + buildbot_user=os.environ["BUILDBOT_GITHUB_USER"], + ), + nix_eval_max_memory_size=int( + os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096") + ), + nix_supported_systems=os.environ.get("NIX_SUPPORTED_SYSTEMS", "auto").split( + " " + ), + ), ] - credentials = os.environ.get("CREDENTIALS_DIRECTORY", ".") c["schedulers"] = [ schedulers.SingleBranchScheduler( name="nixpkgs", @@ -171,62 +49,17 @@ def build_config() -> dict[str, Any]: c["builders"] = [] c["projects"] = [] c["workers"] = [] - - worker_config = json.loads(BUILDBOT_NIX_WORKERS) - worker_names = [] - for item in worker_config: - cores = item.get("cores", 0) - for i in range(cores): - worker_name = f"{item['name']}-{i}" - c["workers"].append(worker.Worker(worker_name, item["pass"])) - worker_names.append(worker_name) - - for project in projects: - config_for_project(c, project, credentials, worker_names) - - c["services"] = [ - reporters.GitHubStatusPush( - token=GITHUB_TOKEN, - # Since we dynamically create build steps, - # we use `virtual_builder_name` in the webinterface - # so that we distinguish what has beeing build - context=Interpolate("buildbot/%(prop:status_name)s"), - ) - ] - - systemd_secrets = secrets.SecretInAFile(dirname=credentials) - c["secretsProviders"] = [systemd_secrets] - + c["services"] = [] c["www"] = { - "avatar_methods": [util.AvatarGitHub()], - "port": int(os.environ.get("PORT", "1810")), - "auth": util.GitHubAuth(GITHUB_OAUTH_ID, GITHUB_OAUTH_SECRET), - "authz": util.Authz( - roleMatchers=[ - util.RolesFromUsername(roles=["admin"], usernames=GITHUB_ADMINS) - ], - allowRules=[ - util.AnyEndpointMatcher(role="admin", defaultDeny=False), - util.AnyControlEndpointMatcher(role="admins"), - ], - ), "plugins": dict( base_react={}, waterfall_view={}, console_view={}, grid_view={} ), - "change_hook_dialects": dict( - github={ - "secret": GITHUB_WEBHOOK_SECRET, - "strict": True, - "token": GITHUB_TOKEN, - "github_property_whitelist": "*", - } - ), + "port": int(os.environ.get("PORT", "1810")), } - c["db"] = {"db_url": BUILDBOT_DB_URL} - + c["db"] = {"db_url": os.environ.get("DB_URL", "sqlite:///state.sqlite")} c["protocols"] = {"pb": {"port": "tcp:9989:interface=\\:\\:"}} - c["buildbotURL"] = BUILDBOT_URL + c["buildbotURL"] = os.environ["BUILDBOT_URL"] return c diff --git a/flake.nix b/flake.nix index 311c696..d1909aa 100644 --- a/flake.nix +++ b/flake.nix @@ -6,11 +6,16 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + + # used for development + treefmt-nix.url = "github:numtide/treefmt-nix"; + treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = inputs@{ self, flake-parts, ... }: flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }: { + imports = inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix; systems = [ "x86_64-linux" "aarch64-linux" ]; flake = { nixosModules.buildbot-master = ./nix/master.nix; diff --git a/nix/master.nix b/nix/master.nix index d4bd89f..4cbfa49 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -50,7 +50,8 @@ in }; githubAdmins = lib.mkOption { type = lib.types.listOf lib.types.str; - description = "Users that are allowed to login to buildbot and do stuff"; + default = [ ]; + description = "Users that are allowed to login to buildbot, trigger builds and change settings"; }; }; workersFile = lib.mkOption {