auxolotl/buildbot-nix
26
1
Fork 0
Code Issues Pull requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #6 from Mic92/master-refactor

Browse source 
[wip] Master refactor
This commit is contained in:
Jörg Thalheim 2023-10-09 16:19:45 +02:00 committed by GitHub
parent cb356fc006 0d39562812
commit 9d5838fa37
Failed to generate hash of commit
10 changed files with 118 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
buildbot_nix/buildbot_nix.py
View file

@ -685,6 +685,7 @@ class NixConfigurator(ConfiguratorBase):
config["workers"].append(worker.Worker(worker_name, item["pass"]))
worker_names.append(worker_name)
config["projects"] = config.get("projects", [])
for project in projects:
config_for_project(
config,

67
buildbot_nix/master.py
View file

@ -1,67 +0,0 @@
#!/usr/bin/env python3
import os
import sys
from datetime import timedelta
from pathlib import Path
from typing import Any
from buildbot.plugins import schedulers, util
# allow to import modules
sys.path.append(str(Path(__file__).parent))
from buildbot_nix import GithubConfig, NixConfigurator # noqa: E402
def build_config() -> dict[str, Any]:
c: dict[str, Any] = {}
c["buildbotNetUsageData"] = None
# configure a janitor which will delete all logs older than one month, and will run on sundays at noon
c["configurators"] = [
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6),
NixConfigurator(
github=GithubConfig(
oauth_id=os.environ["GITHUB_OAUTH_ID"],
admins=os.environ.get("GITHUB_ADMINS", "").split(" "),
buildbot_user=os.environ["BUILDBOT_GITHUB_USER"],
),
nix_eval_max_memory_size=int(
os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096")
),
nix_supported_systems=os.environ.get("NIX_SUPPORTED_SYSTEMS", "auto").split(
" "
),
),
]
c["schedulers"] = [
schedulers.SingleBranchScheduler(
name="nixpkgs",
change_filter=util.ChangeFilter(
repository_re=r"https://github\.com/.*/nixpkgs",
filter_fn=lambda c: c.branch
== c.properties.getProperty("github.repository.default_branch"),
),
treeStableTimer=20,
builderNames=["Mic92/dotfiles/update-flake"],
),
]
c["builders"] = []
c["projects"] = []
c["workers"] = []
c["services"] = []
c["www"] = {
"plugins": dict(
base_react={}, waterfall_view={}, console_view={}, grid_view={}
),
"port": int(os.environ.get("PORT", "1810")),
}
c["db"] = {"db_url": os.environ.get("DB_URL", "sqlite:///state.sqlite")}
c["protocols"] = {"pb": {"port": "tcp:9989:interface=\\:\\:"}}
c["buildbotURL"] = os.environ["BUILDBOT_URL"]
return c
BuildmasterConfig = build_config()

18
flake.lock
View file

@ -22,16 +22,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1694310981,
"narHash": "sha256-Rf6ICXzUXfi2ILwEggBJMmVVR06AKS5DMcXZd1qi24o=",
"owner": "NixOS",
"lastModified": 1695522116,
"narHash": "sha256-hISZSYVmlpJYKNR+kRWk5JBTQUX9lOJYSFPfObwOkDk=",
"owner": "Mic92",
"repo": "nixpkgs",
"rev": "49772c4112bf70d28086b459d31710408745f842",
"rev": "7031d27d0086c378212c138e15bda5d70037bd0f",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"owner": "Mic92",
"ref": "buildbot",
"repo": "nixpkgs",
"type": "github"
}
@ -50,11 +50,11 @@
]
},
"locked": {
"lastModified": 1694528738,
"narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=",
"lastModified": 1695290086,
"narHash": "sha256-ol6licpIAzc9oMsEai/9YZhgSMcrnlnD/3ulMLGNKL0=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8",
"rev": "e951529be2e7c669487de78f5aef8597bbae5fca",
"type": "github"
},
"original": {

6
flake.nix
View file

@ -3,7 +3,7 @@
description = "A nixos module to make buildbot a proper Nix-CI.";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
nixpkgs.url = "github:Mic92/nixpkgs/buildbot";
flake-parts.url = "github:hercules-ci/flake-parts";
flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
@ -15,7 +15,9 @@
outputs = inputs@{ self, flake-parts, ... }:
flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }:
{
imports = inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
imports = [
./nix/checks/flake-module.nix
] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
systems = [ "x86_64-linux" "aarch64-linux" ];
flake = {
nixosModules.buildbot-master = ./nix/master.nix;

15
nix/checks/flake-module.nix Normal file
View file

@ -0,0 +1,15 @@
{ self, ... }: {
perSystem =
{ pkgs
, ...
}: {
checks =
let
# this gives us a reference to our flake but also all flake inputs
checkArgs = { inherit self pkgs; };
in
{
master = import ./master.nix checkArgs;
};
};
}

20
nix/checks/lib.nix Normal file
View file

@ -0,0 +1,20 @@
# tests/lib.nix
# The first argument to this function is the test module itself
test:
# These arguments are provided by `flake.nix` on import, see checkArgs
{ pkgs, self }:
let
inherit (pkgs) lib;
# this imports the nixos library that contains our testing framework
nixos-lib = import (pkgs.path + "/nixos/lib") { };
in
(nixos-lib.runTest {
hostPkgs = pkgs;
# This speeds up the evaluation by skipping evaluating documentation (optional)
defaults.documentation.enable = lib.mkDefault false;
# This makes `self` available in the NixOS configuration of our virtual machines.
# This is useful for referencing modules or packages from your own flake
# as well as importing from other flakes.
node.specialArgs = { inherit self; };
imports = [ test ];
}).config.result

30
nix/checks/master.nix Normal file
View file

@ -0,0 +1,30 @@
(import ./lib.nix) {
name = "from-nixos";
nodes = {
# `self` here is set by using specialArgs in `lib.nix`
node1 = { self, ... }: {
imports = [
self.nixosModules.buildbot-master
];
services.buildbot-nix.master = {
enable = true;
domain = "buildbot2.thalheim.io";
workersFile = "/var/lib/secrets/buildbot-nix/workers.json";
github = {
tokenFile = "/var/lib/secrets/buildbot-nix/github-token";
webhookSecretFile = "/var/lib/secrets/buildbot-nix/github-webhook-secret";
oauthSecretFile = "/var/lib/secrets/buildbot-nix/github-oauth-secret";
oauthId = "aaaaaaaaaaaaaaaaaaaa";
githubUser = "mic92-buildbot";
githubAdmins = [ "Mic92" ];
};
};
};
};
# This is the test code that will check if our service is running correctly:
testScript = ''
start_all()
# wait for our service to start
node1.wait_for_unit("buildbot-master")
'';
}

56
nix/master.nix
View file

@ -10,11 +10,6 @@ in
options = {
services.buildbot-nix.master = {
enable = lib.mkEnableOption "buildbot-master";
port = lib.mkOption {
type = lib.types.int;
default = 1810;
description = "Port on which buildbot-master is listening";
};
dbUrl = lib.mkOption {
type = lib.types.str;
default = "postgresql://@/buildbot";
@ -82,7 +77,40 @@ in
config = lib.mkIf cfg.enable {
services.buildbot-master = {
enable = true;
masterCfg = "${../buildbot_nix}/master.py";
extraImports = ''
import sys
sys.path.append("${../buildbot_nix}")
from datetime import timedelta
from buildbot_nix import GithubConfig, NixConfigurator
'';
extraConfig = ''
c["www"]["plugins"] = c["www"].get("plugins", {})
c["www"]["plugins"].update(
dict(base_react={}, waterfall_view={}, console_view={}, grid_view={})
)
'';
configurators = [
''
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
''
''
NixConfigurator(
github=GithubConfig(
oauth_id=${builtins.toJSON cfg.github.oauthId},
admins=${builtins.toJSON cfg.github.githubAdmins},
buildbot_user=${builtins.toJSON cfg.github.githubUser},
),
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_supported_systems=${builtins.toJSON cfg.buildSystems},
)
''
];
buildbotUrl =
let
host = config.services.nginx.virtualHosts.${cfg.domain};
hasSSL = host.forceSSL || host.addSSL;
in
"${if hasSSL then "https" else "http"}://${cfg.domain}/";
dbUrl = config.services.buildbot-nix.master.dbUrl;
pythonPackages = ps: [
ps.requests
@ -95,16 +123,6 @@ in
};
systemd.services.buildbot-master = {
environment = {
PORT = builtins.toString cfg.port;
DB_URL = cfg.dbUrl;
GITHUB_OAUTH_ID = cfg.github.oauthId;
BUILDBOT_URL = "https://${cfg.domain}/";
BUILDBOT_GITHUB_USER = cfg.github.githubUser;
GITHUB_ADMINS = builtins.toString cfg.github.githubAdmins;
NIX_SUPPORTED_SYSTEMS = builtins.toString cfg.buildSystems;
NIX_EVAL_MAX_MEMORY_SIZE = builtins.toString cfg.evalMaxMemorySize;
};
serviceConfig = {
# in master.py we read secrets from $CREDENTIALS_DIRECTORY
LoadCredential = [
@ -129,14 +147,14 @@ in
services.nginx.enable = true;
services.nginx.virtualHosts.${cfg.domain} = {
locations."/".proxyPass = "http://127.0.0.1:${builtins.toString cfg.port}/";
locations."/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/";
locations."/sse" = {
proxyPass = "http://127.0.0.1:${builtins.toString cfg.port}/sse";
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse";
# proxy buffering will prevent sse to work
extraConfig = "proxy_buffering off;";
};
locations."/ws" = {
proxyPass = "http://127.0.0.1:${builtins.toString cfg.port}/ws";
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws";
proxyWebsockets = true;
# raise the proxy timeout for the websocket
extraConfig = "proxy_read_timeout 6000s;";

2
nix/treefmt/flake-module.nix
View file

@ -2,7 +2,7 @@
imports = [
inputs.treefmt-nix.flakeModule
];
perSystem = { config, pkgs, ... }: {
perSystem = { pkgs, ... }: {
treefmt = {
projectRootFile = ".git/config";
programs.nixpkgs-fmt.enable = true;

2
nix/worker.nix
View file

@ -67,7 +67,7 @@ in
WorkingDirectory = "/var/lib/buildbot-worker";
# Restart buildbot with a delay. This time way we can use buildbot to deploy itself.
ExecReload = "+${pkgs.systemd}/bin/systemd-run --on-active=60 ${pkgs.systemd}/bin/systemctl restart buildbot-worker";
ExecReload = "+${config.systemd.package}/bin/systemd-run --on-active=60 ${config.systemd.package}/bin/systemctl restart buildbot-worker";
ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${../buildbot_nix}/worker.py";
};
};