From aeffb167feca9c943f11fcd996281940bfe7cb0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 09:09:41 +0200 Subject: [PATCH 1/4] worker: move global variables to config class --- buildbot_nix/worker.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/buildbot_nix/worker.py b/buildbot_nix/worker.py index 1bb8d5b..7e1181e 100644 --- a/buildbot_nix/worker.py +++ b/buildbot_nix/worker.py @@ -3,6 +3,7 @@ import multiprocessing import os import socket +from dataclasses import dataclass from pathlib import Path from buildbot_worker.bot import Worker @@ -15,13 +16,20 @@ def require_env(key: str) -> str: return val -PASSWD = Path(require_env("WORKER_PASSWORD_FILE")).read_text().strip("\r\n") -BUILDBOT_DIR = require_env("BUILDBOT_DIR") -MASTER_URL = require_env("MASTER_URL") +@dataclass +class WorkerConfig: + password: str = Path(require_env("WORKER_PASSWORD_FILE")).read_text().rstrip("\r\n") + worker_count: int = int( + os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())) + ) + buildbot_dir: str = require_env("BUILDBOT_DIR") + master_url: str = require_env("MASTER_URL") -def setup_worker(application: service.Application, id: int) -> None: - basedir = f"{BUILDBOT_DIR}-{id}" +def setup_worker( + application: service.Application, id: int, config: WorkerConfig +) -> None: + basedir = f"{config.buildbot_dir}-{id}" os.makedirs(basedir, mode=0o700, exist_ok=True) hostname = socket.gethostname() @@ -36,10 +44,10 @@ def setup_worker(application: service.Application, id: int) -> None: None, None, workername, - PASSWD, + config.password, basedir, keepalive, - connection_string=MASTER_URL, + connection_string=config.master_url, umask=umask, maxdelay=maxdelay, numcpus=numcpus, @@ -50,9 +58,12 @@ def setup_worker(application: service.Application, id: int) -> None: s.setServiceParent(application) +def setup_workers(application: service.Application, config: WorkerConfig) -> None: + for i in range(config.worker_count): + setup_worker(application, i, config) + + # note: this line is matched against to check that this is a worker # directory; do not edit it. application = service.Application("buildbot-worker") - -for i in range(multiprocessing.cpu_count()): - setup_worker(application, i) +setup_workers(application, WorkerConfig()) From d10a73c6661153d4775deed3ed6f41518e2f9b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 09:10:02 +0200 Subject: [PATCH 2/4] also test buildbot on aarch64-linux --- examples/default.nix | 6 +++--- flake.nix | 50 +++++++++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/examples/default.nix b/examples/default.nix index 30638b7..2ee51bf 100644 --- a/examples/default.nix +++ b/examples/default.nix @@ -1,4 +1,4 @@ -{ nixpkgs, system, srvos, buildbot-nix, disko, ... }: +{ nixpkgs, system, buildbot-nix, ... }: let # some example configuration to make it eval dummy = { config, modulesPath, ... }: { @@ -15,7 +15,7 @@ let inherit (lib) nixosSystem; in { - example-master = nixosSystem { + "example-master-${system}" = nixosSystem { inherit system; modules = [ dummy @@ -44,7 +44,7 @@ in buildbot-nix.nixosModules.buildbot-master ]; }; - example-worker = nixosSystem { + "example-worker-${system}" = nixosSystem { inherit system; modules = [ dummy diff --git a/flake.nix b/flake.nix index 7e80f5a..311c696 100644 --- a/flake.nix +++ b/flake.nix @@ -9,28 +9,36 @@ }; outputs = inputs@{ self, flake-parts, ... }: - flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }: { - systems = [ "x86_64-linux" ]; - flake = { - nixosModules.buildbot-master = ./nix/master.nix; - nixosModules.buildbot-worker = ./nix/worker.nix; + flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }: + { + systems = [ "x86_64-linux" "aarch64-linux" ]; + flake = { + nixosModules.buildbot-master = ./nix/master.nix; + nixosModules.buildbot-worker = ./nix/worker.nix; - nixosConfigurations = import ./examples { - inherit (inputs) nixpkgs srvos disko; - buildbot-nix = self; - system = "x86_64-linux"; + nixosConfigurations = + let + examplesFor = system: import ./examples { + inherit system; + inherit (inputs) nixpkgs; + buildbot-nix = self; + }; + in + examplesFor "x86_64-linux" // examplesFor "aarch64-linux"; }; - checks.x86_64-linux = { - nixos-master = self.nixosConfigurations.example-master.config.system.build.toplevel; - nixos-worker = self.nixosConfigurations.example-worker.config.system.build.toplevel; + perSystem = { self', pkgs, system, ... }: { + packages.default = pkgs.mkShell { + packages = [ + pkgs.bashInteractive + ]; + }; + checks = + let + nixosMachines = lib.mapAttrs' (name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel) ((lib.filterAttrs (_: config: config.pkgs.system == system)) self.nixosConfigurations); + packages = lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages; + devShells = lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells; + in + nixosMachines // packages // devShells; }; - }; - perSystem = { pkgs, system, ... }: { - packages.default = pkgs.mkShell { - packages = [ - pkgs.bashInteractive - ]; - }; - }; - }); + }); } From 5d10d20bc9b737230882fa89d2f7b3ede1581ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 09:10:19 +0200 Subject: [PATCH 3/4] gitignore result symlinks --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b2be92b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +result From 2afa25ca270f57d9a9c9a546c1d31bc468c3a6b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 09:10:35 +0200 Subject: [PATCH 4/4] master: move DB_URL to global variable --- buildbot_nix/master.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/master.py b/buildbot_nix/master.py index 51a7cc1..b48c560 100644 --- a/buildbot_nix/master.py +++ b/buildbot_nix/master.py @@ -44,6 +44,7 @@ BUILDBOT_URL = os.environ["BUILDBOT_URL"] BUILDBOT_GITHUB_USER = os.environ["BUILDBOT_GITHUB_USER"] NIX_SUPPORTED_SYSTEMS = os.environ["NIX_SUPPORTED_SYSTEMS"].split(" ") NIX_EVAL_MAX_MEMORY_SIZE = int(os.environ.get("NIX_EVAL_MAX_MEMORY_SIZE", "4096")) +BUILDBOT_DB_URL = os.environ.get("DB_URL", "sqlite:///state.sqlite") def config_for_project( @@ -225,7 +226,7 @@ def build_config() -> dict[str, Any]: ), } - c["db"] = {"db_url": os.environ.get("DB_URL", "sqlite:///state.sqlite")} + c["db"] = {"db_url": BUILDBOT_DB_URL} c["protocols"] = {"pb": {"port": "tcp:9989:interface=\\:\\:"}} c["buildbotURL"] = BUILDBOT_URL