From 30b8d7b0909dc79195b8e22b676348e4a01d1bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 6 Nov 2023 07:46:07 +0100 Subject: [PATCH 1/6] Configurator: make sure all config values have defaults set before hand --- buildbot_nix/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 3004f91..a7aad52 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -719,7 +719,11 @@ class NixConfigurator(ConfiguratorBase): projects = [p for p in projects if self.github.topic in p.topics] worker_config = json.loads(read_secret_file(self.nix_workers_secret_name)) worker_names = [] - config["workers"] = config.get("workers", []) + + config.setdefault("projects", []) + config.setdefault("secretsProviders", []) + config.setdefault("www", {}) + for item in worker_config: cores = item.get("cores", 0) for i in range(cores): @@ -727,8 +731,6 @@ class NixConfigurator(ConfiguratorBase): config["workers"].append(worker.Worker(worker_name, item["pass"])) worker_names.append(worker_name) - config["projects"] = config.get("projects", []) - webhook_secret = read_secret_file(self.github.webhook_secret_name) for project in projects: @@ -775,7 +777,6 @@ class NixConfigurator(ConfiguratorBase): ), ] ) - config["services"] = config.get("services", []) config["services"].append( reporters.GitHubStatusPush( token=self.github.token(), @@ -788,9 +789,7 @@ class NixConfigurator(ConfiguratorBase): systemd_secrets = secrets.SecretInAFile( dirname=os.environ["CREDENTIALS_DIRECTORY"] ) - config["secretsProviders"] = config.get("secretsProviders", []) config["secretsProviders"].append(systemd_secrets) - config["www"] = config.get("www", {}) config["www"]["change_hook_dialects"] = config["www"].get( "change_hook_dialects", {} ) From f7573d02bc4ef9237e47a307824fbe5c2b5e34b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 9 Nov 2023 08:52:46 +0100 Subject: [PATCH 2/6] make sure update-flake has a github project field set --- buildbot_nix/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index a7aad52..cbca7cb 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -644,6 +644,13 @@ def config_for_project( name=f"{project.id}-update-flake", builderNames=[f"{project.name}/update-flake"], buttonName="Update flakes", + properties=[ + util.StringParameter( + name="project", + label="Name of the GitHub repository.", + default=project.name, + ) + ], ), # updates flakes once a week # schedulers.Periodic( From abdff1d9270527ac3587325ae4ea42b61f15a9e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 6 Nov 2023 07:46:24 +0100 Subject: [PATCH 3/6] also pass GITHUB_TOKEN to AvatarGithub --- buildbot_nix/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index cbca7cb..3d706f6 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -809,7 +809,9 @@ class NixConfigurator(ConfiguratorBase): if not config["www"].get("auth"): config["www"]["avatar_methods"] = config["www"].get("avatar_methods", []) - config["www"]["avatar_methods"].append(util.AvatarGitHub()) + config["www"]["avatar_methods"].append( + util.AvatarGitHub(self.github.token()) + ) config["www"]["auth"] = util.GitHubAuth( self.github.oauth_id, read_secret_file(self.github.oauth_secret_name) ) From 0b145d91d8d52b52ddad6d341ed649e4a3e1e4bd Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:52:23 +1000 Subject: [PATCH 4/6] master: add prometheus metrics --- nix/master.nix | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nix/master.nix b/nix/master.nix index c168386..3bdc584 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -87,6 +87,12 @@ in default = null; example = "/var/www/buildbot/nix-outputs"; }; + + prometheusExporterPort = lib.mkOption { + type = lib.types.nullOr lib.types.port; + default = null; + description = "Port where metrics will be served"; + }; }; }; config = lib.mkIf cfg.enable { @@ -117,6 +123,9 @@ in c["www"]["plugins"].update( dict(base_react={}, waterfall_view={}, console_view={}, grid_view={}) ) + ${lib.optionalString (cfg.prometheusExporterPort != null) '' + c['services'].append(reporters.Prometheus(port=${builtins.toString cfg.prometheusExporterPort})) + ''} ''; configurators = [ '' @@ -160,7 +169,19 @@ in pkgs.buildbot-plugins.wsgi-dashboards pkgs.buildbot-plugins.badges (pkgs.python3.pkgs.callPackage ../default.nix { }) - ]; + ] ++ lib.optional (cfg.prometheusExporterPort != null) + (ps.buildPythonPackage rec { + pname = "buildbot-prometheus"; + version = "0c81a89bbe34628362652fbea416610e215b5d1e"; + src = pkgs.fetchFromGitHub { + owner = "claws"; + repo = "buildbot-prometheus"; + rev = version; + hash = "sha256-bz2Nv2RZ44i1VoPvQ/XjGMfTT6TmW6jhEVwItPk23SM="; + }; + propagatedBuildInputs = [ ps.prometheus-client ]; + doCheck = false; + }); }; systemd.services.buildbot-master = { From 2a565a8d5a795a677fae8b06628f220eb4a16b87 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Thu, 9 Nov 2023 17:18:03 +1000 Subject: [PATCH 5/6] pad worker numbering --- buildbot_nix/__init__.py | 2 +- buildbot_nix/worker.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 3d706f6..4dc338a 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -734,7 +734,7 @@ class NixConfigurator(ConfiguratorBase): for item in worker_config: cores = item.get("cores", 0) for i in range(cores): - worker_name = f"{item['name']}-{i}" + worker_name = f"{item['name']}-{i:03}" config["workers"].append(worker.Worker(worker_name, item["pass"])) worker_names.append(worker_name) diff --git a/buildbot_nix/worker.py b/buildbot_nix/worker.py index 7e1181e..65d0f8b 100644 --- a/buildbot_nix/worker.py +++ b/buildbot_nix/worker.py @@ -29,11 +29,11 @@ class WorkerConfig: def setup_worker( application: service.Application, id: int, config: WorkerConfig ) -> None: - basedir = f"{config.buildbot_dir}-{id}" + basedir = f"{config.buildbot_dir}-{id:03}" os.makedirs(basedir, mode=0o700, exist_ok=True) hostname = socket.gethostname() - workername = f"{hostname}-{id}" + workername = f"{hostname}-{id:03}" keepalive = 600 umask = None maxdelay = 300 From 524c8fd1507a4dec6a2129cc2f13489c644366f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 10 Nov 2023 14:30:09 +0100 Subject: [PATCH 6/6] buildbot-nix: limit evaluation to one job per worker --- buildbot_nix/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 4dc338a..1abb1d9 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -412,6 +412,7 @@ def nix_eval_config( worker_names: list[str], github_token_secret: str, supported_systems: list[str], + eval_lock: util.WorkerLock, max_memory_size: int = 4096, ) -> util.BuilderConfig: """ @@ -454,6 +455,7 @@ def nix_eval_config( ".#checks", ], haltOnFailure=True, + locks=[eval_lock.access("counting")], ) ) @@ -585,6 +587,7 @@ def config_for_project( github: GithubConfig, nix_supported_systems: list[str], nix_eval_max_memory_size: int, + eval_lock: util.WorkerLock, outputs_path: Path | None = None, ) -> Project: ## get a deterministic jitter for the project @@ -676,6 +679,7 @@ def config_for_project( github_token_secret=github.token_secret_name, supported_systems=nix_supported_systems, max_memory_size=nix_eval_max_memory_size, + eval_lock=eval_lock, ), nix_build_config( project, @@ -739,6 +743,7 @@ class NixConfigurator(ConfiguratorBase): worker_names.append(worker_name) webhook_secret = read_secret_file(self.github.webhook_secret_name) + eval_lock = util.WorkerLock("nix-eval", maxCount=1) for project in projects: create_project_hook( @@ -758,6 +763,7 @@ class NixConfigurator(ConfiguratorBase): self.github, self.nix_supported_systems, self.nix_eval_max_memory_size, + eval_lock, self.outputs_path, )