auxolotl/buildbot-nix
26
1
Fork 0
Code Issues Pull requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #34 from Mic92/fixes

Browse source 
Simplify retry, fix github avatars, make evalWorkerCount configurable
This commit is contained in:
Jörg Thalheim 2023-11-12 07:30:08 +01:00 committed by GitHub
parent 8dbd6a2101 5b4ddd014c
commit 01b1e50bea
Failed to generate hash of commit
3 changed files with 23 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
buildbot_nix/__init__.py
View file

@ -413,7 +413,8 @@ def nix_eval_config(
github_token_secret: str, github_token_secret: str,
supported_systems: list[str], supported_systems: list[str],
eval_lock: util.WorkerLock, eval_lock: util.WorkerLock,
max_memory_size: int = 4096, worker_count: int,
max_memory_size: int,
) -> util.BuilderConfig: ) -> util.BuilderConfig:
""" """
Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel. Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
@ -441,7 +442,7 @@ def nix_eval_config(
command=[ command=[
"nix-eval-jobs", "nix-eval-jobs",
"--workers", "--workers",
multiprocessing.cpu_count(), str(worker_count),
"--max-memory-size", "--max-memory-size",
str(max_memory_size), str(max_memory_size),
"--option", "--option",
@ -586,6 +587,7 @@ def config_for_project(
worker_names: list[str], worker_names: list[str],
github: GithubConfig, github: GithubConfig,
nix_supported_systems: list[str], nix_supported_systems: list[str],
nix_eval_worker_count: int,
nix_eval_max_memory_size: int, nix_eval_max_memory_size: int,
eval_lock: util.WorkerLock, eval_lock: util.WorkerLock,
outputs_path: Path | None = None, outputs_path: Path | None = None,
@ -678,6 +680,7 @@ def config_for_project(
[worker_names[0]], [worker_names[0]],
github_token_secret=github.token_secret_name, github_token_secret=github.token_secret_name,
supported_systems=nix_supported_systems, supported_systems=nix_supported_systems,
worker_count=nix_eval_worker_count,
max_memory_size=nix_eval_max_memory_size, max_memory_size=nix_eval_max_memory_size,
eval_lock=eval_lock, eval_lock=eval_lock,
), ),
@ -708,13 +711,15 @@ class NixConfigurator(ConfiguratorBase):
github: GithubConfig, github: GithubConfig,
url: str, url: str,
nix_supported_systems: list[str], nix_supported_systems: list[str],
nix_eval_max_memory_size: int = 4096, nix_eval_worker_count: int | None,
nix_eval_max_memory_size: int,
nix_workers_secret_name: str = "buildbot-nix-workers", nix_workers_secret_name: str = "buildbot-nix-workers",
outputs_path: str | None = None, outputs_path: str | None = None,
) -> None: ) -> None:
super().__init__() super().__init__()
self.nix_workers_secret_name = nix_workers_secret_name self.nix_workers_secret_name = nix_workers_secret_name
self.nix_eval_max_memory_size = nix_eval_max_memory_size self.nix_eval_max_memory_size = nix_eval_max_memory_size
self.nix_eval_worker_count = nix_eval_worker_count
self.nix_supported_systems = nix_supported_systems self.nix_supported_systems = nix_supported_systems
self.github = github self.github = github
self.url = url self.url = url
@ -762,6 +767,7 @@ class NixConfigurator(ConfiguratorBase):
worker_names, worker_names,
self.github, self.github,
self.nix_supported_systems, self.nix_supported_systems,
self.nix_eval_worker_count or multiprocessing.cpu_count(),
self.nix_eval_max_memory_size, self.nix_eval_max_memory_size,
eval_lock, eval_lock,
self.outputs_path, self.outputs_path,
@ -816,7 +822,7 @@ class NixConfigurator(ConfiguratorBase):
if not config["www"].get("auth"): if not config["www"].get("auth"):
config["www"]["avatar_methods"] = config["www"].get("avatar_methods", []) config["www"]["avatar_methods"] = config["www"].get("avatar_methods", [])
config["www"]["avatar_methods"].append( config["www"]["avatar_methods"].append(
util.AvatarGitHub(self.github.token()) util.AvatarGitHub(token=self.github.token())
) )
config["www"]["auth"] = util.GitHubAuth( config["www"]["auth"] = util.GitHubAuth(
self.github.oauth_id, read_secret_file(self.github.oauth_secret_name) self.github.oauth_id, read_secret_file(self.github.oauth_secret_name)

4
examples/default.nix
View file

@ -38,6 +38,10 @@ in
}; };
# optional # optional
# outputsPath = "/var/www/buildbot/nix-outputs"; # outputsPath = "/var/www/buildbot/nix-outputs";
# optional nix-eval-jobs settings
# evalWorkerCount = 8; # limit number of concurrent evaluations
# evalMaxMemorySize = "2048"; # limit memory usage per evaluation
}; };
}) })
buildbot-nix.nixosModules.buildbot-master buildbot-nix.nixosModules.buildbot-master

9
nix/master.nix
View file

@ -75,6 +75,14 @@ in
restarted. restarted.
''; '';
}; };
evalWorkerCount = lib.mkOption {
type = lib.types.nullOr lib.types.int;
default = null;
description = ''
Number of nix-eval-jobs worker processes. If null, the number of cores is used.
If you experience memory issues (buildbot-workers going out-of-memory), you can reduce this number.
'';
};
domain = lib.mkOption { domain = lib.mkOption {
type = lib.types.str; type = lib.types.str;
description = "Buildbot domain"; description = "Buildbot domain";
@ -141,6 +149,7 @@ in
), ),
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl}, url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize}, nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_eval_worker_count=${builtins.toJSON cfg.evalWorkerCount},
nix_supported_systems=${builtins.toJSON cfg.buildSystems}, nix_supported_systems=${builtins.toJSON cfg.buildSystems},
outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath}, outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
) )