buildbot-nix/buildbot_nix/worker.py

80 lines
2.2 KiB
Python
Raw Permalink Normal View History

2023-09-10 08:11:56 +00:00
import multiprocessing
import os
import socket
2023-12-26 20:56:36 +00:00
from dataclasses import dataclass, field
2023-09-10 08:11:56 +00:00
from pathlib import Path
from buildbot_worker.bot import Worker
from twisted.application import service
2024-01-01 07:32:22 +00:00
from twisted.python import components
2023-09-10 08:11:56 +00:00
def require_env(key: str) -> str:
val = os.environ.get(key)
2024-04-15 13:17:25 +00:00
assert val is not None, f"{key} environment variable is not set"
2023-09-10 08:11:56 +00:00
return val
@dataclass
class WorkerConfig:
2023-12-26 20:56:36 +00:00
password: str = field(
default_factory=lambda: Path(require_env("WORKER_PASSWORD_FILE"))
.read_text()
.rstrip("\r\n")
)
2024-04-15 13:17:25 +00:00
worker_name: str = field(
default_factory=lambda: os.environ.get("WORKER_NAME", socket.gethostname())
)
worker_count: int = int(
2023-12-26 20:56:36 +00:00
os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())),
)
buildbot_dir: Path = field(
default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
)
2023-12-26 20:56:36 +00:00
master_url: str = field(default_factory=lambda: require_env("MASTER_URL"))
2023-09-10 08:11:56 +00:00
def setup_worker(
2024-01-01 07:32:22 +00:00
application: components.Componentized,
2023-12-26 20:56:36 +00:00
builder_id: int,
config: WorkerConfig,
) -> None:
2023-12-26 20:56:36 +00:00
basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}"
basedir.mkdir(parents=True, exist_ok=True, mode=0o700)
2023-09-10 08:11:56 +00:00
2024-04-15 13:17:25 +00:00
workername = f"{config.worker_name}-{builder_id:03}"
2023-09-10 08:11:56 +00:00
keepalive = 600
umask = None
maxdelay = 300
numcpus = None
allow_shutdown = None
2024-04-30 15:55:27 +00:00
print(f"Starting worker {workername}")
2023-09-10 08:11:56 +00:00
s = Worker(
None,
None,
workername,
config.password,
2023-12-27 00:12:08 +00:00
str(basedir),
2023-09-10 08:11:56 +00:00
keepalive,
connection_string=config.master_url,
2023-09-10 08:11:56 +00:00
umask=umask,
maxdelay=maxdelay,
numcpus=numcpus,
allow_shutdown=allow_shutdown,
)
# defaults to 4096, bump to 10MB for nix-eval-jobs
s.bot.max_line_length = 10485760
s.setServiceParent(application)
2024-01-01 07:32:22 +00:00
def setup_workers(application: components.Componentized, config: WorkerConfig) -> None:
for i in range(config.worker_count):
setup_worker(application, i, config)
2023-09-10 08:11:56 +00:00
# note: this line is matched against to check that this is a worker
# directory; do not edit it.
2024-01-01 07:32:22 +00:00
application = service.Application("buildbot-worker") # type: ignore[no-untyped-call]
setup_workers(application, WorkerConfig())