diff --git a/buildbot_nix/buildbot_nix.py b/buildbot_nix/buildbot_nix.py index d46c7c0..1b9ca2c 100644 --- a/buildbot_nix/buildbot_nix.py +++ b/buildbot_nix/buildbot_nix.py @@ -3,6 +3,7 @@ import json import multiprocessing import os +import signal import sys import uuid from collections import defaultdict @@ -23,8 +24,10 @@ from github_projects import ( # noqa: E402 GithubProject, create_project_hook, load_projects, + refresh_projects, ) -from twisted.internet import defer +from twisted.internet import defer, threads +from twisted.python.failure import Failure class BuildTrigger(Trigger): @@ -51,10 +54,10 @@ class BuildTrigger(Trigger): **kwargs, ) - def createTriggerProperties(self, props: Any) -> Any: + def createTriggerProperties(self, props: Any) -> Any: # noqa: N802 return props - def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: + def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802 build_props = self.build.getProperties() repo_name = build_props.getProperty( "github.base.repo.full_name", @@ -94,7 +97,7 @@ class BuildTrigger(Trigger): triggered_schedulers.append((sch, props)) return triggered_schedulers - def getCurrentSummary(self) -> dict[str, str]: + def getCurrentSummary(self) -> dict[str, str]: # noqa: N802 """ The original build trigger will the generic builder name `nix-build` in this case, which is not helpful """ @@ -250,6 +253,58 @@ class UpdateBuildOutput(steps.BuildStep): return util.SUCCESS +class ReloadGithubProjects(steps.BuildStep): + name = "reload_github_projects" + + def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: + self.token = token + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + refresh_projects(self.token, self.project_cache_file) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + log: Log = yield self.addLog("log") + log.addStderr(f"Failed to reload project list: {self.error_msg}") + return util.FAILURE + + +def reload_github_projects( + worker_names: list[str], + github_token_secret: str, + project_cache_file: Path, +) -> util.BuilderConfig: + """ + Updates the flake an opens a PR for it. + """ + factory = util.BuildFactory() + factory.addStep( + ReloadGithubProjects(github_token_secret, project_cache_file=project_cache_file) + ) + return util.BuilderConfig( + name="reload-github-projects", + workernames=worker_names, + factory=factory, + ) + + def nix_update_flake_config( project: GithubProject, worker_names: list[str], @@ -676,6 +731,21 @@ class NixConfigurator(ConfiguratorBase): self.nix_eval_max_memory_size, ) + # Reload github projects + config["builders"].append( + reload_github_projects( + [worker_names[0]], + self.github.token(), + self.github.project_cache_file, + ) + ) + config["schedulers"].append( + schedulers.ForceScheduler( + name="reload-github-projects", + builderNames=["reload-github-projects"], + buttonName="Update projects", + ) + ) config["services"] = config.get("services", []) config["services"].append( reporters.GitHubStatusPush( diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index aec4835..9f7756d 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,7 +1,9 @@ import http.client import json +import os import urllib.request from pathlib import Path +from tempfile import NamedTemporaryFile from typing import Any from twisted.python import log @@ -33,13 +35,13 @@ def http_request( try: resp = urllib.request.urlopen(req) except urllib.request.HTTPError as e: - body = "" + resp_body = "" try: - body = e.fp.read() + resp_body = e.fp.read().decode("utf-8", "replace") except Exception: pass raise Exception( - f"Request for {method} {url} failed with {e.code} {e.reason}: {body}" + f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" ) from e return HttpResponse(resp) @@ -101,11 +103,15 @@ class GithubProject: return self.data["topics"] -def create_project_hook(owner: str, repo: str, token: str, webhook_url: str, webhook_secret) -> None: +def create_project_hook( + owner: str, repo: str, token: str, webhook_url: str, webhook_secret: str +) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", token ) - config = dict(url=webhook_url, content_type="json", insecure_ssl="0", secret=webhook_secret) + config = dict( + url=webhook_url, content_type="json", insecure_ssl="0", secret=webhook_secret + ) data = dict(name="web", active=True, events=["push", "pull_request"], config=config) headers = { "Authorization": f"Bearer {token}", @@ -126,16 +132,25 @@ def create_project_hook(owner: str, repo: str, token: str, webhook_url: str, web ) +def refresh_projects(github_token: str, repo_cache_file: Path) -> None: + repos = paginated_github_request( + "https://api.github.com/user/repos?per_page=100", + github_token, + ) + with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: + try: + f.write(json.dumps(repos)) + f.flush() + os.rename(f.name, repo_cache_file) + except OSError: + os.unlink(f.name) + raise + + def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]: if repo_cache_file.exists(): log.msg("fetching github repositories from cache") repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text()) else: - log.msg("fetching github repositories from api") - repos = paginated_github_request( - "https://api.github.com/user/repos?per_page=100", - github_token, - ) - repo_cache_file.write_text(json.dumps(repos, indent=2)) - + refresh_projects(github_token, repo_cache_file) return [GithubProject(repo) for repo in repos]