From 74fb30f6ffb28e8cb3c73def1d4e4c322b7d3fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 12 Oct 2023 15:59:26 +0200 Subject: [PATCH 1/3] also install webhooks secrets --- buildbot_nix/buildbot_nix.py | 5 ++++- buildbot_nix/github_projects.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/buildbot_nix/buildbot_nix.py b/buildbot_nix/buildbot_nix.py index a58ce67..d46c7c0 100644 --- a/buildbot_nix/buildbot_nix.py +++ b/buildbot_nix/buildbot_nix.py @@ -654,12 +654,15 @@ class NixConfigurator(ConfiguratorBase): config["projects"] = config.get("projects", []) + webhook_secret = read_secret_file(self.github.webhook_secret_name) + for project in projects: create_project_hook( project.owner, project.repo, self.github.token(), f"{self.url}/change_hook/github", + webhook_secret, ) for project in projects: @@ -707,7 +710,7 @@ class NixConfigurator(ConfiguratorBase): "change_hook_dialects", {} ) config["www"]["change_hook_dialects"]["github"] = { - "secret": read_secret_file(self.github.webhook_secret_name), + "secret": webhook_secret, "strict": True, "token": self.github.token(), "github_property_whitelist": "*", diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 94cad9c..aec4835 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -101,11 +101,11 @@ class GithubProject: return self.data["topics"] -def create_project_hook(owner: str, repo: str, token: str, webhook_url: str) -> None: +def create_project_hook(owner: str, repo: str, token: str, webhook_url: str, webhook_secret) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", token ) - config = dict(url=webhook_url, content_type="json", insecure_ssl="0") + config = dict(url=webhook_url, content_type="json", insecure_ssl="0", secret=webhook_secret) data = dict(name="web", active=True, events=["push", "pull_request"], config=config) headers = { "Authorization": f"Bearer {token}", From e82d3a3d918b53d519614b35315073ebde8125d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 12 Oct 2023 23:01:07 +0200 Subject: [PATCH 2/3] add option to reload github projects fix mypy --- buildbot_nix/buildbot_nix.py | 78 +++++++++++++++++++++++++++++++-- buildbot_nix/github_projects.py | 39 ++++++++++++----- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/buildbot_nix/buildbot_nix.py b/buildbot_nix/buildbot_nix.py index d46c7c0..1b9ca2c 100644 --- a/buildbot_nix/buildbot_nix.py +++ b/buildbot_nix/buildbot_nix.py @@ -3,6 +3,7 @@ import json import multiprocessing import os +import signal import sys import uuid from collections import defaultdict @@ -23,8 +24,10 @@ from github_projects import ( # noqa: E402 GithubProject, create_project_hook, load_projects, + refresh_projects, ) -from twisted.internet import defer +from twisted.internet import defer, threads +from twisted.python.failure import Failure class BuildTrigger(Trigger): @@ -51,10 +54,10 @@ class BuildTrigger(Trigger): **kwargs, ) - def createTriggerProperties(self, props: Any) -> Any: + def createTriggerProperties(self, props: Any) -> Any: # noqa: N802 return props - def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: + def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802 build_props = self.build.getProperties() repo_name = build_props.getProperty( "github.base.repo.full_name", @@ -94,7 +97,7 @@ class BuildTrigger(Trigger): triggered_schedulers.append((sch, props)) return triggered_schedulers - def getCurrentSummary(self) -> dict[str, str]: + def getCurrentSummary(self) -> dict[str, str]: # noqa: N802 """ The original build trigger will the generic builder name `nix-build` in this case, which is not helpful """ @@ -250,6 +253,58 @@ class UpdateBuildOutput(steps.BuildStep): return util.SUCCESS +class ReloadGithubProjects(steps.BuildStep): + name = "reload_github_projects" + + def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None: + self.token = token + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def reload_projects(self) -> None: + refresh_projects(self.token, self.project_cache_file) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.reload_projects) + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + else: + log: Log = yield self.addLog("log") + log.addStderr(f"Failed to reload project list: {self.error_msg}") + return util.FAILURE + + +def reload_github_projects( + worker_names: list[str], + github_token_secret: str, + project_cache_file: Path, +) -> util.BuilderConfig: + """ + Updates the flake an opens a PR for it. + """ + factory = util.BuildFactory() + factory.addStep( + ReloadGithubProjects(github_token_secret, project_cache_file=project_cache_file) + ) + return util.BuilderConfig( + name="reload-github-projects", + workernames=worker_names, + factory=factory, + ) + + def nix_update_flake_config( project: GithubProject, worker_names: list[str], @@ -676,6 +731,21 @@ class NixConfigurator(ConfiguratorBase): self.nix_eval_max_memory_size, ) + # Reload github projects + config["builders"].append( + reload_github_projects( + [worker_names[0]], + self.github.token(), + self.github.project_cache_file, + ) + ) + config["schedulers"].append( + schedulers.ForceScheduler( + name="reload-github-projects", + builderNames=["reload-github-projects"], + buttonName="Update projects", + ) + ) config["services"] = config.get("services", []) config["services"].append( reporters.GitHubStatusPush( diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index aec4835..9f7756d 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,7 +1,9 @@ import http.client import json +import os import urllib.request from pathlib import Path +from tempfile import NamedTemporaryFile from typing import Any from twisted.python import log @@ -33,13 +35,13 @@ def http_request( try: resp = urllib.request.urlopen(req) except urllib.request.HTTPError as e: - body = "" + resp_body = "" try: - body = e.fp.read() + resp_body = e.fp.read().decode("utf-8", "replace") except Exception: pass raise Exception( - f"Request for {method} {url} failed with {e.code} {e.reason}: {body}" + f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" ) from e return HttpResponse(resp) @@ -101,11 +103,15 @@ class GithubProject: return self.data["topics"] -def create_project_hook(owner: str, repo: str, token: str, webhook_url: str, webhook_secret) -> None: +def create_project_hook( + owner: str, repo: str, token: str, webhook_url: str, webhook_secret: str +) -> None: hooks = paginated_github_request( f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", token ) - config = dict(url=webhook_url, content_type="json", insecure_ssl="0", secret=webhook_secret) + config = dict( + url=webhook_url, content_type="json", insecure_ssl="0", secret=webhook_secret + ) data = dict(name="web", active=True, events=["push", "pull_request"], config=config) headers = { "Authorization": f"Bearer {token}", @@ -126,16 +132,25 @@ def create_project_hook(owner: str, repo: str, token: str, webhook_url: str, web ) +def refresh_projects(github_token: str, repo_cache_file: Path) -> None: + repos = paginated_github_request( + "https://api.github.com/user/repos?per_page=100", + github_token, + ) + with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: + try: + f.write(json.dumps(repos)) + f.flush() + os.rename(f.name, repo_cache_file) + except OSError: + os.unlink(f.name) + raise + + def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]: if repo_cache_file.exists(): log.msg("fetching github repositories from cache") repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text()) else: - log.msg("fetching github repositories from api") - repos = paginated_github_request( - "https://api.github.com/user/repos?per_page=100", - github_token, - ) - repo_cache_file.write_text(json.dumps(repos, indent=2)) - + refresh_projects(github_token, repo_cache_file) return [GithubProject(repo) for repo in repos] From 22c64dd2971274b19b8ad8e6c57e423dcca0defd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 12 Oct 2023 23:59:45 +0200 Subject: [PATCH 3/3] add mergify --- .mergify.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 0000000..9a4bd8d --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,16 @@ +queue_rules: + - name: default + merge_conditions: + - check-success=buildbot/nix-eval +defaults: + actions: + queue: + allow_merging_configuration_change: true + method: rebase +pull_request_rules: + - name: merge using the merge queue + conditions: + - base=main + - label~=merge-queue|dependencies + actions: + queue: {}