From 372d7dc70bab01524af7a17f3b3bef84bba3e1a2 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 3 Jul 2024 17:01:08 +0200 Subject: [PATCH] Defer hook creation to project reload, both GitHub and Gitea Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 3 +- buildbot_nix/common.py | 59 +++- buildbot_nix/gitea_projects.py | 216 ++++++----- buildbot_nix/github/installation_token.py | 31 +- buildbot_nix/github_projects.py | 413 ++++++++++++++-------- buildbot_nix/projects.py | 9 - 6 files changed, 476 insertions(+), 255 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 040f263..be75479 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -818,7 +818,7 @@ class NixConfigurator(ConfiguratorBase): backends: dict[str, GitBackend] = {} if self.github is not None: - backends["github"] = GithubBackend(self.github) + backends["github"] = GithubBackend(self.github, self.url) if self.gitea is not None: backends["gitea"] = GiteaBackend(self.gitea) @@ -851,7 +851,6 @@ class NixConfigurator(ConfiguratorBase): eval_lock = util.MasterLock("nix-eval") for project in projects: - project.create_project_hook(project.owner, project.repo, self.url) config_for_project( config, project, diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py index b4d3941..2b9f733 100644 --- a/buildbot_nix/common.py +++ b/buildbot_nix/common.py @@ -2,9 +2,20 @@ import contextlib import http.client import json import urllib.request +from abc import ABC, abstractmethod +from collections.abc import Callable from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from buildbot.process.log import StreamLog +from collections.abc import Generator + +from buildbot.plugins import util +from buildbot.process.buildstep import BuildStep +from twisted.internet import defer, threads +from twisted.python.failure import Failure def slugify_project_name(name: str) -> str: @@ -97,3 +108,49 @@ def atomic_write_file(file: Path, data: str) -> None: except OSError: path.unlink() raise + + +def filter_repos_by_topic( + topic: str | None, repos: list[Any], topics: Callable[[Any], list[str]] +) -> list[Any]: + return list( + filter( + lambda repo: topic is None or topic in topics(repo), + repos, + ) + ) + + +class ThreadDeferredBuildStep(BuildStep, ABC): + def __init__( + self, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + + @abstractmethod + def run_deferred(self) -> None: + pass + + @abstractmethod + def run_post(self) -> Any: + pass + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + d = threads.deferToThread(self.run_deferred) # type: ignore[no-untyped-call] + + self.error_msg = "" + + def error_cb(failure: Failure) -> int: + self.error_msg += failure.getTraceback() + return util.FAILURE + + d.addCallbacks(lambda _: util.SUCCESS, error_cb) + res = yield d + if res == util.SUCCESS: + return self.run_post() + else: + log: StreamLog = yield self.addLog("log") + log.addStderr(f"Failed to reload project list: {self.error_msg}") + return util.FAILURE diff --git a/buildbot_nix/gitea_projects.py b/buildbot_nix/gitea_projects.py index cdcbfa5..f7d72de 100644 --- a/buildbot_nix/gitea_projects.py +++ b/buildbot_nix/gitea_projects.py @@ -1,30 +1,26 @@ import json import os import signal -from collections.abc import Generator from dataclasses import dataclass from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Any +from typing import Any from urllib.parse import urlparse from buildbot.config.builder import BuilderConfig from buildbot.plugins import util -from buildbot.process.buildstep import BuildStep from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase from buildbot_gitea.auth import GiteaAuth # type: ignore[import] from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import] -from twisted.internet import defer, threads +from twisted.logger import Logger from twisted.python import log -from twisted.python.failure import Failure - -if TYPE_CHECKING: - from buildbot.process.log import StreamLog from .common import ( + ThreadDeferredBuildStep, + atomic_write_file, + filter_repos_by_topic, http_request, paginated_github_request, slugify_project_name, @@ -32,6 +28,8 @@ from .common import ( from .projects import GitBackend, GitProject from .secrets import read_secret_file +tlog = Logger() + @dataclass class GiteaConfig: @@ -63,46 +61,6 @@ class GiteaProject(GitProject): self.webhook_secret = webhook_secret self.data = data - def create_project_hook( - self, - owner: str, - repo: str, - webhook_url: str, - ) -> None: - hooks = paginated_github_request( - f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100", - self.config.token(), - ) - config = dict( - url=webhook_url + "change_hook/gitea", - content_type="json", - insecure_ssl="0", - secret=self.webhook_secret, - ) - data = dict( - name="web", - active=True, - events=["push", "pull_request"], - config=config, - type="gitea", - ) - headers = { - "Authorization": f"token {self.config.token()}", - "Accept": "application/json", - "Content-Type": "application/json", - } - for hook in hooks: - if hook["config"]["url"] == webhook_url + "change_hook/gitea": - log.msg(f"hook for {owner}/{repo} already exists") - return - - http_request( - f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks", - method="POST", - headers=headers, - data=data, - ) - def get_project_url(self) -> str: url = urlparse(self.config.instance_url) return f"{url.scheme}://git:%(secret:{self.config.token_secret_name})s@{url.hostname}/{self.name}" @@ -153,6 +111,7 @@ class GiteaProject(GitProject): class GiteaBackend(GitBackend): config: GiteaConfig + webhook_secret: str def __init__(self, config: GiteaConfig) -> None: self.config = config @@ -164,6 +123,11 @@ class GiteaBackend(GitBackend): factory.addStep( ReloadGiteaProjects(self.config, self.config.project_cache_file), ) + factory.addStep( + CreateGiteaProjectHooks( + self.config, self.config.project_cache_file, self.webhook_secret + ), + ) return util.BuilderConfig( name=self.reload_builder_name, workernames=worker_names, @@ -201,21 +165,23 @@ class GiteaBackend(GitBackend): if not self.config.project_cache_file.exists(): return [] - repos: list[dict[str, Any]] = sorted( - json.loads(self.config.project_cache_file.read_text()), - key=lambda x: x["full_name"], + repos: list[dict[str, Any]] = filter_repos_by_topic( + self.config.topic, + sorted( + json.loads(self.config.project_cache_file.read_text()), + key=lambda x: x["full_name"], + ), + lambda repo: repo["topics"], ) - return list( - filter( - lambda project: self.config.topic is not None - and self.config.topic in project.topics, - [ - GiteaProject(self.config, self.webhook_secret, repo) - for repo in repos - ], - ) + repo_names: list[str] = [ + repo["owner"]["login"] + "/" + repo["name"] for repo in repos + ] + tlog.info( + f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]" ) + return [GiteaProject(self.config, self.webhook_secret, repo) for repo in repos] + def are_projects_cached(self) -> bool: return self.config.project_cache_file.exists() @@ -236,43 +202,111 @@ class GiteaBackend(GitBackend): return "gitea" -class ReloadGiteaProjects(BuildStep): - name = "reload_gitea_projects" +def create_repo_hook( + token: str, webhook_secret: str, owner: str, repo: str, webhook_url: str +) -> None: + hooks = paginated_github_request( + f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100", + token, + ) + config = dict( + url=webhook_url + "change_hook/gitea", + content_type="json", + insecure_ssl="0", + secret=webhook_secret, + ) + data = dict( + name="web", + active=True, + events=["push", "pull_request"], + config=config, + type="gitea", + ) + headers = { + "Authorization": f"token {token}", + "Accept": "application/json", + "Content-Type": "application/json", + } + for hook in hooks: + if hook["config"]["url"] == webhook_url + "change_hook/gitea": + log.msg(f"hook for {owner}/{repo} already exists") + return + + log.msg(f"creating hook for {owner}/{repo}") + http_request( + f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks", + method="POST", + headers=headers, + data=data, + ) + + +class CreateGiteaProjectHooks(ThreadDeferredBuildStep): + name = "create_gitea_project_hooks" + config: GiteaConfig + project_cache_file: Path + webhook_secret: str def __init__( - self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any + self, + config: GiteaConfig, + project_cache_file: Path, + webhook_secret: str, + **kwargs: Any, + ) -> None: + self.config = config + self.project_cache_file = project_cache_file + self.webhook_secret = webhook_secret + super().__init__(**kwargs) + + def run_deferred(self) -> None: + repos = json.loads(self.project_cache_file.read_text()) + + for repo in repos: + create_repo_hook( + self.config.token(), + self.webhook_secret, + repo["owner"]["login"], + repo["name"], + self.config.instance_url, + ) + + def run_post(self) -> Any: + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + + +class ReloadGiteaProjects(ThreadDeferredBuildStep): + name = "reload_gitea_projects" + + config: GiteaConfig + project_cache_file: Path + + def __init__( + self, + config: GiteaConfig, + project_cache_file: Path, + **kwargs: Any, ) -> None: self.config = config self.project_cache_file = project_cache_file super().__init__(**kwargs) - def reload_projects(self) -> None: - refresh_projects(self.config, self.project_cache_file) + def run_deferred(self) -> None: + repos = filter_repos_by_topic( + self.config.topic, + refresh_projects(self.config, self.project_cache_file), + lambda repo: repo["topics"], + ) - @defer.inlineCallbacks - def run(self) -> Generator[Any, object, Any]: - d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] + atomic_write_file(self.project_cache_file, json.dumps(repos)) - self.error_msg = "" - - def error_cb(failure: Failure) -> int: - self.error_msg += failure.getTraceback() - return util.FAILURE - - d.addCallbacks(lambda _: util.SUCCESS, error_cb) - res = yield d - if res == util.SUCCESS: - # reload the buildbot config - os.kill(os.getpid(), signal.SIGHUP) - return util.SUCCESS - else: - log: StreamLog = yield self.addLog("log") - log.addStderr(f"Failed to reload project list: {self.error_msg}") - return util.FAILURE + def run_post(self) -> Any: + return util.SUCCESS -def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None: +def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]: repos = [] for repo in paginated_github_request( @@ -296,12 +330,4 @@ def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None: except OSError: pass - with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: - path = Path(f.name) - try: - f.write(json.dumps(repos)) - f.flush() - path.rename(repo_cache_file) - except OSError: - path.unlink() - raise + return repos diff --git a/buildbot_nix/github/installation_token.py b/buildbot_nix/github/installation_token.py index da6f9f9..bd0f080 100644 --- a/buildbot_nix/github/installation_token.py +++ b/buildbot_nix/github/installation_token.py @@ -1,7 +1,7 @@ import json from datetime import UTC, datetime, timedelta from pathlib import Path -from typing import Any +from typing import Any, TypedDict from buildbot_nix.common import ( HttpResponse, @@ -13,6 +13,11 @@ from .jwt_token import JWTToken from .repo_token import RepoToken +class InstallationTokenJSON(TypedDict): + expiration: str + token: str + + class InstallationToken(RepoToken): GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60) @@ -66,6 +71,30 @@ class InstallationToken(RepoToken): else: self.token, self.expiration = installation_token + @staticmethod + def new( + jwt_token: JWTToken, installation_id: int, installations_token_map_name: Path + ) -> "InstallationToken": + return InstallationToken( + jwt_token, installation_id, installations_token_map_name + ) + + @staticmethod + def from_json( + jwt_token: JWTToken, + installation_id: int, + installations_token_map_name: Path, + json: InstallationTokenJSON, + ) -> "InstallationToken": + token: str = json["token"] + expiration: datetime = datetime.fromisoformat(json["expiration"]) + return InstallationToken( + jwt_token, + installation_id, + installations_token_map_name, + (token, expiration), + ) + def get(self) -> str: self.verify() return self.token diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index eaf629f..c1da85f 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -2,13 +2,11 @@ import json import os import signal from abc import ABC, abstractmethod -from collections.abc import Callable, Generator +from collections.abc import Callable from dataclasses import dataclass -from datetime import ( - datetime, -) +from itertools import starmap from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.plugins import util @@ -20,16 +18,13 @@ from buildbot.secrets.providers.base import SecretProviderBase from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth -from twisted.internet import defer, threads from twisted.logger import Logger from twisted.python import log -from twisted.python.failure import Failure - -if TYPE_CHECKING: - from buildbot.process.log import StreamLog from .common import ( + ThreadDeferredBuildStep, atomic_write_file, + filter_repos_by_topic, http_request, paginated_github_request, slugify_project_name, @@ -57,13 +52,71 @@ def get_installations(jwt_token: JWTToken) -> list[int]: return [installation["id"] for installation in installations] -class ReloadGithubInstallations(BuildStep): +class CreateGitHubInstallationHooks(ThreadDeferredBuildStep): + name = "create_github_installation_hooks" + + jwt_token: JWTToken + project_cache_file: Path + installation_token_map_name: Path + webhook_secret: str + webhook_url: str + topic: str | None + + def __init__( + self, + jwt_token: JWTToken, + project_cache_file: Path, + installation_token_map_name: Path, + webhook_secret: str, + webhook_url: str, + topic: str | None, + **kwargs: Any, + ) -> None: + self.jwt_token = jwt_token + self.project_cache_file = project_cache_file + self.installation_token_map_name = installation_token_map_name + self.webhook_secret = webhook_secret + self.webhook_url = webhook_url + self.topic = topic + super().__init__(**kwargs) + + def run_deferred(self) -> None: + repos = json.loads(self.project_cache_file.read_text()) + installation_token_map: dict[int, InstallationToken] = dict( + starmap( + lambda k, v: ( + int(k), + InstallationToken.from_json( + self.jwt_token, int(k), self.installation_token_map_name, v + ), + ), + json.loads(self.installation_token_map_name.read_text()).items(), + ) + ) + + for repo in repos: + create_project_hook( + installation_token_map[repo["installation_id"]], + self.webhook_secret, + repo["owner"]["login"], + repo["name"], + self.webhook_url, + ) + + def run_post(self) -> Any: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + + +class ReloadGithubInstallations(ThreadDeferredBuildStep): name = "reload_github_projects" jwt_token: JWTToken project_cache_file: Path installation_token_map_name: Path project_id_map_name: Path + topic: str | None def __init__( self, @@ -71,15 +124,17 @@ class ReloadGithubInstallations(BuildStep): project_cache_file: Path, installation_token_map_name: Path, project_id_map_name: Path, + topic: str | None, **kwargs: Any, ) -> None: self.jwt_token = jwt_token self.installation_token_map_name = installation_token_map_name self.project_id_map_name = project_id_map_name self.project_cache_file = project_cache_file + self.topic = topic super().__init__(**kwargs) - def reload_projects(self) -> None: + def run_deferred(self) -> None: installation_token_map = GithubBackend.create_missing_installations( self.jwt_token, self.installation_token_map_name, @@ -96,89 +151,110 @@ class ReloadGithubInstallations(BuildStep): repos = [] for k, v in installation_token_map.items(): - new_repos = refresh_projects( - v.get(), - self.project_cache_file, - clear=True, - api_endpoint="/installation/repositories", - subkey="repositories", - require_admin=False, + new_repos = filter_repos_by_topic( + self.topic, + refresh_projects( + v.get(), + self.project_cache_file, + clear=True, + api_endpoint="/installation/repositories", + subkey="repositories", + require_admin=False, + ), + lambda repo: repo["topics"], ) for repo in new_repos: repo["installation_id"] = k - repos.extend(new_repos) - - for repo in new_repos: project_id_map[repo["full_name"]] = k + repos.extend(new_repos) + atomic_write_file(self.project_cache_file, json.dumps(repos)) atomic_write_file(self.project_id_map_name, json.dumps(project_id_map)) tlog.info( - f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token." + f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation tokens." ) - @defer.inlineCallbacks - def run(self) -> Generator[Any, object, Any]: - d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] - - self.error_msg = "" - - def error_cb(failure: Failure) -> int: - self.error_msg += failure.getTraceback() - return util.FAILURE - - d.addCallbacks(lambda _: util.SUCCESS, error_cb) - res = yield d - if res == util.SUCCESS: - # reload the buildbot config - os.kill(os.getpid(), signal.SIGHUP) - return util.SUCCESS - else: - log: StreamLog = yield self.addLog("log") - log.addStderr(f"Failed to reload project list: {self.error_msg}") - return util.FAILURE + def run_post(self) -> Any: + return util.SUCCESS -class ReloadGithubProjects(BuildStep): - name = "reload_github_projects" +class CreateGitHubProjectHooks(ThreadDeferredBuildStep): + name = "create_github_project_hooks" + + token: RepoToken + project_cache_file: Path + webhook_secret: str + webhook_url: str + topic: str | None def __init__( - self, token: RepoToken, project_cache_file: Path, **kwargs: Any + self, + token: RepoToken, + project_cache_file: Path, + webhook_secret: str, + webhook_url: str, + topic: str | None, + **kwargs: Any, ) -> None: self.token = token self.project_cache_file = project_cache_file + self.webhook_secret = webhook_secret + self.webhook_url = webhook_url + self.topic = topic super().__init__(**kwargs) - def reload_projects(self) -> None: - repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file) + def run_deferred(self) -> None: + repos = json.loads(self.project_cache_file.read_text()) - log.msg(repos) + for repo in repos: + create_project_hook( + self.token, + self.webhook_secret, + repo["owner"]["login"], + repo["name"], + self.webhook_url, + ) + + def run_post(self) -> Any: + # reload the buildbot config + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + + +class ReloadGithubProjects(ThreadDeferredBuildStep): + name = "reload_github_projects" + + token: RepoToken + project_cache_file: Path + topic: str | None + + def __init__( + self, + token: RepoToken, + project_cache_file: Path, + topic: str | None, + **kwargs: Any, + ) -> None: + self.token = token + self.project_cache_file = project_cache_file + self.topic = topic + super().__init__(**kwargs) + + def run_deferred(self) -> None: + repos: list[Any] = filter_repos_by_topic( + self.topic, + refresh_projects(self.token.get(), self.project_cache_file), + lambda repo: repo["topics"], + ) atomic_write_file(self.project_cache_file, json.dumps(repos)) - @defer.inlineCallbacks - def run(self) -> Generator[Any, object, Any]: - d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call] - - self.error_msg = "" - - def error_cb(failure: Failure) -> int: - self.error_msg += failure.getTraceback() - return util.FAILURE - - d.addCallbacks(lambda _: util.SUCCESS, error_cb) - res = yield d - if res == util.SUCCESS: - # reload the buildbot config - os.kill(os.getpid(), signal.SIGHUP) - return util.SUCCESS - else: - log: StreamLog = yield self.addLog("log") - log.addStderr(f"Failed to reload project list: {self.error_msg}") - return util.FAILURE + def run_post(self) -> Any: + return util.SUCCESS class GitHubAppStatusPush(GitHubStatusPush): @@ -307,7 +383,13 @@ class GithubAuthBackend(ABC): pass @abstractmethod - def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: + def create_reload_builder_steps( + self, + project_cache_file: Path, + webhook_secret: str, + webhook_url: str, + topic: str | None, + ) -> list[BuildStep]: pass @@ -338,10 +420,27 @@ class GithubLegacyAuthBackend(GithubAuthBackend): context=Interpolate("buildbot/%(prop:status_name)s"), ) - def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: - return ReloadGithubProjects( - token=self.token, project_cache_file=project_cache_file - ) + def create_reload_builder_steps( + self, + project_cache_file: Path, + webhook_secret: str, + webhook_url: str, + topic: str | None, + ) -> list[BuildStep]: + return [ + ReloadGithubProjects( + token=self.token, + project_cache_file=project_cache_file, + topic=topic, + ), + CreateGitHubProjectHooks( + token=self.token, + project_cache_file=project_cache_file, + webhook_secret=webhook_secret, + webhook_url=webhook_url, + topic=topic, + ), + ] class GitHubLegacySecretService(SecretProviderBase): @@ -406,13 +505,30 @@ class GithubAppAuthBackend(GithubAuthBackend): context=Interpolate("buildbot/%(prop:status_name)s"), ) - def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: - return ReloadGithubInstallations( - self.jwt_token, - project_cache_file, - self.auth_type.app_installation_token_map_name, - self.auth_type.app_project_id_map_name, - ) + def create_reload_builder_steps( + self, + project_cache_file: Path, + webhook_secret: str, + webhook_url: str, + topic: str | None, + ) -> list[BuildStep]: + return [ + ReloadGithubInstallations( + self.jwt_token, + project_cache_file, + self.auth_type.app_installation_token_map_name, + self.auth_type.app_project_id_map_name, + topic, + ), + CreateGitHubInstallationHooks( + self.jwt_token, + project_cache_file, + self.auth_type.app_installation_token_map_name, + webhook_secret=webhook_secret, + webhook_url=webhook_url, + topic=topic, + ), + ] class GitHubAppSecretService(SecretProviderBase): @@ -453,12 +569,14 @@ class GithubConfig: class GithubBackend(GitBackend): config: GithubConfig webhook_secret: str + webhook_url: str auth_backend: GithubAuthBackend - def __init__(self, config: GithubConfig) -> None: + def __init__(self, config: GithubConfig, webhook_url: str) -> None: self.config = config self.webhook_secret = read_secret_file(self.config.webhook_secret_name) + self.webhook_url = webhook_url if isinstance(self.config.auth_type, AuthTypeLegacy): self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type) @@ -480,13 +598,8 @@ class GithubBackend(GitBackend): installations_map: dict[int, InstallationToken] = {} for iid, installation in initial_installations_map.items(): - token: str = installation["token"] - expiration: datetime = datetime.fromisoformat(installation["expiration"]) - installations_map[int(iid)] = InstallationToken( - jwt_token, - int(iid), - installations_token_map_name, - installation_token=(token, expiration), + installations_map[int(iid)] = InstallationToken.from_json( + jwt_token, int(iid), installations_token_map_name, installation ) return installations_map @@ -499,7 +612,7 @@ class GithubBackend(GitBackend): installations: list[int], ) -> dict[int, InstallationToken]: for installation in set(installations) - installations_map.keys(): - installations_map[installation] = InstallationToken( + installations_map[installation] = InstallationToken.new( jwt_token, installation, installations_token_map_name, @@ -510,9 +623,15 @@ class GithubBackend(GitBackend): def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" factory = util.BuildFactory() - factory.addStep( - self.auth_backend.create_reload_builder_step(self.config.project_cache_file) + steps = self.auth_backend.create_reload_builder_steps( + self.config.project_cache_file, + self.webhook_secret, + self.webhook_url, + self.config.topic, ) + for step in steps: + factory.addStep(step) + return util.BuilderConfig( name=self.reload_builder_name, workernames=worker_names, @@ -563,9 +682,13 @@ class GithubBackend(GitBackend): if not self.config.project_cache_file.exists(): return [] - repos: list[dict[str, Any]] = sorted( - json.loads(self.config.project_cache_file.read_text()), - key=lambda x: x["full_name"], + repos: list[dict[str, Any]] = filter_repos_by_topic( + self.config.topic, + sorted( + json.loads(self.config.project_cache_file.read_text()), + key=lambda x: x["full_name"], + ), + lambda repo: repo["topics"], ) if isinstance(self.auth_backend, GithubAppAuthBackend): @@ -580,22 +703,22 @@ class GithubBackend(GitBackend): tlog.info(f"\tDropping repo {dropped_repo['full_name']}") repos = list(filter(lambda repo: "installation_id" in repo, repos)) - tlog.info(f"Loading {len(repos)} cached repositories.") - return list( - filter( - lambda project: self.config.topic is not None - and self.config.topic in project.topics, - [ - GithubProject( - self.auth_backend.get_repo_token(repo), - self.config, - self.webhook_secret, - repo, - ) - for repo in repos - ], - ) + repo_names: list[str] = [ + repo["owner"]["login"] + "/" + repo["name"] for repo in repos + ] + + tlog.info( + f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]" ) + return [ + GithubProject( + self.auth_backend.get_repo_token(repo), + self.config, + self.webhook_secret, + repo, + ) + for repo in repos + ] def are_projects_cached(self) -> bool: if not self.config.project_cache_file.exists(): @@ -632,6 +755,39 @@ class GithubBackend(GitBackend): return "github" +def create_project_hook( + token: RepoToken, webhook_secret: str, owner: str, repo: str, webhook_url: str +) -> None: + hooks = paginated_github_request( + f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", + token.get(), + ) + config = dict( + url=webhook_url + "change_hook/github", + content_type="json", + insecure_ssl="0", + secret=webhook_secret, + ) + data = dict(name="web", active=True, events=["push", "pull_request"], config=config) + headers = { + "Authorization": f"Bearer {token.get()}", + "Accept": "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + } + for hook in hooks: + if hook["config"]["url"] == webhook_url + "change_hook/github": + log.msg(f"hook for {owner}/{repo} already exists") + return + + http_request( + f"https://api.github.com/repos/{owner}/{repo}/hooks", + method="POST", + headers=headers, + data=data, + ) + + class GithubProject(GitProject): config: GithubConfig webhook_secret: str @@ -650,43 +806,6 @@ class GithubProject(GitProject): self.webhook_secret = webhook_secret self.data = data - def create_project_hook( - self, - owner: str, - repo: str, - webhook_url: str, - ) -> None: - hooks = paginated_github_request( - f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", - self.token.get(), - ) - config = dict( - url=webhook_url + "change_hook/github", - content_type="json", - insecure_ssl="0", - secret=self.webhook_secret, - ) - data = dict( - name="web", active=True, events=["push", "pull_request"], config=config - ) - headers = { - "Authorization": f"Bearer {self.token.get()}", - "Accept": "application/vnd.github+json", - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - } - for hook in hooks: - if hook["config"]["url"] == webhook_url + "change_hook/github": - log.msg(f"hook for {owner}/{repo} already exists") - return - - http_request( - f"https://api.github.com/repos/{owner}/{repo}/hooks", - method="POST", - headers=headers, - data=data, - ) - def get_project_url(self) -> str: return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}" diff --git a/buildbot_nix/projects.py b/buildbot_nix/projects.py index 23ef1db..70a9092 100644 --- a/buildbot_nix/projects.py +++ b/buildbot_nix/projects.py @@ -62,15 +62,6 @@ class GitBackend(ABC): class GitProject(ABC): - @abstractmethod - def create_project_hook( - self, - owner: str, - repo: str, - webhook_url: str, - ) -> None: - pass - @abstractmethod def get_project_url(self) -> str: pass