Merge pull request #203 from MagicRB/defer-hook-creation

Defer hook creation to project reload, both GitHub and Gitea

buildbot_nix/__init__.py

@@ -818,7 +818,7 @@ class NixConfigurator(ConfiguratorBase):
         backends: dict[str, GitBackend] = {}
 
         if self.github is not None:
-            backends["github"] = GithubBackend(self.github)
+            backends["github"] = GithubBackend(self.github, self.url)
 
         if self.gitea is not None:
             backends["gitea"] = GiteaBackend(self.gitea)
@@ -851,7 +851,6 @@ class NixConfigurator(ConfiguratorBase):
         eval_lock = util.MasterLock("nix-eval")
 
         for project in projects:
-            project.create_project_hook(project.owner, project.repo, self.url)
             config_for_project(
                 config,
                 project,

buildbot_nix/common.py

@@ -2,9 +2,20 @@ import contextlib
 import http.client
 import json
 import urllib.request
+from abc import ABC, abstractmethod
+from collections.abc import Callable
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from buildbot.process.log import StreamLog
+from collections.abc import Generator
+
+from buildbot.plugins import util
+from buildbot.process.buildstep import BuildStep
+from twisted.internet import defer, threads
+from twisted.python.failure import Failure
 
 
 def slugify_project_name(name: str) -> str:
@@ -97,3 +108,49 @@ def atomic_write_file(file: Path, data: str) -> None:
         except OSError:
             path.unlink()
             raise
+
+
+def filter_repos_by_topic(
+    topic: str | None, repos: list[Any], topics: Callable[[Any], list[str]]
+) -> list[Any]:
+    return list(
+        filter(
+            lambda repo: topic is None or topic in topics(repo),
+            repos,
+        )
+    )
+
+
+class ThreadDeferredBuildStep(BuildStep, ABC):
+    def __init__(
+        self,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+    @abstractmethod
+    def run_deferred(self) -> None:
+        pass
+
+    @abstractmethod
+    def run_post(self) -> Any:
+        pass
+
+    @defer.inlineCallbacks
+    def run(self) -> Generator[Any, object, Any]:
+        d = threads.deferToThread(self.run_deferred)  # type: ignore[no-untyped-call]
+
+        self.error_msg = ""
+
+        def error_cb(failure: Failure) -> int:
+            self.error_msg += failure.getTraceback()
+            return util.FAILURE
+
+        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
+        res = yield d
+        if res == util.SUCCESS:
+            return self.run_post()
+        else:
+            log: StreamLog = yield self.addLog("log")
+            log.addStderr(f"Failed to reload project list: {self.error_msg}")
+            return util.FAILURE

buildbot_nix/gitea_projects.py

@@ -1,30 +1,26 @@
 import json
 import os
 import signal
-from collections.abc import Generator
 from dataclasses import dataclass
 from pathlib import Path
-from tempfile import NamedTemporaryFile
+from typing import Any
-from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
 from buildbot.config.builder import BuilderConfig
 from buildbot.plugins import util
-from buildbot.process.buildstep import BuildStep
 from buildbot.process.properties import Interpolate
 from buildbot.reporters.base import ReporterBase
 from buildbot.www.auth import AuthBase
 from buildbot.www.avatar import AvatarBase
 from buildbot_gitea.auth import GiteaAuth  # type: ignore[import]
 from buildbot_gitea.reporter import GiteaStatusPush  # type: ignore[import]
-from twisted.internet import defer, threads
+from twisted.logger import Logger
 from twisted.python import log
-from twisted.python.failure import Failure
-
-if TYPE_CHECKING:
-    from buildbot.process.log import StreamLog
 
 from .common import (
+    ThreadDeferredBuildStep,
+    atomic_write_file,
+    filter_repos_by_topic,
     http_request,
     paginated_github_request,
     slugify_project_name,
@@ -32,6 +28,8 @@ from .common import (
 from .projects import GitBackend, GitProject
 from .secrets import read_secret_file
 
+tlog = Logger()
+
 
 @dataclass
 class GiteaConfig:
@@ -63,46 +61,6 @@ class GiteaProject(GitProject):
         self.webhook_secret = webhook_secret
         self.data = data
 
-    def create_project_hook(
-        self,
-        owner: str,
-        repo: str,
-        webhook_url: str,
-    ) -> None:
-        hooks = paginated_github_request(
-            f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
-            self.config.token(),
-        )
-        config = dict(
-            url=webhook_url + "change_hook/gitea",
-            content_type="json",
-            insecure_ssl="0",
-            secret=self.webhook_secret,
-        )
-        data = dict(
-            name="web",
-            active=True,
-            events=["push", "pull_request"],
-            config=config,
-            type="gitea",
-        )
-        headers = {
-            "Authorization": f"token {self.config.token()}",
-            "Accept": "application/json",
-            "Content-Type": "application/json",
-        }
-        for hook in hooks:
-            if hook["config"]["url"] == webhook_url + "change_hook/gitea":
-                log.msg(f"hook for {owner}/{repo} already exists")
-                return
-
-        http_request(
-            f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks",
-            method="POST",
-            headers=headers,
-            data=data,
-        )
-
     def get_project_url(self) -> str:
         url = urlparse(self.config.instance_url)
         return f"{url.scheme}://git:%(secret:{self.config.token_secret_name})s@{url.hostname}/{self.name}"
@@ -153,6 +111,7 @@ class GiteaProject(GitProject):
 
 class GiteaBackend(GitBackend):
     config: GiteaConfig
+    webhook_secret: str
 
     def __init__(self, config: GiteaConfig) -> None:
         self.config = config
@@ -164,6 +123,11 @@ class GiteaBackend(GitBackend):
         factory.addStep(
             ReloadGiteaProjects(self.config, self.config.project_cache_file),
         )
+        factory.addStep(
+            CreateGiteaProjectHooks(
+                self.config, self.config.project_cache_file, self.webhook_secret
+            ),
+        )
         return util.BuilderConfig(
             name=self.reload_builder_name,
             workernames=worker_names,
@@ -201,21 +165,23 @@ class GiteaBackend(GitBackend):
         if not self.config.project_cache_file.exists():
             return []
 
-        repos: list[dict[str, Any]] = sorted(
+        repos: list[dict[str, Any]] = filter_repos_by_topic(
-            json.loads(self.config.project_cache_file.read_text()),
+            self.config.topic,
-            key=lambda x: x["full_name"],
+            sorted(
+                json.loads(self.config.project_cache_file.read_text()),
+                key=lambda x: x["full_name"],
+            ),
+            lambda repo: repo["topics"],
         )
-        return list(
+        repo_names: list[str] = [
-            filter(
+            repo["owner"]["login"] + "/" + repo["name"] for repo in repos
-                lambda project: self.config.topic is not None
+        ]
-                and self.config.topic in project.topics,
+        tlog.info(
-                [
+            f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
-                    GiteaProject(self.config, self.webhook_secret, repo)
-                    for repo in repos
-                ],
-            )
         )
 
+        return [GiteaProject(self.config, self.webhook_secret, repo) for repo in repos]
+
     def are_projects_cached(self) -> bool:
         return self.config.project_cache_file.exists()
 
@@ -236,43 +202,111 @@ class GiteaBackend(GitBackend):
         return "gitea"
 
 
-class ReloadGiteaProjects(BuildStep):
+def create_repo_hook(
-    name = "reload_gitea_projects"
+    token: str, webhook_secret: str, owner: str, repo: str, webhook_url: str
+) -> None:
+    hooks = paginated_github_request(
+        f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
+        token,
+    )
+    config = dict(
+        url=webhook_url + "change_hook/gitea",
+        content_type="json",
+        insecure_ssl="0",
+        secret=webhook_secret,
+    )
+    data = dict(
+        name="web",
+        active=True,
+        events=["push", "pull_request"],
+        config=config,
+        type="gitea",
+    )
+    headers = {
+        "Authorization": f"token {token}",
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+    }
+    for hook in hooks:
+        if hook["config"]["url"] == webhook_url + "change_hook/gitea":
+            log.msg(f"hook for {owner}/{repo} already exists")
+            return
+
+    log.msg(f"creating hook for {owner}/{repo}")
+    http_request(
+        f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks",
+        method="POST",
+        headers=headers,
+        data=data,
+    )
+
+
+class CreateGiteaProjectHooks(ThreadDeferredBuildStep):
+    name = "create_gitea_project_hooks"
+
     config: GiteaConfig
+    project_cache_file: Path
+    webhook_secret: str
 
     def __init__(
-        self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any
+        self,
+        config: GiteaConfig,
+        project_cache_file: Path,
+        webhook_secret: str,
+        **kwargs: Any,
+    ) -> None:
+        self.config = config
+        self.project_cache_file = project_cache_file
+        self.webhook_secret = webhook_secret
+        super().__init__(**kwargs)
+
+    def run_deferred(self) -> None:
+        repos = json.loads(self.project_cache_file.read_text())
+
+        for repo in repos:
+            create_repo_hook(
+                self.config.token(),
+                self.webhook_secret,
+                repo["owner"]["login"],
+                repo["name"],
+                self.config.instance_url,
+            )
+
+    def run_post(self) -> Any:
+        os.kill(os.getpid(), signal.SIGHUP)
+        return util.SUCCESS
+
+
+class ReloadGiteaProjects(ThreadDeferredBuildStep):
+    name = "reload_gitea_projects"
+
+    config: GiteaConfig
+    project_cache_file: Path
+
+    def __init__(
+        self,
+        config: GiteaConfig,
+        project_cache_file: Path,
+        **kwargs: Any,
     ) -> None:
         self.config = config
         self.project_cache_file = project_cache_file
         super().__init__(**kwargs)
 
-    def reload_projects(self) -> None:
+    def run_deferred(self) -> None:
-        refresh_projects(self.config, self.project_cache_file)
+        repos = filter_repos_by_topic(
+            self.config.topic,
+            refresh_projects(self.config, self.project_cache_file),
+            lambda repo: repo["topics"],
+        )
 
-    @defer.inlineCallbacks
+        atomic_write_file(self.project_cache_file, json.dumps(repos))
-    def run(self) -> Generator[Any, object, Any]:
-        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
 
-        self.error_msg = ""
+    def run_post(self) -> Any:
-
+        return util.SUCCESS
-        def error_cb(failure: Failure) -> int:
-            self.error_msg += failure.getTraceback()
-            return util.FAILURE
-
-        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
-        res = yield d
-        if res == util.SUCCESS:
-            # reload the buildbot config
-            os.kill(os.getpid(), signal.SIGHUP)
-            return util.SUCCESS
-        else:
-            log: StreamLog = yield self.addLog("log")
-            log.addStderr(f"Failed to reload project list: {self.error_msg}")
-            return util.FAILURE
 
 
-def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
+def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]:
     repos = []
 
     for repo in paginated_github_request(
@@ -296,12 +330,4 @@ def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
             except OSError:
                 pass
 
-    with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
+    return repos
-        path = Path(f.name)
-        try:
-            f.write(json.dumps(repos))
-            f.flush()
-            path.rename(repo_cache_file)
-        except OSError:
-            path.unlink()
-            raise

buildbot_nix/github/installation_token.py

@@ -1,7 +1,7 @@
 import json
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
-from typing import Any
+from typing import Any, TypedDict
 
 from buildbot_nix.common import (
     HttpResponse,
@@ -13,6 +13,11 @@ from .jwt_token import JWTToken
 from .repo_token import RepoToken
 
 
+class InstallationTokenJSON(TypedDict):
+    expiration: str
+    token: str
+
+
 class InstallationToken(RepoToken):
     GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
 
@@ -66,6 +71,30 @@ class InstallationToken(RepoToken):
         else:
             self.token, self.expiration = installation_token
 
+    @staticmethod
+    def new(
+        jwt_token: JWTToken, installation_id: int, installations_token_map_name: Path
+    ) -> "InstallationToken":
+        return InstallationToken(
+            jwt_token, installation_id, installations_token_map_name
+        )
+
+    @staticmethod
+    def from_json(
+        jwt_token: JWTToken,
+        installation_id: int,
+        installations_token_map_name: Path,
+        json: InstallationTokenJSON,
+    ) -> "InstallationToken":
+        token: str = json["token"]
+        expiration: datetime = datetime.fromisoformat(json["expiration"])
+        return InstallationToken(
+            jwt_token,
+            installation_id,
+            installations_token_map_name,
+            (token, expiration),
+        )
+
     def get(self) -> str:
         self.verify()
         return self.token

buildbot_nix/github_projects.py

@@ -2,13 +2,11 @@ import json
 import os
 import signal
 from abc import ABC, abstractmethod
-from collections.abc import Callable, Generator
+from collections.abc import Callable
 from dataclasses import dataclass
-from datetime import (
+from itertools import starmap
-    datetime,
-)
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from buildbot.config.builder import BuilderConfig
 from buildbot.plugins import util
@@ -20,16 +18,13 @@ from buildbot.secrets.providers.base import SecretProviderBase
 from buildbot.www.auth import AuthBase
 from buildbot.www.avatar import AvatarBase, AvatarGitHub
 from buildbot.www.oauth2 import GitHubAuth
-from twisted.internet import defer, threads
 from twisted.logger import Logger
 from twisted.python import log
-from twisted.python.failure import Failure
-
-if TYPE_CHECKING:
-    from buildbot.process.log import StreamLog
 
 from .common import (
+    ThreadDeferredBuildStep,
     atomic_write_file,
+    filter_repos_by_topic,
     http_request,
     paginated_github_request,
     slugify_project_name,
@@ -57,13 +52,71 @@ def get_installations(jwt_token: JWTToken) -> list[int]:
     return [installation["id"] for installation in installations]
 
 
-class ReloadGithubInstallations(BuildStep):
+class CreateGitHubInstallationHooks(ThreadDeferredBuildStep):
+    name = "create_github_installation_hooks"
+
+    jwt_token: JWTToken
+    project_cache_file: Path
+    installation_token_map_name: Path
+    webhook_secret: str
+    webhook_url: str
+    topic: str | None
+
+    def __init__(
+        self,
+        jwt_token: JWTToken,
+        project_cache_file: Path,
+        installation_token_map_name: Path,
+        webhook_secret: str,
+        webhook_url: str,
+        topic: str | None,
+        **kwargs: Any,
+    ) -> None:
+        self.jwt_token = jwt_token
+        self.project_cache_file = project_cache_file
+        self.installation_token_map_name = installation_token_map_name
+        self.webhook_secret = webhook_secret
+        self.webhook_url = webhook_url
+        self.topic = topic
+        super().__init__(**kwargs)
+
+    def run_deferred(self) -> None:
+        repos = json.loads(self.project_cache_file.read_text())
+        installation_token_map: dict[int, InstallationToken] = dict(
+            starmap(
+                lambda k, v: (
+                    int(k),
+                    InstallationToken.from_json(
+                        self.jwt_token, int(k), self.installation_token_map_name, v
+                    ),
+                ),
+                json.loads(self.installation_token_map_name.read_text()).items(),
+            )
+        )
+
+        for repo in repos:
+            create_project_hook(
+                installation_token_map[repo["installation_id"]],
+                self.webhook_secret,
+                repo["owner"]["login"],
+                repo["name"],
+                self.webhook_url,
+            )
+
+    def run_post(self) -> Any:
+        # reload the buildbot config
+        os.kill(os.getpid(), signal.SIGHUP)
+        return util.SUCCESS
+
+
+class ReloadGithubInstallations(ThreadDeferredBuildStep):
     name = "reload_github_projects"
 
     jwt_token: JWTToken
     project_cache_file: Path
     installation_token_map_name: Path
     project_id_map_name: Path
+    topic: str | None
 
     def __init__(
         self,
@@ -71,15 +124,17 @@ class ReloadGithubInstallations(BuildStep):
         project_cache_file: Path,
         installation_token_map_name: Path,
         project_id_map_name: Path,
+        topic: str | None,
         **kwargs: Any,
     ) -> None:
         self.jwt_token = jwt_token
         self.installation_token_map_name = installation_token_map_name
         self.project_id_map_name = project_id_map_name
         self.project_cache_file = project_cache_file
+        self.topic = topic
         super().__init__(**kwargs)
 
-    def reload_projects(self) -> None:
+    def run_deferred(self) -> None:
         installation_token_map = GithubBackend.create_missing_installations(
             self.jwt_token,
             self.installation_token_map_name,
@@ -96,89 +151,110 @@ class ReloadGithubInstallations(BuildStep):
         repos = []
 
         for k, v in installation_token_map.items():
-            new_repos = refresh_projects(
+            new_repos = filter_repos_by_topic(
-                v.get(),
+                self.topic,
-                self.project_cache_file,
+                refresh_projects(
-                clear=True,
+                    v.get(),
-                api_endpoint="/installation/repositories",
+                    self.project_cache_file,
-                subkey="repositories",
+                    clear=True,
-                require_admin=False,
+                    api_endpoint="/installation/repositories",
+                    subkey="repositories",
+                    require_admin=False,
+                ),
+                lambda repo: repo["topics"],
             )
 
             for repo in new_repos:
                 repo["installation_id"] = k
 
-            repos.extend(new_repos)
-
-            for repo in new_repos:
                 project_id_map[repo["full_name"]] = k
 
+            repos.extend(new_repos)
+
         atomic_write_file(self.project_cache_file, json.dumps(repos))
         atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
 
         tlog.info(
-            f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token."
+            f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation tokens."
         )
 
-    @defer.inlineCallbacks
+    def run_post(self) -> Any:
-    def run(self) -> Generator[Any, object, Any]:
+        return util.SUCCESS
-        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
-
-        self.error_msg = ""
-
-        def error_cb(failure: Failure) -> int:
-            self.error_msg += failure.getTraceback()
-            return util.FAILURE
-
-        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
-        res = yield d
-        if res == util.SUCCESS:
-            # reload the buildbot config
-            os.kill(os.getpid(), signal.SIGHUP)
-            return util.SUCCESS
-        else:
-            log: StreamLog = yield self.addLog("log")
-            log.addStderr(f"Failed to reload project list: {self.error_msg}")
-            return util.FAILURE
 
 
-class ReloadGithubProjects(BuildStep):
+class CreateGitHubProjectHooks(ThreadDeferredBuildStep):
-    name = "reload_github_projects"
+    name = "create_github_project_hooks"
+
+    token: RepoToken
+    project_cache_file: Path
+    webhook_secret: str
+    webhook_url: str
+    topic: str | None
 
     def __init__(
-        self, token: RepoToken, project_cache_file: Path, **kwargs: Any
+        self,
+        token: RepoToken,
+        project_cache_file: Path,
+        webhook_secret: str,
+        webhook_url: str,
+        topic: str | None,
+        **kwargs: Any,
     ) -> None:
         self.token = token
         self.project_cache_file = project_cache_file
+        self.webhook_secret = webhook_secret
+        self.webhook_url = webhook_url
+        self.topic = topic
         super().__init__(**kwargs)
 
-    def reload_projects(self) -> None:
+    def run_deferred(self) -> None:
-        repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file)
+        repos = json.loads(self.project_cache_file.read_text())
 
-        log.msg(repos)
+        for repo in repos:
+            create_project_hook(
+                self.token,
+                self.webhook_secret,
+                repo["owner"]["login"],
+                repo["name"],
+                self.webhook_url,
+            )
+
+    def run_post(self) -> Any:
+        # reload the buildbot config
+        os.kill(os.getpid(), signal.SIGHUP)
+        return util.SUCCESS
+
+
+class ReloadGithubProjects(ThreadDeferredBuildStep):
+    name = "reload_github_projects"
+
+    token: RepoToken
+    project_cache_file: Path
+    topic: str | None
+
+    def __init__(
+        self,
+        token: RepoToken,
+        project_cache_file: Path,
+        topic: str | None,
+        **kwargs: Any,
+    ) -> None:
+        self.token = token
+        self.project_cache_file = project_cache_file
+        self.topic = topic
+        super().__init__(**kwargs)
+
+    def run_deferred(self) -> None:
+        repos: list[Any] = filter_repos_by_topic(
+            self.topic,
+            refresh_projects(self.token.get(), self.project_cache_file),
+            lambda repo: repo["topics"],
+        )
 
         atomic_write_file(self.project_cache_file, json.dumps(repos))
 
-    @defer.inlineCallbacks
+    def run_post(self) -> Any:
-    def run(self) -> Generator[Any, object, Any]:
+        return util.SUCCESS
-        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
-
-        self.error_msg = ""
-
-        def error_cb(failure: Failure) -> int:
-            self.error_msg += failure.getTraceback()
-            return util.FAILURE
-
-        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
-        res = yield d
-        if res == util.SUCCESS:
-            # reload the buildbot config
-            os.kill(os.getpid(), signal.SIGHUP)
-            return util.SUCCESS
-        else:
-            log: StreamLog = yield self.addLog("log")
-            log.addStderr(f"Failed to reload project list: {self.error_msg}")
-            return util.FAILURE
 
 
 class GitHubAppStatusPush(GitHubStatusPush):
@@ -307,7 +383,13 @@ class GithubAuthBackend(ABC):
         pass
 
     @abstractmethod
-    def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
+    def create_reload_builder_steps(
+        self,
+        project_cache_file: Path,
+        webhook_secret: str,
+        webhook_url: str,
+        topic: str | None,
+    ) -> list[BuildStep]:
         pass
 
 
@@ -338,10 +420,27 @@ class GithubLegacyAuthBackend(GithubAuthBackend):
             context=Interpolate("buildbot/%(prop:status_name)s"),
         )
 
-    def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
+    def create_reload_builder_steps(
-        return ReloadGithubProjects(
+        self,
-            token=self.token, project_cache_file=project_cache_file
+        project_cache_file: Path,
-        )
+        webhook_secret: str,
+        webhook_url: str,
+        topic: str | None,
+    ) -> list[BuildStep]:
+        return [
+            ReloadGithubProjects(
+                token=self.token,
+                project_cache_file=project_cache_file,
+                topic=topic,
+            ),
+            CreateGitHubProjectHooks(
+                token=self.token,
+                project_cache_file=project_cache_file,
+                webhook_secret=webhook_secret,
+                webhook_url=webhook_url,
+                topic=topic,
+            ),
+        ]
 
 
 class GitHubLegacySecretService(SecretProviderBase):
@@ -406,13 +505,30 @@ class GithubAppAuthBackend(GithubAuthBackend):
             context=Interpolate("buildbot/%(prop:status_name)s"),
         )
 
-    def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
+    def create_reload_builder_steps(
-        return ReloadGithubInstallations(
+        self,
-            self.jwt_token,
+        project_cache_file: Path,
-            project_cache_file,
+        webhook_secret: str,
-            self.auth_type.app_installation_token_map_name,
+        webhook_url: str,
-            self.auth_type.app_project_id_map_name,
+        topic: str | None,
-        )
+    ) -> list[BuildStep]:
+        return [
+            ReloadGithubInstallations(
+                self.jwt_token,
+                project_cache_file,
+                self.auth_type.app_installation_token_map_name,
+                self.auth_type.app_project_id_map_name,
+                topic,
+            ),
+            CreateGitHubInstallationHooks(
+                self.jwt_token,
+                project_cache_file,
+                self.auth_type.app_installation_token_map_name,
+                webhook_secret=webhook_secret,
+                webhook_url=webhook_url,
+                topic=topic,
+            ),
+        ]
 
 
 class GitHubAppSecretService(SecretProviderBase):
@@ -453,12 +569,14 @@ class GithubConfig:
 class GithubBackend(GitBackend):
     config: GithubConfig
     webhook_secret: str
+    webhook_url: str
 
     auth_backend: GithubAuthBackend
 
-    def __init__(self, config: GithubConfig) -> None:
+    def __init__(self, config: GithubConfig, webhook_url: str) -> None:
         self.config = config
         self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
+        self.webhook_url = webhook_url
 
         if isinstance(self.config.auth_type, AuthTypeLegacy):
             self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
@@ -480,13 +598,8 @@ class GithubBackend(GitBackend):
         installations_map: dict[int, InstallationToken] = {}
 
         for iid, installation in initial_installations_map.items():
-            token: str = installation["token"]
+            installations_map[int(iid)] = InstallationToken.from_json(
-            expiration: datetime = datetime.fromisoformat(installation["expiration"])
+                jwt_token, int(iid), installations_token_map_name, installation
-            installations_map[int(iid)] = InstallationToken(
-                jwt_token,
-                int(iid),
-                installations_token_map_name,
-                installation_token=(token, expiration),
             )
 
         return installations_map
@@ -499,7 +612,7 @@ class GithubBackend(GitBackend):
         installations: list[int],
     ) -> dict[int, InstallationToken]:
         for installation in set(installations) - installations_map.keys():
-            installations_map[installation] = InstallationToken(
+            installations_map[installation] = InstallationToken.new(
                 jwt_token,
                 installation,
                 installations_token_map_name,
@@ -510,9 +623,15 @@ class GithubBackend(GitBackend):
     def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
         """Updates the flake an opens a PR for it."""
         factory = util.BuildFactory()
-        factory.addStep(
+        steps = self.auth_backend.create_reload_builder_steps(
-            self.auth_backend.create_reload_builder_step(self.config.project_cache_file)
+            self.config.project_cache_file,
+            self.webhook_secret,
+            self.webhook_url,
+            self.config.topic,
         )
+        for step in steps:
+            factory.addStep(step)
+
         return util.BuilderConfig(
             name=self.reload_builder_name,
             workernames=worker_names,
@@ -563,9 +682,13 @@ class GithubBackend(GitBackend):
         if not self.config.project_cache_file.exists():
             return []
 
-        repos: list[dict[str, Any]] = sorted(
+        repos: list[dict[str, Any]] = filter_repos_by_topic(
-            json.loads(self.config.project_cache_file.read_text()),
+            self.config.topic,
-            key=lambda x: x["full_name"],
+            sorted(
+                json.loads(self.config.project_cache_file.read_text()),
+                key=lambda x: x["full_name"],
+            ),
+            lambda repo: repo["topics"],
         )
 
         if isinstance(self.auth_backend, GithubAppAuthBackend):
@@ -580,22 +703,22 @@ class GithubBackend(GitBackend):
                     tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
             repos = list(filter(lambda repo: "installation_id" in repo, repos))
 
-        tlog.info(f"Loading {len(repos)} cached repositories.")
+        repo_names: list[str] = [
-        return list(
+            repo["owner"]["login"] + "/" + repo["name"] for repo in repos
-            filter(
+        ]
-                lambda project: self.config.topic is not None
+
-                and self.config.topic in project.topics,
+        tlog.info(
-                [
+            f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
-                    GithubProject(
-                        self.auth_backend.get_repo_token(repo),
-                        self.config,
-                        self.webhook_secret,
-                        repo,
-                    )
-                    for repo in repos
-                ],
-            )
         )
+        return [
+            GithubProject(
+                self.auth_backend.get_repo_token(repo),
+                self.config,
+                self.webhook_secret,
+                repo,
+            )
+            for repo in repos
+        ]
 
     def are_projects_cached(self) -> bool:
         if not self.config.project_cache_file.exists():
@@ -632,6 +755,39 @@ class GithubBackend(GitBackend):
         return "github"
 
 
+def create_project_hook(
+    token: RepoToken, webhook_secret: str, owner: str, repo: str, webhook_url: str
+) -> None:
+    hooks = paginated_github_request(
+        f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
+        token.get(),
+    )
+    config = dict(
+        url=webhook_url + "change_hook/github",
+        content_type="json",
+        insecure_ssl="0",
+        secret=webhook_secret,
+    )
+    data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
+    headers = {
+        "Authorization": f"Bearer {token.get()}",
+        "Accept": "application/vnd.github+json",
+        "Content-Type": "application/json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    for hook in hooks:
+        if hook["config"]["url"] == webhook_url + "change_hook/github":
+            log.msg(f"hook for {owner}/{repo} already exists")
+            return
+
+    http_request(
+        f"https://api.github.com/repos/{owner}/{repo}/hooks",
+        method="POST",
+        headers=headers,
+        data=data,
+    )
+
+
 class GithubProject(GitProject):
     config: GithubConfig
     webhook_secret: str
@@ -650,43 +806,6 @@ class GithubProject(GitProject):
         self.webhook_secret = webhook_secret
         self.data = data
 
-    def create_project_hook(
-        self,
-        owner: str,
-        repo: str,
-        webhook_url: str,
-    ) -> None:
-        hooks = paginated_github_request(
-            f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
-            self.token.get(),
-        )
-        config = dict(
-            url=webhook_url + "change_hook/github",
-            content_type="json",
-            insecure_ssl="0",
-            secret=self.webhook_secret,
-        )
-        data = dict(
-            name="web", active=True, events=["push", "pull_request"], config=config
-        )
-        headers = {
-            "Authorization": f"Bearer {self.token.get()}",
-            "Accept": "application/vnd.github+json",
-            "Content-Type": "application/json",
-            "X-GitHub-Api-Version": "2022-11-28",
-        }
-        for hook in hooks:
-            if hook["config"]["url"] == webhook_url + "change_hook/github":
-                log.msg(f"hook for {owner}/{repo} already exists")
-                return
-
-        http_request(
-            f"https://api.github.com/repos/{owner}/{repo}/hooks",
-            method="POST",
-            headers=headers,
-            data=data,
-        )
-
     def get_project_url(self) -> str:
         return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}"
 

buildbot_nix/projects.py

@@ -62,15 +62,6 @@ class GitBackend(ABC):
 
 
 class GitProject(ABC):
-    @abstractmethod
-    def create_project_hook(
-        self,
-        owner: str,
-        repo: str,
-        webhook_url: str,
-    ) -> None:
-        pass
-
     @abstractmethod
     def get_project_url(self) -> str:
         pass