Merge pull request #203 from MagicRB/defer-hook-creation

Defer hook creation to project reload, both GitHub and Gitea
This commit is contained in:
Jörg Thalheim 2024-07-05 14:21:29 +02:00 committed by GitHub
commit 04cec58b20
Failed to generate hash of commit
6 changed files with 476 additions and 255 deletions

View file

@ -818,7 +818,7 @@ class NixConfigurator(ConfiguratorBase):
backends: dict[str, GitBackend] = {} backends: dict[str, GitBackend] = {}
if self.github is not None: if self.github is not None:
backends["github"] = GithubBackend(self.github) backends["github"] = GithubBackend(self.github, self.url)
if self.gitea is not None: if self.gitea is not None:
backends["gitea"] = GiteaBackend(self.gitea) backends["gitea"] = GiteaBackend(self.gitea)
@ -851,7 +851,6 @@ class NixConfigurator(ConfiguratorBase):
eval_lock = util.MasterLock("nix-eval") eval_lock = util.MasterLock("nix-eval")
for project in projects: for project in projects:
project.create_project_hook(project.owner, project.repo, self.url)
config_for_project( config_for_project(
config, config,
project, project,

View file

@ -2,9 +2,20 @@ import contextlib
import http.client import http.client
import json import json
import urllib.request import urllib.request
from abc import ABC, abstractmethod
from collections.abc import Callable
from pathlib import Path from pathlib import Path
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from typing import Any from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from collections.abc import Generator
from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from twisted.internet import defer, threads
from twisted.python.failure import Failure
def slugify_project_name(name: str) -> str: def slugify_project_name(name: str) -> str:
@ -97,3 +108,49 @@ def atomic_write_file(file: Path, data: str) -> None:
except OSError: except OSError:
path.unlink() path.unlink()
raise raise
def filter_repos_by_topic(
topic: str | None, repos: list[Any], topics: Callable[[Any], list[str]]
) -> list[Any]:
return list(
filter(
lambda repo: topic is None or topic in topics(repo),
repos,
)
)
class ThreadDeferredBuildStep(BuildStep, ABC):
def __init__(
self,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
@abstractmethod
def run_deferred(self) -> None:
pass
@abstractmethod
def run_post(self) -> Any:
pass
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.run_deferred) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
return self.run_post()
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE

View file

@ -1,30 +1,26 @@
import json import json
import os import os
import signal import signal
from collections.abc import Generator
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from tempfile import NamedTemporaryFile from typing import Any
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse from urllib.parse import urlparse
from buildbot.config.builder import BuilderConfig from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase from buildbot.reporters.base import ReporterBase
from buildbot.www.auth import AuthBase from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase from buildbot.www.avatar import AvatarBase
from buildbot_gitea.auth import GiteaAuth # type: ignore[import] from buildbot_gitea.auth import GiteaAuth # type: ignore[import]
from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import] from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import]
from twisted.internet import defer, threads from twisted.logger import Logger
from twisted.python import log from twisted.python import log
from twisted.python.failure import Failure
if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from .common import ( from .common import (
ThreadDeferredBuildStep,
atomic_write_file,
filter_repos_by_topic,
http_request, http_request,
paginated_github_request, paginated_github_request,
slugify_project_name, slugify_project_name,
@ -32,6 +28,8 @@ from .common import (
from .projects import GitBackend, GitProject from .projects import GitBackend, GitProject
from .secrets import read_secret_file from .secrets import read_secret_file
tlog = Logger()
@dataclass @dataclass
class GiteaConfig: class GiteaConfig:
@ -63,46 +61,6 @@ class GiteaProject(GitProject):
self.webhook_secret = webhook_secret self.webhook_secret = webhook_secret
self.data = data self.data = data
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
hooks = paginated_github_request(
f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
self.config.token(),
)
config = dict(
url=webhook_url + "change_hook/gitea",
content_type="json",
insecure_ssl="0",
secret=self.webhook_secret,
)
data = dict(
name="web",
active=True,
events=["push", "pull_request"],
config=config,
type="gitea",
)
headers = {
"Authorization": f"token {self.config.token()}",
"Accept": "application/json",
"Content-Type": "application/json",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/gitea":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def get_project_url(self) -> str: def get_project_url(self) -> str:
url = urlparse(self.config.instance_url) url = urlparse(self.config.instance_url)
return f"{url.scheme}://git:%(secret:{self.config.token_secret_name})s@{url.hostname}/{self.name}" return f"{url.scheme}://git:%(secret:{self.config.token_secret_name})s@{url.hostname}/{self.name}"
@ -153,6 +111,7 @@ class GiteaProject(GitProject):
class GiteaBackend(GitBackend): class GiteaBackend(GitBackend):
config: GiteaConfig config: GiteaConfig
webhook_secret: str
def __init__(self, config: GiteaConfig) -> None: def __init__(self, config: GiteaConfig) -> None:
self.config = config self.config = config
@ -164,6 +123,11 @@ class GiteaBackend(GitBackend):
factory.addStep( factory.addStep(
ReloadGiteaProjects(self.config, self.config.project_cache_file), ReloadGiteaProjects(self.config, self.config.project_cache_file),
) )
factory.addStep(
CreateGiteaProjectHooks(
self.config, self.config.project_cache_file, self.webhook_secret
),
)
return util.BuilderConfig( return util.BuilderConfig(
name=self.reload_builder_name, name=self.reload_builder_name,
workernames=worker_names, workernames=worker_names,
@ -201,21 +165,23 @@ class GiteaBackend(GitBackend):
if not self.config.project_cache_file.exists(): if not self.config.project_cache_file.exists():
return [] return []
repos: list[dict[str, Any]] = sorted( repos: list[dict[str, Any]] = filter_repos_by_topic(
json.loads(self.config.project_cache_file.read_text()), self.config.topic,
key=lambda x: x["full_name"], sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
),
lambda repo: repo["topics"],
) )
return list( repo_names: list[str] = [
filter( repo["owner"]["login"] + "/" + repo["name"] for repo in repos
lambda project: self.config.topic is not None ]
and self.config.topic in project.topics, tlog.info(
[ f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
GiteaProject(self.config, self.webhook_secret, repo)
for repo in repos
],
)
) )
return [GiteaProject(self.config, self.webhook_secret, repo) for repo in repos]
def are_projects_cached(self) -> bool: def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists() return self.config.project_cache_file.exists()
@ -236,43 +202,111 @@ class GiteaBackend(GitBackend):
return "gitea" return "gitea"
class ReloadGiteaProjects(BuildStep): def create_repo_hook(
name = "reload_gitea_projects" token: str, webhook_secret: str, owner: str, repo: str, webhook_url: str
) -> None:
hooks = paginated_github_request(
f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
token,
)
config = dict(
url=webhook_url + "change_hook/gitea",
content_type="json",
insecure_ssl="0",
secret=webhook_secret,
)
data = dict(
name="web",
active=True,
events=["push", "pull_request"],
config=config,
type="gitea",
)
headers = {
"Authorization": f"token {token}",
"Accept": "application/json",
"Content-Type": "application/json",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/gitea":
log.msg(f"hook for {owner}/{repo} already exists")
return
log.msg(f"creating hook for {owner}/{repo}")
http_request(
f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
class CreateGiteaProjectHooks(ThreadDeferredBuildStep):
name = "create_gitea_project_hooks"
config: GiteaConfig config: GiteaConfig
project_cache_file: Path
webhook_secret: str
def __init__( def __init__(
self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any self,
config: GiteaConfig,
project_cache_file: Path,
webhook_secret: str,
**kwargs: Any,
) -> None:
self.config = config
self.project_cache_file = project_cache_file
self.webhook_secret = webhook_secret
super().__init__(**kwargs)
def run_deferred(self) -> None:
repos = json.loads(self.project_cache_file.read_text())
for repo in repos:
create_repo_hook(
self.config.token(),
self.webhook_secret,
repo["owner"]["login"],
repo["name"],
self.config.instance_url,
)
def run_post(self) -> Any:
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
class ReloadGiteaProjects(ThreadDeferredBuildStep):
name = "reload_gitea_projects"
config: GiteaConfig
project_cache_file: Path
def __init__(
self,
config: GiteaConfig,
project_cache_file: Path,
**kwargs: Any,
) -> None: ) -> None:
self.config = config self.config = config
self.project_cache_file = project_cache_file self.project_cache_file = project_cache_file
super().__init__(**kwargs) super().__init__(**kwargs)
def reload_projects(self) -> None: def run_deferred(self) -> None:
refresh_projects(self.config, self.project_cache_file) repos = filter_repos_by_topic(
self.config.topic,
refresh_projects(self.config, self.project_cache_file),
lambda repo: repo["topics"],
)
@defer.inlineCallbacks atomic_write_file(self.project_cache_file, json.dumps(repos))
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = "" def run_post(self) -> Any:
return util.SUCCESS
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None: def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]:
repos = [] repos = []
for repo in paginated_github_request( for repo in paginated_github_request(
@ -296,12 +330,4 @@ def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
except OSError: except OSError:
pass pass
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: return repos
path = Path(f.name)
try:
f.write(json.dumps(repos))
f.flush()
path.rename(repo_cache_file)
except OSError:
path.unlink()
raise

View file

@ -1,7 +1,7 @@
import json import json
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any, TypedDict
from buildbot_nix.common import ( from buildbot_nix.common import (
HttpResponse, HttpResponse,
@ -13,6 +13,11 @@ from .jwt_token import JWTToken
from .repo_token import RepoToken from .repo_token import RepoToken
class InstallationTokenJSON(TypedDict):
expiration: str
token: str
class InstallationToken(RepoToken): class InstallationToken(RepoToken):
GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60) GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
@ -66,6 +71,30 @@ class InstallationToken(RepoToken):
else: else:
self.token, self.expiration = installation_token self.token, self.expiration = installation_token
@staticmethod
def new(
jwt_token: JWTToken, installation_id: int, installations_token_map_name: Path
) -> "InstallationToken":
return InstallationToken(
jwt_token, installation_id, installations_token_map_name
)
@staticmethod
def from_json(
jwt_token: JWTToken,
installation_id: int,
installations_token_map_name: Path,
json: InstallationTokenJSON,
) -> "InstallationToken":
token: str = json["token"]
expiration: datetime = datetime.fromisoformat(json["expiration"])
return InstallationToken(
jwt_token,
installation_id,
installations_token_map_name,
(token, expiration),
)
def get(self) -> str: def get(self) -> str:
self.verify() self.verify()
return self.token return self.token

View file

@ -2,13 +2,11 @@ import json
import os import os
import signal import signal
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Callable, Generator from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from datetime import ( from itertools import starmap
datetime,
)
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any from typing import Any
from buildbot.config.builder import BuilderConfig from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util from buildbot.plugins import util
@ -20,16 +18,13 @@ from buildbot.secrets.providers.base import SecretProviderBase
from buildbot.www.auth import AuthBase from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.oauth2 import GitHubAuth from buildbot.www.oauth2 import GitHubAuth
from twisted.internet import defer, threads
from twisted.logger import Logger from twisted.logger import Logger
from twisted.python import log from twisted.python import log
from twisted.python.failure import Failure
if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from .common import ( from .common import (
ThreadDeferredBuildStep,
atomic_write_file, atomic_write_file,
filter_repos_by_topic,
http_request, http_request,
paginated_github_request, paginated_github_request,
slugify_project_name, slugify_project_name,
@ -57,13 +52,71 @@ def get_installations(jwt_token: JWTToken) -> list[int]:
return [installation["id"] for installation in installations] return [installation["id"] for installation in installations]
class ReloadGithubInstallations(BuildStep): class CreateGitHubInstallationHooks(ThreadDeferredBuildStep):
name = "create_github_installation_hooks"
jwt_token: JWTToken
project_cache_file: Path
installation_token_map_name: Path
webhook_secret: str
webhook_url: str
topic: str | None
def __init__(
self,
jwt_token: JWTToken,
project_cache_file: Path,
installation_token_map_name: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
**kwargs: Any,
) -> None:
self.jwt_token = jwt_token
self.project_cache_file = project_cache_file
self.installation_token_map_name = installation_token_map_name
self.webhook_secret = webhook_secret
self.webhook_url = webhook_url
self.topic = topic
super().__init__(**kwargs)
def run_deferred(self) -> None:
repos = json.loads(self.project_cache_file.read_text())
installation_token_map: dict[int, InstallationToken] = dict(
starmap(
lambda k, v: (
int(k),
InstallationToken.from_json(
self.jwt_token, int(k), self.installation_token_map_name, v
),
),
json.loads(self.installation_token_map_name.read_text()).items(),
)
)
for repo in repos:
create_project_hook(
installation_token_map[repo["installation_id"]],
self.webhook_secret,
repo["owner"]["login"],
repo["name"],
self.webhook_url,
)
def run_post(self) -> Any:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
class ReloadGithubInstallations(ThreadDeferredBuildStep):
name = "reload_github_projects" name = "reload_github_projects"
jwt_token: JWTToken jwt_token: JWTToken
project_cache_file: Path project_cache_file: Path
installation_token_map_name: Path installation_token_map_name: Path
project_id_map_name: Path project_id_map_name: Path
topic: str | None
def __init__( def __init__(
self, self,
@ -71,15 +124,17 @@ class ReloadGithubInstallations(BuildStep):
project_cache_file: Path, project_cache_file: Path,
installation_token_map_name: Path, installation_token_map_name: Path,
project_id_map_name: Path, project_id_map_name: Path,
topic: str | None,
**kwargs: Any, **kwargs: Any,
) -> None: ) -> None:
self.jwt_token = jwt_token self.jwt_token = jwt_token
self.installation_token_map_name = installation_token_map_name self.installation_token_map_name = installation_token_map_name
self.project_id_map_name = project_id_map_name self.project_id_map_name = project_id_map_name
self.project_cache_file = project_cache_file self.project_cache_file = project_cache_file
self.topic = topic
super().__init__(**kwargs) super().__init__(**kwargs)
def reload_projects(self) -> None: def run_deferred(self) -> None:
installation_token_map = GithubBackend.create_missing_installations( installation_token_map = GithubBackend.create_missing_installations(
self.jwt_token, self.jwt_token,
self.installation_token_map_name, self.installation_token_map_name,
@ -96,89 +151,110 @@ class ReloadGithubInstallations(BuildStep):
repos = [] repos = []
for k, v in installation_token_map.items(): for k, v in installation_token_map.items():
new_repos = refresh_projects( new_repos = filter_repos_by_topic(
v.get(), self.topic,
self.project_cache_file, refresh_projects(
clear=True, v.get(),
api_endpoint="/installation/repositories", self.project_cache_file,
subkey="repositories", clear=True,
require_admin=False, api_endpoint="/installation/repositories",
subkey="repositories",
require_admin=False,
),
lambda repo: repo["topics"],
) )
for repo in new_repos: for repo in new_repos:
repo["installation_id"] = k repo["installation_id"] = k
repos.extend(new_repos)
for repo in new_repos:
project_id_map[repo["full_name"]] = k project_id_map[repo["full_name"]] = k
repos.extend(new_repos)
atomic_write_file(self.project_cache_file, json.dumps(repos)) atomic_write_file(self.project_cache_file, json.dumps(repos))
atomic_write_file(self.project_id_map_name, json.dumps(project_id_map)) atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
tlog.info( tlog.info(
f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token." f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation tokens."
) )
@defer.inlineCallbacks def run_post(self) -> Any:
def run(self) -> Generator[Any, object, Any]: return util.SUCCESS
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
class ReloadGithubProjects(BuildStep): class CreateGitHubProjectHooks(ThreadDeferredBuildStep):
name = "reload_github_projects" name = "create_github_project_hooks"
token: RepoToken
project_cache_file: Path
webhook_secret: str
webhook_url: str
topic: str | None
def __init__( def __init__(
self, token: RepoToken, project_cache_file: Path, **kwargs: Any self,
token: RepoToken,
project_cache_file: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
**kwargs: Any,
) -> None: ) -> None:
self.token = token self.token = token
self.project_cache_file = project_cache_file self.project_cache_file = project_cache_file
self.webhook_secret = webhook_secret
self.webhook_url = webhook_url
self.topic = topic
super().__init__(**kwargs) super().__init__(**kwargs)
def reload_projects(self) -> None: def run_deferred(self) -> None:
repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file) repos = json.loads(self.project_cache_file.read_text())
log.msg(repos) for repo in repos:
create_project_hook(
self.token,
self.webhook_secret,
repo["owner"]["login"],
repo["name"],
self.webhook_url,
)
def run_post(self) -> Any:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
class ReloadGithubProjects(ThreadDeferredBuildStep):
name = "reload_github_projects"
token: RepoToken
project_cache_file: Path
topic: str | None
def __init__(
self,
token: RepoToken,
project_cache_file: Path,
topic: str | None,
**kwargs: Any,
) -> None:
self.token = token
self.project_cache_file = project_cache_file
self.topic = topic
super().__init__(**kwargs)
def run_deferred(self) -> None:
repos: list[Any] = filter_repos_by_topic(
self.topic,
refresh_projects(self.token.get(), self.project_cache_file),
lambda repo: repo["topics"],
)
atomic_write_file(self.project_cache_file, json.dumps(repos)) atomic_write_file(self.project_cache_file, json.dumps(repos))
@defer.inlineCallbacks def run_post(self) -> Any:
def run(self) -> Generator[Any, object, Any]: return util.SUCCESS
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
class GitHubAppStatusPush(GitHubStatusPush): class GitHubAppStatusPush(GitHubStatusPush):
@ -307,7 +383,13 @@ class GithubAuthBackend(ABC):
pass pass
@abstractmethod @abstractmethod
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: def create_reload_builder_steps(
self,
project_cache_file: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
) -> list[BuildStep]:
pass pass
@ -338,10 +420,27 @@ class GithubLegacyAuthBackend(GithubAuthBackend):
context=Interpolate("buildbot/%(prop:status_name)s"), context=Interpolate("buildbot/%(prop:status_name)s"),
) )
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: def create_reload_builder_steps(
return ReloadGithubProjects( self,
token=self.token, project_cache_file=project_cache_file project_cache_file: Path,
) webhook_secret: str,
webhook_url: str,
topic: str | None,
) -> list[BuildStep]:
return [
ReloadGithubProjects(
token=self.token,
project_cache_file=project_cache_file,
topic=topic,
),
CreateGitHubProjectHooks(
token=self.token,
project_cache_file=project_cache_file,
webhook_secret=webhook_secret,
webhook_url=webhook_url,
topic=topic,
),
]
class GitHubLegacySecretService(SecretProviderBase): class GitHubLegacySecretService(SecretProviderBase):
@ -406,13 +505,30 @@ class GithubAppAuthBackend(GithubAuthBackend):
context=Interpolate("buildbot/%(prop:status_name)s"), context=Interpolate("buildbot/%(prop:status_name)s"),
) )
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep: def create_reload_builder_steps(
return ReloadGithubInstallations( self,
self.jwt_token, project_cache_file: Path,
project_cache_file, webhook_secret: str,
self.auth_type.app_installation_token_map_name, webhook_url: str,
self.auth_type.app_project_id_map_name, topic: str | None,
) ) -> list[BuildStep]:
return [
ReloadGithubInstallations(
self.jwt_token,
project_cache_file,
self.auth_type.app_installation_token_map_name,
self.auth_type.app_project_id_map_name,
topic,
),
CreateGitHubInstallationHooks(
self.jwt_token,
project_cache_file,
self.auth_type.app_installation_token_map_name,
webhook_secret=webhook_secret,
webhook_url=webhook_url,
topic=topic,
),
]
class GitHubAppSecretService(SecretProviderBase): class GitHubAppSecretService(SecretProviderBase):
@ -453,12 +569,14 @@ class GithubConfig:
class GithubBackend(GitBackend): class GithubBackend(GitBackend):
config: GithubConfig config: GithubConfig
webhook_secret: str webhook_secret: str
webhook_url: str
auth_backend: GithubAuthBackend auth_backend: GithubAuthBackend
def __init__(self, config: GithubConfig) -> None: def __init__(self, config: GithubConfig, webhook_url: str) -> None:
self.config = config self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name) self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
self.webhook_url = webhook_url
if isinstance(self.config.auth_type, AuthTypeLegacy): if isinstance(self.config.auth_type, AuthTypeLegacy):
self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type) self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
@ -480,13 +598,8 @@ class GithubBackend(GitBackend):
installations_map: dict[int, InstallationToken] = {} installations_map: dict[int, InstallationToken] = {}
for iid, installation in initial_installations_map.items(): for iid, installation in initial_installations_map.items():
token: str = installation["token"] installations_map[int(iid)] = InstallationToken.from_json(
expiration: datetime = datetime.fromisoformat(installation["expiration"]) jwt_token, int(iid), installations_token_map_name, installation
installations_map[int(iid)] = InstallationToken(
jwt_token,
int(iid),
installations_token_map_name,
installation_token=(token, expiration),
) )
return installations_map return installations_map
@ -499,7 +612,7 @@ class GithubBackend(GitBackend):
installations: list[int], installations: list[int],
) -> dict[int, InstallationToken]: ) -> dict[int, InstallationToken]:
for installation in set(installations) - installations_map.keys(): for installation in set(installations) - installations_map.keys():
installations_map[installation] = InstallationToken( installations_map[installation] = InstallationToken.new(
jwt_token, jwt_token,
installation, installation,
installations_token_map_name, installations_token_map_name,
@ -510,9 +623,15 @@ class GithubBackend(GitBackend):
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it.""" """Updates the flake an opens a PR for it."""
factory = util.BuildFactory() factory = util.BuildFactory()
factory.addStep( steps = self.auth_backend.create_reload_builder_steps(
self.auth_backend.create_reload_builder_step(self.config.project_cache_file) self.config.project_cache_file,
self.webhook_secret,
self.webhook_url,
self.config.topic,
) )
for step in steps:
factory.addStep(step)
return util.BuilderConfig( return util.BuilderConfig(
name=self.reload_builder_name, name=self.reload_builder_name,
workernames=worker_names, workernames=worker_names,
@ -563,9 +682,13 @@ class GithubBackend(GitBackend):
if not self.config.project_cache_file.exists(): if not self.config.project_cache_file.exists():
return [] return []
repos: list[dict[str, Any]] = sorted( repos: list[dict[str, Any]] = filter_repos_by_topic(
json.loads(self.config.project_cache_file.read_text()), self.config.topic,
key=lambda x: x["full_name"], sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
),
lambda repo: repo["topics"],
) )
if isinstance(self.auth_backend, GithubAppAuthBackend): if isinstance(self.auth_backend, GithubAppAuthBackend):
@ -580,22 +703,22 @@ class GithubBackend(GitBackend):
tlog.info(f"\tDropping repo {dropped_repo['full_name']}") tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
repos = list(filter(lambda repo: "installation_id" in repo, repos)) repos = list(filter(lambda repo: "installation_id" in repo, repos))
tlog.info(f"Loading {len(repos)} cached repositories.") repo_names: list[str] = [
return list( repo["owner"]["login"] + "/" + repo["name"] for repo in repos
filter( ]
lambda project: self.config.topic is not None
and self.config.topic in project.topics, tlog.info(
[ f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
GithubProject(
self.auth_backend.get_repo_token(repo),
self.config,
self.webhook_secret,
repo,
)
for repo in repos
],
)
) )
return [
GithubProject(
self.auth_backend.get_repo_token(repo),
self.config,
self.webhook_secret,
repo,
)
for repo in repos
]
def are_projects_cached(self) -> bool: def are_projects_cached(self) -> bool:
if not self.config.project_cache_file.exists(): if not self.config.project_cache_file.exists():
@ -632,6 +755,39 @@ class GithubBackend(GitBackend):
return "github" return "github"
def create_project_hook(
token: RepoToken, webhook_secret: str, owner: str, repo: str, webhook_url: str
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
token.get(),
)
config = dict(
url=webhook_url + "change_hook/github",
content_type="json",
insecure_ssl="0",
secret=webhook_secret,
)
data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
headers = {
"Authorization": f"Bearer {token.get()}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/github":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
class GithubProject(GitProject): class GithubProject(GitProject):
config: GithubConfig config: GithubConfig
webhook_secret: str webhook_secret: str
@ -650,43 +806,6 @@ class GithubProject(GitProject):
self.webhook_secret = webhook_secret self.webhook_secret = webhook_secret
self.data = data self.data = data
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
self.token.get(),
)
config = dict(
url=webhook_url + "change_hook/github",
content_type="json",
insecure_ssl="0",
secret=self.webhook_secret,
)
data = dict(
name="web", active=True, events=["push", "pull_request"], config=config
)
headers = {
"Authorization": f"Bearer {self.token.get()}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/github":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def get_project_url(self) -> str: def get_project_url(self) -> str:
return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}" return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}"

View file

@ -62,15 +62,6 @@ class GitBackend(ABC):
class GitProject(ABC): class GitProject(ABC):
@abstractmethod
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
pass
@abstractmethod @abstractmethod
def get_project_url(self) -> str: def get_project_url(self) -> str:
pass pass