auxolotl/buildbot-nix
26
1
Fork 0
Code Issues Pull requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #203 from MagicRB/defer-hook-creation

Browse source 
Defer hook creation to project reload, both GitHub and Gitea
This commit is contained in:
Jörg Thalheim 2024-07-05 14:21:29 +02:00 committed by GitHub
parent 837df7f642 372d7dc70b
commit 04cec58b20
Failed to generate hash of commit
6 changed files with 476 additions and 255 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
buildbot_nix/__init__.py
View file

@ -818,7 +818,7 @@ class NixConfigurator(ConfiguratorBase):
backends: dict[str, GitBackend] = {}
if self.github is not None:
backends["github"] = GithubBackend(self.github)
backends["github"] = GithubBackend(self.github, self.url)
if self.gitea is not None:
backends["gitea"] = GiteaBackend(self.gitea)
@ -851,7 +851,6 @@ class NixConfigurator(ConfiguratorBase):
eval_lock = util.MasterLock("nix-eval")
for project in projects:
project.create_project_hook(project.owner, project.repo, self.url)
config_for_project(
config,
project,

59
buildbot_nix/common.py
View file

@ -2,9 +2,20 @@ import contextlib
import http.client
import json
import urllib.request
from abc import ABC, abstractmethod
from collections.abc import Callable
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from collections.abc import Generator
from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from twisted.internet import defer, threads
from twisted.python.failure import Failure
def slugify_project_name(name: str) -> str:
@ -97,3 +108,49 @@ def atomic_write_file(file: Path, data: str) -> None:
except OSError:
path.unlink()
raise
def filter_repos_by_topic(
topic: str | None, repos: list[Any], topics: Callable[[Any], list[str]]
) -> list[Any]:
return list(
filter(
lambda repo: topic is None or topic in topics(repo),
repos,
)
)
class ThreadDeferredBuildStep(BuildStep, ABC):
def __init__(
self,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
@abstractmethod
def run_deferred(self) -> None:
pass
@abstractmethod
def run_post(self) -> Any:
pass
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.run_deferred) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
return self.run_post()
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE

216
buildbot_nix/gitea_projects.py
View file

@ -1,30 +1,26 @@
import json
import os
import signal
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any
from typing import Any
from urllib.parse import urlparse
from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util
from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate
from buildbot.reporters.base import ReporterBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase
from buildbot_gitea.auth import GiteaAuth # type: ignore[import]
from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import]
from twisted.internet import defer, threads
from twisted.logger import Logger
from twisted.python import log
from twisted.python.failure import Failure
if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from .common import (
ThreadDeferredBuildStep,
atomic_write_file,
filter_repos_by_topic,
http_request,
paginated_github_request,
slugify_project_name,
@ -32,6 +28,8 @@ from .common import (
from .projects import GitBackend, GitProject
from .secrets import read_secret_file
tlog = Logger()
@dataclass
class GiteaConfig:
@ -63,46 +61,6 @@ class GiteaProject(GitProject):
self.webhook_secret = webhook_secret
self.data = data
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
hooks = paginated_github_request(
f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
self.config.token(),
)
config = dict(
url=webhook_url + "change_hook/gitea",
content_type="json",
insecure_ssl="0",
secret=self.webhook_secret,
)
data = dict(
name="web",
active=True,
events=["push", "pull_request"],
config=config,
type="gitea",
)
headers = {
"Authorization": f"token {self.config.token()}",
"Accept": "application/json",
"Content-Type": "application/json",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/gitea":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"{self.config.instance_url}/api/v1/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def get_project_url(self) -> str:
url = urlparse(self.config.instance_url)
return f"{url.scheme}://git:%(secret:{self.config.token_secret_name})s@{url.hostname}/{self.name}"
@ -153,6 +111,7 @@ class GiteaProject(GitProject):
class GiteaBackend(GitBackend):
config: GiteaConfig
webhook_secret: str
def __init__(self, config: GiteaConfig) -> None:
self.config = config
@ -164,6 +123,11 @@ class GiteaBackend(GitBackend):
factory.addStep(
ReloadGiteaProjects(self.config, self.config.project_cache_file),
)
factory.addStep(
CreateGiteaProjectHooks(
self.config, self.config.project_cache_file, self.webhook_secret
),
)
return util.BuilderConfig(
name=self.reload_builder_name,
workernames=worker_names,
@ -201,21 +165,23 @@ class GiteaBackend(GitBackend):
if not self.config.project_cache_file.exists():
return []
repos: list[dict[str, Any]] = sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
repos: list[dict[str, Any]] = filter_repos_by_topic(
self.config.topic,
sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
),
lambda repo: repo["topics"],
)
return list(
filter(
lambda project: self.config.topic is not None
and self.config.topic in project.topics,
[
GiteaProject(self.config, self.webhook_secret, repo)
for repo in repos
],
)
repo_names: list[str] = [
repo["owner"]["login"] + "/" + repo["name"] for repo in repos
]
tlog.info(
f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
)
return [GiteaProject(self.config, self.webhook_secret, repo) for repo in repos]
def are_projects_cached(self) -> bool:
return self.config.project_cache_file.exists()
@ -236,43 +202,111 @@ class GiteaBackend(GitBackend):
return "gitea"
class ReloadGiteaProjects(BuildStep):
name = "reload_gitea_projects"
def create_repo_hook(
token: str, webhook_secret: str, owner: str, repo: str, webhook_url: str
) -> None:
hooks = paginated_github_request(
f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
token,
)
config = dict(
url=webhook_url + "change_hook/gitea",
content_type="json",
insecure_ssl="0",
secret=webhook_secret,
)
data = dict(
name="web",
active=True,
events=["push", "pull_request"],
config=config,
type="gitea",
)
headers = {
"Authorization": f"token {token}",
"Accept": "application/json",
"Content-Type": "application/json",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/gitea":
log.msg(f"hook for {owner}/{repo} already exists")
return
log.msg(f"creating hook for {owner}/{repo}")
http_request(
f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
class CreateGiteaProjectHooks(ThreadDeferredBuildStep):
name = "create_gitea_project_hooks"
config: GiteaConfig
project_cache_file: Path
webhook_secret: str
def __init__(
self, config: GiteaConfig, project_cache_file: Path, **kwargs: Any
self,
config: GiteaConfig,
project_cache_file: Path,
webhook_secret: str,
**kwargs: Any,
) -> None:
self.config = config
self.project_cache_file = project_cache_file
self.webhook_secret = webhook_secret
super().__init__(**kwargs)
def run_deferred(self) -> None:
repos = json.loads(self.project_cache_file.read_text())
for repo in repos:
create_repo_hook(
self.config.token(),
self.webhook_secret,
repo["owner"]["login"],
repo["name"],
self.config.instance_url,
)
def run_post(self) -> Any:
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
class ReloadGiteaProjects(ThreadDeferredBuildStep):
name = "reload_gitea_projects"
config: GiteaConfig
project_cache_file: Path
def __init__(
self,
config: GiteaConfig,
project_cache_file: Path,
**kwargs: Any,
) -> None:
self.config = config
self.project_cache_file = project_cache_file
super().__init__(**kwargs)
def reload_projects(self) -> None:
refresh_projects(self.config, self.project_cache_file)
def run_deferred(self) -> None:
repos = filter_repos_by_topic(
self.config.topic,
refresh_projects(self.config, self.project_cache_file),
lambda repo: repo["topics"],
)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
atomic_write_file(self.project_cache_file, json.dumps(repos))
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
def run_post(self) -> Any:
return util.SUCCESS
def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> list[Any]:
repos = []
for repo in paginated_github_request(
@ -296,12 +330,4 @@ def refresh_projects(config: GiteaConfig, repo_cache_file: Path) -> None:
except OSError:
pass
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
path = Path(f.name)
try:
f.write(json.dumps(repos))
f.flush()
path.rename(repo_cache_file)
except OSError:
path.unlink()
raise
return repos

31
buildbot_nix/github/installation_token.py
View file

@ -1,7 +1,7 @@
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from typing import Any, TypedDict
from buildbot_nix.common import (
HttpResponse,
@ -13,6 +13,11 @@ from .jwt_token import JWTToken
from .repo_token import RepoToken
class InstallationTokenJSON(TypedDict):
expiration: str
token: str
class InstallationToken(RepoToken):
GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
@ -66,6 +71,30 @@ class InstallationToken(RepoToken):
else:
self.token, self.expiration = installation_token
@staticmethod
def new(
jwt_token: JWTToken, installation_id: int, installations_token_map_name: Path
) -> "InstallationToken":
return InstallationToken(
jwt_token, installation_id, installations_token_map_name
)
@staticmethod
def from_json(
jwt_token: JWTToken,
installation_id: int,
installations_token_map_name: Path,
json: InstallationTokenJSON,
) -> "InstallationToken":
token: str = json["token"]
expiration: datetime = datetime.fromisoformat(json["expiration"])
return InstallationToken(
jwt_token,
installation_id,
installations_token_map_name,
(token, expiration),
)
def get(self) -> str:
self.verify()
return self.token

413
buildbot_nix/github_projects.py
View file

@ -2,13 +2,11 @@ import json
import os
import signal
from abc import ABC, abstractmethod
from collections.abc import Callable, Generator
from collections.abc import Callable
from dataclasses import dataclass
from datetime import (
datetime,
)
from itertools import starmap
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import Any
from buildbot.config.builder import BuilderConfig
from buildbot.plugins import util
@ -20,16 +18,13 @@ from buildbot.secrets.providers.base import SecretProviderBase
from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.oauth2 import GitHubAuth
from twisted.internet import defer, threads
from twisted.logger import Logger
from twisted.python import log
from twisted.python.failure import Failure
if TYPE_CHECKING:
from buildbot.process.log import StreamLog
from .common import (
ThreadDeferredBuildStep,
atomic_write_file,
filter_repos_by_topic,
http_request,
paginated_github_request,
slugify_project_name,
@ -57,13 +52,71 @@ def get_installations(jwt_token: JWTToken) -> list[int]:
return [installation["id"] for installation in installations]
class ReloadGithubInstallations(BuildStep):
class CreateGitHubInstallationHooks(ThreadDeferredBuildStep):
name = "create_github_installation_hooks"
jwt_token: JWTToken
project_cache_file: Path
installation_token_map_name: Path
webhook_secret: str
webhook_url: str
topic: str | None
def __init__(
self,
jwt_token: JWTToken,
project_cache_file: Path,
installation_token_map_name: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
**kwargs: Any,
) -> None:
self.jwt_token = jwt_token
self.project_cache_file = project_cache_file
self.installation_token_map_name = installation_token_map_name
self.webhook_secret = webhook_secret
self.webhook_url = webhook_url
self.topic = topic
super().__init__(**kwargs)
def run_deferred(self) -> None:
repos = json.loads(self.project_cache_file.read_text())
installation_token_map: dict[int, InstallationToken] = dict(
starmap(
lambda k, v: (
int(k),
InstallationToken.from_json(
self.jwt_token, int(k), self.installation_token_map_name, v
),
),
json.loads(self.installation_token_map_name.read_text()).items(),
)
)
for repo in repos:
create_project_hook(
installation_token_map[repo["installation_id"]],
self.webhook_secret,
repo["owner"]["login"],
repo["name"],
self.webhook_url,
)
def run_post(self) -> Any:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
class ReloadGithubInstallations(ThreadDeferredBuildStep):
name = "reload_github_projects"
jwt_token: JWTToken
project_cache_file: Path
installation_token_map_name: Path
project_id_map_name: Path
topic: str | None
def __init__(
self,
@ -71,15 +124,17 @@ class ReloadGithubInstallations(BuildStep):
project_cache_file: Path,
installation_token_map_name: Path,
project_id_map_name: Path,
topic: str | None,
**kwargs: Any,
) -> None:
self.jwt_token = jwt_token
self.installation_token_map_name = installation_token_map_name
self.project_id_map_name = project_id_map_name
self.project_cache_file = project_cache_file
self.topic = topic
super().__init__(**kwargs)
def reload_projects(self) -> None:
def run_deferred(self) -> None:
installation_token_map = GithubBackend.create_missing_installations(
self.jwt_token,
self.installation_token_map_name,
@ -96,89 +151,110 @@ class ReloadGithubInstallations(BuildStep):
repos = []
for k, v in installation_token_map.items():
new_repos = refresh_projects(
v.get(),
self.project_cache_file,
clear=True,
api_endpoint="/installation/repositories",
subkey="repositories",
require_admin=False,
new_repos = filter_repos_by_topic(
self.topic,
refresh_projects(
v.get(),
self.project_cache_file,
clear=True,
api_endpoint="/installation/repositories",
subkey="repositories",
require_admin=False,
),
lambda repo: repo["topics"],
)
for repo in new_repos:
repo["installation_id"] = k
repos.extend(new_repos)
for repo in new_repos:
project_id_map[repo["full_name"]] = k
repos.extend(new_repos)
atomic_write_file(self.project_cache_file, json.dumps(repos))
atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
tlog.info(
f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token."
f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation tokens."
)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
def run_post(self) -> Any:
return util.SUCCESS
class ReloadGithubProjects(BuildStep):
name = "reload_github_projects"
class CreateGitHubProjectHooks(ThreadDeferredBuildStep):
name = "create_github_project_hooks"
token: RepoToken
project_cache_file: Path
webhook_secret: str
webhook_url: str
topic: str | None
def __init__(
self, token: RepoToken, project_cache_file: Path, **kwargs: Any
self,
token: RepoToken,
project_cache_file: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
**kwargs: Any,
) -> None:
self.token = token
self.project_cache_file = project_cache_file
self.webhook_secret = webhook_secret
self.webhook_url = webhook_url
self.topic = topic
super().__init__(**kwargs)
def reload_projects(self) -> None:
repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file)
def run_deferred(self) -> None:
repos = json.loads(self.project_cache_file.read_text())
log.msg(repos)
for repo in repos:
create_project_hook(
self.token,
self.webhook_secret,
repo["owner"]["login"],
repo["name"],
self.webhook_url,
)
def run_post(self) -> Any:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
class ReloadGithubProjects(ThreadDeferredBuildStep):
name = "reload_github_projects"
token: RepoToken
project_cache_file: Path
topic: str | None
def __init__(
self,
token: RepoToken,
project_cache_file: Path,
topic: str | None,
**kwargs: Any,
) -> None:
self.token = token
self.project_cache_file = project_cache_file
self.topic = topic
super().__init__(**kwargs)
def run_deferred(self) -> None:
repos: list[Any] = filter_repos_by_topic(
self.topic,
refresh_projects(self.token.get(), self.project_cache_file),
lambda repo: repo["topics"],
)
atomic_write_file(self.project_cache_file, json.dumps(repos))
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
self.error_msg = ""
def error_cb(failure: Failure) -> int:
self.error_msg += failure.getTraceback()
return util.FAILURE
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
res = yield d
if res == util.SUCCESS:
# reload the buildbot config
os.kill(os.getpid(), signal.SIGHUP)
return util.SUCCESS
else:
log: StreamLog = yield self.addLog("log")
log.addStderr(f"Failed to reload project list: {self.error_msg}")
return util.FAILURE
def run_post(self) -> Any:
return util.SUCCESS
class GitHubAppStatusPush(GitHubStatusPush):
@ -307,7 +383,13 @@ class GithubAuthBackend(ABC):
pass
@abstractmethod
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
def create_reload_builder_steps(
self,
project_cache_file: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
) -> list[BuildStep]:
pass
@ -338,10 +420,27 @@ class GithubLegacyAuthBackend(GithubAuthBackend):
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
return ReloadGithubProjects(
token=self.token, project_cache_file=project_cache_file
)
def create_reload_builder_steps(
self,
project_cache_file: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
) -> list[BuildStep]:
return [
ReloadGithubProjects(
token=self.token,
project_cache_file=project_cache_file,
topic=topic,
),
CreateGitHubProjectHooks(
token=self.token,
project_cache_file=project_cache_file,
webhook_secret=webhook_secret,
webhook_url=webhook_url,
topic=topic,
),
]
class GitHubLegacySecretService(SecretProviderBase):
@ -406,13 +505,30 @@ class GithubAppAuthBackend(GithubAuthBackend):
context=Interpolate("buildbot/%(prop:status_name)s"),
)
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
return ReloadGithubInstallations(
self.jwt_token,
project_cache_file,
self.auth_type.app_installation_token_map_name,
self.auth_type.app_project_id_map_name,
)
def create_reload_builder_steps(
self,
project_cache_file: Path,
webhook_secret: str,
webhook_url: str,
topic: str | None,
) -> list[BuildStep]:
return [
ReloadGithubInstallations(
self.jwt_token,
project_cache_file,
self.auth_type.app_installation_token_map_name,
self.auth_type.app_project_id_map_name,
topic,
),
CreateGitHubInstallationHooks(
self.jwt_token,
project_cache_file,
self.auth_type.app_installation_token_map_name,
webhook_secret=webhook_secret,
webhook_url=webhook_url,
topic=topic,
),
]
class GitHubAppSecretService(SecretProviderBase):
@ -453,12 +569,14 @@ class GithubConfig:
class GithubBackend(GitBackend):
config: GithubConfig
webhook_secret: str
webhook_url: str
auth_backend: GithubAuthBackend
def __init__(self, config: GithubConfig) -> None:
def __init__(self, config: GithubConfig, webhook_url: str) -> None:
self.config = config
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
self.webhook_url = webhook_url
if isinstance(self.config.auth_type, AuthTypeLegacy):
self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
@ -480,13 +598,8 @@ class GithubBackend(GitBackend):
installations_map: dict[int, InstallationToken] = {}
for iid, installation in initial_installations_map.items():
token: str = installation["token"]
expiration: datetime = datetime.fromisoformat(installation["expiration"])
installations_map[int(iid)] = InstallationToken(
jwt_token,
int(iid),
installations_token_map_name,
installation_token=(token, expiration),
installations_map[int(iid)] = InstallationToken.from_json(
jwt_token, int(iid), installations_token_map_name, installation
)
return installations_map
@ -499,7 +612,7 @@ class GithubBackend(GitBackend):
installations: list[int],
) -> dict[int, InstallationToken]:
for installation in set(installations) - installations_map.keys():
installations_map[installation] = InstallationToken(
installations_map[installation] = InstallationToken.new(
jwt_token,
installation,
installations_token_map_name,
@ -510,9 +623,15 @@ class GithubBackend(GitBackend):
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it."""
factory = util.BuildFactory()
factory.addStep(
self.auth_backend.create_reload_builder_step(self.config.project_cache_file)
steps = self.auth_backend.create_reload_builder_steps(
self.config.project_cache_file,
self.webhook_secret,
self.webhook_url,
self.config.topic,
)
for step in steps:
factory.addStep(step)
return util.BuilderConfig(
name=self.reload_builder_name,
workernames=worker_names,
@ -563,9 +682,13 @@ class GithubBackend(GitBackend):
if not self.config.project_cache_file.exists():
return []
repos: list[dict[str, Any]] = sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
repos: list[dict[str, Any]] = filter_repos_by_topic(
self.config.topic,
sorted(
json.loads(self.config.project_cache_file.read_text()),
key=lambda x: x["full_name"],
),
lambda repo: repo["topics"],
)
if isinstance(self.auth_backend, GithubAppAuthBackend):
@ -580,22 +703,22 @@ class GithubBackend(GitBackend):
tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
repos = list(filter(lambda repo: "installation_id" in repo, repos))
tlog.info(f"Loading {len(repos)} cached repositories.")
return list(
filter(
lambda project: self.config.topic is not None
and self.config.topic in project.topics,
[
GithubProject(
self.auth_backend.get_repo_token(repo),
self.config,
self.webhook_secret,
repo,
)
for repo in repos
],
)
repo_names: list[str] = [
repo["owner"]["login"] + "/" + repo["name"] for repo in repos
]
tlog.info(
f"Loading {len(repos)} cached repositories: [{', '.join(repo_names)}]"
)
return [
GithubProject(
self.auth_backend.get_repo_token(repo),
self.config,
self.webhook_secret,
repo,
)
for repo in repos
]
def are_projects_cached(self) -> bool:
if not self.config.project_cache_file.exists():
@ -632,6 +755,39 @@ class GithubBackend(GitBackend):
return "github"
def create_project_hook(
token: RepoToken, webhook_secret: str, owner: str, repo: str, webhook_url: str
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
token.get(),
)
config = dict(
url=webhook_url + "change_hook/github",
content_type="json",
insecure_ssl="0",
secret=webhook_secret,
)
data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
headers = {
"Authorization": f"Bearer {token.get()}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/github":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
class GithubProject(GitProject):
config: GithubConfig
webhook_secret: str
@ -650,43 +806,6 @@ class GithubProject(GitProject):
self.webhook_secret = webhook_secret
self.data = data
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
hooks = paginated_github_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
self.token.get(),
)
config = dict(
url=webhook_url + "change_hook/github",
content_type="json",
insecure_ssl="0",
secret=self.webhook_secret,
)
data = dict(
name="web", active=True, events=["push", "pull_request"], config=config
)
headers = {
"Authorization": f"Bearer {self.token.get()}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/github":
log.msg(f"hook for {owner}/{repo} already exists")
return
http_request(
f"https://api.github.com/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
)
def get_project_url(self) -> str:
return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}"

9
buildbot_nix/projects.py
View file

@ -62,15 +62,6 @@ class GitBackend(ABC):
class GitProject(ABC):
@abstractmethod
def create_project_hook(
self,
owner: str,
repo: str,
webhook_url: str,
) -> None:
pass
@abstractmethod
def get_project_url(self) -> str:
pass