commit
7e327b88b6
26
README.md
26
README.md
|
@ -66,16 +66,36 @@ We have the following two roles:
|
||||||
|
|
||||||
### Integration with GitHub
|
### Integration with GitHub
|
||||||
|
|
||||||
To integrate with GitHub:
|
#### GitHub App
|
||||||
|
|
||||||
|
To integrate with GitHub using app authentication:
|
||||||
|
|
||||||
|
1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
|
||||||
|
authentication on the Buildbot dashboard. Enable the following permissions:
|
||||||
|
- Contents: Read-only
|
||||||
|
- Metadata: Read-only
|
||||||
|
- Commit statuses: Read and write
|
||||||
|
- Webhooks: Read and write
|
||||||
|
2. **GitHub App private key**: Get the app private key and app ID from GitHub,
|
||||||
|
configure using the buildbot-nix NixOS module.
|
||||||
|
3. **Install App**: Install the app for an organization or specific user.
|
||||||
|
4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to
|
||||||
|
changes (new repositories or installations) automatically, it is therefore
|
||||||
|
necessary to manually trigger a reload or wait for the next periodic reload.
|
||||||
|
|
||||||
|
#### Legacy Token Auth
|
||||||
|
|
||||||
|
To integrate with GitHub using legacy token authentication:
|
||||||
|
|
||||||
1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo`
|
1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo`
|
||||||
permissions. For GitHub organizations, it's advisable to create a separate
|
permissions. For GitHub organizations, it's advisable to create a separate
|
||||||
GitHub user for managing repository webhooks.
|
GitHub user for managing repository webhooks.
|
||||||
|
|
||||||
#### Optional when using GitHub login
|
### Optional when using GitHub login
|
||||||
|
|
||||||
1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
|
1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
|
||||||
authentication on the Buildbot dashboard.
|
authentication on the Buildbot dashboard. (can be the same as for GitHub App
|
||||||
|
auth)
|
||||||
2. **OAuth Credentials**: After installing the app, generate OAuth credentials
|
2. **OAuth Credentials**: After installing the app, generate OAuth credentials
|
||||||
and configure them in the buildbot-nix NixOS module. Set the callback url to
|
and configure them in the buildbot-nix NixOS module. Set the callback url to
|
||||||
`https://<your-domain>/auth/login`.
|
`https://<your-domain>/auth/login`.
|
||||||
|
|
|
@ -872,6 +872,8 @@ class NixConfigurator(ConfiguratorBase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
config["services"].append(backend.create_reporter())
|
config["services"].append(backend.create_reporter())
|
||||||
|
config.setdefault("secretProviders", [])
|
||||||
|
config["secretsProviders"].extend(backend.create_secret_providers())
|
||||||
|
|
||||||
systemd_secrets = SecretInAFile(
|
systemd_secrets = SecretInAFile(
|
||||||
dirname=os.environ["CREDENTIALS_DIRECTORY"],
|
dirname=os.environ["CREDENTIALS_DIRECTORY"],
|
||||||
|
@ -887,13 +889,15 @@ class NixConfigurator(ConfiguratorBase):
|
||||||
backend.create_change_hook()
|
backend.create_change_hook()
|
||||||
)
|
)
|
||||||
|
|
||||||
if "auth" not in config["www"]:
|
config["www"].setdefault("avatar_methods", [])
|
||||||
config["www"].setdefault("avatar_methods", [])
|
|
||||||
|
|
||||||
for backend in backends.values():
|
for backend in backends.values():
|
||||||
avatar_method = backend.create_avatar_method()
|
avatar_method = backend.create_avatar_method()
|
||||||
if avatar_method is not None:
|
print(avatar_method)
|
||||||
config["www"]["avatar_methods"].append(avatar_method)
|
if avatar_method is not None:
|
||||||
|
config["www"]["avatar_methods"].append(avatar_method)
|
||||||
|
|
||||||
|
if "auth" not in config["www"]:
|
||||||
# TODO one cannot have multiple auth backends...
|
# TODO one cannot have multiple auth backends...
|
||||||
if auth is not None:
|
if auth is not None:
|
||||||
config["www"]["auth"] = auth
|
config["www"]["auth"] = auth
|
||||||
|
|
|
@ -2,6 +2,8 @@ import contextlib
|
||||||
import http.client
|
import http.client
|
||||||
import json
|
import json
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
from tempfile import NamedTemporaryFile
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
@ -9,7 +11,9 @@ def slugify_project_name(name: str) -> str:
|
||||||
return name.replace(".", "-").replace("/", "-")
|
return name.replace(".", "-").replace("/", "-")
|
||||||
|
|
||||||
|
|
||||||
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
|
def paginated_github_request(
|
||||||
|
url: str, token: str, subkey: None | str = None
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
next_url: str | None = url
|
next_url: str | None = url
|
||||||
items = []
|
items = []
|
||||||
while next_url:
|
while next_url:
|
||||||
|
@ -29,7 +33,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
|
||||||
link_parts = link.split(";")
|
link_parts = link.split(";")
|
||||||
if link_parts[1].strip() == 'rel="next"':
|
if link_parts[1].strip() == 'rel="next"':
|
||||||
next_url = link_parts[0][1:-1]
|
next_url = link_parts[0][1:-1]
|
||||||
items += res.json()
|
if subkey is not None:
|
||||||
|
items += res.json()[subkey]
|
||||||
|
else:
|
||||||
|
items += res.json()
|
||||||
return items
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
@ -78,3 +85,15 @@ def http_request(
|
||||||
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
|
msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
|
||||||
raise HttpError(msg) from e
|
raise HttpError(msg) from e
|
||||||
return HttpResponse(resp)
|
return HttpResponse(resp)
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_write_file(file: Path, data: str) -> None:
|
||||||
|
with NamedTemporaryFile("w", delete=False, dir=file.parent) as f:
|
||||||
|
path = Path(f.name)
|
||||||
|
try:
|
||||||
|
f.write(data)
|
||||||
|
f.flush()
|
||||||
|
path.rename(file)
|
||||||
|
except OSError:
|
||||||
|
path.unlink()
|
||||||
|
raise
|
||||||
|
|
1
buildbot_nix/github/__init__.py
Normal file
1
buildbot_nix/github/__init__.py
Normal file
|
@ -0,0 +1 @@
|
||||||
|
|
1
buildbot_nix/github/auth/__init__.py
Normal file
1
buildbot_nix/github/auth/__init__.py
Normal file
|
@ -0,0 +1 @@
|
||||||
|
|
23
buildbot_nix/github/auth/_type.py
Normal file
23
buildbot_nix/github/auth/_type.py
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AuthType:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AuthTypeLegacy(AuthType):
|
||||||
|
token_secret_name: str = "github-token"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AuthTypeApp(AuthType):
|
||||||
|
app_id: int
|
||||||
|
app_secret_key_name: str = "github-app-secret-key"
|
||||||
|
app_installation_token_map_name: Path = Path(
|
||||||
|
"github-app-installation-token-map.json"
|
||||||
|
)
|
||||||
|
app_project_id_map_name: Path = Path("github-app-project-id-map-name.json")
|
||||||
|
app_jwt_token_name: Path = Path("github-app-jwt-token")
|
108
buildbot_nix/github/installation_token.py
Normal file
108
buildbot_nix/github/installation_token.py
Normal file
|
@ -0,0 +1,108 @@
|
||||||
|
import json
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from buildbot_nix.common import (
|
||||||
|
HttpResponse,
|
||||||
|
atomic_write_file,
|
||||||
|
http_request,
|
||||||
|
)
|
||||||
|
|
||||||
|
from .jwt_token import JWTToken
|
||||||
|
from .repo_token import RepoToken
|
||||||
|
|
||||||
|
|
||||||
|
class InstallationToken(RepoToken):
|
||||||
|
GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
|
||||||
|
|
||||||
|
jwt_token: JWTToken
|
||||||
|
installation_id: int
|
||||||
|
|
||||||
|
token: str
|
||||||
|
expiration: datetime
|
||||||
|
installations_token_map_name: Path
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _create_installation_access_token(
|
||||||
|
jwt_token: JWTToken, installation_id: int
|
||||||
|
) -> HttpResponse:
|
||||||
|
return http_request(
|
||||||
|
f"https://api.github.com/app/installations/{installation_id}/access_tokens",
|
||||||
|
data={},
|
||||||
|
headers={"Authorization": f"Bearer {jwt_token.get()}"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _generate_token(
|
||||||
|
jwt_token: JWTToken, installation_id: int
|
||||||
|
) -> tuple[str, datetime]:
|
||||||
|
token = InstallationToken._create_installation_access_token(
|
||||||
|
jwt_token, installation_id
|
||||||
|
).json()["token"]
|
||||||
|
expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME
|
||||||
|
|
||||||
|
return token, expiration
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
jwt_token: JWTToken,
|
||||||
|
installation_id: int,
|
||||||
|
installations_token_map_name: Path,
|
||||||
|
installation_token: None | tuple[str, datetime] = None,
|
||||||
|
) -> None:
|
||||||
|
self.jwt_token = jwt_token
|
||||||
|
self.installation_id = installation_id
|
||||||
|
self.installations_token_map_name = installations_token_map_name
|
||||||
|
|
||||||
|
if installation_token is None:
|
||||||
|
self.token, self.expiration = InstallationToken._generate_token(
|
||||||
|
self.jwt_token, self.installation_id
|
||||||
|
)
|
||||||
|
self._save()
|
||||||
|
else:
|
||||||
|
self.token, self.expiration = installation_token
|
||||||
|
|
||||||
|
def get(self) -> str:
|
||||||
|
self.verify()
|
||||||
|
return self.token
|
||||||
|
|
||||||
|
def get_as_secret(self) -> str:
|
||||||
|
return f"%(secret:github-token-{self.installation_id})"
|
||||||
|
|
||||||
|
def verify(self) -> None:
|
||||||
|
if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8:
|
||||||
|
self.token, self.expiration = InstallationToken._generate_token(
|
||||||
|
self.jwt_token, self.installation_id
|
||||||
|
)
|
||||||
|
self._save()
|
||||||
|
|
||||||
|
def _save(self) -> None:
|
||||||
|
# of format:
|
||||||
|
# {
|
||||||
|
# 123: {
|
||||||
|
# expiration: <datetime>,
|
||||||
|
# token: "token"
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
installations_token_map: dict[str, Any]
|
||||||
|
if self.installations_token_map_name.exists():
|
||||||
|
installations_token_map = json.loads(
|
||||||
|
self.installations_token_map_name.read_text()
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
installations_token_map = {}
|
||||||
|
|
||||||
|
installations_token_map.update(
|
||||||
|
{
|
||||||
|
str(self.installation_id): {
|
||||||
|
"expiration": self.expiration.isoformat(),
|
||||||
|
"token": self.token,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
atomic_write_file(
|
||||||
|
self.installations_token_map_name, json.dumps(installations_token_map)
|
||||||
|
)
|
84
buildbot_nix/github/jwt_token.py
Normal file
84
buildbot_nix/github/jwt_token.py
Normal file
|
@ -0,0 +1,84 @@
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .repo_token import RepoToken
|
||||||
|
|
||||||
|
|
||||||
|
class JWTToken(RepoToken):
|
||||||
|
app_id: int
|
||||||
|
app_private_key: str
|
||||||
|
lifetime: timedelta
|
||||||
|
|
||||||
|
expiration: datetime
|
||||||
|
token: str
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
app_id: int,
|
||||||
|
app_private_key: str,
|
||||||
|
lifetime: timedelta = timedelta(minutes=10),
|
||||||
|
) -> None:
|
||||||
|
self.app_id = app_id
|
||||||
|
self.app_private_key = app_private_key
|
||||||
|
self.lifetime = lifetime
|
||||||
|
|
||||||
|
self.token, self.expiration = JWTToken.generate_token(
|
||||||
|
self.app_id, self.app_private_key, lifetime
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def generate_token(
|
||||||
|
app_id: int, app_private_key: str, lifetime: timedelta
|
||||||
|
) -> tuple[str, datetime]:
|
||||||
|
def build_jwt_payload(
|
||||||
|
app_id: int, lifetime: timedelta
|
||||||
|
) -> tuple[dict[str, Any], datetime]:
|
||||||
|
jwt_iat_drift: timedelta = timedelta(seconds=60)
|
||||||
|
now: datetime = datetime.now(tz=UTC)
|
||||||
|
iat: datetime = now - jwt_iat_drift
|
||||||
|
exp: datetime = iat + lifetime
|
||||||
|
jwt_payload = {
|
||||||
|
"iat": int(iat.timestamp()),
|
||||||
|
"exp": int(exp.timestamp()),
|
||||||
|
"iss": str(app_id),
|
||||||
|
}
|
||||||
|
return (jwt_payload, exp)
|
||||||
|
|
||||||
|
def rs256_sign(data: str, private_key: str) -> str:
|
||||||
|
signature = subprocess.run(
|
||||||
|
["openssl", "dgst", "-binary", "-sha256", "-sign", private_key],
|
||||||
|
input=data.encode("utf-8"),
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
check=True,
|
||||||
|
cwd=os.environ.get("CREDENTIALS_DIRECTORY"),
|
||||||
|
).stdout
|
||||||
|
return base64url(signature)
|
||||||
|
|
||||||
|
def base64url(data: bytes) -> str:
|
||||||
|
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8")
|
||||||
|
|
||||||
|
jwt, expiration = build_jwt_payload(app_id, lifetime)
|
||||||
|
jwt_payload = json.dumps(jwt).encode("utf-8")
|
||||||
|
json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8")
|
||||||
|
encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}"
|
||||||
|
encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key)
|
||||||
|
return (f"{encoded_jwt_parts}.{encoded_mac}", expiration)
|
||||||
|
|
||||||
|
# installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt)
|
||||||
|
|
||||||
|
# return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations))
|
||||||
|
|
||||||
|
def get(self) -> str:
|
||||||
|
if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8:
|
||||||
|
self.token, self.expiration = JWTToken.generate_token(
|
||||||
|
self.app_id, self.app_private_key, self.lifetime
|
||||||
|
)
|
||||||
|
|
||||||
|
return self.token
|
||||||
|
|
||||||
|
def get_as_secret(self) -> str:
|
||||||
|
return "%(secret:github-jwt-token)"
|
14
buildbot_nix/github/legacy_token.py
Normal file
14
buildbot_nix/github/legacy_token.py
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
from .repo_token import RepoToken
|
||||||
|
|
||||||
|
|
||||||
|
class LegacyToken(RepoToken):
|
||||||
|
token: str
|
||||||
|
|
||||||
|
def __init__(self, token: str) -> None:
|
||||||
|
self.token = token
|
||||||
|
|
||||||
|
def get(self) -> str:
|
||||||
|
return self.token
|
||||||
|
|
||||||
|
def get_as_secret(self) -> str:
|
||||||
|
return "%(secret:github-token)"
|
11
buildbot_nix/github/repo_token.py
Normal file
11
buildbot_nix/github/repo_token.py
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
from abc import abstractmethod
|
||||||
|
|
||||||
|
|
||||||
|
class RepoToken:
|
||||||
|
@abstractmethod
|
||||||
|
def get(self) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_as_secret(self) -> str:
|
||||||
|
pass
|
|
@ -1,10 +1,13 @@
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import signal
|
import signal
|
||||||
from collections.abc import Generator
|
from abc import ABC, abstractmethod
|
||||||
|
from collections.abc import Callable, Generator
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from datetime import (
|
||||||
|
datetime,
|
||||||
|
)
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tempfile import NamedTemporaryFile
|
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from buildbot.config.builder import BuilderConfig
|
from buildbot.config.builder import BuilderConfig
|
||||||
|
@ -13,10 +16,12 @@ from buildbot.process.buildstep import BuildStep
|
||||||
from buildbot.process.properties import Interpolate
|
from buildbot.process.properties import Interpolate
|
||||||
from buildbot.reporters.base import ReporterBase
|
from buildbot.reporters.base import ReporterBase
|
||||||
from buildbot.reporters.github import GitHubStatusPush
|
from buildbot.reporters.github import GitHubStatusPush
|
||||||
|
from buildbot.secrets.providers.base import SecretProviderBase
|
||||||
from buildbot.www.auth import AuthBase
|
from buildbot.www.auth import AuthBase
|
||||||
from buildbot.www.avatar import AvatarBase, AvatarGitHub
|
from buildbot.www.avatar import AvatarBase, AvatarGitHub
|
||||||
from buildbot.www.oauth2 import GitHubAuth
|
from buildbot.www.oauth2 import GitHubAuth
|
||||||
from twisted.internet import defer, threads
|
from twisted.internet import defer, threads
|
||||||
|
from twisted.logger import Logger
|
||||||
from twisted.python import log
|
from twisted.python import log
|
||||||
from twisted.python.failure import Failure
|
from twisted.python.failure import Failure
|
||||||
|
|
||||||
|
@ -24,24 +29,96 @@ if TYPE_CHECKING:
|
||||||
from buildbot.process.log import StreamLog
|
from buildbot.process.log import StreamLog
|
||||||
|
|
||||||
from .common import (
|
from .common import (
|
||||||
|
atomic_write_file,
|
||||||
http_request,
|
http_request,
|
||||||
paginated_github_request,
|
paginated_github_request,
|
||||||
slugify_project_name,
|
slugify_project_name,
|
||||||
)
|
)
|
||||||
|
from .github.auth._type import AuthType, AuthTypeApp, AuthTypeLegacy
|
||||||
|
from .github.installation_token import InstallationToken
|
||||||
|
from .github.jwt_token import JWTToken
|
||||||
|
from .github.legacy_token import (
|
||||||
|
LegacyToken,
|
||||||
|
)
|
||||||
|
from .github.repo_token import (
|
||||||
|
RepoToken,
|
||||||
|
)
|
||||||
from .projects import GitBackend, GitProject
|
from .projects import GitBackend, GitProject
|
||||||
from .secrets import read_secret_file
|
from .secrets import read_secret_file
|
||||||
|
|
||||||
|
tlog = Logger()
|
||||||
|
|
||||||
class ReloadGithubProjects(BuildStep):
|
|
||||||
|
def get_installations(jwt_token: JWTToken) -> list[int]:
|
||||||
|
installations = paginated_github_request(
|
||||||
|
"https://api.github.com/app/installations?per_page=100", jwt_token.get()
|
||||||
|
)
|
||||||
|
|
||||||
|
return [installation["id"] for installation in installations]
|
||||||
|
|
||||||
|
|
||||||
|
class ReloadGithubInstallations(BuildStep):
|
||||||
name = "reload_github_projects"
|
name = "reload_github_projects"
|
||||||
|
|
||||||
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
|
jwt_token: JWTToken
|
||||||
self.token = token
|
project_cache_file: Path
|
||||||
|
installation_token_map_name: Path
|
||||||
|
project_id_map_name: Path
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
jwt_token: JWTToken,
|
||||||
|
project_cache_file: Path,
|
||||||
|
installation_token_map_name: Path,
|
||||||
|
project_id_map_name: Path,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> None:
|
||||||
|
self.jwt_token = jwt_token
|
||||||
|
self.installation_token_map_name = installation_token_map_name
|
||||||
|
self.project_id_map_name = project_id_map_name
|
||||||
self.project_cache_file = project_cache_file
|
self.project_cache_file = project_cache_file
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
def reload_projects(self) -> None:
|
def reload_projects(self) -> None:
|
||||||
refresh_projects(self.token, self.project_cache_file)
|
installation_token_map = GithubBackend.create_missing_installations(
|
||||||
|
self.jwt_token,
|
||||||
|
self.installation_token_map_name,
|
||||||
|
GithubBackend.load_installations(
|
||||||
|
self.jwt_token,
|
||||||
|
self.installation_token_map_name,
|
||||||
|
),
|
||||||
|
get_installations(self.jwt_token),
|
||||||
|
)
|
||||||
|
|
||||||
|
repos: list[Any] = []
|
||||||
|
project_id_map: dict[str, int] = {}
|
||||||
|
|
||||||
|
repos = []
|
||||||
|
|
||||||
|
for k, v in installation_token_map.items():
|
||||||
|
new_repos = refresh_projects(
|
||||||
|
v.get(),
|
||||||
|
self.project_cache_file,
|
||||||
|
clear=True,
|
||||||
|
api_endpoint="/installation/repositories",
|
||||||
|
subkey="repositories",
|
||||||
|
require_admin=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
for repo in new_repos:
|
||||||
|
repo["installation_id"] = k
|
||||||
|
|
||||||
|
repos.extend(new_repos)
|
||||||
|
|
||||||
|
for repo in new_repos:
|
||||||
|
project_id_map[repo["full_name"]] = k
|
||||||
|
|
||||||
|
atomic_write_file(self.project_cache_file, json.dumps(repos))
|
||||||
|
atomic_write_file(self.project_id_map_name, json.dumps(project_id_map))
|
||||||
|
|
||||||
|
tlog.info(
|
||||||
|
f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation token."
|
||||||
|
)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
def run(self) -> Generator[Any, object, Any]:
|
||||||
|
@ -65,36 +142,378 @@ class ReloadGithubProjects(BuildStep):
|
||||||
return util.FAILURE
|
return util.FAILURE
|
||||||
|
|
||||||
|
|
||||||
|
class ReloadGithubProjects(BuildStep):
|
||||||
|
name = "reload_github_projects"
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, token: RepoToken, project_cache_file: Path, **kwargs: Any
|
||||||
|
) -> None:
|
||||||
|
self.token = token
|
||||||
|
self.project_cache_file = project_cache_file
|
||||||
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
|
def reload_projects(self) -> None:
|
||||||
|
repos: list[Any] = refresh_projects(self.token.get(), self.project_cache_file)
|
||||||
|
|
||||||
|
log.msg(repos)
|
||||||
|
|
||||||
|
atomic_write_file(self.project_cache_file, json.dumps(repos))
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def run(self) -> Generator[Any, object, Any]:
|
||||||
|
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
|
||||||
|
|
||||||
|
self.error_msg = ""
|
||||||
|
|
||||||
|
def error_cb(failure: Failure) -> int:
|
||||||
|
self.error_msg += failure.getTraceback()
|
||||||
|
return util.FAILURE
|
||||||
|
|
||||||
|
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
|
||||||
|
res = yield d
|
||||||
|
if res == util.SUCCESS:
|
||||||
|
# reload the buildbot config
|
||||||
|
os.kill(os.getpid(), signal.SIGHUP)
|
||||||
|
return util.SUCCESS
|
||||||
|
else:
|
||||||
|
log: StreamLog = yield self.addLog("log")
|
||||||
|
log.addStderr(f"Failed to reload project list: {self.error_msg}")
|
||||||
|
return util.FAILURE
|
||||||
|
|
||||||
|
|
||||||
|
class GitHubAppStatusPush(GitHubStatusPush):
|
||||||
|
token_source: Callable[[int], RepoToken]
|
||||||
|
project_id_source: Callable[[str], int]
|
||||||
|
saved_args: dict[str, Any]
|
||||||
|
saved_kwargs: dict[str, Any]
|
||||||
|
|
||||||
|
def checkConfig(
|
||||||
|
self,
|
||||||
|
token_source: Callable[[int], RepoToken],
|
||||||
|
project_id_source: Callable[[str], int],
|
||||||
|
context: Any = None,
|
||||||
|
baseURL: Any = None,
|
||||||
|
verbose: Any = False,
|
||||||
|
debug: Any = None,
|
||||||
|
verify: Any = None,
|
||||||
|
generators: Any = None,
|
||||||
|
**kwargs: dict[str, Any],
|
||||||
|
) -> Any:
|
||||||
|
if generators is None:
|
||||||
|
generators = self._create_default_generators()
|
||||||
|
|
||||||
|
if "token" in kwargs:
|
||||||
|
del kwargs["token"]
|
||||||
|
super().checkConfig(
|
||||||
|
token="",
|
||||||
|
context=context,
|
||||||
|
baseURL=baseURL,
|
||||||
|
verbose=verbose,
|
||||||
|
debug=debug,
|
||||||
|
verify=verify,
|
||||||
|
generators=generators,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def reconfigService(
|
||||||
|
self,
|
||||||
|
token_source: Callable[[int], RepoToken],
|
||||||
|
project_id_source: Callable[[str], int],
|
||||||
|
context: Any = None,
|
||||||
|
baseURL: Any = None,
|
||||||
|
verbose: Any = False,
|
||||||
|
debug: Any = None,
|
||||||
|
verify: Any = None,
|
||||||
|
generators: Any = None,
|
||||||
|
**kwargs: dict[str, Any],
|
||||||
|
) -> Any:
|
||||||
|
if "saved_args" not in self or self.saved_args is None:
|
||||||
|
self.saved_args = {}
|
||||||
|
self.token_source = token_source
|
||||||
|
self.project_id_source = project_id_source
|
||||||
|
self.saved_kwargs = kwargs
|
||||||
|
self.saved_args["context"] = context
|
||||||
|
self.saved_args["baseURL"] = baseURL
|
||||||
|
self.saved_args["verbose"] = verbose
|
||||||
|
self.saved_args["debug"] = debug
|
||||||
|
self.saved_args["verify"] = verify
|
||||||
|
self.saved_args["generators"] = generators
|
||||||
|
|
||||||
|
if generators is None:
|
||||||
|
generators = self._create_default_generators()
|
||||||
|
|
||||||
|
if "token" in kwargs:
|
||||||
|
del kwargs["token"]
|
||||||
|
super().reconfigService(
|
||||||
|
token="",
|
||||||
|
context=context,
|
||||||
|
baseURL=baseURL,
|
||||||
|
verbose=verbose,
|
||||||
|
debug=debug,
|
||||||
|
verify=verify,
|
||||||
|
generators=generators,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def sendMessage(self, reports: Any) -> Any:
|
||||||
|
build = reports[0]["builds"][0]
|
||||||
|
sourcestamps = build["buildset"].get("sourcestamps")
|
||||||
|
if not sourcestamps:
|
||||||
|
return None
|
||||||
|
|
||||||
|
for sourcestamp in sourcestamps:
|
||||||
|
build["buildset"]["sourcestamps"] = [sourcestamp]
|
||||||
|
|
||||||
|
token: str
|
||||||
|
|
||||||
|
if "project" in sourcestamp and sourcestamp["project"] != "":
|
||||||
|
token = self.token_source(
|
||||||
|
self.project_id_source(sourcestamp["project"])
|
||||||
|
).get()
|
||||||
|
else:
|
||||||
|
token = ""
|
||||||
|
|
||||||
|
super().reconfigService(
|
||||||
|
token,
|
||||||
|
context=self.saved_args["context"],
|
||||||
|
baseURL=self.saved_args["baseURL"],
|
||||||
|
verbose=self.saved_args["verbose"],
|
||||||
|
debug=self.saved_args["debug"],
|
||||||
|
verify=self.saved_args["verify"],
|
||||||
|
generators=self.saved_args["generators"],
|
||||||
|
**self.saved_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
return super().sendMessage(reports)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class GithubAuthBackend(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def get_general_token(self) -> RepoToken:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def create_secret_providers(self) -> list[SecretProviderBase]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def create_reporter(self) -> ReporterBase:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class GithubLegacyAuthBackend(GithubAuthBackend):
|
||||||
|
auth_type: AuthTypeLegacy
|
||||||
|
|
||||||
|
token: LegacyToken
|
||||||
|
|
||||||
|
def __init__(self, auth_type: AuthTypeLegacy) -> None:
|
||||||
|
self.auth_type = auth_type
|
||||||
|
self.token = LegacyToken(read_secret_file(auth_type.token_secret_name))
|
||||||
|
|
||||||
|
def get_general_token(self) -> RepoToken:
|
||||||
|
return self.token
|
||||||
|
|
||||||
|
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
|
||||||
|
return self.token
|
||||||
|
|
||||||
|
def create_secret_providers(self) -> list[SecretProviderBase]:
|
||||||
|
return [GitHubLegacySecretService(self.token)]
|
||||||
|
|
||||||
|
def create_reporter(self) -> ReporterBase:
|
||||||
|
return GitHubStatusPush(
|
||||||
|
token=self.token.get(),
|
||||||
|
# Since we dynamically create build steps,
|
||||||
|
# we use `virtual_builder_name` in the webinterface
|
||||||
|
# so that we distinguish what has beeing build
|
||||||
|
context=Interpolate("buildbot/%(prop:status_name)s"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
|
||||||
|
return ReloadGithubProjects(
|
||||||
|
token=self.token, project_cache_file=project_cache_file
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GitHubLegacySecretService(SecretProviderBase):
|
||||||
|
name = "GitHubLegacySecretService"
|
||||||
|
token: LegacyToken
|
||||||
|
|
||||||
|
def reconfigService(self, token: LegacyToken) -> None:
|
||||||
|
self.token = token
|
||||||
|
|
||||||
|
def get(self, entry: str) -> str | None:
|
||||||
|
"""
|
||||||
|
get the value from the file identified by 'entry'
|
||||||
|
"""
|
||||||
|
if entry.startswith("github-token"):
|
||||||
|
return self.token.get()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class GithubAppAuthBackend(GithubAuthBackend):
|
||||||
|
auth_type: AuthTypeApp
|
||||||
|
|
||||||
|
jwt_token: JWTToken
|
||||||
|
installation_tokens: dict[int, InstallationToken]
|
||||||
|
project_id_map: dict[str, int]
|
||||||
|
|
||||||
|
def __init__(self, auth_type: AuthTypeApp) -> None:
|
||||||
|
self.auth_type = auth_type
|
||||||
|
self.jwt_token = JWTToken(
|
||||||
|
self.auth_type.app_id, self.auth_type.app_secret_key_name
|
||||||
|
)
|
||||||
|
self.installation_tokens = GithubBackend.load_installations(
|
||||||
|
self.jwt_token,
|
||||||
|
self.auth_type.app_installation_token_map_name,
|
||||||
|
)
|
||||||
|
if self.auth_type.app_project_id_map_name.exists():
|
||||||
|
self.project_id_map = json.loads(
|
||||||
|
self.auth_type.app_project_id_map_name.read_text()
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
tlog.info(
|
||||||
|
"~project-id-map~ is not present, GitHub project reload will follow."
|
||||||
|
)
|
||||||
|
self.project_id_map = {}
|
||||||
|
|
||||||
|
def get_general_token(self) -> RepoToken:
|
||||||
|
return self.jwt_token
|
||||||
|
|
||||||
|
def get_repo_token(self, repo: dict[str, Any]) -> RepoToken:
|
||||||
|
assert "installation_id" in repo, f"Missing installation_id in {repo}"
|
||||||
|
return self.installation_tokens[repo["installation_id"]]
|
||||||
|
|
||||||
|
def create_secret_providers(self) -> list[SecretProviderBase]:
|
||||||
|
return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)]
|
||||||
|
|
||||||
|
def create_reporter(self) -> ReporterBase:
|
||||||
|
return GitHubAppStatusPush(
|
||||||
|
token_source=lambda iid: self.installation_tokens[iid],
|
||||||
|
project_id_source=lambda project: self.project_id_map[project],
|
||||||
|
# Since we dynamically create build steps,
|
||||||
|
# we use `virtual_builder_name` in the webinterface
|
||||||
|
# so that we distinguish what has beeing build
|
||||||
|
context=Interpolate("buildbot/%(prop:status_name)s"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_reload_builder_step(self, project_cache_file: Path) -> BuildStep:
|
||||||
|
return ReloadGithubInstallations(
|
||||||
|
self.jwt_token,
|
||||||
|
project_cache_file,
|
||||||
|
self.auth_type.app_installation_token_map_name,
|
||||||
|
self.auth_type.app_project_id_map_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GitHubAppSecretService(SecretProviderBase):
|
||||||
|
name = "GitHubAppSecretService"
|
||||||
|
installation_tokens: dict[int, InstallationToken]
|
||||||
|
jwt_token: JWTToken
|
||||||
|
|
||||||
|
def reconfigService(
|
||||||
|
self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken
|
||||||
|
) -> None:
|
||||||
|
self.installation_tokens = installation_tokens
|
||||||
|
self.jwt_token = jwt_token
|
||||||
|
|
||||||
|
def get(self, entry: str) -> str | None:
|
||||||
|
"""
|
||||||
|
get the value from the file identified by 'entry'
|
||||||
|
"""
|
||||||
|
if entry.startswith("github-token-"):
|
||||||
|
return self.installation_tokens[
|
||||||
|
int(entry.removeprefix("github-token-"))
|
||||||
|
].get()
|
||||||
|
if entry == "github-jwt-token":
|
||||||
|
return self.jwt_token.get()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class GithubConfig:
|
class GithubConfig:
|
||||||
oauth_id: str | None
|
oauth_id: str | None
|
||||||
|
auth_type: AuthType
|
||||||
# TODO unused
|
# TODO unused
|
||||||
buildbot_user: str
|
buildbot_user: str
|
||||||
oauth_secret_name: str = "github-oauth-secret"
|
oauth_secret_name: str = "github-oauth-secret"
|
||||||
token_secret_name: str = "github-token"
|
|
||||||
webhook_secret_name: str = "github-webhook-secret"
|
webhook_secret_name: str = "github-webhook-secret"
|
||||||
project_cache_file: Path = Path("github-project-cache.json")
|
project_cache_file: Path = Path("github-project-cache-v1.json")
|
||||||
topic: str | None = "build-with-buildbot"
|
topic: str | None = "build-with-buildbot"
|
||||||
|
|
||||||
def token(self) -> str:
|
|
||||||
return read_secret_file(self.token_secret_name)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class GithubBackend(GitBackend):
|
class GithubBackend(GitBackend):
|
||||||
config: GithubConfig
|
config: GithubConfig
|
||||||
webhook_secret: str
|
webhook_secret: str
|
||||||
|
|
||||||
|
auth_backend: GithubAuthBackend
|
||||||
|
|
||||||
def __init__(self, config: GithubConfig) -> None:
|
def __init__(self, config: GithubConfig) -> None:
|
||||||
self.config = config
|
self.config = config
|
||||||
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
|
self.webhook_secret = read_secret_file(self.config.webhook_secret_name)
|
||||||
|
|
||||||
|
if isinstance(self.config.auth_type, AuthTypeLegacy):
|
||||||
|
self.auth_backend = GithubLegacyAuthBackend(self.config.auth_type)
|
||||||
|
elif isinstance(self.config.auth_type, AuthTypeApp):
|
||||||
|
self.auth_backend = GithubAppAuthBackend(self.config.auth_type)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load_installations(
|
||||||
|
jwt_token: JWTToken, installations_token_map_name: Path
|
||||||
|
) -> dict[int, InstallationToken]:
|
||||||
|
initial_installations_map: dict[str, Any]
|
||||||
|
if installations_token_map_name.exists():
|
||||||
|
initial_installations_map = json.loads(
|
||||||
|
installations_token_map_name.read_text()
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
initial_installations_map = {}
|
||||||
|
|
||||||
|
installations_map: dict[int, InstallationToken] = {}
|
||||||
|
|
||||||
|
for iid, installation in initial_installations_map.items():
|
||||||
|
token: str = installation["token"]
|
||||||
|
expiration: datetime = datetime.fromisoformat(installation["expiration"])
|
||||||
|
installations_map[int(iid)] = InstallationToken(
|
||||||
|
jwt_token,
|
||||||
|
int(iid),
|
||||||
|
installations_token_map_name,
|
||||||
|
installation_token=(token, expiration),
|
||||||
|
)
|
||||||
|
|
||||||
|
return installations_map
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def create_missing_installations(
|
||||||
|
jwt_token: JWTToken,
|
||||||
|
installations_token_map_name: Path,
|
||||||
|
installations_map: dict[int, InstallationToken],
|
||||||
|
installations: list[int],
|
||||||
|
) -> dict[int, InstallationToken]:
|
||||||
|
for installation in set(installations) - installations_map.keys():
|
||||||
|
installations_map[installation] = InstallationToken(
|
||||||
|
jwt_token,
|
||||||
|
installation,
|
||||||
|
installations_token_map_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
return installations_map
|
||||||
|
|
||||||
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
|
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
|
||||||
"""Updates the flake an opens a PR for it."""
|
"""Updates the flake an opens a PR for it."""
|
||||||
factory = util.BuildFactory()
|
factory = util.BuildFactory()
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
ReloadGithubProjects(self.config.token(), self.config.project_cache_file),
|
self.auth_backend.create_reload_builder_step(self.config.project_cache_file)
|
||||||
)
|
)
|
||||||
return util.BuilderConfig(
|
return util.BuilderConfig(
|
||||||
name=self.reload_builder_name,
|
name=self.reload_builder_name,
|
||||||
|
@ -103,24 +522,33 @@ class GithubBackend(GitBackend):
|
||||||
)
|
)
|
||||||
|
|
||||||
def create_reporter(self) -> ReporterBase:
|
def create_reporter(self) -> ReporterBase:
|
||||||
return GitHubStatusPush(
|
return self.auth_backend.create_reporter()
|
||||||
token=self.config.token(),
|
|
||||||
# Since we dynamically create build steps,
|
|
||||||
# we use `virtual_builder_name` in the webinterface
|
|
||||||
# so that we distinguish what has beeing build
|
|
||||||
context=Interpolate("buildbot/%(prop:status_name)s"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def create_change_hook(self) -> dict[str, Any]:
|
def create_change_hook(self) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"secret": self.webhook_secret,
|
"secret": self.webhook_secret,
|
||||||
"strict": True,
|
"strict": True,
|
||||||
"token": self.config.token(),
|
"token": self.auth_backend.get_general_token().get(),
|
||||||
"github_property_whitelist": ["github.base.sha", "github.head.sha"],
|
"github_property_whitelist": ["github.base.sha", "github.head.sha"],
|
||||||
}
|
}
|
||||||
|
|
||||||
def create_avatar_method(self) -> AvatarBase | None:
|
def create_avatar_method(self) -> AvatarBase | None:
|
||||||
return AvatarGitHub(token=self.config.token())
|
avatar = AvatarGitHub(token=self.auth_backend.get_general_token().get())
|
||||||
|
|
||||||
|
# TODO: not a proper fix, the /users/{username} endpoint is per installation, but I'm not sure
|
||||||
|
# how to tell which installation token to use, unless there is a way to build a huge map of
|
||||||
|
# username -> token, or we just try each one in order
|
||||||
|
def _get_avatar_by_username(self: Any, username: Any) -> Any:
|
||||||
|
return f"https://github.com/{username}.png"
|
||||||
|
|
||||||
|
import types
|
||||||
|
|
||||||
|
avatar._get_avatar_by_username = types.MethodType( # noqa: SLF001
|
||||||
|
_get_avatar_by_username,
|
||||||
|
avatar,
|
||||||
|
)
|
||||||
|
|
||||||
|
return avatar
|
||||||
|
|
||||||
def create_auth(self) -> AuthBase:
|
def create_auth(self) -> AuthBase:
|
||||||
assert self.config.oauth_id is not None, "GitHub OAuth ID is required"
|
assert self.config.oauth_id is not None, "GitHub OAuth ID is required"
|
||||||
|
@ -130,6 +558,9 @@ class GithubBackend(GitBackend):
|
||||||
apiVersion=4,
|
apiVersion=4,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def create_secret_providers(self) -> list[SecretProviderBase]:
|
||||||
|
return self.auth_backend.create_secret_providers()
|
||||||
|
|
||||||
def load_projects(self) -> list["GitProject"]:
|
def load_projects(self) -> list["GitProject"]:
|
||||||
if not self.config.project_cache_file.exists():
|
if not self.config.project_cache_file.exists():
|
||||||
return []
|
return []
|
||||||
|
@ -138,19 +569,53 @@ class GithubBackend(GitBackend):
|
||||||
json.loads(self.config.project_cache_file.read_text()),
|
json.loads(self.config.project_cache_file.read_text()),
|
||||||
key=lambda x: x["full_name"],
|
key=lambda x: x["full_name"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if isinstance(self.auth_backend, GithubAppAuthBackend):
|
||||||
|
dropped_repos = list(
|
||||||
|
filter(lambda repo: not "installation_id" in repo, repos)
|
||||||
|
)
|
||||||
|
if dropped_repos:
|
||||||
|
tlog.info(
|
||||||
|
"Dropped projects follow, refresh will follow after initialisation:"
|
||||||
|
)
|
||||||
|
for dropped_repo in dropped_repos:
|
||||||
|
tlog.info(f"\tDropping repo {dropped_repo['full_name']}")
|
||||||
|
repos = list(filter(lambda repo: "installation_id" in repo, repos))
|
||||||
|
|
||||||
|
tlog.info(f"Loading {len(repos)} cached repositories.")
|
||||||
return list(
|
return list(
|
||||||
filter(
|
filter(
|
||||||
lambda project: self.config.topic is not None
|
lambda project: self.config.topic is not None
|
||||||
and self.config.topic in project.topics,
|
and self.config.topic in project.topics,
|
||||||
[
|
[
|
||||||
GithubProject(self.config, self.webhook_secret, repo)
|
GithubProject(
|
||||||
|
self.auth_backend.get_repo_token(repo),
|
||||||
|
self.config,
|
||||||
|
self.webhook_secret,
|
||||||
|
repo,
|
||||||
|
)
|
||||||
for repo in repos
|
for repo in repos
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def are_projects_cached(self) -> bool:
|
def are_projects_cached(self) -> bool:
|
||||||
return self.config.project_cache_file.exists()
|
if not self.config.project_cache_file.exists():
|
||||||
|
return False
|
||||||
|
|
||||||
|
if (
|
||||||
|
isinstance(self.config.auth_type, AuthTypeApp)
|
||||||
|
and not self.config.auth_type.app_project_id_map_name.exists()
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
|
||||||
|
all_have_installation_id = True
|
||||||
|
for project in json.loads(self.config.project_cache_file.read_text()):
|
||||||
|
if not "installation_id" in project:
|
||||||
|
all_have_installation_id = False
|
||||||
|
break
|
||||||
|
|
||||||
|
return all_have_installation_id
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def type(self) -> str:
|
def type(self) -> str:
|
||||||
|
@ -171,10 +636,18 @@ class GithubBackend(GitBackend):
|
||||||
|
|
||||||
class GithubProject(GitProject):
|
class GithubProject(GitProject):
|
||||||
config: GithubConfig
|
config: GithubConfig
|
||||||
|
webhook_secret: str
|
||||||
|
data: dict[str, Any]
|
||||||
|
token: RepoToken
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self, config: GithubConfig, webhook_secret: str, data: dict[str, Any]
|
self,
|
||||||
|
token: RepoToken,
|
||||||
|
config: GithubConfig,
|
||||||
|
webhook_secret: str,
|
||||||
|
data: dict[str, Any],
|
||||||
) -> None:
|
) -> None:
|
||||||
|
self.token = token
|
||||||
self.config = config
|
self.config = config
|
||||||
self.webhook_secret = webhook_secret
|
self.webhook_secret = webhook_secret
|
||||||
self.data = data
|
self.data = data
|
||||||
|
@ -187,7 +660,7 @@ class GithubProject(GitProject):
|
||||||
) -> None:
|
) -> None:
|
||||||
hooks = paginated_github_request(
|
hooks = paginated_github_request(
|
||||||
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
|
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100",
|
||||||
self.config.token(),
|
self.token.get(),
|
||||||
)
|
)
|
||||||
config = dict(
|
config = dict(
|
||||||
url=webhook_url + "change_hook/github",
|
url=webhook_url + "change_hook/github",
|
||||||
|
@ -199,7 +672,7 @@ class GithubProject(GitProject):
|
||||||
name="web", active=True, events=["push", "pull_request"], config=config
|
name="web", active=True, events=["push", "pull_request"], config=config
|
||||||
)
|
)
|
||||||
headers = {
|
headers = {
|
||||||
"Authorization": f"Bearer {self.config.token()}",
|
"Authorization": f"Bearer {self.token.get()}",
|
||||||
"Accept": "application/vnd.github+json",
|
"Accept": "application/vnd.github+json",
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
"X-GitHub-Api-Version": "2022-11-28",
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
@ -217,7 +690,7 @@ class GithubProject(GitProject):
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_project_url(self) -> str:
|
def get_project_url(self) -> str:
|
||||||
return f"https://git:%(secret:{self.config.token_secret_name})s@github.com/{self.name}"
|
return f"https://git:{self.token.get_as_secret()}s@github.com/{self.name}"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def pretty_type(self) -> str:
|
def pretty_type(self) -> str:
|
||||||
|
@ -260,14 +733,25 @@ class GithubProject(GitProject):
|
||||||
return self.data["owner"]["type"] == "Organization"
|
return self.data["owner"]["type"] == "Organization"
|
||||||
|
|
||||||
|
|
||||||
def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
|
def refresh_projects(
|
||||||
repos = []
|
github_token: str,
|
||||||
|
repo_cache_file: Path,
|
||||||
|
repos: list[Any] | None = None,
|
||||||
|
clear: bool = True,
|
||||||
|
api_endpoint: str = "/user/repos",
|
||||||
|
subkey: None | str = None,
|
||||||
|
require_admin: bool = True,
|
||||||
|
) -> list[Any]:
|
||||||
|
if repos is None:
|
||||||
|
repos = []
|
||||||
|
|
||||||
for repo in paginated_github_request(
|
for repo in paginated_github_request(
|
||||||
"https://api.github.com/user/repos?per_page=100",
|
f"https://api.github.com{api_endpoint}?per_page=100",
|
||||||
github_token,
|
github_token,
|
||||||
|
subkey=subkey,
|
||||||
):
|
):
|
||||||
if not repo["permissions"]["admin"]:
|
# TODO actually check for this properly
|
||||||
|
if not repo["permissions"]["admin"] and require_admin:
|
||||||
name = repo["full_name"]
|
name = repo["full_name"]
|
||||||
log.msg(
|
log.msg(
|
||||||
f"skipping {name} because we do not have admin privileges, needed for hook management",
|
f"skipping {name} because we do not have admin privileges, needed for hook management",
|
||||||
|
@ -275,12 +759,4 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
|
||||||
else:
|
else:
|
||||||
repos.append(repo)
|
repos.append(repo)
|
||||||
|
|
||||||
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
|
return repos
|
||||||
path = Path(f.name)
|
|
||||||
try:
|
|
||||||
f.write(json.dumps(repos))
|
|
||||||
f.flush()
|
|
||||||
path.rename(repo_cache_file)
|
|
||||||
except OSError:
|
|
||||||
path.unlink()
|
|
||||||
raise
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ from typing import Any
|
||||||
|
|
||||||
from buildbot.config.builder import BuilderConfig
|
from buildbot.config.builder import BuilderConfig
|
||||||
from buildbot.reporters.base import ReporterBase
|
from buildbot.reporters.base import ReporterBase
|
||||||
|
from buildbot.secrets.providers.base import SecretProviderBase
|
||||||
from buildbot.www.auth import AuthBase
|
from buildbot.www.auth import AuthBase
|
||||||
from buildbot.www.avatar import AvatarBase
|
from buildbot.www.avatar import AvatarBase
|
||||||
|
|
||||||
|
@ -28,6 +29,9 @@ class GitBackend(ABC):
|
||||||
def create_auth(self) -> AuthBase:
|
def create_auth(self) -> AuthBase:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def create_secret_providers(self) -> list[SecretProviderBase]:
|
||||||
|
return []
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def load_projects(self) -> list["GitProject"]:
|
def load_projects(self) -> list["GitProject"]:
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -21,8 +21,14 @@
|
||||||
github = {
|
github = {
|
||||||
# Github user used as a CI identity
|
# Github user used as a CI identity
|
||||||
user = "mic92-buildbot";
|
user = "mic92-buildbot";
|
||||||
# Github token of the same user
|
authType.legacy = {
|
||||||
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
|
# Github token of the same user
|
||||||
|
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
|
||||||
|
};
|
||||||
|
# authType.app = {
|
||||||
|
# id = "00000000000000000"; # FIXME: replace with App ID obtained from GitHub
|
||||||
|
# secretKeyFile = pkgs.writeText "app-secret.key" "00000000000000000000"; # FIXME: replace with App secret key obtained from GitHub
|
||||||
|
# };
|
||||||
# A random secret used to verify incoming webhooks from GitHub
|
# A random secret used to verify incoming webhooks from GitHub
|
||||||
# buildbot-nix will set up a webhook for each project in the organization
|
# buildbot-nix will set up a webhook for each project in the organization
|
||||||
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
|
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000"; # FIXME: replace this with a secret not stored in the nix store
|
||||||
|
|
|
@ -16,7 +16,9 @@
|
||||||
'';
|
'';
|
||||||
admins = [ "Mic92" ];
|
admins = [ "Mic92" ];
|
||||||
github = {
|
github = {
|
||||||
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
|
authType.legacy = {
|
||||||
|
tokenFile = pkgs.writeText "github-token" "ghp_000000000000000000000000000000000000";
|
||||||
|
};
|
||||||
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
|
webhookSecretFile = pkgs.writeText "webhookSecret" "00000000000000000000";
|
||||||
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
|
oauthSecretFile = pkgs.writeText "oauthSecret" "ffffffffffffffffffffffffffffffffffffffff";
|
||||||
oauthId = "aaaaaaaaaaaaaaaaaaaa";
|
oauthId = "aaaaaaaaaaaaaaaaaaaa";
|
||||||
|
|
|
@ -24,6 +24,24 @@ in
|
||||||
"admins"
|
"admins"
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
(mkRenamedOptionModule
|
||||||
|
[
|
||||||
|
"services"
|
||||||
|
"buildbot-nix"
|
||||||
|
"master"
|
||||||
|
"github"
|
||||||
|
"tokenFile"
|
||||||
|
]
|
||||||
|
[
|
||||||
|
"services"
|
||||||
|
"buildbot-nix"
|
||||||
|
"master"
|
||||||
|
"github"
|
||||||
|
"authType"
|
||||||
|
"legacy"
|
||||||
|
"tokenFile"
|
||||||
|
]
|
||||||
|
)
|
||||||
];
|
];
|
||||||
|
|
||||||
options = {
|
options = {
|
||||||
|
@ -106,10 +124,39 @@ in
|
||||||
default = cfg.authBackend == "github";
|
default = cfg.authBackend == "github";
|
||||||
};
|
};
|
||||||
|
|
||||||
tokenFile = lib.mkOption {
|
authType = lib.mkOption {
|
||||||
type = lib.types.path;
|
type = lib.types.attrTag {
|
||||||
description = "Github token file";
|
legacy = lib.mkOption {
|
||||||
|
description = "GitHub legacy auth backend";
|
||||||
|
type = lib.types.submodule {
|
||||||
|
options.tokenFile = lib.mkOption {
|
||||||
|
type = lib.types.path;
|
||||||
|
description = "Github token file";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
app = lib.mkOption {
|
||||||
|
description = "GitHub legacy auth backend";
|
||||||
|
type = lib.types.submodule {
|
||||||
|
options.id = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
description = ''
|
||||||
|
GitHub app ID.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
options.secretKeyFile = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = ''
|
||||||
|
GitHub app secret key file location.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
webhookSecretFile = lib.mkOption {
|
webhookSecretFile = lib.mkOption {
|
||||||
type = lib.types.path;
|
type = lib.types.path;
|
||||||
description = "Github webhook secret file";
|
description = "Github webhook secret file";
|
||||||
|
@ -243,7 +290,16 @@ in
|
||||||
home = "/var/lib/buildbot";
|
home = "/var/lib/buildbot";
|
||||||
extraImports = ''
|
extraImports = ''
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig, GiteaConfig
|
from buildbot_nix import (
|
||||||
|
GithubConfig,
|
||||||
|
NixConfigurator,
|
||||||
|
CachixConfig,
|
||||||
|
GiteaConfig,
|
||||||
|
)
|
||||||
|
from buildbot_nix.github.auth._type import (
|
||||||
|
AuthTypeLegacy,
|
||||||
|
AuthTypeApp,
|
||||||
|
)
|
||||||
'';
|
'';
|
||||||
configurators = [
|
configurators = [
|
||||||
''
|
''
|
||||||
|
@ -260,6 +316,18 @@ in
|
||||||
oauth_id=${builtins.toJSON cfg.github.oauthId},
|
oauth_id=${builtins.toJSON cfg.github.oauthId},
|
||||||
buildbot_user=${builtins.toJSON cfg.github.user},
|
buildbot_user=${builtins.toJSON cfg.github.user},
|
||||||
topic=${builtins.toJSON cfg.github.topic},
|
topic=${builtins.toJSON cfg.github.topic},
|
||||||
|
auth_type=${
|
||||||
|
if cfg.github.authType ? "legacy" then
|
||||||
|
''AuthTypeLegacy()''
|
||||||
|
else if cfg.github.authType ? "app" then
|
||||||
|
''
|
||||||
|
AuthTypeApp(
|
||||||
|
app_id=${toString cfg.github.authType.app.id},
|
||||||
|
)
|
||||||
|
''
|
||||||
|
else
|
||||||
|
throw "One of AuthTypeApp or AuthTypeLegacy must be enabled"
|
||||||
|
}
|
||||||
)"
|
)"
|
||||||
},
|
},
|
||||||
gitea=${
|
gitea=${
|
||||||
|
@ -323,6 +391,9 @@ in
|
||||||
|
|
||||||
systemd.services.buildbot-master = {
|
systemd.services.buildbot-master = {
|
||||||
after = [ "postgresql.service" ];
|
after = [ "postgresql.service" ];
|
||||||
|
path = [
|
||||||
|
pkgs.openssl
|
||||||
|
];
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
# in master.py we read secrets from $CREDENTIALS_DIRECTORY
|
# in master.py we read secrets from $CREDENTIALS_DIRECTORY
|
||||||
LoadCredential =
|
LoadCredential =
|
||||||
|
@ -337,10 +408,15 @@ in
|
||||||
(
|
(
|
||||||
cfg.cachix.authTokenFile != null
|
cfg.cachix.authTokenFile != null
|
||||||
) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
|
) "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}"
|
||||||
++ lib.optionals (cfg.github.enable) [
|
++ lib.optionals (cfg.github.enable) ([
|
||||||
"github-token:${cfg.github.tokenFile}"
|
|
||||||
"github-webhook-secret:${cfg.github.webhookSecretFile}"
|
"github-webhook-secret:${cfg.github.webhookSecretFile}"
|
||||||
]
|
]
|
||||||
|
++ lib.optionals (cfg.github.authType ? "legacy") [
|
||||||
|
"github-token:${cfg.github.authType.legacy.tokenFile}"
|
||||||
|
]
|
||||||
|
++ lib.optionals (cfg.github.authType ? "app") [
|
||||||
|
"github-app-secret-key:${cfg.github.authType.app.secretKeyFile}"
|
||||||
|
])
|
||||||
++ lib.optionals cfg.gitea.enable [
|
++ lib.optionals cfg.gitea.enable [
|
||||||
"gitea-token:${cfg.gitea.tokenFile}"
|
"gitea-token:${cfg.gitea.tokenFile}"
|
||||||
"gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
|
"gitea-webhook-secret:${cfg.gitea.webhookSecretFile}"
|
||||||
|
|
|
@ -26,6 +26,8 @@ scripts = { buildbot-effects = "hercules_effects.cli:main" }
|
||||||
[tool.setuptools]
|
[tool.setuptools]
|
||||||
packages = [
|
packages = [
|
||||||
"buildbot_nix",
|
"buildbot_nix",
|
||||||
|
"buildbot_nix.github",
|
||||||
|
"buildbot_nix.github.auth",
|
||||||
"buildbot_effects"
|
"buildbot_effects"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -82,6 +84,17 @@ ignore = [
|
||||||
|
|
||||||
# not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html
|
# not compatible with twisted logger: https://docs.twisted.org/en/twisted-18.7.0/core/howto/logger.html
|
||||||
"G010", # Logging statement uses `warn` instead of `warning`
|
"G010", # Logging statement uses `warn` instead of `warning`
|
||||||
|
|
||||||
|
# gives falls positives and isn't hard to check munually
|
||||||
|
"ERA001"
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.ruff.lint.pep8-naming]
|
||||||
|
ignore-names = [
|
||||||
|
"checkConfig",
|
||||||
|
"baseURL",
|
||||||
|
"reconfigService",
|
||||||
|
"sendMessage",
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.mypy]
|
[tool.mypy]
|
||||||
|
|
Loading…
Reference in a new issue