also create webhooks automatically
This commit is contained in:
parent
bf1fed375f
commit
54bcb08fae
161
.gitignore
vendored
161
.gitignore
vendored
|
@ -1 +1,162 @@
|
|||
result
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
|
|
@ -19,7 +19,11 @@ from buildbot.process.project import Project
|
|||
from buildbot.process.properties import Interpolate, Properties
|
||||
from buildbot.process.results import ALL_RESULTS, statusToString
|
||||
from buildbot.steps.trigger import Trigger
|
||||
from github_projects import GithubProject, load_projects # noqa: E402
|
||||
from github_projects import ( # noqa: E402
|
||||
GithubProject,
|
||||
create_project_hook,
|
||||
load_projects,
|
||||
)
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
|
@ -511,7 +515,7 @@ class GithubConfig:
|
|||
webhook_secret_name: str = "github-webhook-secret"
|
||||
token_secret_name: str = "github-token"
|
||||
project_cache_file: Path = Path("github-project-cache.json")
|
||||
topic_filter: str | None = "build-with-buildbot"
|
||||
topic: str | None = "build-with-buildbot"
|
||||
|
||||
def token(self) -> str:
|
||||
return read_secret_file(self.token_secret_name)
|
||||
|
@ -621,6 +625,7 @@ class NixConfigurator(ConfiguratorBase):
|
|||
# Shape of this file:
|
||||
# [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
|
||||
github: GithubConfig,
|
||||
url: str,
|
||||
nix_supported_systems: list[str],
|
||||
nix_eval_max_memory_size: int = 4096,
|
||||
nix_workers_secret_name: str = "buildbot-nix-workers",
|
||||
|
@ -630,12 +635,13 @@ class NixConfigurator(ConfiguratorBase):
|
|||
self.nix_eval_max_memory_size = nix_eval_max_memory_size
|
||||
self.nix_supported_systems = nix_supported_systems
|
||||
self.github = github
|
||||
self.url = url
|
||||
self.systemd_credentials_dir = os.environ["CREDENTIALS_DIRECTORY"]
|
||||
|
||||
def configure(self, config: dict[str, Any]) -> None:
|
||||
projects = load_projects(self.github.token(), self.github.project_cache_file)
|
||||
if self.github.topic_filter is not None:
|
||||
projects = [p for p in projects if self.github.topic_filter in p.topics]
|
||||
if self.github.topic is not None:
|
||||
projects = [p for p in projects if self.github.topic in p.topics]
|
||||
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
|
||||
worker_names = []
|
||||
config["workers"] = config.get("workers", [])
|
||||
|
@ -647,6 +653,15 @@ class NixConfigurator(ConfiguratorBase):
|
|||
worker_names.append(worker_name)
|
||||
|
||||
config["projects"] = config.get("projects", [])
|
||||
|
||||
for project in projects:
|
||||
create_project_hook(
|
||||
project.owner,
|
||||
project.repo,
|
||||
self.github.token(),
|
||||
f"{self.url}/change_hook/github",
|
||||
)
|
||||
|
||||
for project in projects:
|
||||
config_for_project(
|
||||
config,
|
||||
|
@ -657,6 +672,7 @@ class NixConfigurator(ConfiguratorBase):
|
|||
self.nix_supported_systems,
|
||||
self.nix_eval_max_memory_size,
|
||||
)
|
||||
|
||||
config["services"] = config.get("services", [])
|
||||
config["services"].append(
|
||||
reporters.GitHubStatusPush(
|
||||
|
|
|
@ -30,18 +30,28 @@ def http_request(
|
|||
headers = headers.copy()
|
||||
headers["User-Agent"] = "buildbot-nix"
|
||||
req = urllib.request.Request(url, headers=headers, method=method, data=body)
|
||||
resp = urllib.request.urlopen(req)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req)
|
||||
except urllib.request.HTTPError as e:
|
||||
body = ""
|
||||
try:
|
||||
body = e.fp.read()
|
||||
except Exception:
|
||||
pass
|
||||
raise Exception(
|
||||
f"Request for {method} {url} failed with {e.code} {e.reason}: {body}"
|
||||
) from e
|
||||
return HttpResponse(resp)
|
||||
|
||||
|
||||
def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
|
||||
next_url: str | None = url
|
||||
repos = []
|
||||
items = []
|
||||
while next_url:
|
||||
try:
|
||||
res = http_request(
|
||||
next_url,
|
||||
headers={"Authorization": f"token {token}"},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
except OSError as e:
|
||||
raise Exception(f"failed to fetch {next_url}: {e}") from e
|
||||
|
@ -53,34 +63,67 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
|
|||
link_parts = link.split(";")
|
||||
if link_parts[1].strip() == 'rel="next"':
|
||||
next_url = link_parts[0][1:-1]
|
||||
repos += res.json()
|
||||
return repos
|
||||
items += res.json()
|
||||
return items
|
||||
|
||||
|
||||
class GithubProject:
|
||||
def __init__(self, repo: dict[str, Any]) -> None:
|
||||
self.repo = repo
|
||||
def __init__(self, data: dict[str, Any]) -> None:
|
||||
self.data = data
|
||||
|
||||
@property
|
||||
def repo(self) -> str:
|
||||
return self.data["name"]
|
||||
|
||||
@property
|
||||
def owner(self) -> str:
|
||||
return self.data["owner"]["login"]
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.repo["full_name"]
|
||||
return self.data["full_name"]
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.repo["html_url"]
|
||||
return self.data["html_url"]
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
n = self.repo["full_name"]
|
||||
n = self.data["full_name"]
|
||||
return n.replace("/", "-")
|
||||
|
||||
@property
|
||||
def default_branch(self) -> str:
|
||||
return self.repo["default_branch"]
|
||||
return self.data["default_branch"]
|
||||
|
||||
@property
|
||||
def topics(self) -> list[str]:
|
||||
return self.repo["topics"]
|
||||
return self.data["topics"]
|
||||
|
||||
|
||||
def create_project_hook(owner: str, repo: str, token: str, webhook_url: str) -> None:
|
||||
hooks = paginated_github_request(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", token
|
||||
)
|
||||
config = dict(url=webhook_url, content_type="json", insecure_ssl="0")
|
||||
data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Content-Type": "application/json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
for hook in hooks:
|
||||
if hook["config"]["url"] == webhook_url:
|
||||
log.msg(f"hook for {owner}/{repo} already exists")
|
||||
return
|
||||
|
||||
http_request(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/hooks",
|
||||
method="POST",
|
||||
headers=headers,
|
||||
data=data,
|
||||
)
|
||||
|
||||
|
||||
def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]:
|
||||
|
|
|
@ -109,6 +109,7 @@ in
|
|||
buildbot_user=${builtins.toJSON cfg.github.user},
|
||||
topic=${builtins.toJSON cfg.github.topic},
|
||||
),
|
||||
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
|
||||
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
|
||||
nix_supported_systems=${builtins.toJSON cfg.buildSystems},
|
||||
)
|
||||
|
|
Loading…
Reference in a new issue