auxolotl/buildbot-nix
26
1
Fork 0
Code Issues Pull requests 1 Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #217 from MagicRB/combined-build-reports-github

Browse source 
Combined build reports GitHub
This commit is contained in:
Jörg Thalheim 2024-07-26 13:47:13 +02:00 committed by GitHub
parent c9b6d013d7 0ecf33f8d4
commit 2620f2cd31
Failed to generate hash of commit
6 changed files with 196 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
buildbot_nix/__init__.py
View file

@ -59,12 +59,14 @@ class BuildTrigger(Trigger):
builds_scheduler: str,
skipped_builds_scheduler: str,
jobs: list[dict[str, Any]],
report_status: bool,
**kwargs: Any,
) -> None:
if "name" not in kwargs:
kwargs["name"] = "trigger"
self.project = project
self.jobs = jobs
self.report_status = report_status
self.config = None
self.builds_scheduler = builds_scheduler
self.skipped_builds_scheduler = skipped_builds_scheduler
@ -102,6 +104,7 @@ class BuildTrigger(Trigger):
props.setProperty("virtual_builder_name", name, source)
props.setProperty("status_name", f"nix-build .#checks.{attr}", source)
props.setProperty("virtual_builder_tags", "", source)
props.setProperty("report_status", self.report_status, source)
drv_path = job.get("drvPath")
system = job.get("system")
@ -145,6 +148,15 @@ class BuildTrigger(Trigger):
return {"step": f"({', '.join(summary)})"}
class NixBuildCombined(steps.BuildStep):
"""Shows the error message of a failed evaluation."""
name = "nix-build-combined"
def run(self) -> Generator[Any, object, Any]:
return self.build.results
class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
"""Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for
every attribute.
@ -153,7 +165,11 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
project: GitProject
def __init__(
self, project: GitProject, supported_systems: list[str], **kwargs: Any
self,
project: GitProject,
supported_systems: list[str],
job_report_limit: int | None,
**kwargs: Any,
) -> None:
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
@ -161,6 +177,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
self.observer = logobserver.BufferLogObserver()
self.addLogObserver("stdio", self.observer)
self.supported_systems = supported_systems
self.job_report_limit = job_report_limit
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
@ -190,6 +207,8 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
if not system or system in self.supported_systems: # report eval errors
filtered_jobs.append(job)
self.number_of_jobs = len(filtered_jobs)
self.build.addStepsAfterCurrentStep(
[
BuildTrigger(
@ -198,8 +217,28 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
skipped_builds_scheduler=f"{project_id}-nix-skipped-build",
name="build flake",
jobs=filtered_jobs,
report_status=(
self.job_report_limit is None
or self.number_of_jobs <= self.job_report_limit
),
),
],
]
+ (
[
Trigger(
waitForFinish=True,
schedulerNames=[f"{project_id}-nix-build-combined"],
haltOnFailure=True,
flunkOnFailure=True,
sourceStamps=[],
alwaysUseLatest=False,
updateSourceStamp=False,
),
]
if self.job_report_limit is not None
and self.number_of_jobs > self.job_report_limit
else []
),
)
return result
@ -360,6 +399,7 @@ def nix_eval_config(
eval_lock: MasterLock,
worker_count: int,
max_memory_size: int,
job_report_limit: int | None,
) -> BuilderConfig:
"""Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
For each evaluated attribute a new build pipeline is started.
@ -385,6 +425,7 @@ def nix_eval_config(
env={},
name="evaluate flake",
supported_systems=supported_systems,
job_report_limit=job_report_limit,
command=[
"nix-eval-jobs",
"--workers",
@ -492,6 +533,7 @@ def nix_build_config(
updateSourceStamp=False,
doStepIf=do_register_gcroot_if,
copy_properties=["out_path", "attr"],
set_properties={"report_status": False},
),
)
factory.addStep(
@ -556,6 +598,7 @@ def nix_skipped_build_config(
updateSourceStamp=False,
doStepIf=do_register_gcroot_if,
copy_properties=["out_path", "attr"],
set_properties={"report_status": False},
),
)
return util.BuilderConfig(
@ -601,6 +644,24 @@ def nix_register_gcroot_config(
)
def nix_build_combined_config(
project: GitProject,
worker_names: list[str],
) -> BuilderConfig:
factory = util.BuildFactory()
factory.addStep(NixBuildCombined())
return util.BuilderConfig(
name=f"{project.name}/nix-build-combined",
project=project.name,
workernames=worker_names,
collapseRequests=False,
env={},
factory=factory,
properties=dict(status_name="nix-build-combined"),
)
def config_for_project(
config: dict[str, Any],
project: GitProject,
@ -610,6 +671,7 @@ def config_for_project(
nix_eval_max_memory_size: int,
eval_lock: MasterLock,
post_build_steps: list[steps.BuildStep],
job_report_limit: int | None,
outputs_path: Path | None = None,
build_retries: int = 1,
) -> None:
@ -653,6 +715,11 @@ def config_for_project(
name=f"{project.project_id}-nix-skipped-build",
builderNames=[f"{project.name}/nix-skipped-build"],
),
# this is triggered from `nix-eval` when the build contains too many outputs
schedulers.Triggerable(
name=f"{project.project_id}-nix-build-combined",
builderNames=[f"{project.name}/nix-build-combined"],
),
schedulers.Triggerable(
name=f"{project.project_id}-nix-register-gcroot",
builderNames=[f"{project.name}/nix-register-gcroot"],
@ -680,6 +747,7 @@ def config_for_project(
worker_names,
git_url=project.get_project_url(),
supported_systems=nix_supported_systems,
job_report_limit=job_report_limit,
worker_count=nix_eval_worker_count,
max_memory_size=nix_eval_max_memory_size,
eval_lock=eval_lock,
@ -693,6 +761,7 @@ def config_for_project(
),
nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]),
nix_register_gcroot_config(project, worker_names),
nix_build_combined_config(project, worker_names),
],
)
@ -842,7 +911,7 @@ class NixConfigurator(ConfiguratorBase):
backends["github"] = GithubBackend(self.config.github, self.config.url)
if self.config.gitea is not None:
backends["gitea"] = GiteaBackend(self.config.gitea)
backends["gitea"] = GiteaBackend(self.config.gitea, self.config.url)
auth: AuthBase | None = (
backends[self.config.auth_backend].create_auth()
@ -895,6 +964,7 @@ class NixConfigurator(ConfiguratorBase):
self.config.eval_max_memory_size,
eval_lock,
[x.to_buildstep() for x in self.config.post_build_steps],
self.config.job_report_limit,
self.config.outputs_path,
self.config.build_retries,
)

8
buildbot_nix/common.py
View file

@ -172,3 +172,11 @@ def model_validate_project_cache(cls: type[_T], project_cache_file: Path) -> lis
def model_dump_project_cache(repos: list[_T]) -> str:
return json.dumps([repo.model_dump() for repo in repos])
def filter_for_combined_builds(reports: Any) -> Any | None:
properties = reports[0]["builds"][0]["properties"]
if "report_status" in properties and not properties["report_status"][0]:
return None
return reports

78
buildbot_nix/gitea_projects.py
View file

@ -1,5 +1,6 @@
import os
import signal
from collections.abc import Callable
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
@ -13,12 +14,14 @@ from buildbot.www.avatar import AvatarBase
from buildbot_gitea.auth import GiteaAuth # type: ignore[import]
from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import]
from pydantic import BaseModel
from twisted.internet import defer
from twisted.logger import Logger
from twisted.python import log
from .common import (
ThreadDeferredBuildStep,
atomic_write_file,
filter_for_combined_builds,
filter_repos_by_topic,
http_request,
model_dump_project_cache,
@ -105,11 +108,43 @@ class GiteaProject(GitProject):
return False # self.data["owner"]["type"] == "Organization"
class ModifyingGiteaStatusPush(GiteaStatusPush):
def checkConfig(
self,
modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803
**kwargs: Any,
) -> Any:
self.modifyingFilter = modifyingFilter
return super().checkConfig(**kwargs)
def reconfigService(
self,
modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803
**kwargs: Any,
) -> Any:
self.modifyingFilter = modifyingFilter
return super().reconfigService(**kwargs)
@defer.inlineCallbacks
def sendMessage(self, reports: Any) -> Any:
reports = self.modifyingFilter(reports)
if reports is None:
return
result = yield super().sendMessage(reports)
return result
class GiteaBackend(GitBackend):
config: GiteaConfig
webhook_secret: str
instance_url: str
def __init__(self, config: GiteaConfig) -> None:
def __init__(self, config: GiteaConfig, instance_url: str) -> None:
self.config = config
self.instance_url = instance_url
def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
"""Updates the flake an opens a PR for it."""
@ -118,7 +153,10 @@ class GiteaBackend(GitBackend):
ReloadGiteaProjects(self.config, self.config.project_cache_file),
)
factory.addStep(
CreateGiteaProjectHooks(self.config),
CreateGiteaProjectHooks(
self.config,
self.instance_url,
),
)
return util.BuilderConfig(
name=self.reload_builder_name,
@ -127,11 +165,12 @@ class GiteaBackend(GitBackend):
)
def create_reporter(self) -> ReporterBase:
return GiteaStatusPush(
self.config.instance_url,
Interpolate(self.config.token),
return ModifyingGiteaStatusPush(
baseURL=self.config.instance_url,
token=Interpolate(self.config.token),
context=Interpolate("buildbot/%(prop:status_name)s"),
context_pr=Interpolate("buildbot/%(prop:status_name)s"),
modifyingFilter=filter_for_combined_builds,
)
def create_change_hook(self) -> dict[str, Any]:
@ -198,14 +237,19 @@ class GiteaBackend(GitBackend):
def create_repo_hook(
token: str, webhook_secret: str, owner: str, repo: str, webhook_url: str
token: str,
webhook_secret: str,
owner: str,
repo: str,
gitea_url: str,
instance_url: str,
) -> None:
hooks = paginated_github_request(
f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
f"{gitea_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100",
token,
)
config = dict(
url=webhook_url + "change_hook/gitea",
url=instance_url + "change_hook/gitea",
content_type="json",
insecure_ssl="0",
secret=webhook_secret,
@ -223,13 +267,13 @@ def create_repo_hook(
"Content-Type": "application/json",
}
for hook in hooks:
if hook["config"]["url"] == webhook_url + "change_hook/gitea":
if hook["config"]["url"] == instance_url + "change_hook/gitea":
log.msg(f"hook for {owner}/{repo} already exists")
return
log.msg(f"creating hook for {owner}/{repo}")
http_request(
f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks",
f"{gitea_url}/api/v1/repos/{owner}/{repo}/hooks",
method="POST",
headers=headers,
data=data,
@ -240,13 +284,16 @@ class CreateGiteaProjectHooks(ThreadDeferredBuildStep):
name = "create_gitea_project_hooks"
config: GiteaConfig
instance_url: str
def __init__(
self,
config: GiteaConfig,
instance_url: str,
**kwargs: Any,
) -> None:
self.config = config
self.instance_url = instance_url
super().__init__(**kwargs)
def run_deferred(self) -> None:
@ -254,11 +301,12 @@ class CreateGiteaProjectHooks(ThreadDeferredBuildStep):
for repo in repos:
create_repo_hook(
self.config.token,
self.config.webhook_secret,
repo.owner.login,
repo.name,
self.config.instance_url,
token=self.config.token,
webhook_secret=self.config.webhook_secret,
owner=repo.owner.login,
repo=repo.name,
gitea_url=self.config.instance_url,
instance_url=self.instance_url,
)
def run_post(self) -> Any:

38
buildbot_nix/github_projects.py
View file

@ -2,6 +2,7 @@ import json
import os
import signal
from abc import ABC, abstractmethod
from collections.abc import Callable
from dataclasses import dataclass
from itertools import starmap
from pathlib import Path
@ -18,12 +19,14 @@ from buildbot.www.auth import AuthBase
from buildbot.www.avatar import AvatarBase, AvatarGitHub
from buildbot.www.oauth2 import GitHubAuth
from pydantic import BaseModel, ConfigDict, Field
from twisted.internet import defer
from twisted.logger import Logger
from twisted.python import log
from .common import (
ThreadDeferredBuildStep,
atomic_write_file,
filter_for_combined_builds,
filter_repos_by_topic,
http_request,
model_dump_project_cache,
@ -310,6 +313,35 @@ class GithubAuthBackend(ABC):
pass
class ModifyingGitHubStatusPush(GitHubStatusPush):
def checkConfig(
self,
modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803
**kwargs: Any,
) -> Any:
self.modifyingFilter = modifyingFilter
return super().checkConfig(**kwargs)
def reconfigService(
self,
modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803
**kwargs: Any,
) -> Any:
self.modifyingFilter = modifyingFilter
return super().reconfigService(**kwargs)
@defer.inlineCallbacks
def sendMessage(self, reports: Any) -> Any:
reports = self.modifyingFilter(reports)
if reports is None:
return
result = yield super().sendMessage(reports)
return result
class GithubLegacyAuthBackend(GithubAuthBackend):
auth_type: GitHubLegacyConfig
@ -329,12 +361,13 @@ class GithubLegacyAuthBackend(GithubAuthBackend):
return [GitHubLegacySecretService(self.token)]
def create_reporter(self) -> ReporterBase:
return GitHubStatusPush(
return ModifyingGitHubStatusPush(
token=self.token.get(),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
modifyingFilter=filter_for_combined_builds,
)
def create_reload_builder_steps(
@ -416,12 +449,13 @@ class GithubAppAuthBackend(GithubAuthBackend):
self.project_id_map[props["projectname"]]
].get()
return GitHubStatusPush(
return ModifyingGitHubStatusPush(
token=WithProperties("%(github_token)s", github_token=get_github_token),
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:status_name)s"),
modifyingFilter=filter_for_combined_builds,
)
def create_reload_builder_steps(

1
buildbot_nix/models.py
View file

@ -179,6 +179,7 @@ class BuildbotNixConfig(BaseModel):
outputs_path: Path | None
url: str
post_build_steps: list[PostBuildStep]
job_report_limit: int | None
@property
def nix_workers_secret(self) -> str:

16
nix/master.nix
View file

@ -339,6 +339,17 @@ in
default = null;
example = "/var/www/buildbot/nix-outputs";
};
jobReportLimit = lib.mkOption {
type = lib.types.nullOr lib.types.ints.unsigned;
description = ''
The max number of build jobs per `nix-eval` `buildbot-nix` will report to backends (GitHub, Gitea, etc.).
If set to `null`, report everything, if set to `n` (some unsiggned intereger), report builds individually
as long as the number of builds is less than or equal to `n`, then report builds using a combined
`nix-build-combined` build.
'';
default = 50;
};
};
};
config = lib.mkMerge [
@ -487,6 +498,7 @@ in
outputs_path = cfg.outputsPath;
url = config.services.buildbot-nix.master.webhookBaseUrl;
post_build_steps = cfg.postBuildSteps;
job_report_limit=if cfg.jobReportLimit == null then "None" else builtins.toJSON cfg.jobReportLimit;
}}").read_text()))
)
''
@ -500,7 +512,9 @@ in
dbUrl = config.services.buildbot-nix.master.dbUrl;
package = cfg.buildbotNixpkgs.buildbot.overrideAttrs (old: {
patches = old.patches ++ [ ./0001-master-reporters-github-render-token-for-each-reques.patch ];
patches = old.patches ++ [
./0001-master-reporters-github-render-token-for-each-reques.patch
];
});
pythonPackages =
let