From dd2df670090e88f3957457abdfad24ca0e9592fb Mon Sep 17 00:00:00 2001 From: magic_rb Date: Thu, 11 Jul 2024 20:56:15 +0200 Subject: [PATCH 1/7] Combine build reports for GitHub if there are too many Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 55 +++++++++++++++++++++++++++++++-- buildbot_nix/github_projects.py | 35 +++++++++++++++++++-- nix/master.nix | 4 ++- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 637a881..fec1b28 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -59,12 +59,14 @@ class BuildTrigger(Trigger): builds_scheduler: str, skipped_builds_scheduler: str, jobs: list[dict[str, Any]], + report_status: bool, **kwargs: Any, ) -> None: if "name" not in kwargs: kwargs["name"] = "trigger" self.project = project self.jobs = jobs + self.report_status = report_status self.config = None self.builds_scheduler = builds_scheduler self.skipped_builds_scheduler = skipped_builds_scheduler @@ -102,6 +104,7 @@ class BuildTrigger(Trigger): props.setProperty("virtual_builder_name", name, source) props.setProperty("status_name", f"nix-build .#checks.{attr}", source) props.setProperty("virtual_builder_tags", "", source) + props.setProperty("report_status", self.report_status, source) drv_path = job.get("drvPath") system = job.get("system") @@ -145,6 +148,15 @@ class BuildTrigger(Trigger): return {"step": f"({', '.join(summary)})"} +class NixBuildCombined(steps.BuildStep): + """Shows the error message of a failed evaluation.""" + + name = "nix-build-combined" + + def run(self) -> Generator[Any, object, Any]: + return self.build.results + + class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): """Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for every attribute. @@ -190,16 +202,32 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): if not system or system in self.supported_systems: # report eval errors filtered_jobs.append(job) + self.number_of_jobs = len(filtered_jobs) + self.build.addStepsAfterCurrentStep( - [ + [ # noqa: RUF005 BuildTrigger( self.project, builds_scheduler=f"{project_id}-nix-build", skipped_builds_scheduler=f"{project_id}-nix-skipped-build", name="build flake", jobs=filtered_jobs, + report_status=(self.number_of_jobs <= 2), ), - ], + ] + + [ + Trigger( + waitForFinish=True, + schedulerNames=[f"{project_id}-nix-build-combined"], + haltOnFailure=True, + flunkOnFailure=True, + sourceStamps=[], + alwaysUseLatest=False, + updateSourceStamp=False, + ), + ] + if self.number_of_jobs > 2 + else [], ) return result @@ -600,6 +628,23 @@ def nix_register_gcroot_config( factory=factory, ) +def nix_build_combined_config( + project: GitProject, + worker_names: list[str], +) -> BuilderConfig: + factory = util.BuildFactory() + factory.addStep(NixBuildCombined()) + + return util.BuilderConfig( + name=f"{project.name}/nix-build-combined", + project=project.name, + workernames=worker_names, + collapseRequests=False, + env={}, + factory=factory, + properties=dict(status_name="nix-build-combined"), + ) + def config_for_project( config: dict[str, Any], @@ -653,6 +698,11 @@ def config_for_project( name=f"{project.project_id}-nix-skipped-build", builderNames=[f"{project.name}/nix-skipped-build"], ), + # this is triggered from `nix-eval` when the build contains too many outputs + schedulers.Triggerable( + name=f"{project.project_id}-nix-build-combined", + builderNames=[f"{project.name}/nix-build-combined"], + ), schedulers.Triggerable( name=f"{project.project_id}-nix-register-gcroot", builderNames=[f"{project.name}/nix-register-gcroot"], @@ -693,6 +743,7 @@ def config_for_project( ), nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]), nix_register_gcroot_config(project, worker_names), + nix_build_combined_config(project, worker_names), ], ) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 12957e2..b605d72 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from itertools import starmap from pathlib import Path -from typing import Any +from typing import Any, Callable from buildbot.config.builder import BuilderConfig from buildbot.plugins import util @@ -20,6 +20,7 @@ from buildbot.www.oauth2 import GitHubAuth from pydantic import BaseModel, ConfigDict, Field from twisted.logger import Logger from twisted.python import log +from twisted.internet import defer from .common import ( ThreadDeferredBuildStep, @@ -309,6 +310,32 @@ class GithubAuthBackend(ABC): ) -> list[BuildStep]: pass +class ModifyingGitHubStatusPush(GitHubStatusPush): + def checkConfig(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + self.modifyingFilter = modifyingFilter + + return super().checkConfig(**kwargs) + + def reconfigService(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + self.modifyingFilter = modifyingFilter + + return super().reconfigService(**kwargs) + + @defer.inlineCallbacks + def sendMessage(self, reports: Any) -> Any: + reports = self.modifyingFilter(reports) + if reports is None: + return + + result = yield super().sendMessage(reports) + return result + +def filter_for_combined_builds(reports: Any) -> Any | None: + properties = reports[0]["builds"][0]["properties"] + + if "report_status" in properties and not properties["report_status"][0]: + return None + return reports class GithubLegacyAuthBackend(GithubAuthBackend): auth_type: GitHubLegacyConfig @@ -329,12 +356,13 @@ class GithubLegacyAuthBackend(GithubAuthBackend): return [GitHubLegacySecretService(self.token)] def create_reporter(self) -> ReporterBase: - return GitHubStatusPush( + return ModifyingGitHubStatusPush( token=self.token.get(), # Since we dynamically create build steps, # we use `virtual_builder_name` in the webinterface # so that we distinguish what has beeing build context=Interpolate("buildbot/%(prop:status_name)s"), + modifyingFilter=filter_for_combined_builds, ) def create_reload_builder_steps( @@ -416,12 +444,13 @@ class GithubAppAuthBackend(GithubAuthBackend): self.project_id_map[props["projectname"]] ].get() - return GitHubStatusPush( + return ModifyingGitHubStatusPush( token=WithProperties("%(github_token)s", github_token=get_github_token), # Since we dynamically create build steps, # we use `virtual_builder_name` in the webinterface # so that we distinguish what has beeing build context=Interpolate("buildbot/%(prop:status_name)s"), + modifyingFilter=filter_for_combined_builds, ) def create_reload_builder_steps( diff --git a/nix/master.nix b/nix/master.nix index 43c3161..17f11f0 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -482,7 +482,9 @@ in dbUrl = config.services.buildbot-nix.master.dbUrl; package = cfg.buildbotNixpkgs.buildbot.overrideAttrs (old: { - patches = old.patches ++ [ ./0001-master-reporters-github-render-token-for-each-reques.patch ]; + patches = old.patches ++ [ + ./0001-master-reporters-github-render-token-for-each-reques.patch + ]; }); pythonPackages = let From 28b6da6e736ce5dc618e7c757e50eb6f1ac47546 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Fri, 12 Jul 2024 16:44:02 +0200 Subject: [PATCH 2/7] Make the cut off point for combining build reports customizable. Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 48 ++++++++++++++++++++++++++-------------- buildbot_nix/models.py | 1 + nix/master.nix | 12 ++++++++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index fec1b28..36cc22e 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -165,7 +165,11 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): project: GitProject def __init__( - self, project: GitProject, supported_systems: list[str], **kwargs: Any + self, + project: GitProject, + supported_systems: list[str], + job_report_limit: int | None, + **kwargs: Any, ) -> None: kwargs = self.setupShellMixin(kwargs) super().__init__(**kwargs) @@ -173,6 +177,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): self.observer = logobserver.BufferLogObserver() self.addLogObserver("stdio", self.observer) self.supported_systems = supported_systems + self.job_report_limit = job_report_limit @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: @@ -205,29 +210,35 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): self.number_of_jobs = len(filtered_jobs) self.build.addStepsAfterCurrentStep( - [ # noqa: RUF005 + [ BuildTrigger( self.project, builds_scheduler=f"{project_id}-nix-build", skipped_builds_scheduler=f"{project_id}-nix-skipped-build", name="build flake", jobs=filtered_jobs, - report_status=(self.number_of_jobs <= 2), + report_status=( + self.job_report_limit is None + or self.number_of_jobs <= self.job_report_limit + ), ), ] - + [ - Trigger( - waitForFinish=True, - schedulerNames=[f"{project_id}-nix-build-combined"], - haltOnFailure=True, - flunkOnFailure=True, - sourceStamps=[], - alwaysUseLatest=False, - updateSourceStamp=False, - ), - ] - if self.number_of_jobs > 2 - else [], + + ( + [ + Trigger( + waitForFinish=True, + schedulerNames=[f"{project_id}-nix-build-combined"], + haltOnFailure=True, + flunkOnFailure=True, + sourceStamps=[], + alwaysUseLatest=False, + updateSourceStamp=False, + ), + ] + if self.job_report_limit is not None + and self.number_of_jobs > self.job_report_limit + else [] + ), ) return result @@ -388,6 +399,7 @@ def nix_eval_config( eval_lock: MasterLock, worker_count: int, max_memory_size: int, + job_report_limit: int | None, ) -> BuilderConfig: """Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel. For each evaluated attribute a new build pipeline is started. @@ -413,6 +425,7 @@ def nix_eval_config( env={}, name="evaluate flake", supported_systems=supported_systems, + job_report_limit=job_report_limit, command=[ "nix-eval-jobs", "--workers", @@ -655,6 +668,7 @@ def config_for_project( nix_eval_max_memory_size: int, eval_lock: MasterLock, post_build_steps: list[steps.BuildStep], + job_report_limit: int | None, outputs_path: Path | None = None, build_retries: int = 1, ) -> None: @@ -730,6 +744,7 @@ def config_for_project( worker_names, git_url=project.get_project_url(), supported_systems=nix_supported_systems, + job_report_limit=job_report_limit, worker_count=nix_eval_worker_count, max_memory_size=nix_eval_max_memory_size, eval_lock=eval_lock, @@ -946,6 +961,7 @@ class NixConfigurator(ConfiguratorBase): self.config.eval_max_memory_size, eval_lock, [x.to_buildstep() for x in self.config.post_build_steps], + self.config.job_report_limit, self.config.outputs_path, self.config.build_retries, ) diff --git a/buildbot_nix/models.py b/buildbot_nix/models.py index 2acdcd9..6308b71 100644 --- a/buildbot_nix/models.py +++ b/buildbot_nix/models.py @@ -179,6 +179,7 @@ class BuildbotNixConfig(BaseModel): outputs_path: Path | None url: str post_build_steps: list[PostBuildStep] + job_report_limit: int | None @property def nix_workers_secret(self) -> str: diff --git a/nix/master.nix b/nix/master.nix index 17f11f0..aaead46 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -338,6 +338,17 @@ in default = null; example = "/var/www/buildbot/nix-outputs"; }; + + jobReportLimit = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + description = '' + The max number of build jobs per `nix-eval` `buildbot-nix` will report to backends (GitHub, Gitea, etc.). + If set to `null`, report everything, if set to `n` (some positive intereger), report builds individually + as long as the number of builds is less than or equal to `n`, then report builds using a combined + `nix-build-combined` build. + ''; + default = 50; + }; }; }; config = lib.mkMerge [ @@ -469,6 +480,7 @@ in outputs_path = cfg.outputsPath; url = config.services.buildbot-nix.master.webhookBaseUrl; post_build_steps = cfg.postBuildSteps; + job_report_limit=if cfg.jobReportLimit == null then "None" else builtins.toJSON cfg.jobReportLimit; }}").read_text())) ) '' From e53acc66607de33134394aaf0c26df2e20ae703b Mon Sep 17 00:00:00 2001 From: magic_rb Date: Mon, 15 Jul 2024 12:08:36 +0200 Subject: [PATCH 3/7] Support setting `jobReportLimit` to `0`, which always combines bulids Signed-off-by: magic_rb --- nix/master.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/master.nix b/nix/master.nix index aaead46..906454f 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -340,10 +340,10 @@ in }; jobReportLimit = lib.mkOption { - type = lib.types.nullOr lib.types.ints.positive; + type = lib.types.nullOr lib.types.ints.unsigned; description = '' The max number of build jobs per `nix-eval` `buildbot-nix` will report to backends (GitHub, Gitea, etc.). - If set to `null`, report everything, if set to `n` (some positive intereger), report builds individually + If set to `null`, report everything, if set to `n` (some unsiggned intereger), report builds individually as long as the number of builds is less than or equal to `n`, then report builds using a combined `nix-build-combined` build. ''; From 437ebc49b93f968e2f21a147e2bfc8028c4911d5 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Mon, 15 Jul 2024 12:09:07 +0200 Subject: [PATCH 4/7] Support combined builds for Gitea Signed-off-by: magic_rb --- buildbot_nix/common.py | 9 +++++++++ buildbot_nix/gitea_projects.py | 27 +++++++++++++++++++++++++-- buildbot_nix/github_projects.py | 9 ++------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py index 2b0c101..ed0a29d 100644 --- a/buildbot_nix/common.py +++ b/buildbot_nix/common.py @@ -172,3 +172,12 @@ def model_validate_project_cache(cls: type[_T], project_cache_file: Path) -> lis def model_dump_project_cache(repos: list[_T]) -> str: return json.dumps([repo.model_dump() for repo in repos]) + + +def filter_for_combined_builds(reports: Any) -> Any | None: + properties = reports[0]["builds"][0]["properties"] + + if "report_status" in properties and not properties["report_status"][0]: + return None + return reports + diff --git a/buildbot_nix/gitea_projects.py b/buildbot_nix/gitea_projects.py index 9eef8eb..770dc49 100644 --- a/buildbot_nix/gitea_projects.py +++ b/buildbot_nix/gitea_projects.py @@ -1,7 +1,7 @@ import os import signal from pathlib import Path -from typing import Any +from typing import Any, Callable from urllib.parse import urlparse from buildbot.config.builder import BuilderConfig @@ -15,10 +15,12 @@ from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import] from pydantic import BaseModel from twisted.logger import Logger from twisted.python import log +from twisted.internet import defer from .common import ( ThreadDeferredBuildStep, atomic_write_file, + filter_for_combined_builds, filter_repos_by_topic, http_request, model_dump_project_cache, @@ -104,6 +106,26 @@ class GiteaProject(GitProject): # TODO Gitea doesn't include this information return False # self.data["owner"]["type"] == "Organization" +class ModifyingGiteaStatusPush(GiteaStatusPush): + def checkConfig(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + self.modifyingFilter = modifyingFilter + + return super().checkConfig(**kwargs) + + def reconfigService(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + self.modifyingFilter = modifyingFilter + + return super().reconfigService(**kwargs) + + @defer.inlineCallbacks + def sendMessage(self, reports: Any) -> Any: + reports = self.modifyingFilter(reports) + if reports is None: + return + + result = yield super().sendMessage(reports) + return result + class GiteaBackend(GitBackend): config: GiteaConfig @@ -127,11 +149,12 @@ class GiteaBackend(GitBackend): ) def create_reporter(self) -> ReporterBase: - return GiteaStatusPush( + return ModifyingGiteaStatusPush( self.config.instance_url, Interpolate(self.config.token), context=Interpolate("buildbot/%(prop:status_name)s"), context_pr=Interpolate("buildbot/%(prop:status_name)s"), + modifyingFilter=filter_for_combined_builds, ) def create_change_hook(self) -> dict[str, Any]: diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index b605d72..35d5fe5 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -25,6 +25,7 @@ from twisted.internet import defer from .common import ( ThreadDeferredBuildStep, atomic_write_file, + filter_for_combined_builds, filter_repos_by_topic, http_request, model_dump_project_cache, @@ -330,13 +331,6 @@ class ModifyingGitHubStatusPush(GitHubStatusPush): result = yield super().sendMessage(reports) return result -def filter_for_combined_builds(reports: Any) -> Any | None: - properties = reports[0]["builds"][0]["properties"] - - if "report_status" in properties and not properties["report_status"][0]: - return None - return reports - class GithubLegacyAuthBackend(GithubAuthBackend): auth_type: GitHubLegacyConfig @@ -444,6 +438,7 @@ class GithubAppAuthBackend(GithubAuthBackend): self.project_id_map[props["projectname"]] ].get() + return ModifyingGitHubStatusPush( token=WithProperties("%(github_token)s", github_token=get_github_token), # Since we dynamically create build steps, From d44dc3da09249673838822a51c5385e895691330 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 24 Jul 2024 17:24:47 +0200 Subject: [PATCH 5/7] Fix Gitea backend using the Gitea url as a webhook url Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 2 +- buildbot_nix/gitea_projects.py | 43 +++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 36cc22e..2bab8d5 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -908,7 +908,7 @@ class NixConfigurator(ConfiguratorBase): backends["github"] = GithubBackend(self.config.github, self.config.url) if self.config.gitea is not None: - backends["gitea"] = GiteaBackend(self.config.gitea) + backends["gitea"] = GiteaBackend(self.config.gitea, self.config.url) auth: AuthBase | None = ( backends[self.config.auth_backend].create_auth() diff --git a/buildbot_nix/gitea_projects.py b/buildbot_nix/gitea_projects.py index 770dc49..1282b26 100644 --- a/buildbot_nix/gitea_projects.py +++ b/buildbot_nix/gitea_projects.py @@ -129,9 +129,12 @@ class ModifyingGiteaStatusPush(GiteaStatusPush): class GiteaBackend(GitBackend): config: GiteaConfig + webhook_secret: str + instance_url: str - def __init__(self, config: GiteaConfig) -> None: + def __init__(self, config: GiteaConfig, instance_url: str) -> None: self.config = config + self.instance_url = instance_url def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: """Updates the flake an opens a PR for it.""" @@ -140,7 +143,10 @@ class GiteaBackend(GitBackend): ReloadGiteaProjects(self.config, self.config.project_cache_file), ) factory.addStep( - CreateGiteaProjectHooks(self.config), + CreateGiteaProjectHooks( + self.config, + self.instance_url, + ), ) return util.BuilderConfig( name=self.reload_builder_name, @@ -150,8 +156,8 @@ class GiteaBackend(GitBackend): def create_reporter(self) -> ReporterBase: return ModifyingGiteaStatusPush( - self.config.instance_url, - Interpolate(self.config.token), + baseURL=self.config.instance_url, + token=Interpolate(self.config.token), context=Interpolate("buildbot/%(prop:status_name)s"), context_pr=Interpolate("buildbot/%(prop:status_name)s"), modifyingFilter=filter_for_combined_builds, @@ -221,14 +227,19 @@ class GiteaBackend(GitBackend): def create_repo_hook( - token: str, webhook_secret: str, owner: str, repo: str, webhook_url: str + token: str, + webhook_secret: str, + owner: str, + repo: str, + gitea_url: str, + instance_url: str, ) -> None: hooks = paginated_github_request( - f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100", + f"{gitea_url}/api/v1/repos/{owner}/{repo}/hooks?limit=100", token, ) config = dict( - url=webhook_url + "change_hook/gitea", + url=instance_url + "change_hook/gitea", content_type="json", insecure_ssl="0", secret=webhook_secret, @@ -246,13 +257,13 @@ def create_repo_hook( "Content-Type": "application/json", } for hook in hooks: - if hook["config"]["url"] == webhook_url + "change_hook/gitea": + if hook["config"]["url"] == instance_url + "change_hook/gitea": log.msg(f"hook for {owner}/{repo} already exists") return log.msg(f"creating hook for {owner}/{repo}") http_request( - f"{webhook_url}/api/v1/repos/{owner}/{repo}/hooks", + f"{gitea_url}/api/v1/repos/{owner}/{repo}/hooks", method="POST", headers=headers, data=data, @@ -263,13 +274,16 @@ class CreateGiteaProjectHooks(ThreadDeferredBuildStep): name = "create_gitea_project_hooks" config: GiteaConfig + instance_url: str def __init__( self, config: GiteaConfig, + instance_url: str, **kwargs: Any, ) -> None: self.config = config + self.instance_url = instance_url super().__init__(**kwargs) def run_deferred(self) -> None: @@ -277,11 +291,12 @@ class CreateGiteaProjectHooks(ThreadDeferredBuildStep): for repo in repos: create_repo_hook( - self.config.token, - self.config.webhook_secret, - repo.owner.login, - repo.name, - self.config.instance_url, + token=self.config.token, + webhook_secret=self.config.webhook_secret, + owner=repo.owner.login, + repo=repo.name, + gitea_url=self.config.instance_url, + instance_url=self.instance_url, ) def run_post(self) -> Any: From 19d5cdd29a920d2bb885c3129abb9a00c4d8b113 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 24 Jul 2024 17:24:58 +0200 Subject: [PATCH 6/7] Hide gcroot registration steps from the reporters Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 2bab8d5..a4527f0 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -533,6 +533,7 @@ def nix_build_config( updateSourceStamp=False, doStepIf=do_register_gcroot_if, copy_properties=["out_path", "attr"], + set_properties={"report_status": False} ), ) factory.addStep( @@ -597,6 +598,7 @@ def nix_skipped_build_config( updateSourceStamp=False, doStepIf=do_register_gcroot_if, copy_properties=["out_path", "attr"], + set_properties={"report_status": False} ), ) return util.BuilderConfig( From 0ecf33f8d404fc017c0d88e36895bc82a4951442 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 24 Jul 2024 21:22:46 +0200 Subject: [PATCH 7/7] Reformat `gitea_projects.py`, `__init__.py`, and `github_projects.py` Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 5 +++-- buildbot_nix/common.py | 1 - buildbot_nix/gitea_projects.py | 18 ++++++++++++++---- buildbot_nix/github_projects.py | 20 +++++++++++++++----- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index a4527f0..932708b 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -533,7 +533,7 @@ def nix_build_config( updateSourceStamp=False, doStepIf=do_register_gcroot_if, copy_properties=["out_path", "attr"], - set_properties={"report_status": False} + set_properties={"report_status": False}, ), ) factory.addStep( @@ -598,7 +598,7 @@ def nix_skipped_build_config( updateSourceStamp=False, doStepIf=do_register_gcroot_if, copy_properties=["out_path", "attr"], - set_properties={"report_status": False} + set_properties={"report_status": False}, ), ) return util.BuilderConfig( @@ -643,6 +643,7 @@ def nix_register_gcroot_config( factory=factory, ) + def nix_build_combined_config( project: GitProject, worker_names: list[str], diff --git a/buildbot_nix/common.py b/buildbot_nix/common.py index ed0a29d..c2d0ee2 100644 --- a/buildbot_nix/common.py +++ b/buildbot_nix/common.py @@ -180,4 +180,3 @@ def filter_for_combined_builds(reports: Any) -> Any | None: if "report_status" in properties and not properties["report_status"][0]: return None return reports - diff --git a/buildbot_nix/gitea_projects.py b/buildbot_nix/gitea_projects.py index 1282b26..24d6091 100644 --- a/buildbot_nix/gitea_projects.py +++ b/buildbot_nix/gitea_projects.py @@ -1,7 +1,8 @@ import os import signal +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any from urllib.parse import urlparse from buildbot.config.builder import BuilderConfig @@ -13,9 +14,9 @@ from buildbot.www.avatar import AvatarBase from buildbot_gitea.auth import GiteaAuth # type: ignore[import] from buildbot_gitea.reporter import GiteaStatusPush # type: ignore[import] from pydantic import BaseModel +from twisted.internet import defer from twisted.logger import Logger from twisted.python import log -from twisted.internet import defer from .common import ( ThreadDeferredBuildStep, @@ -106,13 +107,22 @@ class GiteaProject(GitProject): # TODO Gitea doesn't include this information return False # self.data["owner"]["type"] == "Organization" + class ModifyingGiteaStatusPush(GiteaStatusPush): - def checkConfig(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + def checkConfig( + self, + modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803 + **kwargs: Any, + ) -> Any: self.modifyingFilter = modifyingFilter return super().checkConfig(**kwargs) - def reconfigService(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + def reconfigService( + self, + modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803 + **kwargs: Any, + ) -> Any: self.modifyingFilter = modifyingFilter return super().reconfigService(**kwargs) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 35d5fe5..0b02b16 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -2,10 +2,11 @@ import json import os import signal from abc import ABC, abstractmethod +from collections.abc import Callable from dataclasses import dataclass from itertools import starmap from pathlib import Path -from typing import Any, Callable +from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.plugins import util @@ -18,9 +19,9 @@ from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase, AvatarGitHub from buildbot.www.oauth2 import GitHubAuth from pydantic import BaseModel, ConfigDict, Field +from twisted.internet import defer from twisted.logger import Logger from twisted.python import log -from twisted.internet import defer from .common import ( ThreadDeferredBuildStep, @@ -311,13 +312,22 @@ class GithubAuthBackend(ABC): ) -> list[BuildStep]: pass + class ModifyingGitHubStatusPush(GitHubStatusPush): - def checkConfig(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + def checkConfig( + self, + modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803 + **kwargs: Any, + ) -> Any: self.modifyingFilter = modifyingFilter return super().checkConfig(**kwargs) - def reconfigService(self, modifyingFilter: Callable[[Any], Any | None] = lambda x: x, **kwargs: Any) -> Any: + def reconfigService( + self, + modifyingFilter: Callable[[Any], Any | None] = lambda x: x, # noqa: N803 + **kwargs: Any, + ) -> Any: self.modifyingFilter = modifyingFilter return super().reconfigService(**kwargs) @@ -331,6 +341,7 @@ class ModifyingGitHubStatusPush(GitHubStatusPush): result = yield super().sendMessage(reports) return result + class GithubLegacyAuthBackend(GithubAuthBackend): auth_type: GitHubLegacyConfig @@ -438,7 +449,6 @@ class GithubAppAuthBackend(GithubAuthBackend): self.project_id_map[props["projectname"]] ].get() - return ModifyingGitHubStatusPush( token=WithProperties("%(github_token)s", github_token=get_github_token), # Since we dynamically create build steps,