From 5163434da625c997ebd0a635e842f7a0fe3f2452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 13 Jul 2024 11:36:57 +0200 Subject: [PATCH] github/reporter: use properties to render github secret --- buildbot_nix/github_projects.py | 11 +- ...allation-tokens-in-GithubStatusPush-.patch | 66 --------- ...-github-render-token-for-each-reques.patch | 133 ++++++++++++++++++ nix/master.nix | 2 +- 4 files changed, 141 insertions(+), 71 deletions(-) delete mode 100644 nix/0001-Support-per-installation-tokens-in-GithubStatusPush-.patch create mode 100644 nix/0001-master-reporters-github-render-token-for-each-reques.patch diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 58af41e..1d567cd 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -10,7 +10,7 @@ from typing import Any from buildbot.config.builder import BuilderConfig from buildbot.plugins import util from buildbot.process.buildstep import BuildStep -from buildbot.process.properties import Interpolate +from buildbot.process.properties import Interpolate, Properties, WithProperties from buildbot.reporters.base import ReporterBase from buildbot.reporters.github import GitHubStatusPush from buildbot.secrets.providers.base import SecretProviderBase @@ -387,10 +387,13 @@ class GithubAppAuthBackend(GithubAuthBackend): return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)] def create_reporter(self) -> ReporterBase: + def get_github_token(props: Properties) -> str: + return self.installation_tokens[ + self.project_id_map[props["projectname"]] + ].get() + return GitHubStatusPush( - token=lambda project: self.installation_tokens[ - self.project_id_map[project] - ].get(), + token=WithProperties("%(github_token)s", github_token=get_github_token), # Since we dynamically create build steps, # we use `virtual_builder_name` in the webinterface # so that we distinguish what has beeing build diff --git a/nix/0001-Support-per-installation-tokens-in-GithubStatusPush-.patch b/nix/0001-Support-per-installation-tokens-in-GithubStatusPush-.patch deleted file mode 100644 index 63facce..0000000 --- a/nix/0001-Support-per-installation-tokens-in-GithubStatusPush-.patch +++ /dev/null @@ -1,66 +0,0 @@ -From 65dd3e63dc0dc34a531eee703baff715ba04d0a0 Mon Sep 17 00:00:00 2001 -From: magic_rb -Date: Wed, 10 Jul 2024 13:00:48 +0200 -Subject: [PATCH 1/3] Support per installation tokens in `GithubStatusPush` - reporter - -Signed-off-by: magic_rb ---- - master/buildbot/reporters/github.py | 20 +++++++++++++++++--- - 1 file changed, 17 insertions(+), 3 deletions(-) - -diff --git a/master/buildbot/reporters/github.py b/master/buildbot/reporters/github.py -index 3873c2676..e61e6495a 100644 ---- a/master/buildbot/reporters/github.py -+++ b/master/buildbot/reporters/github.py -@@ -70,7 +70,14 @@ class GitHubStatusPush(ReporterBase): - generators=None, - **kwargs, - ): -- token = yield self.renderSecrets(token) -+ headers = {} -+ if not callable(token): -+ token = yield self.renderSecrets(token) -+ headers['Authorization'] = 'token ' + token -+ else: -+ self.token_cache = {} -+ self.token_source = token -+ - self.debug = debug - self.verify = verify - self.verbose = verbose -@@ -89,7 +96,7 @@ class GitHubStatusPush(ReporterBase): - self._http = yield httpclientservice.HTTPClientService.getService( - self.master, - baseURL, -- headers={'Authorization': 'token ' + token, 'User-Agent': 'Buildbot'}, -+ headers={'User-Agent': 'Buildbot'} | headers, - debug=self.debug, - verify=self.verify, - ) -@@ -135,6 +142,11 @@ class GitHubStatusPush(ReporterBase): - txgithub is based on twisted's webclient agent, which is much less reliable and featureful - as txrequest (support for proxy, connection pool, keep alive, retry, etc) - """ -+ headers = {} -+ -+ if hasattr(self, 'token_source'): -+ headers['Authorization'] = 'token ' + self.token_source(f"{repo_user}/{repo_name}") -+ - payload = {'state': state} - - if description is not None: -@@ -147,7 +159,9 @@ class GitHubStatusPush(ReporterBase): - payload['context'] = context - - return self._http.post( -- '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]), json=payload -+ '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]), -+ json=payload, -+ headers=headers, - ) - - def is_status_2xx(self, code): --- -2.44.1 - diff --git a/nix/0001-master-reporters-github-render-token-for-each-reques.patch b/nix/0001-master-reporters-github-render-token-for-each-reques.patch new file mode 100644 index 0000000..039b894 --- /dev/null +++ b/nix/0001-master-reporters-github-render-token-for-each-reques.patch @@ -0,0 +1,133 @@ +From 788a484aa700b1541600333ab6bbc08653a4edd1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= +Date: Sat, 13 Jul 2024 10:52:49 +0200 +Subject: [PATCH] master/reporters/github: render token for each request +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Github token can expire i.e. if an GitHub apps are used +the token needs to be renewed every hour from the installation token. +By moving the renderSecrets just when the requests happens, +we provide an interface for users to provide a new refresh GITHUB_TOKEN + +Signed-off-by: Jörg Thalheim +--- + master/buildbot/reporters/github.py | 28 +++++++++++++++---- + .../github-reporter-secret-handling.change | 3 ++ + 2 files changed, 26 insertions(+), 5 deletions(-) + create mode 100644 newsfragments/github-reporter-secret-handling.change + +diff --git a/master/buildbot/reporters/github.py b/master/buildbot/reporters/github.py +index 3873c2676..27242831f 100644 +--- a/master/buildbot/reporters/github.py ++++ b/master/buildbot/reporters/github.py +@@ -70,7 +70,7 @@ def reconfigService( + generators=None, + **kwargs, + ): +- token = yield self.renderSecrets(token) ++ self.token = token + self.debug = debug + self.verify = verify + self.verbose = verbose +@@ -89,7 +89,7 @@ def reconfigService( + self._http = yield httpclientservice.HTTPClientService.getService( + self.master, + baseURL, +- headers={'Authorization': 'token ' + token, 'User-Agent': 'Buildbot'}, ++ headers={'User-Agent': 'Buildbot'}, + debug=self.debug, + verify=self.verify, + ) +@@ -109,6 +109,15 @@ def _create_default_generators(self): + ), + ] + ++ @defer.inlineCallbacks ++ def _get_auth_header(self, props: Properties | None) -> dict[str, str]: ++ if props: ++ token = yield props.render(self.token) ++ else: ++ token = yield self.renderSecrets(self.token) ++ return {'Authorization': f"token {token}"} ++ ++ @defer.inlineCallbacks + def createStatus( + self, + repo_user, +@@ -119,6 +128,7 @@ def createStatus( + context=None, + issue=None, + description=None, ++ props=None, + ): + """ + :param repo_user: GitHub user or organization +@@ -137,6 +147,7 @@ def createStatus( + """ + payload = {'state': state} + ++ + if description is not None: + payload['description'] = description + +@@ -146,9 +157,11 @@ def createStatus( + if context is not None: + payload['context'] = context + +- return self._http.post( +- '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]), json=payload ++ headers = yield self._get_auth_header(props) ++ ret = yield self._http.post( ++ '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]), json=payload, headers=headers + ) ++ return ret + + def is_status_2xx(self, code): + return code // 100 == 2 +@@ -241,6 +254,7 @@ def sendMessage(self, reports): + context=context, + issue=issue, + description=description, ++ props=props, + ) + + if not response: +@@ -305,6 +319,7 @@ def createStatus( + context=None, + issue=None, + description=None, ++ props=None, + ): + """ + :param repo_user: GitHub user or organization +@@ -313,6 +328,8 @@ def createStatus( + :param state: one of the following 'pending', 'success', 'error' + or 'failure'. + :param description: Short description of the status. ++ :param context: Build context ++ :param props: Properties object of the build (used for render GITHUB_TOKEN secret) + :return: A deferred with the result from GitHub. + + This code comes from txgithub by @tomprince. +@@ -328,5 +345,6 @@ def createStatus( + return None + + url = '/'.join(['/repos', repo_user, repo_name, 'issues', issue, 'comments']) +- ret = yield self._http.post(url, json=payload) ++ headers = yield self._get_auth_header(props) ++ ret = yield self._http.post(url, json=payload, headers=headers) + return ret +diff --git a/newsfragments/github-reporter-secret-handling.change b/newsfragments/github-reporter-secret-handling.change +new file mode 100644 +index 000000000..a1ebd5408 +--- /dev/null ++++ b/newsfragments/github-reporter-secret-handling.change +@@ -0,0 +1,3 @@ ++GitHubStatusPush will now render github tokens right before the request. ++This allow to update the token in the configuration file without restarting the server, ++which is in example useful for Githuhb App installations. +-- +2.45.1 + diff --git a/nix/master.nix b/nix/master.nix index b385c00..87dfd5a 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -401,7 +401,7 @@ in dbUrl = config.services.buildbot-nix.master.dbUrl; package = cfg.buildbotNixpkgs.buildbot.overrideAttrs (old: { - patches = old.patches ++ [ ./0001-Support-per-installation-tokens-in-GithubStatusPush-.patch ]; + patches = old.patches ++ [ ./0001-master-reporters-github-render-token-for-each-reques.patch ]; }); pythonPackages = ps: [ ps.requests