github/reporter: use properties to render github secret

buildbot_nix/github_projects.py

@@ -10,7 +10,7 @@ from typing import Any
 from buildbot.config.builder import BuilderConfig
 from buildbot.plugins import util
 from buildbot.process.buildstep import BuildStep
-from buildbot.process.properties import Interpolate
+from buildbot.process.properties import Interpolate, Properties, WithProperties
 from buildbot.reporters.base import ReporterBase
 from buildbot.reporters.github import GitHubStatusPush
 from buildbot.secrets.providers.base import SecretProviderBase
@@ -387,10 +387,13 @@ class GithubAppAuthBackend(GithubAuthBackend):
         return [GitHubAppSecretService(self.installation_tokens, self.jwt_token)]
 
     def create_reporter(self) -> ReporterBase:
+        def get_github_token(props: Properties) -> str:
+            return self.installation_tokens[
+                self.project_id_map[props["projectname"]]
+            ].get()
+
         return GitHubStatusPush(
-            token=lambda project: self.installation_tokens[
+            token=WithProperties("%(github_token)s", github_token=get_github_token),
-                self.project_id_map[project]
-            ].get(),
             # Since we dynamically create build steps,
             # we use `virtual_builder_name` in the webinterface
             # so that we distinguish what has beeing build

nix/0001-Support-per-installation-tokens-in-GithubStatusPush-.patch

@@ -1,66 +0,0 @@
-From 65dd3e63dc0dc34a531eee703baff715ba04d0a0 Mon Sep 17 00:00:00 2001
-From: magic_rb <richard@brezak.sk>
-Date: Wed, 10 Jul 2024 13:00:48 +0200
-Subject: [PATCH 1/3] Support per installation tokens in `GithubStatusPush`
- reporter
-
-Signed-off-by: magic_rb <richard@brezak.sk>
----
- master/buildbot/reporters/github.py | 20 +++++++++++++++++---
- 1 file changed, 17 insertions(+), 3 deletions(-)
-
-diff --git a/master/buildbot/reporters/github.py b/master/buildbot/reporters/github.py
-index 3873c2676..e61e6495a 100644
---- a/master/buildbot/reporters/github.py
-+++ b/master/buildbot/reporters/github.py
-@@ -70,7 +70,14 @@ class GitHubStatusPush(ReporterBase):
-         generators=None,
-         **kwargs,
-     ):
--        token = yield self.renderSecrets(token)
-+        headers = {}
-+        if not callable(token):
-+            token = yield self.renderSecrets(token)
-+            headers['Authorization'] = 'token ' + token
-+        else:
-+            self.token_cache = {}
-+            self.token_source = token
-+
-         self.debug = debug
-         self.verify = verify
-         self.verbose = verbose
-@@ -89,7 +96,7 @@ class GitHubStatusPush(ReporterBase):
-         self._http = yield httpclientservice.HTTPClientService.getService(
-             self.master,
-             baseURL,
--            headers={'Authorization': 'token ' + token, 'User-Agent': 'Buildbot'},
-+            headers={'User-Agent': 'Buildbot'} | headers,
-             debug=self.debug,
-             verify=self.verify,
-         )
-@@ -135,6 +142,11 @@ class GitHubStatusPush(ReporterBase):
-         txgithub is based on twisted's webclient agent, which is much less reliable and featureful
-         as txrequest (support for proxy, connection pool, keep alive, retry, etc)
-         """
-+        headers = {}
-+
-+        if hasattr(self, 'token_source'):
-+            headers['Authorization'] = 'token ' + self.token_source(f"{repo_user}/{repo_name}")
-+
-         payload = {'state': state}
- 
-         if description is not None:
-@@ -147,7 +159,9 @@ class GitHubStatusPush(ReporterBase):
-             payload['context'] = context
- 
-         return self._http.post(
--            '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]), json=payload
-+            '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]),
-+            json=payload,
-+            headers=headers,
-         )
- 
-     def is_status_2xx(self, code):
--- 
-2.44.1
-

nix/0001-master-reporters-github-render-token-for-each-reques.patch

@@ -0,0 +1,133 @@
+From 788a484aa700b1541600333ab6bbc08653a4edd1 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
+Date: Sat, 13 Jul 2024 10:52:49 +0200
+Subject: [PATCH] master/reporters/github: render token for each request
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Github token can expire i.e. if an GitHub apps are used
+the token needs to be renewed every hour from the installation token.
+By moving the renderSecrets just when the requests happens,
+we provide an interface for users to provide a new refresh GITHUB_TOKEN
+
+Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
+---
+ master/buildbot/reporters/github.py           | 28 +++++++++++++++----
+ .../github-reporter-secret-handling.change    |  3 ++
+ 2 files changed, 26 insertions(+), 5 deletions(-)
+ create mode 100644 newsfragments/github-reporter-secret-handling.change
+
+diff --git a/master/buildbot/reporters/github.py b/master/buildbot/reporters/github.py
+index 3873c2676..27242831f 100644
+--- a/master/buildbot/reporters/github.py
++++ b/master/buildbot/reporters/github.py
+@@ -70,7 +70,7 @@ def reconfigService(
+         generators=None,
+         **kwargs,
+     ):
+-        token = yield self.renderSecrets(token)
++        self.token = token
+         self.debug = debug
+         self.verify = verify
+         self.verbose = verbose
+@@ -89,7 +89,7 @@ def reconfigService(
+         self._http = yield httpclientservice.HTTPClientService.getService(
+             self.master,
+             baseURL,
+-            headers={'Authorization': 'token ' + token, 'User-Agent': 'Buildbot'},
++            headers={'User-Agent': 'Buildbot'},
+             debug=self.debug,
+             verify=self.verify,
+         )
+@@ -109,6 +109,15 @@ def _create_default_generators(self):
+             ),
+         ]
+ 
++    @defer.inlineCallbacks
++    def _get_auth_header(self, props: Properties | None) -> dict[str, str]:
++        if props:
++            token = yield props.render(self.token)
++        else:
++            token = yield self.renderSecrets(self.token)
++        return {'Authorization': f"token {token}"}
++
++    @defer.inlineCallbacks
+     def createStatus(
+         self,
+         repo_user,
+@@ -119,6 +128,7 @@ def createStatus(
+         context=None,
+         issue=None,
+         description=None,
++        props=None,
+     ):
+         """
+         :param repo_user: GitHub user or organization
+@@ -137,6 +147,7 @@ def createStatus(
+         """
+         payload = {'state': state}
+ 
++
+         if description is not None:
+             payload['description'] = description
+ 
+@@ -146,9 +157,11 @@ def createStatus(
+         if context is not None:
+             payload['context'] = context
+ 
+-        return self._http.post(
+-            '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]), json=payload
++        headers = yield self._get_auth_header(props)
++        ret = yield self._http.post(
++            '/'.join(['/repos', repo_user, repo_name, 'statuses', sha]), json=payload, headers=headers
+         )
++        return ret
+ 
+     def is_status_2xx(self, code):
+         return code // 100 == 2
+@@ -241,6 +254,7 @@ def sendMessage(self, reports):
+                     context=context,
+                     issue=issue,
+                     description=description,
++                    props=props,
+                 )
+ 
+                 if not response:
+@@ -305,6 +319,7 @@ def createStatus(
+         context=None,
+         issue=None,
+         description=None,
++        props=None,
+     ):
+         """
+         :param repo_user: GitHub user or organization
+@@ -313,6 +328,8 @@ def createStatus(
+         :param state: one of the following 'pending', 'success', 'error'
+                       or 'failure'.
+         :param description: Short description of the status.
++        :param context: Build context
++        :param props: Properties object of the build (used for render GITHUB_TOKEN secret)
+         :return: A deferred with the result from GitHub.
+ 
+         This code comes from txgithub by @tomprince.
+@@ -328,5 +345,6 @@ def createStatus(
+             return None
+ 
+         url = '/'.join(['/repos', repo_user, repo_name, 'issues', issue, 'comments'])
+-        ret = yield self._http.post(url, json=payload)
++        headers = yield self._get_auth_header(props)
++        ret = yield self._http.post(url, json=payload, headers=headers)
+         return ret
+diff --git a/newsfragments/github-reporter-secret-handling.change b/newsfragments/github-reporter-secret-handling.change
+new file mode 100644
+index 000000000..a1ebd5408
+--- /dev/null
++++ b/newsfragments/github-reporter-secret-handling.change
+@@ -0,0 +1,3 @@
++GitHubStatusPush will now render github tokens right before the request.
++This allow to update the token in the configuration file without restarting the server,
++which is in example useful for Githuhb App installations.
+-- 
+2.45.1
+

nix/master.nix

@@ -401,7 +401,7 @@ in
         dbUrl = config.services.buildbot-nix.master.dbUrl;
 
         package = cfg.buildbotNixpkgs.buildbot.overrideAttrs (old: {
-          patches = old.patches ++ [ ./0001-Support-per-installation-tokens-in-GithubStatusPush-.patch ];
+          patches = old.patches ++ [ ./0001-master-reporters-github-render-token-for-each-reques.patch ];
         });
         pythonPackages = ps: [
           ps.requests