diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 1ddd675..be820d4 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -20,12 +20,13 @@ from buildbot.process.project import Project from buildbot.process.properties import Interpolate, Properties from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.steps.trigger import Trigger +from buildbot.util import asyncSleep from twisted.internet import defer, threads from twisted.python.failure import Failure -from .github_projects import ( # noqa: E402 +from .github_projects import ( GithubProject, - create_project_hook, + create_project_hook, # noqa: E402 load_projects, refresh_projects, slugify_project_name, @@ -328,6 +329,25 @@ def reload_github_projects( ) +# The builtin retry mechanism doesn't seem to work for github, +# since github is sometimes not delivering the pull request ref fast enough. +class GitWithRetry(steps.Git): + @defer.inlineCallbacks + def run_vc( + self, branch: str, revision: str, patch: str + ) -> Generator[Any, object, Any]: + retry_counter = 0 + while True: + try: + res = yield super().run_vc(branch, revision, patch) + return res + except Exception as e: + retry_counter += 1 + if retry_counter == 3: + raise e + self.step_status.setText(f"Retrying git clone (error: {e})") + yield asyncSleep(2 << retry_counter) # 2, 4, 8 + def nix_eval_config( project: GithubProject, @@ -348,7 +368,7 @@ def nix_eval_config( f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s" ) factory.addStep( - steps.Git( + GitWithRetry( repourl=url_with_secret, method="clean", submodules=True, @@ -690,8 +710,6 @@ class NixConfigurator(ConfiguratorBase): self.url + "change_hook/github", webhook_secret, ) - - for project in projects: config_for_project( config, project,