Add retry to parts of the import script which were failing from time to time (#249)

This commit is contained in:
Rok Garbas 2020-12-07 10:56:04 +01:00 committed by GitHub
parent bae5a92131
commit 06f5aa9cd7
Failed to generate hash of commit
3 changed files with 23 additions and 2 deletions

View file

@ -1,6 +1,8 @@
import backoff # type: ignore
import boto3 # type: ignore import boto3 # type: ignore
import botocore # type: ignore import botocore # type: ignore
import botocore.client # type: ignore import botocore.client # type: ignore
import botocore.exceptions # type: ignore
import click import click
import click_log # type: ignore import click_log # type: ignore
import dictdiffer # type: ignore import dictdiffer # type: ignore
@ -14,6 +16,7 @@ import os.path
import pypandoc # type: ignore import pypandoc # type: ignore
import re import re
import requests import requests
import requests.exceptions
import shlex import shlex
import subprocess import subprocess
import sys import sys
@ -262,6 +265,7 @@ def parse_query(text):
return tokens return tokens
@backoff.on_exception(backoff.expo, botocore.exceptions.ClientError)
def get_last_evaluation(prefix): def get_last_evaluation(prefix):
logger.debug(f"Retrieving last evaluation for {prefix} prefix.") logger.debug(f"Retrieving last evaluation for {prefix} prefix.")
@ -302,6 +306,7 @@ def get_last_evaluation(prefix):
return evaluation return evaluation
@backoff.on_exception(backoff.expo, requests.exceptions.RequestException)
def get_evaluation_builds(evaluation_id): def get_evaluation_builds(evaluation_id):
logger.debug( logger.debug(
f"get_evaluation_builds: Retrieving list of builds for {evaluation_id} evaluation id" f"get_evaluation_builds: Retrieving list of builds for {evaluation_id} evaluation id"
@ -388,6 +393,7 @@ def remove_attr_set(name):
return name return name
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
def get_packages_raw(evaluation): def get_packages_raw(evaluation):
logger.debug( logger.debug(
f"get_packages: Retrieving list of packages for '{evaluation['git_revision']}' revision" f"get_packages: Retrieving list of packages for '{evaluation['git_revision']}' revision"
@ -497,9 +503,10 @@ def get_packages(evaluation, evaluation_builds):
return len(packages), gen return len(packages), gen
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
def get_options_raw(evaluation): def get_options_raw(evaluation):
logger.debug( logger.debug(
f"get_packages: Retrieving list of options for '{evaluation['git_revision']}' revision" f"get_options: Retrieving list of options for '{evaluation['git_revision']}' revision"
) )
result = subprocess.run( result = subprocess.run(
shlex.split( shlex.split(
@ -521,6 +528,7 @@ def get_options(evaluation):
options = get_options_raw(evaluation) options = get_options_raw(evaluation)
@functools.lru_cache(maxsize=None) @functools.lru_cache(maxsize=None)
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
def jsonToNix(value): def jsonToNix(value):
result = subprocess.run( result = subprocess.run(
shlex.split( shlex.split(

View file

@ -47,6 +47,14 @@ optional = false
python-versions = "*" python-versions = "*"
version = "0.2.0" version = "0.2.0"
[[package]]
category = "main"
description = "Function decoration for backoff and retry"
name = "backoff"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "1.10.0"
[[package]] [[package]]
category = "dev" category = "dev"
description = "The uncompromising code formatter." description = "The uncompromising code formatter."
@ -868,7 +876,7 @@ version = "0.35.1"
test = ["pytest (>=3.0.0)", "pytest-cov"] test = ["pytest (>=3.0.0)", "pytest-cov"]
[metadata] [metadata]
content-hash = "60240c730440563a56cdef8cc567b54ba34a5a819442c6e2c8da09af56c9231b" content-hash = "fdccd4ed5b12d959552f67bcb3f308ce70bb1d3f418ed1a18d8a8a483401dd05"
lock-version = "1.0" lock-version = "1.0"
python-versions = "^3.8" python-versions = "^3.8"
@ -893,6 +901,10 @@ backcall = [
{file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
{file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
] ]
backoff = [
{file = "backoff-1.10.0-py2.py3-none-any.whl", hash = "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd"},
{file = "backoff-1.10.0.tar.gz", hash = "sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4"},
]
black = [ black = [
{file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"}, {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"},
{file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"},

View file

@ -21,6 +21,7 @@ boto3 = "^1.14.5"
tqdm = "^4.46.1" tqdm = "^4.46.1"
pypandoc = "^1.5" pypandoc = "^1.5"
dictdiffer = "^0.8.1" dictdiffer = "^0.8.1"
backoff = "^1.10.0"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
ipdb = "^0.13.2" ipdb = "^0.13.2"