Add retry to parts of the import script which were failing from time to time (#249)
This commit is contained in:
parent
bae5a92131
commit
06f5aa9cd7
|
@ -1,6 +1,8 @@
|
|||
import backoff # type: ignore
|
||||
import boto3 # type: ignore
|
||||
import botocore # type: ignore
|
||||
import botocore.client # type: ignore
|
||||
import botocore.exceptions # type: ignore
|
||||
import click
|
||||
import click_log # type: ignore
|
||||
import dictdiffer # type: ignore
|
||||
|
@ -14,6 +16,7 @@ import os.path
|
|||
import pypandoc # type: ignore
|
||||
import re
|
||||
import requests
|
||||
import requests.exceptions
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
@ -262,6 +265,7 @@ def parse_query(text):
|
|||
return tokens
|
||||
|
||||
|
||||
@backoff.on_exception(backoff.expo, botocore.exceptions.ClientError)
|
||||
def get_last_evaluation(prefix):
|
||||
logger.debug(f"Retrieving last evaluation for {prefix} prefix.")
|
||||
|
||||
|
@ -302,6 +306,7 @@ def get_last_evaluation(prefix):
|
|||
return evaluation
|
||||
|
||||
|
||||
@backoff.on_exception(backoff.expo, requests.exceptions.RequestException)
|
||||
def get_evaluation_builds(evaluation_id):
|
||||
logger.debug(
|
||||
f"get_evaluation_builds: Retrieving list of builds for {evaluation_id} evaluation id"
|
||||
|
@ -388,6 +393,7 @@ def remove_attr_set(name):
|
|||
return name
|
||||
|
||||
|
||||
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
|
||||
def get_packages_raw(evaluation):
|
||||
logger.debug(
|
||||
f"get_packages: Retrieving list of packages for '{evaluation['git_revision']}' revision"
|
||||
|
@ -497,9 +503,10 @@ def get_packages(evaluation, evaluation_builds):
|
|||
return len(packages), gen
|
||||
|
||||
|
||||
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
|
||||
def get_options_raw(evaluation):
|
||||
logger.debug(
|
||||
f"get_packages: Retrieving list of options for '{evaluation['git_revision']}' revision"
|
||||
f"get_options: Retrieving list of options for '{evaluation['git_revision']}' revision"
|
||||
)
|
||||
result = subprocess.run(
|
||||
shlex.split(
|
||||
|
@ -521,6 +528,7 @@ def get_options(evaluation):
|
|||
options = get_options_raw(evaluation)
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
|
||||
def jsonToNix(value):
|
||||
result = subprocess.run(
|
||||
shlex.split(
|
||||
|
|
14
import-scripts/poetry.lock
generated
14
import-scripts/poetry.lock
generated
|
@ -47,6 +47,14 @@ optional = false
|
|||
python-versions = "*"
|
||||
version = "0.2.0"
|
||||
|
||||
[[package]]
|
||||
category = "main"
|
||||
description = "Function decoration for backoff and retry"
|
||||
name = "backoff"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
version = "1.10.0"
|
||||
|
||||
[[package]]
|
||||
category = "dev"
|
||||
description = "The uncompromising code formatter."
|
||||
|
@ -868,7 +876,7 @@ version = "0.35.1"
|
|||
test = ["pytest (>=3.0.0)", "pytest-cov"]
|
||||
|
||||
[metadata]
|
||||
content-hash = "60240c730440563a56cdef8cc567b54ba34a5a819442c6e2c8da09af56c9231b"
|
||||
content-hash = "fdccd4ed5b12d959552f67bcb3f308ce70bb1d3f418ed1a18d8a8a483401dd05"
|
||||
lock-version = "1.0"
|
||||
python-versions = "^3.8"
|
||||
|
||||
|
@ -893,6 +901,10 @@ backcall = [
|
|||
{file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
|
||||
{file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
|
||||
]
|
||||
backoff = [
|
||||
{file = "backoff-1.10.0-py2.py3-none-any.whl", hash = "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd"},
|
||||
{file = "backoff-1.10.0.tar.gz", hash = "sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4"},
|
||||
]
|
||||
black = [
|
||||
{file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"},
|
||||
{file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"},
|
||||
|
|
|
@ -21,6 +21,7 @@ boto3 = "^1.14.5"
|
|||
tqdm = "^4.46.1"
|
||||
pypandoc = "^1.5"
|
||||
dictdiffer = "^0.8.1"
|
||||
backoff = "^1.10.0"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
ipdb = "^0.13.2"
|
||||
|
|
Loading…
Reference in a new issue