From 06f5aa9cd79a4d7d6b8afbae9898b21355550b63 Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Mon, 7 Dec 2020 10:56:04 +0100
Subject: [PATCH] Add retry to parts of the import script which were failing
 from time to time (#249)

---
 import-scripts/import_scripts/channel.py | 10 +++++++++-
 import-scripts/poetry.lock               | 14 +++++++++++++-
 import-scripts/pyproject.toml            |  1 +
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/import-scripts/import_scripts/channel.py b/import-scripts/import_scripts/channel.py
index 2f56f88..6f03569 100644
--- a/import-scripts/import_scripts/channel.py
+++ b/import-scripts/import_scripts/channel.py
@@ -1,6 +1,8 @@
+import backoff  # type: ignore
 import boto3  # type: ignore
 import botocore  # type: ignore
 import botocore.client  # type: ignore
+import botocore.exceptions  # type: ignore
 import click
 import click_log  # type: ignore
 import dictdiffer  # type: ignore
@@ -14,6 +16,7 @@ import os.path
 import pypandoc  # type: ignore
 import re
 import requests
+import requests.exceptions
 import shlex
 import subprocess
 import sys
@@ -262,6 +265,7 @@ def parse_query(text):
     return tokens
 
 
+@backoff.on_exception(backoff.expo, botocore.exceptions.ClientError)
 def get_last_evaluation(prefix):
     logger.debug(f"Retrieving last evaluation for {prefix} prefix.")
 
@@ -302,6 +306,7 @@ def get_last_evaluation(prefix):
     return evaluation
 
 
+@backoff.on_exception(backoff.expo, requests.exceptions.RequestException)
 def get_evaluation_builds(evaluation_id):
     logger.debug(
         f"get_evaluation_builds: Retrieving list of builds for {evaluation_id} evaluation id"
@@ -388,6 +393,7 @@ def remove_attr_set(name):
     return name
 
 
+@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
 def get_packages_raw(evaluation):
     logger.debug(
         f"get_packages: Retrieving list of packages for '{evaluation['git_revision']}' revision"
@@ -497,9 +503,10 @@ def get_packages(evaluation, evaluation_builds):
     return len(packages), gen
 
 
+@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
 def get_options_raw(evaluation):
     logger.debug(
-        f"get_packages: Retrieving list of options for '{evaluation['git_revision']}' revision"
+        f"get_options: Retrieving list of options for '{evaluation['git_revision']}' revision"
     )
     result = subprocess.run(
         shlex.split(
@@ -521,6 +528,7 @@ def get_options(evaluation):
     options = get_options_raw(evaluation)
 
     @functools.lru_cache(maxsize=None)
+    @backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
     def jsonToNix(value):
         result = subprocess.run(
             shlex.split(
diff --git a/import-scripts/poetry.lock b/import-scripts/poetry.lock
index 394078f..79a6e26 100644
--- a/import-scripts/poetry.lock
+++ b/import-scripts/poetry.lock
@@ -47,6 +47,14 @@ optional = false
 python-versions = "*"
 version = "0.2.0"
 
+[[package]]
+category = "main"
+description = "Function decoration for backoff and retry"
+name = "backoff"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+version = "1.10.0"
+
 [[package]]
 category = "dev"
 description = "The uncompromising code formatter."
@@ -868,7 +876,7 @@ version = "0.35.1"
 test = ["pytest (>=3.0.0)", "pytest-cov"]
 
 [metadata]
-content-hash = "60240c730440563a56cdef8cc567b54ba34a5a819442c6e2c8da09af56c9231b"
+content-hash = "fdccd4ed5b12d959552f67bcb3f308ce70bb1d3f418ed1a18d8a8a483401dd05"
 lock-version = "1.0"
 python-versions = "^3.8"
 
@@ -893,6 +901,10 @@ backcall = [
     {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
     {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
 ]
+backoff = [
+    {file = "backoff-1.10.0-py2.py3-none-any.whl", hash = "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd"},
+    {file = "backoff-1.10.0.tar.gz", hash = "sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4"},
+]
 black = [
     {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"},
     {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"},
diff --git a/import-scripts/pyproject.toml b/import-scripts/pyproject.toml
index 26c6379..118d352 100644
--- a/import-scripts/pyproject.toml
+++ b/import-scripts/pyproject.toml
@@ -21,6 +21,7 @@ boto3 = "^1.14.5"
 tqdm = "^4.46.1"
 pypandoc = "^1.5"
 dictdiffer = "^0.8.1"
+backoff = "^1.10.0"
 
 [tool.poetry.dev-dependencies]
 ipdb = "^0.13.2"