some logging added and hidden behind --verbose flag (#45)
This commit is contained in:
parent
4629bc39cc
commit
b264d7af55
|
@ -12,6 +12,8 @@
|
||||||
|
|
||||||
import boto3
|
import boto3
|
||||||
import click
|
import click
|
||||||
|
import logging
|
||||||
|
import click_log
|
||||||
import elasticsearch
|
import elasticsearch
|
||||||
import elasticsearch.helpers
|
import elasticsearch.helpers
|
||||||
import json
|
import json
|
||||||
|
@ -22,6 +24,9 @@ import tqdm
|
||||||
import botocore.client
|
import botocore.client
|
||||||
import botocore
|
import botocore
|
||||||
|
|
||||||
|
logger = logging.getLogger("import-channel")
|
||||||
|
click_log.basic_config(logger)
|
||||||
|
|
||||||
|
|
||||||
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
||||||
|
@ -68,17 +73,27 @@ ANALYSIS = {
|
||||||
|
|
||||||
|
|
||||||
def get_last_evaluation(channel):
|
def get_last_evaluation(channel):
|
||||||
|
logger.debug(f"Retriving last evaluation for {channel} channel")
|
||||||
|
|
||||||
project, project_version = channel.split("-", 1)
|
project, project_version = channel.split("-", 1)
|
||||||
|
logger.debug(f"get_last_evaluation: project='{project}'")
|
||||||
|
logger.debug(f"get_last_evaluation: project_version='{project_version}'")
|
||||||
|
|
||||||
|
bucket = "nix-releases"
|
||||||
|
prefix = f"{project}/{project_version}/"
|
||||||
|
logger.debug(
|
||||||
|
f"get_last_evaluation: list all evaluation in '{bucket}' bucker under '{prefix}' prefix"
|
||||||
|
)
|
||||||
|
|
||||||
s3 = boto3.client(
|
s3 = boto3.client(
|
||||||
"s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED)
|
"s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED)
|
||||||
)
|
)
|
||||||
s3_result = s3.list_objects(
|
s3_result = s3.list_objects(Bucket=bucket, Prefix=prefix, Delimiter="/",)
|
||||||
Bucket="nix-releases", Prefix=f"{project}/{project_version}/", Delimiter="/",
|
|
||||||
)
|
|
||||||
evaluations = []
|
evaluations = []
|
||||||
for item in s3_result.get("CommonPrefixes"):
|
for item in s3_result.get("CommonPrefixes"):
|
||||||
if not item:
|
if not item:
|
||||||
continue
|
continue
|
||||||
|
logger.debug(f"get_last_evaluation: evaluation in raw {item}")
|
||||||
prefix = item.get("Prefix")
|
prefix = item.get("Prefix")
|
||||||
evaluation = prefix[len(f"{project}/{project_version}/{channel}") :]
|
evaluation = prefix[len(f"{project}/{project_version}/{channel}") :]
|
||||||
if evaluation.startswith("beta"):
|
if evaluation.startswith("beta"):
|
||||||
|
@ -89,19 +104,27 @@ def get_last_evaluation(channel):
|
||||||
)
|
)
|
||||||
except Exception as e: # noqa
|
except Exception as e: # noqa
|
||||||
continue
|
continue
|
||||||
evaluations.append(
|
evaluation = {
|
||||||
dict(
|
"revisions_since_start": int(revisions_since_start),
|
||||||
revisions_since_start=int(revisions_since_start),
|
"git_revision": git_revision,
|
||||||
git_revision=git_revision,
|
"prefix": prefix,
|
||||||
prefix=prefix,
|
}
|
||||||
)
|
logger.debug(f"get_last_evaluation: evaluation {evaluation}")
|
||||||
)
|
evaluations.append(evaluation)
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
f"get_last_evaluation: {len(evaluations)} evaluations found for {channel} channel"
|
||||||
|
)
|
||||||
evaluations = sorted(evaluations, key=lambda i: i["revisions_since_start"])
|
evaluations = sorted(evaluations, key=lambda i: i["revisions_since_start"])
|
||||||
|
|
||||||
|
logger.debug(f"get_last_evaluation: last evaluation is: {evaluations[-1]}")
|
||||||
return evaluations[-1]
|
return evaluations[-1]
|
||||||
|
|
||||||
|
|
||||||
def get_packages(evaluation):
|
def get_packages(evaluation):
|
||||||
|
logger.debug(
|
||||||
|
f"get_packages: Retriving list of packages for '{evaluation['git_revision']}' revision"
|
||||||
|
)
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
shlex.split(
|
shlex.split(
|
||||||
f"nix-env -f '<nixpkgs>' -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/{evaluation['git_revision']}.tar.gz --arg config 'import {CURRENT_DIR}/packages-config.nix' -qa --json"
|
f"nix-env -f '<nixpkgs>' -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/{evaluation['git_revision']}.tar.gz --arg config 'import {CURRENT_DIR}/packages-config.nix' -qa --json"
|
||||||
|
@ -170,6 +193,7 @@ def get_packages(evaluation):
|
||||||
)
|
)
|
||||||
yield doc
|
yield doc
|
||||||
|
|
||||||
|
logger.debug(f"get_packages: Found {len(packages)} packages")
|
||||||
return len(packages), gen
|
return len(packages), gen
|
||||||
|
|
||||||
|
|
||||||
|
@ -212,10 +236,14 @@ def get_options(evaluation):
|
||||||
|
|
||||||
|
|
||||||
def recreate_index(es, channel):
|
def recreate_index(es, channel):
|
||||||
if es.indices.exists(f"{channel}-packages"):
|
packages_index = f"{channel}-packages"
|
||||||
es.indices.delete(index=f"{channel}-packages")
|
if es.indices.exists(packages_index):
|
||||||
|
es.indices.delete(index=packages_index)
|
||||||
|
logger.debug(
|
||||||
|
f"recreate_index: index '{packages_index}' already exists and was deleted"
|
||||||
|
)
|
||||||
es.indices.create(
|
es.indices.create(
|
||||||
index=f"{channel}-packages",
|
index=packages_index,
|
||||||
body=dict(
|
body=dict(
|
||||||
settings=dict(number_of_shards=1, analysis=ANALYSIS),
|
settings=dict(number_of_shards=1, analysis=ANALYSIS),
|
||||||
mappings=dict(
|
mappings=dict(
|
||||||
|
@ -251,10 +279,16 @@ def recreate_index(es, channel):
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
if es.indices.exists(f"{channel}-options"):
|
logger.debug(f"recreate_index: index '{packages_index}' was created")
|
||||||
es.indices.delete(index=f"{channel}-options")
|
|
||||||
|
options_index = f"{channel}-options"
|
||||||
|
if es.indices.exists(options_index):
|
||||||
|
es.indices.delete(index=options_index)
|
||||||
|
logger.debug(
|
||||||
|
f"recreate_index: index '{options_index}' already exists and was deleted"
|
||||||
|
)
|
||||||
es.indices.create(
|
es.indices.create(
|
||||||
index=f"{channel}-options",
|
index=options_index,
|
||||||
body=dict(
|
body=dict(
|
||||||
settings=dict(number_of_shards=1, analysis=ANALYSIS),
|
settings=dict(number_of_shards=1, analysis=ANALYSIS),
|
||||||
mappings=dict(
|
mappings=dict(
|
||||||
|
@ -269,12 +303,25 @@ def recreate_index(es, channel):
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
logger.debug(f"recreate_index: index '{options_index}' was created")
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
@click.command()
|
||||||
@click.option("--es-url", help="Elasticsearch connection url")
|
@click.option("-u", "--es-url", help="Elasticsearch connection url")
|
||||||
@click.option("--channel")
|
@click.option("-c", "--channel", help="NixOS channel name")
|
||||||
def main(es_url, channel):
|
@click.option("-v", "--verbose", count=True)
|
||||||
|
def main(es_url, channel, verbose):
|
||||||
|
|
||||||
|
logging_level = "CRITICAL"
|
||||||
|
if verbose == 1:
|
||||||
|
logging_level = "WARNING"
|
||||||
|
elif verbose >= 2:
|
||||||
|
logging_level = "DEBUG"
|
||||||
|
|
||||||
|
logger.setLevel(getattr(logging, logging_level))
|
||||||
|
logger.debug(f"Verbosity is {verbose}")
|
||||||
|
logger.debug(f"Logging set to {logging_level}")
|
||||||
|
|
||||||
evaluation = get_last_evaluation(channel)
|
evaluation = get_last_evaluation(channel)
|
||||||
es = elasticsearch.Elasticsearch([es_url])
|
es = elasticsearch.Elasticsearch([es_url])
|
||||||
recreate_index(es, channel)
|
recreate_index(es, channel)
|
||||||
|
@ -290,7 +337,7 @@ def main(es_url, channel):
|
||||||
):
|
):
|
||||||
progress.update(1)
|
progress.update(1)
|
||||||
successes += ok
|
successes += ok
|
||||||
print("Indexed %d/%d packages" % (successes, number_of_packages))
|
click.echo("Indexed %d/%d packages" % (successes, number_of_packages))
|
||||||
|
|
||||||
# write options
|
# write options
|
||||||
number_of_options, gen_options = get_options(evaluation)
|
number_of_options, gen_options = get_options(evaluation)
|
||||||
|
@ -308,3 +355,5 @@ def main(es_url, channel):
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
||||||
|
# vi:ft=python
|
||||||
|
|
Loading…
Reference in a new issue