From 7bcf9f17d86c67d75a38ef100998a201c35e95ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 30 Nov 2020 17:58:42 +0100 Subject: [PATCH] Fix import scripts, remove 19.09 branch (#239) * Fix rendering of default and example values * bump the version * reduce nix-instantiate calls * Allow literalExample in default values, handle null correctly * using py38 for now * literalExample is not always string * in case literalExample is not string process it as we would normally * Remove 19.09 branch * also remove 19.09 channel from github actions * improve testing of empty lists and dicts Co-authored-by: Rok Garbas --- .github/workflows/cron.yml | 1 - VERSION | 2 +- import-scripts/import_scripts/channel.py | 58 ++++++++++++++++++------ src/Search.elm | 10 +--- 4 files changed, 46 insertions(+), 25 deletions(-) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 1d1b782..ad16a6a 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -18,7 +18,6 @@ jobs: - unstable - 20.09 - 20.03 - - 19.09 env: S3_URL: s3://nix-releases/nixpkgs diff --git a/VERSION b/VERSION index b1bd38b..8351c19 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13 +14 diff --git a/import-scripts/import_scripts/channel.py b/import-scripts/import_scripts/channel.py index 5405045..4e91406 100644 --- a/import-scripts/import_scripts/channel.py +++ b/import-scripts/import_scripts/channel.py @@ -6,6 +6,7 @@ import click_log # type: ignore import dictdiffer # type: ignore import elasticsearch # type: ignore import elasticsearch.helpers # type: ignore +import functools import json import logging import os @@ -29,7 +30,6 @@ INDEX_SCHEMA_VERSION = os.environ.get("INDEX_SCHEMA_VERSION", 0) DIFF_OUTPUT = ["json", "stats"] CHANNELS = { "unstable": "nixos/unstable/nixos-21.03pre", - "19.09": "nixos/19.09/nixos-19.09.", "20.03": "nixos/20.03/nixos-20.03.", "20.09": "nixos/20.09/nixos-20.09.", } @@ -266,7 +266,7 @@ def parse_query(text): def get_last_evaluation(prefix): - logger.debug(f"Retriving last evaluation for {prefix} prefix.") + logger.debug(f"Retrieving last evaluation for {prefix} prefix.") s3 = boto3.client( "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED) @@ -307,7 +307,7 @@ def get_last_evaluation(prefix): def get_evaluation_builds(evaluation_id): logger.debug( - f"get_evaluation_builds: Retriving list of builds for {evaluation_id} evaluation id" + f"get_evaluation_builds: Retrieving list of builds for {evaluation_id} evaluation id" ) filename = f"eval-{evaluation_id}.json" if not os.path.exists(filename): @@ -393,7 +393,7 @@ def remove_attr_set(name): def get_packages_raw(evaluation): logger.debug( - f"get_packages: Retriving list of packages for '{evaluation['git_revision']}' revision" + f"get_packages: Retrieving list of packages for '{evaluation['git_revision']}' revision" ) result = subprocess.run( shlex.split( @@ -502,7 +502,7 @@ def get_packages(evaluation, evaluation_builds): def get_options_raw(evaluation): logger.debug( - f"get_packages: Retriving list of options for '{evaluation['git_revision']}' revision" + f"get_packages: Retrieving list of options for '{evaluation['git_revision']}' revision" ) result = subprocess.run( shlex.split( @@ -523,18 +523,48 @@ def get_options_raw(evaluation): def get_options(evaluation): options = get_options_raw(evaluation) + @functools.lru_cache(maxsize=None) + def jsonToNix(value): + result = subprocess.run( + shlex.split( + "nix-instantiate --eval --strict -E 'builtins.fromJSON (builtins.readFile /dev/stdin)'" + ), + input=value.encode(), + stdout=subprocess.PIPE, + check=True, + ) + return result.stdout.strip().decode() + + def toNix(value): + if isinstance(value, dict) and value.get("_type") == "literalExample": + if isinstance(value["text"], str): + return value["text"] + value = value["text"] + + if value is None: + return "null" + elif isinstance(value, int) or isinstance(value, float): + return str(value) + elif isinstance(value, bool): + return "true" if value else "false" + elif isinstance(value, list) and not value: + return "[ ]" + elif isinstance(value, dict) and not value: + return "{ }" + else: + return jsonToNix(json.dumps(value)) + def gen(): for name, option in options: - default = option.get("default") - if default is not None: - default = json.dumps(default) + if "default" in option: + default = toNix(option.get("default")) + else: + default = None - example = option.get("example") - if example is not None: - if type(example) == dict and example.get("_type") == "literalExample": - example = json.dumps(example["text"]) - else: - example = json.dumps(example) + if "example" in option: + example = toNix(option.get("example")) + else: + example = None description = option.get("description") if description is not None: diff --git a/src/Search.elm b/src/Search.elm index e11bc96..38359f4 100644 --- a/src/Search.elm +++ b/src/Search.elm @@ -275,7 +275,6 @@ createUrl toRoute model = type Channel = Unstable - | Release_19_09 | Release_20_03 | Release_20_09 @@ -302,9 +301,6 @@ channelDetails channel = Unstable -> ChannelDetails "unstable" "unstable" "nixos/trunk-combined" "nixos-unstable" - Release_19_09 -> - ChannelDetails "19.09" "19.09" "nixos/release-19.09" "nixos-19.09" - Release_20_03 -> ChannelDetails "20.03" "20.03" "nixos/release-20.03" "nixos-20.03" @@ -318,9 +314,6 @@ channelFromId channel_id = "unstable" -> Just Unstable - "19.09" -> - Just Release_19_09 - "20.03" -> Just Release_20_03 @@ -339,8 +332,7 @@ channelDetailsFromId channel_id = channels : List String channels = - [ "19.09" - , "20.03" + [ "20.03" , "20.09" , "unstable" ]