Fix import scripts, remove 19.09 branch (#239)

* Fix rendering of default and example values

* bump the version

* reduce nix-instantiate calls

* Allow literalExample in default values, handle null correctly

* using py38 for now

* literalExample is not always string

* in case literalExample is not string process it as we would normally

* Remove 19.09 branch

* also remove 19.09 channel from github actions

* improve testing of empty lists and dicts

Co-authored-by: Rok Garbas <rok@garbas.si>
This commit is contained in:
Naïm Favier 2020-11-30 17:58:42 +01:00 committed by GitHub
parent 060daed760
commit 7bcf9f17d8
Failed to generate hash of commit
4 changed files with 46 additions and 25 deletions

View file

@ -18,7 +18,6 @@ jobs:
- unstable
- 20.09
- 20.03
- 19.09
env:
S3_URL: s3://nix-releases/nixpkgs

View file

@ -1 +1 @@
13
14

View file

@ -6,6 +6,7 @@ import click_log # type: ignore
import dictdiffer # type: ignore
import elasticsearch # type: ignore
import elasticsearch.helpers # type: ignore
import functools
import json
import logging
import os
@ -29,7 +30,6 @@ INDEX_SCHEMA_VERSION = os.environ.get("INDEX_SCHEMA_VERSION", 0)
DIFF_OUTPUT = ["json", "stats"]
CHANNELS = {
"unstable": "nixos/unstable/nixos-21.03pre",
"19.09": "nixos/19.09/nixos-19.09.",
"20.03": "nixos/20.03/nixos-20.03.",
"20.09": "nixos/20.09/nixos-20.09.",
}
@ -266,7 +266,7 @@ def parse_query(text):
def get_last_evaluation(prefix):
logger.debug(f"Retriving last evaluation for {prefix} prefix.")
logger.debug(f"Retrieving last evaluation for {prefix} prefix.")
s3 = boto3.client(
"s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED)
@ -307,7 +307,7 @@ def get_last_evaluation(prefix):
def get_evaluation_builds(evaluation_id):
logger.debug(
f"get_evaluation_builds: Retriving list of builds for {evaluation_id} evaluation id"
f"get_evaluation_builds: Retrieving list of builds for {evaluation_id} evaluation id"
)
filename = f"eval-{evaluation_id}.json"
if not os.path.exists(filename):
@ -393,7 +393,7 @@ def remove_attr_set(name):
def get_packages_raw(evaluation):
logger.debug(
f"get_packages: Retriving list of packages for '{evaluation['git_revision']}' revision"
f"get_packages: Retrieving list of packages for '{evaluation['git_revision']}' revision"
)
result = subprocess.run(
shlex.split(
@ -502,7 +502,7 @@ def get_packages(evaluation, evaluation_builds):
def get_options_raw(evaluation):
logger.debug(
f"get_packages: Retriving list of options for '{evaluation['git_revision']}' revision"
f"get_packages: Retrieving list of options for '{evaluation['git_revision']}' revision"
)
result = subprocess.run(
shlex.split(
@ -523,18 +523,48 @@ def get_options_raw(evaluation):
def get_options(evaluation):
options = get_options_raw(evaluation)
@functools.lru_cache(maxsize=None)
def jsonToNix(value):
result = subprocess.run(
shlex.split(
"nix-instantiate --eval --strict -E 'builtins.fromJSON (builtins.readFile /dev/stdin)'"
),
input=value.encode(),
stdout=subprocess.PIPE,
check=True,
)
return result.stdout.strip().decode()
def toNix(value):
if isinstance(value, dict) and value.get("_type") == "literalExample":
if isinstance(value["text"], str):
return value["text"]
value = value["text"]
if value is None:
return "null"
elif isinstance(value, int) or isinstance(value, float):
return str(value)
elif isinstance(value, bool):
return "true" if value else "false"
elif isinstance(value, list) and not value:
return "[ ]"
elif isinstance(value, dict) and not value:
return "{ }"
else:
return jsonToNix(json.dumps(value))
def gen():
for name, option in options:
default = option.get("default")
if default is not None:
default = json.dumps(default)
if "default" in option:
default = toNix(option.get("default"))
else:
default = None
example = option.get("example")
if example is not None:
if type(example) == dict and example.get("_type") == "literalExample":
example = json.dumps(example["text"])
else:
example = json.dumps(example)
if "example" in option:
example = toNix(option.get("example"))
else:
example = None
description = option.get("description")
if description is not None:

View file

@ -275,7 +275,6 @@ createUrl toRoute model =
type Channel
= Unstable
| Release_19_09
| Release_20_03
| Release_20_09
@ -302,9 +301,6 @@ channelDetails channel =
Unstable ->
ChannelDetails "unstable" "unstable" "nixos/trunk-combined" "nixos-unstable"
Release_19_09 ->
ChannelDetails "19.09" "19.09" "nixos/release-19.09" "nixos-19.09"
Release_20_03 ->
ChannelDetails "20.03" "20.03" "nixos/release-20.03" "nixos-20.03"
@ -318,9 +314,6 @@ channelFromId channel_id =
"unstable" ->
Just Unstable
"19.09" ->
Just Release_19_09
"20.03" ->
Just Release_20_03
@ -339,8 +332,7 @@ channelDetailsFromId channel_id =
channels : List String
channels =
[ "19.09"
, "20.03"
[ "20.03"
, "20.09"
, "unstable"
]