Pretty print nix examples and default fields in options (#252)

VERSION

@@ -1 +1 @@
-16
+17

import-scripts/default.nix

@@ -13,9 +13,12 @@ mkPoetryApplication {
       '';
     });
   });
-  nativeBuildInputs = [
+  nativeBuildInputs = with pkgs; [
-    pkgs.poetry
+    poetry
+    fd
+    entr
   ];
+  #doCheck = false;
   checkPhase = ''
     export PYTHONPATH=$PWD:$PYTHONPATH
     black --diff --check import_scripts/ tests/

import-scripts/import_scripts/channel.py

@@ -8,7 +8,7 @@ import click_log  # type: ignore
 import dictdiffer  # type: ignore
 import elasticsearch  # type: ignore
 import elasticsearch.helpers  # type: ignore
-import functools
+import import_scripts.nix  # type: ignore
 import json
 import logging
 import os
@@ -527,47 +527,15 @@ def get_options_raw(evaluation):
 def get_options(evaluation):
     options = get_options_raw(evaluation)
 
-    @functools.lru_cache(maxsize=None)
-    @backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
-    def jsonToNix(value):
-        result = subprocess.run(
-            shlex.split(
-                "nix-instantiate --eval --strict -E 'builtins.fromJSON (builtins.readFile /dev/stdin)'"
-            ),
-            input=value.encode(),
-            stdout=subprocess.PIPE,
-            check=True,
-        )
-        return result.stdout.strip().decode()
-
-    def toNix(value):
-        if isinstance(value, dict) and value.get("_type") == "literalExample":
-            if isinstance(value["text"], str):
-                return value["text"]
-            value = value["text"]
-
-        if value is None:
-            return "null"
-        elif type(value) in [int, float]:
-            return str(value)
-        elif type(value) == bool:
-            return "true" if value else "false"
-        elif type(value) == list and not value:
-            return "[ ]"
-        elif type(value) == dict and not value:
-            return "{ }"
-        else:
-            return jsonToNix(json.dumps(value))
-
     def gen():
         for name, option in options:
             if "default" in option:
-                default = toNix(option.get("default"))
+                default = import_scripts.nix.prettyPrint(option.get("default"))
             else:
                 default = None
 
             if "example" in option:
-                example = toNix(option.get("example"))
+                example = import_scripts.nix.prettyPrint(option.get("example"))
             else:
                 example = None
 
@@ -626,15 +594,15 @@ def ensure_index(es, index, mapping, force=False):
     return True
 
 
-def create_index_name(channel, evaluation_packages, evaluation_options):
+def create_index_name(channel, evaluation):
     evaluation_name = "-".join(
         [
-            evaluation_packages["id"],
+            evaluation["id"],
-            str(evaluation_packages["revisions_since_start"]),
+            str(evaluation["revisions_since_start"]),
-            evaluation_packages["git_revision"],
+            evaluation["git_revision"],
-            evaluation_options["id"],
+            evaluation["id"],
-            str(evaluation_options["revisions_since_start"]),
+            str(evaluation["revisions_since_start"]),
-            evaluation_options["git_revision"],
+            evaluation["git_revision"],
         ]
     )
     return (
@@ -699,27 +667,20 @@ def setup_logging(verbose):
 def run_import(es_url, channel, force, verbose):
     setup_logging(verbose)
 
-    evaluation_packages = get_last_evaluation(CHANNELS[channel])
+    evaluation = get_last_evaluation(CHANNELS[channel])
-    evaluation_options = get_last_evaluation(CHANNELS[channel])
+    evaluation_builds = dict()
-    evaluation_packages_builds = (
+    # evaluation_builds = get_evaluation_builds(evaluation["id"])
-        dict()
-    )  # get_evaluation_builds(evaluation_packages["id"])
 
     es = elasticsearch.Elasticsearch([es_url])
 
-    alias_name, index_name = create_index_name(
+    alias_name, index_name = create_index_name(channel, evaluation)
-        channel, evaluation_packages, evaluation_options
-    )
     index_created = ensure_index(es, index_name, MAPPING, force)
 
     if index_created:
         write(
-            "packages",
+            "packages", es, index_name, *get_packages(evaluation, evaluation_builds),
-            es,
-            index_name,
-            *get_packages(evaluation_packages, evaluation_packages_builds),
         )
-        write("options", es, index_name, *get_options(evaluation_options))
+        write("options", es, index_name, *get_options(evaluation))
 
     update_alias(es, alias_name, index_name)
 
@@ -813,4 +774,4 @@ def run_diff(channel_from, channel_to, output, verbose):
 
 
 if __name__ == "__main__":
-    run_diff()
+    run_import()

import-scripts/import_scripts/nix.py

@@ -0,0 +1,56 @@
+def prettyPrintAttrName(attr_name):
+    if "." in attr_name:
+        return prettyPrint(attr_name)
+    return attr_name
+
+
+def prettyPrint(item, level=""):
+    next_level = level + "  "
+    if item is None:
+        return "null"
+
+    elif type(item) == bool:
+        if item:
+            return "true"
+        return "false"
+
+    elif type(item) in (int, float):
+        return f"{item}"
+
+    elif type(item) == str:
+        if "\n" in item:
+            return f"''{item}''"
+        return f'"{item}"'
+
+    elif type(item) == list:
+        if len(item) == 0:
+            return "[ ]"
+        return (
+            "[\n"
+            + ("".join([f"{level}  {prettyPrint(i, next_level)}\n" for i in item]))
+            + f"{level}]"
+        )
+
+    elif type(item) == dict:
+        if len(item) == 0:
+            return "{ }"
+        if item.get("_type") == "literalExample":
+            if type(item["text"]) == str:
+                return item["text"]
+            else:
+                return prettyPrint(item["text"], next_level)
+        return (
+            "{\n"
+            + (
+                "".join(
+                    [
+                        f"{level}  {prettyPrintAttrName(n)} = {prettyPrint(v, next_level)};\n"
+                        for n, v in item.items()
+                    ]
+                )
+            )
+            + f"{level}}}"
+        )
+
+    else:
+        raise NotImplementedError(item)

import-scripts/tests/test_nix.py

@@ -0,0 +1,44 @@
+import pytest  # type: ignore
+
+
+@pytest.mark.parametrize(
+    "item,expected",
+    [
+        (None, "null",),
+        (True, "true",),
+        ("text", '"text"',),
+        (123, "123",),
+        (123.123, "123.123",),
+        ([False, "text"], ("[\n" "  false\n" '  "text"\n' "]"),),
+        (
+            {"name1": "value1", "name.2": True, "name3": [False, "text"]},
+            (
+                "{\n"
+                '  name1 = "value1";\n'
+                '  "name.2" = true;\n'
+                "  name3 = [\n"
+                "    false\n"
+                '    "text"\n'
+                "  ];\n"
+                "}"
+            ),
+        ),
+        (
+            [{"name1": ["value1", "value2"]}],
+            (
+                "[\n"
+                "  {\n"
+                "    name1 = [\n"
+                '      "value1"\n'
+                '      "value2"\n'
+                "    ];\n"
+                "  }\n"
+                "]"
+            ),
+        ),
+    ],
+)
+def test_convert(item, expected):
+    import import_scripts.nix
+
+    assert import_scripts.nix.prettyPrint(item) == expected

src/Page/Options.elm

@@ -196,7 +196,7 @@ viewResultItemDetails channel item =
                     Ok nodes ->
                         Html.Parser.Util.toVirtualDom nodes
 
-                    Err _ ->
+                    Err e ->
                         []
 
         asPre value =
@@ -205,6 +205,14 @@ viewResultItemDetails channel item =
         asCode value =
             code [] [ text value ]
 
+        asPreCode value =
+            div [] [ pre [] [ code [] [ text value ] ] ]
+
+        encodeHtml value =
+            value
+                |> String.replace "<" "&lt;"
+                |> String.replace ">" "&gt;"
+
         githubUrlPrefix branch =
             "https://github.com/NixOS/nixpkgs/blob/" ++ branch ++ "/"
 
@@ -248,15 +256,15 @@ viewResultItemDetails channel item =
     in
     dl [ class "dl-horizontal" ]
         [ dt [] [ text "Name" ]
-        , dd [] [ withEmpty asText (Just item.source.name) ]
+        , dd [] [ withEmpty asText (Just (encodeHtml item.source.name)) ]
         , dt [] [ text "Description" ]
         , dd [] [ withEmpty asText item.source.description ]
         , dt [] [ text "Default value" ]
-        , dd [] [ withEmpty asCode item.source.default ]
+        , dd [] [ withEmpty (wrapped asPreCode) item.source.default ]
         , dt [] [ text "Type" ]
         , dd [] [ withEmpty asPre item.source.type_ ]
         , dt [] [ text "Example value" ]
-        , dd [] [ withEmpty (wrapped asCode) item.source.example ]
+        , dd [] [ withEmpty (wrapped asPreCode) item.source.example ]
         , dt [] [ text "Declared in" ]
         , dd [] [ withEmpty asGithubLink item.source.source ]
         ]

src/Route.elm

@@ -44,7 +44,8 @@ searchQueryParser url =
         rawQuery =
             Route.SearchQuery.toRawQuery url
 
-        maybeQuery = Maybe.andThen (Route.SearchQuery.searchString "query") rawQuery
+        maybeQuery =
+            Maybe.andThen (Route.SearchQuery.searchString "query") rawQuery
     in
     Url.Parser.map (SearchArgs maybeQuery) <|
         Url.Parser.top