Provide correct link to hydra job or if exists build (#81)

also add --force option to import-channel script which will always recreate index

fixes #72

scripts/import-channel

@@ -29,8 +29,23 @@ logger = logging.getLogger("import-channel")
 click_log.basic_config(logger)
 
 
+S3_BUCKET = "nix-releases"
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-INDEX_SCHEMA_VERSION = 4
+INDEX_SCHEMA_VERSION = 5
+CHANNELS = {
+    "unstable": {
+        "packages": "nixpkgs/nixpkgs-20.09pre",
+        "options": "nixos/unstable/nixos-20.09pre",
+    },
+    "19.09": {
+        "packages": "nixpkgs/nixpkgs-19.09pre",
+        "options": "nixos/19.09/nixos-19.09.",
+    },
+    "20.03": {
+        "packages": "nixpkgs/nixpkgs-19.09pre",
+        "options": "nixos/20.03/nixos-20.03.",
+    },
+}
 ANALYSIS = {
     "analyzer": {
         "nixAttrName": {
@@ -109,19 +124,25 @@ MAPPING = {
     "properties": {
         "type": {"type": "keyword"},
         # Package fields
-        "package_hydra_build_id": {"type": "keyword"},
+        "package_hydra_build": {
-        "package_hydra_build_status": {"type": "keyword"},
+            "type": "nested",
-        "package_hydra_project": {"type": "keyword"},
+            "properties": {
-        "package_hydra_job": {"type": "keyword"},
+                "build_id": {"type": "keyword"},
-        "package_hydra_jobset": {"type": "keyword"},
+                "build_status": {"type": "keyword"},
-        "package_hydra_path": {
+                "platform": {"type": "keyword"},
+                "project": {"type": "keyword"},
+                "jobset": {"type": "keyword"},
+                "job": {"type": "keyword"},
+                "path": {
                     "type": "nested",
                     "properties": {
                         "output": {"type": "keyword"},
                         "path": {"type": "keyword"}
-            }
                     },
-        "package_hydra_drvpath": {"type": "keyword"},
+                },
+                "drv_path": {"type": "keyword"},
+            },
+        },
         "package_attr_name": {
             "type": "text",
             "analyzer": "nixAttrName",
@@ -172,54 +193,38 @@ MAPPING = {
 }
 
 
-def get_last_evaluation(channel):
+def get_last_evaluation(prefix):
-    logger.debug(f"Retriving last evaluation for {channel} channel")
+    logger.debug(f"Retriving last evaluation for {prefix} prefix.")
-
-    project, project_version = channel.split("-", 1)
-    logger.debug(f"get_last_evaluation: project='{project}'")
-    logger.debug(f"get_last_evaluation: project_version='{project_version}'")
-
-    bucket = "nix-releases"
-    prefix = f"{project}/{project_version}/"
-    logger.debug(
-        f"get_last_evaluation: list all evaluation in '{bucket}' bucker under '{prefix}' prefix"
-    )
 
     s3 = boto3.client(
         "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED)
     )
-    s3_result = s3.list_objects(Bucket=bucket, Prefix=prefix, Delimiter="/",)
+    s3_result = s3.list_objects(Bucket=S3_BUCKET, Prefix=prefix, Delimiter="/",)
     evaluations = []
     for item in s3_result.get("CommonPrefixes"):
         if not item:
             continue
         logger.debug(f"get_last_evaluation: evaluation in raw {item}")
-        prefix = item.get("Prefix")
-        evaluation = prefix[len(f"{project}/{project_version}/{channel}") :]
-        if evaluation.startswith("beta"):
-            evaluation = evaluation[len("beta") :]
         try:
-            revisions_since_start, git_revision = (
+            revisions_since_start, git_revision = item['Prefix'][len(prefix):].rstrip('/').split('.')
-                evaluation.lstrip(".").rstrip("/").split(".")
+        except: 
-            )
+            __import__('pdb').set_trace()
-        except Exception as e:  # noqa
-            continue
         evaluation = {
             "revisions_since_start": int(revisions_since_start),
             "git_revision": git_revision,
-            "prefix": prefix,
+            "prefix": item['Prefix'].rstrip('/'),
         }
         logger.debug(f"get_last_evaluation: evaluation {evaluation}")
         evaluations.append(evaluation)
 
     logger.debug(
-        f"get_last_evaluation: {len(evaluations)} evaluations found for {channel} channel"
+        f"get_last_evaluation: {len(evaluations)} evaluations found for {prefix} prefix"
     )
     evaluations = sorted(evaluations, key=lambda i: i["revisions_since_start"])
 
     evaluation = evaluations[-1]
 
-    result = s3.get_object(Bucket=bucket, Key=f"{evaluation['prefix']}src-url")
+    result = s3.get_object(Bucket=S3_BUCKET, Key=f"{evaluation['prefix']}/src-url")
     evaluation['id'] = result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/"):]
 
     logger.debug(f"get_last_evaluation: last evaluation is: {evaluation}")
@@ -250,10 +255,12 @@ def get_evaluation_builds(evaluation_id):
     with open(filename) as f:
         builds = json.loads(f.read())
 
-    return {
+    result = {}
-        f"{build['nixname']}.{build['system']}": build 
+    for build in builds:
-        for build in builds
+        result.setdefault(build['nixname'], {})
-    }
+        result[build['nixname']][build['system']] = build
+
+    return result
 
 
 def get_packages(evaluation, evaluation_builds):
@@ -315,38 +322,30 @@ def get_packages(evaluation, evaluation_builds):
                 ):
                     attr_set = None
 
-            hydra_build_id = None
+            hydra = None
-            hydra_build_status = None
+            if data['name'] in evaluation_builds:
-            hydra_job = None
+                hydra = []
-            hydra_jobset = None
+                for platform, build in evaluation_builds[data['name']].items():
-            hydra_path = None
+                    hydra.append({
-            hydra_drvpath = None
+                        "build_id": build['id'],
-            build_key = f"{data['name']}.{data['system']}"
+                        "build_status":  build['buildstatus'],
-            if build_key in evaluation_builds:
+                        "platform": build['system'],
-                build = evaluation_builds[build_key]
+                        "project": build['project'],
-                hydra_build_id = build['id']
+                        "jobset": build['jobset'],
-                hydra_build_status = build['buildstatus']
+                        "job": build['job'],
-                hydra_project = build['project']
+                        "path": [
-                hydra_job = build['job']
-                hydra_jobset = build['jobset']
-                hydra_path = [
                             {
                                 "output": output,
                                 "path": item['path'],
                             }
                             for output, item in build['buildoutputs'].items()
-                ]
+                        ],
-                hydra_drvpath = build['drvpath']
+                        "drv_path": build['drvpath'],
+                    })
 
             yield dict(
                 type="package",
-                package_hydra_build_id=hydra_build_id,
+                package_hydra=hydra,
-                package_hydra_build_status=hydra_build_status,
-                package_hydra_project=hydra_project,
-                package_hydra_job=hydra_job,
-                package_hydra_jobset=hydra_jobset,
-                package_hydra_path=hydra_path,
-                package_hydra_drvpath=hydra_drvpath,
                 package_attr_name=attr_name,
                 package_attr_set=attr_set,
                 package_pname=data["pname"],
@@ -403,11 +402,15 @@ def get_options(evaluation):
     return len(options), gen
 
 
-def ensure_index(es, index, mapping):
+def ensure_index(es, index, mapping, force=False):
     if es.indices.exists(index):
         logger.debug(f"ensure_index: index '{index}' already exists")
+        if not force:
             return False
 
+        logger.debug(f"ensure_index: Deleting index '{index}'")
+        es.indices.delete(index)
+
     es.indices.create(
         index=index,
         body={
@@ -420,10 +423,18 @@ def ensure_index(es, index, mapping):
     return True
 
 
-def create_index_name(channel, evaluation):
+def create_index_name(channel, evaluation_packages, evaluation_options):
+    evaluation_name = '-'.join([
+        evaluation_packages['id'],
+        str(evaluation_packages['revisions_since_start']),
+        evaluation_packages['git_revision'],
+        evaluation_options['id'],
+        str(evaluation_options['revisions_since_start']),
+        evaluation_options['git_revision'],
+    ])
     return (
         f"latest-{INDEX_SCHEMA_VERSION}-{channel}",
-        f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation['id']}-{evaluation['revisions_since_start']}-{evaluation['git_revision']}",
+        f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation_name}",
     )
 
 
@@ -464,10 +475,11 @@ def write(unit, es, index_name, number_of_items, item_generator):
 
 
 @click.command()
-@click.option("-u", "--es-url", help="Elasticsearch connection url")
+@click.option("-u", "--es-url", help="Elasticsearch connection url.")
-@click.option("-c", "--channel", help="NixOS channel name")
+@click.option("-c", "--channel", type=click.Choice(CHANNELS.keys()), help="Channel.")
+@click.option("-f", "--force", is_flag=True, help="Force channel recreation.")
 @click.option("-v", "--verbose", count=True)
-def main(es_url, channel, verbose):
+def main(es_url, channel, force, verbose):
 
     logging_level = "CRITICAL"
     if verbose == 1:
@@ -479,18 +491,18 @@ def main(es_url, channel, verbose):
     logger.debug(f"Verbosity is {verbose}")
     logger.debug(f"Logging set to {logging_level}")
 
-    evaluation = get_last_evaluation(channel)
+    evaluation_packages = get_last_evaluation(CHANNELS[channel]['packages'])
-    evaluation_builds = get_evaluation_builds(evaluation['id'])
+    evaluation_options = get_last_evaluation(CHANNELS[channel]['options'])
+    evaluation_packages_builds = get_evaluation_builds(evaluation_packages['id'])
 
     es = elasticsearch.Elasticsearch([es_url])
 
-    # ensure indexes exist
+    alias_name, index_name = create_index_name(channel, evaluation_packages, evaluation_options)
-    alias_name, index_name = create_index_name(channel, evaluation)
+    index_created = ensure_index(es, index_name, MAPPING, force)
-    index_created = ensure_index(es, index_name, MAPPING)
 
     if index_created:
-        write("packages", es, index_name, *get_packages(evaluation, evaluation_builds))
+        write("packages", es, index_name, *get_packages(evaluation_packages, evaluation_packages_builds))
-        write("options", es, index_name, *get_options(evaluation))
+        write("options", es, index_name, *get_options(evaluation_options))
 
     update_alias(es, alias_name, index_name)
 

src/Page/Options.elm

@@ -376,7 +376,7 @@ makeRequest :
 makeRequest options channel query from size =
     Search.makeRequest
         (makeRequestBody query from size)
-        ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-nixos-" ++ channel)
+        ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel)
         decodeResultItemSource
         options
         query

src/Page/Packages.elm

@@ -65,6 +65,7 @@ type alias ResultItemSource =
     , position : Maybe String
     , homepage : Maybe String
     , system : String
+    , hydra : Maybe (List ResultPackageHydra)
     }
 
 
@@ -81,6 +82,24 @@ type alias ResultPackageMaintainer =
     }
 
 
+type alias ResultPackageHydra =
+    { build_id : Int
+    , build_status : Int
+    , platform : String
+    , project : String
+    , jobset : String
+    , job : String
+    , path : List ResultPackageHydraPath
+    , drv_path : String
+    }
+
+
+type alias ResultPackageHydraPath =
+    { output : String
+    , path : String
+    }
+
+
 init :
     Maybe String
     -> Maybe String
@@ -221,7 +240,7 @@ viewResultItemDetails channel item =
                 Just value ->
                     wrapWith value
 
-        allowedPlatforms platform =
+        mainPlatforms platform =
             List.member platform
                 [ "x86_64-linux"
                 , "aarch64-linux"
@@ -229,22 +248,42 @@ viewResultItemDetails channel item =
                 , "i686-linux"
                 ]
 
-        showPlatforms platforms =
+        getHydraDetailsForPlatform hydra platform =
-            platforms
+            hydra
-                |> List.filter allowedPlatforms
+                |> Maybe.andThen
-                |> List.map showPlatform
+                    (\hydras ->
+                        hydras
+                            |> List.filter (\x -> x.platform == platform)
+                            |> List.head
+                    )
 
-        showPlatform platform =
+        showPlatforms hydra platforms =
+            platforms
+                |> List.filter mainPlatforms
+                |> List.map (showPlatform hydra)
+
+        showPlatform hydra platform =
             li []
-                [ case Search.channelDetailsFromId channel of
+                [ case
-                    Just channelDetails ->
+                    ( getHydraDetailsForPlatform hydra platform
+                    , Search.channelDetailsFromId channel
+                    )
+                  of
+                    ( Just hydraDetails, _ ) ->
                         a
-                            [ href <| "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ item.source.system
+                            [ href <| "https://hydra.nixos.org/build/" ++ String.fromInt hydraDetails.build_id
                             ]
                             [ text platform
                             ]
 
-                    Nothing ->
+                    ( Nothing, Just channelDetails ) ->
+                        a
+                            [ href <| "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ platform
+                            ]
+                            [ text platform
+                            ]
+
+                    ( _, _ ) ->
                         text platform
                 ]
 
@@ -284,7 +323,7 @@ viewResultItemDetails channel item =
         , dt [] [ text <| "Nix expression" ]
         , dd [] [ withDefault asGithubLink item.source.position ]
         , dt [] [ text "Platforms" ]
-        , dd [] [ ul [ class "inline" ] <| showPlatforms item.source.platforms ]
+        , dd [] [ ul [ class "inline" ] <| showPlatforms item.source.hydra item.source.platforms ]
         , dt [] [ text "Homepage" ]
         , dd [] [ withDefault asLink item.source.homepage ]
         , dt [] [ text "Licenses" ]
@@ -437,7 +476,7 @@ makeRequest :
 makeRequest options channel query from size =
     Search.makeRequest
         (makeRequestBody query from size)
-        ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-nixos-" ++ channel)
+        ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel)
         decodeResultItemSource
         options
         query
@@ -464,6 +503,7 @@ decodeResultItemSource =
         |> Json.Decode.Pipeline.required "package_position" (Json.Decode.nullable Json.Decode.string)
         |> Json.Decode.Pipeline.required "package_homepage" (Json.Decode.nullable Json.Decode.string)
         |> Json.Decode.Pipeline.required "package_system" Json.Decode.string
+        |> Json.Decode.Pipeline.required "package_hydra" (Json.Decode.nullable (Json.Decode.list decodeResultPackageHydra))
 
 
 decodeResultPackageLicense : Json.Decode.Decoder ResultPackageLicense
@@ -479,3 +519,24 @@ decodeResultPackageMaintainer =
         (Json.Decode.field "name" Json.Decode.string)
         (Json.Decode.field "email" Json.Decode.string)
         (Json.Decode.field "github" (Json.Decode.nullable Json.Decode.string))
+
+
+decodeResultPackageHydra : Json.Decode.Decoder ResultPackageHydra
+decodeResultPackageHydra =
+    Json.Decode.succeed ResultPackageHydra
+        |> Json.Decode.Pipeline.required "build_id" Json.Decode.int
+        |> Json.Decode.Pipeline.required "build_status" Json.Decode.int
+        |> Json.Decode.Pipeline.required "platform" Json.Decode.string
+        |> Json.Decode.Pipeline.required "project" Json.Decode.string
+        |> Json.Decode.Pipeline.required "jobset" Json.Decode.string
+        |> Json.Decode.Pipeline.required "job" Json.Decode.string
+        |> Json.Decode.Pipeline.required "path" (Json.Decode.list decodeResultPackageHydraPath)
+        |> Json.Decode.Pipeline.required "drv_path" Json.Decode.string
+
+
+decodeResultPackageHydraPath : Json.Decode.Decoder ResultPackageHydraPath
+decodeResultPackageHydraPath =
+    Json.Decode.map2 ResultPackageHydraPath
+        (Json.Decode.field "output" Json.Decode.string)
+        (Json.Decode.field "path" Json.Decode.string)
+

src/index.js

@@ -6,7 +6,7 @@ const {Elm} = require('./Main');
 
 Elm.Main.init({
   flags: {
-    elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 4,
+    elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 5,
     elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443',
     elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR',
     elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG'