diff --git a/scripts/import-channel b/scripts/import-channel index 145d104..f689179 100755 --- a/scripts/import-channel +++ b/scripts/import-channel @@ -29,8 +29,23 @@ logger = logging.getLogger("import-channel") click_log.basic_config(logger) +S3_BUCKET = "nix-releases" CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -INDEX_SCHEMA_VERSION = 4 +INDEX_SCHEMA_VERSION = 5 +CHANNELS = { + "unstable": { + "packages": "nixpkgs/nixpkgs-20.09pre", + "options": "nixos/unstable/nixos-20.09pre", + }, + "19.09": { + "packages": "nixpkgs/nixpkgs-19.09pre", + "options": "nixos/19.09/nixos-19.09.", + }, + "20.03": { + "packages": "nixpkgs/nixpkgs-19.09pre", + "options": "nixos/20.03/nixos-20.03.", + }, +} ANALYSIS = { "analyzer": { "nixAttrName": { @@ -109,19 +124,25 @@ MAPPING = { "properties": { "type": {"type": "keyword"}, # Package fields - "package_hydra_build_id": {"type": "keyword"}, - "package_hydra_build_status": {"type": "keyword"}, - "package_hydra_project": {"type": "keyword"}, - "package_hydra_job": {"type": "keyword"}, - "package_hydra_jobset": {"type": "keyword"}, - "package_hydra_path": { + "package_hydra_build": { "type": "nested", "properties": { - "output": {"type": "keyword"}, - "path": {"type": "keyword"} - } + "build_id": {"type": "keyword"}, + "build_status": {"type": "keyword"}, + "platform": {"type": "keyword"}, + "project": {"type": "keyword"}, + "jobset": {"type": "keyword"}, + "job": {"type": "keyword"}, + "path": { + "type": "nested", + "properties": { + "output": {"type": "keyword"}, + "path": {"type": "keyword"} + }, + }, + "drv_path": {"type": "keyword"}, + }, }, - "package_hydra_drvpath": {"type": "keyword"}, "package_attr_name": { "type": "text", "analyzer": "nixAttrName", @@ -172,54 +193,38 @@ MAPPING = { } -def get_last_evaluation(channel): - logger.debug(f"Retriving last evaluation for {channel} channel") - - project, project_version = channel.split("-", 1) - logger.debug(f"get_last_evaluation: project='{project}'") - logger.debug(f"get_last_evaluation: project_version='{project_version}'") - - bucket = "nix-releases" - prefix = f"{project}/{project_version}/" - logger.debug( - f"get_last_evaluation: list all evaluation in '{bucket}' bucker under '{prefix}' prefix" - ) +def get_last_evaluation(prefix): + logger.debug(f"Retriving last evaluation for {prefix} prefix.") s3 = boto3.client( "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED) ) - s3_result = s3.list_objects(Bucket=bucket, Prefix=prefix, Delimiter="/",) + s3_result = s3.list_objects(Bucket=S3_BUCKET, Prefix=prefix, Delimiter="/",) evaluations = [] for item in s3_result.get("CommonPrefixes"): if not item: continue logger.debug(f"get_last_evaluation: evaluation in raw {item}") - prefix = item.get("Prefix") - evaluation = prefix[len(f"{project}/{project_version}/{channel}") :] - if evaluation.startswith("beta"): - evaluation = evaluation[len("beta") :] try: - revisions_since_start, git_revision = ( - evaluation.lstrip(".").rstrip("/").split(".") - ) - except Exception as e: # noqa - continue + revisions_since_start, git_revision = item['Prefix'][len(prefix):].rstrip('/').split('.') + except: + __import__('pdb').set_trace() evaluation = { "revisions_since_start": int(revisions_since_start), "git_revision": git_revision, - "prefix": prefix, + "prefix": item['Prefix'].rstrip('/'), } logger.debug(f"get_last_evaluation: evaluation {evaluation}") evaluations.append(evaluation) logger.debug( - f"get_last_evaluation: {len(evaluations)} evaluations found for {channel} channel" + f"get_last_evaluation: {len(evaluations)} evaluations found for {prefix} prefix" ) evaluations = sorted(evaluations, key=lambda i: i["revisions_since_start"]) evaluation = evaluations[-1] - result = s3.get_object(Bucket=bucket, Key=f"{evaluation['prefix']}src-url") + result = s3.get_object(Bucket=S3_BUCKET, Key=f"{evaluation['prefix']}/src-url") evaluation['id'] = result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/"):] logger.debug(f"get_last_evaluation: last evaluation is: {evaluation}") @@ -250,10 +255,12 @@ def get_evaluation_builds(evaluation_id): with open(filename) as f: builds = json.loads(f.read()) - return { - f"{build['nixname']}.{build['system']}": build - for build in builds - } + result = {} + for build in builds: + result.setdefault(build['nixname'], {}) + result[build['nixname']][build['system']] = build + + return result def get_packages(evaluation, evaluation_builds): @@ -315,38 +322,30 @@ def get_packages(evaluation, evaluation_builds): ): attr_set = None - hydra_build_id = None - hydra_build_status = None - hydra_job = None - hydra_jobset = None - hydra_path = None - hydra_drvpath = None - build_key = f"{data['name']}.{data['system']}" - if build_key in evaluation_builds: - build = evaluation_builds[build_key] - hydra_build_id = build['id'] - hydra_build_status = build['buildstatus'] - hydra_project = build['project'] - hydra_job = build['job'] - hydra_jobset = build['jobset'] - hydra_path = [ - { - "output": output, - "path": item['path'], - } - for output, item in build['buildoutputs'].items() - ] - hydra_drvpath = build['drvpath'] + hydra = None + if data['name'] in evaluation_builds: + hydra = [] + for platform, build in evaluation_builds[data['name']].items(): + hydra.append({ + "build_id": build['id'], + "build_status": build['buildstatus'], + "platform": build['system'], + "project": build['project'], + "jobset": build['jobset'], + "job": build['job'], + "path": [ + { + "output": output, + "path": item['path'], + } + for output, item in build['buildoutputs'].items() + ], + "drv_path": build['drvpath'], + }) yield dict( type="package", - package_hydra_build_id=hydra_build_id, - package_hydra_build_status=hydra_build_status, - package_hydra_project=hydra_project, - package_hydra_job=hydra_job, - package_hydra_jobset=hydra_jobset, - package_hydra_path=hydra_path, - package_hydra_drvpath=hydra_drvpath, + package_hydra=hydra, package_attr_name=attr_name, package_attr_set=attr_set, package_pname=data["pname"], @@ -403,10 +402,14 @@ def get_options(evaluation): return len(options), gen -def ensure_index(es, index, mapping): +def ensure_index(es, index, mapping, force=False): if es.indices.exists(index): logger.debug(f"ensure_index: index '{index}' already exists") - return False + if not force: + return False + + logger.debug(f"ensure_index: Deleting index '{index}'") + es.indices.delete(index) es.indices.create( index=index, @@ -420,10 +423,18 @@ def ensure_index(es, index, mapping): return True -def create_index_name(channel, evaluation): +def create_index_name(channel, evaluation_packages, evaluation_options): + evaluation_name = '-'.join([ + evaluation_packages['id'], + str(evaluation_packages['revisions_since_start']), + evaluation_packages['git_revision'], + evaluation_options['id'], + str(evaluation_options['revisions_since_start']), + evaluation_options['git_revision'], + ]) return ( f"latest-{INDEX_SCHEMA_VERSION}-{channel}", - f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation['id']}-{evaluation['revisions_since_start']}-{evaluation['git_revision']}", + f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation_name}", ) @@ -464,10 +475,11 @@ def write(unit, es, index_name, number_of_items, item_generator): @click.command() -@click.option("-u", "--es-url", help="Elasticsearch connection url") -@click.option("-c", "--channel", help="NixOS channel name") +@click.option("-u", "--es-url", help="Elasticsearch connection url.") +@click.option("-c", "--channel", type=click.Choice(CHANNELS.keys()), help="Channel.") +@click.option("-f", "--force", is_flag=True, help="Force channel recreation.") @click.option("-v", "--verbose", count=True) -def main(es_url, channel, verbose): +def main(es_url, channel, force, verbose): logging_level = "CRITICAL" if verbose == 1: @@ -479,18 +491,18 @@ def main(es_url, channel, verbose): logger.debug(f"Verbosity is {verbose}") logger.debug(f"Logging set to {logging_level}") - evaluation = get_last_evaluation(channel) - evaluation_builds = get_evaluation_builds(evaluation['id']) + evaluation_packages = get_last_evaluation(CHANNELS[channel]['packages']) + evaluation_options = get_last_evaluation(CHANNELS[channel]['options']) + evaluation_packages_builds = get_evaluation_builds(evaluation_packages['id']) es = elasticsearch.Elasticsearch([es_url]) - # ensure indexes exist - alias_name, index_name = create_index_name(channel, evaluation) - index_created = ensure_index(es, index_name, MAPPING) + alias_name, index_name = create_index_name(channel, evaluation_packages, evaluation_options) + index_created = ensure_index(es, index_name, MAPPING, force) if index_created: - write("packages", es, index_name, *get_packages(evaluation, evaluation_builds)) - write("options", es, index_name, *get_options(evaluation)) + write("packages", es, index_name, *get_packages(evaluation_packages, evaluation_packages_builds)) + write("options", es, index_name, *get_options(evaluation_options)) update_alias(es, alias_name, index_name) diff --git a/src/Page/Options.elm b/src/Page/Options.elm index b78c5df..dfa4d5f 100644 --- a/src/Page/Options.elm +++ b/src/Page/Options.elm @@ -376,7 +376,7 @@ makeRequest : makeRequest options channel query from size = Search.makeRequest (makeRequestBody query from size) - ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-nixos-" ++ channel) + ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel) decodeResultItemSource options query diff --git a/src/Page/Packages.elm b/src/Page/Packages.elm index 8e084e0..356914d 100644 --- a/src/Page/Packages.elm +++ b/src/Page/Packages.elm @@ -65,6 +65,7 @@ type alias ResultItemSource = , position : Maybe String , homepage : Maybe String , system : String + , hydra : Maybe (List ResultPackageHydra) } @@ -81,6 +82,24 @@ type alias ResultPackageMaintainer = } +type alias ResultPackageHydra = + { build_id : Int + , build_status : Int + , platform : String + , project : String + , jobset : String + , job : String + , path : List ResultPackageHydraPath + , drv_path : String + } + + +type alias ResultPackageHydraPath = + { output : String + , path : String + } + + init : Maybe String -> Maybe String @@ -221,7 +240,7 @@ viewResultItemDetails channel item = Just value -> wrapWith value - allowedPlatforms platform = + mainPlatforms platform = List.member platform [ "x86_64-linux" , "aarch64-linux" @@ -229,22 +248,42 @@ viewResultItemDetails channel item = , "i686-linux" ] - showPlatforms platforms = - platforms - |> List.filter allowedPlatforms - |> List.map showPlatform + getHydraDetailsForPlatform hydra platform = + hydra + |> Maybe.andThen + (\hydras -> + hydras + |> List.filter (\x -> x.platform == platform) + |> List.head + ) - showPlatform platform = + showPlatforms hydra platforms = + platforms + |> List.filter mainPlatforms + |> List.map (showPlatform hydra) + + showPlatform hydra platform = li [] - [ case Search.channelDetailsFromId channel of - Just channelDetails -> + [ case + ( getHydraDetailsForPlatform hydra platform + , Search.channelDetailsFromId channel + ) + of + ( Just hydraDetails, _ ) -> a - [ href <| "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ item.source.system + [ href <| "https://hydra.nixos.org/build/" ++ String.fromInt hydraDetails.build_id ] [ text platform ] - Nothing -> + ( Nothing, Just channelDetails ) -> + a + [ href <| "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ platform + ] + [ text platform + ] + + ( _, _ ) -> text platform ] @@ -284,7 +323,7 @@ viewResultItemDetails channel item = , dt [] [ text <| "Nix expression" ] , dd [] [ withDefault asGithubLink item.source.position ] , dt [] [ text "Platforms" ] - , dd [] [ ul [ class "inline" ] <| showPlatforms item.source.platforms ] + , dd [] [ ul [ class "inline" ] <| showPlatforms item.source.hydra item.source.platforms ] , dt [] [ text "Homepage" ] , dd [] [ withDefault asLink item.source.homepage ] , dt [] [ text "Licenses" ] @@ -437,7 +476,7 @@ makeRequest : makeRequest options channel query from size = Search.makeRequest (makeRequestBody query from size) - ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-nixos-" ++ channel) + ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel) decodeResultItemSource options query @@ -464,6 +503,7 @@ decodeResultItemSource = |> Json.Decode.Pipeline.required "package_position" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_homepage" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_system" Json.Decode.string + |> Json.Decode.Pipeline.required "package_hydra" (Json.Decode.nullable (Json.Decode.list decodeResultPackageHydra)) decodeResultPackageLicense : Json.Decode.Decoder ResultPackageLicense @@ -479,3 +519,24 @@ decodeResultPackageMaintainer = (Json.Decode.field "name" Json.Decode.string) (Json.Decode.field "email" Json.Decode.string) (Json.Decode.field "github" (Json.Decode.nullable Json.Decode.string)) + + +decodeResultPackageHydra : Json.Decode.Decoder ResultPackageHydra +decodeResultPackageHydra = + Json.Decode.succeed ResultPackageHydra + |> Json.Decode.Pipeline.required "build_id" Json.Decode.int + |> Json.Decode.Pipeline.required "build_status" Json.Decode.int + |> Json.Decode.Pipeline.required "platform" Json.Decode.string + |> Json.Decode.Pipeline.required "project" Json.Decode.string + |> Json.Decode.Pipeline.required "jobset" Json.Decode.string + |> Json.Decode.Pipeline.required "job" Json.Decode.string + |> Json.Decode.Pipeline.required "path" (Json.Decode.list decodeResultPackageHydraPath) + |> Json.Decode.Pipeline.required "drv_path" Json.Decode.string + + +decodeResultPackageHydraPath : Json.Decode.Decoder ResultPackageHydraPath +decodeResultPackageHydraPath = + Json.Decode.map2 ResultPackageHydraPath + (Json.Decode.field "output" Json.Decode.string) + (Json.Decode.field "path" Json.Decode.string) + diff --git a/src/index.js b/src/index.js index db6b247..7d24c99 100644 --- a/src/index.js +++ b/src/index.js @@ -6,7 +6,7 @@ const {Elm} = require('./Main'); Elm.Main.init({ flags: { - elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 4, + elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 5, elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443', elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR', elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG'