Provide correct link to hydra job or if exists build (#81)

also add --force option to import-channel script which will always recreate index

fixes #72
This commit is contained in:
Rok Garbas 2020-06-10 13:48:53 +02:00 committed by GitHub
parent 7a9d4cd9d4
commit 9413e66d90
Failed to generate hash of commit
4 changed files with 170 additions and 97 deletions

View file

@ -29,8 +29,23 @@ logger = logging.getLogger("import-channel")
click_log.basic_config(logger) click_log.basic_config(logger)
S3_BUCKET = "nix-releases"
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
INDEX_SCHEMA_VERSION = 4 INDEX_SCHEMA_VERSION = 5
CHANNELS = {
"unstable": {
"packages": "nixpkgs/nixpkgs-20.09pre",
"options": "nixos/unstable/nixos-20.09pre",
},
"19.09": {
"packages": "nixpkgs/nixpkgs-19.09pre",
"options": "nixos/19.09/nixos-19.09.",
},
"20.03": {
"packages": "nixpkgs/nixpkgs-19.09pre",
"options": "nixos/20.03/nixos-20.03.",
},
}
ANALYSIS = { ANALYSIS = {
"analyzer": { "analyzer": {
"nixAttrName": { "nixAttrName": {
@ -109,19 +124,25 @@ MAPPING = {
"properties": { "properties": {
"type": {"type": "keyword"}, "type": {"type": "keyword"},
# Package fields # Package fields
"package_hydra_build_id": {"type": "keyword"}, "package_hydra_build": {
"package_hydra_build_status": {"type": "keyword"},
"package_hydra_project": {"type": "keyword"},
"package_hydra_job": {"type": "keyword"},
"package_hydra_jobset": {"type": "keyword"},
"package_hydra_path": {
"type": "nested", "type": "nested",
"properties": { "properties": {
"output": {"type": "keyword"}, "build_id": {"type": "keyword"},
"path": {"type": "keyword"} "build_status": {"type": "keyword"},
} "platform": {"type": "keyword"},
"project": {"type": "keyword"},
"jobset": {"type": "keyword"},
"job": {"type": "keyword"},
"path": {
"type": "nested",
"properties": {
"output": {"type": "keyword"},
"path": {"type": "keyword"}
},
},
"drv_path": {"type": "keyword"},
},
}, },
"package_hydra_drvpath": {"type": "keyword"},
"package_attr_name": { "package_attr_name": {
"type": "text", "type": "text",
"analyzer": "nixAttrName", "analyzer": "nixAttrName",
@ -172,54 +193,38 @@ MAPPING = {
} }
def get_last_evaluation(channel): def get_last_evaluation(prefix):
logger.debug(f"Retriving last evaluation for {channel} channel") logger.debug(f"Retriving last evaluation for {prefix} prefix.")
project, project_version = channel.split("-", 1)
logger.debug(f"get_last_evaluation: project='{project}'")
logger.debug(f"get_last_evaluation: project_version='{project_version}'")
bucket = "nix-releases"
prefix = f"{project}/{project_version}/"
logger.debug(
f"get_last_evaluation: list all evaluation in '{bucket}' bucker under '{prefix}' prefix"
)
s3 = boto3.client( s3 = boto3.client(
"s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED) "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED)
) )
s3_result = s3.list_objects(Bucket=bucket, Prefix=prefix, Delimiter="/",) s3_result = s3.list_objects(Bucket=S3_BUCKET, Prefix=prefix, Delimiter="/",)
evaluations = [] evaluations = []
for item in s3_result.get("CommonPrefixes"): for item in s3_result.get("CommonPrefixes"):
if not item: if not item:
continue continue
logger.debug(f"get_last_evaluation: evaluation in raw {item}") logger.debug(f"get_last_evaluation: evaluation in raw {item}")
prefix = item.get("Prefix")
evaluation = prefix[len(f"{project}/{project_version}/{channel}") :]
if evaluation.startswith("beta"):
evaluation = evaluation[len("beta") :]
try: try:
revisions_since_start, git_revision = ( revisions_since_start, git_revision = item['Prefix'][len(prefix):].rstrip('/').split('.')
evaluation.lstrip(".").rstrip("/").split(".") except:
) __import__('pdb').set_trace()
except Exception as e: # noqa
continue
evaluation = { evaluation = {
"revisions_since_start": int(revisions_since_start), "revisions_since_start": int(revisions_since_start),
"git_revision": git_revision, "git_revision": git_revision,
"prefix": prefix, "prefix": item['Prefix'].rstrip('/'),
} }
logger.debug(f"get_last_evaluation: evaluation {evaluation}") logger.debug(f"get_last_evaluation: evaluation {evaluation}")
evaluations.append(evaluation) evaluations.append(evaluation)
logger.debug( logger.debug(
f"get_last_evaluation: {len(evaluations)} evaluations found for {channel} channel" f"get_last_evaluation: {len(evaluations)} evaluations found for {prefix} prefix"
) )
evaluations = sorted(evaluations, key=lambda i: i["revisions_since_start"]) evaluations = sorted(evaluations, key=lambda i: i["revisions_since_start"])
evaluation = evaluations[-1] evaluation = evaluations[-1]
result = s3.get_object(Bucket=bucket, Key=f"{evaluation['prefix']}src-url") result = s3.get_object(Bucket=S3_BUCKET, Key=f"{evaluation['prefix']}/src-url")
evaluation['id'] = result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/"):] evaluation['id'] = result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/"):]
logger.debug(f"get_last_evaluation: last evaluation is: {evaluation}") logger.debug(f"get_last_evaluation: last evaluation is: {evaluation}")
@ -250,10 +255,12 @@ def get_evaluation_builds(evaluation_id):
with open(filename) as f: with open(filename) as f:
builds = json.loads(f.read()) builds = json.loads(f.read())
return { result = {}
f"{build['nixname']}.{build['system']}": build for build in builds:
for build in builds result.setdefault(build['nixname'], {})
} result[build['nixname']][build['system']] = build
return result
def get_packages(evaluation, evaluation_builds): def get_packages(evaluation, evaluation_builds):
@ -315,38 +322,30 @@ def get_packages(evaluation, evaluation_builds):
): ):
attr_set = None attr_set = None
hydra_build_id = None hydra = None
hydra_build_status = None if data['name'] in evaluation_builds:
hydra_job = None hydra = []
hydra_jobset = None for platform, build in evaluation_builds[data['name']].items():
hydra_path = None hydra.append({
hydra_drvpath = None "build_id": build['id'],
build_key = f"{data['name']}.{data['system']}" "build_status": build['buildstatus'],
if build_key in evaluation_builds: "platform": build['system'],
build = evaluation_builds[build_key] "project": build['project'],
hydra_build_id = build['id'] "jobset": build['jobset'],
hydra_build_status = build['buildstatus'] "job": build['job'],
hydra_project = build['project'] "path": [
hydra_job = build['job'] {
hydra_jobset = build['jobset'] "output": output,
hydra_path = [ "path": item['path'],
{ }
"output": output, for output, item in build['buildoutputs'].items()
"path": item['path'], ],
} "drv_path": build['drvpath'],
for output, item in build['buildoutputs'].items() })
]
hydra_drvpath = build['drvpath']
yield dict( yield dict(
type="package", type="package",
package_hydra_build_id=hydra_build_id, package_hydra=hydra,
package_hydra_build_status=hydra_build_status,
package_hydra_project=hydra_project,
package_hydra_job=hydra_job,
package_hydra_jobset=hydra_jobset,
package_hydra_path=hydra_path,
package_hydra_drvpath=hydra_drvpath,
package_attr_name=attr_name, package_attr_name=attr_name,
package_attr_set=attr_set, package_attr_set=attr_set,
package_pname=data["pname"], package_pname=data["pname"],
@ -403,10 +402,14 @@ def get_options(evaluation):
return len(options), gen return len(options), gen
def ensure_index(es, index, mapping): def ensure_index(es, index, mapping, force=False):
if es.indices.exists(index): if es.indices.exists(index):
logger.debug(f"ensure_index: index '{index}' already exists") logger.debug(f"ensure_index: index '{index}' already exists")
return False if not force:
return False
logger.debug(f"ensure_index: Deleting index '{index}'")
es.indices.delete(index)
es.indices.create( es.indices.create(
index=index, index=index,
@ -420,10 +423,18 @@ def ensure_index(es, index, mapping):
return True return True
def create_index_name(channel, evaluation): def create_index_name(channel, evaluation_packages, evaluation_options):
evaluation_name = '-'.join([
evaluation_packages['id'],
str(evaluation_packages['revisions_since_start']),
evaluation_packages['git_revision'],
evaluation_options['id'],
str(evaluation_options['revisions_since_start']),
evaluation_options['git_revision'],
])
return ( return (
f"latest-{INDEX_SCHEMA_VERSION}-{channel}", f"latest-{INDEX_SCHEMA_VERSION}-{channel}",
f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation['id']}-{evaluation['revisions_since_start']}-{evaluation['git_revision']}", f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation_name}",
) )
@ -464,10 +475,11 @@ def write(unit, es, index_name, number_of_items, item_generator):
@click.command() @click.command()
@click.option("-u", "--es-url", help="Elasticsearch connection url") @click.option("-u", "--es-url", help="Elasticsearch connection url.")
@click.option("-c", "--channel", help="NixOS channel name") @click.option("-c", "--channel", type=click.Choice(CHANNELS.keys()), help="Channel.")
@click.option("-f", "--force", is_flag=True, help="Force channel recreation.")
@click.option("-v", "--verbose", count=True) @click.option("-v", "--verbose", count=True)
def main(es_url, channel, verbose): def main(es_url, channel, force, verbose):
logging_level = "CRITICAL" logging_level = "CRITICAL"
if verbose == 1: if verbose == 1:
@ -479,18 +491,18 @@ def main(es_url, channel, verbose):
logger.debug(f"Verbosity is {verbose}") logger.debug(f"Verbosity is {verbose}")
logger.debug(f"Logging set to {logging_level}") logger.debug(f"Logging set to {logging_level}")
evaluation = get_last_evaluation(channel) evaluation_packages = get_last_evaluation(CHANNELS[channel]['packages'])
evaluation_builds = get_evaluation_builds(evaluation['id']) evaluation_options = get_last_evaluation(CHANNELS[channel]['options'])
evaluation_packages_builds = get_evaluation_builds(evaluation_packages['id'])
es = elasticsearch.Elasticsearch([es_url]) es = elasticsearch.Elasticsearch([es_url])
# ensure indexes exist alias_name, index_name = create_index_name(channel, evaluation_packages, evaluation_options)
alias_name, index_name = create_index_name(channel, evaluation) index_created = ensure_index(es, index_name, MAPPING, force)
index_created = ensure_index(es, index_name, MAPPING)
if index_created: if index_created:
write("packages", es, index_name, *get_packages(evaluation, evaluation_builds)) write("packages", es, index_name, *get_packages(evaluation_packages, evaluation_packages_builds))
write("options", es, index_name, *get_options(evaluation)) write("options", es, index_name, *get_options(evaluation_options))
update_alias(es, alias_name, index_name) update_alias(es, alias_name, index_name)

View file

@ -376,7 +376,7 @@ makeRequest :
makeRequest options channel query from size = makeRequest options channel query from size =
Search.makeRequest Search.makeRequest
(makeRequestBody query from size) (makeRequestBody query from size)
("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-nixos-" ++ channel) ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel)
decodeResultItemSource decodeResultItemSource
options options
query query

View file

@ -65,6 +65,7 @@ type alias ResultItemSource =
, position : Maybe String , position : Maybe String
, homepage : Maybe String , homepage : Maybe String
, system : String , system : String
, hydra : Maybe (List ResultPackageHydra)
} }
@ -81,6 +82,24 @@ type alias ResultPackageMaintainer =
} }
type alias ResultPackageHydra =
{ build_id : Int
, build_status : Int
, platform : String
, project : String
, jobset : String
, job : String
, path : List ResultPackageHydraPath
, drv_path : String
}
type alias ResultPackageHydraPath =
{ output : String
, path : String
}
init : init :
Maybe String Maybe String
-> Maybe String -> Maybe String
@ -221,7 +240,7 @@ viewResultItemDetails channel item =
Just value -> Just value ->
wrapWith value wrapWith value
allowedPlatforms platform = mainPlatforms platform =
List.member platform List.member platform
[ "x86_64-linux" [ "x86_64-linux"
, "aarch64-linux" , "aarch64-linux"
@ -229,22 +248,42 @@ viewResultItemDetails channel item =
, "i686-linux" , "i686-linux"
] ]
showPlatforms platforms = getHydraDetailsForPlatform hydra platform =
platforms hydra
|> List.filter allowedPlatforms |> Maybe.andThen
|> List.map showPlatform (\hydras ->
hydras
|> List.filter (\x -> x.platform == platform)
|> List.head
)
showPlatform platform = showPlatforms hydra platforms =
platforms
|> List.filter mainPlatforms
|> List.map (showPlatform hydra)
showPlatform hydra platform =
li [] li []
[ case Search.channelDetailsFromId channel of [ case
Just channelDetails -> ( getHydraDetailsForPlatform hydra platform
, Search.channelDetailsFromId channel
)
of
( Just hydraDetails, _ ) ->
a a
[ href <| "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ item.source.system [ href <| "https://hydra.nixos.org/build/" ++ String.fromInt hydraDetails.build_id
] ]
[ text platform [ text platform
] ]
Nothing -> ( Nothing, Just channelDetails ) ->
a
[ href <| "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ platform
]
[ text platform
]
( _, _ ) ->
text platform text platform
] ]
@ -284,7 +323,7 @@ viewResultItemDetails channel item =
, dt [] [ text <| "Nix expression" ] , dt [] [ text <| "Nix expression" ]
, dd [] [ withDefault asGithubLink item.source.position ] , dd [] [ withDefault asGithubLink item.source.position ]
, dt [] [ text "Platforms" ] , dt [] [ text "Platforms" ]
, dd [] [ ul [ class "inline" ] <| showPlatforms item.source.platforms ] , dd [] [ ul [ class "inline" ] <| showPlatforms item.source.hydra item.source.platforms ]
, dt [] [ text "Homepage" ] , dt [] [ text "Homepage" ]
, dd [] [ withDefault asLink item.source.homepage ] , dd [] [ withDefault asLink item.source.homepage ]
, dt [] [ text "Licenses" ] , dt [] [ text "Licenses" ]
@ -437,7 +476,7 @@ makeRequest :
makeRequest options channel query from size = makeRequest options channel query from size =
Search.makeRequest Search.makeRequest
(makeRequestBody query from size) (makeRequestBody query from size)
("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-nixos-" ++ channel) ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel)
decodeResultItemSource decodeResultItemSource
options options
query query
@ -464,6 +503,7 @@ decodeResultItemSource =
|> Json.Decode.Pipeline.required "package_position" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_position" (Json.Decode.nullable Json.Decode.string)
|> Json.Decode.Pipeline.required "package_homepage" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_homepage" (Json.Decode.nullable Json.Decode.string)
|> Json.Decode.Pipeline.required "package_system" Json.Decode.string |> Json.Decode.Pipeline.required "package_system" Json.Decode.string
|> Json.Decode.Pipeline.required "package_hydra" (Json.Decode.nullable (Json.Decode.list decodeResultPackageHydra))
decodeResultPackageLicense : Json.Decode.Decoder ResultPackageLicense decodeResultPackageLicense : Json.Decode.Decoder ResultPackageLicense
@ -479,3 +519,24 @@ decodeResultPackageMaintainer =
(Json.Decode.field "name" Json.Decode.string) (Json.Decode.field "name" Json.Decode.string)
(Json.Decode.field "email" Json.Decode.string) (Json.Decode.field "email" Json.Decode.string)
(Json.Decode.field "github" (Json.Decode.nullable Json.Decode.string)) (Json.Decode.field "github" (Json.Decode.nullable Json.Decode.string))
decodeResultPackageHydra : Json.Decode.Decoder ResultPackageHydra
decodeResultPackageHydra =
Json.Decode.succeed ResultPackageHydra
|> Json.Decode.Pipeline.required "build_id" Json.Decode.int
|> Json.Decode.Pipeline.required "build_status" Json.Decode.int
|> Json.Decode.Pipeline.required "platform" Json.Decode.string
|> Json.Decode.Pipeline.required "project" Json.Decode.string
|> Json.Decode.Pipeline.required "jobset" Json.Decode.string
|> Json.Decode.Pipeline.required "job" Json.Decode.string
|> Json.Decode.Pipeline.required "path" (Json.Decode.list decodeResultPackageHydraPath)
|> Json.Decode.Pipeline.required "drv_path" Json.Decode.string
decodeResultPackageHydraPath : Json.Decode.Decoder ResultPackageHydraPath
decodeResultPackageHydraPath =
Json.Decode.map2 ResultPackageHydraPath
(Json.Decode.field "output" Json.Decode.string)
(Json.Decode.field "path" Json.Decode.string)

View file

@ -6,7 +6,7 @@ const {Elm} = require('./Main');
Elm.Main.init({ Elm.Main.init({
flags: { flags: {
elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 4, elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 5,
elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443', elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443',
elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR', elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR',
elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG' elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG'