c8h4/aux-search
1
0
Fork 0
Code Issues Pull requests Packages Releases Activity

Improve option search ranking (#107)

Browse source
This commit is contained in:
Rok Garbas 2020-06-19 08:53:49 +02:00 committed by GitHub
parent bbc97d17f0
commit 3ecce08a94
Failed to generate hash of commit
8 changed files with 343 additions and 386 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
VERSION
View file

@ -1 +1 @@
7 8

92
import-scripts/import_scripts/channel.py
View file

@ -49,83 +49,6 @@ ANALYSIS = {
"tokenizer": "keyword", "tokenizer": "keyword",
"filter": ["lowercase"], "filter": ["lowercase"],
}, },
"nixOptionName": {
"type": "custom",
"tokenizer": "nix_option_name",
"filter": ["lowercase"],
},
"nixOptionNameGranular": {
"type": "custom",
"tokenizer": "nix_option_name_granular",
"filter": ["lowercase"],
},
},
"tokenizer": {
"nix_package_query": {"type": "pattern", "pattern": "|".join(["[ ]"])},
"nix_package_attr_name": {
"type": "pattern",
# Split on attrname separators like _, .
"pattern": "|".join(
[
"[_.-]", # Common separators like underscores, dots and dashes
"\\d+?Packages", # python37Packages -> python
"\\d+?Plugins", # vimPlugins -> vim
"\\d+?Extensions", # php74Extensions -> php
"\\d+?Interpreters", # perlInterpreters -> perl
# Camelcase tokenizer adapted from
# https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-pattern-analyzer.html
"".join(
[
"(?<=[\\p{L}&&[^\\p{Lu}]])" # lower case
"(?=\\p{Lu})", # followed by upper case
"|",
"(?<=\\p{Lu})" # or upper case
"(?=\\p{Lu}[\\p{L}&&[^\\p{Lu}]])", # followed by lower case
]
),
]
),
},
"nix_option_name": {"type": "pattern", "pattern": "[.]"},
# Lower priority (virtualHost -> [virtual, host])
"nix_option_name_granular": {
"type": "pattern",
# Split on attrname separators like _, .
"pattern": "|".join(
[
"[_.-]", # Common separators like underscores, dots and dashes
# Camelcase tokenizer adapted from
# https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-pattern-analyzer.html
"".join(
[
"(?<=[\\p{L}&&[^\\p{Lu}]])" # lower case
"(?=\\p{Lu})", # followed by upper case
"|",
"(?<=\\p{Lu})" # or upper case
"(?=\\p{Lu}[\\p{L}&&[^\\p{Lu}]])", # followed by lower case
]
),
]
),
},
},
"filter": {
"nix_stopwords": {
"type": "stop",
"ignore_case": True,
"stopwords": [
"packages",
"package",
"options",
"option",
"plugins",
"plugin",
"extensions",
"extension",
"interpreters",
"interpreter",
],
},
}, },
} }
MAPPING = { MAPPING = {
@ -175,15 +98,8 @@ MAPPING = {
"package_homepage": {"type": "keyword"}, "package_homepage": {"type": "keyword"},
"package_system": {"type": "keyword"}, "package_system": {"type": "keyword"},
# Options fields # Options fields
"option_name": { "option_name": {"type": "keyword", "normalizer": "lowercase"},
"type": "text", "option_name_query": {"type": "keyword", "normalizer": "lowercase"},
"analyzer": "nixOptionName",
"fielddata": True,
"fields": {
"raw": {"type": "keyword"},
"granular": {"type": "text", "analyzer": "nixOptionNameGranular"},
},
},
"option_description": {"type": "text"}, "option_description": {"type": "text"},
"option_type": {"type": "keyword"}, "option_type": {"type": "keyword"},
"option_default": {"type": "text"}, "option_default": {"type": "text"},
@ -340,7 +256,8 @@ def remove_attr_set(name):
# Plugins # Plugins
"elasticsearch", "elasticsearch",
"graylog", "graylog",
"tmuxplugin" "vimplugin", "tmuxplugin",
"vimplugin",
] ]
# TODO: is this correct # TODO: is this correct
if any([name.startswith(i) for i in sets]): if any([name.startswith(i) for i in sets]):
@ -495,6 +412,7 @@ def get_options(evaluation):
yield dict( yield dict(
type="option", type="option",
option_name=name, option_name=name,
option_name_query=split_query(name),
option_description=description, option_description=description,
option_type=option.get("type"), option_type=option.get("type"),
option_default=default, option_default=default,

26
src/Page/Home.elm
View file

@ -1,28 +1,38 @@
module Page.Home exposing (Model, Msg, init, update, view) module Page.Home exposing (Model, Msg, init, update, view)
import Html exposing (Html, text, div ) import Html exposing (Html, div, text)
-- MODEL -- MODEL
type alias Model = ()
type alias Model =
()
init : (Model, Cmd Msg) init : ( Model, Cmd Msg )
init = init =
((), Cmd.none) ( (), Cmd.none )
-- UPDATE -- UPDATE
type Msg = NoOp
type Msg
= NoOp
update : Msg -> Model -> ( Model, Cmd Msg ) update : Msg -> Model -> ( Model, Cmd Msg )
update msg model = update msg model =
(model, Cmd.none) ( model, Cmd.none )
-- VIEW -- VIEW
view : Model -> Html Msg view : Model -> Html Msg
view model = view model =
div [] [text "Home"] div [] [ text "Home" ]

298
src/Page/Options.elm
View file

@ -17,6 +17,8 @@ import Html
, div , div
, dl , dl
, dt , dt
, li
, p
, pre , pre
, span , span
, table , table
@ -26,6 +28,7 @@ import Html
, th , th
, thead , thead
, tr , tr
, ul
) )
import Html.Attributes import Html.Attributes
exposing exposing
@ -42,6 +45,7 @@ import Html.Parser.Util
import Http import Http
import Json.Decode import Json.Decode
import Json.Encode import Json.Encode
import Regex
import Search import Search
@ -145,10 +149,30 @@ viewResultItem show item =
else else
[] []
in in
tr [ onClick (SearchMsg (Search.ShowDetails item.source.name)) ] []
-- DEBUG: |> List.append
-- DEBUG: [ tr []
-- DEBUG: [ td [ colspan 1 ]
-- DEBUG: [ p [] [ text <| "score: " ++ String.fromFloat item.score ]
-- DEBUG: , p []
-- DEBUG: [ text <|
-- DEBUG: "matched queries: "
-- DEBUG: , ul []
-- DEBUG: (item.matched_queries
-- DEBUG: |> Maybe.withDefault []
-- DEBUG: |> List.sort
-- DEBUG: |> List.map (\q -> li [] [ text q ])
-- DEBUG: )
-- DEBUG: ]
-- DEBUG: ]
-- DEBUG: ]
-- DEBUG: ]
|> List.append
(tr [ onClick (SearchMsg (Search.ShowDetails item.source.name)) ]
[ td [] [ text item.source.name ] [ td [] [ text item.source.name ]
] ]
:: packageDetails :: packageDetails
)
viewResultItemDetails : viewResultItemDetails :
@ -229,160 +253,6 @@ viewResultItemDetails item =
-- API -- API
makeRequestBody :
String
-> Int
-> Int
-> Http.Body
makeRequestBody query from size =
-- Prefix Query
-- example query for "python"
-- {
-- "from": 0,
-- "size": 1000,
-- "query": {
-- "bool": {
-- "must": {
-- "bool": {
-- "should": [
-- {
-- "term": {
-- "option_name.raw": {
-- "value": "nginx",
-- "boost": 2.0
-- }
-- }
-- },
-- {
-- "term": {
-- "option_name": {
-- "value": "nginx",
-- "boost": 1.0
-- }
-- }
-- },
-- {
-- "term": {
-- "option_name.granular": {
-- "value": "nginx",
-- "boost": 0.6
-- }
-- }
-- },
-- {
-- "term": {
-- "option_description": {
-- "value": "nginx",
-- "boost": 0.3
-- }
-- }
-- }
-- ]
-- }
-- },
-- "filter": [
-- {
-- "match": {
-- "type": "option"
-- }
-- }
-- ]
-- }
-- },
-- "rescore" : {
-- "window_size": 500,
-- "query" : {
-- "score_mode": "total",
-- "rescore_query" : {
-- "function_score" : {
-- "script_score": {
-- "script": {
-- "source": "
-- int i = 1;
-- for (token in doc['option_name.raw'][0].splitOnToken('.')) {
-- if (token == \"nginx\") {
-- return 10000 - (i * 100);
-- }
-- i++;
-- }
-- return 10;
-- "
-- }
-- }
-- }
-- }
-- }
-- }
-- }
let
stringIn name value =
[ ( name, Json.Encode.string value ) ]
listIn name type_ value =
[ ( name, Json.Encode.list type_ value ) ]
objectIn name value =
[ ( name, Json.Encode.object value ) ]
encodeTerm ( name, boost ) =
[ ( "term"
, Json.Encode.object
[ ( name
, Json.Encode.object
[ ( "value", Json.Encode.string query )
, ( "boost", Json.Encode.float boost )
]
)
]
)
]
in
[ ( "option_name.raw", 2.0 )
, ( "option_name", 1.0 )
, ( "option_name.granular", 0.6 )
, ( "option_description", 0.3 )
]
|> List.map encodeTerm
|> listIn "should" Json.Encode.object
|> objectIn "bool"
|> objectIn "must"
|> ([ ( "type", Json.Encode.string "option" ) ]
|> objectIn "match"
|> objectIn "filter"
|> List.append
)
|> objectIn "bool"
|> objectIn "query"
|> List.append
("""int i = 1;
for (token in doc['option_name.raw'][0].splitOnToken('.')) {
if (token == '"""
++ query
++ """') {
return 10000 - (i * 100);
}
i++;
}
return 10;
"""
|> stringIn "source"
|> objectIn "script"
|> objectIn "script_score"
|> objectIn "function_score"
|> objectIn "rescore_query"
|> List.append ("total" |> stringIn "score_mode")
|> objectIn "query"
|> List.append [ ( "window_size", Json.Encode.int 1000 ) ]
|> objectIn "rescore"
)
|> List.append
[ ( "from", Json.Encode.int from )
, ( "size", Json.Encode.int size )
]
|> Json.Encode.object
|> Http.jsonBody
makeRequest : makeRequest :
Search.Options Search.Options
-> String -> String
@ -390,9 +260,123 @@ makeRequest :
-> Int -> Int
-> Int -> Int
-> Cmd Msg -> Cmd Msg
makeRequest options channel query from size = makeRequest options channel queryRaw from size =
let
query =
queryRaw
|> String.trim
delimiters =
Maybe.withDefault Regex.never (Regex.fromString "[. ]")
should_match boost_base =
List.indexedMap
(\i ( field, boost ) ->
[ ( "match"
, Json.Encode.object
[ ( field
, Json.Encode.object
[ ( "query", Json.Encode.string query )
, ( "boost", Json.Encode.float boost )
, ( "analyzer", Json.Encode.string "whitespace" )
, ( "fuzziness", Json.Encode.string "1" )
, ( "_name"
, Json.Encode.string <|
"should_match_"
++ String.fromInt (i + 1)
)
]
)
]
)
]
)
[ ( "option_name", 1 )
, ( "option_name_query", 1 )
, ( "option_description", 1 )
]
should_match_bool_prefix boost_base =
List.indexedMap
(\i ( field, boost ) ->
[ ( "match_bool_prefix"
, Json.Encode.object
[ ( field
, Json.Encode.object
[ ( "query", Json.Encode.string query )
, ( "boost", Json.Encode.float boost )
, ( "analyzer", Json.Encode.string "whitespace" )
, ( "fuzziness", Json.Encode.string "1" )
, ( "_name"
, Json.Encode.string <|
"should_match_bool_prefix_"
++ String.fromInt (i + 1)
)
]
)
]
)
]
)
[ ( "option_name", 1 )
, ( "option_name_query", 1 )
]
should_terms boost_base =
List.indexedMap
(\i ( field, boost ) ->
[ ( "terms"
, Json.Encode.object
[ ( field
, Json.Encode.list Json.Encode.string (Regex.split delimiters query)
)
, ( "boost", Json.Encode.float <| boost_base * boost )
, ( "_name"
, Json.Encode.string <|
"should_terms_"
++ String.fromInt (i + 1)
)
]
)
]
)
[ ( "option_name", 1 )
, ( "option_name_query", 1 )
]
should_term boost_base =
List.indexedMap
(\i ( field, boost ) ->
[ ( "term"
, Json.Encode.object
[ ( field
, Json.Encode.object
[ ( "value", Json.Encode.string query )
, ( "boost", Json.Encode.float <| boost_base * boost )
, ( "_name"
, Json.Encode.string <|
"should_term_"
++ String.fromInt (i + 1)
)
]
)
]
)
]
)
[ ( "option_name", 1 )
, ( "option_name_query", 1 )
]
should_queries =
[]
|> List.append (should_term 10000)
|> List.append (should_terms 1000)
|> List.append (should_match_bool_prefix 100)
|> List.append (should_match 10)
in
Search.makeRequest Search.makeRequest
makeRequestBody (Search.makeRequestBody query from size "option" "option_name_query" should_queries)
("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel) ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel)
decodeResultItemSource decodeResultItemSource
options options

133
src/Page/Packages.elm
View file

@ -362,12 +362,14 @@ viewResultItemDetails channel item =
-- API -- API
makeRequestBody : makeRequest :
String Search.Options
-> String
-> String
-> Int -> Int
-> Int -> Int
-> Http.Body -> Cmd Msg
makeRequestBody queryRaw from size = makeRequest options channel queryRaw from size =
let let
query = query =
queryRaw queryRaw
@ -481,130 +483,15 @@ makeRequestBody queryRaw from size =
, ( "package_pname", 1 ) , ( "package_pname", 1 )
] ]
filter_packages = should_queries =
( "term" []
, Json.Encode.object
[ ( "type"
, Json.Encode.object
[ ( "value", Json.Encode.string "package" )
, ( "_name", Json.Encode.string "filter_packages" )
]
)
]
)
filter_queries =
let
filterQuery =
query
|> String.replace "." " "
in
filterQuery
|> String.words
|> List.indexedMap
(\i query_word ->
let
isLast =
List.length (String.words filterQuery) == i + 1
in
[ if isLast then
( "bool"
, Json.Encode.object
[ ( "should"
, Json.Encode.list Json.Encode.object
[ [ ( "match"
, Json.Encode.object
[ ( "package_attr_name_query"
, Json.Encode.object
[ ( "query", Json.Encode.string query_word )
, ( "fuzziness", Json.Encode.string "1" )
, ( "_name", Json.Encode.string <| "filter_queries_" ++ String.fromInt (i + 1) ++ "_should_match" )
]
)
]
)
]
, [ ( "match_bool_prefix"
, Json.Encode.object
[ ( "package_attr_name_query"
, Json.Encode.object
[ ( "query", Json.Encode.string query_word )
, ( "_name"
, Json.Encode.string <| "filter_queries_" ++ String.fromInt (i + 1) ++ "_should_prefix"
)
]
)
]
)
]
]
)
]
)
else
( "match_bool_prefix"
, Json.Encode.object
[ ( "package_attr_name_query"
, Json.Encode.object
[ ( "query", Json.Encode.string query_word )
, ( "_name"
, Json.Encode.string <| "filter_queries_" ++ String.fromInt (i + 1) ++ "_prefix"
)
]
)
]
)
]
)
in
Http.jsonBody
(Json.Encode.object
[ ( "from"
, Json.Encode.int from
)
, ( "size"
, Json.Encode.int size
)
, ( "query"
, Json.Encode.object
[ ( "bool"
, Json.Encode.object
[ ( "filter"
, Json.Encode.list Json.Encode.object
(List.append
[ [ filter_packages ] ]
filter_queries
)
)
, ( "should"
, Json.Encode.list
Json.Encode.object
([]
|> List.append (should_term 10000) |> List.append (should_term 10000)
|> List.append (should_terms 1000) |> List.append (should_terms 1000)
|> List.append (should_match_bool_prefix 100) |> List.append (should_match_bool_prefix 100)
|> List.append (should_match 10) |> List.append (should_match 10)
) in
)
]
)
]
)
]
)
makeRequest :
Search.Options
-> String
-> String
-> Int
-> Int
-> Cmd Msg
makeRequest options channel query from size =
Search.makeRequest Search.makeRequest
makeRequestBody (Search.makeRequestBody query from size "package" "package_attr_name_query" should_queries)
("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel) ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel)
decodeResultItemSource decodeResultItemSource
options options

160
src/Search.elm
View file

@ -8,6 +8,7 @@ module Search exposing
, decodeResult , decodeResult
, init , init
, makeRequest , makeRequest
, makeRequestBody
, update , update
, view , view
) )
@ -559,8 +560,161 @@ type alias Options =
} }
filter_by_type :
String
-> ( String, Json.Encode.Value )
filter_by_type type_ =
( "term"
, Json.Encode.object
[ ( "type"
, Json.Encode.object
[ ( "value", Json.Encode.string type_ )
, ( "_name", Json.Encode.string <| "filter_" ++ type_ ++ "s" )
]
)
]
)
filter_by_query : String -> String -> List (List ( String, Json.Encode.Value ))
filter_by_query field queryRaw =
let
query =
queryRaw
|> String.trim
in
query
|> String.replace "." " "
|> String.words
|> List.indexedMap
(\i query_word ->
let
isLast =
List.length (String.words query) == i + 1
in
[ if isLast then
( "bool"
, Json.Encode.object
[ ( "should"
, Json.Encode.list Json.Encode.object
[ [ ( "match"
, Json.Encode.object
[ ( field
, Json.Encode.object
[ ( "query", Json.Encode.string query_word )
, ( "fuzziness", Json.Encode.string "1" )
, ( "_name", Json.Encode.string <| "filter_queries_" ++ String.fromInt (i + 1) ++ "_should_match" )
]
)
]
)
]
, [ ( "match_bool_prefix"
, Json.Encode.object
[ ( field
, Json.Encode.object
[ ( "query", Json.Encode.string query_word )
, ( "_name"
, Json.Encode.string <| "filter_queries_" ++ String.fromInt (i + 1) ++ "_should_prefix"
)
]
)
]
)
]
]
)
]
)
else
( "match_bool_prefix"
, Json.Encode.object
[ ( field
, Json.Encode.object
[ ( "query", Json.Encode.string query_word )
, ( "_name"
, Json.Encode.string <| "filter_queries_" ++ String.fromInt (i + 1) ++ "_prefix"
)
]
)
]
)
]
)
makeRequestBody :
String
-> Int
-> Int
-> String
-> String
-> List (List ( String, Json.Encode.Value ))
-> Http.Body
makeRequestBody query from size type_ query_field should_queries =
-- TODO: rescore how close the query is to the root of the name
-- |> List.append
-- ("""int i = 1;
-- for (token in doc['option_name.raw'][0].splitOnToken('.')) {
-- if (token == '"""
-- ++ query
-- ++ """') {
-- return 10000 - (i * 100);
-- }
-- i++;
-- }
-- return 10;
-- """
-- |> stringIn "source"
-- |> objectIn "script"
-- |> objectIn "script_score"
-- |> objectIn "function_score"
-- |> objectIn "rescore_query"
-- |> List.append ("total" |> stringIn "score_mode")
-- |> List.append ("total" |> stringIn "score_mode")
-- |> objectIn "query"
-- |> List.append [ ( "window_size", Json.Encode.int 1000 ) ]
-- |> objectIn "rescore"
-- )
-- |> List.append
-- [ ( "from", Json.Encode.int from )
-- , ( "size", Json.Encode.int size )
-- ]
-- |> Json.Encode.object
-- |> Http.jsonBody
Http.jsonBody
(Json.Encode.object
[ ( "from"
, Json.Encode.int from
)
, ( "size"
, Json.Encode.int size
)
, ( "query"
, Json.Encode.object
[ ( "bool"
, Json.Encode.object
[ ( "filter"
, Json.Encode.list Json.Encode.object
(List.append
[ [ filter_by_type type_ ] ]
(filter_by_query query_field query)
)
)
, ( "should"
, Json.Encode.list Json.Encode.object should_queries
)
]
)
]
)
]
)
makeRequest : makeRequest :
(String -> Int -> Int -> Http.Body) Http.Body
-> String -> String
-> Json.Decode.Decoder a -> Json.Decode.Decoder a
-> Options -> Options
@ -568,7 +722,7 @@ makeRequest :
-> Int -> Int
-> Int -> Int
-> Cmd (Msg a) -> Cmd (Msg a)
makeRequest makeRequestBody index decodeResultItemSource options query from sizeRaw = makeRequest body index decodeResultItemSource options query from sizeRaw =
let let
-- you can not request more then 10000 results otherwise it will return 404 -- you can not request more then 10000 results otherwise it will return 404
size = size =
@ -584,7 +738,7 @@ makeRequest makeRequestBody index decodeResultItemSource options query from size
[ Http.header "Authorization" ("Basic " ++ Base64.encode (options.username ++ ":" ++ options.password)) [ Http.header "Authorization" ("Basic " ++ Base64.encode (options.username ++ ":" ++ options.password))
] ]
, url = options.url ++ "/" ++ index ++ "/_search" , url = options.url ++ "/" ++ index ++ "/_search"
, body = makeRequestBody query from size , body = body
, expect = , expect =
Http.expectJson Http.expectJson
(RemoteData.fromResult >> QueryResponse) (RemoteData.fromResult >> QueryResponse)

3
src/index.js
View file

@ -4,9 +4,10 @@ require("./index.scss");
const {Elm} = require('./Main'); const {Elm} = require('./Main');
console.log("WORKS: " + process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION);
Elm.Main.init({ Elm.Main.init({
flags: { flags: {
elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 0, elasticsearchMappingSchemaVersion: parseInt(process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION) || 0,
elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443', elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443',
elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR', elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR',
elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG' elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG'

3
webpack.config.js
View file

@ -26,6 +26,9 @@ var common = {
filename: MODE == "production" ? "[name]-[hash].js" : "index.js" filename: MODE == "production" ? "[name]-[hash].js" : "index.js"
}, },
plugins: [ plugins: [
new webpack.EnvironmentPlugin([
"ELASTICSEARCH_MAPPING_SCHEMA_VERSION"
]),
new HTMLWebpackPlugin({ new HTMLWebpackPlugin({
// Use this template to get basic responsive meta tags // Use this template to get basic responsive meta tags
template: "src/index.html", template: "src/index.html",