This commit is contained in:
Rok Garbas 2020-06-19 00:41:01 +02:00 committed by GitHub
parent c420d05815
commit bbc97d17f0
Failed to generate hash of commit
13 changed files with 1086 additions and 146 deletions

View file

@ -17,10 +17,23 @@ jobs:
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Installing Nix - name: Installing Nix
uses: cachix/install-nix-action@v8 uses: cachix/install-nix-action@v8
- name: Installing NixFlakes
run: |
nix-env -iA nixpkgs.nixFlakes
echo "::add-path::$HOME/.nix-profile/bin"
echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
nix --version
cat /etc/nix/nix.conf
- name: Building import_scripts
run: |
nix build ./#packages.x86_64-linux.import_scripts
- name: Import ${{ matrix.channel }} channel - name: Import ${{ matrix.channel }} channel
run: | run: |
cd scripts/ ./result/bin/import-channel --es-url ${{ secrets.ELASTICSEARCH_URL }} --channel ${{ matrix.channel }} -vvv
./import-channel --es-url ${{ secrets.ELASTICSEARCH_URL }} --channel ${{ matrix.channel }} -vvv
if: github.repository == 'NixOS/nixos-search' if: github.repository == 'NixOS/nixos-search'

View file

@ -12,13 +12,28 @@ jobs:
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Installing Nix - name: Installing Nix
uses: cachix/install-nix-action@v8 uses: cachix/install-nix-action@v9
- name: Installing NixFlakes
run: |
nix-env -iA nixpkgs.nixFlakes
echo "::add-path::$HOME/.nix-profile/bin"
echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
nix --version
cat /etc/nix/nix.conf
- name: Building import_scripts
run: |
nix build ./#packages.x86_64-linux.import_scripts
- name: Building search.nixos.org - name: Building search.nixos.org
run: | run: |
nix-build nix build ./#packages.x86_64-linux.frontend
mkdir ./dist mkdir ./dist
cp -RL ./result/* ./dist/ cp -RL ./result/* ./dist/
- name: Deploy to Netlify - name: Deploy to Netlify
uses: nwtgck/actions-netlify@v1.0.5 uses: nwtgck/actions-netlify@v1.0.5
env: env:

1
VERSION Normal file
View file

@ -0,0 +1 @@
7

View file

@ -1,15 +1,9 @@
{ pkgs ? import <nixpkgs> {} { pkgs ? import <nixpkgs> { }
, version ? "0"
}: }:
let let
package = builtins.fromJSON (builtins.readFile ./package.json); package = builtins.fromJSON (builtins.readFile ./package.json);
yarn2nix = import (pkgs.fetchFromGitHub { yarnPkg = pkgs.yarn2nix-moretea.mkYarnPackage rec {
owner = "moretea";
repo = "yarn2nix";
rev = "9e7279edde2a4e0f5ec04c53f5cd64440a27a1ae";
sha256 = "sha256-x77mYSvwA4bd/uCwL0rRw+8mwH+dR/UfWnn4YXmm4n8=";
}) { inherit pkgs; };
yarnPkg = yarn2nix.mkYarnPackage rec {
name = "${package.name}-yarn-${package.version}"; name = "${package.name}-yarn-${package.version}";
src = null; src = null;
dontUnpack = true; dontUnpack = true;
@ -28,7 +22,7 @@ let
''; '';
pkgConfig = { pkgConfig = {
node-sass = { node-sass = {
buildInputs = [ pkgs.python pkgs.libsass pkgs.pkgconfig]; buildInputs = [ pkgs.python pkgs.libsass pkgs.pkgconfig ];
postInstall = '' postInstall = ''
LIBSASS_EXT=auto yarn --offline run build LIBSASS_EXT=auto yarn --offline run build
rm build/config.gypi rm build/config.gypi
@ -36,16 +30,19 @@ let
}; };
}; };
publishBinsFor = publishBinsFor =
[ "webpack" [
"webpack"
"webpack-dev-server" "webpack-dev-server"
]; ];
}; };
in pkgs.stdenv.mkDerivation { in
pkgs.stdenv.mkDerivation {
name = "${package.name}-${package.version}"; name = "${package.name}-${package.version}";
src = pkgs.lib.cleanSource ./.; src = pkgs.lib.cleanSource ./.;
buildInputs = buildInputs =
[ yarnPkg [
yarnPkg
] ++ ] ++
(with pkgs; [ (with pkgs; [
nodejs nodejs
@ -76,6 +73,7 @@ in pkgs.stdenv.mkDerivation {
buildPhase = '' buildPhase = ''
# Yarn writes cache directories etc to $HOME. # Yarn writes cache directories etc to $HOME.
export HOME=$PWD/yarn_home export HOME=$PWD/yarn_home
sed -i -e "s|process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION|${version}|" src/index.js
yarn prod yarn prod
''; '';
@ -88,6 +86,7 @@ in pkgs.stdenv.mkDerivation {
rm -rf node_modules rm -rf node_modules
ln -sf ${yarnPkg}/libexec/${package.name}/node_modules . ln -sf ${yarnPkg}/libexec/${package.name}/node_modules .
export PATH=$PWD/node_modules/.bin:$PATH export PATH=$PWD/node_modules/.bin:$PATH
export ELASTICSEARCH_MAPPING_SCHEMA_VERSION=${version}
echo "============================" echo "============================"
echo "= To develop run: yarn dev =" echo "= To develop run: yarn dev ="
echo "============================" echo "============================"

65
flake.lock Normal file
View file

@ -0,0 +1,65 @@
{
"nodes": {
"nixpkgs": {
"info": {
"lastModified": 1592376843,
"narHash": "sha256-vKchUGsNMQC9vCmTnpssHMV/+cKXA5QhbEoFm0zua1U="
},
"locked": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "14fcd947a0150b78d7a37bef56c87fcd57808247",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-unstable",
"type": "indirect"
}
},
"nixpkgs_2": {
"info": {
"lastModified": 1592514044,
"narHash": "sha256-jGQJmq14vh9ASt9d5Z9yBPsrbUgroiNVNhsoBabsw0k="
},
"locked": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "e051dab9ff8a58455f6b8375856ac2802c6cd261",
"type": "github"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"poetry2nix": {
"info": {
"lastModified": 1592121810,
"narHash": "sha256-9cGTKtL7JpllwABxa4iRVU4Qw/YwniLni4RATyONz+s="
},
"inputs": {
"nixpkgs": "nixpkgs_2"
},
"locked": {
"owner": "nix-community",
"repo": "poetry2nix",
"rev": "e7c69a288c10e4d97816fdabda5ae3f38e21914e",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "poetry2nix",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"poetry2nix": "poetry2nix"
}
}
},
"root": "root",
"version": 5
}

35
flake.nix Normal file
View file

@ -0,0 +1,35 @@
{
description = "Code behind search.nixos.org";
inputs = {
nixpkgs = { url = "nixpkgs/nixos-unstable"; };
poetry2nix = { url = "github:nix-community/poetry2nix"; };
};
outputs = { self, nixpkgs, poetry2nix }:
let
systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
allPackages = system:
let
pkgs = import nixpkgs {
inherit system;
overlays = [
poetry2nix.overlay
];
};
version = pkgs.lib.removeSuffix "\n" (builtins.readFile "${self}/VERSION");
in
{
import_scripts = import ./import-scripts {
inherit pkgs version;
};
frontend = import ./. {
inherit pkgs version;
};
};
in
{
packages = forAllSystems allPackages;
};
}

View file

@ -0,0 +1,23 @@
{ pkgs ? import <nixpkgs> { }
, version ? "0"
}:
let
inherit (pkgs.poetry2nix) mkPoetryApplication overrides;
in
mkPoetryApplication {
projectDir = ./.;
overrides = overrides.withDefaults (self: super: {
pypandoc = super.pypandoc.overridePythonAttrs (old: {
postPatch = ''
sed -i '/^__pandoc_path = None$/c__pandoc_path = "${pkgs.pandoc}/bin/pandoc"' pypandoc/__init__.py
'';
});
});
checkPhase = ''
black --diff --check ./import_scripts
flake8 --ignore W503,E501,E265,E203 ./import_scripts
'';
postInstall = ''
wrapProgram $out/bin/import-channel --set INDEX_SCHEMA_VERSION "${version}"
'';
}

View file

@ -1,15 +1,3 @@
#! /usr/bin/env nix-shell
#! nix-shell -i python3 -p python3 python3Packages.requests python3Packages.click python3Packages.click-log python3Packages.elasticsearch python3Packages.boto3 python3Packages.tqdm python3Packages.pypandoc
# develop:
# $ nix-shell -p python3Packages.black python3Packages.mypy python3Packages.flake8
#
# format:
# $ nix-shell -p python3Packages.black --command "black import-channel"
#
# lint:
# $ nix-shell -p python3Packages.flake8 --command "flake8 --ignore E501,E265 import-channel"
import boto3 import boto3
import botocore import botocore
import botocore.client import botocore.client
@ -19,12 +7,14 @@ import elasticsearch
import elasticsearch.helpers import elasticsearch.helpers
import json import json
import logging import logging
import os
import os.path import os.path
import pypandoc import pypandoc
import re import re
import requests import requests
import shlex import shlex
import subprocess import subprocess
import sys
import tqdm import tqdm
import xml.etree.ElementTree import xml.etree.ElementTree
@ -34,7 +24,7 @@ click_log.basic_config(logger)
S3_BUCKET = "nix-releases" S3_BUCKET = "nix-releases"
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
INDEX_SCHEMA_VERSION = 6 INDEX_SCHEMA_VERSION = os.environ.get("INDEX_SCHEMA_VERSION", 0)
CHANNELS = { CHANNELS = {
"unstable": { "unstable": {
"packages": "nixpkgs/nixpkgs-20.09pre", "packages": "nixpkgs/nixpkgs-20.09pre",
@ -51,11 +41,7 @@ CHANNELS = {
} }
ANALYSIS = { ANALYSIS = {
"normalizer": { "normalizer": {
"lowercase": { "lowercase": {"type": "custom", "char_filter": [], "filter": ["lowercase"]}
"type": "custom",
"char_filter": [],
"filter": ["lowercase"],
}
}, },
"analyzer": { "analyzer": {
"lowercase": { "lowercase": {
@ -75,14 +61,7 @@ ANALYSIS = {
}, },
}, },
"tokenizer": { "tokenizer": {
"nix_package_query": { "nix_package_query": {"type": "pattern", "pattern": "|".join(["[ ]"])},
"type": "pattern",
"pattern": "|".join(
[
"[ ]",
]
),
},
"nix_package_attr_name": { "nix_package_attr_name": {
"type": "pattern", "type": "pattern",
# Split on attrname separators like _, . # Split on attrname separators like _, .
@ -107,10 +86,7 @@ ANALYSIS = {
] ]
), ),
}, },
"nix_option_name": { "nix_option_name": {"type": "pattern", "pattern": "[.]"},
"type": "pattern",
"pattern": "[.]",
},
# Lower priority (virtualHost -> [virtual, host]) # Lower priority (virtualHost -> [virtual, host])
"nix_option_name_granular": { "nix_option_name_granular": {
"type": "pattern", "type": "pattern",
@ -169,28 +145,16 @@ MAPPING = {
"type": "nested", "type": "nested",
"properties": { "properties": {
"output": {"type": "keyword"}, "output": {"type": "keyword"},
"path": {"type": "keyword"} "path": {"type": "keyword"},
}, },
}, },
"drv_path": {"type": "keyword"}, "drv_path": {"type": "keyword"},
}, },
}, },
"package_attr_name": { "package_attr_name": {"type": "keyword", "normalizer": "lowercase"},
"type": "keyword", "package_attr_name_query": {"type": "keyword", "normalizer": "lowercase"},
"normalizer": "lowercase", "package_attr_set": {"type": "keyword", "normalizer": "lowercase"},
}, "package_pname": {"type": "keyword", "normalizer": "lowercase"},
"package_attr_name_query": {
"type": "keyword",
"normalizer": "lowercase",
},
"package_attr_set": {
"type": "keyword",
"normalizer": "lowercase",
},
"package_pname": {
"type": "keyword",
"normalizer": "lowercase",
},
"package_pversion": {"type": "keyword"}, "package_pversion": {"type": "keyword"},
"package_description": {"type": "text"}, "package_description": {"type": "text"},
"package_longDescription": {"type": "text"}, "package_longDescription": {"type": "text"},
@ -216,13 +180,8 @@ MAPPING = {
"analyzer": "nixOptionName", "analyzer": "nixOptionName",
"fielddata": True, "fielddata": True,
"fields": { "fields": {
"raw": { "raw": {"type": "keyword"},
"type": "keyword" "granular": {"type": "text", "analyzer": "nixOptionNameGranular"},
},
"granular": {
"type": "text",
"analyzer": "nixOptionNameGranular",
},
}, },
}, },
"option_description": {"type": "text"}, "option_description": {"type": "text"},
@ -257,8 +216,10 @@ def split_query(text):
- test2 - test2
""" """
tokens = [] tokens = []
regex = re.compile(".+?(?:(?<=[a-z])(?=[1-9A-Z])|(?<=[1-9A-Z])(?=[A-Z][a-z])|[\._-]|$)") regex = re.compile(
parts = [m.group(0) for m in regex.finditer(text)] ".+?(?:(?<=[a-z])(?=[1-9A-Z])|(?<=[1-9A-Z])(?=[A-Z][a-z])|[._-]|$)"
)
parts = [m.group(0) for m in regex.finditer(text)]
for index in range(len(parts)): for index in range(len(parts)):
prev_parts = "" prev_parts = ""
for part in parts[index:]: for part in parts[index:]:
@ -279,14 +240,13 @@ def get_last_evaluation(prefix):
if not item: if not item:
continue continue
logger.debug(f"get_last_evaluation: evaluation in raw {item}") logger.debug(f"get_last_evaluation: evaluation in raw {item}")
try: revisions_since_start, git_revision = (
revisions_since_start, git_revision = item['Prefix'][len(prefix):].rstrip('/').split('.') item["Prefix"][len(prefix) :].rstrip("/").split(".")
except: )
__import__('pdb').set_trace()
evaluation = { evaluation = {
"revisions_since_start": int(revisions_since_start), "revisions_since_start": int(revisions_since_start),
"git_revision": git_revision, "git_revision": git_revision,
"prefix": item['Prefix'].rstrip('/'), "prefix": item["Prefix"].rstrip("/"),
} }
logger.debug(f"get_last_evaluation: evaluation {evaluation}") logger.debug(f"get_last_evaluation: evaluation {evaluation}")
evaluations.append(evaluation) evaluations.append(evaluation)
@ -299,7 +259,9 @@ def get_last_evaluation(prefix):
evaluation = evaluations[-1] evaluation = evaluations[-1]
result = s3.get_object(Bucket=S3_BUCKET, Key=f"{evaluation['prefix']}/src-url") result = s3.get_object(Bucket=S3_BUCKET, Key=f"{evaluation['prefix']}/src-url")
evaluation['id'] = result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/"):] evaluation["id"] = (
result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/") :]
)
logger.debug(f"get_last_evaluation: last evaluation is: {evaluation}") logger.debug(f"get_last_evaluation: last evaluation is: {evaluation}")
@ -307,21 +269,21 @@ def get_last_evaluation(prefix):
def get_evaluation_builds(evaluation_id): def get_evaluation_builds(evaluation_id):
logger.debug(f"get_evaluation_builds: Retriving list of builds for {evaluation_id} evaluation id") logger.debug(
f"get_evaluation_builds: Retriving list of builds for {evaluation_id} evaluation id"
)
filename = f"eval-{evaluation_id}.json" filename = f"eval-{evaluation_id}.json"
if not os.path.exists(filename): if not os.path.exists(filename):
url = f"https://hydra.nixos.org/eval/{evaluation_id}/builds" url = f"https://hydra.nixos.org/eval/{evaluation_id}/builds"
logger.debug(f"get_evaluation_builds: Fetching builds from {url} url.") logger.debug(f"get_evaluation_builds: Fetching builds from {url} url.")
headers = { headers = {"Content-Type": "application/json"}
"Content-Type": "application/json"
}
r = requests.get(url, headers=headers, stream=True) r = requests.get(url, headers=headers, stream=True)
with tqdm.tqdm.wrapattr( with tqdm.tqdm.wrapattr(
open(filename, "wb"), open(filename, "wb"),
"write", "write",
miniters=1, miniters=1,
total=int(r.headers.get('content-length', 0)), total=int(r.headers.get("content-length", 0)),
desc=filename desc=filename,
) as f: ) as f:
for chunk in r.iter_content(chunk_size=4096): for chunk in r.iter_content(chunk_size=4096):
f.write(chunk) f.write(chunk)
@ -331,8 +293,8 @@ def get_evaluation_builds(evaluation_id):
result = {} result = {}
for build in builds: for build in builds:
result.setdefault(build['nixname'], {}) result.setdefault(build["nixname"], {})
result[build['nixname']][build['system']] = build result[build["nixname"]][build["system"]] = build
return result return result
@ -341,18 +303,16 @@ def get_maintainer(maintainer):
maintainers = [] maintainers = []
if type(maintainer) == str: if type(maintainer) == str:
maintainers.append(dict( maintainers.append(dict(name=maintainer, email=None, github=None,))
name=maintainer,
email=None,
github=None,
))
elif type(maintainer) == dict: elif type(maintainer) == dict:
maintainers.append(dict( maintainers.append(
name=maintainer.get("name"), dict(
email=maintainer.get("email"), name=maintainer.get("name"),
github=maintainer.get("github"), email=maintainer.get("email"),
)) github=maintainer.get("github"),
)
)
elif type(maintainer) == list: elif type(maintainer) == list:
for item in maintainer: for item in maintainer:
@ -380,8 +340,7 @@ def remove_attr_set(name):
# Plugins # Plugins
"elasticsearch", "elasticsearch",
"graylog", "graylog",
"tmuxplugin" "tmuxplugin" "vimplugin",
"vimplugin"
] ]
# TODO: is this correct # TODO: is this correct
if any([name.startswith(i) for i in sets]): if any([name.startswith(i) for i in sets]):
@ -389,7 +348,7 @@ def remove_attr_set(name):
# node does things a bit different # node does things a bit different
elif name.startswith("node_"): elif name.startswith("node_"):
name = name[len("node_"):] name = name[len("node_") :]
return name return name
@ -440,31 +399,32 @@ def get_packages(evaluation, evaluation_builds):
attr_set = None attr_set = None
if "." in attr_name: if "." in attr_name:
attr_set = attr_name.split(".")[0] attr_set = attr_name.split(".")[0]
if not attr_set.endswith("Packages") and \ if (
not attr_set.endswith("Plugins") and \ not attr_set.endswith("Packages")
not attr_set.endswith("Extensions"): and not attr_set.endswith("Plugins")
and not attr_set.endswith("Extensions")
):
attr_set = None attr_set = None
hydra = None hydra = None
if data['name'] in evaluation_builds: if data["name"] in evaluation_builds:
hydra = [] hydra = []
for platform, build in evaluation_builds[data['name']].items(): for platform, build in evaluation_builds[data["name"]].items():
hydra.append({ hydra.append(
"build_id": build['id'], {
"build_status": build['buildstatus'], "build_id": build["id"],
"platform": build['system'], "build_status": build["buildstatus"],
"project": build['project'], "platform": build["system"],
"jobset": build['jobset'], "project": build["project"],
"job": build['job'], "jobset": build["jobset"],
"path": [ "job": build["job"],
{ "path": [
"output": output, {"output": output, "path": item["path"]}
"path": item['path'], for output, item in build["buildoutputs"].items()
} ],
for output, item in build['buildoutputs'].items() "drv_path": build["drvpath"],
], }
"drv_path": build['drvpath'], )
})
yield dict( yield dict(
type="package", type="package",
@ -512,7 +472,7 @@ def get_options(evaluation):
example = option.get("example") example = option.get("example")
if example is not None: if example is not None:
if (type(example) == dict and example.get("_type") == "literalExample"): if type(example) == dict and example.get("_type") == "literalExample":
example = json.dumps(example["text"]) example = json.dumps(example["text"])
else: else:
example = json.dumps(example) example = json.dumps(example)
@ -520,18 +480,16 @@ def get_options(evaluation):
description = option.get("description") description = option.get("description")
if description is not None: if description is not None:
xml_description = ( xml_description = (
f"<xml xmlns:xlink=\"http://www.w3.org/1999/xlink\">" f'<xml xmlns:xlink="http://www.w3.org/1999/xlink">'
f"<para>{description}</para>" f"<para>{description}</para>"
f"</xml>" f"</xml>"
) )
# we first check if there are some xml elements before using pypandoc # we first check if there are some xml elements before using pypandoc
# since pypandoc calls are quite slow # since pypandoc calls are quite slow
root = xml.etree.ElementTree.fromstring(xml_description) root = xml.etree.ElementTree.fromstring(xml_description)
if len(list(root.find('para'))) > 0: if len(list(root.find("para"))) > 0:
description = pypandoc.convert_text( description = pypandoc.convert_text(
xml_description, xml_description, "html", format="docbook",
"html",
format="docbook",
) )
yield dict( yield dict(
@ -569,14 +527,16 @@ def ensure_index(es, index, mapping, force=False):
def create_index_name(channel, evaluation_packages, evaluation_options): def create_index_name(channel, evaluation_packages, evaluation_options):
evaluation_name = '-'.join([ evaluation_name = "-".join(
evaluation_packages['id'], [
str(evaluation_packages['revisions_since_start']), evaluation_packages["id"],
evaluation_packages['git_revision'], str(evaluation_packages["revisions_since_start"]),
evaluation_options['id'], evaluation_packages["git_revision"],
str(evaluation_options['revisions_since_start']), evaluation_options["id"],
evaluation_options['git_revision'], str(evaluation_options["revisions_since_start"]),
]) evaluation_options["git_revision"],
]
)
return ( return (
f"latest-{INDEX_SCHEMA_VERSION}-{channel}", f"latest-{INDEX_SCHEMA_VERSION}-{channel}",
f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation_name}", f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation_name}",
@ -624,7 +584,7 @@ def write(unit, es, index_name, number_of_items, item_generator):
@click.option("-c", "--channel", type=click.Choice(CHANNELS.keys()), help="Channel.") @click.option("-c", "--channel", type=click.Choice(CHANNELS.keys()), help="Channel.")
@click.option("-f", "--force", is_flag=True, help="Force channel recreation.") @click.option("-f", "--force", is_flag=True, help="Force channel recreation.")
@click.option("-v", "--verbose", count=True) @click.option("-v", "--verbose", count=True)
def main(es_url, channel, force, verbose): def run(es_url, channel, force, verbose):
logging_level = "CRITICAL" logging_level = "CRITICAL"
if verbose == 1: if verbose == 1:
@ -636,23 +596,30 @@ def main(es_url, channel, force, verbose):
logger.debug(f"Verbosity is {verbose}") logger.debug(f"Verbosity is {verbose}")
logger.debug(f"Logging set to {logging_level}") logger.debug(f"Logging set to {logging_level}")
evaluation_packages = get_last_evaluation(CHANNELS[channel]['packages']) evaluation_packages = get_last_evaluation(CHANNELS[channel]["packages"])
evaluation_options = get_last_evaluation(CHANNELS[channel]['options']) evaluation_options = get_last_evaluation(CHANNELS[channel]["options"])
evaluation_packages_builds = get_evaluation_builds(evaluation_packages['id']) evaluation_packages_builds = get_evaluation_builds(evaluation_packages["id"])
es = elasticsearch.Elasticsearch([es_url]) es = elasticsearch.Elasticsearch([es_url])
alias_name, index_name = create_index_name(channel, evaluation_packages, evaluation_options) alias_name, index_name = create_index_name(
channel, evaluation_packages, evaluation_options
)
index_created = ensure_index(es, index_name, MAPPING, force) index_created = ensure_index(es, index_name, MAPPING, force)
if index_created: if index_created:
write("packages", es, index_name, *get_packages(evaluation_packages, evaluation_packages_builds)) write(
"packages",
es,
index_name,
*get_packages(evaluation_packages, evaluation_packages_builds),
)
write("options", es, index_name, *get_options(evaluation_options)) write("options", es, index_name, *get_options(evaluation_options))
update_alias(es, alias_name, index_name) update_alias(es, alias_name, index_name)
if __name__ == "__main__": if __name__ == "__main__":
main() run()
# vi:ft=python # vi:ft=python

791
import-scripts/poetry.lock generated Normal file
View file

@ -0,0 +1,791 @@
[[package]]
category = "dev"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
name = "appdirs"
optional = false
python-versions = "*"
version = "1.4.4"
[[package]]
category = "dev"
description = "Disable App Nap on OS X 10.9"
marker = "python_version >= \"3.4\" and sys_platform == \"darwin\""
name = "appnope"
optional = false
python-versions = "*"
version = "0.1.0"
[[package]]
category = "dev"
description = "Classes Without Boilerplate"
name = "attrs"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "19.3.0"
[package.extras]
azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"]
dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"]
docs = ["sphinx", "zope.interface"]
tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
[[package]]
category = "dev"
description = "Specifications for callback functions passed in to an API"
marker = "python_version >= \"3.4\""
name = "backcall"
optional = false
python-versions = "*"
version = "0.2.0"
[[package]]
category = "dev"
description = "The uncompromising code formatter."
name = "black"
optional = false
python-versions = ">=3.6"
version = "19.10b0"
[package.dependencies]
appdirs = "*"
attrs = ">=18.1.0"
click = ">=6.5"
pathspec = ">=0.6,<1"
regex = "*"
toml = ">=0.9.4"
typed-ast = ">=1.4.0"
[package.extras]
d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
[[package]]
category = "main"
description = "The AWS SDK for Python"
name = "boto3"
optional = false
python-versions = "*"
version = "1.14.5"
[package.dependencies]
botocore = ">=1.17.5,<1.18.0"
jmespath = ">=0.7.1,<1.0.0"
s3transfer = ">=0.3.0,<0.4.0"
[[package]]
category = "main"
description = "Low-level, data-driven core of boto 3."
name = "botocore"
optional = false
python-versions = "*"
version = "1.17.5"
[package.dependencies]
docutils = ">=0.10,<0.16"
jmespath = ">=0.7.1,<1.0.0"
python-dateutil = ">=2.1,<3.0.0"
[package.dependencies.urllib3]
python = "<3.4.0 || >=3.5.0"
version = ">=1.20,<1.26"
[[package]]
category = "main"
description = "Python package for providing Mozilla's CA Bundle."
name = "certifi"
optional = false
python-versions = "*"
version = "2020.4.5.2"
[[package]]
category = "main"
description = "Universal encoding detector for Python 2 and 3"
name = "chardet"
optional = false
python-versions = "*"
version = "3.0.4"
[[package]]
category = "main"
description = "Composable command line interface toolkit"
name = "click"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "7.1.2"
[[package]]
category = "main"
description = "Logging integration for Click"
name = "click-log"
optional = false
python-versions = "*"
version = "0.3.2"
[package.dependencies]
click = "*"
[[package]]
category = "dev"
description = "Cross-platform colored terminal text."
marker = "python_version >= \"3.4\" and sys_platform == \"win32\""
name = "colorama"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.4.3"
[[package]]
category = "dev"
description = "Decorators for Humans"
marker = "python_version >= \"3.4\""
name = "decorator"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*"
version = "4.4.2"
[[package]]
category = "main"
description = "Docutils -- Python Documentation Utilities"
name = "docutils"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
version = "0.15.2"
[[package]]
category = "main"
description = "Python client for Elasticsearch"
name = "elasticsearch"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4"
version = "7.8.0"
[package.dependencies]
certifi = "*"
urllib3 = ">=1.21.1"
[package.extras]
async = ["aiohttp (>=3,<4)", "yarl"]
develop = ["requests (>=2.0.0,<3.0.0)", "coverage", "mock", "pyyaml", "pytest", "pytest-cov", "sphinx (<1.7)", "sphinx-rtd-theme", "black", "jinja2"]
docs = ["sphinx (<1.7)", "sphinx-rtd-theme"]
requests = ["requests (>=2.4.0,<3.0.0)"]
[[package]]
category = "dev"
description = "the modular source code checker: pep8 pyflakes and co"
name = "flake8"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
version = "3.8.3"
[package.dependencies]
mccabe = ">=0.6.0,<0.7.0"
pycodestyle = ">=2.6.0a1,<2.7.0"
pyflakes = ">=2.2.0,<2.3.0"
[[package]]
category = "main"
description = "Internationalized Domain Names in Applications (IDNA)"
name = "idna"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.9"
[[package]]
category = "dev"
description = "IPython-enabled pdb"
name = "ipdb"
optional = false
python-versions = ">=2.7"
version = "0.13.2"
[package.dependencies]
setuptools = "*"
[package.dependencies.ipython]
python = ">=3.4"
version = ">=5.1.0"
[[package]]
category = "dev"
description = "IPython: Productive Interactive Computing"
marker = "python_version >= \"3.4\""
name = "ipython"
optional = false
python-versions = ">=3.6"
version = "7.15.0"
[package.dependencies]
appnope = "*"
backcall = "*"
colorama = "*"
decorator = "*"
jedi = ">=0.10"
pexpect = "*"
pickleshare = "*"
prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0"
pygments = "*"
setuptools = ">=18.5"
traitlets = ">=4.2"
[package.extras]
all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.14)", "pygments", "qtconsole", "requests", "testpath"]
doc = ["Sphinx (>=1.3)"]
kernel = ["ipykernel"]
nbconvert = ["nbconvert"]
nbformat = ["nbformat"]
notebook = ["notebook", "ipywidgets"]
parallel = ["ipyparallel"]
qtconsole = ["qtconsole"]
test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.14)"]
[[package]]
category = "dev"
description = "Vestigial utilities from IPython"
marker = "python_version >= \"3.4\""
name = "ipython-genutils"
optional = false
python-versions = "*"
version = "0.2.0"
[[package]]
category = "dev"
description = "An autocompletion tool for Python that can be used for text editors."
marker = "python_version >= \"3.4\""
name = "jedi"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.17.0"
[package.dependencies]
parso = ">=0.7.0"
[package.extras]
qa = ["flake8 (3.7.9)"]
testing = ["colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"]
[[package]]
category = "main"
description = "JSON Matching Expressions"
name = "jmespath"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
version = "0.10.0"
[[package]]
category = "dev"
description = "McCabe checker, plugin for flake8"
name = "mccabe"
optional = false
python-versions = "*"
version = "0.6.1"
[[package]]
category = "dev"
description = "Optional static typing for Python"
name = "mypy"
optional = false
python-versions = ">=3.5"
version = "0.780"
[package.dependencies]
mypy-extensions = ">=0.4.3,<0.5.0"
typed-ast = ">=1.4.0,<1.5.0"
typing-extensions = ">=3.7.4"
[package.extras]
dmypy = ["psutil (>=4.0)"]
[[package]]
category = "dev"
description = "Experimental type system extensions for programs checked with the mypy typechecker."
name = "mypy-extensions"
optional = false
python-versions = "*"
version = "0.4.3"
[[package]]
category = "dev"
description = "A Python Parser"
marker = "python_version >= \"3.4\""
name = "parso"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.7.0"
[package.extras]
testing = ["docopt", "pytest (>=3.0.7)"]
[[package]]
category = "dev"
description = "Utility library for gitignore style pattern matching of file paths."
name = "pathspec"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.8.0"
[[package]]
category = "dev"
description = "Pexpect allows easy control of interactive console applications."
marker = "python_version >= \"3.4\" and sys_platform != \"win32\""
name = "pexpect"
optional = false
python-versions = "*"
version = "4.8.0"
[package.dependencies]
ptyprocess = ">=0.5"
[[package]]
category = "dev"
description = "Tiny 'shelve'-like database with concurrency support"
marker = "python_version >= \"3.4\""
name = "pickleshare"
optional = false
python-versions = "*"
version = "0.7.5"
[[package]]
category = "dev"
description = "Library for building powerful interactive command lines in Python"
marker = "python_version >= \"3.4\""
name = "prompt-toolkit"
optional = false
python-versions = ">=3.6.1"
version = "3.0.5"
[package.dependencies]
wcwidth = "*"
[[package]]
category = "dev"
description = "Run a subprocess in a pseudo terminal"
marker = "python_version >= \"3.4\" and sys_platform != \"win32\""
name = "ptyprocess"
optional = false
python-versions = "*"
version = "0.6.0"
[[package]]
category = "dev"
description = "Python style guide checker"
name = "pycodestyle"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.6.0"
[[package]]
category = "dev"
description = "passive checker of Python programs"
name = "pyflakes"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.2.0"
[[package]]
category = "dev"
description = "Pygments is a syntax highlighting package written in Python."
marker = "python_version >= \"3.4\""
name = "pygments"
optional = false
python-versions = ">=3.5"
version = "2.6.1"
[[package]]
category = "main"
description = "Thin wrapper for pandoc."
name = "pypandoc"
optional = false
python-versions = "*"
version = "1.5"
[package.dependencies]
pip = ">=8.1.0"
setuptools = "*"
wheel = ">=0.25.0"
[[package]]
category = "main"
description = "Extensions to the standard Python datetime module"
name = "python-dateutil"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
version = "2.8.1"
[package.dependencies]
six = ">=1.5"
[[package]]
category = "dev"
description = "Alternative regular expression module, to replace re."
name = "regex"
optional = false
python-versions = "*"
version = "2020.6.8"
[[package]]
category = "main"
description = "Python HTTP for Humans."
name = "requests"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "2.24.0"
[package.dependencies]
certifi = ">=2017.4.17"
chardet = ">=3.0.2,<4"
idna = ">=2.5,<3"
urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26"
[package.extras]
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
[[package]]
category = "main"
description = "An Amazon S3 Transfer Manager"
name = "s3transfer"
optional = false
python-versions = "*"
version = "0.3.3"
[package.dependencies]
botocore = ">=1.12.36,<2.0a.0"
[[package]]
category = "main"
description = "Python 2 and 3 compatibility utilities"
name = "six"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
version = "1.15.0"
[[package]]
category = "dev"
description = "Python Library for Tom's Obvious, Minimal Language"
name = "toml"
optional = false
python-versions = "*"
version = "0.10.1"
[[package]]
category = "main"
description = "Fast, Extensible Progress Meter"
name = "tqdm"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*"
version = "4.46.1"
[package.extras]
dev = ["py-make (>=0.1.0)", "twine", "argopt", "pydoc-markdown"]
[[package]]
category = "dev"
description = "Traitlets Python config system"
marker = "python_version >= \"3.4\""
name = "traitlets"
optional = false
python-versions = "*"
version = "4.3.3"
[package.dependencies]
decorator = "*"
ipython-genutils = "*"
six = "*"
[package.extras]
test = ["pytest", "mock"]
[[package]]
category = "dev"
description = "a fork of Python 2 and 3 ast modules with type comment support"
name = "typed-ast"
optional = false
python-versions = "*"
version = "1.4.1"
[[package]]
category = "dev"
description = "Backported and Experimental Type Hints for Python 3.5+"
name = "typing-extensions"
optional = false
python-versions = "*"
version = "3.7.4.2"
[[package]]
category = "main"
description = "HTTP library with thread-safe connection pooling, file post, and more."
name = "urllib3"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
version = "1.25.9"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
[[package]]
category = "dev"
description = "Measures the displayed width of unicode strings in a terminal"
marker = "python_version >= \"3.4\""
name = "wcwidth"
optional = false
python-versions = "*"
version = "0.2.4"
[[package]]
category = "main"
description = "A built-package format for Python"
name = "wheel"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.34.2"
[package.extras]
test = ["pytest (>=3.0.0)", "pytest-cov"]
[metadata]
content-hash = "b75343832402daab2b4e4c515ff32a0acbe74e1b1f22703e58060a14ea231904"
python-versions = "^3.8"
[metadata.files]
appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
]
appnope = [
{file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"},
{file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"},
]
attrs = [
{file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"},
{file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"},
]
backcall = [
{file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
{file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
]
black = [
{file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"},
{file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"},
]
boto3 = [
{file = "boto3-1.14.5-py2.py3-none-any.whl", hash = "sha256:ed9b0852c0dc8ca88f5f851357acf238b954d1cced44f48e0d930d306150de7c"},
{file = "boto3-1.14.5.tar.gz", hash = "sha256:7098937f6432e3ae4e5fe9b93f561b06117c5df736effb8cc166f6fb2bb41ab8"},
]
botocore = [
{file = "botocore-1.17.5-py2.py3-none-any.whl", hash = "sha256:072c82c64906996f1d7953da1a61d8e6debf0ee5acaa267ec777f05b30755b66"},
{file = "botocore-1.17.5.tar.gz", hash = "sha256:fcfc3762472aa1d758583d818faaa59b933d839a87f372688836d49d66ad9a7a"},
]
certifi = [
{file = "certifi-2020.4.5.2-py2.py3-none-any.whl", hash = "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"},
{file = "certifi-2020.4.5.2.tar.gz", hash = "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1"},
]
chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
{file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
]
click = [
{file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
{file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
]
click-log = [
{file = "click-log-0.3.2.tar.gz", hash = "sha256:16fd1ca3fc6b16c98cea63acf1ab474ea8e676849dc669d86afafb0ed7003124"},
{file = "click_log-0.3.2-py2.py3-none-any.whl", hash = "sha256:eee14dc37cdf3072158570f00406572f9e03e414accdccfccd4c538df9ae322c"},
]
colorama = [
{file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
{file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"},
]
decorator = [
{file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"},
{file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"},
]
docutils = [
{file = "docutils-0.15.2-py2-none-any.whl", hash = "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827"},
{file = "docutils-0.15.2-py3-none-any.whl", hash = "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0"},
{file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"},
]
elasticsearch = [
{file = "elasticsearch-7.8.0-py2.py3-none-any.whl", hash = "sha256:6fb566dd23b91b5871ce12212888674b4cf33374e92b71b1080916c931e44dcb"},
{file = "elasticsearch-7.8.0.tar.gz", hash = "sha256:e637d8cf4e27e279b5ff8ca8edc0c086f4b5df4bf2b48e2f950b7833aca3a792"},
]
flake8 = [
{file = "flake8-3.8.3-py2.py3-none-any.whl", hash = "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c"},
{file = "flake8-3.8.3.tar.gz", hash = "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208"},
]
idna = [
{file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"},
{file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"},
]
ipdb = [
{file = "ipdb-0.13.2.tar.gz", hash = "sha256:77fb1c2a6fccdfee0136078c9ed6fe547ab00db00bebff181f1e8c9e13418d49"},
]
ipython = [
{file = "ipython-7.15.0-py3-none-any.whl", hash = "sha256:1b85d65632211bf5d3e6f1406f3393c8c429a47d7b947b9a87812aa5bce6595c"},
{file = "ipython-7.15.0.tar.gz", hash = "sha256:0ef1433879816a960cd3ae1ae1dc82c64732ca75cec8dab5a4e29783fb571d0e"},
]
ipython-genutils = [
{file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"},
{file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"},
]
jedi = [
{file = "jedi-0.17.0-py2.py3-none-any.whl", hash = "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798"},
{file = "jedi-0.17.0.tar.gz", hash = "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030"},
]
jmespath = [
{file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"},
{file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"},
]
mccabe = [
{file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
{file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
]
mypy = [
{file = "mypy-0.780-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:d3b4941de44341227ece1caaf5b08b23e42ad4eeb8b603219afb11e9d4cfb437"},
{file = "mypy-0.780-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:1f3976a945ad7f0a0727aafdc5651c2d3278e3c88dee94e2bf75cd3386b7b2f4"},
{file = "mypy-0.780-cp35-cp35m-win_amd64.whl", hash = "sha256:eadb865126da4e3c4c95bdb47fe1bb087a3e3ea14d39a3b13224b8a4d9f9a102"},
{file = "mypy-0.780-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:75eed74d2faf2759f79c5f56f17388defd2fc994222312ec54ee921e37b31ad4"},
{file = "mypy-0.780-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2f8c098f12b402c19b735aec724cc9105cc1a9eea405d08814eb4b14a6fb1a41"},
{file = "mypy-0.780-cp36-cp36m-win_amd64.whl", hash = "sha256:62eb5dd4ea86bda8ce386f26684f7f26e4bfe6283c9f2b6ca6d17faf704dcfad"},
{file = "mypy-0.780-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:00cb1964a7476e871d6108341ac9c1a857d6bd20bf5877f4773ac5e9d92cd3cd"},
{file = "mypy-0.780-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:974bebe3699b9b46278a7f076635d219183da26e1a675c1f8243a69221758273"},
{file = "mypy-0.780-cp37-cp37m-win_amd64.whl", hash = "sha256:64c36eb0936d0bfb7d8da49f92c18e312ad2e3ed46e5548ae4ca997b0d33bd59"},
{file = "mypy-0.780-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d31291df31bafb997952dc0a17ebb2737f802c754aed31dd155a8bfe75112c57"},
{file = "mypy-0.780-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a5e5bb12b7982b179af513dddb06fca12285f0316d74f3964078acbfcf4c68f2"},
{file = "mypy-0.780-cp38-cp38-win_amd64.whl", hash = "sha256:5d142f219bf8c7894dfa79ebfb7d352c4c63a325e75f10dfb4c3db9417dcd135"},
{file = "mypy-0.780-py3-none-any.whl", hash = "sha256:127de5a9b817a03a98c5ae8a0c46a20dc44442af6dcfa2ae7f96cb519b312efa"},
{file = "mypy-0.780.tar.gz", hash = "sha256:4ef13b619a289aa025f2273e05e755f8049bb4eaba6d703a425de37d495d178d"},
]
mypy-extensions = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
]
parso = [
{file = "parso-0.7.0-py2.py3-none-any.whl", hash = "sha256:158c140fc04112dc45bca311633ae5033c2c2a7b732fa33d0955bad8152a8dd0"},
{file = "parso-0.7.0.tar.gz", hash = "sha256:908e9fae2144a076d72ae4e25539143d40b8e3eafbaeae03c1bfe226f4cdf12c"},
]
pathspec = [
{file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"},
{file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"},
]
pexpect = [
{file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"},
{file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"},
]
pickleshare = [
{file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"},
{file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"},
]
prompt-toolkit = [
{file = "prompt_toolkit-3.0.5-py3-none-any.whl", hash = "sha256:df7e9e63aea609b1da3a65641ceaf5bc7d05e0a04de5bd45d05dbeffbabf9e04"},
{file = "prompt_toolkit-3.0.5.tar.gz", hash = "sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8"},
]
ptyprocess = [
{file = "ptyprocess-0.6.0-py2.py3-none-any.whl", hash = "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"},
{file = "ptyprocess-0.6.0.tar.gz", hash = "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0"},
]
pycodestyle = [
{file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"},
{file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"},
]
pyflakes = [
{file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"},
{file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"},
]
pygments = [
{file = "Pygments-2.6.1-py3-none-any.whl", hash = "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324"},
{file = "Pygments-2.6.1.tar.gz", hash = "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44"},
]
pypandoc = [
{file = "pypandoc-1.5.tar.gz", hash = "sha256:14a49977ab1fbc9b14ef3087dcb101f336851837fca55ca79cf33846cc4976ff"},
]
python-dateutil = [
{file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
{file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
]
regex = [
{file = "regex-2020.6.8-cp27-cp27m-win32.whl", hash = "sha256:fbff901c54c22425a5b809b914a3bfaf4b9570eee0e5ce8186ac71eb2025191c"},
{file = "regex-2020.6.8-cp27-cp27m-win_amd64.whl", hash = "sha256:112e34adf95e45158c597feea65d06a8124898bdeac975c9087fe71b572bd938"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:92d8a043a4241a710c1cf7593f5577fbb832cf6c3a00ff3fc1ff2052aff5dd89"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bae83f2a56ab30d5353b47f9b2a33e4aac4de9401fb582b55c42b132a8ac3868"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:b2ba0f78b3ef375114856cbdaa30559914d081c416b431f2437f83ce4f8b7f2f"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:95fa7726d073c87141f7bbfb04c284901f8328e2d430eeb71b8ffdd5742a5ded"},
{file = "regex-2020.6.8-cp36-cp36m-win32.whl", hash = "sha256:e3cdc9423808f7e1bb9c2e0bdb1c9dc37b0607b30d646ff6faf0d4e41ee8fee3"},
{file = "regex-2020.6.8-cp36-cp36m-win_amd64.whl", hash = "sha256:c78e66a922de1c95a208e4ec02e2e5cf0bb83a36ceececc10a72841e53fbf2bd"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:08997a37b221a3e27d68ffb601e45abfb0093d39ee770e4257bd2f5115e8cb0a"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2f6f211633ee8d3f7706953e9d3edc7ce63a1d6aad0be5dcee1ece127eea13ae"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:55b4c25cbb3b29f8d5e63aeed27b49fa0f8476b0d4e1b3171d85db891938cc3a"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:89cda1a5d3e33ec9e231ece7307afc101b5217523d55ef4dc7fb2abd6de71ba3"},
{file = "regex-2020.6.8-cp37-cp37m-win32.whl", hash = "sha256:690f858d9a94d903cf5cada62ce069b5d93b313d7d05456dbcd99420856562d9"},
{file = "regex-2020.6.8-cp37-cp37m-win_amd64.whl", hash = "sha256:1700419d8a18c26ff396b3b06ace315b5f2a6e780dad387e4c48717a12a22c29"},
{file = "regex-2020.6.8-cp38-cp38-manylinux1_i686.whl", hash = "sha256:654cb773b2792e50151f0e22be0f2b6e1c3a04c5328ff1d9d59c0398d37ef610"},
{file = "regex-2020.6.8-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:52e1b4bef02f4040b2fd547357a170fc1146e60ab310cdbdd098db86e929b387"},
{file = "regex-2020.6.8-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:cf59bbf282b627130f5ba68b7fa3abdb96372b24b66bdf72a4920e8153fc7910"},
{file = "regex-2020.6.8-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:5aaa5928b039ae440d775acea11d01e42ff26e1561c0ffcd3d805750973c6baf"},
{file = "regex-2020.6.8-cp38-cp38-win32.whl", hash = "sha256:97712e0d0af05febd8ab63d2ef0ab2d0cd9deddf4476f7aa153f76feef4b2754"},
{file = "regex-2020.6.8-cp38-cp38-win_amd64.whl", hash = "sha256:6ad8663c17db4c5ef438141f99e291c4d4edfeaacc0ce28b5bba2b0bf273d9b5"},
{file = "regex-2020.6.8.tar.gz", hash = "sha256:e9b64e609d37438f7d6e68c2546d2cb8062f3adb27e6336bc129b51be20773ac"},
]
requests = [
{file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"},
{file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"},
]
s3transfer = [
{file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"},
{file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"},
]
six = [
{file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
{file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
]
toml = [
{file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"},
{file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"},
]
tqdm = [
{file = "tqdm-4.46.1-py2.py3-none-any.whl", hash = "sha256:07c06493f1403c1380b630ae3dcbe5ae62abcf369a93bbc052502279f189ab8c"},
{file = "tqdm-4.46.1.tar.gz", hash = "sha256:cd140979c2bebd2311dfb14781d8f19bd5a9debb92dcab9f6ef899c987fcf71f"},
]
traitlets = [
{file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"},
{file = "traitlets-4.3.3.tar.gz", hash = "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7"},
]
typed-ast = [
{file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"},
{file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb"},
{file = "typed_ast-1.4.1-cp35-cp35m-win32.whl", hash = "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919"},
{file = "typed_ast-1.4.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01"},
{file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"},
{file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"},
{file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"},
{file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"},
{file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"},
{file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"},
{file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"},
{file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"},
{file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"},
{file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"},
{file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"},
{file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"},
{file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"},
{file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"},
{file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"},
{file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"},
{file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"},
]
typing-extensions = [
{file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"},
{file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"},
{file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"},
]
urllib3 = [
{file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"},
{file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"},
]
wcwidth = [
{file = "wcwidth-0.2.4-py2.py3-none-any.whl", hash = "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f"},
{file = "wcwidth-0.2.4.tar.gz", hash = "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f"},
]
wheel = [
{file = "wheel-0.34.2-py2.py3-none-any.whl", hash = "sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e"},
{file = "wheel-0.34.2.tar.gz", hash = "sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96"},
]

View file

@ -0,0 +1,31 @@
[tool.poetry]
name = "import-scripts"
version = "0.1.0"
description = ""
authors = ["Rok Garbas <rok@garbas.si>"]
include = [
"import_scripts/packages-config.nix"
]
[tool.poetry.scripts]
import-channel = 'import_scripts.channel:run'
[tool.poetry.dependencies]
python = "^3.8"
requests = "^2.24.0"
click = "^7.1.2"
click-log = "^0.3.2"
elasticsearch = "^7.8.0"
boto3 = "^1.14.5"
tqdm = "^4.46.1"
pypandoc = "^1.5"
[tool.poetry.dev-dependencies]
ipdb = "^0.13.2"
black = "^19.10b0"
flake8 = "^3.8.3"
mypy = "^0.780"
[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"

View file

@ -6,7 +6,7 @@ const {Elm} = require('./Main');
Elm.Main.init({ Elm.Main.init({
flags: { flags: {
elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 6, elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 0,
elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443', elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443',
elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR', elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR',
elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG' elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG'