This commit is contained in:
Rok Garbas 2020-06-19 00:41:01 +02:00 committed by GitHub
parent c420d05815
commit bbc97d17f0
Failed to generate hash of commit
13 changed files with 1086 additions and 146 deletions

View file

@ -17,10 +17,23 @@ jobs:
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Installing Nix
uses: cachix/install-nix-action@v8
- name: Installing NixFlakes
run: |
nix-env -iA nixpkgs.nixFlakes
echo "::add-path::$HOME/.nix-profile/bin"
echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
nix --version
cat /etc/nix/nix.conf
- name: Building import_scripts
run: |
nix build ./#packages.x86_64-linux.import_scripts
- name: Import ${{ matrix.channel }} channel
run: |
cd scripts/
./import-channel --es-url ${{ secrets.ELASTICSEARCH_URL }} --channel ${{ matrix.channel }} -vvv
./result/bin/import-channel --es-url ${{ secrets.ELASTICSEARCH_URL }} --channel ${{ matrix.channel }} -vvv
if: github.repository == 'NixOS/nixos-search'

View file

@ -12,13 +12,28 @@ jobs:
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Installing Nix
uses: cachix/install-nix-action@v8
uses: cachix/install-nix-action@v9
- name: Installing NixFlakes
run: |
nix-env -iA nixpkgs.nixFlakes
echo "::add-path::$HOME/.nix-profile/bin"
echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
nix --version
cat /etc/nix/nix.conf
- name: Building import_scripts
run: |
nix build ./#packages.x86_64-linux.import_scripts
- name: Building search.nixos.org
run: |
nix-build
nix build ./#packages.x86_64-linux.frontend
mkdir ./dist
cp -RL ./result/* ./dist/
- name: Deploy to Netlify
uses: nwtgck/actions-netlify@v1.0.5
env:

1
VERSION Normal file
View file

@ -0,0 +1 @@
7

View file

@ -1,15 +1,9 @@
{ pkgs ? import <nixpkgs> {}
{ pkgs ? import <nixpkgs> { }
, version ? "0"
}:
let
package = builtins.fromJSON (builtins.readFile ./package.json);
yarn2nix = import (pkgs.fetchFromGitHub {
owner = "moretea";
repo = "yarn2nix";
rev = "9e7279edde2a4e0f5ec04c53f5cd64440a27a1ae";
sha256 = "sha256-x77mYSvwA4bd/uCwL0rRw+8mwH+dR/UfWnn4YXmm4n8=";
}) { inherit pkgs; };
yarnPkg = yarn2nix.mkYarnPackage rec {
yarnPkg = pkgs.yarn2nix-moretea.mkYarnPackage rec {
name = "${package.name}-yarn-${package.version}";
src = null;
dontUnpack = true;
@ -28,7 +22,7 @@ let
'';
pkgConfig = {
node-sass = {
buildInputs = [ pkgs.python pkgs.libsass pkgs.pkgconfig];
buildInputs = [ pkgs.python pkgs.libsass pkgs.pkgconfig ];
postInstall = ''
LIBSASS_EXT=auto yarn --offline run build
rm build/config.gypi
@ -36,16 +30,19 @@ let
};
};
publishBinsFor =
[ "webpack"
[
"webpack"
"webpack-dev-server"
];
};
in pkgs.stdenv.mkDerivation {
in
pkgs.stdenv.mkDerivation {
name = "${package.name}-${package.version}";
src = pkgs.lib.cleanSource ./.;
buildInputs =
[ yarnPkg
[
yarnPkg
] ++
(with pkgs; [
nodejs
@ -76,6 +73,7 @@ in pkgs.stdenv.mkDerivation {
buildPhase = ''
# Yarn writes cache directories etc to $HOME.
export HOME=$PWD/yarn_home
sed -i -e "s|process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION|${version}|" src/index.js
yarn prod
'';
@ -88,6 +86,7 @@ in pkgs.stdenv.mkDerivation {
rm -rf node_modules
ln -sf ${yarnPkg}/libexec/${package.name}/node_modules .
export PATH=$PWD/node_modules/.bin:$PATH
export ELASTICSEARCH_MAPPING_SCHEMA_VERSION=${version}
echo "============================"
echo "= To develop run: yarn dev ="
echo "============================"

65
flake.lock Normal file
View file

@ -0,0 +1,65 @@
{
"nodes": {
"nixpkgs": {
"info": {
"lastModified": 1592376843,
"narHash": "sha256-vKchUGsNMQC9vCmTnpssHMV/+cKXA5QhbEoFm0zua1U="
},
"locked": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "14fcd947a0150b78d7a37bef56c87fcd57808247",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-unstable",
"type": "indirect"
}
},
"nixpkgs_2": {
"info": {
"lastModified": 1592514044,
"narHash": "sha256-jGQJmq14vh9ASt9d5Z9yBPsrbUgroiNVNhsoBabsw0k="
},
"locked": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "e051dab9ff8a58455f6b8375856ac2802c6cd261",
"type": "github"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"poetry2nix": {
"info": {
"lastModified": 1592121810,
"narHash": "sha256-9cGTKtL7JpllwABxa4iRVU4Qw/YwniLni4RATyONz+s="
},
"inputs": {
"nixpkgs": "nixpkgs_2"
},
"locked": {
"owner": "nix-community",
"repo": "poetry2nix",
"rev": "e7c69a288c10e4d97816fdabda5ae3f38e21914e",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "poetry2nix",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"poetry2nix": "poetry2nix"
}
}
},
"root": "root",
"version": 5
}

35
flake.nix Normal file
View file

@ -0,0 +1,35 @@
{
description = "Code behind search.nixos.org";
inputs = {
nixpkgs = { url = "nixpkgs/nixos-unstable"; };
poetry2nix = { url = "github:nix-community/poetry2nix"; };
};
outputs = { self, nixpkgs, poetry2nix }:
let
systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
allPackages = system:
let
pkgs = import nixpkgs {
inherit system;
overlays = [
poetry2nix.overlay
];
};
version = pkgs.lib.removeSuffix "\n" (builtins.readFile "${self}/VERSION");
in
{
import_scripts = import ./import-scripts {
inherit pkgs version;
};
frontend = import ./. {
inherit pkgs version;
};
};
in
{
packages = forAllSystems allPackages;
};
}

View file

@ -0,0 +1,23 @@
{ pkgs ? import <nixpkgs> { }
, version ? "0"
}:
let
inherit (pkgs.poetry2nix) mkPoetryApplication overrides;
in
mkPoetryApplication {
projectDir = ./.;
overrides = overrides.withDefaults (self: super: {
pypandoc = super.pypandoc.overridePythonAttrs (old: {
postPatch = ''
sed -i '/^__pandoc_path = None$/c__pandoc_path = "${pkgs.pandoc}/bin/pandoc"' pypandoc/__init__.py
'';
});
});
checkPhase = ''
black --diff --check ./import_scripts
flake8 --ignore W503,E501,E265,E203 ./import_scripts
'';
postInstall = ''
wrapProgram $out/bin/import-channel --set INDEX_SCHEMA_VERSION "${version}"
'';
}

View file

@ -1,15 +1,3 @@
#! /usr/bin/env nix-shell
#! nix-shell -i python3 -p python3 python3Packages.requests python3Packages.click python3Packages.click-log python3Packages.elasticsearch python3Packages.boto3 python3Packages.tqdm python3Packages.pypandoc
# develop:
# $ nix-shell -p python3Packages.black python3Packages.mypy python3Packages.flake8
#
# format:
# $ nix-shell -p python3Packages.black --command "black import-channel"
#
# lint:
# $ nix-shell -p python3Packages.flake8 --command "flake8 --ignore E501,E265 import-channel"
import boto3
import botocore
import botocore.client
@ -19,12 +7,14 @@ import elasticsearch
import elasticsearch.helpers
import json
import logging
import os
import os.path
import pypandoc
import re
import requests
import shlex
import subprocess
import sys
import tqdm
import xml.etree.ElementTree
@ -34,7 +24,7 @@ click_log.basic_config(logger)
S3_BUCKET = "nix-releases"
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
INDEX_SCHEMA_VERSION = 6
INDEX_SCHEMA_VERSION = os.environ.get("INDEX_SCHEMA_VERSION", 0)
CHANNELS = {
"unstable": {
"packages": "nixpkgs/nixpkgs-20.09pre",
@ -51,11 +41,7 @@ CHANNELS = {
}
ANALYSIS = {
"normalizer": {
"lowercase": {
"type": "custom",
"char_filter": [],
"filter": ["lowercase"],
}
"lowercase": {"type": "custom", "char_filter": [], "filter": ["lowercase"]}
},
"analyzer": {
"lowercase": {
@ -75,14 +61,7 @@ ANALYSIS = {
},
},
"tokenizer": {
"nix_package_query": {
"type": "pattern",
"pattern": "|".join(
[
"[ ]",
]
),
},
"nix_package_query": {"type": "pattern", "pattern": "|".join(["[ ]"])},
"nix_package_attr_name": {
"type": "pattern",
# Split on attrname separators like _, .
@ -107,10 +86,7 @@ ANALYSIS = {
]
),
},
"nix_option_name": {
"type": "pattern",
"pattern": "[.]",
},
"nix_option_name": {"type": "pattern", "pattern": "[.]"},
# Lower priority (virtualHost -> [virtual, host])
"nix_option_name_granular": {
"type": "pattern",
@ -169,28 +145,16 @@ MAPPING = {
"type": "nested",
"properties": {
"output": {"type": "keyword"},
"path": {"type": "keyword"}
"path": {"type": "keyword"},
},
},
"drv_path": {"type": "keyword"},
},
},
"package_attr_name": {
"type": "keyword",
"normalizer": "lowercase",
},
"package_attr_name_query": {
"type": "keyword",
"normalizer": "lowercase",
},
"package_attr_set": {
"type": "keyword",
"normalizer": "lowercase",
},
"package_pname": {
"type": "keyword",
"normalizer": "lowercase",
},
"package_attr_name": {"type": "keyword", "normalizer": "lowercase"},
"package_attr_name_query": {"type": "keyword", "normalizer": "lowercase"},
"package_attr_set": {"type": "keyword", "normalizer": "lowercase"},
"package_pname": {"type": "keyword", "normalizer": "lowercase"},
"package_pversion": {"type": "keyword"},
"package_description": {"type": "text"},
"package_longDescription": {"type": "text"},
@ -216,13 +180,8 @@ MAPPING = {
"analyzer": "nixOptionName",
"fielddata": True,
"fields": {
"raw": {
"type": "keyword"
},
"granular": {
"type": "text",
"analyzer": "nixOptionNameGranular",
},
"raw": {"type": "keyword"},
"granular": {"type": "text", "analyzer": "nixOptionNameGranular"},
},
},
"option_description": {"type": "text"},
@ -257,7 +216,9 @@ def split_query(text):
- test2
"""
tokens = []
regex = re.compile(".+?(?:(?<=[a-z])(?=[1-9A-Z])|(?<=[1-9A-Z])(?=[A-Z][a-z])|[\._-]|$)")
regex = re.compile(
".+?(?:(?<=[a-z])(?=[1-9A-Z])|(?<=[1-9A-Z])(?=[A-Z][a-z])|[._-]|$)"
)
parts = [m.group(0) for m in regex.finditer(text)]
for index in range(len(parts)):
prev_parts = ""
@ -279,14 +240,13 @@ def get_last_evaluation(prefix):
if not item:
continue
logger.debug(f"get_last_evaluation: evaluation in raw {item}")
try:
revisions_since_start, git_revision = item['Prefix'][len(prefix):].rstrip('/').split('.')
except:
__import__('pdb').set_trace()
revisions_since_start, git_revision = (
item["Prefix"][len(prefix) :].rstrip("/").split(".")
)
evaluation = {
"revisions_since_start": int(revisions_since_start),
"git_revision": git_revision,
"prefix": item['Prefix'].rstrip('/'),
"prefix": item["Prefix"].rstrip("/"),
}
logger.debug(f"get_last_evaluation: evaluation {evaluation}")
evaluations.append(evaluation)
@ -299,7 +259,9 @@ def get_last_evaluation(prefix):
evaluation = evaluations[-1]
result = s3.get_object(Bucket=S3_BUCKET, Key=f"{evaluation['prefix']}/src-url")
evaluation['id'] = result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/"):]
evaluation["id"] = (
result.get("Body").read().decode()[len("https://hydra.nixos.org/eval/") :]
)
logger.debug(f"get_last_evaluation: last evaluation is: {evaluation}")
@ -307,21 +269,21 @@ def get_last_evaluation(prefix):
def get_evaluation_builds(evaluation_id):
logger.debug(f"get_evaluation_builds: Retriving list of builds for {evaluation_id} evaluation id")
logger.debug(
f"get_evaluation_builds: Retriving list of builds for {evaluation_id} evaluation id"
)
filename = f"eval-{evaluation_id}.json"
if not os.path.exists(filename):
url = f"https://hydra.nixos.org/eval/{evaluation_id}/builds"
logger.debug(f"get_evaluation_builds: Fetching builds from {url} url.")
headers = {
"Content-Type": "application/json"
}
headers = {"Content-Type": "application/json"}
r = requests.get(url, headers=headers, stream=True)
with tqdm.tqdm.wrapattr(
open(filename, "wb"),
"write",
miniters=1,
total=int(r.headers.get('content-length', 0)),
desc=filename
total=int(r.headers.get("content-length", 0)),
desc=filename,
) as f:
for chunk in r.iter_content(chunk_size=4096):
f.write(chunk)
@ -331,8 +293,8 @@ def get_evaluation_builds(evaluation_id):
result = {}
for build in builds:
result.setdefault(build['nixname'], {})
result[build['nixname']][build['system']] = build
result.setdefault(build["nixname"], {})
result[build["nixname"]][build["system"]] = build
return result
@ -341,18 +303,16 @@ def get_maintainer(maintainer):
maintainers = []
if type(maintainer) == str:
maintainers.append(dict(
name=maintainer,
email=None,
github=None,
))
maintainers.append(dict(name=maintainer, email=None, github=None,))
elif type(maintainer) == dict:
maintainers.append(dict(
maintainers.append(
dict(
name=maintainer.get("name"),
email=maintainer.get("email"),
github=maintainer.get("github"),
))
)
)
elif type(maintainer) == list:
for item in maintainer:
@ -380,8 +340,7 @@ def remove_attr_set(name):
# Plugins
"elasticsearch",
"graylog",
"tmuxplugin"
"vimplugin"
"tmuxplugin" "vimplugin",
]
# TODO: is this correct
if any([name.startswith(i) for i in sets]):
@ -389,7 +348,7 @@ def remove_attr_set(name):
# node does things a bit different
elif name.startswith("node_"):
name = name[len("node_"):]
name = name[len("node_") :]
return name
@ -440,31 +399,32 @@ def get_packages(evaluation, evaluation_builds):
attr_set = None
if "." in attr_name:
attr_set = attr_name.split(".")[0]
if not attr_set.endswith("Packages") and \
not attr_set.endswith("Plugins") and \
not attr_set.endswith("Extensions"):
if (
not attr_set.endswith("Packages")
and not attr_set.endswith("Plugins")
and not attr_set.endswith("Extensions")
):
attr_set = None
hydra = None
if data['name'] in evaluation_builds:
if data["name"] in evaluation_builds:
hydra = []
for platform, build in evaluation_builds[data['name']].items():
hydra.append({
"build_id": build['id'],
"build_status": build['buildstatus'],
"platform": build['system'],
"project": build['project'],
"jobset": build['jobset'],
"job": build['job'],
"path": [
for platform, build in evaluation_builds[data["name"]].items():
hydra.append(
{
"output": output,
"path": item['path'],
}
for output, item in build['buildoutputs'].items()
"build_id": build["id"],
"build_status": build["buildstatus"],
"platform": build["system"],
"project": build["project"],
"jobset": build["jobset"],
"job": build["job"],
"path": [
{"output": output, "path": item["path"]}
for output, item in build["buildoutputs"].items()
],
"drv_path": build['drvpath'],
})
"drv_path": build["drvpath"],
}
)
yield dict(
type="package",
@ -512,7 +472,7 @@ def get_options(evaluation):
example = option.get("example")
if example is not None:
if (type(example) == dict and example.get("_type") == "literalExample"):
if type(example) == dict and example.get("_type") == "literalExample":
example = json.dumps(example["text"])
else:
example = json.dumps(example)
@ -520,18 +480,16 @@ def get_options(evaluation):
description = option.get("description")
if description is not None:
xml_description = (
f"<xml xmlns:xlink=\"http://www.w3.org/1999/xlink\">"
f'<xml xmlns:xlink="http://www.w3.org/1999/xlink">'
f"<para>{description}</para>"
f"</xml>"
)
# we first check if there are some xml elements before using pypandoc
# since pypandoc calls are quite slow
root = xml.etree.ElementTree.fromstring(xml_description)
if len(list(root.find('para'))) > 0:
if len(list(root.find("para"))) > 0:
description = pypandoc.convert_text(
xml_description,
"html",
format="docbook",
xml_description, "html", format="docbook",
)
yield dict(
@ -569,14 +527,16 @@ def ensure_index(es, index, mapping, force=False):
def create_index_name(channel, evaluation_packages, evaluation_options):
evaluation_name = '-'.join([
evaluation_packages['id'],
str(evaluation_packages['revisions_since_start']),
evaluation_packages['git_revision'],
evaluation_options['id'],
str(evaluation_options['revisions_since_start']),
evaluation_options['git_revision'],
])
evaluation_name = "-".join(
[
evaluation_packages["id"],
str(evaluation_packages["revisions_since_start"]),
evaluation_packages["git_revision"],
evaluation_options["id"],
str(evaluation_options["revisions_since_start"]),
evaluation_options["git_revision"],
]
)
return (
f"latest-{INDEX_SCHEMA_VERSION}-{channel}",
f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation_name}",
@ -624,7 +584,7 @@ def write(unit, es, index_name, number_of_items, item_generator):
@click.option("-c", "--channel", type=click.Choice(CHANNELS.keys()), help="Channel.")
@click.option("-f", "--force", is_flag=True, help="Force channel recreation.")
@click.option("-v", "--verbose", count=True)
def main(es_url, channel, force, verbose):
def run(es_url, channel, force, verbose):
logging_level = "CRITICAL"
if verbose == 1:
@ -636,23 +596,30 @@ def main(es_url, channel, force, verbose):
logger.debug(f"Verbosity is {verbose}")
logger.debug(f"Logging set to {logging_level}")
evaluation_packages = get_last_evaluation(CHANNELS[channel]['packages'])
evaluation_options = get_last_evaluation(CHANNELS[channel]['options'])
evaluation_packages_builds = get_evaluation_builds(evaluation_packages['id'])
evaluation_packages = get_last_evaluation(CHANNELS[channel]["packages"])
evaluation_options = get_last_evaluation(CHANNELS[channel]["options"])
evaluation_packages_builds = get_evaluation_builds(evaluation_packages["id"])
es = elasticsearch.Elasticsearch([es_url])
alias_name, index_name = create_index_name(channel, evaluation_packages, evaluation_options)
alias_name, index_name = create_index_name(
channel, evaluation_packages, evaluation_options
)
index_created = ensure_index(es, index_name, MAPPING, force)
if index_created:
write("packages", es, index_name, *get_packages(evaluation_packages, evaluation_packages_builds))
write(
"packages",
es,
index_name,
*get_packages(evaluation_packages, evaluation_packages_builds),
)
write("options", es, index_name, *get_options(evaluation_options))
update_alias(es, alias_name, index_name)
if __name__ == "__main__":
main()
run()
# vi:ft=python

791
import-scripts/poetry.lock generated Normal file
View file

@ -0,0 +1,791 @@
[[package]]
category = "dev"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
name = "appdirs"
optional = false
python-versions = "*"
version = "1.4.4"
[[package]]
category = "dev"
description = "Disable App Nap on OS X 10.9"
marker = "python_version >= \"3.4\" and sys_platform == \"darwin\""
name = "appnope"
optional = false
python-versions = "*"
version = "0.1.0"
[[package]]
category = "dev"
description = "Classes Without Boilerplate"
name = "attrs"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "19.3.0"
[package.extras]
azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"]
dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"]
docs = ["sphinx", "zope.interface"]
tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
[[package]]
category = "dev"
description = "Specifications for callback functions passed in to an API"
marker = "python_version >= \"3.4\""
name = "backcall"
optional = false
python-versions = "*"
version = "0.2.0"
[[package]]
category = "dev"
description = "The uncompromising code formatter."
name = "black"
optional = false
python-versions = ">=3.6"
version = "19.10b0"
[package.dependencies]
appdirs = "*"
attrs = ">=18.1.0"
click = ">=6.5"
pathspec = ">=0.6,<1"
regex = "*"
toml = ">=0.9.4"
typed-ast = ">=1.4.0"
[package.extras]
d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
[[package]]
category = "main"
description = "The AWS SDK for Python"
name = "boto3"
optional = false
python-versions = "*"
version = "1.14.5"
[package.dependencies]
botocore = ">=1.17.5,<1.18.0"
jmespath = ">=0.7.1,<1.0.0"
s3transfer = ">=0.3.0,<0.4.0"
[[package]]
category = "main"
description = "Low-level, data-driven core of boto 3."
name = "botocore"
optional = false
python-versions = "*"
version = "1.17.5"
[package.dependencies]
docutils = ">=0.10,<0.16"
jmespath = ">=0.7.1,<1.0.0"
python-dateutil = ">=2.1,<3.0.0"
[package.dependencies.urllib3]
python = "<3.4.0 || >=3.5.0"
version = ">=1.20,<1.26"
[[package]]
category = "main"
description = "Python package for providing Mozilla's CA Bundle."
name = "certifi"
optional = false
python-versions = "*"
version = "2020.4.5.2"
[[package]]
category = "main"
description = "Universal encoding detector for Python 2 and 3"
name = "chardet"
optional = false
python-versions = "*"
version = "3.0.4"
[[package]]
category = "main"
description = "Composable command line interface toolkit"
name = "click"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "7.1.2"
[[package]]
category = "main"
description = "Logging integration for Click"
name = "click-log"
optional = false
python-versions = "*"
version = "0.3.2"
[package.dependencies]
click = "*"
[[package]]
category = "dev"
description = "Cross-platform colored terminal text."
marker = "python_version >= \"3.4\" and sys_platform == \"win32\""
name = "colorama"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.4.3"
[[package]]
category = "dev"
description = "Decorators for Humans"
marker = "python_version >= \"3.4\""
name = "decorator"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*"
version = "4.4.2"
[[package]]
category = "main"
description = "Docutils -- Python Documentation Utilities"
name = "docutils"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
version = "0.15.2"
[[package]]
category = "main"
description = "Python client for Elasticsearch"
name = "elasticsearch"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4"
version = "7.8.0"
[package.dependencies]
certifi = "*"
urllib3 = ">=1.21.1"
[package.extras]
async = ["aiohttp (>=3,<4)", "yarl"]
develop = ["requests (>=2.0.0,<3.0.0)", "coverage", "mock", "pyyaml", "pytest", "pytest-cov", "sphinx (<1.7)", "sphinx-rtd-theme", "black", "jinja2"]
docs = ["sphinx (<1.7)", "sphinx-rtd-theme"]
requests = ["requests (>=2.4.0,<3.0.0)"]
[[package]]
category = "dev"
description = "the modular source code checker: pep8 pyflakes and co"
name = "flake8"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
version = "3.8.3"
[package.dependencies]
mccabe = ">=0.6.0,<0.7.0"
pycodestyle = ">=2.6.0a1,<2.7.0"
pyflakes = ">=2.2.0,<2.3.0"
[[package]]
category = "main"
description = "Internationalized Domain Names in Applications (IDNA)"
name = "idna"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.9"
[[package]]
category = "dev"
description = "IPython-enabled pdb"
name = "ipdb"
optional = false
python-versions = ">=2.7"
version = "0.13.2"
[package.dependencies]
setuptools = "*"
[package.dependencies.ipython]
python = ">=3.4"
version = ">=5.1.0"
[[package]]
category = "dev"
description = "IPython: Productive Interactive Computing"
marker = "python_version >= \"3.4\""
name = "ipython"
optional = false
python-versions = ">=3.6"
version = "7.15.0"
[package.dependencies]
appnope = "*"
backcall = "*"
colorama = "*"
decorator = "*"
jedi = ">=0.10"
pexpect = "*"
pickleshare = "*"
prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0"
pygments = "*"
setuptools = ">=18.5"
traitlets = ">=4.2"
[package.extras]
all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.14)", "pygments", "qtconsole", "requests", "testpath"]
doc = ["Sphinx (>=1.3)"]
kernel = ["ipykernel"]
nbconvert = ["nbconvert"]
nbformat = ["nbformat"]
notebook = ["notebook", "ipywidgets"]
parallel = ["ipyparallel"]
qtconsole = ["qtconsole"]
test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.14)"]
[[package]]
category = "dev"
description = "Vestigial utilities from IPython"
marker = "python_version >= \"3.4\""
name = "ipython-genutils"
optional = false
python-versions = "*"
version = "0.2.0"
[[package]]
category = "dev"
description = "An autocompletion tool for Python that can be used for text editors."
marker = "python_version >= \"3.4\""
name = "jedi"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.17.0"
[package.dependencies]
parso = ">=0.7.0"
[package.extras]
qa = ["flake8 (3.7.9)"]
testing = ["colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"]
[[package]]
category = "main"
description = "JSON Matching Expressions"
name = "jmespath"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
version = "0.10.0"
[[package]]
category = "dev"
description = "McCabe checker, plugin for flake8"
name = "mccabe"
optional = false
python-versions = "*"
version = "0.6.1"
[[package]]
category = "dev"
description = "Optional static typing for Python"
name = "mypy"
optional = false
python-versions = ">=3.5"
version = "0.780"
[package.dependencies]
mypy-extensions = ">=0.4.3,<0.5.0"
typed-ast = ">=1.4.0,<1.5.0"
typing-extensions = ">=3.7.4"
[package.extras]
dmypy = ["psutil (>=4.0)"]
[[package]]
category = "dev"
description = "Experimental type system extensions for programs checked with the mypy typechecker."
name = "mypy-extensions"
optional = false
python-versions = "*"
version = "0.4.3"
[[package]]
category = "dev"
description = "A Python Parser"
marker = "python_version >= \"3.4\""
name = "parso"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.7.0"
[package.extras]
testing = ["docopt", "pytest (>=3.0.7)"]
[[package]]
category = "dev"
description = "Utility library for gitignore style pattern matching of file paths."
name = "pathspec"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.8.0"
[[package]]
category = "dev"
description = "Pexpect allows easy control of interactive console applications."
marker = "python_version >= \"3.4\" and sys_platform != \"win32\""
name = "pexpect"
optional = false
python-versions = "*"
version = "4.8.0"
[package.dependencies]
ptyprocess = ">=0.5"
[[package]]
category = "dev"
description = "Tiny 'shelve'-like database with concurrency support"
marker = "python_version >= \"3.4\""
name = "pickleshare"
optional = false
python-versions = "*"
version = "0.7.5"
[[package]]
category = "dev"
description = "Library for building powerful interactive command lines in Python"
marker = "python_version >= \"3.4\""
name = "prompt-toolkit"
optional = false
python-versions = ">=3.6.1"
version = "3.0.5"
[package.dependencies]
wcwidth = "*"
[[package]]
category = "dev"
description = "Run a subprocess in a pseudo terminal"
marker = "python_version >= \"3.4\" and sys_platform != \"win32\""
name = "ptyprocess"
optional = false
python-versions = "*"
version = "0.6.0"
[[package]]
category = "dev"
description = "Python style guide checker"
name = "pycodestyle"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.6.0"
[[package]]
category = "dev"
description = "passive checker of Python programs"
name = "pyflakes"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.2.0"
[[package]]
category = "dev"
description = "Pygments is a syntax highlighting package written in Python."
marker = "python_version >= \"3.4\""
name = "pygments"
optional = false
python-versions = ">=3.5"
version = "2.6.1"
[[package]]
category = "main"
description = "Thin wrapper for pandoc."
name = "pypandoc"
optional = false
python-versions = "*"
version = "1.5"
[package.dependencies]
pip = ">=8.1.0"
setuptools = "*"
wheel = ">=0.25.0"
[[package]]
category = "main"
description = "Extensions to the standard Python datetime module"
name = "python-dateutil"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
version = "2.8.1"
[package.dependencies]
six = ">=1.5"
[[package]]
category = "dev"
description = "Alternative regular expression module, to replace re."
name = "regex"
optional = false
python-versions = "*"
version = "2020.6.8"
[[package]]
category = "main"
description = "Python HTTP for Humans."
name = "requests"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "2.24.0"
[package.dependencies]
certifi = ">=2017.4.17"
chardet = ">=3.0.2,<4"
idna = ">=2.5,<3"
urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26"
[package.extras]
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
[[package]]
category = "main"
description = "An Amazon S3 Transfer Manager"
name = "s3transfer"
optional = false
python-versions = "*"
version = "0.3.3"
[package.dependencies]
botocore = ">=1.12.36,<2.0a.0"
[[package]]
category = "main"
description = "Python 2 and 3 compatibility utilities"
name = "six"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
version = "1.15.0"
[[package]]
category = "dev"
description = "Python Library for Tom's Obvious, Minimal Language"
name = "toml"
optional = false
python-versions = "*"
version = "0.10.1"
[[package]]
category = "main"
description = "Fast, Extensible Progress Meter"
name = "tqdm"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*"
version = "4.46.1"
[package.extras]
dev = ["py-make (>=0.1.0)", "twine", "argopt", "pydoc-markdown"]
[[package]]
category = "dev"
description = "Traitlets Python config system"
marker = "python_version >= \"3.4\""
name = "traitlets"
optional = false
python-versions = "*"
version = "4.3.3"
[package.dependencies]
decorator = "*"
ipython-genutils = "*"
six = "*"
[package.extras]
test = ["pytest", "mock"]
[[package]]
category = "dev"
description = "a fork of Python 2 and 3 ast modules with type comment support"
name = "typed-ast"
optional = false
python-versions = "*"
version = "1.4.1"
[[package]]
category = "dev"
description = "Backported and Experimental Type Hints for Python 3.5+"
name = "typing-extensions"
optional = false
python-versions = "*"
version = "3.7.4.2"
[[package]]
category = "main"
description = "HTTP library with thread-safe connection pooling, file post, and more."
name = "urllib3"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
version = "1.25.9"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
[[package]]
category = "dev"
description = "Measures the displayed width of unicode strings in a terminal"
marker = "python_version >= \"3.4\""
name = "wcwidth"
optional = false
python-versions = "*"
version = "0.2.4"
[[package]]
category = "main"
description = "A built-package format for Python"
name = "wheel"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.34.2"
[package.extras]
test = ["pytest (>=3.0.0)", "pytest-cov"]
[metadata]
content-hash = "b75343832402daab2b4e4c515ff32a0acbe74e1b1f22703e58060a14ea231904"
python-versions = "^3.8"
[metadata.files]
appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
]
appnope = [
{file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"},
{file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"},
]
attrs = [
{file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"},
{file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"},
]
backcall = [
{file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
{file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
]
black = [
{file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"},
{file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"},
]
boto3 = [
{file = "boto3-1.14.5-py2.py3-none-any.whl", hash = "sha256:ed9b0852c0dc8ca88f5f851357acf238b954d1cced44f48e0d930d306150de7c"},
{file = "boto3-1.14.5.tar.gz", hash = "sha256:7098937f6432e3ae4e5fe9b93f561b06117c5df736effb8cc166f6fb2bb41ab8"},
]
botocore = [
{file = "botocore-1.17.5-py2.py3-none-any.whl", hash = "sha256:072c82c64906996f1d7953da1a61d8e6debf0ee5acaa267ec777f05b30755b66"},
{file = "botocore-1.17.5.tar.gz", hash = "sha256:fcfc3762472aa1d758583d818faaa59b933d839a87f372688836d49d66ad9a7a"},
]
certifi = [
{file = "certifi-2020.4.5.2-py2.py3-none-any.whl", hash = "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"},
{file = "certifi-2020.4.5.2.tar.gz", hash = "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1"},
]
chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
{file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
]
click = [
{file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
{file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
]
click-log = [
{file = "click-log-0.3.2.tar.gz", hash = "sha256:16fd1ca3fc6b16c98cea63acf1ab474ea8e676849dc669d86afafb0ed7003124"},
{file = "click_log-0.3.2-py2.py3-none-any.whl", hash = "sha256:eee14dc37cdf3072158570f00406572f9e03e414accdccfccd4c538df9ae322c"},
]
colorama = [
{file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
{file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"},
]
decorator = [
{file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"},
{file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"},
]
docutils = [
{file = "docutils-0.15.2-py2-none-any.whl", hash = "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827"},
{file = "docutils-0.15.2-py3-none-any.whl", hash = "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0"},
{file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"},
]
elasticsearch = [
{file = "elasticsearch-7.8.0-py2.py3-none-any.whl", hash = "sha256:6fb566dd23b91b5871ce12212888674b4cf33374e92b71b1080916c931e44dcb"},
{file = "elasticsearch-7.8.0.tar.gz", hash = "sha256:e637d8cf4e27e279b5ff8ca8edc0c086f4b5df4bf2b48e2f950b7833aca3a792"},
]
flake8 = [
{file = "flake8-3.8.3-py2.py3-none-any.whl", hash = "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c"},
{file = "flake8-3.8.3.tar.gz", hash = "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208"},
]
idna = [
{file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"},
{file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"},
]
ipdb = [
{file = "ipdb-0.13.2.tar.gz", hash = "sha256:77fb1c2a6fccdfee0136078c9ed6fe547ab00db00bebff181f1e8c9e13418d49"},
]
ipython = [
{file = "ipython-7.15.0-py3-none-any.whl", hash = "sha256:1b85d65632211bf5d3e6f1406f3393c8c429a47d7b947b9a87812aa5bce6595c"},
{file = "ipython-7.15.0.tar.gz", hash = "sha256:0ef1433879816a960cd3ae1ae1dc82c64732ca75cec8dab5a4e29783fb571d0e"},
]
ipython-genutils = [
{file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"},
{file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"},
]
jedi = [
{file = "jedi-0.17.0-py2.py3-none-any.whl", hash = "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798"},
{file = "jedi-0.17.0.tar.gz", hash = "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030"},
]
jmespath = [
{file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"},
{file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"},
]
mccabe = [
{file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
{file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
]
mypy = [
{file = "mypy-0.780-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:d3b4941de44341227ece1caaf5b08b23e42ad4eeb8b603219afb11e9d4cfb437"},
{file = "mypy-0.780-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:1f3976a945ad7f0a0727aafdc5651c2d3278e3c88dee94e2bf75cd3386b7b2f4"},
{file = "mypy-0.780-cp35-cp35m-win_amd64.whl", hash = "sha256:eadb865126da4e3c4c95bdb47fe1bb087a3e3ea14d39a3b13224b8a4d9f9a102"},
{file = "mypy-0.780-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:75eed74d2faf2759f79c5f56f17388defd2fc994222312ec54ee921e37b31ad4"},
{file = "mypy-0.780-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2f8c098f12b402c19b735aec724cc9105cc1a9eea405d08814eb4b14a6fb1a41"},
{file = "mypy-0.780-cp36-cp36m-win_amd64.whl", hash = "sha256:62eb5dd4ea86bda8ce386f26684f7f26e4bfe6283c9f2b6ca6d17faf704dcfad"},
{file = "mypy-0.780-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:00cb1964a7476e871d6108341ac9c1a857d6bd20bf5877f4773ac5e9d92cd3cd"},
{file = "mypy-0.780-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:974bebe3699b9b46278a7f076635d219183da26e1a675c1f8243a69221758273"},
{file = "mypy-0.780-cp37-cp37m-win_amd64.whl", hash = "sha256:64c36eb0936d0bfb7d8da49f92c18e312ad2e3ed46e5548ae4ca997b0d33bd59"},
{file = "mypy-0.780-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d31291df31bafb997952dc0a17ebb2737f802c754aed31dd155a8bfe75112c57"},
{file = "mypy-0.780-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a5e5bb12b7982b179af513dddb06fca12285f0316d74f3964078acbfcf4c68f2"},
{file = "mypy-0.780-cp38-cp38-win_amd64.whl", hash = "sha256:5d142f219bf8c7894dfa79ebfb7d352c4c63a325e75f10dfb4c3db9417dcd135"},
{file = "mypy-0.780-py3-none-any.whl", hash = "sha256:127de5a9b817a03a98c5ae8a0c46a20dc44442af6dcfa2ae7f96cb519b312efa"},
{file = "mypy-0.780.tar.gz", hash = "sha256:4ef13b619a289aa025f2273e05e755f8049bb4eaba6d703a425de37d495d178d"},
]
mypy-extensions = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
]
parso = [
{file = "parso-0.7.0-py2.py3-none-any.whl", hash = "sha256:158c140fc04112dc45bca311633ae5033c2c2a7b732fa33d0955bad8152a8dd0"},
{file = "parso-0.7.0.tar.gz", hash = "sha256:908e9fae2144a076d72ae4e25539143d40b8e3eafbaeae03c1bfe226f4cdf12c"},
]
pathspec = [
{file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"},
{file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"},
]
pexpect = [
{file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"},
{file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"},
]
pickleshare = [
{file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"},
{file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"},
]
prompt-toolkit = [
{file = "prompt_toolkit-3.0.5-py3-none-any.whl", hash = "sha256:df7e9e63aea609b1da3a65641ceaf5bc7d05e0a04de5bd45d05dbeffbabf9e04"},
{file = "prompt_toolkit-3.0.5.tar.gz", hash = "sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8"},
]
ptyprocess = [
{file = "ptyprocess-0.6.0-py2.py3-none-any.whl", hash = "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"},
{file = "ptyprocess-0.6.0.tar.gz", hash = "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0"},
]
pycodestyle = [
{file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"},
{file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"},
]
pyflakes = [
{file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"},
{file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"},
]
pygments = [
{file = "Pygments-2.6.1-py3-none-any.whl", hash = "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324"},
{file = "Pygments-2.6.1.tar.gz", hash = "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44"},
]
pypandoc = [
{file = "pypandoc-1.5.tar.gz", hash = "sha256:14a49977ab1fbc9b14ef3087dcb101f336851837fca55ca79cf33846cc4976ff"},
]
python-dateutil = [
{file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
{file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
]
regex = [
{file = "regex-2020.6.8-cp27-cp27m-win32.whl", hash = "sha256:fbff901c54c22425a5b809b914a3bfaf4b9570eee0e5ce8186ac71eb2025191c"},
{file = "regex-2020.6.8-cp27-cp27m-win_amd64.whl", hash = "sha256:112e34adf95e45158c597feea65d06a8124898bdeac975c9087fe71b572bd938"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:92d8a043a4241a710c1cf7593f5577fbb832cf6c3a00ff3fc1ff2052aff5dd89"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bae83f2a56ab30d5353b47f9b2a33e4aac4de9401fb582b55c42b132a8ac3868"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:b2ba0f78b3ef375114856cbdaa30559914d081c416b431f2437f83ce4f8b7f2f"},
{file = "regex-2020.6.8-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:95fa7726d073c87141f7bbfb04c284901f8328e2d430eeb71b8ffdd5742a5ded"},
{file = "regex-2020.6.8-cp36-cp36m-win32.whl", hash = "sha256:e3cdc9423808f7e1bb9c2e0bdb1c9dc37b0607b30d646ff6faf0d4e41ee8fee3"},
{file = "regex-2020.6.8-cp36-cp36m-win_amd64.whl", hash = "sha256:c78e66a922de1c95a208e4ec02e2e5cf0bb83a36ceececc10a72841e53fbf2bd"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:08997a37b221a3e27d68ffb601e45abfb0093d39ee770e4257bd2f5115e8cb0a"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2f6f211633ee8d3f7706953e9d3edc7ce63a1d6aad0be5dcee1ece127eea13ae"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:55b4c25cbb3b29f8d5e63aeed27b49fa0f8476b0d4e1b3171d85db891938cc3a"},
{file = "regex-2020.6.8-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:89cda1a5d3e33ec9e231ece7307afc101b5217523d55ef4dc7fb2abd6de71ba3"},
{file = "regex-2020.6.8-cp37-cp37m-win32.whl", hash = "sha256:690f858d9a94d903cf5cada62ce069b5d93b313d7d05456dbcd99420856562d9"},
{file = "regex-2020.6.8-cp37-cp37m-win_amd64.whl", hash = "sha256:1700419d8a18c26ff396b3b06ace315b5f2a6e780dad387e4c48717a12a22c29"},
{file = "regex-2020.6.8-cp38-cp38-manylinux1_i686.whl", hash = "sha256:654cb773b2792e50151f0e22be0f2b6e1c3a04c5328ff1d9d59c0398d37ef610"},
{file = "regex-2020.6.8-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:52e1b4bef02f4040b2fd547357a170fc1146e60ab310cdbdd098db86e929b387"},
{file = "regex-2020.6.8-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:cf59bbf282b627130f5ba68b7fa3abdb96372b24b66bdf72a4920e8153fc7910"},
{file = "regex-2020.6.8-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:5aaa5928b039ae440d775acea11d01e42ff26e1561c0ffcd3d805750973c6baf"},
{file = "regex-2020.6.8-cp38-cp38-win32.whl", hash = "sha256:97712e0d0af05febd8ab63d2ef0ab2d0cd9deddf4476f7aa153f76feef4b2754"},
{file = "regex-2020.6.8-cp38-cp38-win_amd64.whl", hash = "sha256:6ad8663c17db4c5ef438141f99e291c4d4edfeaacc0ce28b5bba2b0bf273d9b5"},
{file = "regex-2020.6.8.tar.gz", hash = "sha256:e9b64e609d37438f7d6e68c2546d2cb8062f3adb27e6336bc129b51be20773ac"},
]
requests = [
{file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"},
{file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"},
]
s3transfer = [
{file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"},
{file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"},
]
six = [
{file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
{file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
]
toml = [
{file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"},
{file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"},
]
tqdm = [
{file = "tqdm-4.46.1-py2.py3-none-any.whl", hash = "sha256:07c06493f1403c1380b630ae3dcbe5ae62abcf369a93bbc052502279f189ab8c"},
{file = "tqdm-4.46.1.tar.gz", hash = "sha256:cd140979c2bebd2311dfb14781d8f19bd5a9debb92dcab9f6ef899c987fcf71f"},
]
traitlets = [
{file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"},
{file = "traitlets-4.3.3.tar.gz", hash = "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7"},
]
typed-ast = [
{file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"},
{file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb"},
{file = "typed_ast-1.4.1-cp35-cp35m-win32.whl", hash = "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919"},
{file = "typed_ast-1.4.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01"},
{file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"},
{file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"},
{file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"},
{file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"},
{file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"},
{file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"},
{file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"},
{file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"},
{file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"},
{file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"},
{file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"},
{file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"},
{file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"},
{file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"},
{file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"},
{file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"},
{file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"},
]
typing-extensions = [
{file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"},
{file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"},
{file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"},
]
urllib3 = [
{file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"},
{file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"},
]
wcwidth = [
{file = "wcwidth-0.2.4-py2.py3-none-any.whl", hash = "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f"},
{file = "wcwidth-0.2.4.tar.gz", hash = "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f"},
]
wheel = [
{file = "wheel-0.34.2-py2.py3-none-any.whl", hash = "sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e"},
{file = "wheel-0.34.2.tar.gz", hash = "sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96"},
]

View file

@ -0,0 +1,31 @@
[tool.poetry]
name = "import-scripts"
version = "0.1.0"
description = ""
authors = ["Rok Garbas <rok@garbas.si>"]
include = [
"import_scripts/packages-config.nix"
]
[tool.poetry.scripts]
import-channel = 'import_scripts.channel:run'
[tool.poetry.dependencies]
python = "^3.8"
requests = "^2.24.0"
click = "^7.1.2"
click-log = "^0.3.2"
elasticsearch = "^7.8.0"
boto3 = "^1.14.5"
tqdm = "^4.46.1"
pypandoc = "^1.5"
[tool.poetry.dev-dependencies]
ipdb = "^0.13.2"
black = "^19.10b0"
flake8 = "^3.8.3"
mypy = "^0.780"
[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"

View file

@ -6,7 +6,7 @@ const {Elm} = require('./Main');
Elm.Main.init({
flags: {
elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 6,
elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 0,
elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443',
elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR',
elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG'