From 5ad71362e505484a34e1b35edbfed0236aeef4ad Mon Sep 17 00:00:00 2001 From: Yannik Sander Date: Tue, 17 Aug 2021 10:55:08 +0200 Subject: [PATCH] Rewrite import script in Rust (#320) --- .github/workflows/cron-nixpkgs.yml | 71 + .github/workflows/cron.yml | 98 - .github/workflows/main.yml | 3 +- .github/workflows/pulls.yml | 57 - .gitignore | 3 +- VERSION | 2 +- flake-info/Cargo.lock | 1725 +++++++++++++++++ flake-info/Cargo.toml | 30 + flake-info/README.md | 206 ++ flake-info/default.nix | 16 + flake-info/examples/.-..json | 48 + flake-info/examples/adaspark-offen.json | 346 ++++ flake-info/examples/examples.in.json | 13 + flake-info/examples/examples.txt | 4 + .../examples/github:W95Psp-LiterateFStar.json | 46 + .../examples/github:ngi-nix-openpgp-ca.json | 50 + .../examples/github:serokell-deploy-rs.json | 50 + flake-info/examples/pull.sh | 15 + flake-info/src/bin/flake-info.rs | 354 ++++ flake-info/src/commands/flake_info.nix | 146 ++ flake-info/src/commands/mod.rs | 8 + flake-info/src/commands/nix_flake_attrs.rs | 51 + flake-info/src/commands/nix_flake_info.rs | 39 + flake-info/src/commands/nix_gc.rs | 54 + flake-info/src/commands/nixpkgs_info.rs | 96 + flake-info/src/commands/packages-config.nix | 49 + flake-info/src/data/export.rs | 369 ++++ flake-info/src/data/flake.rs | 87 + flake-info/src/data/import.rs | 321 +++ flake-info/src/data/mod.rs | 10 + flake-info/src/data/source.rs | 111 ++ flake-info/src/data/system.rs | 38 + flake-info/src/data/utility.rs | 237 +++ flake-info/src/elastic.rs | 515 +++++ flake-info/src/lib.rs | 52 + flake.nix | 19 +- overlay.nix | 1 + src/Main.elm | 25 +- src/Page/Flakes.elm | 297 +++ src/Page/Options.elm | 2 +- src/Page/Packages.elm | 31 +- src/Route.elm | 6 + src/Search.elm | 17 +- 43 files changed, 5544 insertions(+), 174 deletions(-) create mode 100644 .github/workflows/cron-nixpkgs.yml delete mode 100644 .github/workflows/cron.yml delete mode 100644 .github/workflows/pulls.yml create mode 100644 flake-info/Cargo.lock create mode 100644 flake-info/Cargo.toml create mode 100644 flake-info/README.md create mode 100644 flake-info/default.nix create mode 100644 flake-info/examples/.-..json create mode 100644 flake-info/examples/adaspark-offen.json create mode 100644 flake-info/examples/examples.in.json create mode 100644 flake-info/examples/examples.txt create mode 100644 flake-info/examples/github:W95Psp-LiterateFStar.json create mode 100644 flake-info/examples/github:ngi-nix-openpgp-ca.json create mode 100644 flake-info/examples/github:serokell-deploy-rs.json create mode 100755 flake-info/examples/pull.sh create mode 100644 flake-info/src/bin/flake-info.rs create mode 100644 flake-info/src/commands/flake_info.nix create mode 100644 flake-info/src/commands/mod.rs create mode 100644 flake-info/src/commands/nix_flake_attrs.rs create mode 100644 flake-info/src/commands/nix_flake_info.rs create mode 100644 flake-info/src/commands/nix_gc.rs create mode 100644 flake-info/src/commands/nixpkgs_info.rs create mode 100644 flake-info/src/commands/packages-config.nix create mode 100644 flake-info/src/data/export.rs create mode 100644 flake-info/src/data/flake.rs create mode 100644 flake-info/src/data/import.rs create mode 100644 flake-info/src/data/mod.rs create mode 100644 flake-info/src/data/source.rs create mode 100644 flake-info/src/data/system.rs create mode 100644 flake-info/src/data/utility.rs create mode 100644 flake-info/src/elastic.rs create mode 100644 flake-info/src/lib.rs create mode 100644 src/Page/Flakes.elm diff --git a/.github/workflows/cron-nixpkgs.yml b/.github/workflows/cron-nixpkgs.yml new file mode 100644 index 0000000..808a473 --- /dev/null +++ b/.github/workflows/cron-nixpkgs.yml @@ -0,0 +1,71 @@ +name: "Nixpkgs: Hourly import to Elasticsearch" + +on: + + schedule: + - cron: '0 * * * *' + +jobs: + + import-channel: + + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + channel: + - unstable + - 21.05 + - 20.09 + + env: + RUST_LOG: debug + FI_ES_EXISTS_STRATEGY: abort + FI_ES_URL: ${{ secrets.ELASTICSEARCH_URL }} + + steps: + + - name: Checking out the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Installing Nix + uses: cachix/install-nix-action@v13 + with: + install_url: https://nixos-nix-install-tests.cachix.org/serve/i6laym9jw3wg9mw6ncyrk6gjx4l34vvx/install + install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve' + extra_nix_config: | + experimental-features = nix-command flakes + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + + - uses: cachix/cachix-action@v10 + with: + name: nixos-search + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + + - name: Install unstable channel + run: | + nix-channel --add https://nixos.org/channels/nixpkgs-unstable + nix-channel --update + + - name: Installing jq + run: | + nix-env -iA nixpkgs.nixFlakes nixpkgs.jq + + - name: Building import_scripts + run: | + nix build ./#packages.x86_64-linux.flake_info + + - name: Import ${{ matrix.channel }} channel + run: | + ./result/bin/flake-info --push --elastic-schema-version=$(cat ./VERSION) nixpkgs ${{ matrix.channel }} + if: github.repository == 'NixOS/nixos-search' + + - name: Warmup ${{ matrix.channel }} channel + run: | + curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-nixpkgs-${{ matrix.channel }}/_search | jq '.took' + curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-nixpkgs-${{ matrix.channel }}/_search | jq '.took' + curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-nixpkgs-${{ matrix.channel }}/_search | jq '.took' + if: github.repository == 'NixOS/nixos-search' diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml deleted file mode 100644 index ad4ca71..0000000 --- a/.github/workflows/cron.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: "Hourly import channel to Elasticsearch" - -on: - - schedule: - - cron: '0 * * * *' - -jobs: - - hourly-import-channel: - - runs-on: ubuntu-latest - - strategy: - fail-fast: true - matrix: - channel: - - unstable - - 21.05 - - 20.09 - - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_S3_URL: s3://nix-releases/nixpkgs - - steps: - - - name: Checking out the repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Installing Nix - uses: cachix/install-nix-action@v13 - - - uses: cachix/cachix-action@v10 - with: - name: nixos-search - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - - - name: Install unstable channel - run: | - nix-channel --add https://nixos.org/channels/nixpkgs-unstable - nix-channel --update - - - name: Installing awscli - run: | - nix-env -iA nixpkgs.awscli2 - - - name: Check for latest evaluation in ${{ matrix.channel }} channel - run: | - if [ "${{ matrix.channel }}" = "unstable" ]; then - RELEASE=$(aws s3 ls --no-sign-request "$AWS_S3_URL/" | grep '/$' | cut -d' ' -f29 | sort | tail -1 | sed 's|/||') - else - RELEASE=$(aws s3 ls --no-sign-request "$AWS_S3_URL/" | grep 'nixpkgs-${{ matrix.channel }}pre' | grep '/$' | cut -d' ' -f29 | sort | tail -1 | sed 's|/||') - fi - aws s3 cp --no-sign-request "$AWS_S3_URL/$RELEASE/src-url" ./ - EVAL_ID=$(cat src-url | cut -c30-) - echo "EVAL_ID=${EVAL_ID}" >> $GITHUB_ENV - - - name: Download latest builds for ${{ matrix.channel }} channel (if needed) - if: steps.eval-cache.outputs.cache-hit != 'true' - run: | - mkdir -p ./eval-cache - cp ./src-url ./eval-cache/ - curl -H "Content-Type: application/json" "$(cat ./eval-cache/src-url)/builds" -o ./eval-cache/builds.json - - - name: Cache ${{ matrix.channel }} channel builds - id: eval-cache - uses: actions/cache@v2 - with: - path: ./eval-cache - key: eval-cache-${{ env.EVAL_ID }} - - - name: Installing nixFlakes (and jq) - run: | - nix-env -iA nixpkgs.nixFlakes nixpkgs.jq - echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf - nix --version - cat /etc/nix/nix.conf - echo "$HOME/.nix-profile/bin" >> $GITHUB_PATH - - - name: Building import_scripts - run: | - nix build ./#packages.x86_64-linux.import_scripts - - - name: Import ${{ matrix.channel }} channel - run: | - cp ./eval-cache/builds.json ./eval-${{ env.EVAL_ID }}.json - ./result/bin/import-channel --es-url ${{ secrets.ELASTICSEARCH_URL }} --channel ${{ matrix.channel }} -vvv - if: github.repository == 'NixOS/nixos-search' - - - name: Warmup ${{ matrix.channel }} channel - run: | - curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-${{ matrix.channel }}/_search | jq '.took' - curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-${{ matrix.channel }}/_search | jq '.took' - curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-${{ matrix.channel }}/_search | jq '.took' - if: github.repository == 'NixOS/nixos-search' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 62adc88..dd6186e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,6 @@ name: "Build & Deploy to Netlify" on: + pull_request: push: branches: - main @@ -50,7 +51,7 @@ jobs: NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} with: production-branch: 'main' - production-deploy: true + production-deploy: ${{ github.event_name == 'push' }} publish-dir: './dist' github-token: ${{ secrets.GITHUB_TOKEN }} deploy-message: 'Deploy from GitHub Actions' diff --git a/.github/workflows/pulls.yml b/.github/workflows/pulls.yml deleted file mode 100644 index f8a2957..0000000 --- a/.github/workflows/pulls.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: "Build & Deploy to Netlify" -on: - pull_request: -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - - name: Checking out the repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Installing Nix - uses: cachix/install-nix-action@v13 - - - uses: cachix/cachix-action@v10 - with: - name: nixos-search - - - name: Install unstable channel - run: | - nix-channel --add https://nixos.org/channels/nixpkgs-unstable - nix-channel --update - - - name: Installing NixFlakes - run: | - nix-env -iA nixpkgs.nixFlakes - echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf - nix --version - cat /etc/nix/nix.conf - echo "$HOME/.nix-profile/bin" >> $GITHUB_PATH - - - name: Building import_scripts - run: | - nix build ./#packages.x86_64-linux.import_scripts - - - name: Building search.nixos.org - run: | - nix build ./#packages.x86_64-linux.frontend - mkdir ./dist - cp -RL ./result/* ./dist/ - - - name: Deploy to Netlify - uses: nwtgck/actions-netlify@v1.2 - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} - with: - production-branch: 'main' - publish-dir: './dist' - github-token: ${{ secrets.GITHUB_TOKEN }} - deploy-message: 'Deploy from GitHub Actions' - enable-pull-request-comment: true - enable-commit-comment: true - enable-commit-status: true - overwrites-pull-request-comment: false - if: github.repository == 'NixOS/nixos-search' diff --git a/.gitignore b/.gitignore index 91a8fc6..fc3bbb4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .vscode .node_repl_history .npm +target/ build/Release dist elm-stuff/ @@ -18,4 +19,4 @@ npm-debug.log* package-lock.json repl-temp-* result -src-url \ No newline at end of file +src-url diff --git a/VERSION b/VERSION index 209e3ef..aabe6ec 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -20 +21 diff --git a/flake-info/Cargo.lock b/flake-info/Cargo.lock new file mode 100644 index 0000000..ac7ce83 --- /dev/null +++ b/flake-info/Cargo.lock @@ -0,0 +1,1725 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" + +[[package]] +name = "async-compression" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bit-set" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "command-run" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d89d1b108bbcdd5d3edba0644210660ce441cf29f002e6e2bbe43427ac0e7ae" +dependencies = [ + "log", + "os_pipe", +] + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "cpufeatures" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "darling" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d6ddad5866bb2170686ed03f6839d31a76e5407d80b1c334a2c24618543ffa" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ced1fd13dc386d5a8315899de465708cf34ee2a6d9394654515214e67bb846" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7a1445d54b2f9792e3b31a3e715feabbace393f38dc4ffd49d94ee9bc487d5" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dyn-clone" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" + +[[package]] +name = "elasticsearch" +version = "8.0.0-alpha.1" +source = "git+https://github.com/elastic/elasticsearch-rs#d96bfa25f1d9695e624e7ae393b55ac2ccd6735a" +dependencies = [ + "base64 0.11.0", + "bytes", + "dyn-clone", + "lazy_static", + "percent-encoding", + "reqwest", + "rustc_version", + "serde", + "serde_json", + "serde_with", + "url", + "void", +] + +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "fancy-regex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3371ac125221b609ce0cf587a53fbb624fb56267bbe85cf1929350043bf360" +dependencies = [ + "bit-set", + "regex", +] + +[[package]] +name = "flake-info" +version = "0.3.0" +dependencies = [ + "anyhow", + "clap", + "command-run", + "elasticsearch", + "env_logger", + "fancy-regex", + "lazy_static", + "log", + "reqwest", + "serde", + "serde_json", + "sha2", + "structopt", + "tempfile", + "thiserror", + "tokio", +] + +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" + +[[package]] +name = "futures-io" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0e06c393068f3a6ef246c75cdca793d6a46347e75286933e5e75fd2fd11582" + +[[package]] +name = "futures-macro" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" + +[[package]] +name = "futures-task" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" + +[[package]] +name = "futures-util" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "futures-util", + "hyper", + "log", + "rustls", + "tokio", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" + +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "os_pipe" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4" +dependencies = [ + "async-compression", + "base64 0.13.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-rustls", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tokio-util", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustversion" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb5d2a036dc6d2d8fd16fde3498b04306e29bd193bf306a57427019b823d5acd" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e557c650adfb38b32a5aec07082053253c703bc3cec654b27a5dbcf61995bb9b" +dependencies = [ + "rustversion", + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48b35457e9d855d3dc05ef32a73e0df1e2c0fd72c38796a4ee909160c8eeec2" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +dependencies = [ + "block-buffer", + "cfg-if", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "structopt" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5143d049e85af7fbc36f5454d990e62c2df705b3589f123b71f441b6b59f443f" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" + +[[package]] +name = "unicode-bidi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" +dependencies = [ + "cfg-if", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" + +[[package]] +name = "web-sys" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] diff --git a/flake-info/Cargo.toml b/flake-info/Cargo.toml new file mode 100644 index 0000000..583e282 --- /dev/null +++ b/flake-info/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "flake-info" +version = "0.3.0" +authors = ["Yannik Sander "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = "^2.33" +serde = {version="1.0", features = ["derive"]} +serde_json = "1.0" +anyhow = "1.0" +thiserror = "1.0" +structopt = "0.3" +command-run = "0.13" +env_logger = "0.8" +log = "0.4" +tempfile = "3" +lazy_static = "1.4" +fancy-regex = "0.6" +tokio = { version = "*", features = ["full"] } +reqwest = { version = "0.11", features = ["json", "blocking"] } +sha2 = "0.9" + +elasticsearch = {git = "https://github.com/elastic/elasticsearch-rs", features = ["rustls-tls"]} + +[lib] +name = "flake_info" +path = "./src/lib.rs" diff --git a/flake-info/README.md b/flake-info/README.md new file mode 100644 index 0000000..fd87f32 --- /dev/null +++ b/flake-info/README.md @@ -0,0 +1,206 @@ +# Flake Info + +A tool that fetches packages and apps from nix flakes. + +## Usage + +``` +flake-info 0.3.0 +Extracts various information from a given flake + +USAGE: + flake-info [FLAGS] [OPTIONS] [extra]... + +FLAGS: + --push Push to Elasticsearch (Configure using FI_ES_* environment variables) + -h, --help Prints help information + --json Print ElasticSeach Compatible JSON output + -V, --version Prints version information + +OPTIONS: + --elastic-exists + How to react to existing indices [env: FI_ES_EXISTS_STRATEGY=] [default: abort] [possible values: Abort, + Ignore, Recreate] + --elastic-index-name Name of the index to store results to [env: FI_ES_INDEX=] + -p, --elastic-pw + Elasticsearch password (unimplemented) [env: FI_ES_PASSWORD=] + + --elastic-schema-version + Which schema version to associate with the operation [env: FI_ES_VERSION=] + + --elastic-url + Elasticsearch instance url [env: FI_ES_URL=] [default: http://localhost:9200] + + -u, --elastic-user Elasticsearch username (unimplemented) [env: FI_ES_USER=] + -k, --kind + Kind of data to extract (packages|options|apps|all) [default: all] + + +ARGS: + ... Extra arguments that are passed to nix as it + +SUBCOMMANDS: + flake + group + help Prints this message or the help of the given subcommand(s) + nixpkgs +``` + +### flake + +Flakes can be imported using the flake subcommand + +``` +USAGE: + flake-info flake [FLAGS] + +FLAGS: + --gc Whether to gc the store after info or not + -h, --help Prints help information + --temp-store Whether to use a temporary store or not. Located at /tmp/flake-info-store + -V, --version Prints version information + +ARGS: + Flake identifier passed to nix to gather information about +``` + +The `` argument should contain a valid reference to a flake. It accepts all formats nix accepts: + +> use git+ to checkout a git repository at +> use /local/absolute/path or ./relative/path to load a local source +> use gitlab://github:/ to shortcut gitlab or github repositories + + +Optionally, analyzing can be done in a temporary store enabled by the `--temp-store` option. + +#### Example + +``` +$ flake-info flake github:ngi-nix/offen +``` + +### nixpkgs + +nixpkgs currently have to be imported in a different way. This is what the `nixpkgs` subcommand exists for. + +It takes any valid git reference to the upstream [`nixos/nixpkgs`](https://github.com/iixos/nixpkgs/) repo as an argument and produces a complete output. + +**This operation may take a short while and produces lots of output** + +#### Example + +``` +$ flake-info nixpkgs nixos-21.05 +``` + +### group + +to perform a bulk import grouping multiple inputs under the same name/index use the group command. + +It expects a json file as input that contains references to flakes or nixpkgs. If those resources are on github or gitlab they can be extended with more meta information including pinning the commit hash/ref. + +The second argument is the group name that is used to provide the index name. + +#### Example + +An example `targets.json` file can look like the following + +```json +[ + { + "type": "git", + "url": "./." + }, + { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + { + "type": "github", + "owner": "ngi-nix", + "repo": "offen", + "hash": "4052febf151d60aa4352fa1960cf3ae088f600aa", + "description": "Hier könnte Ihre Werbung stehen" + } +] +``` + +``` +$ flake-info group ./targets.json small-group +``` + +### Elasticsearch + +A number of flags is dedicated to pushing to elasticsearch. + +``` + --elastic-exists + How to react to existing indices [env: FI_ES_EXISTS_STRATEGY=] [default: abort] + [possible values: Abort, Ignore, Recreate] + --elastic-index-name + Name of the index to store results to [env: FI_ES_INDEX=] +-p, --elastic-pw + Elasticsearch password (unimplemented) [env: FI_ES_PASSWORD=] + + --elastic-schema-version + Which schema version to associate with the operation [env: FI_ES_VERSION=] + + --elastic-url + Elasticsearch instance url [env: FI_ES_URL=] [default: http://localhost:9200] + +-u, --elastic-user Elasticsearch username (unimplemented) [env: FI_ES_USER=] +``` + + +#### Example + +``` +$ flake-info --push \ + --elastic-url http://localhost:5555 \ + --elastic-index-name latest-21-21.05 + --elastic-schema-version 21 group ./examples/ngi-nix.json ngi-nix +``` + + +## Installation + +### Preparations + +This tool requires your system to have Nix installed! + +You can install nix using this installer: https://nixos.org/guides/install-nix.html +Also, see https://nixos.wiki/wiki/Nix_Installation_Guide if your system is ✨special✨. + +### Preparations (Docker) + +If you do not want to install nix on your system, using Docker is an alternative. + +Enter the [nixos/nix](https://hub.docker.com/u/nixos/) docker image and proceed + +### Setup nix flakes + +Note that you also need to have nix flakes support. + +Once you have nix installed run the following commands: + +1. ``` + $ nix-shell -I nixpkgs=channel:nixos-21.05 -p nixFlakes + ``` + to enter a shell with the preview version of nix flakes installed. +2. ``` + $ mkdir -p ~/.config/nix + $ echo "experimental-features = nix-command flakes" > .config/nix/nix.conf + ``` + to enable flake support + +### Installation, finally + +This project is defined as a flake therefore you can build the tool using + +``` +$ nix build +or +$ nix build github:miszkur/github-search +``` + +Replace `build` with run if you want to run the tool directly. diff --git a/flake-info/default.nix b/flake-info/default.nix new file mode 100644 index 0000000..eaeb252 --- /dev/null +++ b/flake-info/default.nix @@ -0,0 +1,16 @@ +{ pkgs ? import { } }: with pkgs; + + + + +rustPlatform.buildRustPackage rec { + name = "flake-info"; + src = ./.; + cargoSha256 = "sha256-TA1WEvmOfnxQ+rRwkIPN1t4VPrDL6pq+WnPHVu2/CPE="; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ openssl openssl.dev ] ++ lib.optional pkgs.stdenv.isDarwin [libiconv darwin.apple_sdk.frameworks.Security]; + checkFlags = [ + "--skip elastic::tests" + "--skip nix_gc::tests" + ]; +} diff --git a/flake-info/examples/.-..json b/flake-info/examples/.-..json new file mode 100644 index 0000000..8c39885 --- /dev/null +++ b/flake-info/examples/.-..json @@ -0,0 +1,48 @@ +[ + { + "flake_description": "Extracting information from flakes", + "flake_resolved": { + "type": "git", + "url": "file:///Volumes/projects/Uni/courses/kth/DD2476-Search-Engines-and-Information-Retrieval-Systems/project?dir=flake-info" + }, + "flake_name": "", + "flake_source": { + "type": "git", + "url": "./." + }, + "package_attr_name": "flake-info", + "package_pname": "flake-info", + "package_pversion": "", + "package_platforms": [ + "x86_64-linux", + "x86_64-darwin", + "i686-linux", + "aarch64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Extracting information from flakes", + "flake_resolved": { + "type": "git", + "url": "file:///Volumes/projects/Uni/courses/kth/DD2476-Search-Engines-and-Information-Retrieval-Systems/project?dir=flake-info" + }, + "flake_name": "", + "flake_source": { + "type": "git", + "url": "./." + }, + "app_bin": "/nix/store/4akx0is6fgh9ci2ak5sbskwzykr0xj85-flake-info/bin/flake-info", + "app_attr_name": "flake-info", + "app_platforms": [ + "x86_64-linux", + "x86_64-darwin", + "i686-linux", + "aarch64-linux" + ], + "app_type": "app" + } +] diff --git a/flake-info/examples/adaspark-offen.json b/flake-info/examples/adaspark-offen.json new file mode 100644 index 0000000..b7453c9 --- /dev/null +++ b/flake-info/examples/adaspark-offen.json @@ -0,0 +1,346 @@ +[ + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "adaspark", + "package_pname": "adaspark", + "package_pversion": "", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "asis", + "package_pname": "ASIS", + "package_pversion": "gcc-10.1.0", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "aunit", + "package_pname": "AUnit", + "package_pversion": "20.2", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "gnat", + "package_pname": "gnat-10.2.0", + "package_pversion": "10.2.0", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out", + "man", + "info" + ], + "package_description": "GNU Compiler Collection, version 10.2.0 (wrapper script)", + "package_license": { + "license_long": "GNU General Public License v3.0 or later", + "license": "gpl3Plus", + "license_url": "https://spdx.org/licenses/GPL-3.0-or-later.html" + } + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "gnat_util", + "package_pname": "gnat_util", + "package_pversion": "10.1.0", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "gnatcoll-core", + "package_pname": "gnatcoll-core", + "package_pversion": "20.2", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "gpr", + "package_pname": "gprbuild", + "package_pversion": "20.2", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "spark", + "package_pname": "SPARK2014", + "package_pversion": "20.2", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Compilers and tools for SPARK2014 Ada development", + "flake_resolved": { + "type": "github", + "owner": "fluffynukeit", + "repo": "adaspark" + }, + "flake_name": "adaspark", + "flake_source": { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + "package_attr_name": "xmlada", + "package_pname": "xmlada", + "package_pversion": "20.2", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Offen, a fair web analytics tool", + "flake_resolved": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen" + }, + "flake_name": "offen", + "flake_source": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen", + "description": "Hier könnte Ihre Werbung stehen", + "git_ref": "4052febf151d60aa4352fa1960cf3ae088f600aa" + }, + "package_attr_name": "license_finder", + "package_pname": "license_finder", + "package_pversion": "", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Offen, a fair web analytics tool", + "flake_resolved": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen" + }, + "flake_name": "offen", + "flake_source": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen", + "description": "Hier könnte Ihre Werbung stehen", + "git_ref": "4052febf151d60aa4352fa1960cf3ae088f600aa" + }, + "package_attr_name": "offen", + "package_pname": "offen-20210115", + "package_pversion": "20210115", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Offen, a fair web analytics tool", + "flake_resolved": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen" + }, + "flake_name": "offen", + "flake_source": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen", + "description": "Hier könnte Ihre Werbung stehen", + "git_ref": "4052febf151d60aa4352fa1960cf3ae088f600aa" + }, + "package_attr_name": "offen-auditorium", + "package_pname": "offen-auditorium", + "package_pversion": "20210115", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Offen, a fair web analytics tool", + "flake_resolved": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen" + }, + "flake_name": "offen", + "flake_source": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen", + "description": "Hier könnte Ihre Werbung stehen", + "git_ref": "4052febf151d60aa4352fa1960cf3ae088f600aa" + }, + "package_attr_name": "offen-script", + "package_pname": "offen-script", + "package_pversion": "20210115", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "Offen, a fair web analytics tool", + "flake_resolved": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen" + }, + "flake_name": "offen", + "flake_source": { + "type": "github", + "owner": "ngi-nix", + "repo": "offen", + "description": "Hier könnte Ihre Werbung stehen", + "git_ref": "4052febf151d60aa4352fa1960cf3ae088f600aa" + }, + "package_attr_name": "offen-vault", + "package_pname": "offen-vault", + "package_pversion": "20210115", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + } +] diff --git a/flake-info/examples/examples.in.json b/flake-info/examples/examples.in.json new file mode 100644 index 0000000..43f42da --- /dev/null +++ b/flake-info/examples/examples.in.json @@ -0,0 +1,13 @@ +[ + { + "type": "git", + "url": "github:fluffynukeit/adaspark" + }, + { + "type": "github", + "owner": "ngi-nix", + "repo": "offen", + "git_ref": "4052febf151d60aa4352fa1960cf3ae088f600aa", + "description": "Hier könnte Ihre Werbung stehen" + } +] diff --git a/flake-info/examples/examples.txt b/flake-info/examples/examples.txt new file mode 100644 index 0000000..d505327 --- /dev/null +++ b/flake-info/examples/examples.txt @@ -0,0 +1,4 @@ +github:serokell/deploy-rs +github:W95Psp/LiterateFStar +github:ngi-nix/openpgp-ca +./. diff --git a/flake-info/examples/github:W95Psp-LiterateFStar.json b/flake-info/examples/github:W95Psp-LiterateFStar.json new file mode 100644 index 0000000..810993c --- /dev/null +++ b/flake-info/examples/github:W95Psp-LiterateFStar.json @@ -0,0 +1,46 @@ +[ + { + "flake_description": "LiterateFStar", + "flake_resolved": { + "type": "github", + "owner": "W95Psp", + "repo": "LiterateFStar" + }, + "flake_name": "LiterateFStar", + "flake_source": { + "type": "git", + "url": "github:W95Psp/LiterateFStar" + }, + "package_attr_name": "fstar", + "package_pname": "fstar-c671957efe8769b8fc421cd3e9da47b3fa57d510", + "package_pversion": "", + "package_platforms": [ + "x86_64-linux", + "x86_64-darwin" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "LiterateFStar", + "flake_resolved": { + "type": "github", + "owner": "W95Psp", + "repo": "LiterateFStar" + }, + "flake_name": "LiterateFStar", + "flake_source": { + "type": "git", + "url": "github:W95Psp/LiterateFStar" + }, + "app_bin": "/nix/store/mwwn9wzbgkdfac4ijj176akbkr9bxk5k-build", + "app_attr_name": "build", + "app_platforms": [ + "x86_64-linux", + "x86_64-darwin" + ], + "app_type": "derivation" + } +] diff --git a/flake-info/examples/github:ngi-nix-openpgp-ca.json b/flake-info/examples/github:ngi-nix-openpgp-ca.json new file mode 100644 index 0000000..f184f5b --- /dev/null +++ b/flake-info/examples/github:ngi-nix-openpgp-ca.json @@ -0,0 +1,50 @@ +[ + { + "flake_description": "OpenPGP CA is a tool for managing OpenPGP keys within an organization.", + "flake_resolved": { + "type": "github", + "owner": "ngi-nix", + "repo": "openpgp-ca" + }, + "flake_name": "openpgp-ca", + "flake_source": { + "type": "git", + "url": "github:ngi-nix/openpgp-ca" + }, + "package_attr_name": "openpgp-ca", + "package_pname": "openpgp-ca", + "package_pversion": "20200717", + "package_platforms": [ + "x86_64-linux", + "x86_64-darwin" + ], + "package_outputs": [ + "out" + ], + "package_description": "OpenPGP CA is a tool for managing OpenPGP keys within an organization.", + "package_license": {} + }, + { + "flake_description": "OpenPGP CA is a tool for managing OpenPGP keys within an organization.", + "flake_resolved": { + "type": "github", + "owner": "ngi-nix", + "repo": "openpgp-ca" + }, + "flake_name": "openpgp-ca", + "flake_source": { + "type": "git", + "url": "github:ngi-nix/openpgp-ca" + }, + "package_attr_name": "openpgp-ca-docker", + "package_pname": "docker-image-openpgp-ca.tar.gz", + "package_pversion": "", + "package_platforms": [ + "x86_64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + } +] diff --git a/flake-info/examples/github:serokell-deploy-rs.json b/flake-info/examples/github:serokell-deploy-rs.json new file mode 100644 index 0000000..8fc3148 --- /dev/null +++ b/flake-info/examples/github:serokell-deploy-rs.json @@ -0,0 +1,50 @@ +[ + { + "flake_description": "A Simple multi-profile Nix-flake deploy tool.", + "flake_resolved": { + "type": "github", + "owner": "serokell", + "repo": "deploy-rs" + }, + "flake_name": "deploy-rs", + "flake_source": { + "type": "git", + "url": "github:serokell/deploy-rs" + }, + "package_attr_name": "deploy-rs", + "package_pname": "deploy-rs-0.1.0", + "package_pversion": "0.1.0", + "package_platforms": [ + "x86_64-linux", + "x86_64-darwin", + "i686-linux", + "aarch64-linux" + ], + "package_outputs": [ + "out" + ], + "package_license": {} + }, + { + "flake_description": "A Simple multi-profile Nix-flake deploy tool.", + "flake_resolved": { + "type": "github", + "owner": "serokell", + "repo": "deploy-rs" + }, + "flake_name": "deploy-rs", + "flake_source": { + "type": "git", + "url": "github:serokell/deploy-rs" + }, + "app_bin": "/nix/store/lw8c19dkrr8a766qbl89nsfwbgwhp43q-deploy-rs-0.1.0/bin/deploy", + "app_attr_name": "deploy-rs", + "app_platforms": [ + "x86_64-linux", + "x86_64-darwin", + "i686-linux", + "aarch64-linux" + ], + "app_type": "app" + } +] diff --git a/flake-info/examples/pull.sh b/flake-info/examples/pull.sh new file mode 100755 index 0000000..ab4b25a --- /dev/null +++ b/flake-info/examples/pull.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +# Run from cargo root as +# $ ./examples/pull.sh + +echo "pulling examples in examples.txt" +examples=$(cat ./examples/examples.txt) +for flake in $examples; do + + cargo run -- --flake "$flake" | jq > examples/"$(echo "$flake" | tr "/" "-")".json + +done + +echo "pulling excamples using json file" +cargo run -- --targets ./examples/examples.in.json | jq > examples/adaspark-offen.json diff --git a/flake-info/src/bin/flake-info.rs b/flake-info/src/bin/flake-info.rs new file mode 100644 index 0000000..620f23f --- /dev/null +++ b/flake-info/src/bin/flake-info.rs @@ -0,0 +1,354 @@ +use anyhow::{Context, Result}; +use commands::run_gc; +use flake_info::data::import::{Kind, NixOption}; +use flake_info::data::{self, Export, Nixpkgs, Source}; +use flake_info::elastic::{ElasticsearchError, ExistsStrategy}; +use flake_info::{commands, elastic}; +use log::{debug, error, info, warn}; +use sha2::Digest; +use std::fs; +use std::path::{Path, PathBuf}; +use std::ptr::hash; +use structopt::{clap::ArgGroup, StructOpt}; +use thiserror::Error; + +#[derive(StructOpt, Debug)] +#[structopt( + name = "flake-info", + about = "Extracts various information from a given flake", + group = ArgGroup::with_name("sources").required(false) +)] +struct Args { + #[structopt(subcommand)] + command: Command, + + #[structopt( + short, + long, + help = "Kind of data to extract (packages|options|apps|all)", + default_value + )] + kind: data::import::Kind, + + #[structopt(flatten)] + elastic: ElasticOpts, + + #[structopt(help = "Extra arguments that are passed to nix as it")] + extra: Vec, +} + +#[derive(StructOpt, Debug)] +enum Command { + Flake { + #[structopt(help = "Flake identifier passed to nix to gather information about")] + flake: String, + + #[structopt( + long, + help = "Whether to use a temporary store or not. Located at /tmp/flake-info-store" + )] + temp_store: bool, + + #[structopt(long, help = "Whether to gc the store after info or not")] + gc: bool, + }, + Nixpkgs { + #[structopt(help = "Nixpkgs channel to import")] + channel: String, + }, + Group { + #[structopt(help = "Points to a JSON file containing info targets")] + targets: PathBuf, + + name: String, + + #[structopt( + long, + help = "Whether to use a temporary store or not. Located at /tmp/flake-info-store" + )] + temp_store: bool, + + #[structopt(long, help = "Whether to gc the store after info or not")] + gc: bool, + }, +} + +#[derive(StructOpt, Debug)] +struct ElasticOpts { + #[structopt(long = "json", help = "Print ElasticSeach Compatible JSON output")] + json: bool, + + #[structopt( + long = "push", + help = "Push to Elasticsearch (Configure using FI_ES_* environment variables)", + requires("elastic-schema-version") + )] + enable: bool, + + #[structopt( + long, + short = "u", + env = "FI_ES_USER", + help = "Elasticsearch username (unimplemented)" + )] + elastic_user: Option, + + #[structopt( + long, + short = "p", + env = "FI_ES_PASSWORD", + help = "Elasticsearch password (unimplemented)" + )] + elastic_pw: Option, + + #[structopt( + long, + env = "FI_ES_URL", + default_value = "http://localhost:9200", + help = "Elasticsearch instance url" + )] + elastic_url: String, + + #[structopt( + long, + help = "Name of the index to store results to", + env = "FI_ES_INDEX", + required_if("enable", "true") + )] + elastic_index_name: Option, + + #[structopt( + long, + help = "How to react to existing indices", + possible_values = &ExistsStrategy::variants(), + case_insensitive = true, + default_value = "abort", + env = "FI_ES_EXISTS_STRATEGY" + )] + elastic_exists: ExistsStrategy, + + #[structopt( + long, + help = "Which schema version to associate with the operation", + env = "FI_ES_VERSION" + )] + elastic_schema_version: Option, + + #[structopt( + long, + help = "Whether to disable `latest` alias creation", + env = "FI_ES_VERSION" + )] + no_alias: bool, +} + +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + + let args = Args::from_args(); + + let command_result = run_command(args.command, args.kind, &args.extra).await; + + if let Err(error) = command_result { + match error { + FlakeInfoError::Flake(ref e) + | FlakeInfoError::Nixpkgs(ref e) + | FlakeInfoError::IO(ref e) => { + error!("{}", e); + } + FlakeInfoError::Group(ref el) => { + el.iter().for_each(|e| error!("{}", e)); + } + } + + return Err(error.into()); + } + + let (successes, ident) = command_result.unwrap(); + + if args.elastic.enable { + push_to_elastic(&args.elastic, &successes, ident).await?; + } + + if args.elastic.json { + println!("{}", serde_json::to_string(&successes)?); + } + Ok(()) +} + +#[derive(Debug, Error)] +enum FlakeInfoError { + #[error("Getting flake info caused an error: {0}")] + Flake(anyhow::Error), + #[error("Getting nixpkgs info caused an error: {0}")] + Nixpkgs(anyhow::Error), + #[error("Getting group info caused one or more errors: {0:?}")] + Group(Vec), + + #[error("Couldn't perform IO: {0}")] + IO(anyhow::Error), +} + +async fn run_command( + command: Command, + kind: Kind, + extra: &[String], +) -> Result<(Vec, (String, String, String)), FlakeInfoError> { + match command { + Command::Flake { + flake, + temp_store, + gc, + } => { + let source = Source::Git { url: flake }; + let exports = flake_info::process_flake(&source, &kind, temp_store, extra) + .map_err(FlakeInfoError::Flake)?; + + let info = flake_info::get_flake_info(source.to_flake_ref(), temp_store, extra) + .map_err(FlakeInfoError::Flake)?; + + let ident = ("flake".to_owned(), info.name, info.revision); + + Ok((exports, ident)) + } + Command::Nixpkgs { channel } => { + let nixpkgs = Source::nixpkgs(channel) + .await + .map_err(FlakeInfoError::Nixpkgs)?; + let ident = ( + "nixpkgs".to_owned(), + nixpkgs.channel.clone(), + nixpkgs.git_ref.clone(), + ); + let exports = flake_info::process_nixpkgs(&Source::Nixpkgs(nixpkgs), &kind) + .map_err(FlakeInfoError::Nixpkgs)?; + + Ok((exports, ident)) + } + Command::Group { + targets, + temp_store, + gc, + name, + } => { + let sources = Source::read_sources_file(&targets).map_err(FlakeInfoError::IO)?; + let (exports_and_hashes, errors) = sources + .iter() + .map(|source| match source { + Source::Nixpkgs(nixpkgs) => flake_info::process_nixpkgs(source, &kind) + .map(|result| (result, nixpkgs.git_ref.to_owned())), + _ => flake_info::process_flake(source, &kind, temp_store, &extra).and_then( + |result| { + flake_info::get_flake_info(source.to_flake_ref(), temp_store, extra) + .map(|info| (result, info.revision)) + }, + ), + }) + .partition::, _>(Result::is_ok); + + let (exports, hashes) = exports_and_hashes + .into_iter() + .map(|result| result.unwrap()) + .fold( + (Vec::new(), Vec::new()), + |(mut exports, mut hashes), (export, hash)| { + exports.extend(export); + hashes.push(hash); + (exports, hashes) + }, + ); + + let errors = errors + .into_iter() + .map(Result::unwrap_err) + .collect::>(); + + if !errors.is_empty() { + return Err(FlakeInfoError::Group(errors)); + } + + let hash = { + let mut sha = sha2::Sha256::new(); + for hash in hashes { + sha.update(hash); + } + format!("{:08x}", sha.finalize()) + }; + + let ident = ("group".to_owned(), name, hash); + + Ok((exports, ident)) + } + } +} + +async fn push_to_elastic( + elastic: &ElasticOpts, + successes: &[Export], + ident: (String, String, String), +) -> Result<()> { + let (index, alias) = elastic + .elastic_index_name + .to_owned() + .map(|ident| { + ( + format!("{}-{}", elastic.elastic_schema_version.unwrap(), ident), + None, + ) + }) + .or_else(|| { + let (kind, name, hash) = ident; + let ident = format!( + "{}-{}-{}-{}", + kind, + elastic.elastic_schema_version.unwrap(), + &name, + hash + ); + let alias = format!( + "latest-{}-{}-{}", + elastic.elastic_schema_version.unwrap(), + kind, + &name + ); + + warn!("Using automatic index identifier: {}", ident); + Some((ident, Some(alias))) + }) + .unwrap(); + + info!("Pushing to elastic"); + let es = elastic::Elasticsearch::new(elastic.elastic_url.as_str())?; + let config = elastic::Config { + index: &index, + exists_strategy: elastic.elastic_exists, + }; + + // catch error variant if abort strategy was triggered + let ensure = es.ensure_index(&config).await; + if let Err(ElasticsearchError::IndexExistsError(_)) = ensure { + // abort on abort + return Ok(()); + } else { + // throw error if present + ensure?; + } + + + es.push_exports(&config, successes) + .await + .with_context(|| "Failed to push results to elasticsearch".to_string())?; + + if let Some(alias) = alias { + if !elastic.no_alias { + es.write_alias(&config, &index, &alias) + .await + .with_context(|| "Failed to create alias".to_string())?; + } else { + warn!("Creating alias disabled") + } + } + + Ok(()) +} diff --git a/flake-info/src/commands/flake_info.nix b/flake-info/src/commands/flake_info.nix new file mode 100644 index 0000000..3066369 --- /dev/null +++ b/flake-info/src/commands/flake_info.nix @@ -0,0 +1,146 @@ +{ flake }: +let + resolved = builtins.getFlake (toString flake); + + nixpkgs = (import {}); + lib = nixpkgs.lib; + + + default = drv: attr: default: if drv ? ${attr} then drv.${attr} else default; + + # filter = lib.filterAttrs (key: _ : key == "apps" || key == "packages"); + + withSystem = fn: lib.mapAttrs (system: drvs: (fn system drvs)); + isValid = d: + let + r = builtins.tryEval (lib.isDerivation d && ! (lib.attrByPath [ "meta" "broken" ] false d) && builtins.seq d.name true && d ? outputs); + in + r.success && r.value; + all = pkgs: + let + validPkgs = lib.filterAttrs (k: v: isValid v) pkgs; + in + validPkgs; + + + + readPackages = system: drvs: lib.mapAttrsToList ( + attribute_name: drv: ( + # if isValid drv then + { + attribute_name = attribute_name; + system = system; + name = drv.name; + # TODO consider using `builtins.parseDrvName` + version = default drv "version" ""; + outputs = drv.outputs; + # paths = builtins.listToAttrs ( map (output: {name = output; value = drv.${output};}) drv.outputs ); + } + // lib.optionalAttrs (drv ? meta && drv.meta ? description) { inherit (drv.meta) description; } + // lib.optionalAttrs (drv ? meta && drv.meta ? license) { inherit (drv.meta) license; } + + # else {} + ) + ) (all drvs); + readApps = system: apps: lib.mapAttrsToList ( + attribute_name: app: ( + { + attribute_name = attribute_name; + system = system; + } + // lib.optionalAttrs (app ? outPath) { bin = app.outPath; } + // lib.optionalAttrs (app ? program) { bin = app.program; } + // lib.optionalAttrs (app ? type) { type = app.type; } + ) + ) apps; + + readOptions = modules: isNixOS: let + + declarations = module: ( + lib.evalModules { + modules = (if lib.isList module then module else [ module ]) ++ [ + ( + { ... }: { + _module.check = false; + nixpkgs.system = lib.mkDefault "x86_64-linux"; + nixpkgs.config.allowBroken = true; + } + ) + ]; + } + ).options; + + cleanUpOption = module: opt: + let + applyOnAttr = n: f: lib.optionalAttrs (lib.hasAttr n opt) { ${n} = f opt.${n}; }; + # mkDeclaration = decl: rec { + # path = stripModulePathPrefixes decl; + # url = mkModuleUrl path; + # channelPath = "${channelName}/${path}"; + # }; + # Replace functions by the string + substFunction = x: + if builtins.isAttrs x then + lib.mapAttrs (name: substFunction) x + else if builtins.isList x then + map substFunction x + else if lib.isFunction x then + "" + else + x; + in + opt + // applyOnAttr "example" substFunction + // applyOnAttr "default" substFunction + // applyOnAttr "type" substFunction + // lib.optionalAttrs (!isNixOS) { flake = [ flake module ]; }; + # // applyOnAttr "declarations" (map mkDeclaration) + + + options = lib.mapAttrs ( + attr: module: let + list = lib.optionAttrSetToDocList (declarations module); + in + map (cleanUpOption attr) (lib.filter (x: !x.internal) list ) + ) modules; + in + lib.flatten (builtins.attrValues options); + + + read = reader: set: lib.flatten (lib.attrValues (withSystem reader set)); + + legacyPackages' = read readPackages (default resolved "legacyPackages" {}); + packages' = read readPackages (default resolved "packages" {}); + + apps' = read readApps (default resolved "apps" {}); + + + collectSystems = lib.lists.foldr ( + drv@{ attribute_name, system, ... }: set: + let + present = default set "${attribute_name}" ({ platforms = []; } // drv); + + drv' = present // { + platforms = present.platforms ++ [ system ]; + }; + drv'' = removeAttrs drv' [ "system" ]; + in + set // { + ${attribute_name} = drv''; + } + ) {}; + +in + +rec { + legacyPackages = lib.attrValues (collectSystems legacyPackages'); + packages = lib.attrValues (collectSystems packages'); + apps = lib.attrValues (collectSystems apps'); + options = readOptions (default resolved "nixosModules" {}) false; + nixos-options = readOptions ( + { + "nixos" = import "${builtins.fetchTarball { url = flake; }}/nixos/modules/module-list.nix"; + } + ) true; + all = packages ++ apps ++ options; +} diff --git a/flake-info/src/commands/mod.rs b/flake-info/src/commands/mod.rs new file mode 100644 index 0000000..4cc1efb --- /dev/null +++ b/flake-info/src/commands/mod.rs @@ -0,0 +1,8 @@ +mod nix_flake_attrs; +mod nix_flake_info; +mod nix_gc; +mod nixpkgs_info; +pub use nix_flake_attrs::get_derivation_info; +pub use nix_flake_info::get_flake_info; +pub use nix_gc::run_gc; +pub use nixpkgs_info::{get_nixpkgs_info, get_nixpkgs_options}; diff --git a/flake-info/src/commands/nix_flake_attrs.rs b/flake-info/src/commands/nix_flake_attrs.rs new file mode 100644 index 0000000..8f048be --- /dev/null +++ b/flake-info/src/commands/nix_flake_attrs.rs @@ -0,0 +1,51 @@ +use crate::data::import::{FlakeEntry, Kind}; +use anyhow::{Context, Result}; +use command_run::{Command, LogTo}; +use log::debug; +use std::fmt::Display; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; + +const SCRIPT: &str = include_str!("flake_info.nix"); +const ARGS: [&str; 3] = ["eval", "--json", "--no-write-lock-file"]; + +/// Uses `nix` to fetch the provided flake and read general information +/// about it using `nix flake info` +pub fn get_derivation_info + Display>( + flake_ref: T, + kind: Kind, + temp_store: bool, + extra: &[String], +) -> Result> { + let script_dir = tempfile::tempdir()?; + let script_path = script_dir.path().join("extract.nix"); + writeln!(File::create(&script_path)?, "{}", SCRIPT)?; + + let mut command = Command::with_args("nix", ARGS.iter()); + command.add_arg_pair("-f", script_path.as_os_str()); + let command = command.add_args(["--arg", "flake", flake_ref.as_ref()].iter()); + let command = command.add_arg(kind.as_ref()); + if temp_store { + let temp_store_path = PathBuf::from("/tmp/flake-info-store"); + if !temp_store_path.exists() { + std::fs::create_dir_all(&temp_store_path) + .with_context(|| "Couldn't create temporary store path")?; + } + command.add_arg_pair("--store", temp_store_path.canonicalize()?); + } + command.add_args(extra); + let mut command = command.enable_capture(); + command.log_to = LogTo::Log; + command.log_output_on_error = true; + + let parsed: Result> = command + .run() + .with_context(|| format!("Failed to gather information about {}", flake_ref)) + .and_then(|o| { + debug!("stderr: {}", o.stderr_string_lossy()); + serde_json::de::from_str(&o.stdout_string_lossy()) + .with_context(|| format!("Failed to analyze flake {}", flake_ref)) + }); + parsed +} diff --git a/flake-info/src/commands/nix_flake_info.rs b/flake-info/src/commands/nix_flake_info.rs new file mode 100644 index 0000000..e7cdbbf --- /dev/null +++ b/flake-info/src/commands/nix_flake_info.rs @@ -0,0 +1,39 @@ +use anyhow::{Context, Result}; +use command_run::{Command, LogTo}; +use std::fmt::Display; +use std::path::PathBuf; + +use crate::data::Flake; + +/// Uses `nix` to fetch the provided flake and read general information +/// about it using `nix flake info` +pub fn get_flake_info + Display>( + flake_ref: T, + temp_store: bool, + extra: &[String], +) -> Result { + let args = ["flake", "info", "--json", "--no-write-lock-file"].iter(); + let mut command = Command::with_args("nix", args); + let command = command.add_arg(flake_ref.as_ref()); + if temp_store { + let temp_store_path = PathBuf::from("/tmp/flake-info-store"); + if !temp_store_path.exists() { + std::fs::create_dir_all(&temp_store_path) + .with_context(|| "Couldn't create temporary store path")?; + } + command.add_arg_pair("--store", temp_store_path.canonicalize()?); + } + command.add_args(extra); + let mut command = command.enable_capture(); + command.log_to = LogTo::Log; + command.log_output_on_error = true; + + command + .run() + .with_context(|| format!("Failed to gather information about {}", flake_ref)) + .and_then(|o| { + let deserialized: Result = + serde_json::de::from_str(o.stdout_string_lossy().to_string().as_str()); + Ok(deserialized?.resolve_name()) + }) +} diff --git a/flake-info/src/commands/nix_gc.rs b/flake-info/src/commands/nix_gc.rs new file mode 100644 index 0000000..ad968c3 --- /dev/null +++ b/flake-info/src/commands/nix_gc.rs @@ -0,0 +1,54 @@ +use anyhow::{Context, Result}; +use log::warn; +use std::{ + path::{self, PathBuf}, + process::Command, +}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum GCError { + #[error("Unexpected exit status: {0}")] + ExitStatusError(String), +} + +pub fn run_gc() -> Result<()> { + let temp_store_path = PathBuf::from("/tmp/flake-info-store"); + if !temp_store_path.exists() { + warn!("Temporary store path does not exist, was a temporary store used?"); + return Ok(()); + } + + let mut command = Command::new("nix-store"); + command.args(&[ + "--gc", + "--store", + temp_store_path.canonicalize()?.to_str().unwrap(), + ]); + + dbg!(&command); + + let mut child = command + .spawn() + .with_context(|| "failed to start `nix-store gc` subprocess")?; + + let result = child.wait()?; + + if !result.success() { + return Err(GCError::ExitStatusError(format!("Code: {}", result.code().unwrap())).into()); + } + + std::fs::remove_dir_all(temp_store_path).with_context(|| "failed to clean up temp dir")?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gc() { + run_gc().unwrap(); + } +} diff --git a/flake-info/src/commands/nixpkgs_info.rs b/flake-info/src/commands/nixpkgs_info.rs new file mode 100644 index 0000000..f544688 --- /dev/null +++ b/flake-info/src/commands/nixpkgs_info.rs @@ -0,0 +1,96 @@ +use anyhow::{Context, Result}; +use std::io::Write; +use std::{collections::HashMap, fmt::Display, fs::File}; + +use command_run::Command; +use log::{debug, error}; + +use crate::data::import::{NixOption, NixpkgsEntry, Package}; + +const NIXPKGS_SCRIPT: &str = include_str!("packages-config.nix"); +const FLAKE_INFO_SCRIPT: &str = include_str!("flake_info.nix"); + +pub fn get_nixpkgs_info + Display>(nixpkgs_channel: T) -> Result> { + let script_dir = tempfile::tempdir()?; + let script_path = script_dir.path().join("packages-config.nix"); + writeln!(File::create(&script_path)?, "{}", NIXPKGS_SCRIPT)?; + + let mut command = Command::new("nix-env"); + let command = command.enable_capture(); + let command = command.add_args(&[ + "-f", + "", + "-I", + format!("nixpkgs={}", nixpkgs_channel.as_ref()).as_str(), + "--arg", + "config", + format!("import {}", script_path.to_str().unwrap()).as_str(), + "-qa", + "--json", + ]); + + let parsed: Result> = command + .run() + .with_context(|| { + format!( + "Failed to gather information about nixpkgs {}", + nixpkgs_channel.as_ref() + ) + }) + .and_then(|o| { + debug!("stderr: {}", o.stderr_string_lossy()); + let attr_set: HashMap = + serde_json::de::from_str(&o.stdout_string_lossy())?; + Ok(attr_set + .into_iter() + .map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package }) + .collect()) + }); + + parsed +} + +pub fn get_nixpkgs_options + Display>( + nixpkgs_channel: T, +) -> Result> { + let script_dir = tempfile::tempdir()?; + let script_path = script_dir.path().join("flake_info.nix"); + writeln!(File::create(&script_path)?, "{}", FLAKE_INFO_SCRIPT)?; + + let mut command = Command::new("nix"); + let command = command.enable_capture(); + let mut command = command.add_args(&[ + "eval", + "--json", + "-f", + script_path.to_str().unwrap(), + "--arg", + "flake", + nixpkgs_channel.as_ref(), + "nixos-options", + ]); + + // Nix might fail to evaluate some options that reference insecure packages + let mut env = HashMap::new(); + env.insert("NIXPKGS_ALLOW_INSECURE".into(), "1".into()); + env.insert("NIXPKGS_ALLOW_UNFREE".into(), "1".into()); + + command.env = env; + + let parsed = command.run().with_context(|| { + format!( + "Failed to gather information about nixpkgs {}", + nixpkgs_channel.as_ref() + ) + }); + + if let Err(ref e) = parsed { + error!("Command error: {}", e); + } + + parsed.and_then(|o| { + debug!("stderr: {}", o.stderr_string_lossy()); + let attr_set: Vec = serde_json::de::from_str(&o.stdout_string_lossy())?; + Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect()) + }) +} diff --git a/flake-info/src/commands/packages-config.nix b/flake-info/src/commands/packages-config.nix new file mode 100644 index 0000000..fcb2019 --- /dev/null +++ b/flake-info/src/commands/packages-config.nix @@ -0,0 +1,49 @@ +{ + # Ensures no aliases are in the results. + allowAliases = false; + + # Also list unfree packages + allowUnfree = true; + + # Enable recursion into attribute sets that nix-env normally doesn't look into + # so that we can get a more complete picture of the available packages for the + # purposes of the index. + packageOverrides = super: + let + recurseIntoAttrs = sets: + super.lib.genAttrs + (builtins.filter (set: builtins.hasAttr set super) sets) + (set: super.recurseIntoAttrs (builtins.getAttr set super)); + in recurseIntoAttrs [ + "roundcubePlugins" + "emscriptenfastcompPackages" + "fdbPackages" + "nodePackages_latest" + "nodePackages" + "platformioPackages" + "haskellPackages" + "haskell.compiler" + "idrisPackages" + "sconsPackages" + "gns3Packages" + "quicklispPackagesClisp" + "quicklispPackagesSBCL" + "rPackages" + "apacheHttpdPackages_2_4" + "zabbix44" + "zabbix40" + "zabbix30" + "fusePackages" + "nvidiaPackages" + "sourceHanPackages" + "atomPackages" + "emacs25Packages" + "emacs26Packages" + "emacs25.pkgs" + "emacs26.pkgs" + "emacs27.pkgs" + "steamPackages" + "ut2004Packages" + "zeroadPackages" + ]; +} diff --git a/flake-info/src/data/export.rs b/flake-info/src/data/export.rs new file mode 100644 index 0000000..1e1a28f --- /dev/null +++ b/flake-info/src/data/export.rs @@ -0,0 +1,369 @@ +/// This module defines the unified putput format as expected by the elastic search +/// Additionally, we implement converseions from the two possible input formats, i.e. +/// Flakes, or Nixpkgs. +use std::path::PathBuf; + +use crate::data::import::NixOption; +use serde::{Deserialize, Serialize}; + +use super::{ + import, + system::System, + utility::{AttributeQuery, Flatten, OneOrMany, Reverse}, +}; + +type Flake = super::Flake; + +#[allow(non_snake_case)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct License { + url: Option, + fullName: String, +} + +impl From for License { + #[allow(non_snake_case)] + fn from(license: import::License) -> Self { + match license { + import::License::None { .. } => License { + url: None, + fullName: "No License Specified".to_string(), + }, + import::License::Simple { license } => License { + url: None, + fullName: license, + }, + import::License::Full { fullName, url, .. } => License { url, fullName }, + import::License::Url { url } => License { + url: Some(url), + fullName: "No Name".into(), + }, + } + } +} + +// ----- Unified derivation representation + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum Derivation { + #[serde(rename = "package")] + Package { + package_attr_name: String, + package_attr_name_reverse: Reverse, + package_attr_name_query: AttributeQuery, + package_attr_name_query_reverse: Reverse, + package_attr_set: String, + package_attr_set_reverse: Reverse, + package_pname: String, + package_pname_reverse: Reverse, + package_pversion: String, + package_platforms: Vec, + package_outputs: Vec, + package_license: Vec, + package_license_set: Vec, + package_maintainers: Vec, + package_maintainers_set: Vec, + package_description: Option, + package_description_reverse: Option>, + package_longDescription: Option, + package_longDescription_reverse: Option>, + package_hydra: (), + package_system: String, + package_homepage: Vec, + package_position: Option, + }, + #[serde(rename = "app")] + App { + app_attr_name: String, + app_platforms: Vec, + + app_type: Option, + + app_bin: Option, + }, + #[serde(rename = "option")] + Option { + option_source: Option, + option_name: String, + option_name_reverse: Reverse, + option_name_query: AttributeQuery, + option_name_query_reverse: Reverse, + + option_description: Option, + option_description_reverse: Option>, + + option_type: Option, + + option_default: Option, + + option_example: Option, + + option_flake: Option<(String, String)>, + }, +} + +// ----- Conversions + +impl From<(import::FlakeEntry, super::Flake)> for Derivation { + fn from((d, f): (import::FlakeEntry, super::Flake)) -> Self { + match d { + import::FlakeEntry::Package { + attribute_name, + name, + version, + platforms, + outputs, + description, + license, + } => { + let package_attr_set: Vec<_> = attribute_name.split(".").collect(); + let package_attr_set: String = (if package_attr_set.len() > 1 { + package_attr_set[0] + } else { + "No package set" + }) + .into(); + + let package_attr_set_reverse = Reverse(package_attr_set.clone()); + + let package_license: Vec = vec![license.into()]; + let package_license_set: Vec = package_license + .iter() + .clone() + .map(|l| l.fullName.to_owned()) + .collect(); + + let maintainer: Maintainer = f.into(); + + Derivation::Package { + package_attr_name_query: AttributeQuery::new(&attribute_name), + package_attr_name_query_reverse: Reverse(AttributeQuery::new(&attribute_name)), + package_attr_name: attribute_name.clone(), + package_attr_name_reverse: Reverse(attribute_name), + package_attr_set, + package_attr_set_reverse, + package_pname: name.clone(), + package_pname_reverse: Reverse(name), + package_pversion: version, + package_platforms: platforms, + package_outputs: outputs, + package_license, + package_license_set, + package_description: description.clone(), + package_maintainers: vec![maintainer.clone()], + package_maintainers_set: maintainer.name.map_or(vec![], |n| vec![n]), + package_description_reverse: description.map(Reverse), + package_longDescription: None, + package_longDescription_reverse: None, + package_hydra: (), + package_system: String::new(), + package_homepage: Vec::new(), + package_position: None, + } + } + import::FlakeEntry::App { + bin, + attribute_name, + platforms, + app_type, + } => Derivation::App { + app_attr_name: attribute_name, + app_platforms: platforms, + app_bin: bin, + app_type, + }, + import::FlakeEntry::Option(option) => option.into(), + } + } +} + +impl From for Derivation { + fn from(entry: import::NixpkgsEntry) -> Self { + match entry { + import::NixpkgsEntry::Derivation { attribute, package } => { + let package_attr_set: Vec<_> = attribute.split(".").collect(); + let package_attr_set: String = (if package_attr_set.len() > 1 { + package_attr_set[0] + } else { + "No package set" + }) + .into(); + + let package_attr_set_reverse = Reverse(package_attr_set.clone()); + + let package_license: Vec<_> = package + .meta + .license + .map(OneOrMany::into_list) + .unwrap_or_default() + .into_iter() + .map(|sos| sos.0.into()) + .collect(); + + let package_license_set = package_license + .iter() + .map(|l: &License| l.fullName.to_owned()) + .collect(); + + let package_maintainers = package + .meta + .maintainers + .map_or(Default::default(), Flatten::flatten); + + let package_maintainers_set = package_maintainers + .iter() + .filter(|m| m.name.is_some()) + .map(|m| m.name.to_owned().unwrap()) + .collect(); + + Derivation::Package { + package_attr_name: attribute.clone(), + package_attr_name_reverse: Reverse(attribute.clone()), + package_attr_name_query: AttributeQuery::new(&attribute), + package_attr_name_query_reverse: Reverse(AttributeQuery::new(&attribute)), + package_attr_set, + package_attr_set_reverse, + package_pname: package.pname.clone(), + package_pname_reverse: Reverse(package.pname), + package_pversion: package.version, + package_platforms: package + .meta + .platforms + .map(Flatten::flatten) + .unwrap_or_default(), + package_outputs: package.meta.outputs.unwrap_or_default(), + package_license, + package_license_set, + package_maintainers, + package_maintainers_set, + package_description: package.meta.description.clone(), + package_description_reverse: package.meta.description.map(Reverse), + package_longDescription: package.meta.long_description.clone(), + package_longDescription_reverse: package.meta.long_description.map(Reverse), + package_hydra: (), + package_system: package.system, + package_homepage: package + .meta + .homepage + .map_or(Default::default(), OneOrMany::into_list), + package_position: package.meta.position, + } + } + import::NixpkgsEntry::Option(option) => option.into(), + } + } +} + +impl From for Derivation { + fn from( + NixOption { + declarations, + description, + name, + option_type, + default, + example, + flake, + }: import::NixOption, + ) -> Self { + Derivation::Option { + option_source: declarations.get(0).map(Clone::clone), + option_name: name.clone(), + option_name_reverse: Reverse(name.clone()), + option_description: description.clone(), + option_description_reverse: description.map(Reverse), + option_default: default.map(|v| { + v.as_str().map_or_else( + || serde_json::to_string_pretty(&v).unwrap(), + |s| s.to_owned(), + ) + }), + option_example: example.map(|v| { + v.as_str().map_or_else( + || serde_json::to_string_pretty(&v).unwrap(), + |s| s.to_owned(), + ) + }), + option_flake: flake, + option_type, + option_name_query: AttributeQuery::new(&name), + option_name_query_reverse: Reverse(AttributeQuery::new(&name)), + } + } +} + +type Maintainer = import::Maintainer; + +impl From for Maintainer { + fn from(flake: super::Flake) -> Self { + let github = flake + .source + .and_then(|source| match source { + super::Source::Github { owner, .. } => Some(owner), + _ => None, + }) + .unwrap_or_else(|| "Maintainer Unknown".to_string()); + + Maintainer { + github: Some(github), + email: None, + name: None, + } + } +} + +// ----- output type + +/// Export type that brings together derivation and optional flake info +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Export { + #[serde(flatten)] + flake: Option, + + #[serde(flatten)] + item: Derivation, +} + +impl Export { + /// Construct Export from Flake and Flake entry + pub fn flake(flake: Flake, item: import::FlakeEntry) -> Self { + Self { + flake: Some(flake.clone()), + item: Derivation::from((item, flake)), + } + } + + /// Construct Export from NixpkgsEntry + pub fn nixpkgs(item: import::NixpkgsEntry) -> Self { + Self { + flake: None, + item: Derivation::from(item), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_option() { + let option: NixOption = serde_json::from_str(r#" + { + "declarations":["/nix/store/s1q1238ahiks5a4g6j6qhhfb3rlmamvz-source/nixos/modules/system/boot/luksroot.nix"], + "default":"", + "description":"Commands that should be run right after we have mounted our LUKS device.\n", + "example":"oneline\ntwoline\nthreeline\n", + "internal":false, + "loc":["boot","initrd","luks","devices","","postOpenCommands"], + "name":"boot.initrd.luks.devices..postOpenCommands", + "readOnly":false,"type": + "strings concatenated with \"\\n\"","visible":true + }"#).unwrap(); + + let option: Derivation = option.into(); + + println!("{}", serde_json::to_string_pretty(&option).unwrap()); + } +} diff --git a/flake-info/src/data/flake.rs b/flake-info/src/data/flake.rs new file mode 100644 index 0000000..119dcf4 --- /dev/null +++ b/flake-info/src/data/flake.rs @@ -0,0 +1,87 @@ +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; + +use super::Source; + +/// Holds general infoamtion about a flake +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Flake { + #[serde( + rename(serialize = "flake_description"), + skip_serializing_if = "Option::is_none" + )] + pub description: Option, + #[serde(rename(serialize = "flake_path"), skip_serializing)] + pub path: PathBuf, + #[serde(rename(serialize = "flake_resolved"))] + pub resolved: Repo, + + #[serde(rename(serialize = "flake_name"), skip_deserializing)] + pub name: String, + + pub revision: String, + + #[serde( + skip_deserializing, + rename(serialize = "flake_source"), + skip_serializing_if = "Option::is_none" + )] + pub source: Option, +} + +impl Flake { + pub(crate) fn resolve_name(mut self) -> Self { + self.name = match &self.resolved { + Repo::Git { .. } => Default::default(), + Repo::GitHub { repo, .. } => repo.clone(), + Repo::Gitlab { repo, .. } => repo.clone(), + }; + self + } +} + +/// Information about the flake origin +/// Supports (local/raw) Git, GitHub and Gitlab repos +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum Repo { + Git { url: PathBuf }, + GitHub { owner: String, repo: String }, + Gitlab { owner: String, repo: String }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gitlab_flake() { + let nix_info_out = r#"{"description":"neuropil is a secure messaging library for IoT, robotics and more.","lastModified":1616059502,"locked":{"lastModified":1616059502,"narHash":"sha256-fHB1vyjDXQq/E2/Xb6Xs3caAAc0VkUlnzu5kl/PvFW4=","owner":"pi-lar","repo":"neuropil","rev":"9e2f634ffa45da3f5feb158a12ee32e1673bfe35","type":"gitlab"},"original":{"owner":"pi-lar","repo":"neuropil","type":"gitlab"},"originalUrl":"gitlab:pi-lar/neuropil","path":"/nix/store/z4fp2fc9hca40nnvxi0116pfbrla5zgl-source","resolved":{"owner":"pi-lar","repo":"neuropil","type":"gitlab"},"resolvedUrl":"gitlab:pi-lar/neuropil","revision":"9e2f634ffa45da3f5feb158a12ee32e1673bfe35","url":"gitlab:pi-lar/neuropil/9e2f634ffa45da3f5feb158a12ee32e1673bfe35"}"#; + + assert_eq!( + serde_json::de::from_str::(nix_info_out).unwrap(), + Flake { + description: Some( + "neuropil is a secure messaging library for IoT, robotics and more.".into() + ), + path: "/nix/store/z4fp2fc9hca40nnvxi0116pfbrla5zgl-source".into(), + resolved: Repo::Gitlab { + owner: "pi-lar".into(), + repo: "neuropil".into() + }, + name: "".into(), + source: None, + revision: "9e2f634ffa45da3f5feb158a12ee32e1673bfe35".into() + } + ); + + assert_eq!( + serde_json::de::from_str::(nix_info_out) + .unwrap() + .resolve_name() + .name, + "neuropil" + ); + } +} diff --git a/flake-info/src/data/import.rs b/flake-info/src/data/import.rs new file mode 100644 index 0000000..1cf1ebd --- /dev/null +++ b/flake-info/src/data/import.rs @@ -0,0 +1,321 @@ +use std::fmt::{self, write, Display}; +use std::marker::PhantomData; +use std::{path::PathBuf, str::FromStr}; + +use serde::de::{self, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; +use thiserror::Error; + +use super::system::System; +use super::utility::{Flatten, OneOrMany}; + +/// Holds information about a specific derivation +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum FlakeEntry { + /// A package as it may be defined in a flake + /// + /// Note: As flakes do not enforce any particular structure to be necessarily + /// present, the data represented is an idealization that _should_ match in + /// most cases and is open to extension. + Package { + attribute_name: String, + name: String, + version: String, + platforms: Vec, + outputs: Vec, + description: Option, + #[serde(deserialize_with = "string_or_struct", default)] + license: License, + }, + /// An "application" that can be called using nix run <..> + App { + bin: Option, + attribute_name: String, + platforms: Vec, + app_type: Option, + }, + /// an option defined in a module of a flake + Option(NixOption), +} + +/// The representation of an option that is part of some module and can be used +/// in some nixos configuration +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NixOption { + /// Location of the defining module(s) + pub declarations: Vec, + + pub description: Option, + pub name: String, + #[serde(rename = "type")] + /// Nix generated description of the options type + pub option_type: Option, + pub default: Option, + pub example: Option, + + /// If defined in a flake, contains defining flake and module + pub flake: Option<(String, String)>, +} + +/// Package as defined in nixpkgs +/// These packages usually have a "more" homogenic structure that is given by +/// nixpkgs +/// note: This is the parsing module that deals with nested input. A flattened, +/// unified representation can be found in [crate::data::export::Derivation] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Package { + pub pname: String, + pub version: String, + pub system: String, + pub meta: Meta, +} + +/// The nixpkgs output lists attribute names as keys of a map. +/// Name and Package definition are combined using this struct +#[derive(Debug, Clone)] +pub enum NixpkgsEntry { + Derivation { attribute: String, package: Package }, + Option(NixOption), +} + +/// Most information about packages in nixpkgs is contained in the meta key +/// This struct represents a subset of that metadata +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Meta { + #[serde(rename = "outputsToInstall")] + pub outputs: Option>, + pub license: Option>>, + pub maintainers: Option>, + pub homepage: Option>, + pub platforms: Option>, + pub position: Option, + pub description: Option, + #[serde(rename = "longDescription")] + pub long_description: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Maintainer { + pub name: Option, + pub github: Option, + pub email: Option, +} + +/// The type of derivation (placed in packages. or apps.) +/// Used to command the extraction script +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub enum Kind { + App, + Package, + Option, + All, +} + +impl AsRef for Kind { + fn as_ref(&self) -> &str { + match self { + Kind::App => "app", + Kind::Package => "packages", + Kind::Option => "options", + Kind::All => "all", + } + } +} + +impl Display for Kind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.as_ref()) + } +} + +#[derive(Debug, Error)] +pub enum ParseKindError { + #[error("Failed to parse kind: {0}")] + UnknownKind(String), +} + +impl FromStr for Kind { + type Err = ParseKindError; + + fn from_str(s: &str) -> Result { + let kind = match s { + "app" => Kind::App, + "packages" => Kind::Package, + "options" => Kind::Option, + "all" => Kind::All, + _ => return Err(ParseKindError::UnknownKind(s.into())), + }; + Ok(kind) + } +} + +impl Default for Kind { + fn default() -> Self { + Kind::All + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct StringOrStruct(pub T); + +impl<'de, T> Deserialize<'de> for StringOrStruct +where + T: Deserialize<'de> + FromStr, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(StringOrStruct(string_or_struct(deserializer)?)) + } +} + +/// Different representations of the licence attribute +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum License { + None { + #[serde(skip_serializing)] + license: (), + }, + Simple { + license: String, + }, + Full { + fullName: String, + // shortName: String, + url: Option, + }, + Url { + url: String, + }, +} + +impl Default for License { + fn default() -> Self { + License::None { license: () } + } +} + +impl FromStr for License { + // This implementation of `from_str` can never fail, so use the impossible + // `Void` type as the error type. + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(License::Simple { + license: s.to_string(), + }) + } +} + +/// Deserialization helper that parses an item using either serde or fromString +fn string_or_struct<'de, T, D>(deserializer: D) -> Result +where + T: Deserialize<'de> + FromStr, + D: Deserializer<'de>, +{ + // This is a Visitor that forwards string types to T's `FromStr` impl and + // forwards map types to T's `Deserialize` impl. The `PhantomData` is to + // keep the compiler from complaining about T being an unused generic type + // parameter. We need T in order to know the Value type for the Visitor + // impl. + struct StringOrStruct(PhantomData T>); + + impl<'de, T> Visitor<'de> for StringOrStruct + where + T: Deserialize<'de> + FromStr, + { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("string or map") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(FromStr::from_str(value).unwrap()) + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + // `MapAccessDeserializer` is a wrapper that turns a `MapAccess` + // into a `Deserializer`, allowing it to be used as the input to T's + // `Deserialize` implementation. T then deserializes itself using + // the entries from the map visitor. + Deserialize::deserialize(de::value::MapAccessDeserializer::new(map)) + } + } + + deserializer.deserialize_any(StringOrStruct(PhantomData)) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use serde_json::Value; + + use super::*; + + #[test] + fn test_nixpkgs_deserialize() { + let json = r#" + { + "nixpkgs-unstable._0verkill": { + "name": "0verkill-unstable-2011-01-13", + "pname": "0verkill-unstable", + "version": "2011-01-13", + "system": "x86_64-darwin", + "meta": { + "available": true, + "broken": false, + "description": "ASCII-ART bloody 2D action deathmatch-like game", + "homepage": "https://github.com/hackndev/0verkill", + "insecure": false, + "license": { + "fullName": "GNU General Public License v2.0 only", + "shortName": "gpl2Only", + "spdxId": "GPL-2.0-only", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + }, + "maintainers": [ + { + "email": "torres.anderson.85@protonmail.com", + "github": "AndersonTorres", + "githubId": 5954806, + "name": "Anderson Torres" + } + ], + "name": "0verkill-unstable-2011-01-13", + "outputsToInstall": [ + "out" + ], + "platforms": [ + "powerpc64-linux", + "powerpc64le-linux", + "riscv32-linux", + "riscv64-linux" + ], + "position": "/nix/store/97lxf2n6zip41j5flbv6b0928mxv9za8-nixpkgs-unstable-21.03pre268853.d9c6f13e13f/nixpkgs-unstable/pkgs/games/0verkill/default.nix:34", + "unfree": false, + "unsupported": false + } + } + } + "#; + + let map: HashMap = serde_json::from_str(json).unwrap(); + + let _: Vec = map + .into_iter() + .map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package }) + .collect(); + } +} diff --git a/flake-info/src/data/mod.rs b/flake-info/src/data/mod.rs new file mode 100644 index 0000000..0c65104 --- /dev/null +++ b/flake-info/src/data/mod.rs @@ -0,0 +1,10 @@ +mod export; +mod flake; +pub mod import; +mod source; +mod system; +mod utility; + +pub use export::Export; +pub use flake::{Flake, Repo}; +pub use source::{FlakeRef, Hash, Nixpkgs, Source}; diff --git a/flake-info/src/data/source.rs b/flake-info/src/data/source.rs new file mode 100644 index 0000000..663c41d --- /dev/null +++ b/flake-info/src/data/source.rs @@ -0,0 +1,111 @@ +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, File}, + path::Path, +}; + +pub type Hash = String; +pub type FlakeRef = String; + +/// Information about the flake origin +/// Supports (local/raw) Git, GitHub and Gitlab repos +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum Source { + Github { + owner: String, + repo: String, + description: Option, + #[serde(rename(deserialize = "hash"))] + git_ref: Option, + }, + Gitlab { + owner: String, + repo: String, + git_ref: Option, + }, + Git { + url: String, + }, + Nixpkgs(Nixpkgs), +} + +impl Source { + pub fn to_flake_ref(&self) -> FlakeRef { + match self { + Source::Github { + owner, + repo, + git_ref, + .. + } => format!( + "github:{}/{}{}", + owner, + repo, + git_ref + .as_ref() + .map_or("".to_string(), |f| format!("?ref={}", f)) + ), + Source::Gitlab { + owner, + repo, + git_ref, + } => format!( + "gitlab:{}/{}{}", + owner, + repo, + git_ref + .as_ref() + .map_or("".to_string(), |f| format!("?ref={}", f)) + ), + Source::Git { url } => url.to_string(), + Source::Nixpkgs(Nixpkgs { git_ref, .. }) => format!( + "https://api.github.com/repos/NixOS/nixpkgs/tarball/{}", + git_ref + ), + } + } + + pub fn read_sources_file(path: &Path) -> Result> { + let file = File::open(path).with_context(|| "Failed to open input file")?; + + Ok(serde_json::from_reader(file)?) + } + + pub async fn nixpkgs(channel: String) -> Result { + #[derive(Deserialize, Debug)] + struct ApiResult { + commit: Commit, + } + + #[derive(Deserialize, Debug)] + struct Commit { + sha: String, + } + + let git_ref = reqwest::Client::builder() + .user_agent("curl") // thank you github + .build()? + .get(format!( + "https://api.github.com/repos/nixos/nixpkgs/branches/nixos-{}", + channel + )) + .send() + .await? + .json::() + .await? + .commit + .sha; + + let nixpkgs = Nixpkgs { channel, git_ref }; + + Ok(nixpkgs) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Nixpkgs { + pub channel: String, + pub git_ref: String, +} diff --git a/flake-info/src/data/system.rs b/flake-info/src/data/system.rs new file mode 100644 index 0000000..8ef8bc3 --- /dev/null +++ b/flake-info/src/data/system.rs @@ -0,0 +1,38 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum System { + Plain(String), + Detailed { cpu: Cpu, kernel: Kernel }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Cpu { + family: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Kernel { + name: String, +} + +impl ToString for System { + fn to_string(&self) -> String { + match self { + System::Plain(system) => system.to_owned(), + System::Detailed { cpu, kernel } => format!("{}-{}", cpu.family, kernel.name), + } + } +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct InstancePlatform { + system: System, + version: String, +} + +#[cfg(test)] +mod tests { + use super::*; +} diff --git a/flake-info/src/data/utility.rs b/flake-info/src/data/utility.rs new file mode 100644 index 0000000..a07c2ea --- /dev/null +++ b/flake-info/src/data/utility.rs @@ -0,0 +1,237 @@ +use fancy_regex::Regex; +use lazy_static::lazy_static; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +#[derive(Debug, Clone, PartialEq)] +pub struct Reverse(pub T); + +pub trait Reversable { + fn reverse(&self) -> Self; +} + +impl Reversable for String { + fn reverse(&self) -> Self { + self.chars().rev().collect::() + } +} + +impl Reversable for Vec { + fn reverse(&self) -> Self { + self.iter().cloned().map(|item| item.reverse()).collect() + } +} + +impl Reversable for Reverse +where + T: Reversable + Serialize, +{ + fn reverse(&self) -> Self { + Reverse(self.0.reverse()) + } +} + +impl Serialize for Reverse +where + T: Reversable + Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.0.reverse().serialize(serializer) + } +} + +impl<'de, T> Deserialize<'de> for Reverse +where + T: Reversable + Serialize + Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + Ok(Reverse(T::deserialize(deserializer)?.reverse())) + } +} + +/// A utility type that can represent the presence of either a single associated +/// value or a list of those. Adding absence can be achieved by wrapping the type +/// in an [Option] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum OneOrMany { + #[serde(serialize_with = "list")] + One(T), + Many(Vec), +} + +impl OneOrMany { + pub fn into_list(self) -> Vec { + match self { + OneOrMany::One(one) => vec![one], + OneOrMany::Many(many) => many, + } + } +} + +/// A utility type that flattens lists of lists as seen with `maintainers` and `platforms` on selected packages +/// in an [Option] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Flatten { + #[serde(serialize_with = "list")] + Single(T), + Deep(Vec>), +} + +impl Flatten { + pub fn flatten(self) -> Vec { + match self { + Flatten::Single(s) => vec![s], + Flatten::Deep(v) => v.into_iter().map(Flatten::flatten).flatten().collect(), + } + } +} + +// TODO: use this or a to_ist function? +/// Serialization helper that serializes single elements as a list with a single +/// item +pub fn list(item: &T, s: S) -> Result +where + T: Serialize, + S: Serializer, +{ + s.collect_seq(vec![item].iter()) +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AttributeQuery(Vec); + +lazy_static! { + static ref QUERY: Regex = + Regex::new(".+?(?:(?<=[a-z])(?=[1-9A-Z])|(?<=[1-9A-Z])(?=[A-Z][a-z])|[._-]|$)").unwrap(); +} + +impl AttributeQuery { + pub fn new(attribute_name: &str) -> Self { + const SUFFIX: &[char] = &['-', '.', '_']; + + let matches = QUERY + .find_iter(attribute_name) + .map(|found| found.unwrap().as_str()) + .collect::>(); + + let tokens = (0..matches.len()) + .flat_map(|index| { + let (_, tokens) = matches.iter().skip(index).fold( + (String::new(), Vec::new()), + |(prev_parts, mut tokens), part| { + let token: String = prev_parts + part; + tokens.push(token.trim_end_matches(SUFFIX).to_owned()); + (token, tokens) + }, + ); + + tokens + }) + .collect::>(); + + AttributeQuery(tokens) + } + + pub fn query(&self) -> &[String] { + &self.0 + } +} + +impl Reversable for AttributeQuery { + fn reverse(&self) -> Self { + AttributeQuery(self.query().to_owned().reverse()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn attr_query_test() { + assert_eq!( + { + let mut q = AttributeQuery::new("services.nginx.extraConfig") + .query() + .to_owned(); + q.sort(); + q + }, + { + let mut ex = [ + "services.nginx.extraConfig", + "services.nginx.extra", + "services.nginx", + "services", + "nginx.extraConfig", + "nginx.extra", + "nginx", + "extraConfig", + "extra", + "Config", + ]; + ex.sort_unstable(); + ex + }, + ); + + assert_eq!( + { + let mut q = AttributeQuery::new("python37Packages.test1_name-test2") + .query() + .to_owned(); + q.sort(); + q + }, + { + let mut ex = [ + "python37Packages.test1_name-test2", + "python37Packages.test1_name-test", + "python37Packages.test1_name", + "python37Packages.test1", + "python37Packages.test", + "python37Packages", + "python37", + "python", + "37Packages.test1_name-test2", + "37Packages.test1_name-test", + "37Packages.test1_name", + "37Packages.test1", + "37Packages.test", + "37Packages", + "37", + "Packages.test1_name-test2", + "Packages.test1_name-test", + "Packages.test1_name", + "Packages.test1", + "Packages.test", + "Packages", + "test1_name-test2", + "test1_name-test", + "test1_name", + "test1", + "test", + "1_name-test2", + "1_name-test", + "1_name", + "1", + "name-test2", + "name-test", + "name", + "test2", + "test", + "2", + ]; + ex.sort_unstable(); + ex + } + ); + } +} diff --git a/flake-info/src/elastic.rs b/flake-info/src/elastic.rs new file mode 100644 index 0000000..35b76ed --- /dev/null +++ b/flake-info/src/elastic.rs @@ -0,0 +1,515 @@ +use std::{borrow::Borrow, collections::HashMap}; + +use clap::arg_enum; +pub use elasticsearch::http::transport::Transport; +use elasticsearch::{BulkOperation, Elasticsearch as Client, http::response::{self, Response}, indices::{IndicesCreateParts, IndicesDeleteAliasParts, IndicesDeleteParts, IndicesExistsParts, IndicesGetAliasParts, IndicesPutAliasParts, IndicesUpdateAliasesParts}}; +use lazy_static::lazy_static; +use log::{info, warn}; +use serde_json::{json, Value}; +use thiserror::Error; + +use crate::data::Export; +lazy_static! { + static ref MAPPING: Value = json!({ + "mappings": { + "properties": { + "type": {"type": "keyword"}, + "flake_name": { + "type": "text", + "analyzer": "english", + }, + "flake_description": { + "type": "text", + "analyzer": "english", + }, + "flake_resolved": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + }, + "repo": { + "type": "keyword" + }, + } + }, + "flake_source": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + }, + "repo": { + "type": "keyword" + }, + "desciption": { + "type": "text", + "analyzer": "english", + }, + "git_ref": { + "type": "keyword" + }, + "url": { + "type": "keyword" + }, + } + }, + "package_attr_name": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_attr_name_reverse": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_attr_name_query": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_attr_name_query_reverse": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_attr_set": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_attr_set_reverse": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_pname": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_pname_reverse": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_pversion": { + "type": "keyword" + }, + "package_platforms": { + "type": "keyword" + }, + "package_system": { + "type": "keyword" + }, + "package_position": { + "type": "text" + }, + "package_outputs": { + "type": "keyword" + }, + "package_description": { + "type": "text", + "analyzer": "english", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_description_reverse": { + "type": "text", + "analyzer": "english", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_longDescription": { + "type": "text", + "analyzer": "english", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_longDescription_reverse": { + "type": "text", + "analyzer": "english", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "package_license": { + "type": "nested", + "properties": { + "fullName": {"type": "text"}, + "url": {"type": "text"}}, + }, + "package_license_set": {"type": "keyword"}, + "package_maintainers": { + "type": "nested", + "properties": { + "name": {"type": "text"}, + "email": {"type": "text"}, + "github": {"type": "text"}, + }, + }, + "package_maintainers_set": {"type": "keyword"}, + "package_homepage": { + "type": "keyword" + }, + // Options fields + "option_name": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "option_name_reverse": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "option_name": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "option_name_reverse": { + "type": "keyword", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "option_description": { + "type": "text", + "analyzer": "english", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "option_description_reverse": { + "type": "text", + "analyzer": "english", + "fields": {"edge": {"type": "text", "analyzer": "edge"}}, + }, + "option_type": {"type": "keyword"}, + "option_default": {"type": "text"}, + "option_example": {"type": "text"}, + "option_source": {"type": "keyword"}, + } + }, + "settings": { + "analysis": { + "normalizer": { + "lowercase": {"type": "custom", "char_filter": [], "filter": ["lowercase"]} + }, + "tokenizer": { + "edge": { + "type": "edge_ngram", + "min_gram": 2, + "max_gram": 50, + "token_chars": [ + "letter", + "digit", + // Either we use them or we would need to strip them before that. + "punctuation", + "symbol", + ], + }, + }, + "analyzer": { + "edge": {"tokenizer": "edge", "filter": ["lowercase"]}, + "lowercase": { + "type": "custom", + "tokenizer": "keyword", + "filter": ["lowercase"], + }, + }, + } + } + }); +} + +#[derive(Default)] +pub struct Elasticsearch { + client: Client, +} + +#[derive(Error, Debug)] +pub enum ElasticsearchError { + #[error("Transport failed to initialize: {0}")] + TransportInitError(elasticsearch::Error), + + #[error("Failed to send push exports: {0}")] + PushError(elasticsearch::Error), + #[error("Push exports returned bad result: {0:?}")] + PushResponseError(response::Exception), + + #[error("Failed to iitialize index: {0}")] + InitIndexError(elasticsearch::Error), + #[error("Push exports returned bad result: {0:?}")] + InitResponseError(response::Exception), + + #[error("An unexpected error occured in the elastic search client: {0}")] + ClientError(elasticsearch::Error), + + #[error("Failed to serialize exported data: {0}")] + SerializationError(#[from] serde_json::Error), + + #[error("An index with the name \"{0}\" already exists and the (default) stategy is abort")] + IndexExistsError(String), +} + +impl Elasticsearch { + pub fn new(url: &str) -> Result { + let transport = + Transport::single_node(url).map_err(ElasticsearchError::TransportInitError)?; + let client = Client::new(transport); + Ok(Elasticsearch { client }) + } + pub fn with_transport(transport: Transport) -> Self { + let client = Client::new(transport); + Elasticsearch { client } + } + + pub async fn push_exports( + &self, + config: &Config<'_>, + exports: &[Export], + ) -> Result<(), ElasticsearchError> { + // let exports: Result, serde_json::Error> = exports.iter().map(serde_json::to_value).collect(); + // let exports = exports?; + let bodies = exports.chunks(10_000).map(|chunk| { + chunk + .iter() + .map(|e| BulkOperation::from(BulkOperation::index(e))) + }); + + for body in bodies { + let response = self + .client + .bulk(elasticsearch::BulkParts::Index(config.index)) + .body(body.collect()) + .send() + .await + .map_err(ElasticsearchError::PushError)?; + + dbg!(response) + .exception() + .await + .map_err(ElasticsearchError::ClientError)? + .map(ElasticsearchError::PushResponseError) + .map_or(Ok(()), Err)?; + } + + Ok(()) + } + + pub async fn ensure_index(&self, config: &Config<'_>) -> Result<(), ElasticsearchError> { + let exists = self.check_index(config).await?; + + if exists { + match config.exists_strategy { + ExistsStrategy::Abort => { + warn!( + "Index \"{}\" already exists, strategy is: Abort push", + config.index + ); + return Err(ElasticsearchError::IndexExistsError( + config.index.to_owned(), + )); + } + ExistsStrategy::Ignore => { + warn!( + "Index \"{}\" already exists, strategy is: Ignore, proceed push", + config.index + ); + return Ok(()); + } + ExistsStrategy::Recreate => { + warn!( + "Index \"{}\" already exists, strategy is: Recreate index", + config.index + ); + self.clear_index(config).await?; + } + } + } + + let response = self + .client + .indices() + .create(IndicesCreateParts::Index(config.index)) + .body(MAPPING.as_object()) + .send() + .await + .map_err(ElasticsearchError::InitIndexError)?; + + dbg!(response) + .exception() + .await + .map_err(ElasticsearchError::ClientError)? + .map(ElasticsearchError::PushResponseError) + .map_or(Ok(()), Err)?; + + Ok(()) + } + + pub async fn check_index(&self, config: &Config<'_>) -> Result { + let response = self + .client + .indices() + .exists(IndicesExistsParts::Index(&[config.index])) + .send() + .await + .map_err(ElasticsearchError::InitIndexError)?; + + Ok(response.status_code() == 200) + } + + pub async fn clear_index(&self, config: &Config<'_>) -> Result<(), ElasticsearchError> { + let response = self + .client + .indices() + .delete(IndicesDeleteParts::Index(&[config.index])) + .send() + .await + .map_err(ElasticsearchError::InitIndexError)?; + + dbg!(response) + .exception() + .await + .map_err(ElasticsearchError::ClientError)? + .map(ElasticsearchError::PushResponseError) + .map_or(Ok(()), Err) + } + + pub async fn write_alias( + &self, + config: &Config<'_>, + index: &str, + alias: &str, + ) -> Result<(), ElasticsearchError> { + // delete old alias + info!("Try deletig old alias"); + let response = self.client.indices().get_alias(IndicesGetAliasParts::Name(&[alias])).send().await + .map_err(ElasticsearchError::InitIndexError)?; + let indices = response.json::>().await.map_err(ElasticsearchError::InitIndexError)?.keys().cloned().collect::>(); + + self + .client + .indices() + .delete_alias(IndicesDeleteAliasParts::IndexName(&indices.iter().map(AsRef::as_ref).collect::>(), &[alias])) + .send() + .await + .map_err(ElasticsearchError::InitIndexError)?; + + // put new alias + info!("Putting new alias"); + let response = self + .client + .indices() + .put_alias(IndicesPutAliasParts::IndexName(&[index], alias)) + .send() + .await + .map_err(ElasticsearchError::InitIndexError)?; + + dbg!(response) + .exception() + .await + .map_err(ElasticsearchError::ClientError)? + .map(ElasticsearchError::PushResponseError) + .map_or(Ok(()), Err) + } +} + +#[derive(Debug)] +pub struct Config<'a> { + pub index: &'a str, + pub exists_strategy: ExistsStrategy, +} + +arg_enum! { + /// Different strategies to deal with eisting indices + /// Abort: cancel push, return with an error + /// Ignore: Reuse existing index, appending new data + /// Recreate: Drop the existing index and start with a new one + #[derive(Debug, Clone, Copy)] + pub enum ExistsStrategy { + Abort, + Ignore, + Recreate, + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + use crate::{ + data::{self, import::Kind}, + process_flake, + }; + + #[tokio::test] + async fn test_delete() -> Result<(), Box> { + let es = Elasticsearch::new("http://localhost:9200").unwrap(); + let config = &Config { + index: "flakes_index", + exists_strategy: ExistsStrategy::Ignore, + }; + es.ensure_index(config).await?; + es.clear_index(config).await?; + + let exists = es.check_index(config).await?; + assert!(!exists); + + Ok(()) + } + + #[tokio::test] + async fn test_init() -> Result<(), Box> { + let es = Elasticsearch::new("http://localhost:9200").unwrap(); + let config = &Config { + index: "flakes_index", + exists_strategy: ExistsStrategy::Recreate, + }; + + es.ensure_index(config).await?; + + let exists = es.check_index(config).await?; + assert!(exists, "Index should exist"); + + Ok(()) + } + + #[tokio::test] + async fn test_push() -> Result<(), Box> { + let sources: Vec = + data::Source::read_sources_file(Path::new("./examples/examples.in.json"))?; + + let exports = sources + .iter() + .flat_map(|s| process_flake(s, &Kind::All, false, &[])) + .flatten() + .collect::>(); + println!("{}", serde_json::to_string(&exports[1]).unwrap()); + + let es = Elasticsearch::new("http://localhost:9200").unwrap(); + let config = &Config { + index: "flakes_index", + exists_strategy: ExistsStrategy::Recreate, + }; + + es.ensure_index(config).await?; + es.push_exports(config, &exports).await?; + + Ok(()) + } + + #[tokio::test] + async fn test_abort_if_index_exists() -> Result<(), Box> { + let es = Elasticsearch::new("http://localhost:9200").unwrap(); + let config = &Config { + index: "flakes_index", + exists_strategy: ExistsStrategy::Abort, + }; + + es.ensure_index(&Config { + exists_strategy: ExistsStrategy::Ignore, + ..*config + }) + .await?; + + assert!(matches!( + es.ensure_index(config).await, + Err(ElasticsearchError::IndexExistsError(_)), + )); + + es.clear_index(config).await?; + + Ok(()) + } +} diff --git a/flake-info/src/lib.rs b/flake-info/src/lib.rs new file mode 100644 index 0000000..f46f2fa --- /dev/null +++ b/flake-info/src/lib.rs @@ -0,0 +1,52 @@ +#![recursion_limit = "256"] + +use std::path::PathBuf; + +use anyhow::Result; +use data::{import::Kind, Export, Source}; + +pub mod commands; +pub mod data; +pub mod elastic; + +pub use commands::get_flake_info; + +pub fn process_flake( + source: &Source, + kind: &data::import::Kind, + temp_store: bool, + extra: &[String], +) -> Result> { + let mut info = commands::get_flake_info(source.to_flake_ref(), temp_store, extra)?; + info.source = Some(source.clone()); + let packages = commands::get_derivation_info(source.to_flake_ref(), *kind, temp_store, extra)?; + eprintln!("{:#?}", info); + eprintln!("{:#?}", packages); + + let exports: Vec = packages + .into_iter() + .map(|p| Export::flake(info.clone(), p)) + .collect(); + + Ok(exports) +} + +pub fn process_nixpkgs(nixpkgs: &Source, kind: &Kind) -> Result, anyhow::Error> { + let drvs = if matches!(kind, Kind::All | Kind::Package) { + commands::get_nixpkgs_info(nixpkgs.to_flake_ref())? + } else { + Vec::new() + }; + + let mut options = if matches!(kind, Kind::All | Kind::Option) { + commands::get_nixpkgs_options(nixpkgs.to_flake_ref())? + } else { + Vec::new() + }; + + let mut all = drvs; + all.append(&mut options); + + let exports = all.into_iter().map(Export::nixpkgs).collect(); + Ok(exports) +} diff --git a/flake.nix b/flake.nix index d3d8db7..3e88268 100644 --- a/flake.nix +++ b/flake.nix @@ -21,11 +21,22 @@ packages = system: { import_scripts = mkPackage ./import-scripts system; + flake_info = mkPackage ./flake-info system; frontend = mkPackage ./. system; }; + + devShell = system: + nixpkgs.legacyPackages.${system}.mkShell { + inputsFrom = builtins.attrValues (packages system); + shellHook = '' + # undo import_scripts' shell hook + cd .. + ''; + }; in - { - defaultPackage = forAllSystems (mkPackage ./.); - packages = forAllSystems packages; - }; + { + defaultPackage = forAllSystems (mkPackage ./.); + packages = forAllSystems packages; + devShell = forAllSystems devShell; + }; } diff --git a/overlay.nix b/overlay.nix index cad0b00..4f20f0f 100644 --- a/overlay.nix +++ b/overlay.nix @@ -3,5 +3,6 @@ final: prev: nixos-search = { frontend = import ./. { pkgs = prev; }; import_scripts = import ./import-scripts { pkgs = prev; }; + flake_info = import ./flake-info { pkgs = prev; }; }; } diff --git a/src/Main.elm b/src/Main.elm index fb9b241..bfc04c8 100644 --- a/src/Main.elm +++ b/src/Main.elm @@ -29,6 +29,7 @@ import Html.Attributes import Page.Home import Page.Options import Page.Packages +import Page.Flakes import Route import Search import Url @@ -59,6 +60,7 @@ type Page | Home Page.Home.Model | Packages Page.Packages.Model | Options Page.Options.Model + | Flakes Page.Flakes.Model init : @@ -93,6 +95,7 @@ type Msg | HomeMsg Page.Home.Msg | PackagesMsg Page.Packages.Msg | OptionsMsg Page.Options.Msg + | FlakesMsg Page.Flakes.Msg updateWith : @@ -234,6 +237,21 @@ changeRouteTo currentModel url = |> avoidReinit |> attemptQuery + Route.Flakes searchArgs -> + let + modelPage = + case model.page of + Flakes x -> + Just x + + _ -> + Nothing + in + Page.Flakes.init searchArgs modelPage + |> updateWith Flakes FlakesMsg model + |> avoidReinit + |> attemptQuery + update : Msg -> Model -> ( Model, Cmd Msg ) update msg model = @@ -296,6 +314,9 @@ view model = Options _ -> "NixOS Search - Options" + Flakes _ -> + "NixOS Search - Flakes (Experimental)" + _ -> "NixOS Search" in @@ -362,6 +383,7 @@ viewNavigation route = (viewNavigationItem route) [ ( toRoute Route.Packages, "Packages" ) , ( toRoute Route.Options, "Options" ) + --, ( toRoute Route.Flakes, "Flakes (Experimental)" ) ] @@ -390,7 +412,8 @@ viewPage model = Options optionsModel -> Html.map (\m -> OptionsMsg m) <| Page.Options.view optionsModel - + Flakes flakesModel -> + Html.map (\m -> FlakesMsg m) <| Page.Flakes.view flakesModel -- SUBSCRIPTIONS diff --git a/src/Page/Flakes.elm b/src/Page/Flakes.elm new file mode 100644 index 0000000..07f0946 --- /dev/null +++ b/src/Page/Flakes.elm @@ -0,0 +1,297 @@ +module Page.Flakes exposing (Model, Msg, init, update, view) + +import Browser.Navigation +import Html exposing (Html, a, code, div, li, pre, strong, text, ul) +import Html.Attributes exposing (class, classList, href, target) +import Html.Events exposing (onClick) +import Html.Parser +import Html.Parser.Util +import Json.Decode +import Route +import Search + + + +-- MODEL + + +type alias Model = + Search.Model ResultItemSource ResultAggregations + + +type alias ResultItemSource = + { name : String + , description : Maybe String + , type_ : Maybe String + , default : Maybe String + , example : Maybe String + , source : Maybe String + } + + +type alias ResultAggregations = + { all : AggregationsAll + } + + +type alias AggregationsAll = + { doc_count : Int + } + + +init : Route.SearchArgs -> Maybe Model -> ( Model, Cmd Msg ) +init searchArgs model = + let + ( newModel, newCmd ) = + Search.init searchArgs model + in + ( newModel + , Cmd.map SearchMsg newCmd + ) + + + +-- UPDATE + + +type Msg + = SearchMsg (Search.Msg ResultItemSource ResultAggregations) + + +update : + Browser.Navigation.Key + -> Msg + -> Model + -> ( Model, Cmd Msg ) +update navKey msg model = + case msg of + SearchMsg subMsg -> + let + ( newModel, newCmd ) = + Search.update + Route.Options + navKey + subMsg + model + in + ( newModel, Cmd.map SearchMsg newCmd ) + + + +-- VIEW + + +view : Model -> Html Msg +view model = + Search.view { toRoute = Route.Options, categoryName = "options" } + [ text "Search more than " + , strong [] [ text "10 000 options" ] + ] + model + viewSuccess + viewBuckets + SearchMsg + + +viewBuckets : + Maybe String + -> Search.SearchResult ResultItemSource ResultAggregations + -> List (Html Msg) +viewBuckets _ _ = + [] + + +viewSuccess : + String + -> Bool + -> Maybe String + -> List (Search.ResultItem ResultItemSource) + -> Html Msg +viewSuccess channel showNixOSDetails show hits = + ul [] + (List.map + (viewResultItem channel showNixOSDetails show) + hits + ) + + +viewResultItem : + String + -> Bool + -> Maybe String + -> Search.ResultItem ResultItemSource + -> Html Msg +viewResultItem channel _ show item = + let + showHtml value = + case Html.Parser.run value of + Ok nodes -> + Html.Parser.Util.toVirtualDom nodes + + Err _ -> + [] + + default = + "Not given" + + asPre value = + pre [] [ text value ] + + asPreCode value = + div [] [ pre [] [ code [ class "code-block" ] [ text value ] ] ] + + githubUrlPrefix branch = + "https://github.com/NixOS/nixpkgs/blob/" ++ branch ++ "/" + + cleanPosition value = + if String.startsWith "source/" value then + String.dropLeft 7 value + + else + value + + asGithubLink value = + case Search.channelDetailsFromId channel of + Just channelDetails -> + a + [ href <| githubUrlPrefix channelDetails.branch ++ (value |> String.replace ":" "#L") + , target "_blank" + ] + [ text value ] + + Nothing -> + text <| cleanPosition value + + withEmpty wrapWith maybe = + case maybe of + Nothing -> + asPre default + + Just "" -> + asPre default + + Just value -> + wrapWith value + + wrapped wrapWith value = + case value of + "" -> + wrapWith <| "\"" ++ value ++ "\"" + + _ -> + wrapWith value + + showDetails = + if Just item.source.name == show then + div [ Html.Attributes.map SearchMsg Search.trapClick ] + [ div [] [ text "Name" ] + , div [] [ wrapped asPreCode item.source.name ] + , div [] [ text "Description" ] + , div [] <| + (item.source.description + |> Maybe.map showHtml + |> Maybe.withDefault [] + ) + , div [] [ text "Default value" ] + , div [] [ withEmpty (wrapped asPreCode) item.source.default ] + , div [] [ text "Type" ] + , div [] [ withEmpty asPre item.source.type_ ] + , div [] [ text "Example" ] + , div [] [ withEmpty (wrapped asPreCode) item.source.example ] + , div [] [ text "Declared in" ] + , div [] [ withEmpty asGithubLink item.source.source ] + ] + |> Just + + else + Nothing + + toggle = + SearchMsg (Search.ShowDetails item.source.name) + + isOpen = + Just item.source.name == show + in + li + [ class "option" + , classList [ ( "opened", isOpen ) ] + , Search.elementId item.source.name + ] + <| + List.filterMap identity + [ Just <| + Html.a + [ class "search-result-button" + , onClick toggle + , href "" + ] + [ text item.source.name ] + , showDetails + ] + + + +-- API + + +makeRequest : + Search.Options + -> String + -> String + -> Int + -> Int + -> Maybe String + -> Search.Sort + -> Cmd Msg +makeRequest options channel query from size _ sort = + Search.makeRequest + (Search.makeRequestBody + (String.trim query) + from + size + sort + "option" + "option_name" + [] + [] + [] + "option_name" + [ ( "option_name", 6.0 ) + , ( "option_name_query", 3.0 ) + , ( "option_description", 1.0 ) + ] + ) + channel + decodeResultItemSource + decodeResultAggregations + options + Search.QueryResponse + (Just "query-options") + |> Cmd.map SearchMsg + + + +-- JSON + + +decodeResultItemSource : Json.Decode.Decoder ResultItemSource +decodeResultItemSource = + Json.Decode.map6 ResultItemSource + (Json.Decode.field "option_name" Json.Decode.string) + (Json.Decode.field "option_description" (Json.Decode.nullable Json.Decode.string)) + (Json.Decode.field "option_type" (Json.Decode.nullable Json.Decode.string)) + (Json.Decode.field "option_default" (Json.Decode.nullable Json.Decode.string)) + (Json.Decode.field "option_example" (Json.Decode.nullable Json.Decode.string)) + (Json.Decode.field "option_source" (Json.Decode.nullable Json.Decode.string)) + + +decodeResultAggregations : Json.Decode.Decoder ResultAggregations +decodeResultAggregations = + Json.Decode.map ResultAggregations + (Json.Decode.field "all" decodeResultAggregationsAll) + + +decodeResultAggregationsAll : Json.Decode.Decoder AggregationsAll +decodeResultAggregationsAll = + Json.Decode.map AggregationsAll + (Json.Decode.field "doc_count" Json.Decode.int) diff --git a/src/Page/Options.elm b/src/Page/Options.elm index f0fe724..d193048 100644 --- a/src/Page/Options.elm +++ b/src/Page/Options.elm @@ -289,7 +289,7 @@ makeRequest options channel query from size _ sort = , ( "option_description", 1.0 ) ] ) - ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel) + channel decodeResultItemSource decodeResultAggregations options diff --git a/src/Page/Packages.elm b/src/Page/Packages.elm index 9482db7..5eb696a 100644 --- a/src/Page/Packages.elm +++ b/src/Page/Packages.elm @@ -40,6 +40,7 @@ import Regex import Route import Search import Utils +import Search exposing (channelDetailsFromId) @@ -153,6 +154,15 @@ init searchArgs model = ) +platforms: List String +platforms = + [ "x86_64-linux" + , "aarch64-linux" + , "i686-linux" + , "x86_64-darwin" + , "aarch64-darwin" + ] + -- UPDATE @@ -240,11 +250,14 @@ viewBuckets bucketsAsString result = selectedBucket.maintainers |> viewBucket "Platforms" - (result.aggregations.package_platforms.buckets |> sortBuckets) + (result.aggregations.package_platforms.buckets |> sortBuckets |> filterPlatformsBucket) (createBucketsMsg .platforms (\s v -> { s | platforms = v })) selectedBucket.platforms +filterPlatformsBucket : List {a | key : String} -> List {a | key : String} +filterPlatformsBucket = List.filter (\a -> List.member a.key platforms) + viewBucket : String -> List Search.AggregationsBucketItem @@ -574,7 +587,7 @@ makeRequest : -> Search.Sort -> Cmd Msg makeRequest options channel query from size maybeBuckets sort = - let + let currentBuckets = initBuckets maybeBuckets @@ -647,7 +660,7 @@ makeRequest options channel query from size maybeBuckets sort = , ( "package_longDescription", 1.0 ) ] ) - ("latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ channel) + channel decodeResultItemSource decodeResultAggregations options @@ -689,13 +702,23 @@ decodeResultItemSource = |> Json.Decode.Pipeline.required "package_longDescription" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_license" (Json.Decode.list decodeResultPackageLicense) |> Json.Decode.Pipeline.required "package_maintainers" (Json.Decode.list decodeResultPackageMaintainer) - |> Json.Decode.Pipeline.required "package_platforms" (Json.Decode.list Json.Decode.string) + |> Json.Decode.Pipeline.required "package_platforms" (Json.Decode.map filterPlatforms (Json.Decode.list Json.Decode.string)) |> Json.Decode.Pipeline.required "package_position" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_homepage" decodeHomepage |> Json.Decode.Pipeline.required "package_system" Json.Decode.string |> Json.Decode.Pipeline.required "package_hydra" (Json.Decode.nullable (Json.Decode.list decodeResultPackageHydra)) +filterPlatforms : List String -> List String +filterPlatforms = + let + flip : (a -> b -> c) -> b -> a -> c + flip function argB argA = + function argA argB + in + List.filter (flip List.member platforms) + + decodeHomepage : Json.Decode.Decoder (List String) decodeHomepage = Json.Decode.oneOf diff --git a/src/Route.elm b/src/Route.elm index 06f6ff8..37c8ceb 100644 --- a/src/Route.elm +++ b/src/Route.elm @@ -77,6 +77,7 @@ type Route | Home | Packages SearchArgs | Options SearchArgs + | Flakes SearchArgs parser : Url.Url -> Url.Parser.Parser (Route -> msg) msg @@ -86,6 +87,7 @@ parser url = , Url.Parser.map NotFound <| Url.Parser.s "not-found" , Url.Parser.map Packages <| Url.Parser.s "packages" searchQueryParser url , Url.Parser.map Options <| Url.Parser.s "options" searchQueryParser url + , Url.Parser.map Flakes <| Url.Parser.s "flakes" searchQueryParser url ] @@ -143,3 +145,7 @@ routeToPieces page = Options searchArgs -> searchArgsToUrl searchArgs |> (\( query, raw ) -> ( [ "options" ], query, raw )) + + Flakes searchArgs -> + searchArgsToUrl searchArgs + |> (\( query, raw ) -> ( [ "flakes" ], query, raw )) diff --git a/src/Search.elm b/src/Search.elm index e221868..b7b3c29 100644 --- a/src/Search.elm +++ b/src/Search.elm @@ -395,14 +395,13 @@ channelDetails : Channel -> ChannelDetails channelDetails channel = case channel of Unstable -> - ChannelDetails "unstable" "unstable" "nixos/trunk-combined" "nixos-unstable" + ChannelDetails "unstable" "unstable" "nixos/trunk-combined" "nixpkgs-unstable" Release_20_09 -> - ChannelDetails "20.09" "20.09" "nixos/release-20.09" "nixos-20.09" + ChannelDetails "20.09" "20.09" "nixos/release-20.09" "nixpkgs-20.09" Release_21_05 -> - ChannelDetails "21.05" "21.05" "nixos/release-21.05" "nixos-21.05" - + ChannelDetails "21.05" "21.05" "nixos/release-21.05" "nixpkgs-21.05" channelFromId : String -> Maybe Channel channelFromId channel_id = @@ -457,6 +456,9 @@ toAggregations bucketsFields = [ ( "field" , Json.Encode.string field ) + , ( "size" + , Json.Encode.int 20 + ) ] ) ] @@ -477,7 +479,7 @@ toAggregations bucketsFields = ) ] in - ( "aggregations" + ( "aggs" , Json.Encode.object <| List.append fields allFields ) @@ -1136,7 +1138,10 @@ makeRequest : -> (RemoteData.WebData (SearchResult a b) -> Msg a b) -> Maybe String -> Cmd (Msg a b) -makeRequest body index decodeResultItemSource decodeResultAggregations options responseMsg tracker = +makeRequest body channel decodeResultItemSource decodeResultAggregations options responseMsg tracker = + let branch = Maybe.map (\details -> details.branch) (channelDetailsFromId channel) |> Maybe.withDefault "" + index = "latest-" ++ String.fromInt options.mappingSchemaVersion ++ "-" ++ branch + in Http.riskyRequest { method = "POST" , headers =