From dff4ba7132982f7d27bdc2f680c03ec553023eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 7 Feb 2023 16:31:17 +0100 Subject: [PATCH] Search programs provided by a package (#610) * Search programs provided by a package Use the `programs.sqlite` database provided with nixpkgs channels to populate a `package_programs` field so that searches for e.g. `make` return `gnumake` with a higher priority. * Bump VERSION * frontend: show programs --- VERSION | 2 +- flake-info/Cargo.lock | 31 ++++++ flake-info/Cargo.toml | 1 + flake-info/src/commands/nixpkgs_info.rs | 123 ++++++++++++++++-------- flake-info/src/data/export.rs | 9 +- flake-info/src/data/import.rs | 12 ++- flake-info/src/elastic.rs | 3 + flake-info/src/lib.rs | 4 +- frontend/src/Page/Packages.elm | 18 +++- frontend/src/Search.elm | 7 +- 10 files changed, 159 insertions(+), 51 deletions(-) diff --git a/VERSION b/VERSION index 8f92bfd..7facc89 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -35 +36 diff --git a/flake-info/Cargo.lock b/flake-info/Cargo.lock index 1d1ad99..d05d0ae 100644 --- a/flake-info/Cargo.lock +++ b/flake-info/Cargo.lock @@ -340,6 +340,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "sha2", + "sqlite", "structopt", "thiserror", "tokio", @@ -1347,6 +1348,36 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "sqlite" +version = "0.30.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48" +dependencies = [ + "libc", + "sqlite3-sys", +] + +[[package]] +name = "sqlite3-src" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "sqlite3-sys" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee" +dependencies = [ + "libc", + "sqlite3-src", +] + [[package]] name = "strsim" version = "0.8.0" diff --git a/flake-info/Cargo.toml b/flake-info/Cargo.toml index 9ba1834..923ffca 100644 --- a/flake-info/Cargo.toml +++ b/flake-info/Cargo.toml @@ -25,6 +25,7 @@ reqwest = { version = "0.11", features = ["json", "blocking"] } sha2 = "0.9" pandoc = "0.8.10" semver = "1.0" +sqlite = "0.30" elasticsearch = {git = "https://github.com/elastic/elasticsearch-rs", features = ["rustls-tls"], optional = true} diff --git a/flake-info/src/commands/nixpkgs_info.rs b/flake-info/src/commands/nixpkgs_info.rs index f7f1c5a..2c3f921 100644 --- a/flake-info/src/commands/nixpkgs_info.rs +++ b/flake-info/src/commands/nixpkgs_info.rs @@ -1,20 +1,21 @@ use anyhow::{Context, Result}; use serde_json::Deserializer; -use std::{collections::HashMap, fmt::Display}; +use std::collections::{HashMap, HashSet}; use command_run::{Command, LogTo}; -use log::error; use crate::data::import::{NixOption, NixpkgsEntry, Package}; +use crate::data::Nixpkgs; +use crate::Source; -pub fn get_nixpkgs_info + Display>(nixpkgs_channel: T) -> Result> { +pub fn get_nixpkgs_info(nixpkgs: &Source) -> Result> { let mut command = Command::new("nix-env"); command.add_args(&[ "--json", "-f", "", "-I", - format!("nixpkgs={}", nixpkgs_channel.as_ref()).as_str(), + format!("nixpkgs={}", nixpkgs.to_flake_ref()).as_str(), "--arg", "config", "import ", @@ -26,56 +27,94 @@ pub fn get_nixpkgs_info + Display>(nixpkgs_channel: T) -> Result> = command + let cow = command .run() - .with_context(|| { - format!( - "Failed to gather information about nixpkgs {}", - nixpkgs_channel.as_ref() - ) - }) - .and_then(|o| { - let output = &*o.stdout_string_lossy(); - let de = &mut Deserializer::from_str(output); - let attr_set: HashMap = - serde_path_to_error::deserialize(de).with_context(|| "Could not parse packages")?; - Ok(attr_set - .into_iter() - .map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package }) - .collect()) - }); + .with_context(|| "Failed to gather information about nixpkgs packages")?; - parsed + let output = &*cow.stdout_string_lossy(); + let de = &mut Deserializer::from_str(output); + let attr_set: HashMap = + serde_path_to_error::deserialize(de).with_context(|| "Could not parse packages")?; + + let mut programs = match nixpkgs { + Source::Nixpkgs(nixpkgs) => get_nixpkgs_programs(nixpkgs)?, + _ => Default::default(), + }; + + Ok(attr_set + .into_iter() + .map(|(attribute, package)| { + let programs = programs + .remove(&attribute) + .unwrap_or_default() + .into_iter() + .collect(); + NixpkgsEntry::Derivation { + attribute, + package, + programs, + } + }) + .collect()) } -pub fn get_nixpkgs_options + Display>( - nixpkgs_channel: T, -) -> Result> { +pub fn get_nixpkgs_programs(nixpkgs: &Nixpkgs) -> Result>> { + let mut command = Command::new("nix-instantiate"); + command.add_args(&[ + "--eval", + "--json", + "-I", + format!("nixpkgs=channel:nixos-{}", nixpkgs.channel).as_str(), + "--expr", + "toString ", + ]); + + command.enable_capture(); + command.log_to = LogTo::Log; + command.log_output_on_error = true; + + let cow = command + .run() + .with_context(|| "Failed to gather information about nixpkgs programs")?; + + let output = &*cow.stdout_string_lossy(); + let programs_db: &str = serde_json::from_str(output)?; + let conn = sqlite::open(programs_db)?; + let cur = conn + .prepare("SELECT name, package FROM Programs")? + .into_iter(); + + let mut programs: HashMap> = HashMap::new(); + for row in cur.map(|r| r.unwrap()) { + let name: &str = row.read("name"); + let package: &str = row.read("package"); + programs + .entry(package.into()) + .or_default() + .insert(name.into()); + } + + Ok(programs) +} + +pub fn get_nixpkgs_options(nixpkgs: &Source) -> Result> { let mut command = Command::with_args("nix", &["eval", "--json"]); command.add_arg_pair("-f", super::EXTRACT_SCRIPT.clone()); - command.add_arg_pair("-I", format!("nixpkgs={}", nixpkgs_channel.as_ref())); + command.add_arg_pair("-I", format!("nixpkgs={}", nixpkgs.to_flake_ref())); command.add_arg("nixos-options"); command.enable_capture(); command.log_to = LogTo::Log; command.log_output_on_error = true; - let parsed = command.run().with_context(|| { - format!( - "Failed to gather information about nixpkgs {}", - nixpkgs_channel.as_ref() - ) - }); + let cow = command + .run() + .with_context(|| "Failed to gather information about nixpkgs options")?; - if let Err(ref e) = parsed { - error!("Command error: {}", e); - } + let output = &*cow.stdout_string_lossy(); + let de = &mut Deserializer::from_str(output); + let attr_set: Vec = + serde_path_to_error::deserialize(de).with_context(|| "Could not parse options")?; - parsed.and_then(|o| { - let output = &*o.stdout_string_lossy(); - let de = &mut Deserializer::from_str(output); - let attr_set: Vec = - serde_path_to_error::deserialize(de).with_context(|| "Could not parse options")?; - Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect()) - }) + Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect()) } diff --git a/flake-info/src/data/export.rs b/flake-info/src/data/export.rs index ccd837d..932c66e 100644 --- a/flake-info/src/data/export.rs +++ b/flake-info/src/data/export.rs @@ -63,6 +63,7 @@ pub enum Derivation { package_platforms: Vec, package_outputs: Vec, package_default_output: Option, + package_programs: Vec, package_license: Vec, package_license_set: Vec, package_maintainers: Vec, @@ -150,6 +151,7 @@ impl TryFrom<(import::FlakeEntry, super::Flake)> for Derivation { package_platforms: platforms, package_outputs: outputs, package_default_output: Some(default_output), + package_programs: Vec::new(), package_license, package_license_set, package_description: description.clone(), @@ -183,7 +185,11 @@ impl TryFrom for Derivation { fn try_from(entry: import::NixpkgsEntry) -> Result { Ok(match entry { - import::NixpkgsEntry::Derivation { attribute, package } => { + import::NixpkgsEntry::Derivation { + attribute, + package, + programs, + } => { let package_attr_set: Vec<_> = attribute.split(".").collect(); let package_attr_set: String = (if package_attr_set.len() > 1 { package_attr_set[0] @@ -249,6 +255,7 @@ impl TryFrom for Derivation { package_platforms: platforms, package_outputs: package.outputs.into_keys().collect(), package_default_output: package.default_output, + package_programs: programs, package_license, package_license_set, package_maintainers, diff --git a/flake-info/src/data/import.rs b/flake-info/src/data/import.rs index 3d47b71..affeb3d 100644 --- a/flake-info/src/data/import.rs +++ b/flake-info/src/data/import.rs @@ -179,7 +179,11 @@ pub struct Package { /// Name and Package definition are combined using this struct #[derive(Debug, Clone)] pub enum NixpkgsEntry { - Derivation { attribute: String, package: Package }, + Derivation { + attribute: String, + package: Package, + programs: Vec, + }, Option(NixOption), } @@ -435,7 +439,11 @@ mod tests { let _: Vec = map .into_iter() - .map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package }) + .map(|(attribute, package)| NixpkgsEntry::Derivation { + attribute, + package, + programs: Vec::new(), + }) .collect(); } diff --git a/flake-info/src/elastic.rs b/flake-info/src/elastic.rs index 39f9b0b..5f41c4e 100644 --- a/flake-info/src/elastic.rs +++ b/flake-info/src/elastic.rs @@ -97,6 +97,9 @@ lazy_static! { "package_default_output": { "type": "keyword" }, + "package_programs": { + "type": "keyword" + }, "package_description": { "type": "text", "analyzer": "english", diff --git a/flake-info/src/lib.rs b/flake-info/src/lib.rs index 744898a..6b59c31 100644 --- a/flake-info/src/lib.rs +++ b/flake-info/src/lib.rs @@ -41,13 +41,13 @@ pub fn process_flake( pub fn process_nixpkgs(nixpkgs: &Source, kind: &Kind) -> Result, anyhow::Error> { let drvs = if matches!(kind, Kind::All | Kind::Package) { - commands::get_nixpkgs_info(nixpkgs.to_flake_ref())? + commands::get_nixpkgs_info(nixpkgs)? } else { Vec::new() }; let mut options = if matches!(kind, Kind::All | Kind::Option) { - commands::get_nixpkgs_options(nixpkgs.to_flake_ref())? + commands::get_nixpkgs_options(nixpkgs)? } else { Vec::new() }; diff --git a/frontend/src/Page/Packages.elm b/frontend/src/Page/Packages.elm index 4abb98d..6c80e23 100644 --- a/frontend/src/Page/Packages.elm +++ b/frontend/src/Page/Packages.elm @@ -72,6 +72,7 @@ type alias ResultItemSource = , pversion : String , outputs : List String , default_output : Maybe String + , programs : List String , description : Maybe String , longDescription : Maybe String , licenses : List ResultPackageLicense @@ -476,7 +477,7 @@ viewResultItem nixosChannels channel showInstallDetails show item = li [] [ text platform ] maintainersAndPlatforms = - [ div [] + div [] [ div [] (List.append [ h4 [] [ text "Maintainers" ] ] (if List.isEmpty item.source.maintainers then @@ -500,7 +501,16 @@ viewResultItem nixosChannels channel showInstallDetails show item = ) ) ] - ] + + programs = + div [] + [ h4 [] [ text "Programs in ", code [] [ text "/bin" ] ] + , if List.isEmpty item.source.programs then + p [] [ text "This package provides no programs." ] + + else + p [] (List.intersperse (text " ") (List.map (\p -> code [] [ text p ]) item.source.programs)) + ] longerPackageDetails = optionals (Just item.source.attr_name == show) @@ -722,6 +732,8 @@ viewResultItem nixosChannels channel showInstallDetails show item = Maybe.map Tuple.first item.source.flakeUrl ] :: maintainersAndPlatforms + :: programs + :: [] ) ] @@ -908,6 +920,7 @@ makeRequestBody query from size maybeBuckets sort = filterByBuckets "package_attr_name" [ ( "package_attr_name", 9.0 ) + , ( "package_programs", 9.0 ) , ( "package_pname", 6.0 ) , ( "package_description", 1.3 ) , ( "package_longDescription", 1.0 ) @@ -946,6 +959,7 @@ decodeResultItemSource = |> Json.Decode.Pipeline.required "package_pversion" Json.Decode.string |> Json.Decode.Pipeline.required "package_outputs" (Json.Decode.list Json.Decode.string) |> Json.Decode.Pipeline.required "package_default_output" (Json.Decode.nullable Json.Decode.string) + |> Json.Decode.Pipeline.required "package_programs" (Json.Decode.list Json.Decode.string) |> Json.Decode.Pipeline.required "package_description" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_longDescription" (Json.Decode.nullable Json.Decode.string) |> Json.Decode.Pipeline.required "package_license" (Json.Decode.list decodeResultPackageLicense) diff --git a/frontend/src/Search.elm b/frontend/src/Search.elm index 8d05fb9..2d3bda9 100644 --- a/frontend/src/Search.elm +++ b/frontend/src/Search.elm @@ -1232,7 +1232,12 @@ searchFields query mainField fields = let allFields = fields - |> List.map (\( field, score ) -> [ field ++ "^" ++ String.fromFloat score, field ++ ".*^" ++ String.fromFloat score ]) + |> List.map + (\( field, score ) -> + [ field ++ "^" ++ String.fromFloat score + , field ++ ".*^" ++ String.fromFloat (score * 0.6) + ] + ) |> List.concat queryWords =