Search programs provided by a package (#610)
* Search programs provided by a package Use the `programs.sqlite` database provided with nixpkgs channels to populate a `package_programs` field so that searches for e.g. `make` return `gnumake` with a higher priority. * Bump VERSION * frontend: show programs
This commit is contained in:
parent
32097fc62e
commit
dff4ba7132
31
flake-info/Cargo.lock
generated
31
flake-info/Cargo.lock
generated
|
@ -340,6 +340,7 @@ dependencies = [
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_path_to_error",
|
"serde_path_to_error",
|
||||||
"sha2",
|
"sha2",
|
||||||
|
"sqlite",
|
||||||
"structopt",
|
"structopt",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
@ -1347,6 +1348,36 @@ version = "0.5.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
|
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sqlite"
|
||||||
|
version = "0.30.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"sqlite3-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sqlite3-src"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"pkg-config",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sqlite3-sys"
|
||||||
|
version = "0.14.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"sqlite3-src",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "strsim"
|
name = "strsim"
|
||||||
version = "0.8.0"
|
version = "0.8.0"
|
||||||
|
|
|
@ -25,6 +25,7 @@ reqwest = { version = "0.11", features = ["json", "blocking"] }
|
||||||
sha2 = "0.9"
|
sha2 = "0.9"
|
||||||
pandoc = "0.8.10"
|
pandoc = "0.8.10"
|
||||||
semver = "1.0"
|
semver = "1.0"
|
||||||
|
sqlite = "0.30"
|
||||||
|
|
||||||
elasticsearch = {git = "https://github.com/elastic/elasticsearch-rs", features = ["rustls-tls"], optional = true}
|
elasticsearch = {git = "https://github.com/elastic/elasticsearch-rs", features = ["rustls-tls"], optional = true}
|
||||||
|
|
||||||
|
|
|
@ -1,20 +1,21 @@
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use serde_json::Deserializer;
|
use serde_json::Deserializer;
|
||||||
use std::{collections::HashMap, fmt::Display};
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
use command_run::{Command, LogTo};
|
use command_run::{Command, LogTo};
|
||||||
use log::error;
|
|
||||||
|
|
||||||
use crate::data::import::{NixOption, NixpkgsEntry, Package};
|
use crate::data::import::{NixOption, NixpkgsEntry, Package};
|
||||||
|
use crate::data::Nixpkgs;
|
||||||
|
use crate::Source;
|
||||||
|
|
||||||
pub fn get_nixpkgs_info<T: AsRef<str> + Display>(nixpkgs_channel: T) -> Result<Vec<NixpkgsEntry>> {
|
pub fn get_nixpkgs_info(nixpkgs: &Source) -> Result<Vec<NixpkgsEntry>> {
|
||||||
let mut command = Command::new("nix-env");
|
let mut command = Command::new("nix-env");
|
||||||
command.add_args(&[
|
command.add_args(&[
|
||||||
"--json",
|
"--json",
|
||||||
"-f",
|
"-f",
|
||||||
"<nixpkgs>",
|
"<nixpkgs>",
|
||||||
"-I",
|
"-I",
|
||||||
format!("nixpkgs={}", nixpkgs_channel.as_ref()).as_str(),
|
format!("nixpkgs={}", nixpkgs.to_flake_ref()).as_str(),
|
||||||
"--arg",
|
"--arg",
|
||||||
"config",
|
"config",
|
||||||
"import <nixpkgs/pkgs/top-level/packages-config.nix>",
|
"import <nixpkgs/pkgs/top-level/packages-config.nix>",
|
||||||
|
@ -26,56 +27,94 @@ pub fn get_nixpkgs_info<T: AsRef<str> + Display>(nixpkgs_channel: T) -> Result<V
|
||||||
command.log_to = LogTo::Log;
|
command.log_to = LogTo::Log;
|
||||||
command.log_output_on_error = true;
|
command.log_output_on_error = true;
|
||||||
|
|
||||||
let parsed: Result<Vec<NixpkgsEntry>> = command
|
let cow = command
|
||||||
.run()
|
.run()
|
||||||
.with_context(|| {
|
.with_context(|| "Failed to gather information about nixpkgs packages")?;
|
||||||
format!(
|
|
||||||
"Failed to gather information about nixpkgs {}",
|
|
||||||
nixpkgs_channel.as_ref()
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.and_then(|o| {
|
|
||||||
let output = &*o.stdout_string_lossy();
|
|
||||||
let de = &mut Deserializer::from_str(output);
|
|
||||||
let attr_set: HashMap<String, Package> =
|
|
||||||
serde_path_to_error::deserialize(de).with_context(|| "Could not parse packages")?;
|
|
||||||
Ok(attr_set
|
|
||||||
.into_iter()
|
|
||||||
.map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package })
|
|
||||||
.collect())
|
|
||||||
});
|
|
||||||
|
|
||||||
parsed
|
let output = &*cow.stdout_string_lossy();
|
||||||
|
let de = &mut Deserializer::from_str(output);
|
||||||
|
let attr_set: HashMap<String, Package> =
|
||||||
|
serde_path_to_error::deserialize(de).with_context(|| "Could not parse packages")?;
|
||||||
|
|
||||||
|
let mut programs = match nixpkgs {
|
||||||
|
Source::Nixpkgs(nixpkgs) => get_nixpkgs_programs(nixpkgs)?,
|
||||||
|
_ => Default::default(),
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(attr_set
|
||||||
|
.into_iter()
|
||||||
|
.map(|(attribute, package)| {
|
||||||
|
let programs = programs
|
||||||
|
.remove(&attribute)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.collect();
|
||||||
|
NixpkgsEntry::Derivation {
|
||||||
|
attribute,
|
||||||
|
package,
|
||||||
|
programs,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_nixpkgs_options<T: AsRef<str> + Display>(
|
pub fn get_nixpkgs_programs(nixpkgs: &Nixpkgs) -> Result<HashMap<String, HashSet<String>>> {
|
||||||
nixpkgs_channel: T,
|
let mut command = Command::new("nix-instantiate");
|
||||||
) -> Result<Vec<NixpkgsEntry>> {
|
command.add_args(&[
|
||||||
|
"--eval",
|
||||||
|
"--json",
|
||||||
|
"-I",
|
||||||
|
format!("nixpkgs=channel:nixos-{}", nixpkgs.channel).as_str(),
|
||||||
|
"--expr",
|
||||||
|
"toString <nixpkgs/programs.sqlite>",
|
||||||
|
]);
|
||||||
|
|
||||||
|
command.enable_capture();
|
||||||
|
command.log_to = LogTo::Log;
|
||||||
|
command.log_output_on_error = true;
|
||||||
|
|
||||||
|
let cow = command
|
||||||
|
.run()
|
||||||
|
.with_context(|| "Failed to gather information about nixpkgs programs")?;
|
||||||
|
|
||||||
|
let output = &*cow.stdout_string_lossy();
|
||||||
|
let programs_db: &str = serde_json::from_str(output)?;
|
||||||
|
let conn = sqlite::open(programs_db)?;
|
||||||
|
let cur = conn
|
||||||
|
.prepare("SELECT name, package FROM Programs")?
|
||||||
|
.into_iter();
|
||||||
|
|
||||||
|
let mut programs: HashMap<String, HashSet<String>> = HashMap::new();
|
||||||
|
for row in cur.map(|r| r.unwrap()) {
|
||||||
|
let name: &str = row.read("name");
|
||||||
|
let package: &str = row.read("package");
|
||||||
|
programs
|
||||||
|
.entry(package.into())
|
||||||
|
.or_default()
|
||||||
|
.insert(name.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(programs)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_nixpkgs_options(nixpkgs: &Source) -> Result<Vec<NixpkgsEntry>> {
|
||||||
let mut command = Command::with_args("nix", &["eval", "--json"]);
|
let mut command = Command::with_args("nix", &["eval", "--json"]);
|
||||||
command.add_arg_pair("-f", super::EXTRACT_SCRIPT.clone());
|
command.add_arg_pair("-f", super::EXTRACT_SCRIPT.clone());
|
||||||
command.add_arg_pair("-I", format!("nixpkgs={}", nixpkgs_channel.as_ref()));
|
command.add_arg_pair("-I", format!("nixpkgs={}", nixpkgs.to_flake_ref()));
|
||||||
command.add_arg("nixos-options");
|
command.add_arg("nixos-options");
|
||||||
|
|
||||||
command.enable_capture();
|
command.enable_capture();
|
||||||
command.log_to = LogTo::Log;
|
command.log_to = LogTo::Log;
|
||||||
command.log_output_on_error = true;
|
command.log_output_on_error = true;
|
||||||
|
|
||||||
let parsed = command.run().with_context(|| {
|
let cow = command
|
||||||
format!(
|
.run()
|
||||||
"Failed to gather information about nixpkgs {}",
|
.with_context(|| "Failed to gather information about nixpkgs options")?;
|
||||||
nixpkgs_channel.as_ref()
|
|
||||||
)
|
|
||||||
});
|
|
||||||
|
|
||||||
if let Err(ref e) = parsed {
|
let output = &*cow.stdout_string_lossy();
|
||||||
error!("Command error: {}", e);
|
let de = &mut Deserializer::from_str(output);
|
||||||
}
|
let attr_set: Vec<NixOption> =
|
||||||
|
serde_path_to_error::deserialize(de).with_context(|| "Could not parse options")?;
|
||||||
|
|
||||||
parsed.and_then(|o| {
|
Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect())
|
||||||
let output = &*o.stdout_string_lossy();
|
|
||||||
let de = &mut Deserializer::from_str(output);
|
|
||||||
let attr_set: Vec<NixOption> =
|
|
||||||
serde_path_to_error::deserialize(de).with_context(|| "Could not parse options")?;
|
|
||||||
Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect())
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -63,6 +63,7 @@ pub enum Derivation {
|
||||||
package_platforms: Vec<String>,
|
package_platforms: Vec<String>,
|
||||||
package_outputs: Vec<String>,
|
package_outputs: Vec<String>,
|
||||||
package_default_output: Option<String>,
|
package_default_output: Option<String>,
|
||||||
|
package_programs: Vec<String>,
|
||||||
package_license: Vec<License>,
|
package_license: Vec<License>,
|
||||||
package_license_set: Vec<String>,
|
package_license_set: Vec<String>,
|
||||||
package_maintainers: Vec<Maintainer>,
|
package_maintainers: Vec<Maintainer>,
|
||||||
|
@ -150,6 +151,7 @@ impl TryFrom<(import::FlakeEntry, super::Flake)> for Derivation {
|
||||||
package_platforms: platforms,
|
package_platforms: platforms,
|
||||||
package_outputs: outputs,
|
package_outputs: outputs,
|
||||||
package_default_output: Some(default_output),
|
package_default_output: Some(default_output),
|
||||||
|
package_programs: Vec::new(),
|
||||||
package_license,
|
package_license,
|
||||||
package_license_set,
|
package_license_set,
|
||||||
package_description: description.clone(),
|
package_description: description.clone(),
|
||||||
|
@ -183,7 +185,11 @@ impl TryFrom<import::NixpkgsEntry> for Derivation {
|
||||||
|
|
||||||
fn try_from(entry: import::NixpkgsEntry) -> Result<Self, Self::Error> {
|
fn try_from(entry: import::NixpkgsEntry) -> Result<Self, Self::Error> {
|
||||||
Ok(match entry {
|
Ok(match entry {
|
||||||
import::NixpkgsEntry::Derivation { attribute, package } => {
|
import::NixpkgsEntry::Derivation {
|
||||||
|
attribute,
|
||||||
|
package,
|
||||||
|
programs,
|
||||||
|
} => {
|
||||||
let package_attr_set: Vec<_> = attribute.split(".").collect();
|
let package_attr_set: Vec<_> = attribute.split(".").collect();
|
||||||
let package_attr_set: String = (if package_attr_set.len() > 1 {
|
let package_attr_set: String = (if package_attr_set.len() > 1 {
|
||||||
package_attr_set[0]
|
package_attr_set[0]
|
||||||
|
@ -249,6 +255,7 @@ impl TryFrom<import::NixpkgsEntry> for Derivation {
|
||||||
package_platforms: platforms,
|
package_platforms: platforms,
|
||||||
package_outputs: package.outputs.into_keys().collect(),
|
package_outputs: package.outputs.into_keys().collect(),
|
||||||
package_default_output: package.default_output,
|
package_default_output: package.default_output,
|
||||||
|
package_programs: programs,
|
||||||
package_license,
|
package_license,
|
||||||
package_license_set,
|
package_license_set,
|
||||||
package_maintainers,
|
package_maintainers,
|
||||||
|
|
|
@ -179,7 +179,11 @@ pub struct Package {
|
||||||
/// Name and Package definition are combined using this struct
|
/// Name and Package definition are combined using this struct
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum NixpkgsEntry {
|
pub enum NixpkgsEntry {
|
||||||
Derivation { attribute: String, package: Package },
|
Derivation {
|
||||||
|
attribute: String,
|
||||||
|
package: Package,
|
||||||
|
programs: Vec<String>,
|
||||||
|
},
|
||||||
Option(NixOption),
|
Option(NixOption),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -435,7 +439,11 @@ mod tests {
|
||||||
|
|
||||||
let _: Vec<NixpkgsEntry> = map
|
let _: Vec<NixpkgsEntry> = map
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package })
|
.map(|(attribute, package)| NixpkgsEntry::Derivation {
|
||||||
|
attribute,
|
||||||
|
package,
|
||||||
|
programs: Vec::new(),
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -97,6 +97,9 @@ lazy_static! {
|
||||||
"package_default_output": {
|
"package_default_output": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
|
"package_programs": {
|
||||||
|
"type": "keyword"
|
||||||
|
},
|
||||||
"package_description": {
|
"package_description": {
|
||||||
"type": "text",
|
"type": "text",
|
||||||
"analyzer": "english",
|
"analyzer": "english",
|
||||||
|
|
|
@ -41,13 +41,13 @@ pub fn process_flake(
|
||||||
|
|
||||||
pub fn process_nixpkgs(nixpkgs: &Source, kind: &Kind) -> Result<Vec<Export>, anyhow::Error> {
|
pub fn process_nixpkgs(nixpkgs: &Source, kind: &Kind) -> Result<Vec<Export>, anyhow::Error> {
|
||||||
let drvs = if matches!(kind, Kind::All | Kind::Package) {
|
let drvs = if matches!(kind, Kind::All | Kind::Package) {
|
||||||
commands::get_nixpkgs_info(nixpkgs.to_flake_ref())?
|
commands::get_nixpkgs_info(nixpkgs)?
|
||||||
} else {
|
} else {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut options = if matches!(kind, Kind::All | Kind::Option) {
|
let mut options = if matches!(kind, Kind::All | Kind::Option) {
|
||||||
commands::get_nixpkgs_options(nixpkgs.to_flake_ref())?
|
commands::get_nixpkgs_options(nixpkgs)?
|
||||||
} else {
|
} else {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
|
|
|
@ -72,6 +72,7 @@ type alias ResultItemSource =
|
||||||
, pversion : String
|
, pversion : String
|
||||||
, outputs : List String
|
, outputs : List String
|
||||||
, default_output : Maybe String
|
, default_output : Maybe String
|
||||||
|
, programs : List String
|
||||||
, description : Maybe String
|
, description : Maybe String
|
||||||
, longDescription : Maybe String
|
, longDescription : Maybe String
|
||||||
, licenses : List ResultPackageLicense
|
, licenses : List ResultPackageLicense
|
||||||
|
@ -476,7 +477,7 @@ viewResultItem nixosChannels channel showInstallDetails show item =
|
||||||
li [] [ text platform ]
|
li [] [ text platform ]
|
||||||
|
|
||||||
maintainersAndPlatforms =
|
maintainersAndPlatforms =
|
||||||
[ div []
|
div []
|
||||||
[ div []
|
[ div []
|
||||||
(List.append [ h4 [] [ text "Maintainers" ] ]
|
(List.append [ h4 [] [ text "Maintainers" ] ]
|
||||||
(if List.isEmpty item.source.maintainers then
|
(if List.isEmpty item.source.maintainers then
|
||||||
|
@ -500,7 +501,16 @@ viewResultItem nixosChannels channel showInstallDetails show item =
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
]
|
|
||||||
|
programs =
|
||||||
|
div []
|
||||||
|
[ h4 [] [ text "Programs in ", code [] [ text "/bin" ] ]
|
||||||
|
, if List.isEmpty item.source.programs then
|
||||||
|
p [] [ text "This package provides no programs." ]
|
||||||
|
|
||||||
|
else
|
||||||
|
p [] (List.intersperse (text " ") (List.map (\p -> code [] [ text p ]) item.source.programs))
|
||||||
|
]
|
||||||
|
|
||||||
longerPackageDetails =
|
longerPackageDetails =
|
||||||
optionals (Just item.source.attr_name == show)
|
optionals (Just item.source.attr_name == show)
|
||||||
|
@ -722,6 +732,8 @@ viewResultItem nixosChannels channel showInstallDetails show item =
|
||||||
Maybe.map Tuple.first item.source.flakeUrl
|
Maybe.map Tuple.first item.source.flakeUrl
|
||||||
]
|
]
|
||||||
:: maintainersAndPlatforms
|
:: maintainersAndPlatforms
|
||||||
|
:: programs
|
||||||
|
:: []
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -908,6 +920,7 @@ makeRequestBody query from size maybeBuckets sort =
|
||||||
filterByBuckets
|
filterByBuckets
|
||||||
"package_attr_name"
|
"package_attr_name"
|
||||||
[ ( "package_attr_name", 9.0 )
|
[ ( "package_attr_name", 9.0 )
|
||||||
|
, ( "package_programs", 9.0 )
|
||||||
, ( "package_pname", 6.0 )
|
, ( "package_pname", 6.0 )
|
||||||
, ( "package_description", 1.3 )
|
, ( "package_description", 1.3 )
|
||||||
, ( "package_longDescription", 1.0 )
|
, ( "package_longDescription", 1.0 )
|
||||||
|
@ -946,6 +959,7 @@ decodeResultItemSource =
|
||||||
|> Json.Decode.Pipeline.required "package_pversion" Json.Decode.string
|
|> Json.Decode.Pipeline.required "package_pversion" Json.Decode.string
|
||||||
|> Json.Decode.Pipeline.required "package_outputs" (Json.Decode.list Json.Decode.string)
|
|> Json.Decode.Pipeline.required "package_outputs" (Json.Decode.list Json.Decode.string)
|
||||||
|> Json.Decode.Pipeline.required "package_default_output" (Json.Decode.nullable Json.Decode.string)
|
|> Json.Decode.Pipeline.required "package_default_output" (Json.Decode.nullable Json.Decode.string)
|
||||||
|
|> Json.Decode.Pipeline.required "package_programs" (Json.Decode.list Json.Decode.string)
|
||||||
|> Json.Decode.Pipeline.required "package_description" (Json.Decode.nullable Json.Decode.string)
|
|> Json.Decode.Pipeline.required "package_description" (Json.Decode.nullable Json.Decode.string)
|
||||||
|> Json.Decode.Pipeline.required "package_longDescription" (Json.Decode.nullable Json.Decode.string)
|
|> Json.Decode.Pipeline.required "package_longDescription" (Json.Decode.nullable Json.Decode.string)
|
||||||
|> Json.Decode.Pipeline.required "package_license" (Json.Decode.list decodeResultPackageLicense)
|
|> Json.Decode.Pipeline.required "package_license" (Json.Decode.list decodeResultPackageLicense)
|
||||||
|
|
|
@ -1232,7 +1232,12 @@ searchFields query mainField fields =
|
||||||
let
|
let
|
||||||
allFields =
|
allFields =
|
||||||
fields
|
fields
|
||||||
|> List.map (\( field, score ) -> [ field ++ "^" ++ String.fromFloat score, field ++ ".*^" ++ String.fromFloat score ])
|
|> List.map
|
||||||
|
(\( field, score ) ->
|
||||||
|
[ field ++ "^" ++ String.fromFloat score
|
||||||
|
, field ++ ".*^" ++ String.fromFloat (score * 0.6)
|
||||||
|
]
|
||||||
|
)
|
||||||
|> List.concat
|
|> List.concat
|
||||||
|
|
||||||
queryWords =
|
queryWords =
|
||||||
|
|
Loading…
Reference in a new issue