c8h4/aux-search
1
0
Fork 0
Code Issues Pull requests Packages Releases Activity

Search programs provided by a package (#610)

Browse source 
* Search programs provided by a package

Use the `programs.sqlite` database provided with nixpkgs channels to
populate a `package_programs` field so that searches for e.g. `make` return
`gnumake` with a higher priority.

* Bump VERSION

* frontend: show programs
This commit is contained in:
Naïm Favier 2023-02-07 16:31:17 +01:00 committed by GitHub
parent 32097fc62e
commit dff4ba7132
Failed to generate hash of commit
10 changed files with 159 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
VERSION
View file

@ -1 +1 @@
35
36

31
flake-info/Cargo.lock generated
View file

@ -340,6 +340,7 @@ dependencies = [
"serde_json",
"serde_path_to_error",
"sha2",
"sqlite",
"structopt",
"thiserror",
"tokio",
@ -1347,6 +1348,36 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "sqlite"
version = "0.30.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48"
dependencies = [
"libc",
"sqlite3-sys",
]
[[package]]
name = "sqlite3-src"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
dependencies = [
"cc",
"pkg-config",
]
[[package]]
name = "sqlite3-sys"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
dependencies = [
"libc",
"sqlite3-src",
]
[[package]]
name = "strsim"
version = "0.8.0"

1
flake-info/Cargo.toml
View file

@ -25,6 +25,7 @@ reqwest = { version = "0.11", features = ["json", "blocking"] }
sha2 = "0.9"
pandoc = "0.8.10"
semver = "1.0"
sqlite = "0.30"
elasticsearch = {git = "https://github.com/elastic/elasticsearch-rs", features = ["rustls-tls"], optional = true}

123
flake-info/src/commands/nixpkgs_info.rs
View file

@ -1,20 +1,21 @@
use anyhow::{Context, Result};
use serde_json::Deserializer;
use std::{collections::HashMap, fmt::Display};
use std::collections::{HashMap, HashSet};
use command_run::{Command, LogTo};
use log::error;
use crate::data::import::{NixOption, NixpkgsEntry, Package};
use crate::data::Nixpkgs;
use crate::Source;
pub fn get_nixpkgs_info<T: AsRef<str> + Display>(nixpkgs_channel: T) -> Result<Vec<NixpkgsEntry>> {
pub fn get_nixpkgs_info(nixpkgs: &Source) -> Result<Vec<NixpkgsEntry>> {
let mut command = Command::new("nix-env");
command.add_args(&[
"--json",
"-f",
"<nixpkgs>",
"-I",
format!("nixpkgs={}", nixpkgs_channel.as_ref()).as_str(),
format!("nixpkgs={}", nixpkgs.to_flake_ref()).as_str(),
"--arg",
"config",
"import <nixpkgs/pkgs/top-level/packages-config.nix>",
@ -26,56 +27,94 @@ pub fn get_nixpkgs_info<T: AsRef<str> + Display>(nixpkgs_channel: T) -> Result<V
command.log_to = LogTo::Log;
command.log_output_on_error = true;
let parsed: Result<Vec<NixpkgsEntry>> = command
let cow = command
.run()
.with_context(|| {
format!(
"Failed to gather information about nixpkgs {}",
nixpkgs_channel.as_ref()
)
})
.and_then(|o| {
let output = &*o.stdout_string_lossy();
let de = &mut Deserializer::from_str(output);
let attr_set: HashMap<String, Package> =
serde_path_to_error::deserialize(de).with_context(|| "Could not parse packages")?;
Ok(attr_set
.into_iter()
.map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package })
.collect())
});
.with_context(|| "Failed to gather information about nixpkgs packages")?;
parsed
let output = &*cow.stdout_string_lossy();
let de = &mut Deserializer::from_str(output);
let attr_set: HashMap<String, Package> =
serde_path_to_error::deserialize(de).with_context(|| "Could not parse packages")?;
let mut programs = match nixpkgs {
Source::Nixpkgs(nixpkgs) => get_nixpkgs_programs(nixpkgs)?,
_ => Default::default(),
};
Ok(attr_set
.into_iter()
.map(|(attribute, package)| {
let programs = programs
.remove(&attribute)
.unwrap_or_default()
.into_iter()
.collect();
NixpkgsEntry::Derivation {
attribute,
package,
programs,
}
})
.collect())
}
pub fn get_nixpkgs_options<T: AsRef<str> + Display>(
nixpkgs_channel: T,
) -> Result<Vec<NixpkgsEntry>> {
pub fn get_nixpkgs_programs(nixpkgs: &Nixpkgs) -> Result<HashMap<String, HashSet<String>>> {
let mut command = Command::new("nix-instantiate");
command.add_args(&[
"--eval",
"--json",
"-I",
format!("nixpkgs=channel:nixos-{}", nixpkgs.channel).as_str(),
"--expr",
"toString <nixpkgs/programs.sqlite>",
]);
command.enable_capture();
command.log_to = LogTo::Log;
command.log_output_on_error = true;
let cow = command
.run()
.with_context(|| "Failed to gather information about nixpkgs programs")?;
let output = &*cow.stdout_string_lossy();
let programs_db: &str = serde_json::from_str(output)?;
let conn = sqlite::open(programs_db)?;
let cur = conn
.prepare("SELECT name, package FROM Programs")?
.into_iter();
let mut programs: HashMap<String, HashSet<String>> = HashMap::new();
for row in cur.map(|r| r.unwrap()) {
let name: &str = row.read("name");
let package: &str = row.read("package");
programs
.entry(package.into())
.or_default()
.insert(name.into());
}
Ok(programs)
}
pub fn get_nixpkgs_options(nixpkgs: &Source) -> Result<Vec<NixpkgsEntry>> {
let mut command = Command::with_args("nix", &["eval", "--json"]);
command.add_arg_pair("-f", super::EXTRACT_SCRIPT.clone());
command.add_arg_pair("-I", format!("nixpkgs={}", nixpkgs_channel.as_ref()));
command.add_arg_pair("-I", format!("nixpkgs={}", nixpkgs.to_flake_ref()));
command.add_arg("nixos-options");
command.enable_capture();
command.log_to = LogTo::Log;
command.log_output_on_error = true;
let parsed = command.run().with_context(|| {
format!(
"Failed to gather information about nixpkgs {}",
nixpkgs_channel.as_ref()
)
});
let cow = command
.run()
.with_context(|| "Failed to gather information about nixpkgs options")?;
if let Err(ref e) = parsed {
error!("Command error: {}", e);
}
let output = &*cow.stdout_string_lossy();
let de = &mut Deserializer::from_str(output);
let attr_set: Vec<NixOption> =
serde_path_to_error::deserialize(de).with_context(|| "Could not parse options")?;
parsed.and_then(|o| {
let output = &*o.stdout_string_lossy();
let de = &mut Deserializer::from_str(output);
let attr_set: Vec<NixOption> =
serde_path_to_error::deserialize(de).with_context(|| "Could not parse options")?;
Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect())
})
Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect())
}

9
flake-info/src/data/export.rs
View file

@ -63,6 +63,7 @@ pub enum Derivation {
package_platforms: Vec<String>,
package_outputs: Vec<String>,
package_default_output: Option<String>,
package_programs: Vec<String>,
package_license: Vec<License>,
package_license_set: Vec<String>,
package_maintainers: Vec<Maintainer>,
@ -150,6 +151,7 @@ impl TryFrom<(import::FlakeEntry, super::Flake)> for Derivation {
package_platforms: platforms,
package_outputs: outputs,
package_default_output: Some(default_output),
package_programs: Vec::new(),
package_license,
package_license_set,
package_description: description.clone(),
@ -183,7 +185,11 @@ impl TryFrom<import::NixpkgsEntry> for Derivation {
fn try_from(entry: import::NixpkgsEntry) -> Result<Self, Self::Error> {
Ok(match entry {
import::NixpkgsEntry::Derivation { attribute, package } => {
import::NixpkgsEntry::Derivation {
attribute,
package,
programs,
} => {
let package_attr_set: Vec<_> = attribute.split(".").collect();
let package_attr_set: String = (if package_attr_set.len() > 1 {
package_attr_set[0]
@ -249,6 +255,7 @@ impl TryFrom<import::NixpkgsEntry> for Derivation {
package_platforms: platforms,
package_outputs: package.outputs.into_keys().collect(),
package_default_output: package.default_output,
package_programs: programs,
package_license,
package_license_set,
package_maintainers,

12
flake-info/src/data/import.rs
View file

@ -179,7 +179,11 @@ pub struct Package {
/// Name and Package definition are combined using this struct
#[derive(Debug, Clone)]
pub enum NixpkgsEntry {
Derivation { attribute: String, package: Package },
Derivation {
attribute: String,
package: Package,
programs: Vec<String>,
},
Option(NixOption),
}
@ -435,7 +439,11 @@ mod tests {
let _: Vec<NixpkgsEntry> = map
.into_iter()
.map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package })
.map(|(attribute, package)| NixpkgsEntry::Derivation {
attribute,
package,
programs: Vec::new(),
})
.collect();
}

3
flake-info/src/elastic.rs
View file

@ -97,6 +97,9 @@ lazy_static! {
"package_default_output": {
"type": "keyword"
},
"package_programs": {
"type": "keyword"
},
"package_description": {
"type": "text",
"analyzer": "english",

4
flake-info/src/lib.rs
View file

@ -41,13 +41,13 @@ pub fn process_flake(
pub fn process_nixpkgs(nixpkgs: &Source, kind: &Kind) -> Result<Vec<Export>, anyhow::Error> {
let drvs = if matches!(kind, Kind::All | Kind::Package) {
commands::get_nixpkgs_info(nixpkgs.to_flake_ref())?
commands::get_nixpkgs_info(nixpkgs)?
} else {
Vec::new()
};
let mut options = if matches!(kind, Kind::All | Kind::Option) {
commands::get_nixpkgs_options(nixpkgs.to_flake_ref())?
commands::get_nixpkgs_options(nixpkgs)?
} else {
Vec::new()
};

18
frontend/src/Page/Packages.elm
View file

@ -72,6 +72,7 @@ type alias ResultItemSource =
, pversion : String
, outputs : List String
, default_output : Maybe String
, programs : List String
, description : Maybe String
, longDescription : Maybe String
, licenses : List ResultPackageLicense
@ -476,7 +477,7 @@ viewResultItem nixosChannels channel showInstallDetails show item =
li [] [ text platform ]
maintainersAndPlatforms =
[ div []
div []
[ div []
(List.append [ h4 [] [ text "Maintainers" ] ]
(if List.isEmpty item.source.maintainers then
@ -500,7 +501,16 @@ viewResultItem nixosChannels channel showInstallDetails show item =
)
)
]
]
programs =
div []
[ h4 [] [ text "Programs in ", code [] [ text "/bin" ] ]
, if List.isEmpty item.source.programs then
p [] [ text "This package provides no programs." ]
else
p [] (List.intersperse (text " ") (List.map (\p -> code [] [ text p ]) item.source.programs))
]
longerPackageDetails =
optionals (Just item.source.attr_name == show)
@ -722,6 +732,8 @@ viewResultItem nixosChannels channel showInstallDetails show item =
Maybe.map Tuple.first item.source.flakeUrl
]
:: maintainersAndPlatforms
:: programs
:: []
)
]
@ -908,6 +920,7 @@ makeRequestBody query from size maybeBuckets sort =
filterByBuckets
"package_attr_name"
[ ( "package_attr_name", 9.0 )
, ( "package_programs", 9.0 )
, ( "package_pname", 6.0 )
, ( "package_description", 1.3 )
, ( "package_longDescription", 1.0 )
@ -946,6 +959,7 @@ decodeResultItemSource =
|> Json.Decode.Pipeline.required "package_pversion" Json.Decode.string
|> Json.Decode.Pipeline.required "package_outputs" (Json.Decode.list Json.Decode.string)
|> Json.Decode.Pipeline.required "package_default_output" (Json.Decode.nullable Json.Decode.string)
|> Json.Decode.Pipeline.required "package_programs" (Json.Decode.list Json.Decode.string)
|> Json.Decode.Pipeline.required "package_description" (Json.Decode.nullable Json.Decode.string)
|> Json.Decode.Pipeline.required "package_longDescription" (Json.Decode.nullable Json.Decode.string)
|> Json.Decode.Pipeline.required "package_license" (Json.Decode.list decodeResultPackageLicense)

7
frontend/src/Search.elm
View file

@ -1232,7 +1232,12 @@ searchFields query mainField fields =
let
allFields =
fields
|> List.map (\( field, score ) -> [ field ++ "^" ++ String.fromFloat score, field ++ ".*^" ++ String.fromFloat score ])
|> List.map
(\( field, score ) ->
[ field ++ "^" ++ String.fromFloat score
, field ++ ".*^" ++ String.fromFloat (score * 0.6)
]
)
|> List.concat
queryWords =