From 3dfcf4fd453c516a34bcc1052568fd4037a26a54 Mon Sep 17 00:00:00 2001 From: Yannik Sander Date: Sun, 26 Dec 2021 16:03:09 +0100 Subject: [PATCH] Fix/explicit printing (#395) * Extract pandoc function to own module * Bump schema version * Make string handling more explicit * Update logging and pandoc crates * Improve serializing error handling * Serialize values as string (for elastic) * Perform option doc parsing entirely in rust * Show non pandoc'ed results as code elements * Parse correct html string * Change expected `option_type` type to String * Allow treat string unparsable as html like non-docbook strings * Improve deserializing error reporting using serde_path_to_error * Format code --- VERSION | 2 +- flake-info/Cargo.lock | 19 +++-- flake-info/Cargo.toml | 5 +- flake-info/default.nix | 1 - flake-info/src/bin/flake-info.rs | 7 +- flake-info/src/commands/flake_info.nix | 14 ++-- flake-info/src/commands/nix_flake_attrs.rs | 6 +- flake-info/src/commands/nixpkgs_info.rs | 12 ++-- flake-info/src/data/export.rs | 84 ++++++++-------------- flake-info/src/data/import.rs | 52 +++++++++++++- flake-info/src/data/mod.rs | 1 + flake-info/src/data/pandoc.rs | 64 +++++++++++++++++ src/Page/Options.elm | 24 +++++-- 13 files changed, 202 insertions(+), 89 deletions(-) create mode 100644 flake-info/src/data/pandoc.rs diff --git a/VERSION b/VERSION index a45fd52..7273c0f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24 +25 diff --git a/flake-info/Cargo.lock b/flake-info/Cargo.lock index db3b4d7..1fc0212 100644 --- a/flake-info/Cargo.lock +++ b/flake-info/Cargo.lock @@ -273,9 +273,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.8.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ "atty", "humantime", @@ -310,6 +310,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serde_path_to_error", "sha2", "structopt", "tempfile", @@ -822,8 +823,9 @@ dependencies = [ [[package]] name = "pandoc" -version = "0.8.6" -source = "git+https://github.com/ysndr/rust-pandoc#c16ba426cdea58084be731ef8028ba58ca670b40" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eac785b7de8de25c5ec48b3a9df1be552de03906f99145ed6d7da3d696c0dbb" dependencies = [ "itertools", ] @@ -1197,6 +1199,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0421d4f173fab82d72d6babf36d57fae38b994ca5c2d78e704260ba6d12118b" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.0" diff --git a/flake-info/Cargo.toml b/flake-info/Cargo.toml index 3b153e6..cb5e679 100644 --- a/flake-info/Cargo.toml +++ b/flake-info/Cargo.toml @@ -10,12 +10,13 @@ edition = "2018" clap = "^2.33" serde = {version="1.0", features = ["derive"]} serde_json = "1.0" +serde_path_to_error = "0.1.5" toml = "0.5" anyhow = "1.0" thiserror = "1.0" structopt = "0.3" command-run = "0.13" -env_logger = "0.8" +env_logger = "0.9" log = "0.4" tempfile = "3" lazy_static = "1.4" @@ -23,7 +24,7 @@ fancy-regex = "0.6" tokio = { version = "*", features = ["full"] } reqwest = { version = "0.11", features = ["json", "blocking"] } sha2 = "0.9" -pandoc = { git = "https://github.com/ysndr/rust-pandoc" } +pandoc = "0.8" elasticsearch = {git = "https://github.com/elastic/elasticsearch-rs", features = ["rustls-tls"]} diff --git a/flake-info/default.nix b/flake-info/default.nix index 05395d9..3d381cc 100644 --- a/flake-info/default.nix +++ b/flake-info/default.nix @@ -6,7 +6,6 @@ rustPlatform.buildRustPackage rec { lockFile = ./Cargo.lock; outputHashes = { "elasticsearch-8.0.0-alpha.1" = "sha256-gjmk3Q3LTAvLhzQ+k1knSp1HBwtqNiubjXNnLy/cS5M="; - "pandoc-0.8.6" = "sha256-NsHDzqWjQ17cznjOSpXOdUOhJjAO28Z6QZ6Mn6afVVs="; }; }; nativeBuildInputs = [ pkg-config ]; diff --git a/flake-info/src/bin/flake-info.rs b/flake-info/src/bin/flake-info.rs index 37ec618..52791c0 100644 --- a/flake-info/src/bin/flake-info.rs +++ b/flake-info/src/bin/flake-info.rs @@ -273,14 +273,15 @@ async fn run_command( .collect::>(); if !errors.is_empty() { - if exports.is_empty() { return Err(FlakeInfoError::Group(errors)); } warn!("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); - warn!("Some group members could not be evaluated: {}", FlakeInfoError::Group(errors)); + warn!( + "Some group members could not be evaluated: {}", + FlakeInfoError::Group(errors) + ); warn!("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); - } let hash = { diff --git a/flake-info/src/commands/flake_info.nix b/flake-info/src/commands/flake_info.nix index 76db6d3..58d2a52 100644 --- a/flake-info/src/commands/flake_info.nix +++ b/flake-info/src/commands/flake_info.nix @@ -72,7 +72,7 @@ let cleanUpOption = module: opt: let - applyOnAttr = n: f: lib.optionalAttrs (lib.hasAttr n opt) { ${n} = f opt.${n}; }; + applyOnAttr = n: f: lib.optionalAttrs (builtins.hasAttr n opt) { ${n} = f opt.${n}; }; mkDeclaration = decl: let discard = lib.concatStringsSep "/" (lib.take 4 (lib.splitString "/" decl)) + "/"; @@ -82,20 +82,18 @@ let # Replace functions by the string substFunction = x: - if x ? _type && (x._type == "literalExample" || x._type == "literalExpression" || x._type == "literalDocBook") then - x.text - else if builtins.isAttrs x then - lib.mapAttrs (name: substFunction) x + if builtins.isAttrs x then + lib.mapAttrs (_:substFunction ) x else if builtins.isList x then map substFunction x else if lib.isFunction x then - "" + "function" else - x; + x; in opt - // applyOnAttr "example" substFunction // applyOnAttr "default" substFunction + // applyOnAttr "example" substFunction # (_: { __type = "function"; }) // applyOnAttr "type" substFunction // applyOnAttr "declarations" (map mkDeclaration) // lib.optionalAttrs (!isNixOS) { flake = [ flake module ]; }; diff --git a/flake-info/src/commands/nix_flake_attrs.rs b/flake-info/src/commands/nix_flake_attrs.rs index 8f048be..2bf4bcf 100644 --- a/flake-info/src/commands/nix_flake_attrs.rs +++ b/flake-info/src/commands/nix_flake_attrs.rs @@ -2,6 +2,7 @@ use crate::data::import::{FlakeEntry, Kind}; use anyhow::{Context, Result}; use command_run::{Command, LogTo}; use log::debug; +use serde_json::Deserializer; use std::fmt::Display; use std::fs::File; use std::io::Write; @@ -43,8 +44,9 @@ pub fn get_derivation_info + Display>( .run() .with_context(|| format!("Failed to gather information about {}", flake_ref)) .and_then(|o| { - debug!("stderr: {}", o.stderr_string_lossy()); - serde_json::de::from_str(&o.stdout_string_lossy()) + let output = &*o.stdout_string_lossy(); + let de = &mut Deserializer::from_str(output); + serde_path_to_error::deserialize(de) .with_context(|| format!("Failed to analyze flake {}", flake_ref)) }); parsed diff --git a/flake-info/src/commands/nixpkgs_info.rs b/flake-info/src/commands/nixpkgs_info.rs index 09069ae..9fed34b 100644 --- a/flake-info/src/commands/nixpkgs_info.rs +++ b/flake-info/src/commands/nixpkgs_info.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use serde_json::Deserializer; use std::io::Write; use std::{collections::HashMap, fmt::Display, fs::File}; @@ -38,9 +39,10 @@ pub fn get_nixpkgs_info + Display>(nixpkgs_channel: T) -> Result = - serde_json::de::from_str(&o.stdout_string_lossy())?; + serde_path_to_error::deserialize(de).with_context(|| "Could not parse packages")?; Ok(attr_set .into_iter() .map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package }) @@ -91,8 +93,10 @@ pub fn get_nixpkgs_options + Display>( } parsed.and_then(|o| { - debug!("stderr: {}", o.stderr_string_lossy()); - let attr_set: Vec = serde_json::de::from_str(&o.stdout_string_lossy())?; + let output = &*o.stdout_string_lossy(); + let de = &mut Deserializer::from_str(output); + let attr_set: Vec = + serde_path_to_error::deserialize(de).with_context(|| "Could not parse options")?; Ok(attr_set.into_iter().map(NixpkgsEntry::Option).collect()) }) } diff --git a/flake-info/src/data/export.rs b/flake-info/src/data/export.rs index 4e08e41..ad55480 100644 --- a/flake-info/src/data/export.rs +++ b/flake-info/src/data/export.rs @@ -1,14 +1,13 @@ /// This module defines the unified putput format as expected by the elastic search /// Additionally, we implement converseions from the two possible input formats, i.e. /// Flakes, or Nixpkgs. -use std::path::PathBuf; +use std::{convert::TryInto, path::PathBuf}; +use super::{import::DocValue, pandoc::PandocExt}; use crate::data::import::NixOption; use log::error; -use pandoc::{ - InputFormat, InputKind, OutputFormat, OutputKind, PandocError, PandocOption, PandocOutput, -}; use serde::{Deserialize, Serialize}; +use serde_json::Value; use super::{ import, @@ -107,9 +106,9 @@ pub enum Derivation { option_type: Option, - option_default: Option, + option_default: Option, - option_example: Option, + option_example: Option, option_flake: Option<(String, String)>, }, @@ -290,49 +289,23 @@ impl From for Derivation { flake, }: import::NixOption, ) -> Self { - let citeref_filter = { - let mut p = FILTERS_PATH.clone(); - p.push("docbook-reader/citerefentry-to-rst-role.lua"); - p - }; - let man_filter = { - let mut p = FILTERS_PATH.clone(); - p.push("link-unix-man-references.lua"); - p - }; - - let description = if let Some(description) = description { - let mut pandoc = pandoc::new(); - let description_xml = format!( - " - - {} - - ", - description - ); - - pandoc.set_input(InputKind::Pipe(description_xml)); - pandoc.set_input_format(InputFormat::DocBook, Vec::new()); - pandoc.set_output(OutputKind::Pipe); - pandoc.set_output_format(OutputFormat::Html, Vec::new()); - pandoc.add_options(&[ - PandocOption::LuaFilter(citeref_filter), - PandocOption::LuaFilter(man_filter), - ]); - - let result = pandoc.execute().expect(&format!( - "Pandoc could not parse documentation of '{}'", - name - )); - - match result { - PandocOutput::ToBuffer(description) => Some(description), - _ => unreachable!(), - } - } else { - description - }; + let description = description + .as_ref() + .map(PandocExt::render) + .transpose() + .expect(&format!("Could not render descript of `{}`", name)); + let option_default = default; + // .map(TryInto::try_into) + // .transpose() + // .expect(&format!("Could not render option_default of `{}`", name)); + let option_example = example; + // .map(TryInto::try_into) + // .transpose() + // .expect(&format!("Could not render option_example of `{}`", name)); + let option_type = option_type; + // .map(TryInto::try_into) + // .transpose() + // .expect(&format!("Could not render option_type of `{}`", name)); Derivation::Option { option_source: declarations.get(0).map(Clone::clone), @@ -340,8 +313,8 @@ impl From for Derivation { option_name_reverse: Reverse(name.clone()), option_description: description.clone(), option_description_reverse: description.map(Reverse), - option_default: default.map(print_value), - option_example: example.map(print_value), + option_default, + option_example, option_flake: flake, option_type, option_name_query: AttributeQuery::new(&name), @@ -435,14 +408,15 @@ mod tests { let option: NixOption = serde_json::from_str(r#" { "declarations":["/nix/store/s1q1238ahiks5a4g6j6qhhfb3rlmamvz-source/nixos/modules/system/boot/luksroot.nix"], - "default":"", + "default": {"one": 1, "two" : { "three": "tree", "four": []}}, "description":"Commands that should be run right after we have mounted our LUKS device.\n", - "example":"oneline\ntwoline\nthreeline\n", + "example":null, "internal":false, "loc":["boot","initrd","luks","devices","","postOpenCommands"], "name":"boot.initrd.luks.devices..postOpenCommands", - "readOnly":false,"type": - "strings concatenated with \"\\n\"","visible":true + "readOnly":false, + "type": "boolean", + "visible":true }"#).unwrap(); let option: Derivation = option.into(); diff --git a/flake-info/src/data/import.rs b/flake-info/src/data/import.rs index 78024c7..599e01a 100644 --- a/flake-info/src/data/import.rs +++ b/flake-info/src/data/import.rs @@ -1,13 +1,19 @@ +use std::collections::{BTreeMap, HashMap}; +use std::convert::TryInto; use std::fmt::{self, write, Display}; use std::marker::PhantomData; use std::{path::PathBuf, str::FromStr}; use clap::arg_enum; +use log::warn; +use pandoc::PandocError; use serde::de::{self, MapAccess, Visitor}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; use thiserror::Error; +use super::pandoc::PandocExt; +use super::prettyprint::{self, print_value}; use super::system::System; use super::utility::{Flatten, OneOrMany}; @@ -50,16 +56,55 @@ pub struct NixOption { pub description: Option, pub name: String, + #[serde(rename = "type")] /// Nix generated description of the options type pub option_type: Option, - pub default: Option, - pub example: Option, + pub default: Option, + pub example: Option, /// If defined in a flake, contains defining flake and module pub flake: Option<(String, String)>, } +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum DocValue { + Literal(Literal), + Value(Value), +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(tag = "_type", content = "text")] +pub enum Literal { + #[serde(rename = "literalExpression")] + LiteralExpression(Value), + #[serde(rename = "literalExample")] + LiteralExample(Value), + #[serde(rename = "literalDocBook")] + LiteralDocBook(String), +} + +impl Serialize for DocValue { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + DocValue::Literal(Literal::LiteralExample(s) | Literal::LiteralExpression(s)) => { + return serializer.serialize_some(&s); + } + DocValue::Literal(Literal::LiteralDocBook(doc_book)) => { + return serializer.serialize_str(&doc_book.render().unwrap_or_else(|e| { + warn!("Could not render docbook content: {}", e); + doc_book.to_owned() + })); + } + DocValue::Value(v) => serializer.serialize_str(&print_value(v.to_owned())), + } + } +} + /// Package as defined in nixpkgs /// These packages usually have a "more" homogenic structure that is given by /// nixpkgs @@ -299,4 +344,7 @@ mod tests { .map(|(attribute, package)| NixpkgsEntry::Derivation { attribute, package }) .collect(); } + + #[test] + fn test_option_parsing() {} } diff --git a/flake-info/src/data/mod.rs b/flake-info/src/data/mod.rs index 590d73b..2915a92 100644 --- a/flake-info/src/data/mod.rs +++ b/flake-info/src/data/mod.rs @@ -1,6 +1,7 @@ mod export; mod flake; pub mod import; +mod pandoc; mod prettyprint; mod source; mod system; diff --git a/flake-info/src/data/pandoc.rs b/flake-info/src/data/pandoc.rs new file mode 100644 index 0000000..4f56d3b --- /dev/null +++ b/flake-info/src/data/pandoc.rs @@ -0,0 +1,64 @@ +use std::path::PathBuf; + +use lazy_static::lazy_static; +use log::debug; +use pandoc::{ + InputFormat, InputKind, OutputFormat, OutputKind, PandocError, PandocOption, PandocOutput, +}; + +lazy_static! { + static ref FILTERS_PATH: PathBuf = std::env::var("NIXPKGS_PANDOC_FILTERS_PATH") + .unwrap_or("".into()) + .into(); +} + +pub trait PandocExt { + fn render(&self) -> Result; +} + +impl> PandocExt for T { + fn render(&self) -> Result { + if !self.as_ref().contains("{}", + self.as_ref() + )); + } + + let citeref_filter = { + let mut p = FILTERS_PATH.clone(); + p.push("docbook-reader/citerefentry-to-rst-role.lua"); + p + }; + let man_filter = { + let mut p = FILTERS_PATH.clone(); + p.push("link-unix-man-references.lua"); + p + }; + let mut pandoc = pandoc::new(); + let wrapper_xml = format!( + " + + {} + + ", + self.as_ref() + ); + + pandoc.set_input(InputKind::Pipe(wrapper_xml)); + pandoc.set_input_format(InputFormat::DocBook, Vec::new()); + pandoc.set_output(OutputKind::Pipe); + pandoc.set_output_format(OutputFormat::Html, Vec::new()); + pandoc.add_options(&[ + PandocOption::LuaFilter(citeref_filter), + PandocOption::LuaFilter(man_filter), + ]); + + pandoc.execute().map(|result| match result { + PandocOutput::ToBuffer(description) => { + format!("{}", description) + } + _ => unreachable!(), + }) + } +} diff --git a/src/Page/Options.elm b/src/Page/Options.elm index d35b66e..b4ee8af 100644 --- a/src/Page/Options.elm +++ b/src/Page/Options.elm @@ -171,12 +171,14 @@ viewResultItem : viewResultItem channel _ show item = let showHtml value = - case Html.Parser.run value of - Ok nodes -> - Html.Parser.Util.toVirtualDom nodes + case Html.Parser.run <| String.trim value of + Ok [ Html.Parser.Element "rendered-docbook" _ nodes ] -> + Just <| Html.Parser.Util.toVirtualDom nodes + Ok _ -> + Nothing Err _ -> - [] + Nothing default = "Not given" @@ -214,15 +216,23 @@ viewResultItem channel _ show item = , div [] [ text "Description" ] , div [] <| (item.source.description - |> Maybe.map showHtml + |> Maybe.andThen showHtml |> Maybe.withDefault [] ) , div [] [ text "Default value" ] - , div [] [ withEmpty (wrapped asPreCode) item.source.default ] + , div [] <| + (item.source.default + |> Maybe.map (\value -> Maybe.withDefault [ asPreCode value ] (showHtml value)) + |> Maybe.withDefault [ asPre default ] + ) , div [] [ text "Type" ] , div [] [ withEmpty asPre item.source.type_ ] , div [] [ text "Example" ] - , div [] [ withEmpty (wrapped asPreCode) item.source.example ] + , div [] <| + (item.source.example + |> Maybe.map (\value -> Maybe.withDefault [ asPreCode value ] (showHtml value)) + |> Maybe.withDefault [ asPre default ] + ) , div [] [ text "Declared in" ] , div [] <| findSource channel item.source ]