Show package outputs (#419)

As mentioned in [^1] this PR includes a bump in swap size of the github action evaluating nixpkgs.
This is subject to change once a tentative change to Nix is merged.

^1: https://github.com/NixOS/nixos-search/pull/419#issuecomment-1065356169

—

* flake-info: use packages-config.nix straight from nixpkgs

No need to override anything anymore, see discussion at
https://github.com/NixOS/nixos-search/pull/343#issuecomment-1021147104

* flake-info: query package outputs

`package_outputs` is now set to the actual outputs of the derivation
instead of `meta.outputsToInstall`

Also updates nixpkgs to get Rust 1.57 which has HashMap.into_keys

* frontend: show package outputs

* Drop 21.05 channel

* Increase swap space in import-nixpkgs worker

* Bump VERSION

* frontend: improvements and refactoring

- move licenses to the end (without a line break)
- pluralise "Licenses:" conditionally
- change the homepage emoji from a house to a world
- replace backwards List.append pipelines with ++ chains
- avoid text nodes in <ul> outside of <li>
- improve rendering of maintainers

* frontend: sort package outputs

.github/workflows/import-nixpkgs.yml

@@ -17,7 +17,6 @@ jobs:
         channel:
           - unstable
           - 21.11
-          - 21.05
 
     env:
       RUST_LOG: debug
@@ -25,6 +24,11 @@ jobs:
       FI_ES_URL: ${{ secrets.ELASTICSEARCH_URL }}
 
     steps:
+    - name: Increase swap space
+      uses: pierotofy/set-swap-space@v1.0
+      with:
+        swap-size-gb: 10
+
     - name: Checking out the repository
       uses: actions/checkout@v3
 

VERSION

@@ -1 +1 @@
-26
+27

flake-info/src/commands/nix_flake_attrs.rs

@@ -25,8 +25,8 @@ pub fn get_derivation_info<T: AsRef<str> + Display>(
 
     let mut command = Command::with_args("nix", ARGS.iter());
     command.add_arg_pair("-f", script_path.as_os_str());
-    let command = command.add_args(["--arg", "flake", flake_ref.as_ref()].iter());
+    command.add_args(["--arg", "flake", flake_ref.as_ref()].iter());
-    let command = command.add_arg(kind.as_ref());
+    command.add_arg(kind.as_ref());
     if temp_store {
         let temp_store_path = PathBuf::from("/tmp/flake-info-store");
         if !temp_store_path.exists() {
@@ -36,7 +36,7 @@ pub fn get_derivation_info<T: AsRef<str> + Display>(
         command.add_arg_pair("--store", temp_store_path.canonicalize()?);
     }
     command.add_args(extra);
-    let mut command = command.enable_capture();
+    command.enable_capture();
     command.log_to = LogTo::Log;
     command.log_output_on_error = true;
 

flake-info/src/commands/nix_flake_info.rs

@@ -14,7 +14,7 @@ pub fn get_flake_info<T: AsRef<str> + Display>(
 ) -> Result<Flake> {
     let args = ["flake", "metadata", "--json", "--no-write-lock-file"];
     let mut command = Command::with_args("nix", args);
-    let command = command.add_arg(flake_ref.as_ref());
+    command.add_arg(flake_ref.as_ref());
     if temp_store {
         let temp_store_path = PathBuf::from("/tmp/flake-info-store");
         if !temp_store_path.exists() {
@@ -24,7 +24,7 @@ pub fn get_flake_info<T: AsRef<str> + Display>(
         command.add_arg_pair("--store", temp_store_path.canonicalize()?);
     }
     command.add_args(extra);
-    let mut command = command.enable_capture();
+    command.enable_capture();
     command.log_to = LogTo::Log;
     command.log_output_on_error = true;
 

flake-info/src/commands/nixpkgs_info.rs

@@ -3,34 +3,41 @@ use serde_json::Deserializer;
 use std::io::Write;
 use std::{collections::HashMap, fmt::Display, fs::File};
 
-use command_run::Command;
+use command_run::{Command, LogTo};
 use log::{debug, error};
 
 use crate::data::import::{NixOption, NixpkgsEntry, Package};
 
-const NIXPKGS_SCRIPT: &str = include_str!("packages-config.nix");
 const FLAKE_INFO_SCRIPT: &str = include_str!("flake_info.nix");
 
 pub fn get_nixpkgs_info<T: AsRef<str> + Display>(nixpkgs_channel: T) -> Result<Vec<NixpkgsEntry>> {
-    let script_dir = tempfile::tempdir()?;
-    let script_path = script_dir.path().join("packages-config.nix");
-    writeln!(File::create(&script_path)?, "{}", NIXPKGS_SCRIPT)?;
-
     let mut command = Command::new("nix-env");
-    let command = command.enable_capture();
+    command.add_args(&[
-    let command = command.add_args(&[
         "-f",
         "<nixpkgs>",
         "-I",
         format!("nixpkgs={}", nixpkgs_channel.as_ref()).as_str(),
         "--arg",
         "config",
-        format!("import {}", script_path.to_str().unwrap()).as_str(),
+        "import <nixpkgs/pkgs/top-level/packages-config.nix>",
         "-qa",
         "--meta",
+        "--out-path",
         "--json",
     ]);
 
+    // Nix might fail to evaluate some disallowed packages
+    let mut env = HashMap::new();
+    env.insert("NIXPKGS_ALLOW_BROKEN".into(), "1".into());
+    env.insert("NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM".into(), "1".into());
+    env.insert("NIXPKGS_ALLOW_UNFREE".into(), "1".into());
+    env.insert("NIXPKGS_ALLOW_INSECURE".into(), "1".into());
+    command.env = env;
+
+    command.enable_capture();
+    command.log_to = LogTo::Log;
+    command.log_output_on_error = true;
+
     let parsed: Result<Vec<NixpkgsEntry>> = command
         .run()
         .with_context(|| {
@@ -61,8 +68,7 @@ pub fn get_nixpkgs_options<T: AsRef<str> + Display>(
     writeln!(File::create(&script_path)?, "{}", FLAKE_INFO_SCRIPT)?;
 
     let mut command = Command::new("nix");
-    let command = command.enable_capture();
+    command.add_args(&[
-    let mut command = command.add_args(&[
         "eval",
         "--json",
         "-f",
@@ -75,13 +81,18 @@ pub fn get_nixpkgs_options<T: AsRef<str> + Display>(
         "nixos-options",
     ]);
 
-    // Nix might fail to evaluate some options that reference insecure packages
+    // Nix might fail to evaluate some options that reference disallowed packages
     let mut env = HashMap::new();
-    env.insert("NIXPKGS_ALLOW_INSECURE".into(), "1".into());
+    env.insert("NIXPKGS_ALLOW_BROKEN".into(), "1".into());
+    env.insert("NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM".into(), "1".into());
     env.insert("NIXPKGS_ALLOW_UNFREE".into(), "1".into());
-
+    env.insert("NIXPKGS_ALLOW_INSECURE".into(), "1".into());
     command.env = env;
 
+    command.enable_capture();
+    command.log_to = LogTo::Log;
+    command.log_output_on_error = true;
+
     let parsed = command.run().with_context(|| {
         format!(
             "Failed to gather information about nixpkgs {}",

flake-info/src/commands/packages-config.nix

@@ -1,4 +0,0 @@
-import <nixpkgs/pkgs/top-level/packages-config.nix> // {
-  # Do *NOT* list unfree packages
-  allowUnfree = false;
-}

flake-info/src/data/export.rs

@@ -254,7 +254,7 @@ impl From<import::NixpkgsEntry> for Derivation {
                         .platforms
                         .map(Flatten::flatten)
                         .unwrap_or_default(),
-                    package_outputs: package.meta.outputs.unwrap_or_default(),
+                    package_outputs: package.outputs.into_keys().collect(),
                     package_license,
                     package_license_set,
                     package_maintainers,

flake-info/src/data/import.rs

@@ -124,7 +124,10 @@ impl Serialize for DocValue {
 pub struct Package {
     pub pname: String,
     pub version: String,
+    #[serde(default)]
+    pub outputs: HashMap<String, String>,
     pub system: String,
+    #[serde(default)]
     pub meta: Meta,
 }
 
@@ -138,10 +141,8 @@ pub enum NixpkgsEntry {
 
 /// Most information about packages in nixpkgs is contained in the meta key
 /// This struct represents a subset of that metadata
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
 pub struct Meta {
-    #[serde(rename = "outputsToInstall")]
-    pub outputs: Option<Vec<String>>,
     pub license: Option<OneOrMany<StringOrStruct<License>>>,
     pub maintainers: Option<Flatten<Maintainer>>,
     pub homepage: Option<OneOrMany<String>>,

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1631118067,
+        "lastModified": 1642903813,
-        "narHash": "sha256-tEcFvm3a6ToeBGwHdjfB2mVQwa4LZCZTQYE2LnY3ycA=",
+        "narHash": "sha256-0lNfGW8sNfyTrixoQhVG00Drl/ECaf5GbfKAQ1ZDoyE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "09cd65b33c5653d7d2954fef4b9f0e718c899743",
+        "rev": "689b76bcf36055afdeb2e9852f5ecdd2bf483f87",
         "type": "github"
       },
       "original": {

src/Page/Packages.elm

@@ -67,6 +67,7 @@ type alias ResultItemSource =
     { attr_name : String
     , pname : String
     , pversion : String
+    , outputs : List String
     , description : Maybe String
     , longDescription : Maybe String
     , licenses : List ResultPackageLicense
@@ -302,6 +303,8 @@ viewResultItem :
     -> Html Msg
 viewResultItem channel showInstallDetails show item =
     let
+        optionals b l = if b then l else []
+
         cleanPosition =
             Regex.fromString "^[0-9a-f]+\\.tar\\.gz\\/"
                 |> Maybe.withDefault Regex.never
@@ -324,38 +327,42 @@ viewResultItem channel showInstallDetails show item =
                 [ text title ]
 
         shortPackageDetails =
-            ul []
+            ul [] (
-                (renderSource item channel trapClick createShortDetailsItem createGithubUrl
+                [ li []
-                    |> List.append
+                    [ text "Name: "
-                        (item.source.homepage
+                    , code [] [ text item.source.pname ]
+                    ]
+                ]
+                ++ optionals (item.source.pversion /= "")
+                    [ li []
+                        [ text "Version: "
+                        , strong [] [ text item.source.pversion ]
+                        ]
+                    ]
+                ++ optionals (List.length item.source.outputs > 1)
+                    [ li [] (
+                        text "Outputs: "
+                        :: (item.source.outputs
+                            |> List.sort
+                            |> List.map (\o -> code [] [ text o ])
+                            |> List.intersperse (text " "))
+                        )
+                    ]
+                ++ (
+                    item.source.homepage
                     |> List.head
                     |> Maybe.map
                         (\x ->
                             [ li [ trapClick ]
-                                        [ createShortDetailsItem "🏡 Homepage" x ]
+                                [ createShortDetailsItem "🌐 Homepage" x ]
                             ]
                         )
                     |> Maybe.withDefault []
                 )
-                    |> List.append
+                ++ renderSource item channel trapClick createShortDetailsItem createGithubUrl
-                        (if item.source.pversion == "" then
+                ++ (
-                            []
+                    let
-
+                        licenses = item.source.licenses |> List.filterMap
-                         else
-                            [ text "Version: "
-                            , li [] [ strong [] [ text item.source.pversion ] ]
-                            ]
-                        )
-                    |> List.append
-                        [ text "Name: "
-                        , li [] [ code [] [ text item.source.pname ] ]
-                        ]
-                    |> List.append
-                        [ br [] []
-                        ]
-                    |> List.append
-                        (item.source.licenses
-                            |> List.filterMap
                             (\license ->
                                 case ( license.fullName, license.url ) of
                                     ( Nothing, Nothing ) ->
@@ -370,63 +377,54 @@ viewResultItem channel showInstallDetails show item =
                                     ( Just fullName, Just url ) ->
                                         Just (createShortDetailsItem fullName url)
                             )
-                            |> List.intersperse (text " ▪ ")
+                    in
-                            |> (\x -> [ li [] (List.append [ text "License(s): " ] x) ])
+                    optionals (licenses /= [])
+                        [ li [] (
+                            text ("License" ++ (if List.length licenses == 1 then "" else "s") ++ ": ")
+                            :: List.intersperse (text " ▪ ") licenses
+                        ) ]
                 )
             )
 
         showMaintainer maintainer =
-            li []
+            let
-                [ div []
+                optionalLink url node = case url of
-                    [ a
+                    Just u -> a [ href u] [ node ]
-                        [ href <|
+                    Nothing -> node
-                            case maintainer.github of
-                                Just github ->
-                                    "https://github.com/" ++ github
 
-                                Nothing ->
+                maybe m d = Maybe.withDefault d m
-                                    "#"
+            in
-                        ]
+            li [] (
-                        [ text <| Maybe.withDefault "" maintainer.name ++ " <" ++ Maybe.withDefault "" maintainer.email ++ ">" ]
+                optionalLink
-                    , a
+                    (Maybe.map (String.append "https://github.com/") maintainer.github)
-                        [ href <|
+                    (text <| maybe maintainer.name <| maybe maintainer.github "Unknown")
-                            case maintainer.email of
+                :: case maintainer.email of
                     Just email ->
-                                    "mailto:" ++ email
+                        [ text " <"
-
+                        , a [ href ("mailto:" ++ email) ] [ text email ]
-                                Nothing ->
+                        , text ">" ]
-                                    "#"
+                    Nothing -> []
-                        ]
+                )
-                        [ text "(mail)" ]
-                    ]
-                ]
 
         mailtoAllMaintainers maintainers =
             let
-                maintainerMails =
+                maintainerMails = List.filterMap (\m -> m.email) maintainers
-                    List.filterMap (\m -> m.email) maintainers
             in
-            li []
+            optionals (List.length maintainerMails > 1)
+                [ li []
                     [ a
-                    [ href <|
+                        [ href ("mailto:" ++ String.join "," maintainerMails) ]
-                        ("mailto:" ++ String.join "," maintainerMails)
+                        [ text "✉️ Mail to all maintainers" ]
                     ]
-                    [ text "Mail to all maintainers" ]
                 ]
 
         showPlatform platform =
             case Search.channelDetailsFromId channel of
                 Just channelDetails ->
                     let
-                        url =
+                        url = "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ platform
-                            "https://hydra.nixos.org/job/" ++ channelDetails.jobset ++ "/nixpkgs." ++ item.source.attr_name ++ "." ++ platform
                     in
-                    li []
+                    li [] [ a [ href url ] [ text platform ] ]
-                        [ a
-                            [ href url
-                            ]
-                            [ text platform ]
-                        ]
 
                 Nothing ->
                     li [] [ text platform ]
@@ -439,11 +437,10 @@ viewResultItem channel showInstallDetails show item =
                             [ p [] [ text "This package has no maintainers." ] ]
 
                          else
-                            [ ul []
+                            [ ul [] (
-                                (List.singleton (mailtoAllMaintainers item.source.maintainers)
+                                List.map showMaintainer item.source.maintainers
-                                    |> List.append (List.map showMaintainer item.source.maintainers)
+                                ++ mailtoAllMaintainers item.source.maintainers
-                                )
+                            ) ]
-                            ]
                         )
                     )
                 , div []
@@ -459,15 +456,9 @@ viewResultItem channel showInstallDetails show item =
             ]
 
         longerPackageDetails =
-            if Just item.source.attr_name == show then
+            optionals (Just item.source.attr_name == show)
                 [ div [ trapClick ]
-                    (maintainersAndPlatforms
+                    (
-                        |> List.append
-                            (item.source.longDescription
-                                |> Maybe.map (\desc -> [ p [] [ text desc ] ])
-                                |> Maybe.withDefault []
-                            )
-                        |> List.append
                         [ div []
                             [ h4 []
                                 [ text "How to install "
@@ -575,12 +566,15 @@ viewResultItem channel showInstallDetails show item =
                                         Maybe.map Tuple.first item.source.flakeUrl
                             ]
                         ]
+                        ++ (
+                            item.source.longDescription
+                            |> Maybe.map (\desc -> [ p [] [ text desc ] ])
+                            |> Maybe.withDefault []
+                        )
+                        ++ maintainersAndPlatforms
                     )
                 ]
 
-            else
-                []
-
         toggle =
             SearchMsg (Search.ShowDetails item.source.attr_name)
 
@@ -603,7 +597,6 @@ viewResultItem channel showInstallDetails show item =
                             [ text item.source.attr_name ]
                       ]
 
-
                 _ ->
                     [ a
                         [ onClick toggle
@@ -617,52 +610,39 @@ viewResultItem channel showInstallDetails show item =
         , classList [ ( "opened", isOpen ) ]
         , Search.elementId item.source.attr_name
         ]
-        ([]
+        (
-            |> List.append longerPackageDetails
-            |> List.append
             [ span [] flakeOrNixpkgs
             , div [] [ text <| Maybe.withDefault "" item.source.description ]
             , shortPackageDetails
             , Search.showMoreButton toggle isOpen
-                ]
+            ] ++ longerPackageDetails
         )
 
 
 renderSource : Search.ResultItem ResultItemSource -> String -> Html.Attribute Msg -> (String -> String -> Html Msg) -> (String -> String -> String) -> List (Html Msg)
 renderSource item channel trapClick createShortDetailsItem createGithubUrl =
     let
-        postion =
+        makeLink text url = [ li [ trapClick ] [ createShortDetailsItem text url ] ]
+
+        position =
             item.source.position
                 |> Maybe.map
-                    (\position ->
+                    (\pos ->
                         case Search.channelDetailsFromId channel of
                             Nothing ->
                                 []
-
                             Just channelDetails ->
-                                [ li [ trapClick ]
+                                makeLink "📦 Source" (createGithubUrl channelDetails.branch pos)
-                                    [ createShortDetailsItem
-                                        "📦 Source"
-                                        (createGithubUrl channelDetails.branch position)
-                                    ]
-                                ]
                     )
 
         flakeDef =
             Maybe.map2
-                (\name resolved ->
+                (\name resolved -> makeLink ("Flake: " ++ name) resolved)
-                    [ li [ trapClick ]
-                        [ createShortDetailsItem
-                            ("Flake: " ++ name)
-                            resolved
-                        ]
-                    ]
-                )
                 item.source.flakeName
             <|
                 Maybe.map Tuple.second item.source.flakeUrl
     in
-    Maybe.withDefault (Maybe.withDefault [] flakeDef) postion
+    Maybe.withDefault (Maybe.withDefault [] flakeDef) position
 
 
 
@@ -795,6 +775,7 @@ decodeResultItemSource =
         |> Json.Decode.Pipeline.required "package_attr_name" Json.Decode.string
         |> Json.Decode.Pipeline.required "package_pname" Json.Decode.string
         |> Json.Decode.Pipeline.required "package_pversion" Json.Decode.string
+        |> Json.Decode.Pipeline.required "package_outputs" (Json.Decode.list Json.Decode.string)
         |> Json.Decode.Pipeline.required "package_description" (Json.Decode.nullable Json.Decode.string)
         |> Json.Decode.Pipeline.required "package_longDescription" (Json.Decode.nullable Json.Decode.string)
         |> Json.Decode.Pipeline.required "package_license" (Json.Decode.list decodeResultPackageLicense)

src/Search.elm

@@ -15,8 +15,8 @@ module Search exposing
     , decodeResult
     , defaultFlakeId
     , elementId
-    , flakeFromId
+    -- , flakeFromId
-    , flakes
+    -- , flakes
     , fromSortId
     , init
     , makeRequest
@@ -269,7 +269,7 @@ ensureLoading :
     Model a b
     -> Model a b
 ensureLoading model =
-    if model.query /= Nothing && model.query /= Just "" && (List.member model.channel channels || List.member model.channel flakeIds) then
+    if model.query /= Nothing && model.query /= Just "" && List.member model.channel channels then
         { model | result = RemoteData.Loading }
 
     else
@@ -477,7 +477,6 @@ createUrl toRoute model =
 
 type Channel
     = Unstable
-    | Release_21_05
     | Release_21_11
 
 
@@ -503,9 +502,6 @@ channelDetails channel =
         Unstable ->
             ChannelDetails "unstable" "unstable" "nixos/trunk-combined" "nixos-unstable"
 
-        Release_21_05 ->
-            ChannelDetails "21.05" "21.05" "nixos/release-21.05" "nixos-21.05"
-
         Release_21_11 ->
             ChannelDetails "21.11" "21.11" "nixos/release-21.11" "nixos-21.11"
 
@@ -516,9 +512,6 @@ channelFromId channel_id =
         "unstable" ->
             Just Unstable
 
-        "21.05" ->
-            Just Release_21_05
-
         "21.11" ->
             Just Release_21_11
 
@@ -534,8 +527,7 @@ channelDetailsFromId channel_id =
 
 channels : List String
 channels =
-    [ "21.05"
+    [ "21.11"
-    , "21.11"
     , "unstable"
     ]
 
@@ -553,58 +545,58 @@ defaultFlakeId =
     "group-manual"
 
 
-flakeFromId : String -> Maybe Flake
+-- flakeFromId : String -> Maybe Flake
-flakeFromId flake_id =
+-- flakeFromId flake_id =
-    let
+--     let
-        find : String -> List Flake -> Maybe Flake
+--         find : String -> List Flake -> Maybe Flake
-        find id_ list =
+--         find id_ list =
-            case list of
+--             case list of
-                flake :: rest ->
+--                 flake :: rest ->
-                    if flake.id == id_ then
+--                     if flake.id == id_ then
-                        Just flake
+--                         Just flake
-
+--
-                    else
+--                     else
-                        find id_ rest
+--                         find id_ rest
-
+--
-                [] ->
+--                 [] ->
-                    Nothing
+--                     Nothing
-    in
+--     in
-    find flake_id flakes
+--     find flake_id flakes
-
+--
-
+--
-flakeIds : List String
+-- flakeIds : List String
-flakeIds =
+-- flakeIds =
-    List.map .id flakes
+--     List.map .id flakes
-
+--
-
+--
-flakes : List Flake
+-- flakes : List Flake
-flakes =
+-- flakes =
-    [ { id = "latest-nixos-21.11-latest"
+--     [ { id = "latest-nixos-21.11-latest"
-      , isNixpkgs = True
+--       , isNixpkgs = True
-      , title = "Nixpkgs 21.11"
+--       , title = "Nixpkgs 21.11"
-      , source = ""
+--       , source = ""
-      }
+--       }
-    , { id = "latest-nixos-21.05-latest"
+--     , { id = "latest-nixos-21.05-latest"
-      , isNixpkgs = True
+--       , isNixpkgs = True
-      , title = "Nixpkgs 21.05"
+--       , title = "Nixpkgs 21.05"
-      , source = ""
+--       , source = ""
-      }
+--       }
-    , { id = "nixos-21.09-latest"
+--     , { id = "nixos-21.09-latest"
-      , isNixpkgs = True
+--       , isNixpkgs = True
-      , title = "Nixpkgs 21.09"
+--       , title = "Nixpkgs 21.09"
-      , source = ""
+--       , source = ""
-      }
+--       }
-    , { id = "latest-nixos-unstable"
+--     , { id = "latest-nixos-unstable"
-      , isNixpkgs = True
+--       , isNixpkgs = True
-      , title = "Nixpkgs Unstable"
+--       , title = "Nixpkgs Unstable"
-      , source = ""
+--       , source = ""
-      }
+--       }
-    , { id = "flakes"
+--     , { id = "flakes"
-      , isNixpkgs = False
+--       , isNixpkgs = False
-      , title = "Public Flakes"
+--       , title = "Public Flakes"
-      , source = ""
+--       , source = ""
-      }
+--       }
-    ]
+--     ]
 
 
 sortBy : List Sort