From 62bc2f4eee8473e168165088e54b31f6ba0eaf2d Mon Sep 17 00:00:00 2001 From: Jake Hamilton Date: Mon, 8 Jul 2024 23:07:59 -0700 Subject: [PATCH] feat: propagating hooks and context --- lib/src/dag/default.nix | 48 +++++---- lib/src/lists/default.nix | 37 ++++--- tidepool/flake.lock | 16 +-- tidepool/src/builders/basic.nix | 51 ++++++--- tidepool/src/export.nix | 1 + tidepool/src/lib/packages.nix | 168 +++++++++++++++++++++++++++++- tidepool/src/lib/types.nix | 36 +++++-- tidepool/src/packages/aux/b.nix | 10 ++ tidepool/src/packages/default.nix | 116 +++++++++------------ 9 files changed, 352 insertions(+), 131 deletions(-) diff --git a/lib/src/dag/default.nix b/lib/src/dag/default.nix index 87f2275..d6f1888 100644 --- a/lib/src/dag/default.nix +++ b/lib/src/dag/default.nix @@ -47,10 +47,12 @@ lib: { in if sorted ? result then { - result = builtins.map (value: { - name = value.name; - value = value.value; - }) sorted.result; + result = builtins.map + (value: { + name = value.name; + value = value.value; + }) + sorted.result; } else sorted; @@ -64,24 +66,26 @@ lib: { defaults = graph: defaults: let - result = builtins.mapAttrs ( - name: entry: - if defaults ? ${name} then - if builtins.isString entry then - { - value = entry; - before = defaults.${name}.before or [ ]; - after = defaults.${name}.after or [ ]; - } - else - entry - // { - before = (entry.before or [ ]) ++ (defaults.${name}.before or [ ]); - after = (entry.after or [ ]) ++ (defaults.${name}.after or [ ]); - } - else - entry - ) graph; + result = builtins.mapAttrs + ( + name: entry: + if defaults ? ${name} then + if builtins.isString entry then + { + value = entry; + before = defaults.${name}.before or [ ]; + after = defaults.${name}.after or [ ]; + } + else + entry + // { + before = (entry.before or [ ]) ++ (defaults.${name}.before or [ ]); + after = (entry.after or [ ]) ++ (defaults.${name}.after or [ ]); + } + else + entry + ) + graph; in defaults // result; }; diff --git a/lib/src/lists/default.nix b/lib/src/lists/default.nix index b8cca9b..22a3d29 100644 --- a/lib/src/lists/default.nix +++ b/lib/src/lists/default.nix @@ -23,10 +23,12 @@ lib: { parts = lib.strings.split "(0|[1-9][0-9]*)" string; in builtins.map serialize parts; - prepared = builtins.map (value: [ - (vectorize value) - value - ]) list; + prepared = builtins.map + (value: [ + (vectorize value) + value + ]) + list; isLess = a: b: (lib.lists.compare lib.numbers.compare (builtins.head a) (builtins.head b)) < 0; in builtins.map (x: builtins.elemAt x 1) (builtins.sort isLess prepared); @@ -123,8 +125,8 @@ lib: { ## @type List a -> a last = list: - assert lib.errors.trace (list != [ ]) "List cannot be empty"; - builtins.elemAt list (builtins.length list - 1); + assert lib.errors.trace (list != [ ]) "List cannot be empty"; + builtins.elemAt list (builtins.length list - 1); ## Slice part of a list to create a new list. ## @@ -163,8 +165,8 @@ lib: { ## @type List -> List init = list: - assert lib.errors.trace (builtins.length list != 0) "lib.lists.init: list must not be empty."; - lib.lists.take (builtins.length list - 1) list; + assert lib.errors.trace (builtins.length list != 0) "lib.lists.init: list must not be empty."; + lib.lists.take (builtins.length list - 1) list; ## Reverse a list. ## @@ -189,10 +191,12 @@ lib: { list else builtins.tail ( - builtins.concatMap (part: [ - separator - part - ]) list + builtins.concatMap + (part: [ + separator + part + ]) + list ); ## Create a list of integers from a starting number to an ending @@ -224,5 +228,14 @@ lib: { filter = result: value: if builtins.elem value result then result else result ++ [ value ]; in builtins.foldl' filter [ ] list; + + ## Flatten a list of lists into a single list. + ## + ## @type List (List a) -> List a + flatten = value: + if builtins.isList value then + builtins.concatMap lib.lists.flatten value + else + [ value ]; }; } diff --git a/tidepool/flake.lock b/tidepool/flake.lock index add5ace..68df33a 100644 --- a/tidepool/flake.lock +++ b/tidepool/flake.lock @@ -8,10 +8,10 @@ }, "locked": { "dir": "foundation", - "dirtyRev": "2be3111b2c0911f40b47fe0a1fb22b5f5188cf59-dirty", - "dirtyShortRev": "2be3111-dirty", - "lastModified": 1719251485, - "narHash": "sha256-9G1TPBdlQNXCZf6A66bCT9m2vhodkSF+rDtqOVuFteY=", + "dirtyRev": "42e69f7d43c0fb108ac70fde118c4a8ff6a777b0-dirty", + "dirtyShortRev": "42e69f7-dirty", + "lastModified": 1720466389, + "narHash": "sha256-Zrmbcb+42r6eZV05QbcG2znHXrOh0sbdRaEZYxV0C7A=", "type": "git", "url": "file:../?dir=foundation" }, @@ -24,10 +24,10 @@ "lib": { "locked": { "dir": "lib", - "dirtyRev": "2be3111b2c0911f40b47fe0a1fb22b5f5188cf59-dirty", - "dirtyShortRev": "2be3111-dirty", - "lastModified": 1719251485, - "narHash": "sha256-9G1TPBdlQNXCZf6A66bCT9m2vhodkSF+rDtqOVuFteY=", + "dirtyRev": "42e69f7d43c0fb108ac70fde118c4a8ff6a777b0-dirty", + "dirtyShortRev": "42e69f7-dirty", + "lastModified": 1720466389, + "narHash": "sha256-Zrmbcb+42r6eZV05QbcG2znHXrOh0sbdRaEZYxV0C7A=", "type": "git", "url": "file:../?dir=lib" }, diff --git a/tidepool/src/builders/basic.nix b/tidepool/src/builders/basic.nix index b109ecc..aeedd97 100644 --- a/tidepool/src/builders/basic.nix +++ b/tidepool/src/builders/basic.nix @@ -1,10 +1,8 @@ -{ lib, config }: +{ config }: let cfg = config.builders.basic; - lib' = config.lib; - - inherit (config) foundation; + inherit (config) lib foundation; in { config.builders = { @@ -14,7 +12,36 @@ in build = package: let - phases = lib.dag.apply.defaults package.phases { + system = package.platform.build.double; + + dependencies = lib.packages.dependencies.collect package; + + context = lib.packages.context.create dependencies { }; + + hooks = lib.packages.hooks.create dependencies context; + + cflags = builtins.concatStringsSep " " (context.build.host.cflags or [ ]); + + phasesWithHooks = + let + all = lib.lists.flatten [ + hooks.build.only + hooks.build.build + hooks.build.host + hooks.build.target + hooks.host.only + hooks.host.host + hooks.host.target + hooks.target.only + hooks.target.target + ]; + in + builtins.foldl' + (final: defaults: lib.dag.apply.defaults final defaults) + package.phases + all; + + phases = lib.dag.apply.defaults phasesWithHooks { unpack = lib.dag.entry.before [ "patch" ] ""; patch = lib.dag.entry.between [ "unpack" ] [ "configure" ] ""; @@ -25,13 +52,12 @@ in install = lib.dag.entry.after [ "build" ] ""; }; + sorted = lib.dag.sort.topological phases; - script = lib.strings.concatMapSep "\n" ( - entry: if builtins.isFunction entry.value then entry.value package else entry.value - ) sorted.result; - - system = package.platform.build.double; + script = lib.strings.concatMapSep "\n" + (entry: entry.value) + sorted.result; built = builtins.derivation ( package.env @@ -46,7 +72,7 @@ in PATH = let bins = lib.paths.bin ( - (lib'.packages.dependencies.getPackages package.deps.build.host) + (lib.packages.dependencies.get dependencies.build.host) ++ [ foundation.stage2-bash foundation.stage2-coreutils @@ -85,7 +111,8 @@ in inherit (package) meta; extras = { inherit package; - }; + phases = builtins.listToAttrs sorted.result; + } // package.extras; }; }; }; diff --git a/tidepool/src/export.nix b/tidepool/src/export.nix index 56a470e..8eb98c3 100644 --- a/tidepool/src/export.nix +++ b/tidepool/src/export.nix @@ -14,6 +14,7 @@ in packages = { aux-a = config.packages.aux.a; + aux-b = config.packages.aux.b; # foundation-gcc-x86_64 = # (config.packages.foundation.gcc.versions."13.2.0".extend (args: { diff --git a/tidepool/src/lib/packages.nix b/tidepool/src/lib/packages.nix index f981a57..bfa4648 100644 --- a/tidepool/src/lib/packages.nix +++ b/tidepool/src/lib/packages.nix @@ -6,18 +6,178 @@ in config = { lib.packages = { dependencies = { - getPackages = + get = dependencies: let - available = builtins.filter (dependency: !(builtins.isNull dependency)) ( - builtins.attrValues dependencies - ); + exists = value: ! (builtins.isNull value); + available = builtins.filter exists dependencies; in builtins.map (dependency: dependency.package) available; build = build': host': target': builtins.mapAttrs (name: dep: lib'.packages.build dep build' host' target'); + + collect = package: + let + isPropagated = name: package: package.propagate or false; + getPropagatedDependencies = target: builtins.attrValues (lib.attrs.filter isPropagated target); + + process = dependencies: + let + getDeps = name: dependency: + let + deps = + { + build = { + only = getPropagatedDependencies dependency.deps.build.only ++ process dependency.deps.build.only; + build = getPropagatedDependencies dependency.deps.build.build ++ process dependency.deps.build.build; + host = getPropagatedDependencies dependency.deps.build.host ++ process dependency.deps.build.host; + target = getPropagatedDependencies dependency.deps.build.target ++ process dependency.deps.build.target; + }; + host = { + only = getPropagatedDependencies dependency.deps.host.only ++ process dependency.deps.host.only; + host = getPropagatedDependencies dependency.deps.host.host ++ process dependency.deps.host.host; + target = getPropagatedDependencies dependency.deps.host.target ++ process dependency.deps.host.target; + }; + target = { + only = getPropagatedDependencies dependency.deps.target.only ++ process dependency.deps.target.only; + target = getPropagatedDependencies dependency.deps.target.target ++ process dependency.deps.target.target; + }; + }; + in + lib.lists.flatten + [ + deps.build.only + deps.build.build + deps.build.host + deps.build.target + deps.host.only + deps.host.host + deps.host.target + deps.target.only + deps.target.target + ]; + + propagated = lib.attrs.mapToList getDeps dependencies; + in + lib.lists.flatten propagated; + in + { + build = { + only = builtins.attrValues package.deps.build.only ++ process package.deps.build.only; + build = builtins.attrValues package.deps.build.build ++ process package.deps.build.build; + host = builtins.attrValues package.deps.build.host ++ process package.deps.build.host; + target = builtins.attrValues package.deps.build.target ++ process package.deps.build.target; + }; + host = { + only = builtins.attrValues package.deps.host.only ++ process package.deps.host.only; + host = builtins.attrValues package.deps.host.host ++ process package.deps.host.host; + target = builtins.attrValues package.deps.host.target ++ process package.deps.host.target; + }; + target = { + only = builtins.attrValues package.deps.target.only ++ process package.deps.target.only; + target = builtins.attrValues package.deps.target.target ++ process package.deps.target.target; + }; + }; + }; + + context = { + create = collected: ctx: + let + process = path: + let + dependencies = lib.attrs.selectOrThrow path collected; + contexts = builtins.map (dependency: dependency.context or { }) dependencies; + base = ctx // { + target = builtins.concatStringsSep "." path; + }; + result = lib.modules.run { + modules = builtins.map + (context: { config }: { + config = context; + }) + contexts + ++ [ + { + freeform = lib.types.any; + + options = config.packages.context.options // { + target = lib.options.create { + description = "The dependency target that is being generated."; + type = lib.types.enum [ + "build.only" + "build.build" + "build.host" + "build.target" + "host.only" + "host.host" + "host.target" + "target.only" + "target.target" + ]; + }; + }; + + config = base; + } + ]; + }; + in + result.config; + in + { + build = { + only = process [ "build" "only" ]; + build = process [ "build" "build" ]; + host = process [ "build" "host" ]; + target = process [ "build" "target" ]; + }; + host = { + only = process [ "host" "only" ]; + host = process [ "host" "host" ]; + target = process [ "host" "target" ]; + }; + target = { + only = process [ "target" "only" ]; + target = process [ "target" "target" ]; + }; + }; + }; + + hooks = { + create = collected: ctx: + let + process = path: + let + dependencies = lib.attrs.selectOrThrow path collected; + hooks = builtins.map + (dependency: + let + getHooks = dependency.hooks or (lib.fp.const { }); + in + getHooks ctx) + dependencies; + in + hooks; + in + { + build = { + only = process [ "build" "only" ]; + build = process [ "build" "build" ]; + host = process [ "build" "host" ]; + target = process [ "build" "target" ]; + }; + host = { + only = process [ "host" "only" ]; + host = process [ "host" "host" ]; + target = process [ "host" "target" ]; + }; + target = { + only = process [ "target" "only" ]; + target = process [ "target" "target" ]; + }; + }; }; getLatest = diff --git a/tidepool/src/lib/types.nix b/tidepool/src/lib/types.nix index 3444c14..2128b46 100644 --- a/tidepool/src/lib/types.nix +++ b/tidepool/src/lib/types.nix @@ -280,6 +280,12 @@ in }; }; + extras = lib.options.create { + description = "Extra information for the package."; + type = lib.types.attrs.of lib.types.any; + default.value = { }; + }; + platform = { build = lib.options.create { description = "The build platform for the package."; @@ -327,12 +333,6 @@ in type = lib'.types.builder; }; - phases = lib.options.create { - description = "The phases for the package."; - type = lib.types.dag.of lib.types.string; - default.value = { }; - }; - env = lib.options.create { description = "The environment for the package."; type = lib.types.attrs.of lib.types.string; @@ -345,6 +345,30 @@ in default.value = config.builder.build config; }; + phases = lib.options.create { + description = "The phases for the package."; + type = lib.types.dag.of lib.types.string; + default.value = { }; + }; + + context = lib.options.create { + description = "The context information that the package provides."; + type = lib.types.attrs.of lib.types.raw; + default.value = { }; + }; + + hooks = lib.options.create { + description = "The hooks that the package provides."; + type = lib.types.function (lib.types.dag.of lib.types.string); + default.value = ctx: { }; + }; + + propagate = lib.options.create { + description = "Whether the package should propagate its hooks and context."; + type = lib.types.bool; + default.value = false; + }; + deps = lib.options.create { description = "The dependencies for the package."; type = deps build host target; diff --git a/tidepool/src/packages/aux/b.nix b/tidepool/src/packages/aux/b.nix index b3d52f1..329806b 100644 --- a/tidepool/src/packages/aux/b.nix +++ b/tidepool/src/packages/aux/b.nix @@ -26,6 +26,16 @@ in builder = builders.basic; + context = { + cflags = [ "-I $AUX_B/include" ]; + }; + + hooks = ctx: { + "aux:b:env" = lib.dag.entry.after + [ "unpack" ] + ''export AUX_B=${config.package}''; + }; + phases = { install = '' echo "b" > $out diff --git a/tidepool/src/packages/default.nix b/tidepool/src/packages/default.nix index ababd40..725d760 100644 --- a/tidepool/src/packages/default.nix +++ b/tidepool/src/packages/default.nix @@ -19,14 +19,24 @@ in type = lib.types.submodule { freeform = lib.types.packages; - options.cross = lib.attrs.generate doubles ( - system: - lib.options.create { - description = "The cross-compiled package set for the ${system} target."; - type = lib.types.packages; - default = { }; - } - ); + options = { + cross = lib.attrs.generate doubles ( + system: + lib.options.create { + description = "The cross-compiled package set for the ${system} target."; + type = lib.types.packages; + default.value = { }; + } + ); + + # NOTE: We may offer a way to set default context values. For this reason we have + # nested `options` under `context` rather than using a plain option directly under `packages`. + context.options = lib.options.create { + description = "The available options for package contexts."; + default.value = { }; + type = lib.types.attrs.of lib.types.option; + }; + }; }; }; @@ -42,65 +52,37 @@ in }; }; - config.packages.cross = lib.attrs.generate doubles ( - system: - builtins.mapAttrs ( - namespace: - builtins.mapAttrs ( - name: alias: - let - setHost = - package: - package - // { - __modules__ = package.__modules__ ++ [ - { - config.platform = { - host = lib.modules.override 5 system; - target = lib.modules.override 5 system; + config = { + packages.cross = lib.attrs.generate doubles ( + system: + builtins.mapAttrs + ( + namespace: + builtins.mapAttrs ( + name: alias: + let + setHost = + package: + package + // { + __modules__ = package.__modules__ ++ [ + { + config.platform = { + host = lib.modules.override 5 system; + target = lib.modules.override 5 system; + }; + } + ]; }; - } - ]; - }; - # if package != {} - # then - # (package.extend ( - # {config}: { - # config = { - # platform = { - # host = lib.modules.overrides.force system; - # target = lib.modules.overrides.default system; - # }; - # deps = { - # build = { - # only = setHost package.deps.build.only; - # build = setHost package.deps.build.build; - # host = setHost package.deps.build.host; - # target = setHost package.deps.build.target; - # }; - # host = { - # only = setHost package.deps.host.only; - # host = setHost package.deps.host.host; - # target = setHost package.deps.host.target; - # }; - # target = { - # only = setHost package.deps.target.only; - # target = setHost package.deps.target.target; - # }; - # }; - # }; - # } - # )) - # .config - # else package; - - updated = alias // { - versions = builtins.mapAttrs (version: package: setHost package) alias.versions; - }; - in - updated - ) - ) packages - ); + updated = alias // { + versions = builtins.mapAttrs (version: package: setHost package) alias.versions; + }; + in + updated + ) + ) + packages + ); + }; }