From a6ad4027a121bdb5e7a452737f27a6cfda491114 Mon Sep 17 00:00:00 2001 From: Austreelis Date: Fri, 14 Jun 2024 04:10:14 +0200 Subject: [PATCH 1/6] feat: namespaced includes This commit adds some logic to the modules normalization process to allow including modules under a user-defined namespace. It achieves it by: - flattening any attribute sets in `includes` that are non-empty nor contain any of a module's valid attributes (`lib.modules.VALID_KEYS`). - Erroring out on dupplicate namespaces. - Mapping namespaced includes do normal modules declaring an option `${namespace}` of the include as a submodule. This allows specifying includes in a module like: ``` { includes.mynamespace0 = ./mymodule0.nix; includes.mynamespace1 = ./mymodule1.nix; } ``` and is approximatively desugared by `lib.modules.normalize` into: ``` { includes = [ { options.mynamespace0 = lib.submodule mymodule0.nix; } { options.mynamespace1 = lib.submodule mymodule1.nix; } ]; } ``` This was inspired by nixpkgs' `lib.modules.doRename`. --- lib/src/modules/default.nix | 61 +++++++++++++++++++- lib/src/modules/default.test.nix | 99 ++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 2 deletions(-) diff --git a/lib/src/modules/default.nix b/lib/src/modules/default.nix index 63105ef..286564b 100644 --- a/lib/src/modules/default.nix +++ b/lib/src/modules/default.nix @@ -168,12 +168,69 @@ lib: { let invalid = builtins.removeAttrs module lib.modules.VALID_KEYS; invalidKeys = builtins.concatStringsSep ", " (builtins.attrNames invalid); + + __key__ = builtins.toString module.__key__ or key; + + normalizeIncludes = + includes: + let + flattened = builtins.concatMap ( + include: + if + builtins.isAttrs include + && include != { } + && !builtins.any (n: builtins.elem n lib.modules.VALID_KEYS) (builtins.attrNames include) + then + lib.attrs.mapToList (namespace: module: { + inherit namespace; + include = module; + }) include + else + [ + { + inherit include; + namespace = null; + } + ] + ) (lib.lists.when (includes != { }) includes); + + throwOnConflict = + let + filter = + namespaces: + { namespace, include }: + if namespace != null && builtins.elem namespace namespaces then + builtins.throw "Module ${key} declares several includes under the same namespace '${namespace}'" + else + namespaces ++ [ namespace ]; + in + builtins.foldl' filter [ ] flattened; + + doNamespace = + { namespace, include }: + if namespace == null then + include + else + { + __key__ = "${__key__}:include-${namespace}"; + options.${namespace} = lib.options.create { + description = "options and configuration included from ${namespace}"; + default.value = { }; + type = lib.types.submodules.of { + modules = [ include ]; + description = "include ${namespace}"; + }; + }; + # config.${namespace} = lib.modules.override 1000 {}; + }; + in + builtins.seq throwOnConflict builtins.map doNamespace flattened; in if lib.modules.validate.keys module then { + inherit __key__; __file__ = builtins.toString module.__file__ or file; - __key__ = builtins.toString module.__key__ or key; - includes = module.includes or [ ]; + includes = normalizeIncludes module.includes or [ ]; excludes = module.excludes or [ ]; options = module.options or { }; config = diff --git a/lib/src/modules/default.test.nix b/lib/src/modules/default.test.nix index ff98eb5..bbe82db 100644 --- a/lib/src/modules/default.test.nix +++ b/lib/src/modules/default.test.nix @@ -236,6 +236,81 @@ in }; in actual == expected; + + "includes" = { + + "handles an empty list" = + let + expected = [ ]; + actual = (lib.modules.normalize "/aux/example.nix" "example" { includes = [ ]; }).includes; + in + expected == actual; + + "handles an empty set" = + let + expected = [ ]; + actual = (lib.modules.normalize "/aux/example.nix" "example" { includes = { }; }).includes; + in + expected == actual; + + "handles a mixed list" = + let + expected = [ + { } + { a = null; } + ]; + actual = + # Because includes leverage submodules, we can't match the actual + # included namespaced submodule under "a". So we just assert the + # namespace was gotten right and do not evaluate the included value. + builtins.map (include: builtins.mapAttrs (_: _: null) include.options or include) + (lib.modules.normalize "/aux/example.nix" "example" { + includes = [ + { } + { a = null; } + ]; + }).includes; + in + expected == actual; + + "rejects conflicting namespaces" = + let + normalized = lib.modules.normalize "/aux/example.nix" "example" { + includes = [ + { a = { }; } + { a = { }; } + ]; + }; + in + !(builtins.tryEval normalized.includes).success; + + "allows multiple without namespace" = + let + normalized = lib.modules.normalize "/aux/example.nix" "example" { + includes = [ + { } + { } + ]; + }; + in + (builtins.tryEval normalized.includes).success; + + "handles multiple without namespace" = + let + expected = [ + { } + { } + ]; + actual = + (lib.modules.normalize "/aux/example.nix" "example" { + includes = [ + { } + { } + ]; + }).includes; + in + expected == actual; + }; }; "resolve" = { @@ -699,5 +774,29 @@ in }; in (evaluated.config.aux.message == evaluated.config.aux.message2) && evaluated.config.aux.exists; + + "namespaced includes" = + let + expected = { + msg = "hello"; + myinclude.msg = "hello"; + }; + evaluated = lib.modules.run { + modules = [ + { + includes = [ + { + myinclude = { + options.msg = lib.options.create { default.value = "hello"; }; + }; + } + { options.msg = lib.options.create { default.value = "hello"; }; } + ]; + } + ]; + }; + actual = evaluated.config; + in + expected == actual; }; } -- 2.45.2 From 71bc768dc18efa6f4f8f7369bbcf8cffcfd30c33 Mon Sep 17 00:00:00 2001 From: Austreelis Date: Sat, 22 Jun 2024 11:40:22 +0200 Subject: [PATCH 2/6] Code cleanup and readability --- lib/src/modules/default.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/src/modules/default.nix b/lib/src/modules/default.nix index 286564b..487424b 100644 --- a/lib/src/modules/default.nix +++ b/lib/src/modules/default.nix @@ -206,7 +206,7 @@ lib: { in builtins.foldl' filter [ ] flattened; - doNamespace = + createNamespacedModule = { namespace, include }: if namespace == null then include @@ -221,16 +221,17 @@ lib: { description = "include ${namespace}"; }; }; - # config.${namespace} = lib.modules.override 1000 {}; }; + + namespacedModules = builtins.map createNamespacedModule flattened; in - builtins.seq throwOnConflict builtins.map doNamespace flattened; + builtins.seq throwOnConflict namespacedModules; in if lib.modules.validate.keys module then { inherit __key__; - __file__ = builtins.toString module.__file__ or file; - includes = normalizeIncludes module.includes or [ ]; + __file__ = builtins.toString (module.__file__ or file); + includes = normalizeIncludes (module.includes or [ ]); excludes = module.excludes or [ ]; options = module.options or { }; config = -- 2.45.2 From feb9987300c9a0239e7825152b6ef60363aab53f Mon Sep 17 00:00:00 2001 From: Austreelis Date: Sat, 22 Jun 2024 15:47:39 +0200 Subject: [PATCH 3/6] Use include object syntax instead of shorthand --- lib/src/modules/default.nix | 80 ++++++++++++++++++++++---------- lib/src/modules/default.test.nix | 52 ++++++++++++--------- 2 files changed, 86 insertions(+), 46 deletions(-) diff --git a/lib/src/modules/default.nix b/lib/src/modules/default.nix index 487424b..131175e 100644 --- a/lib/src/modules/default.nix +++ b/lib/src/modules/default.nix @@ -142,6 +142,18 @@ lib: { invalid = builtins.removeAttrs module lib.modules.VALID_KEYS; in invalid == { }; + + ## Check that an include object has the required attributes and otherwise + ## only specifies supported attributes. + ## + ## @type (Attrs | String | Path | Function) -> Bool + include = + include: + let + invalidKeys = builtins.removeAttrs include lib.modules.VALID_INCLUDE_KEYS; + namespaceIsString = include ? namespace -> builtins.isString include.namespace; + in + builtins.isAttrs include -> (include ? module && namespaceIsString && invalidKeys == { }); }; ## Modules only support certain keys at the root level. This list determines @@ -159,6 +171,15 @@ lib: { "meta" ]; + ## Include objects only support certain keys. This list determines the + ## valid attributes that users can supply. + ## + ## @type List String + VALID_INCLUDE_KEYS = [ + "module" + "namespace" + ]; + ## Normalize a module to a standard structure. All other information will be ## lost in the conversion. ## @@ -168,48 +189,59 @@ lib: { let invalid = builtins.removeAttrs module lib.modules.VALID_KEYS; invalidKeys = builtins.concatStringsSep ", " (builtins.attrNames invalid); + throwError = msg: builtins.throw ("Module ${key} (${file}) " + msg); __key__ = builtins.toString module.__key__ or key; normalizeIncludes = includes: let - flattened = builtins.concatMap ( - include: - if - builtins.isAttrs include - && include != { } - && !builtins.any (n: builtins.elem n lib.modules.VALID_KEYS) (builtins.attrNames include) - then - lib.attrs.mapToList (namespace: module: { - inherit namespace; - include = module; - }) include - else - [ + normalized = lib.lists.mapWithIndex1 ( + n: include: + let + invalid = builtins.removeAttrs module lib.modules.VALID_INCLUDE_KEYS; + invalidKeys = builtins.concatStringsSep ", " (builtins.attrNames invalid); + hasInvalidKeys = invalid != { }; + throwError' = msg: throwError ("has invalid include at position ${builtins.toString n}" + msg); + in + if lib.modules.validate.include include then + if builtins.isAttrs include then { - inherit include; + inherit (include) module; + namespace = include.namespace or null; + } + else + { + module = include; namespace = null; } - ] - ) (lib.lists.when (includes != { }) includes); + else if !include ? module then + { + module = include; + namespace = null; + } + else if hasInvalidKeys then + throwError' " with unsupported attribute(s): ${invalidKeys}" + else + throwError' ": namespace is not a string" + ) includes; throwOnConflict = let filter = namespaces: - { namespace, include }: + { namespace, ... }: if namespace != null && builtins.elem namespace namespaces then - builtins.throw "Module ${key} declares several includes under the same namespace '${namespace}'" + throwError "declares several includes under the same namespace '${namespace}'" else namespaces ++ [ namespace ]; in - builtins.foldl' filter [ ] flattened; + builtins.foldl' filter [ ] normalized; createNamespacedModule = - { namespace, include }: + { namespace, module, ... }: if namespace == null then - include + module else { __key__ = "${__key__}:include-${namespace}"; @@ -217,13 +249,13 @@ lib: { description = "options and configuration included from ${namespace}"; default.value = { }; type = lib.types.submodules.of { - modules = [ include ]; + modules = [ module ]; description = "include ${namespace}"; }; }; }; - namespacedModules = builtins.map createNamespacedModule flattened; + namespacedModules = builtins.map createNamespacedModule normalized; in builtins.seq throwOnConflict namespacedModules; in @@ -259,7 +291,7 @@ lib: { withFreeform (withMeta base); } else - builtins.throw "Module `${key}` (${file}) has unsupported attribute(s): ${invalidKeys}"; + throwError "has unsupported attribute(s): ${invalidKeys}"; ## Convert a module that is either a function or an attribute set into ## a resolved attribute set. If the module was a function then it will diff --git a/lib/src/modules/default.test.nix b/lib/src/modules/default.test.nix index bbe82db..d05d4b7 100644 --- a/lib/src/modules/default.test.nix +++ b/lib/src/modules/default.test.nix @@ -246,28 +246,31 @@ in in expected == actual; - "handles an empty set" = - let - expected = [ ]; - actual = (lib.modules.normalize "/aux/example.nix" "example" { includes = { }; }).includes; - in - expected == actual; - "handles a mixed list" = let expected = [ - { } + null { a = null; } ]; actual = # Because includes leverage submodules, we can't match the actual # included namespaced submodule under "a". So we just assert the # namespace was gotten right and do not evaluate the included value. - builtins.map (include: builtins.mapAttrs (_: _: null) include.options or include) + builtins.map + ( + include: + if builtins.isAttrs include then + builtins.mapAttrs (_: _: null) include.options or include + else + include + ) (lib.modules.normalize "/aux/example.nix" "example" { includes = [ - { } - { a = null; } + null + { + module = null; + namespace = "a"; + } ]; }).includes; in @@ -277,8 +280,14 @@ in let normalized = lib.modules.normalize "/aux/example.nix" "example" { includes = [ - { a = { }; } - { a = { }; } + { + module = null; + namespace = "a"; + } + { + module = null; + namespace = "a"; + } ]; }; in @@ -288,8 +297,8 @@ in let normalized = lib.modules.normalize "/aux/example.nix" "example" { includes = [ - { } - { } + null + null ]; }; in @@ -298,14 +307,14 @@ in "handles multiple without namespace" = let expected = [ - { } - { } + null + null ]; actual = (lib.modules.normalize "/aux/example.nix" "example" { includes = [ - { } - { } + null + null ]; }).includes; in @@ -786,9 +795,8 @@ in { includes = [ { - myinclude = { - options.msg = lib.options.create { default.value = "hello"; }; - }; + namespace = "myinclude"; + module.options.msg = lib.options.create { default.value = "hello"; }; } { options.msg = lib.options.create { default.value = "hello"; }; } ]; -- 2.45.2 From 5098f7789b6a7797d09d80d4202b4df83b684260 Mon Sep 17 00:00:00 2001 From: Austreelis Date: Sat, 22 Jun 2024 18:16:32 +0200 Subject: [PATCH 4/6] Add test checking mixed module/include keys --- lib/src/modules/default.test.nix | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/lib/src/modules/default.test.nix b/lib/src/modules/default.test.nix index d05d4b7..9b816f2 100644 --- a/lib/src/modules/default.test.nix +++ b/lib/src/modules/default.test.nix @@ -200,6 +200,57 @@ in in !value; }; + + "cannot mix include and module keys" = + let + # Simple backtracking algorithm to generate all combinations of a list's + # elements without ordering, in . + combinations = set: + let + n = builtins.length set; + genIndices = combinations': + let + prev = builtins.head combinations'; + k = builtins.length prev; + first = builtins.head prev; + next' = + # Generate the next combination of length k + if k > 0 && first+1 < n then [(first+1)] ++ (builtins.tail prev) + # We have generated all combinations of length k, generate the first + # (trivial) combination of length k+1 + else builtins.genList (i: i) (k+1); + next = builtins.deepSeq next' next'; + in + # Generate the very first (trivial) combination + if combinations' == [ ] then genIndices [ [ ] ] + # We have generated all combinations of length 0 to n, return + else if k >= n then combinations' + # Generate the next combination and continue + else genIndices ([next] ++ combinations'); + select = indices: builtins.map (builtins.elemAt set) indices; + in + if set == [ ] then [ ] + else builtins.map select (genIndices []); + + keys = combinations (lib.modules.VALID_KEYS ++ lib.modules.VALID_INCLUDE_KEYS); + sets = + builtins.map + (names: + builtins.listToAttrs + (builtins.map (name: { value = null; inherit name; }) names) + ) + keys; + validated = + builtins.map + (set: { + inherit set; + include = lib.modules.validate.include set; + module = lib.modules.validate.keys set; + }) + sets; + bothValid = {include, module, ...}: include && module; + in + ! builtins.any bothValid validated; }; "normalize" = { -- 2.45.2 From 4b9b9c3506dd04f466f7b9eb5e9a0ed1dfafbadb Mon Sep 17 00:00:00 2001 From: Austreelis Date: Sat, 22 Jun 2024 18:38:34 +0200 Subject: [PATCH 5/6] format code --- lib/src/modules/default.test.nix | 55 +++++++++++++++++--------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/lib/src/modules/default.test.nix b/lib/src/modules/default.test.nix index 9b816f2..00e9396 100644 --- a/lib/src/modules/default.test.nix +++ b/lib/src/modules/default.test.nix @@ -205,52 +205,57 @@ in let # Simple backtracking algorithm to generate all combinations of a list's # elements without ordering, in . - combinations = set: + combinations = + set: let n = builtins.length set; - genIndices = combinations': + genIndices = + combinations': let prev = builtins.head combinations'; k = builtins.length prev; first = builtins.head prev; next' = # Generate the next combination of length k - if k > 0 && first+1 < n then [(first+1)] ++ (builtins.tail prev) + if k > 0 && first + 1 < n then + [ (first + 1) ] ++ (builtins.tail prev) # We have generated all combinations of length k, generate the first # (trivial) combination of length k+1 - else builtins.genList (i: i) (k+1); + else + builtins.genList (i: i) (k + 1); next = builtins.deepSeq next' next'; in # Generate the very first (trivial) combination - if combinations' == [ ] then genIndices [ [ ] ] + if combinations' == [ ] then + genIndices [ [ ] ] # We have generated all combinations of length 0 to n, return - else if k >= n then combinations' + else if k >= n then + combinations' # Generate the next combination and continue - else genIndices ([next] ++ combinations'); + else + genIndices ([ next ] ++ combinations'); select = indices: builtins.map (builtins.elemAt set) indices; in - if set == [ ] then [ ] - else builtins.map select (genIndices []); + if set == [ ] then [ ] else builtins.map select (genIndices [ ]); keys = combinations (lib.modules.VALID_KEYS ++ lib.modules.VALID_INCLUDE_KEYS); - sets = - builtins.map - (names: - builtins.listToAttrs - (builtins.map (name: { value = null; inherit name; }) names) + sets = builtins.map ( + names: + builtins.listToAttrs ( + builtins.map (name: { + value = null; + inherit name; + }) names ) - keys; - validated = - builtins.map - (set: { - inherit set; - include = lib.modules.validate.include set; - module = lib.modules.validate.keys set; - }) - sets; - bothValid = {include, module, ...}: include && module; + ) keys; + validated = builtins.map (set: { + inherit set; + include = lib.modules.validate.include set; + module = lib.modules.validate.keys set; + }) sets; + bothValid = { include, module, ... }: include && module; in - ! builtins.any bothValid validated; + !builtins.any bothValid validated; }; "normalize" = { -- 2.45.2 From 8288e215553f22fad7e767256be5c2af4e68d5ab Mon Sep 17 00:00:00 2001 From: Austreelis Date: Sat, 22 Jun 2024 20:09:37 +0200 Subject: [PATCH 6/6] Apply namespace in load rather than lib.modules.normalize --- lib/src/modules/default.nix | 50 +++++++++++++------------- lib/src/modules/default.test.nix | 61 ++++++++++++++++---------------- 2 files changed, 55 insertions(+), 56 deletions(-) diff --git a/lib/src/modules/default.nix b/lib/src/modules/default.nix index 131175e..a03a1f6 100644 --- a/lib/src/modules/default.nix +++ b/lib/src/modules/default.nix @@ -237,27 +237,8 @@ lib: { namespaces ++ [ namespace ]; in builtins.foldl' filter [ ] normalized; - - createNamespacedModule = - { namespace, module, ... }: - if namespace == null then - module - else - { - __key__ = "${__key__}:include-${namespace}"; - options.${namespace} = lib.options.create { - description = "options and configuration included from ${namespace}"; - default.value = { }; - type = lib.types.submodules.of { - modules = [ module ]; - description = "include ${namespace}"; - }; - }; - }; - - namespacedModules = builtins.map createNamespacedModule normalized; in - builtins.seq throwOnConflict namespacedModules; + builtins.seq throwOnConflict normalized; in if lib.modules.validate.keys module then { @@ -587,19 +568,36 @@ lib: { collect = let load = - args: file: key: module: + args: file: key: module': let + namespace = module'.namespace or null; + module = module'.module or module'; + moduleFromValue = lib.modules.normalize file key (lib.modules.resolve key module args); moduleFromPath = lib.modules.normalize (builtins.toString module) (builtins.toString module) ( lib.modules.resolve (builtins.toString module) (import module) args ); in - if builtins.isAttrs module || builtins.isFunction module then - moduleFromValue - else if builtins.isString module || builtins.isPath module then - moduleFromPath + if namespace == null then + if builtins.isAttrs module || builtins.isFunction module then + moduleFromValue + else if builtins.isString module || builtins.isPath module then + moduleFromPath + else + builtins.throw "The provided module must be either an attribute set, function, or path but got ${builtins.typeOf module}" + # We don't resolve the module yet if we're under a namespace else - builtins.throw "The provided module must be either an attribute set, function, or path but got ${builtins.typeOf module}"; + lib.modules.normalize file key { + __key__ = "${key}:include-${namespace}"; + options.${namespace} = lib.options.create { + description = "options and configuration included from ${namespace}"; + default.value = { }; + type = lib.types.submodules.of { + modules = [ module ]; + description = "include ${namespace}"; + }; + }; + }; normalize = parentFile: parentKey: modules: args: diff --git a/lib/src/modules/default.test.nix b/lib/src/modules/default.test.nix index 00e9396..cf37ea4 100644 --- a/lib/src/modules/default.test.nix +++ b/lib/src/modules/default.test.nix @@ -305,30 +305,25 @@ in "handles a mixed list" = let expected = [ - null - { a = null; } + { + module = { }; + namespace = null; + } + { + module = { }; + namespace = "a"; + } ]; actual = - # Because includes leverage submodules, we can't match the actual - # included namespaced submodule under "a". So we just assert the - # namespace was gotten right and do not evaluate the included value. - builtins.map - ( - include: - if builtins.isAttrs include then - builtins.mapAttrs (_: _: null) include.options or include - else - include - ) - (lib.modules.normalize "/aux/example.nix" "example" { - includes = [ - null - { - module = null; - namespace = "a"; - } - ]; - }).includes; + (lib.modules.normalize "/aux/example.nix" "example" { + includes = [ + { } + { + module = { }; + namespace = "a"; + } + ]; + }).includes; in expected == actual; @@ -337,11 +332,11 @@ in normalized = lib.modules.normalize "/aux/example.nix" "example" { includes = [ { - module = null; + module = { }; namespace = "a"; } { - module = null; + module = { }; namespace = "a"; } ]; @@ -353,8 +348,8 @@ in let normalized = lib.modules.normalize "/aux/example.nix" "example" { includes = [ - null - null + { } + { } ]; }; in @@ -363,14 +358,20 @@ in "handles multiple without namespace" = let expected = [ - null - null + { + namespace = null; + module = { }; + } + { + namespace = null; + module = { }; + } ]; actual = (lib.modules.normalize "/aux/example.nix" "example" { includes = [ - null - null + { } + { } ]; }).includes; in -- 2.45.2