From a793eae8faa205651174a89deb5b475ed760fa65 Mon Sep 17 00:00:00 2001 From: tcmal Date: Mon, 13 May 2024 23:50:12 +0100 Subject: [PATCH] refactor: cleanup stdenv building --- pkgs/by-name/li/libffi/default.nix | 4 +- pkgs/by-name/py/python/default.nix | 2 + pkgs/stdenv/adapters.nix | 6 +- pkgs/stdenv/default.nix | 4 +- pkgs/stdenv/linux/default.nix | 8 +- pkgs/top-level/base-packages.nix | 6 +- pkgs/top-level/default.nix | 53 ++----- pkgs/top-level/splice.nix | 12 +- pkgs/top-level/stage.nix | 226 ++++++++++------------------- 9 files changed, 107 insertions(+), 214 deletions(-) diff --git a/pkgs/by-name/li/libffi/default.nix b/pkgs/by-name/li/libffi/default.nix index 455b667..943e3d4 100644 --- a/pkgs/by-name/li/libffi/default.nix +++ b/pkgs/by-name/li/libffi/default.nix @@ -2,8 +2,7 @@ # test suite depends on dejagnu which cannot be used during bootstrapping # dejagnu also requires tcl which can't be built statically at the moment -, doCheck ? !(stdenv.hostPlatform.isStatic), dejagnu, nix-update-script, testers -}: +, doCheck ? !(stdenv.hostPlatform.isStatic), dejagnu, testers }: stdenv.mkDerivation (finalAttrs: { pname = "libffi"; @@ -47,7 +46,6 @@ stdenv.mkDerivation (finalAttrs: { nativeCheckInputs = [ dejagnu ]; passthru = { - updateScript = nix-update-script { }; tests = { pkg-config = testers.hasPkgConfigModules { package = finalAttrs.finalPackage; }; diff --git a/pkgs/by-name/py/python/default.nix b/pkgs/by-name/py/python/default.nix index 4ec359a..b79c3b4 100644 --- a/pkgs/by-name/py/python/default.nix +++ b/pkgs/by-name/py/python/default.nix @@ -103,6 +103,8 @@ in { configd = null; sqlite = null; tzdata = null; + libX11 = null; + xorgproto = null; libffi = libffiBoot; # without test suite stripConfig = true; stripIdlelib = true; diff --git a/pkgs/stdenv/adapters.nix b/pkgs/stdenv/adapters.nix index 213045c..8770168 100644 --- a/pkgs/stdenv/adapters.nix +++ b/pkgs/stdenv/adapters.nix @@ -1,6 +1,6 @@ -/* This file contains various functions that take a stdenv and return - a new stdenv with different behaviour, e.g. using a different C - compiler. +/* * + Various functions that take a stdenv and return a new stdenv with different behaviour. + e.g. using a different C compiler. */ { lib, pkgs, config }: diff --git a/pkgs/stdenv/default.nix b/pkgs/stdenv/default.nix index 4a1668a..887b566 100644 --- a/pkgs/stdenv/default.nix +++ b/pkgs/stdenv/default.nix @@ -7,7 +7,7 @@ { # Args just for stdenvs' usage lib # Args to pass on to the pkgset builder, too -, localSystem, crossSystem, config, overlays, crossOverlays ? [ ] }@args: +, localSystem, crossSystem, config }@args: let # The native (i.e., impure) build environment. This one uses the @@ -34,7 +34,7 @@ let stagesCustom = import ./custom args; # Select the appropriate stages for the platform `system'. -in if crossSystem != localSystem || crossOverlays != [ ] then +in if crossSystem != localSystem then stagesCross else if config ? replaceStdenv then stagesCustom diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 44c954f..1403452 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -53,7 +53,7 @@ # For a TUI (rather than CLI) view, you can use: # # $ nix-tree --derivation $(nix-instantiate -A stdenv) -{ lib, localSystem, crossSystem, config, overlays, crossOverlays ? [ ] +{ lib, localSystem, crossSystem, config , bootstrapFiles ? let table = { @@ -224,7 +224,7 @@ let }; in { - inherit config overlays; + inherit config; stdenv = thisStdenv; }; @@ -628,7 +628,7 @@ in assert bootstrapTools.passthru.isFromBootstrapFiles or false; # sanity check assert isBuiltByNixpkgsCompiler prevStage.coreutils; assert isBuiltByNixpkgsCompiler prevStage.gnugrep; assert isBuiltByNixpkgsCompiler prevStage.patchelf; { - inherit config overlays; + inherit config; stdenv = import ../generic lib rec { name = "stdenv-linux"; @@ -740,6 +740,6 @@ in assert bootstrapTools.passthru.isFromBootstrapFiles or false; # sanity check assert isBuiltByNixpkgsCompiler prevStage.coreutils; assert isBuiltByNixpkgsCompiler prevStage.gnugrep; assert isBuiltByNixpkgsCompiler prevStage.patchelf; { - inherit (prevStage) config overlays stdenv; + inherit (prevStage) config stdenv; }) ] diff --git a/pkgs/top-level/base-packages.nix b/pkgs/top-level/base-packages.nix index addba49..d8fb2e8 100644 --- a/pkgs/top-level/base-packages.nix +++ b/pkgs/top-level/base-packages.nix @@ -4,7 +4,7 @@ # strict sorting has been long lost due to merges. Please use the full-text # search of your editor. ;) # Hint: ### starts category names. -{ lib, noSysDirs, config, overlays }: +{ lib, noSysDirs, config }: res: pkgs: super: with pkgs; @@ -87,7 +87,7 @@ with pkgs; path = ../..; ### Helper functions. - inherit lib config overlays; + inherit lib config; # do not import 'appendToName' to get consistent package-names with the same # set of package-parameters: https://github.com/NixOS/nixpkgs/issues/68519 @@ -113,8 +113,6 @@ with pkgs; packages, like using pkgs/top-level/release-attrpaths-superset.nix. ''; - tests = callPackages ../test { }; - # These are used when buiding compiler-rt / libgcc, prior to building libc. preLibcCrossHeaders = let inherit (stdenv.targetPlatform) libc; in if stdenv.targetPlatform.isMinGW then diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index d27cb0b..efc076c 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -17,50 +17,29 @@ or dot-files. */ -{ # The system packages will be built on. See the manual for the +{ +# The system packages will be built on. See the manual for the # subtle division of labor between these two `*System`s and the three # `*Platform`s. -localSystem - -, lib ? import - -, # The system packages will ultimately be run on. -crossSystem ? localSystem - -, # Allow a configuration attribute set to be passed in as an argument. -config ? { } - -, # List of overlays layers used to extend Nixpkgs. -overlays ? [ ] - -, # List of overlays to apply to target packages only. -crossOverlays ? [ ] - -, # A function booting the final package set for a specific standard -# environment. See below for the arguments given to that function, the type of -# list it returns. -stdenvStages ? import ../stdenv +localSystem, lib ? import + # The system packages will ultimately be run on. +, crossSystem ? localSystem + # Allow a configuration attribute set to be passed in as an argument. +, config ? { } + # A function booting the final package set for a specific standard + # environment. See below for the arguments given to that function, the type of + # list it returns. +, stdenvStages ? import ../stdenv , # Ignore unexpected args. ... }@args: - -let # Rename the function arguments +let + # Rename the function arguments config0 = config; crossSystem0 = crossSystem; - in let inherit (lib) throwIfNot; - checked = throwIfNot (lib.isList overlays) - "The overlays argument to nixpkgs must be a list." lib.foldr (x: - throwIfNot (lib.isFunction x) - "All overlays passed to nixpkgs must be functions.") (r: r) overlays - throwIfNot (lib.isList crossOverlays) - "The crossOverlays argument to nixpkgs must be a list." lib.foldr (x: - throwIfNot (lib.isFunction x) - "All crossOverlays passed to nixpkgs must be functions.") (r: r) - crossOverlays; - localSystem = lib.systems.elaborate args.localSystem; # Condition preserves sharing which in turn affects equality. @@ -135,10 +114,8 @@ in let boot = import ../stdenv/booter.nix { inherit lib allPackages; }; - stages = stdenvStages { - inherit lib localSystem crossSystem config overlays crossOverlays; - }; + stages = stdenvStages { inherit lib localSystem crossSystem config; }; pkgs = boot stages; -in checked pkgs +in pkgs diff --git a/pkgs/top-level/splice.nix b/pkgs/top-level/splice.nix index cf4bfa3..200a11c 100644 --- a/pkgs/top-level/splice.nix +++ b/pkgs/top-level/splice.nix @@ -105,14 +105,6 @@ let inherit (pkgs.stdenv) buildPlatform targetPlatform hostPlatform; }; - splicedPackagesWithXorg = splicedPackages - // builtins.removeAttrs splicedPackages.xorg [ - "callPackage" - "newScope" - "overrideScope" - "packages" - ]; - in { inherit splicePackages; @@ -121,9 +113,9 @@ in { # `newScope' for sets of packages in `pkgs' (see e.g. `gnome' below). callPackage = pkgs.newScope { }; - callPackages = lib.callPackagesWith splicedPackagesWithXorg; + callPackages = lib.callPackagesWith splicedPackages; - newScope = extra: lib.callPackageWith (splicedPackagesWithXorg // extra); + newScope = extra: lib.callPackageWith (splicedPackages // extra); # prefill 2 fields of the function for convenience makeScopeWithSplicing = diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index f361ecc..9184483 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -14,22 +14,13 @@ let # By defining it at the top of the file, # this value gets reused even if this file is imported multiple times, # thanks to Nix's import-value cache. - # autoCalledPackages = import ./by-name-overlay.nix ../by-name; + autoCalledPackages = import ./by-name-overlay.nix ../by-name; in { -## Misc parameters kept the same for all stages -## +# Misc parameters kept the same for all stages +lib, nixpkgsFun -# Utility functions, could just import but passing in for efficiency -lib - -, # Use to reevaluate Nixpkgs -nixpkgsFun - -## Other parameters -## - -, # Either null or an object in the form: +# Either null or an object in the form: # # { # pkgsBuildBuild = ...; @@ -45,75 +36,30 @@ nixpkgsFun # they are instead defined internally as the current stage. This allows us to # avoid expensive splicing. `pkgsHostTarget` is skipped because it is always # defined as the current stage. -adjacentPackages +, adjacentPackages -, # The standard environment to use for building packages. -stdenv +# The standard environment to use for building packages. +, stdenv -, # This is used because stdenv replacement and the stdenvCross do benefit from +# This is used because stdenv replacement and the stdenvCross do benefit from # the overridden configuration provided by the user, as opposed to the normal # bootstrapping stdenvs. -allowCustomOverrides +, allowCustomOverrides -, # Non-GNU/Linux OSes are currently "impure" platforms, with their libc +# Non-GNU/Linux OSes are currently "impure" platforms, with their libc # outside of the store. Thus, GCC, GFortran, & co. must always look for files # in standard system directories (/usr/include, etc.) -noSysDirs ? stdenv.buildPlatform.system != "x86_64-freebsd" +, noSysDirs ? stdenv.buildPlatform.system != "x86_64-freebsd" && stdenv.buildPlatform.system != "i686-freebsd" && stdenv.buildPlatform.system != "x86_64-solaris" && stdenv.buildPlatform.system != "x86_64-kfreebsd-gnu" -, # The configuration attribute set -config - -, # A list of overlays (Additional `self: super: { .. }` customization -# functions) to be fixed together in the produced package set -overlays + # The configuration attribute set +, config }@args: let - # This is a function from parsed platforms (like - # stdenv.hostPlatform.parsed) to parsed platforms. - makeMuslParsedPlatform = parsed: - # The following line guarantees that the output of this function - # is a well-formed platform with no missing fields. It will be - # uncommented in a separate PR, in case it breaks the build. - #(x: lib.trivial.pipe x [ (x: builtins.removeAttrs x [ "_type" ]) lib.systems.parse.mkSystem ]) - (parsed // { - abi = { - gnu = lib.systems.parse.abis.musl; - gnueabi = lib.systems.parse.abis.musleabi; - gnueabihf = lib.systems.parse.abis.musleabihf; - gnuabin32 = lib.systems.parse.abis.muslabin32; - gnuabi64 = lib.systems.parse.abis.muslabi64; - gnuabielfv2 = lib.systems.parse.abis.musl; - gnuabielfv1 = lib.systems.parse.abis.musl; - # The following two entries ensure that this function is idempotent. - musleabi = lib.systems.parse.abis.musleabi; - musleabihf = lib.systems.parse.abis.musleabihf; - muslabin32 = lib.systems.parse.abis.muslabin32; - muslabi64 = lib.systems.parse.abis.muslabi64; - }.${parsed.abi.name} or lib.systems.parse.abis.musl; - }); - - stdenvAdapters = self: super: - let - res = import ../stdenv/adapters.nix { - inherit lib config; - pkgs = self; - }; - in res // { stdenvAdapters = res; }; - - trivialBuilders = self: super: - import ../build-support/trivial-builders { - inherit lib; - inherit (self) config; - inherit (self) runtimeShell stdenv stdenvNoCC; - inherit (self.pkgsBuildHost) jq shellcheck-minimal; - inherit (self.pkgsBuildHost.xorg) lndir; - }; - stdenvBootstappingAndPlatforms = self: super: let withFallback = thisPkgs: @@ -148,31 +94,25 @@ let inherit stdenv; }; + stdenvAdapters = self: super: + let + res = import ../stdenv/adapters.nix { + inherit lib config; + pkgs = self; + }; + in res // { stdenvAdapters = res; }; + + trivialBuilders = self: super: + import ../build-support/trivial-builders { + inherit lib; + inherit (self) config; + inherit (self) runtimeShell stdenv stdenvNoCC; + inherit (self.pkgsBuildHost) jq shellcheck-minimal; + inherit (self.pkgsBuildHost.xorg) lndir; + }; + splice = self: super: import ./splice.nix lib self (adjacentPackages != null); - basePackages = self: super: - let - res = - import ./base-packages.nix { inherit lib noSysDirs config overlays; } - res self super; - in res; - - commonUpdaterPackages = self: super: - let - res = import ../common-updater/packages.nix { - inherit lib noSysDirs config overlays; - } res self super; - in res; - - buildSupportPackages = self: super: - let - res = import ../build-support/packages.nix { - inherit lib noSysDirs config overlays; - } res self super; - in res; - - # aliases = self: super: lib.optionalAttrs config.allowAliases (import ./aliases.nix lib self super); - byNamePackages = self: super: let inherit (builtins) readDir; @@ -208,27 +148,47 @@ let _internalCallByNamePackageFile = file: self.callPackage file { }; } // (mapAttrs (name: self._internalCallByNamePackageFile) packageFiles) // lib.foldl (acc: path: - (import "${path}/packages.nix" { - inherit lib noSysDirs config overlays; - } res self super) // acc) { } packageGroupFiles; + (import "${path}/packages.nix" { inherit lib noSysDirs config; } res + self super) // acc) { } packageGroupFiles; in res; - # stdenvOverrides is used to avoid having multiple of versions - # of certain dependencies that were used in bootstrapping the - # standard environment. - stdenvOverrides = self: super: - (super.stdenv.overrides or (_: _: { })) self super; + basePackages = self: super: + let + res = + import ./base-packages.nix { inherit lib noSysDirs config; } res self + super; + in res; - # Allow packages to be overridden globally via the `packageOverrides' - # configuration option, which must be a function that takes `pkgs' - # as an argument and returns a set of new or overridden packages. - # The `packageOverrides' function is called with the *original* - # (un-overridden) set of packages, allowing packageOverrides - # attributes to refer to the original attributes (e.g. "foo = - # ... pkgs.foo ..."). - configOverrides = self: super: - lib.optionalAttrs allowCustomOverrides - ((config.packageOverrides or (super: { })) super); + buildSupportPackages = self: super: + let + res = + import ../build-support/packages.nix { inherit lib noSysDirs config; } + res self super; + in res; + + # This is a function from parsed platforms (like + # stdenv.hostPlatform.parsed) to parsed platforms. + makeMuslParsedPlatform = parsed: + # The following line guarantees that the output of this function + # is a well-formed platform with no missing fields. It will be + # uncommented in a separate PR, in case it breaks the build. + #(x: lib.trivial.pipe x [ (x: builtins.removeAttrs x [ "_type" ]) lib.systems.parse.mkSystem ]) + (parsed // { + abi = { + gnu = lib.systems.parse.abis.musl; + gnueabi = lib.systems.parse.abis.musleabi; + gnueabihf = lib.systems.parse.abis.musleabihf; + gnuabin32 = lib.systems.parse.abis.muslabin32; + gnuabi64 = lib.systems.parse.abis.muslabi64; + gnuabielfv2 = lib.systems.parse.abis.musl; + gnuabielfv1 = lib.systems.parse.abis.musl; + # The following two entries ensure that this function is idempotent. + musleabi = lib.systems.parse.abis.musleabi; + musleabihf = lib.systems.parse.abis.musleabihf; + muslabin32 = lib.systems.parse.abis.muslabin32; + muslabi64 = lib.systems.parse.abis.muslabi64; + }.${parsed.abi.name} or lib.systems.parse.abis.musl; + }); # Convenience attributes for instantitating package sets. Each of # these will instantiate a new version of allPackages. Currently the @@ -248,7 +208,6 @@ let lib.systems.examples; pkgsLLVM = nixpkgsFun { - overlays = [ (self': super': { pkgsLLVM = super'; }) ] ++ overlays; # Bootstrap a cross stdenv using the LLVM toolchain. # This is currently not possible when compiling natively, # so we don't need to check hostPlatform != buildPlatform. @@ -264,7 +223,6 @@ let pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then nixpkgsFun { - overlays = [ (self': super': { pkgsMusl = super'; }) ] ++ overlays; ${ if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" @@ -282,8 +240,6 @@ let pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then nixpkgsFun { - overlays = [ (self': super': { pkgsi686Linux = super'; }) ] - ++ overlays; ${ if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" @@ -301,8 +257,6 @@ let # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then nixpkgsFun { - overlays = [ (self': super': { pkgsx86_64Darwin = super'; }) ] - ++ overlays; localSystem = { parsed = stdenv.hostPlatform.parsed // { cpu = lib.systems.parse.cpuTypes.x86_64; @@ -323,28 +277,9 @@ let lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; }; - # Extend the package set with zero or more overlays. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - appendOverlays = extraOverlays: - if extraOverlays == [ ] then - self - else - nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; - - # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB - # of allocations. DO NOT USE THIS IN NIXPKGS. - # - # Extend the package set with a single overlay. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - # Prefer appendOverlays if used repeatedly. - extend = f: self.appendOverlays [ f ]; - # Fully static packages. # Currently uses Musl on Linux (couldn’t get static glibc to work). pkgsStatic = nixpkgsFun ({ - overlays = [ (self': super': { pkgsStatic = super'; }) ] ++ overlays; crossSystem = { isStatic = true; parsed = if stdenv.isLinux then @@ -355,22 +290,15 @@ let gcc.abi = "elfv2"; }; }); - - pkgsExtraHardening = nixpkgsFun { - overlays = [ - (self': super': { - pkgsExtraHardening = super'; - stdenv = super'.withDefaultHardeningFlags - (super'.stdenv.cc.defaultHardeningFlags - ++ [ "zerocallusedregs" "trivialautovarinit" ]) super'.stdenv; - }) - ] ++ overlays; - }; }; + # stdenvOverrides is used to avoid having multiple of versions + # of certain dependencies that were used in bootstrapping the + # standard environment. + stdenvOverrides = self: super: + (super.stdenv.overrides or (_: _: { })) self super; + # The complete chain of package set builders, applied from top to bottom. - # stdenvOverlays must be last as it brings package forward from the - # previous bootstrapping phases which have already been overlayed. toFix = lib.foldl' (lib.flip lib.extends) (self: { }) ([ stdenvBootstappingAndPlatforms stdenvAdapters @@ -378,12 +306,10 @@ let splice byNamePackages basePackages - commonUpdaterPackages buildSupportPackages otherPackageSets - # aliases - configOverrides - ] ++ overlays ++ [ stdenvOverrides ]); + stdenvOverrides + ]); # Return the complete set of packages. in lib.fix toFix