auxolotl/core
26
2
Fork 1
Code Issues 1 Pull requests 5 Actions Packages Projects Releases Wiki Activity

refactor: cleanup stdenv building

Browse source
This commit is contained in:
tcmal 2024-05-13 23:50:12 +01:00
parent d9b1d23b5e
commit a793eae8fa
9 changed files with 107 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkgs/by-name/li/libffi/default.nix
View file

@ -2,8 +2,7 @@
# test suite depends on dejagnu which cannot be used during bootstrapping
# dejagnu also requires tcl which can't be built statically at the moment
, doCheck ? !(stdenv.hostPlatform.isStatic), dejagnu, nix-update-script, testers
}:
, doCheck ? !(stdenv.hostPlatform.isStatic), dejagnu, testers }:
stdenv.mkDerivation (finalAttrs: {
pname = "libffi";
@ -47,7 +46,6 @@ stdenv.mkDerivation (finalAttrs: {
nativeCheckInputs = [ dejagnu ];
passthru = {
updateScript = nix-update-script { };
tests = {
pkg-config =
testers.hasPkgConfigModules { package = finalAttrs.finalPackage; };

2
pkgs/by-name/py/python/default.nix
View file

@ -103,6 +103,8 @@ in {
configd = null;
sqlite = null;
tzdata = null;
libX11 = null;
xorgproto = null;
libffi = libffiBoot; # without test suite
stripConfig = true;
stripIdlelib = true;

6
pkgs/stdenv/adapters.nix
View file

@ -1,6 +1,6 @@
/* This file contains various functions that take a stdenv and return
a new stdenv with different behaviour, e.g. using a different C
compiler.
/* *
Various functions that take a stdenv and return a new stdenv with different behaviour.
e.g. using a different C compiler.
*/
{ lib, pkgs, config }:

4
pkgs/stdenv/default.nix
View file

@ -7,7 +7,7 @@
{ # Args just for stdenvs' usage
lib
# Args to pass on to the pkgset builder, too
, localSystem, crossSystem, config, overlays, crossOverlays ? [ ] }@args:
, localSystem, crossSystem, config }@args:
let
# The native (i.e., impure) build environment. This one uses the
@ -34,7 +34,7 @@ let
stagesCustom = import ./custom args;
# Select the appropriate stages for the platform `system'.
in if crossSystem != localSystem || crossOverlays != [ ] then
in if crossSystem != localSystem then
stagesCross
else if config ? replaceStdenv then
stagesCustom

8
pkgs/stdenv/linux/default.nix
View file

@ -53,7 +53,7 @@
# For a TUI (rather than CLI) view, you can use:
#
# $ nix-tree --derivation $(nix-instantiate -A stdenv)
{ lib, localSystem, crossSystem, config, overlays, crossOverlays ? [ ]
{ lib, localSystem, crossSystem, config
, bootstrapFiles ? let
table = {
@ -224,7 +224,7 @@ let
};
in {
inherit config overlays;
inherit config;
stdenv = thisStdenv;
};
@ -628,7 +628,7 @@ in assert bootstrapTools.passthru.isFromBootstrapFiles or false; # sanity check
assert isBuiltByNixpkgsCompiler prevStage.coreutils;
assert isBuiltByNixpkgsCompiler prevStage.gnugrep;
assert isBuiltByNixpkgsCompiler prevStage.patchelf; {
inherit config overlays;
inherit config;
stdenv = import ../generic lib rec {
name = "stdenv-linux";
@ -740,6 +740,6 @@ in assert bootstrapTools.passthru.isFromBootstrapFiles or false; # sanity check
assert isBuiltByNixpkgsCompiler prevStage.coreutils;
assert isBuiltByNixpkgsCompiler prevStage.gnugrep;
assert isBuiltByNixpkgsCompiler prevStage.patchelf; {
inherit (prevStage) config overlays stdenv;
inherit (prevStage) config stdenv;
})
]

6
pkgs/top-level/base-packages.nix
View file

@ -4,7 +4,7 @@
# strict sorting has been long lost due to merges. Please use the full-text
# search of your editor. ;)
# Hint: ### starts category names.
{ lib, noSysDirs, config, overlays }:
{ lib, noSysDirs, config }:
res: pkgs: super:
with pkgs;
@ -87,7 +87,7 @@ with pkgs;
path = ../..;
### Helper functions.
inherit lib config overlays;
inherit lib config;
# do not import 'appendToName' to get consistent package-names with the same
# set of package-parameters: https://github.com/NixOS/nixpkgs/issues/68519
@ -113,8 +113,6 @@ with pkgs;
packages, like using pkgs/top-level/release-attrpaths-superset.nix.
'';
tests = callPackages ../test { };
# These are used when buiding compiler-rt / libgcc, prior to building libc.
preLibcCrossHeaders = let inherit (stdenv.targetPlatform) libc;
in if stdenv.targetPlatform.isMinGW then

53
pkgs/top-level/default.nix
View file

@ -17,50 +17,29 @@
or dot-files.
*/
{ # The system packages will be built on. See the manual for the
{
# The system packages will be built on. See the manual for the
# subtle division of labor between these two `*System`s and the three
# `*Platform`s.
localSystem
, lib ? import <auxlib>
, # The system packages will ultimately be run on.
crossSystem ? localSystem
, # Allow a configuration attribute set to be passed in as an argument.
config ? { }
, # List of overlays layers used to extend Nixpkgs.
overlays ? [ ]
, # List of overlays to apply to target packages only.
crossOverlays ? [ ]
, # A function booting the final package set for a specific standard
# environment. See below for the arguments given to that function, the type of
# list it returns.
stdenvStages ? import ../stdenv
localSystem, lib ? import <auxlib>
# The system packages will ultimately be run on.
, crossSystem ? localSystem
# Allow a configuration attribute set to be passed in as an argument.
, config ? { }
# A function booting the final package set for a specific standard
# environment. See below for the arguments given to that function, the type of
# list it returns.
, stdenvStages ? import ../stdenv
, # Ignore unexpected args.
... }@args:
let # Rename the function arguments
let
# Rename the function arguments
config0 = config;
crossSystem0 = crossSystem;
in let
inherit (lib) throwIfNot;
checked = throwIfNot (lib.isList overlays)
"The overlays argument to nixpkgs must be a list." lib.foldr (x:
throwIfNot (lib.isFunction x)
"All overlays passed to nixpkgs must be functions.") (r: r) overlays
throwIfNot (lib.isList crossOverlays)
"The crossOverlays argument to nixpkgs must be a list." lib.foldr (x:
throwIfNot (lib.isFunction x)
"All crossOverlays passed to nixpkgs must be functions.") (r: r)
crossOverlays;
localSystem = lib.systems.elaborate args.localSystem;
# Condition preserves sharing which in turn affects equality.
@ -135,10 +114,8 @@ in let
boot = import ../stdenv/booter.nix { inherit lib allPackages; };
stages = stdenvStages {
inherit lib localSystem crossSystem config overlays crossOverlays;
};
stages = stdenvStages { inherit lib localSystem crossSystem config; };
pkgs = boot stages;
in checked pkgs
in pkgs

12
pkgs/top-level/splice.nix
View file

@ -105,14 +105,6 @@ let
inherit (pkgs.stdenv) buildPlatform targetPlatform hostPlatform;
};
splicedPackagesWithXorg = splicedPackages
// builtins.removeAttrs splicedPackages.xorg [
"callPackage"
"newScope"
"overrideScope"
"packages"
];
in {
inherit splicePackages;
@ -121,9 +113,9 @@ in {
# `newScope' for sets of packages in `pkgs' (see e.g. `gnome' below).
callPackage = pkgs.newScope { };
callPackages = lib.callPackagesWith splicedPackagesWithXorg;
callPackages = lib.callPackagesWith splicedPackages;
newScope = extra: lib.callPackageWith (splicedPackagesWithXorg // extra);
newScope = extra: lib.callPackageWith (splicedPackages // extra);
# prefill 2 fields of the function for convenience
makeScopeWithSplicing =

226
pkgs/top-level/stage.nix
View file

@ -14,22 +14,13 @@ let
# By defining it at the top of the file,
# this value gets reused even if this file is imported multiple times,
# thanks to Nix's import-value cache.
# autoCalledPackages = import ./by-name-overlay.nix ../by-name;
autoCalledPackages = import ./by-name-overlay.nix ../by-name;
in {
## Misc parameters kept the same for all stages
##
# Misc parameters kept the same for all stages
lib, nixpkgsFun
# Utility functions, could just import but passing in for efficiency
lib
, # Use to reevaluate Nixpkgs
nixpkgsFun
## Other parameters
##
, # Either null or an object in the form:
# Either null or an object in the form:
#
# {
# pkgsBuildBuild = ...;
@ -45,75 +36,30 @@ nixpkgsFun
# they are instead defined internally as the current stage. This allows us to
# avoid expensive splicing. `pkgsHostTarget` is skipped because it is always
# defined as the current stage.
adjacentPackages
, adjacentPackages
, # The standard environment to use for building packages.
stdenv
# The standard environment to use for building packages.
, stdenv
, # This is used because stdenv replacement and the stdenvCross do benefit from
# This is used because stdenv replacement and the stdenvCross do benefit from
# the overridden configuration provided by the user, as opposed to the normal
# bootstrapping stdenvs.
allowCustomOverrides
, allowCustomOverrides
, # Non-GNU/Linux OSes are currently "impure" platforms, with their libc
# Non-GNU/Linux OSes are currently "impure" platforms, with their libc
# outside of the store. Thus, GCC, GFortran, & co. must always look for files
# in standard system directories (/usr/include, etc.)
noSysDirs ? stdenv.buildPlatform.system != "x86_64-freebsd"
, noSysDirs ? stdenv.buildPlatform.system != "x86_64-freebsd"
&& stdenv.buildPlatform.system != "i686-freebsd"
&& stdenv.buildPlatform.system != "x86_64-solaris"
&& stdenv.buildPlatform.system != "x86_64-kfreebsd-gnu"
, # The configuration attribute set
config
, # A list of overlays (Additional `self: super: { .. }` customization
# functions) to be fixed together in the produced package set
overlays
# The configuration attribute set
, config
}@args:
let
# This is a function from parsed platforms (like
# stdenv.hostPlatform.parsed) to parsed platforms.
makeMuslParsedPlatform = parsed:
# The following line guarantees that the output of this function
# is a well-formed platform with no missing fields. It will be
# uncommented in a separate PR, in case it breaks the build.
#(x: lib.trivial.pipe x [ (x: builtins.removeAttrs x [ "_type" ]) lib.systems.parse.mkSystem ])
(parsed // {
abi = {
gnu = lib.systems.parse.abis.musl;
gnueabi = lib.systems.parse.abis.musleabi;
gnueabihf = lib.systems.parse.abis.musleabihf;
gnuabin32 = lib.systems.parse.abis.muslabin32;
gnuabi64 = lib.systems.parse.abis.muslabi64;
gnuabielfv2 = lib.systems.parse.abis.musl;
gnuabielfv1 = lib.systems.parse.abis.musl;
# The following two entries ensure that this function is idempotent.
musleabi = lib.systems.parse.abis.musleabi;
musleabihf = lib.systems.parse.abis.musleabihf;
muslabin32 = lib.systems.parse.abis.muslabin32;
muslabi64 = lib.systems.parse.abis.muslabi64;
}.${parsed.abi.name} or lib.systems.parse.abis.musl;
});
stdenvAdapters = self: super:
let
res = import ../stdenv/adapters.nix {
inherit lib config;
pkgs = self;
};
in res // { stdenvAdapters = res; };
trivialBuilders = self: super:
import ../build-support/trivial-builders {
inherit lib;
inherit (self) config;
inherit (self) runtimeShell stdenv stdenvNoCC;
inherit (self.pkgsBuildHost) jq shellcheck-minimal;
inherit (self.pkgsBuildHost.xorg) lndir;
};
stdenvBootstappingAndPlatforms = self: super:
let
withFallback = thisPkgs:
@ -148,31 +94,25 @@ let
inherit stdenv;
};
stdenvAdapters = self: super:
let
res = import ../stdenv/adapters.nix {
inherit lib config;
pkgs = self;
};
in res // { stdenvAdapters = res; };
trivialBuilders = self: super:
import ../build-support/trivial-builders {
inherit lib;
inherit (self) config;
inherit (self) runtimeShell stdenv stdenvNoCC;
inherit (self.pkgsBuildHost) jq shellcheck-minimal;
inherit (self.pkgsBuildHost.xorg) lndir;
};
splice = self: super: import ./splice.nix lib self (adjacentPackages != null);
basePackages = self: super:
let
res =
import ./base-packages.nix { inherit lib noSysDirs config overlays; }
res self super;
in res;
commonUpdaterPackages = self: super:
let
res = import ../common-updater/packages.nix {
inherit lib noSysDirs config overlays;
} res self super;
in res;
buildSupportPackages = self: super:
let
res = import ../build-support/packages.nix {
inherit lib noSysDirs config overlays;
} res self super;
in res;
# aliases = self: super: lib.optionalAttrs config.allowAliases (import ./aliases.nix lib self super);
byNamePackages = self: super:
let
inherit (builtins) readDir;
@ -208,27 +148,47 @@ let
_internalCallByNamePackageFile = file: self.callPackage file { };
} // (mapAttrs (name: self._internalCallByNamePackageFile) packageFiles)
// lib.foldl (acc: path:
(import "${path}/packages.nix" {
inherit lib noSysDirs config overlays;
} res self super) // acc) { } packageGroupFiles;
(import "${path}/packages.nix" { inherit lib noSysDirs config; } res
self super) // acc) { } packageGroupFiles;
in res;
# stdenvOverrides is used to avoid having multiple of versions
# of certain dependencies that were used in bootstrapping the
# standard environment.
stdenvOverrides = self: super:
(super.stdenv.overrides or (_: _: { })) self super;
basePackages = self: super:
let
res =
import ./base-packages.nix { inherit lib noSysDirs config; } res self
super;
in res;
# Allow packages to be overridden globally via the `packageOverrides'
# configuration option, which must be a function that takes `pkgs'
# as an argument and returns a set of new or overridden packages.
# The `packageOverrides' function is called with the *original*
# (un-overridden) set of packages, allowing packageOverrides
# attributes to refer to the original attributes (e.g. "foo =
# ... pkgs.foo ...").
configOverrides = self: super:
lib.optionalAttrs allowCustomOverrides
((config.packageOverrides or (super: { })) super);
buildSupportPackages = self: super:
let
res =
import ../build-support/packages.nix { inherit lib noSysDirs config; }
res self super;
in res;
# This is a function from parsed platforms (like
# stdenv.hostPlatform.parsed) to parsed platforms.
makeMuslParsedPlatform = parsed:
# The following line guarantees that the output of this function
# is a well-formed platform with no missing fields. It will be
# uncommented in a separate PR, in case it breaks the build.
#(x: lib.trivial.pipe x [ (x: builtins.removeAttrs x [ "_type" ]) lib.systems.parse.mkSystem ])
(parsed // {
abi = {
gnu = lib.systems.parse.abis.musl;
gnueabi = lib.systems.parse.abis.musleabi;
gnueabihf = lib.systems.parse.abis.musleabihf;
gnuabin32 = lib.systems.parse.abis.muslabin32;
gnuabi64 = lib.systems.parse.abis.muslabi64;
gnuabielfv2 = lib.systems.parse.abis.musl;
gnuabielfv1 = lib.systems.parse.abis.musl;
# The following two entries ensure that this function is idempotent.
musleabi = lib.systems.parse.abis.musleabi;
musleabihf = lib.systems.parse.abis.musleabihf;
muslabin32 = lib.systems.parse.abis.muslabin32;
muslabi64 = lib.systems.parse.abis.muslabi64;
}.${parsed.abi.name} or lib.systems.parse.abis.musl;
});
# Convenience attributes for instantitating package sets. Each of
# these will instantiate a new version of allPackages. Currently the
@ -248,7 +208,6 @@ let
lib.systems.examples;
pkgsLLVM = nixpkgsFun {
overlays = [ (self': super': { pkgsLLVM = super'; }) ] ++ overlays;
# Bootstrap a cross stdenv using the LLVM toolchain.
# This is currently not possible when compiling natively,
# so we don't need to check hostPlatform != buildPlatform.
@ -264,7 +223,6 @@ let
pkgsMusl =
if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then
nixpkgsFun {
overlays = [ (self': super': { pkgsMusl = super'; }) ] ++ overlays;
${
if stdenv.hostPlatform == stdenv.buildPlatform then
"localSystem"
@ -282,8 +240,6 @@ let
pkgsi686Linux =
if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then
nixpkgsFun {
overlays = [ (self': super': { pkgsi686Linux = super'; }) ]
++ overlays;
${
if stdenv.hostPlatform == stdenv.buildPlatform then
"localSystem"
@ -301,8 +257,6 @@ let
# x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages
pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then
nixpkgsFun {
overlays = [ (self': super': { pkgsx86_64Darwin = super'; }) ]
++ overlays;
localSystem = {
parsed = stdenv.hostPlatform.parsed // {
cpu = lib.systems.parse.cpuTypes.x86_64;
@ -323,28 +277,9 @@ let
lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux";
};
# Extend the package set with zero or more overlays. This preserves
# preexisting overlays. Prefer to initialize with the right overlays
# in one go when calling Nixpkgs, for performance and simplicity.
appendOverlays = extraOverlays:
if extraOverlays == [ ] then
self
else
nixpkgsFun { overlays = args.overlays ++ extraOverlays; };
# NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB
# of allocations. DO NOT USE THIS IN NIXPKGS.
#
# Extend the package set with a single overlay. This preserves
# preexisting overlays. Prefer to initialize with the right overlays
# in one go when calling Nixpkgs, for performance and simplicity.
# Prefer appendOverlays if used repeatedly.
extend = f: self.appendOverlays [ f ];
# Fully static packages.
# Currently uses Musl on Linux (couldn’t get static glibc to work).
pkgsStatic = nixpkgsFun ({
overlays = [ (self': super': { pkgsStatic = super'; }) ] ++ overlays;
crossSystem = {
isStatic = true;
parsed = if stdenv.isLinux then
@ -355,22 +290,15 @@ let
gcc.abi = "elfv2";
};
});
pkgsExtraHardening = nixpkgsFun {
overlays = [
(self': super': {
pkgsExtraHardening = super';
stdenv = super'.withDefaultHardeningFlags
(super'.stdenv.cc.defaultHardeningFlags
++ [ "zerocallusedregs" "trivialautovarinit" ]) super'.stdenv;
})
] ++ overlays;
};
};
# stdenvOverrides is used to avoid having multiple of versions
# of certain dependencies that were used in bootstrapping the
# standard environment.
stdenvOverrides = self: super:
(super.stdenv.overrides or (_: _: { })) self super;
# The complete chain of package set builders, applied from top to bottom.
# stdenvOverlays must be last as it brings package forward from the
# previous bootstrapping phases which have already been overlayed.
toFix = lib.foldl' (lib.flip lib.extends) (self: { }) ([
stdenvBootstappingAndPlatforms
stdenvAdapters
@ -378,12 +306,10 @@ let
splice
byNamePackages
basePackages
commonUpdaterPackages
buildSupportPackages
otherPackageSets
# aliases
configOverrides
] ++ overlays ++ [ stdenvOverrides ]);
stdenvOverrides
]);
# Return the complete set of packages.
in lib.fix toFix