From 69a8760bd315633b63f7987f5cc08fbaade3e6fd Mon Sep 17 00:00:00 2001 From: Jake Hamilton Date: Mon, 17 Jun 2024 02:15:45 -0700 Subject: [PATCH] feat: working linux-headers build --- foundation/flake.lock | 8 +- tidepool/flake.lock | 16 +-- tidepool/src/export.nix | 4 + tidepool/src/exports/packages.nix | 2 +- tidepool/src/lib/packages.nix | 30 ++--- tidepool/src/lib/systems.nix | 16 ++- tidepool/src/lib/types.nix | 49 +++++--- .../packages/foundation/binutils/default.nix | 117 ++++++++++++++++++ .../binutils/patches/deterministic.patch | 13 ++ tidepool/src/packages/foundation/default.nix | 2 + .../src/packages/foundation/gcc/default.nix | 15 ++- .../src/packages/foundation/glibc/default.nix | 112 +++++++++++++++++ .../foundation/linux-headers/default.nix | 89 +++++++++++++ 13 files changed, 418 insertions(+), 55 deletions(-) create mode 100644 tidepool/src/packages/foundation/binutils/default.nix create mode 100644 tidepool/src/packages/foundation/binutils/patches/deterministic.patch create mode 100644 tidepool/src/packages/foundation/glibc/default.nix create mode 100644 tidepool/src/packages/foundation/linux-headers/default.nix diff --git a/foundation/flake.lock b/foundation/flake.lock index 4e69788..f4e0cb0 100644 --- a/foundation/flake.lock +++ b/foundation/flake.lock @@ -3,10 +3,10 @@ "lib": { "locked": { "dir": "lib", - "dirtyRev": "9c29945531c58ad81f05cd1f4958c8894a733216-dirty", - "dirtyShortRev": "9c29945-dirty", - "lastModified": 1718255029, - "narHash": "sha256-iPMsyNszFA+EzjtIpjmu9EAG7zdjLbuugLtfa64dJos=", + "dirtyRev": "9850da8aa9dc9be22e237c9b424a18e801e53ecb-dirty", + "dirtyShortRev": "9850da8-dirty", + "lastModified": 1718529861, + "narHash": "sha256-tv/0C7ixH+9Ij+r+5nua48OlXXXnbdEsnenxX4eG/Sk=", "type": "git", "url": "file:../?dir=lib" }, diff --git a/tidepool/flake.lock b/tidepool/flake.lock index 9eadd0d..805747f 100644 --- a/tidepool/flake.lock +++ b/tidepool/flake.lock @@ -8,10 +8,10 @@ }, "locked": { "dir": "foundation", - "dirtyRev": "0312e3c4cc261e2384fcf372c766a0cf245f3213-dirty", - "dirtyShortRev": "0312e3c-dirty", - "lastModified": 1718460525, - "narHash": "sha256-+ToaXY8ISWLx9AtO/CjfY/6SGuKu8A/d9ncqE4h2H20=", + "dirtyRev": "9850da8aa9dc9be22e237c9b424a18e801e53ecb-dirty", + "dirtyShortRev": "9850da8-dirty", + "lastModified": 1718529861, + "narHash": "sha256-X1Wd6mDz8GTaoxt1ylkvZfrJOcZtspJrEjXMtJ2ZyG0=", "type": "git", "url": "file:../?dir=foundation" }, @@ -24,10 +24,10 @@ "lib": { "locked": { "dir": "lib", - "dirtyRev": "0312e3c4cc261e2384fcf372c766a0cf245f3213-dirty", - "dirtyShortRev": "0312e3c-dirty", - "lastModified": 1718460525, - "narHash": "sha256-+ToaXY8ISWLx9AtO/CjfY/6SGuKu8A/d9ncqE4h2H20=", + "dirtyRev": "9850da8aa9dc9be22e237c9b424a18e801e53ecb-dirty", + "dirtyShortRev": "9850da8-dirty", + "lastModified": 1718529861, + "narHash": "sha256-X1Wd6mDz8GTaoxt1ylkvZfrJOcZtspJrEjXMtJ2ZyG0=", "type": "git", "url": "file:../?dir=lib" }, diff --git a/tidepool/src/export.nix b/tidepool/src/export.nix index c0a8c14..880c25d 100644 --- a/tidepool/src/export.nix +++ b/tidepool/src/export.nix @@ -24,6 +24,10 @@ in { # })) # .config; foundation-gcc = config.packages.foundation.gcc; + foundation-binutils = config.packages.foundation.binutils; + foundation-linux-headers = config.packages.foundation.linux-headers.versions.latest.extend { + platform.host = lib.modules.overrides.force "x86_64-linux"; + }; # example-x = config.packages.example.x; # cross-example-x-x86_64-linux = config.packages.cross.x86_64-linux.example.x; }; diff --git a/tidepool/src/exports/packages.nix b/tidepool/src/exports/packages.nix index c58b846..111c34c 100644 --- a/tidepool/src/exports/packages.nix +++ b/tidepool/src/exports/packages.nix @@ -24,7 +24,7 @@ in { builtins.mapAttrs ( name: package: let - result = lib'.packages.build package system system; + result = lib'.packages.build package system system system; in result ) diff --git a/tidepool/src/lib/packages.nix b/tidepool/src/lib/packages.nix index b6bc0ce..11d152d 100644 --- a/tidepool/src/lib/packages.nix +++ b/tidepool/src/lib/packages.nix @@ -21,37 +21,37 @@ in builtins.head sorted; - build = package: system: cross: let + build = package: build: host: target: let resolved = if package ? versions then package.versions.${config.preferences.packages.version} or (package.versions.${lib'.packages.getLatest package}) else package; - buildDependencies = builtins.mapAttrs (name: dep: lib'.packages.build dep system cross); + buildDependencies = build': host': target': builtins.mapAttrs (name: dep: lib'.packages.build dep build' host' target'); result = resolved.extend ({config}: { config = { platform = { - build = system; - host = cross; - target = lib.modules.override 150 cross; + build = build; + host = host; + target = lib.modules.override 150 target; }; deps = { build = { - only = buildDependencies resolved.deps.build.only; - build = buildDependencies resolved.deps.build.build; - host = buildDependencies resolved.deps.build.host; - target = buildDependencies resolved.deps.build.target; + only = buildDependencies build build build resolved.deps.build.only; + build = buildDependencies build build target resolved.deps.build.build; + host = buildDependencies build host target resolved.deps.build.host; + target = buildDependencies build target target resolved.deps.build.target; }; host = { - only = buildDependencies resolved.deps.host.only; - host = buildDependencies resolved.deps.host.host; - target = buildDependencies resolved.deps.host.target; + only = buildDependencies host host host resolved.deps.host.only; + host = buildDependencies host host target resolved.deps.host.host; + target = buildDependencies host target target resolved.deps.host.target; }; target = { - only = buildDependencies resolved.deps.target.only; - target = buildDependencies resolved.deps.target.target; + only = buildDependencies target target target resolved.deps.target.only; + target = buildDependencies target target target resolved.deps.target.target; }; }; @@ -59,7 +59,7 @@ }; }); in - result.config; + result; }; }; } diff --git a/tidepool/src/lib/systems.nix b/tidepool/src/lib/systems.nix index 94ea624..b97109c 100644 --- a/tidepool/src/lib/systems.nix +++ b/tidepool/src/lib/systems.nix @@ -14,7 +14,13 @@ matchAnyAttrs = patterns: if builtins.isList patterns - then value: builtins.any (pattern: lib.attrs.match pattern value) patterns + then + value: + builtins.any (pattern: + if builtins.isFunction pattern + then pattern value + else matchAnyAttrs pattern value) + patterns else lib.attrs.match patterns; getDoubles = predicate: @@ -1586,8 +1592,8 @@ in { }; isArmv7 = map ({arch, ...}: {cpu = {inherit arch;};}) - (lib.filter (cpu: lib.hasPrefix "armv7" cpu.arch or "") - (lib.attrValues types.cpus)); + (builtins.filter (cpu: lib.strings.hasPrefix "armv7" cpu.arch or "") + (builtins.attrValues types.cpus)); isAarch64 = { cpu = { family = "arm"; @@ -1926,7 +1932,7 @@ in { // lib'.systems.platforms.select resolved) linux-kernel gcc - rust + rustc ; double = lib'.systems.into.double resolved.system; @@ -2162,7 +2168,7 @@ in { } // builtins.mapAttrs (name: match: match resolved.system) lib'.systems.match // builtins.mapAttrs (name: validate: validate (resolved.gcc.arch or "default")) lib'.systems.validate.architecture - // settings; + // (builtins.removeAttrs settings ["system"]); assertions = builtins.foldl' diff --git a/tidepool/src/lib/types.nix b/tidepool/src/lib/types.nix index 8945019..7a579ac 100644 --- a/tidepool/src/lib/types.nix +++ b/tidepool/src/lib/types.nix @@ -82,13 +82,15 @@ extend = lib.options.create { description = "Extend the package's submodules with additional configuration."; type = lib.types.function lib.types.raw; - default.value = value: - meta.extend { + default.value = value: let + result = meta.extend { modules = if builtins.isAttrs value then [{config = value;}] else lib.lists.from.any value; }; + in + result.config; }; name = lib.options.create { @@ -172,40 +174,47 @@ default.value = "x86_64-linux"; apply = raw: let system = lib'.systems.from.string raw; - in { - inherit raw system; - - double = lib'.systems.into.double system; - triple = lib'.systems.into.triple system; - }; + x = lib'.systems.withBuildInfo raw; + in + x; }; host = lib.options.create { description = "The host platform for the package."; type = lib.types.string; default.value = "x86_64-linux"; + # apply = raw: let + # system = lib'.systems.from.string raw; + # in { + # inherit raw system; + + # double = lib'.systems.into.double system; + # triple = lib'.systems.into.triple system; + # }; apply = raw: let system = lib'.systems.from.string raw; - in { - inherit raw system; - - double = lib'.systems.into.double system; - triple = lib'.systems.into.triple system; - }; + x = lib'.systems.withBuildInfo raw; + in + x; }; target = lib.options.create { description = "The target platform for the package."; type = lib.types.string; default.value = "x86_64-linux"; + # apply = raw: let + # system = lib'.systems.from.string raw; + # in { + # inherit raw system; + + # double = lib'.systems.into.double system; + # triple = lib'.systems.into.triple system; + # }; apply = raw: let system = lib'.systems.from.string raw; - in { - inherit raw system; - - double = lib'.systems.into.double system; - triple = lib'.systems.into.triple system; - }; + x = lib'.systems.withBuildInfo raw; + in + x; }; }; diff --git a/tidepool/src/packages/foundation/binutils/default.nix b/tidepool/src/packages/foundation/binutils/default.nix new file mode 100644 index 0000000..1121354 --- /dev/null +++ b/tidepool/src/packages/foundation/binutils/default.nix @@ -0,0 +1,117 @@ +{ + lib, + lib', + config, + options, +}: let + inherit + (config) + mirrors + builders + # These are the upstream foundational packages exported from the Aux Foundation project. + + foundation + ; +in { + config.packages.foundation.binutils = { + versions = { + "latest" = { + config, + meta, + }: { + options = { + src = lib.options.create { + type = lib.types.derivation; + description = "Source for the package."; + }; + }; + + config = { + meta = { + platforms = ["i686-linux"]; + }; + + pname = "binutils"; + version = "2.41"; + + builder = builders.basic; + + env = { + PATH = lib.paths.bin [ + foundation.stage2-gcc + foundation.stage2-binutils + foundation.stage2-gnumake + foundation.stage2-gnupatch + foundation.stage2-gnused + foundation.stage2-gnugrep + foundation.stage2-gawk + foundation.stage2-diffutils + foundation.stage2-findutils + foundation.stage2-gnutar + foundation.stage1-xz + ]; + }; + + phases = let + patches = [ + # Make binutils output deterministic by default. + ./patches/deterministic.patch + ]; + + configureFlags = [ + # "CC=musl-gcc" + "LDFLAGS=--static" + "--prefix=${builtins.placeholder "out"}" + "--build=${config.platform.build.triple}" + "--host=${config.platform.host.triple}" + "--target=${config.platform.target.triple}" + + "--with-sysroot=/" + "--enable-deterministic-archives" + # depends on bison + "--disable-gprofng" + + # Turn on --enable-new-dtags by default to make the linker set + # RUNPATH instead of RPATH on binaries. This is important because + # RUNPATH can be overridden using LD_LIBRARY_PATH at runtime. + "--enable-new-dtags" + + # By default binutils searches $libdir for libraries. This brings in + # libbfd and libopcodes into a default visibility. Drop default lib + # path to force users to declare their use of these libraries. + "--with-lib-path=:" + + "--disable-multilib" + ]; + in { + unpack = lib.dag.entry.before ["patch"] '' + tar xf ${config.src} + cd binutils-${config.version} + ''; + + patch = lib.dag.entry.between ["configure"] ["unpack"] '' + ${lib.strings.concatMapSep "\n" (file: "patch -Np1 -i ${file}") patches} + ''; + + configure = lib.dag.entry.between ["build"] ["patch"] '' + bash ./configure ${builtins.concatStringsSep " " configureFlags} + ''; + + build = lib.dag.entry.between ["install"] ["configure"] '' + make -j $NIX_BUILD_CORES + ''; + + install = lib.dag.entry.after ["build"] '' + make -j $NIX_BUILD_CORES install-strip + ''; + }; + + src = builtins.fetchurl { + url = "${mirrors.gnu}/binutils/binutils-${config.version}.tar.xz"; + sha256 = "rppXieI0WeWWBuZxRyPy0//DHAMXQZHvDQFb3wYAdFA="; + }; + }; + }; + }; + }; +} diff --git a/tidepool/src/packages/foundation/binutils/patches/deterministic.patch b/tidepool/src/packages/foundation/binutils/patches/deterministic.patch new file mode 100644 index 0000000..5b479ba --- /dev/null +++ b/tidepool/src/packages/foundation/binutils/patches/deterministic.patch @@ -0,0 +1,13 @@ +diff -ur orig/binutils-2.23.1/ld/ldlang.c binutils-2.23.1/ld/ldlang.c +--- orig/ld/ldlang.c ++++ new/ld/ldlang.c +@@ -3095,6 +3095,8 @@ + ldfile_output_machine)) + einfo (_("%P%F:%s: can not set architecture: %E\n"), name); + ++ link_info.output_bfd->flags |= BFD_DETERMINISTIC_OUTPUT; ++ + link_info.hash = bfd_link_hash_table_create (link_info.output_bfd); + if (link_info.hash == NULL) + einfo (_("%P%F: can not create hash table: %E\n")); + diff --git a/tidepool/src/packages/foundation/default.nix b/tidepool/src/packages/foundation/default.nix index 5772791..8b62be3 100644 --- a/tidepool/src/packages/foundation/default.nix +++ b/tidepool/src/packages/foundation/default.nix @@ -6,6 +6,8 @@ }: { includes = [ ./gcc + ./binutils + ./linux-headers ]; config = { diff --git a/tidepool/src/packages/foundation/gcc/default.nix b/tidepool/src/packages/foundation/gcc/default.nix index 86af967..03214a9 100644 --- a/tidepool/src/packages/foundation/gcc/default.nix +++ b/tidepool/src/packages/foundation/gcc/default.nix @@ -15,7 +15,7 @@ in { config.packages.foundation.gcc = { versions = { - "13.2.0" = { + "latest" = { config, meta, }: { @@ -98,9 +98,12 @@ in { then foundation.stage2-gcc # Otherwise we are going to need a cross-compiler. else + # TODO: Create a gcc-cross package. (meta.extend (args: { config = { platform = { + build = config.platform.build.triple; + host = config.platform.build.triple; target = lib.modules.override.force config.platform.host.triple; }; }; @@ -126,6 +129,14 @@ in { phases = let host = lib'.systems.withBuildInfo config.platform.host; + + mbits = + if host.system.cpu.family == "x86" + then + if host.is64bit + then "-m64" + else "-m32" + else ""; in { unpack = lib.dag.entry.before ["patch"] '' # Unpack @@ -150,7 +161,7 @@ in { configure = lib.dag.entry.between ["build"] ["patch"] '' # Configure - export CC="gcc -Wl,-dynamic-linker -march=${host.gcc.arch or host.system.cpu.arch} -Wl,${foundation.stage1-musl}/lib/libc.so" + export CC="gcc -Wl,-dynamic-linker -march=${host.gcc.arch or host.system.cpu.family} ${mbits} -Wl,${foundation.stage1-musl}/lib/libc.so" export CXX="g++ -Wl,-dynamic-linker -Wl,${foundation.stage1-musl}/lib/libc.so" export CFLAGS_FOR_TARGET="-Wl,-dynamic-linker -Wl,${foundation.stage1-musl}/lib/libc.so" export LIBRARY_PATH="${foundation.stage1-musl}/lib" diff --git a/tidepool/src/packages/foundation/glibc/default.nix b/tidepool/src/packages/foundation/glibc/default.nix new file mode 100644 index 0000000..6fe7e50 --- /dev/null +++ b/tidepool/src/packages/foundation/glibc/default.nix @@ -0,0 +1,112 @@ +{ + lib, + lib', + config, + options, +}: let + inherit + (config) + mirrors + builders + # These are the upstream foundational packages exported from the Aux Foundation project. + + foundation + ; +in { + config.packages.foundation.glibc = { + versions = { + "latest" = { + config, + meta, + }: { + options = { + src = lib.options.create { + type = lib.types.derivation; + description = "Source for the package."; + }; + }; + + config = { + meta = { + platforms = ["i686-linux"]; + }; + + pname = "gcc"; + version = "2.38"; + + src = builtins.fetchurl { + url = "${mirrors.gnu}/libc/glibc-${config.version}.tar.xz"; + sha256 = "+4KZiZiyspllRnvBtp0VLpwwfSzzAcnq+0VVt3DvP9I="; + }; + + builder = builders.basic; + + env = { + PATH = let + gcc = + if config.platform.build.triple == config.platform.host.triple + # If we're on the same system then we can use the existing GCC instance. + then foundation.stage2-gcc + # Otherwise we are going to need a cross-compiler. + else + (meta.extend (args: { + config = { + platform = { + build = config.platform.build.triple; + host = config.platform.build.triple; + target = lib.modules.override.force config.platform.host.triple; + }; + }; + })) + .config + .package; + in + lib.paths.bin [ + foundation.stage2-gcc + foundation.stage2-binutils + foundation.stage2-gnumake + foundation.stage2-gnused + foundation.stage2-gnugrep + foundation.stage2-gawk + foundation.stage2-diffutils + foundation.stage2-findutils + foundation.stage2-gnutar + foundation.stage2-gzip + foundation.stage2-bzip2 + foundation.stage1-xz + ]; + }; + + phases = { + unpack = lib.dag.entry.before ["patch"] '' + tar xf ${config.src} + cd glibc-${config.version} + ''; + + configure = lib.dag.entry.between ["build"] ["patch"] '' + mkdir build + cd build + # libstdc++.so is built against musl and fails to link + export CXX=false + bash ../configure \ + --prefix=$out \ + --build=${config.platform.build.triple} \ + --host=${config.platform.host.triple} \ + --with-headers=${foundation.stage1-linux-headers}/include + ''; + + build = lib.dag.entry.between ["install"] ["configure"] '' + # Build + make -j $NIX_BUILD_CORES + ''; + + install = lib.dag.entry.after ["build"] '' + # Install + make -j $NIX_BUILD_CORES install-strip + ''; + }; + }; + }; + }; + }; +} diff --git a/tidepool/src/packages/foundation/linux-headers/default.nix b/tidepool/src/packages/foundation/linux-headers/default.nix new file mode 100644 index 0000000..d8ba05e --- /dev/null +++ b/tidepool/src/packages/foundation/linux-headers/default.nix @@ -0,0 +1,89 @@ +{ + lib, + lib', + config, + options, +}: let + inherit + (config) + mirrors + builders + # These are the upstream foundational packages exported from the Aux Foundation project. + + foundation + ; +in { + config.packages.foundation.linux-headers = { + versions = { + "latest" = { + config, + meta, + }: { + options = { + src = lib.options.create { + type = lib.types.derivation; + description = "Source for the package."; + }; + }; + + config = { + meta = { + platforms = ["i686-linux"]; + }; + + pname = "linux-headers"; + version = "6.5.6"; + + builder = builders.basic; + + env = { + PATH = lib.paths.bin [ + foundation.stage2-gcc + foundation.stage1-musl + foundation.stage2-binutils + foundation.stage2-gnumake + foundation.stage2-gnupatch + foundation.stage2-gnused + foundation.stage2-gnugrep + foundation.stage2-gawk + foundation.stage2-diffutils + foundation.stage2-findutils + foundation.stage2-gnutar + foundation.stage1-xz + ]; + }; + + phases = { + unpack = lib.dag.entry.before ["patch"] '' + tar xf ${config.src} + cd linux-${config.version} + ''; + + patch = + lib.dag.entry.between ["configure"] ["unpack"] '' + ''; + + configure = + lib.dag.entry.between ["build"] ["patch"] '' + ''; + + build = lib.dag.entry.between ["install"] ["configure"] '' + make -j $NIX_BUILD_CORES CC=musl-gcc HOSTCC=musl-gcc ARCH=${config.platform.host.linux.arch} headers + ''; + + install = lib.dag.entry.after ["build"] '' + find usr/include -name '.*' -exec rm {} + + mkdir -p $out + cp -rv usr/include $out/ + ''; + }; + + src = builtins.fetchurl { + url = "https://cdn.kernel.org/pub/linux/kernel/v${lib.versions.major config.version}.x/linux-${config.version}.tar.xz"; + sha256 = "eONtQhRUcFHCTfIUD0zglCjWxRWtmnGziyjoCUqV0vY="; + }; + }; + }; + }; + }; +}