core/pkgs/build-support/build-bazel-package/default.nix
2024-05-13 11:34:52 -04:00

295 lines
10 KiB
Nix
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{ stdenv
, cacert
, lib
, writeCBin
}:
args@{
name ? "${args.pname}-${args.version}"
, bazel
, bazelFlags ? []
, bazelBuildFlags ? []
, bazelTestFlags ? []
, bazelRunFlags ? []
, runTargetFlags ? []
, bazelFetchFlags ? []
, bazelTargets ? []
, bazelTestTargets ? []
, bazelRunTarget ? null
, buildAttrs
, fetchAttrs
# Newer versions of Bazel are moving away from built-in rules_cc and instead
# allow fetching it as an external dependency in a WORKSPACE file[1]. If
# removed in the fixed-output fetch phase, building will fail to download it.
# This can be seen e.g. in #73097
#
# This option allows configuring the removal of rules_cc in cases where a
# project depends on it via an external dependency.
#
# [1]: https://github.com/bazelbuild/rules_cc
, removeRulesCC ? true
, removeLocalConfigCc ? true
, removeLocal ? true
# Use build --nobuild instead of fetch. This allows fetching the dependencies
# required for the build as configured, rather than fetching all the dependencies
# which may not work in some situations (e.g. Java code which ends up relying on
# Debian-specific /usr/share/java paths, but doesn't in the configured build).
, fetchConfigured ? true
# Dont add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE /
# NIX_LDFLAGS. This is necessary when using a custom toolchain which
# Bazel wants all headers / libraries to come from, like when using
# CROSSTOOL. Weirdly, we can still get the flags through the wrapped
# compiler.
, dontAddBazelOpts ? false
, ...
}:
let
fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" "removeRulesCC" ] // {
inherit
name
bazelFlags
bazelBuildFlags
bazelTestFlags
bazelRunFlags
runTargetFlags
bazelFetchFlags
bazelTargets
bazelTestTargets
bazelRunTarget
dontAddBazelOpts
;
};
fBuildAttrs = fArgs // buildAttrs;
fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ];
bazelCmd = { cmd, additionalFlags, targets, targetRunFlags ? [ ] }:
lib.optionalString (targets != [ ]) ''
# See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]
BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
USER=homeless-shelter \
bazel \
--batch \
--output_base="$bazelOut" \
--output_user_root="$bazelUserRoot" \
${cmd} \
--curses=no \
"''${copts[@]}" \
"''${host_copts[@]}" \
"''${linkopts[@]}" \
"''${host_linkopts[@]}" \
$bazelFlags \
${lib.strings.concatStringsSep " " additionalFlags} \
${lib.strings.concatStringsSep " " targets} \
${lib.optionalString (targetRunFlags != []) " -- " + lib.strings.concatStringsSep " " targetRunFlags}
'';
# we need this to chmod dangling symlinks on darwin, gnu coreutils refuses to do so:
# chmod: cannot operate on dangling symlink '$symlink'
chmodder = writeCBin "chmodder" ''
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <string.h>
int main(int argc, char** argv) {
mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO;
if (argc != 2) {
fprintf(stderr, "usage: chmodder file");
exit(EXIT_FAILURE);
}
if (lchmod(argv[1], mode) != 0) {
fprintf(stderr, "failed to lchmod '%s': %s", argv[0], strerror(errno));
exit(EXIT_FAILURE);
}
}
'';
in
stdenv.mkDerivation (fBuildAttrs // {
deps = stdenv.mkDerivation (fFetchAttrs // {
name = "${name}-deps.tar.gz";
impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ fFetchAttrs.impureEnvVars or [];
nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ];
preHook = fFetchAttrs.preHook or "" + ''
export bazelOut="$(echo ''${NIX_BUILD_TOP}/output | sed -e 's,//,/,g')"
export bazelUserRoot="$(echo ''${NIX_BUILD_TOP}/tmp | sed -e 's,//,/,g')"
export HOME="$NIX_BUILD_TOP"
export USER="nix"
# This is needed for git_repository with https remotes
export GIT_SSL_CAINFO="${cacert}/etc/ssl/certs/ca-bundle.crt"
# This is needed for Bazel fetchers that are themselves programs (e.g.
# rules_go using the go toolchain)
export SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt"
'';
buildPhase = fFetchAttrs.buildPhase or ''
runHook preBuild
${
bazelCmd {
cmd = if fetchConfigured then "build --nobuild" else "fetch";
additionalFlags = [
# We disable multithreading for the fetching phase since it can lead to timeouts with many dependencies/threads:
# https://github.com/bazelbuild/bazel/issues/6502
"--loading_phase_threads=1"
"$bazelFetchFlags"
] ++ (if fetchConfigured then ["--jobs" "$NIX_BUILD_CORES"] else []);
targets = fFetchAttrs.bazelTargets ++ fFetchAttrs.bazelTestTargets;
}
}
runHook postBuild
'';
installPhase = fFetchAttrs.installPhase or (''
runHook preInstall
# Remove all built in external workspaces, Bazel will recreate them when building
rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
${lib.optionalString removeRulesCC "rm -rf $bazelOut/external/{rules_cc,\\@rules_cc.marker}"}
rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
${lib.optionalString removeLocalConfigCc "rm -rf $bazelOut/external/{local_config_cc,\\@local_config_cc.marker}"}
${lib.optionalString removeLocal "rm -rf $bazelOut/external/{local_*,\\@local_*.marker}"}
# Clear markers
find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
# Remove all vcs files
rm -rf $(find $bazelOut/external -type d -name .git)
rm -rf $(find $bazelOut/external -type d -name .svn)
rm -rf $(find $bazelOut/external -type d -name .hg)
# Removing top-level symlinks along with their markers.
# This is needed because they sometimes point to temporary paths (?).
# For example, in Tensorflow-gpu build:
# platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms
find $bazelOut/external -maxdepth 1 -type l | while read symlink; do
name="$(basename "$symlink")"
rm "$symlink"
test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker" || true
done
# Patching symlinks to remove build directory reference
find $bazelOut/external -type l | while read symlink; do
new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")"
rm "$symlink"
ln -sf "$new_target" "$symlink"
'' + lib.optionalString stdenv.isDarwin ''
# on linux symlink permissions cannot be modified, so we modify those on darwin to match the linux ones
${chmodder}/bin/chmodder "$symlink"
'' + ''
done
echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version
(cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/)
runHook postInstall
'');
dontFixup = true;
allowedRequisites = [];
outputHashAlgo = "sha256";
outputHash = fetchAttrs.sha256;
});
nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ];
preHook = fBuildAttrs.preHook or "" + ''
export bazelOut="$NIX_BUILD_TOP/output"
export bazelUserRoot="$NIX_BUILD_TOP/tmp"
export HOME="$NIX_BUILD_TOP"
'';
preConfigure = ''
mkdir -p "$bazelOut"
(cd $bazelOut && tar xfz $deps)
test "${bazel.name}" = "$(<$bazelOut/external/.nix-bazel-version)" || {
echo "fixed output derivation was built for a different bazel version" >&2
echo " got: $(<$bazelOut/external/.nix-bazel-version)" >&2
echo "expected: ${bazel.name}" >&2
exit 1
}
chmod -R +w $bazelOut
find $bazelOut -type l | while read symlink; do
if [[ $(readlink "$symlink") == *NIX_BUILD_TOP* ]]; then
ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink"
fi
done
'' + fBuildAttrs.preConfigure or "";
buildPhase = fBuildAttrs.buildPhase or ''
runHook preBuild
# Bazel sandboxes the execution of the tools it invokes, so even though we are
# calling the correct nix wrappers, the values of the environment variables
# the wrappers are expecting will not be set. So instead of relying on the
# wrappers picking them up, pass them in explicitly via `--copt`, `--linkopt`
# and related flags.
copts=()
host_copts=()
linkopts=()
host_linkopts=()
if [ -z "''${dontAddBazelOpts:-}" ]; then
for flag in $NIX_CFLAGS_COMPILE; do
copts+=( "--copt=$flag" )
host_copts+=( "--host_copt=$flag" )
done
for flag in $NIX_CXXSTDLIB_COMPILE; do
copts+=( "--copt=$flag" )
host_copts+=( "--host_copt=$flag" )
done
for flag in $NIX_LDFLAGS; do
linkopts+=( "--linkopt=-Wl,$flag" )
host_linkopts+=( "--host_linkopt=-Wl,$flag" )
done
fi
${
bazelCmd {
cmd = "test";
additionalFlags =
["--test_output=errors"] ++ fBuildAttrs.bazelTestFlags ++ ["--jobs" "$NIX_BUILD_CORES"];
targets = fBuildAttrs.bazelTestTargets;
}
}
${
bazelCmd {
cmd = "build";
additionalFlags = fBuildAttrs.bazelBuildFlags ++ ["--jobs" "$NIX_BUILD_CORES"];
targets = fBuildAttrs.bazelTargets;
}
}
${
bazelCmd {
cmd = "run";
additionalFlags = fBuildAttrs.bazelRunFlags ++ [ "--jobs" "$NIX_BUILD_CORES" ];
# Bazel run only accepts a single target, but `bazelCmd` expects `targets` to be a list.
targets = lib.optionals (fBuildAttrs.bazelRunTarget != null) [ fBuildAttrs.bazelRunTarget ];
targetRunFlags = fBuildAttrs.runTargetFlags;
}
}
runHook postBuild
'';
})
# [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]:
# Bazel computes the default value of output_user_root before parsing the
# flag. The computation of the default value involves getting the $USER
# from the environment. Code here :
# https://github.com/bazelbuild/bazel/blob/9323c57607d37f9c949b60e293b573584906da46/src/main/cpp/startup_options.cc#L123-L124
#
# On macOS Bazel will use the system installed Xcode or CLT toolchain instead of the one in the PATH unless we pass BAZEL_USE_CPP_ONLY_TOOLCHAIN.