core/pkgs/build-support/build-bazel-package/default.nix

295 lines
10 KiB
Nix
Raw Normal View History

2024-05-02 00:46:19 +00:00
{ stdenv
, cacert
, lib
, writeCBin
}:
args@{
name ? "${args.pname}-${args.version}"
, bazel
, bazelFlags ? []
, bazelBuildFlags ? []
, bazelTestFlags ? []
, bazelRunFlags ? []
, runTargetFlags ? []
, bazelFetchFlags ? []
, bazelTargets ? []
, bazelTestTargets ? []
, bazelRunTarget ? null
, buildAttrs
, fetchAttrs
# Newer versions of Bazel are moving away from built-in rules_cc and instead
# allow fetching it as an external dependency in a WORKSPACE file[1]. If
# removed in the fixed-output fetch phase, building will fail to download it.
# This can be seen e.g. in #73097
#
# This option allows configuring the removal of rules_cc in cases where a
# project depends on it via an external dependency.
#
# [1]: https://github.com/bazelbuild/rules_cc
, removeRulesCC ? true
, removeLocalConfigCc ? true
, removeLocal ? true
# Use build --nobuild instead of fetch. This allows fetching the dependencies
# required for the build as configured, rather than fetching all the dependencies
# which may not work in some situations (e.g. Java code which ends up relying on
# Debian-specific /usr/share/java paths, but doesn't in the configured build).
, fetchConfigured ? true
# Dont add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE /
# NIX_LDFLAGS. This is necessary when using a custom toolchain which
# Bazel wants all headers / libraries to come from, like when using
# CROSSTOOL. Weirdly, we can still get the flags through the wrapped
# compiler.
, dontAddBazelOpts ? false
, ...
}:
let
fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" "removeRulesCC" ] // {
inherit
name
bazelFlags
bazelBuildFlags
bazelTestFlags
bazelRunFlags
runTargetFlags
bazelFetchFlags
bazelTargets
bazelTestTargets
bazelRunTarget
dontAddBazelOpts
;
};
fBuildAttrs = fArgs // buildAttrs;
fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ];
bazelCmd = { cmd, additionalFlags, targets, targetRunFlags ? [ ] }:
lib.optionalString (targets != [ ]) ''
# See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]
BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
USER=homeless-shelter \
bazel \
--batch \
--output_base="$bazelOut" \
--output_user_root="$bazelUserRoot" \
${cmd} \
--curses=no \
"''${copts[@]}" \
"''${host_copts[@]}" \
"''${linkopts[@]}" \
"''${host_linkopts[@]}" \
$bazelFlags \
${lib.strings.concatStringsSep " " additionalFlags} \
${lib.strings.concatStringsSep " " targets} \
${lib.optionalString (targetRunFlags != []) " -- " + lib.strings.concatStringsSep " " targetRunFlags}
'';
# we need this to chmod dangling symlinks on darwin, gnu coreutils refuses to do so:
# chmod: cannot operate on dangling symlink '$symlink'
chmodder = writeCBin "chmodder" ''
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <string.h>
int main(int argc, char** argv) {
mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO;
if (argc != 2) {
fprintf(stderr, "usage: chmodder file");
exit(EXIT_FAILURE);
}
if (lchmod(argv[1], mode) != 0) {
fprintf(stderr, "failed to lchmod '%s': %s", argv[0], strerror(errno));
exit(EXIT_FAILURE);
}
}
'';
in
stdenv.mkDerivation (fBuildAttrs // {
deps = stdenv.mkDerivation (fFetchAttrs // {
name = "${name}-deps.tar.gz";
impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ fFetchAttrs.impureEnvVars or [];
nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ];
preHook = fFetchAttrs.preHook or "" + ''
export bazelOut="$(echo ''${NIX_BUILD_TOP}/output | sed -e 's,//,/,g')"
export bazelUserRoot="$(echo ''${NIX_BUILD_TOP}/tmp | sed -e 's,//,/,g')"
export HOME="$NIX_BUILD_TOP"
export USER="nix"
# This is needed for git_repository with https remotes
export GIT_SSL_CAINFO="${cacert}/etc/ssl/certs/ca-bundle.crt"
# This is needed for Bazel fetchers that are themselves programs (e.g.
# rules_go using the go toolchain)
export SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt"
'';
buildPhase = fFetchAttrs.buildPhase or ''
runHook preBuild
${
bazelCmd {
cmd = if fetchConfigured then "build --nobuild" else "fetch";
additionalFlags = [
# We disable multithreading for the fetching phase since it can lead to timeouts with many dependencies/threads:
# https://github.com/bazelbuild/bazel/issues/6502
"--loading_phase_threads=1"
"$bazelFetchFlags"
] ++ (if fetchConfigured then ["--jobs" "$NIX_BUILD_CORES"] else []);
targets = fFetchAttrs.bazelTargets ++ fFetchAttrs.bazelTestTargets;
}
}
runHook postBuild
'';
installPhase = fFetchAttrs.installPhase or (''
runHook preInstall
# Remove all built in external workspaces, Bazel will recreate them when building
rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
${lib.optionalString removeRulesCC "rm -rf $bazelOut/external/{rules_cc,\\@rules_cc.marker}"}
rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
${lib.optionalString removeLocalConfigCc "rm -rf $bazelOut/external/{local_config_cc,\\@local_config_cc.marker}"}
${lib.optionalString removeLocal "rm -rf $bazelOut/external/{local_*,\\@local_*.marker}"}
# Clear markers
find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
# Remove all vcs files
rm -rf $(find $bazelOut/external -type d -name .git)
rm -rf $(find $bazelOut/external -type d -name .svn)
rm -rf $(find $bazelOut/external -type d -name .hg)
# Removing top-level symlinks along with their markers.
# This is needed because they sometimes point to temporary paths (?).
# For example, in Tensorflow-gpu build:
# platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms
find $bazelOut/external -maxdepth 1 -type l | while read symlink; do
name="$(basename "$symlink")"
rm "$symlink"
test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker" || true
done
# Patching symlinks to remove build directory reference
find $bazelOut/external -type l | while read symlink; do
new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")"
rm "$symlink"
ln -sf "$new_target" "$symlink"
'' + lib.optionalString stdenv.isDarwin ''
# on linux symlink permissions cannot be modified, so we modify those on darwin to match the linux ones
${chmodder}/bin/chmodder "$symlink"
'' + ''
done
echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version
(cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/)
runHook postInstall
'');
dontFixup = true;
allowedRequisites = [];
outputHashAlgo = "sha256";
outputHash = fetchAttrs.sha256;
});
nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ];
preHook = fBuildAttrs.preHook or "" + ''
export bazelOut="$NIX_BUILD_TOP/output"
export bazelUserRoot="$NIX_BUILD_TOP/tmp"
export HOME="$NIX_BUILD_TOP"
'';
preConfigure = ''
mkdir -p "$bazelOut"
(cd $bazelOut && tar xfz $deps)
test "${bazel.name}" = "$(<$bazelOut/external/.nix-bazel-version)" || {
echo "fixed output derivation was built for a different bazel version" >&2
echo " got: $(<$bazelOut/external/.nix-bazel-version)" >&2
echo "expected: ${bazel.name}" >&2
exit 1
}
chmod -R +w $bazelOut
find $bazelOut -type l | while read symlink; do
if [[ $(readlink "$symlink") == *NIX_BUILD_TOP* ]]; then
ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink"
fi
done
'' + fBuildAttrs.preConfigure or "";
buildPhase = fBuildAttrs.buildPhase or ''
runHook preBuild
# Bazel sandboxes the execution of the tools it invokes, so even though we are
# calling the correct nix wrappers, the values of the environment variables
# the wrappers are expecting will not be set. So instead of relying on the
# wrappers picking them up, pass them in explicitly via `--copt`, `--linkopt`
# and related flags.
copts=()
host_copts=()
linkopts=()
host_linkopts=()
if [ -z "''${dontAddBazelOpts:-}" ]; then
for flag in $NIX_CFLAGS_COMPILE; do
copts+=( "--copt=$flag" )
host_copts+=( "--host_copt=$flag" )
done
for flag in $NIX_CXXSTDLIB_COMPILE; do
copts+=( "--copt=$flag" )
host_copts+=( "--host_copt=$flag" )
done
for flag in $NIX_LDFLAGS; do
linkopts+=( "--linkopt=-Wl,$flag" )
host_linkopts+=( "--host_linkopt=-Wl,$flag" )
done
fi
${
bazelCmd {
cmd = "test";
additionalFlags =
["--test_output=errors"] ++ fBuildAttrs.bazelTestFlags ++ ["--jobs" "$NIX_BUILD_CORES"];
targets = fBuildAttrs.bazelTestTargets;
}
}
${
bazelCmd {
cmd = "build";
additionalFlags = fBuildAttrs.bazelBuildFlags ++ ["--jobs" "$NIX_BUILD_CORES"];
targets = fBuildAttrs.bazelTargets;
}
}
${
bazelCmd {
cmd = "run";
additionalFlags = fBuildAttrs.bazelRunFlags ++ [ "--jobs" "$NIX_BUILD_CORES" ];
# Bazel run only accepts a single target, but `bazelCmd` expects `targets` to be a list.
targets = lib.optionals (fBuildAttrs.bazelRunTarget != null) [ fBuildAttrs.bazelRunTarget ];
targetRunFlags = fBuildAttrs.runTargetFlags;
}
}
runHook postBuild
'';
})
# [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]:
# Bazel computes the default value of output_user_root before parsing the
# flag. The computation of the default value involves getting the $USER
# from the environment. Code here :
# https://github.com/bazelbuild/bazel/blob/9323c57607d37f9c949b60e293b573584906da46/src/main/cpp/startup_options.cc#L123-L124
#
# On macOS Bazel will use the system installed Xcode or CLT toolchain instead of the one in the PATH unless we pass BAZEL_USE_CPP_ONLY_TOOLCHAIN.