core/pkgs/by-name/py/python/mk-python-derivation.nix
2024-05-13 11:34:52 -04:00

350 lines
11 KiB
Nix

# Generic builder.
{ lib
, config
, python
, wrapPython
, unzip
, ensureNewerSourcesForZipFilesHook
# Whether the derivation provides a Python module or not.
, toPythonModule
, namePrefix
, update-python-libraries
, setuptools
, pypaBuildHook
, pypaInstallHook
, pythonCatchConflictsHook
, pythonImportsCheckHook
, pythonNamespacesHook
, pythonOutputDistHook
, pythonRemoveBinBytecodeHook
, pythonRemoveTestsDirHook
, pythonRuntimeDepsCheckHook
, setuptoolsBuildHook
, setuptoolsCheckHook
, wheelUnpackHook
, eggUnpackHook
, eggBuildHook
, eggInstallHook
}:
let
inherit (builtins) unsafeGetAttrPos;
inherit (lib)
elem optionalString max stringLength fixedWidthString getName
optional optionals optionalAttrs hasSuffix escapeShellArgs
extendDerivation head splitString isBool;
leftPadName = name: against: let
len = max (stringLength name) (stringLength against);
in fixedWidthString len " " name;
isPythonModule = drv:
# all pythonModules have the pythonModule attribute
(drv ? "pythonModule")
# Some pythonModules are turned in to a pythonApplication by setting the field to false
&& (!isBool drv.pythonModule);
isMismatchedPython = drv: drv.pythonModule != python;
withDistOutput' = lib.flip elem ["pyproject" "setuptools" "wheel"];
isBootstrapInstallPackage' = lib.flip elem [ "flit-core" "installer" ];
isBootstrapPackage' = lib.flip elem ([
"build" "packaging" "pyproject-hooks" "wheel"
] ++ optionals (python.pythonOlder "3.11") [
"tomli"
]);
isSetuptoolsDependency' = lib.flip elem [ "setuptools" "wheel" ];
cleanAttrs = lib.flip removeAttrs [
"disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format"
"disabledTestPaths" "outputs" "stdenv"
"dependencies" "optional-dependencies" "build-system"
];
in
{ name ? "${attrs.pname}-${attrs.version}"
# Build-time dependencies for the package
, nativeBuildInputs ? []
# Run-time dependencies for the package
, buildInputs ? []
# Dependencies needed for running the checkPhase.
# These are added to buildInputs when doCheck = true.
, checkInputs ? []
, nativeCheckInputs ? []
# propagate build dependencies so in case we have A -> B -> C,
# C can import package A propagated by B
, propagatedBuildInputs ? []
# Python module dependencies.
# These are named after PEP-621.
, dependencies ? []
, optional-dependencies ? {}
# Python PEP-517 build systems.
, build-system ? []
# DEPRECATED: use propagatedBuildInputs
, pythonPath ? []
# Enabled to detect some (native)BuildInputs mistakes
, strictDeps ? true
, outputs ? [ "out" ]
# used to disable derivation, useful for specific python versions
, disabled ? false
# Raise an error if two packages are installed with the same name
# TODO: For cross we probably need a different PYTHONPATH, or not
# add the runtime deps until after buildPhase.
, catchConflicts ? (python.stdenv.hostPlatform == python.stdenv.buildPlatform)
# Additional arguments to pass to the makeWrapper function, which wraps
# generated binaries.
, makeWrapperArgs ? []
# Skip wrapping of python programs altogether
, dontWrapPythonPrograms ? false
# Don't use Pip to install a wheel
# Note this is actually a variable for the pipInstallPhase in pip's setupHook.
# It's included here to prevent an infinite recursion.
, dontUsePipInstall ? false
# Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
, permitUserSite ? false
# Remove bytecode from bin folder.
# When a Python script has the extension `.py`, bytecode is generated
# Typically, executables in bin have no extension, so no bytecode is generated.
# However, some packages do provide executables with extensions, and thus bytecode is generated.
, removeBinBytecode ? true
# pyproject = true <-> format = "pyproject"
# pyproject = false <-> format = "other"
# https://github.com/NixOS/nixpkgs/issues/253154
, pyproject ? null
# Several package formats are supported.
# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
# "wheel" : Install from a pre-compiled wheel.
# "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel.
# "egg": Install a package from an egg.
# "other" : Provide your own buildPhase and installPhase.
, format ? null
, meta ? {}
, doCheck ? config.doCheckByDefault or false
, disabledTestPaths ? []
# Allow passing in a custom stdenv to buildPython*
, stdenv ? python.stdenv
, ... } @ attrs:
assert (pyproject != null) -> (format == null);
let
format' =
if pyproject != null then
if pyproject then
"pyproject"
else
"other"
else if format != null then
format
else
"setuptools";
withDistOutput = withDistOutput' format';
validatePythonMatches = let
throwMismatch = attrName: drv: let
myName = "'${namePrefix}${name}'";
theirName = "'${drv.name}'";
optionalLocation = let
pos = unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name") attrs;
in optionalString (pos != null) " at ${pos.file}:${toString pos.line}:${toString pos.column}";
in throw ''
Python version mismatch in ${myName}:
The Python derivation ${myName} depends on a Python derivation
named ${theirName}, but the two derivations use different versions
of Python:
${leftPadName myName theirName} uses ${python}
${leftPadName theirName myName} uses ${toString drv.pythonModule}
Possible solutions:
* If ${theirName} is a Python library, change the reference to ${theirName}
in the ${attrName} of ${myName} to use a ${theirName} built from the same
version of Python
* If ${theirName} is used as a tool during the build, move the reference to
${theirName} in ${myName} from ${attrName} to nativeBuildInputs
* If ${theirName} provides executables that are called at run time, pass its
bin path to makeWrapperArgs:
makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${getName drv } ] }" ];
${optionalLocation}
'';
checkDrv = attrName: drv:
if (isPythonModule drv) && (isMismatchedPython drv) then throwMismatch attrName drv
else drv;
in attrName: inputs: map (checkDrv attrName) inputs;
isBootstrapInstallPackage = isBootstrapInstallPackage' (attrs.pname or null);
isBootstrapPackage = isBootstrapInstallPackage || isBootstrapPackage' (attrs.pname or null);
isSetuptoolsDependency = isSetuptoolsDependency' (attrs.pname or null);
passthru =
attrs.passthru or { }
// {
updateScript = let
filename = head (splitString ":" self.meta.position);
in attrs.passthru.updateScript or [ update-python-libraries filename ];
}
// optionalAttrs (dependencies != []) {
inherit dependencies;
}
// optionalAttrs (optional-dependencies != {}) {
inherit optional-dependencies;
}
// optionalAttrs (build-system != []) {
inherit build-system;
};
# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
self = toPythonModule (stdenv.mkDerivation ((cleanAttrs attrs) // {
name = namePrefix + name;
nativeBuildInputs = [
python
wrapPython
ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)?
pythonRemoveTestsDirHook
] ++ optionals (catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [
#
# 1. When building a package that is also part of the bootstrap chain, we
# must ignore conflicts after installation, because there will be one with
# the package in the bootstrap.
#
# 2. When a package is a dependency of setuptools, we must ignore conflicts
# because the hook that checks for conflicts uses setuptools.
#
pythonCatchConflictsHook
] ++ optionals removeBinBytecode [
pythonRemoveBinBytecodeHook
] ++ optionals (hasSuffix "zip" (attrs.src.name or "")) [
unzip
] ++ optionals (format' == "setuptools") [
setuptoolsBuildHook
] ++ optionals (format' == "pyproject") [(
if isBootstrapPackage then
pypaBuildHook.override {
inherit (python.pythonOnBuildForHost.pkgs.bootstrap) build;
wheel = null;
}
else
pypaBuildHook
) (
if isBootstrapPackage then
pythonRuntimeDepsCheckHook.override {
inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging;
}
else
pythonRuntimeDepsCheckHook
)] ++ optionals (format' == "wheel") [
wheelUnpackHook
] ++ optionals (format' == "egg") [
eggUnpackHook eggBuildHook eggInstallHook
] ++ optionals (format' != "other") [(
if isBootstrapInstallPackage then
pypaInstallHook.override {
inherit (python.pythonOnBuildForHost.pkgs.bootstrap) installer;
}
else
pypaInstallHook
)] ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
# This is a test, however, it should be ran independent of the checkPhase and checkInputs
pythonImportsCheckHook
] ++ optionals (python.pythonAtLeast "3.3") [
# Optionally enforce PEP420 for python3
pythonNamespacesHook
] ++ optionals withDistOutput [
pythonOutputDistHook
] ++ nativeBuildInputs ++ build-system;
buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);
propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ dependencies ++ [
# we propagate python even for packages transformed with 'toPythonApplication'
# this pollutes the PATH but avoids rebuilds
# see https://github.com/NixOS/nixpkgs/issues/170887 for more context
python
]);
inherit strictDeps;
LANG = "${if python.stdenv.isDarwin then "en_US" else "C"}.UTF-8";
# Python packages don't have a checkPhase, only an installCheckPhase
doCheck = false;
doInstallCheck = attrs.doCheck or true;
nativeInstallCheckInputs = [
] ++ optionals (format' == "setuptools") [
# Longer-term we should get rid of this and require
# users of this function to set the `installCheckPhase` or
# pass in a hook that sets it.
setuptoolsCheckHook
] ++ nativeCheckInputs;
installCheckInputs = checkInputs;
postFixup = optionalString (!dontWrapPythonPrograms) ''
wrapPythonPrograms
'' + attrs.postFixup or "";
# Python packages built through cross-compilation are always for the host platform.
disallowedReferences = optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonOnBuildForHost ];
outputs = outputs ++ optional withDistOutput "dist";
inherit passthru;
meta = {
# default to python's platforms
platforms = python.meta.platforms;
isBuildPythonPackage = python.meta.platforms;
} // meta;
} // optionalAttrs (attrs?checkPhase) {
# If given use the specified checkPhase, otherwise use the setup hook.
# Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
installCheckPhase = attrs.checkPhase;
} // optionalAttrs (disabledTestPaths != []) {
disabledTestPaths = escapeShellArgs disabledTestPaths;
}));
in extendDerivation
(disabled -> throw "${name} not supported for interpreter ${python.executable}")
passthru
self