core/pkgs/by-name/py/python/mk-python-derivation.nix

# Generic builder.

{ lib, config, python, wrapPython, unzip, ensureNewerSourcesForZipFilesHook
# Whether the derivation provides a Python module or not.
, toPythonModule, namePrefix, update-python-libraries, setuptools, pypaBuildHook
, pypaInstallHook, pythonCatchConflictsHook, pythonImportsCheckHook
, pythonNamespacesHook, pythonOutputDistHook, pythonRemoveBinBytecodeHook
, pythonRemoveTestsDirHook, pythonRuntimeDepsCheckHook, setuptoolsBuildHook
, setuptoolsCheckHook, wheelUnpackHook, eggUnpackHook, eggBuildHook
, eggInstallHook }:

let
  inherit (builtins) unsafeGetAttrPos;
  inherit (lib)
    elem optionalString max stringLength fixedWidthString getName optional
    optionals optionalAttrs hasSuffix escapeShellArgs extendDerivation head
    splitString isBool;

  leftPadName = name: against:
    let len = max (stringLength name) (stringLength against);
    in fixedWidthString len " " name;

  isPythonModule = drv:
    # all pythonModules have the pythonModule attribute
    (drv ? "pythonModule")
    # Some pythonModules are turned in to a pythonApplication by setting the field to false
    && (!isBool drv.pythonModule);

  isMismatchedPython = drv: drv.pythonModule != python;

  withDistOutput' = lib.flip elem [ "pyproject" "setuptools" "wheel" ];

  isBootstrapInstallPackage' = lib.flip elem [ "flit-core" "installer" ];

  isBootstrapPackage' = lib.flip elem
    ([ "build" "packaging" "pyproject-hooks" "wheel" ]
      ++ optionals (python.pythonOlder "3.11") [ "tomli" ]);

  isSetuptoolsDependency' = lib.flip elem [ "setuptools" "wheel" ];

  cleanAttrs = lib.flip removeAttrs [
    "disabled"
    "checkPhase"
    "checkInputs"
    "nativeCheckInputs"
    "doCheck"
    "doInstallCheck"
    "dontWrapPythonPrograms"
    "catchConflicts"
    "pyproject"
    "format"
    "disabledTestPaths"
    "outputs"
    "stdenv"
    "dependencies"
    "optional-dependencies"
    "build-system"
  ];

in { name ? "${attrs.pname}-${attrs.version}"

  # Build-time dependencies for the package
, nativeBuildInputs ? [ ]

  # Run-time dependencies for the package
, buildInputs ? [ ]

  # Dependencies needed for running the checkPhase.
  # These are added to buildInputs when doCheck = true.
, checkInputs ? [ ], nativeCheckInputs ? [ ]

  # propagate build dependencies so in case we have A -> B -> C,
  # C can import package A propagated by B
, propagatedBuildInputs ? [ ]

  # Python module dependencies.
  # These are named after PEP-621.
, dependencies ? [ ], optional-dependencies ? { }

  # Python PEP-517 build systems.
, build-system ? [ ]

  # DEPRECATED: use propagatedBuildInputs
, pythonPath ? [ ]

  # Enabled to detect some (native)BuildInputs mistakes
, strictDeps ? true

, outputs ? [
  "out"
]

# used to disable derivation, useful for specific python versions
, disabled ? false

  # Raise an error if two packages are installed with the same name
  # TODO: For cross we probably need a different PYTHONPATH, or not
  # add the runtime deps until after buildPhase.
, catchConflicts ? (python.stdenv.hostPlatform == python.stdenv.buildPlatform)

# Additional arguments to pass to the makeWrapper function, which wraps
# generated binaries.
, makeWrapperArgs ? [ ]

  # Skip wrapping of python programs altogether
, dontWrapPythonPrograms ? false

  # Don't use Pip to install a wheel
  # Note this is actually a variable for the pipInstallPhase in pip's setupHook.
  # It's included here to prevent an infinite recursion.
, dontUsePipInstall ? false

  # Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
, permitUserSite ? false

  # Remove bytecode from bin folder.
  # When a Python script has the extension `.py`, bytecode is generated
  # Typically, executables in bin have no extension, so no bytecode is generated.
  # However, some packages do provide executables with extensions, and thus bytecode is generated.
, removeBinBytecode ? true

  # pyproject = true <-> format = "pyproject"
  # pyproject = false <-> format = "other"
  # https://github.com/NixOS/nixpkgs/issues/253154
, pyproject ? null

  # Several package formats are supported.
  # "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
  # "wheel" : Install from a pre-compiled wheel.
  # "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel.
  # "egg": Install a package from an egg.
  # "other" : Provide your own buildPhase and installPhase.
, format ? null

, meta ? { }

, doCheck ? config.doCheckByDefault or false

, disabledTestPaths ? [ ]

  # Allow passing in a custom stdenv to buildPython*
, stdenv ? python.stdenv

, ... }@attrs:

assert (pyproject != null) -> (format == null);

let
  format' = if pyproject != null then
    if pyproject then "pyproject" else "other"
  else if format != null then
    format
  else
    "setuptools";

  withDistOutput = withDistOutput' format';

  validatePythonMatches = let
    throwMismatch = attrName: drv:
      let
        myName = "'${namePrefix}${name}'";
        theirName = "'${drv.name}'";
        optionalLocation = let
          pos = unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name")
            attrs;
        in optionalString (pos != null)
        " at ${pos.file}:${toString pos.line}:${toString pos.column}";
      in throw ''
        Python version mismatch in ${myName}:

        The Python derivation ${myName} depends on a Python derivation
        named ${theirName}, but the two derivations use different versions
        of Python:

            ${leftPadName myName theirName} uses ${python}
            ${leftPadName theirName myName} uses ${toString drv.pythonModule}

        Possible solutions:

          * If ${theirName} is a Python library, change the reference to ${theirName}
            in the ${attrName} of ${myName} to use a ${theirName} built from the same
            version of Python

          * If ${theirName} is used as a tool during the build, move the reference to
            ${theirName} in ${myName} from ${attrName} to nativeBuildInputs

          * If ${theirName} provides executables that are called at run time, pass its
            bin path to makeWrapperArgs:

                makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${
                  getName drv
                } ] }" ];

        ${optionalLocation}
      '';

    checkDrv = attrName: drv:
      if (isPythonModule drv) && (isMismatchedPython drv) then
        throwMismatch attrName drv
      else
        drv;

  in attrName: inputs: map (checkDrv attrName) inputs;

  isBootstrapInstallPackage = isBootstrapInstallPackage' (attrs.pname or null);

  isBootstrapPackage = isBootstrapInstallPackage
    || isBootstrapPackage' (attrs.pname or null);

  isSetuptoolsDependency = isSetuptoolsDependency' (attrs.pname or null);

  passthru = attrs.passthru or { } // {
    updateScript = let filename = head (splitString ":" self.meta.position);
    in attrs.passthru.updateScript or [ update-python-libraries filename ];
  } // optionalAttrs (dependencies != [ ]) { inherit dependencies; }
    // optionalAttrs (optional-dependencies != { }) {
      inherit optional-dependencies;
    } // optionalAttrs (build-system != [ ]) { inherit build-system; };

  # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
  self = toPythonModule (stdenv.mkDerivation ((cleanAttrs attrs) // {

    name = namePrefix + name;

    nativeBuildInputs = [
      python
      wrapPython
      ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)?
      pythonRemoveTestsDirHook
    ] ++ optionals
      (catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [
        #
        # 1. When building a package that is also part of the bootstrap chain, we
        #    must ignore conflicts after installation, because there will be one with
        #    the package in the bootstrap.
        #
        # 2. When a package is a dependency of setuptools, we must ignore conflicts
        #    because the hook that checks for conflicts uses setuptools.
        #
        pythonCatchConflictsHook
      ] ++ optionals removeBinBytecode [ pythonRemoveBinBytecodeHook ]
      ++ optionals (hasSuffix "zip" (attrs.src.name or "")) [ unzip ]
      ++ optionals (format' == "setuptools") [ setuptoolsBuildHook ]
      ++ optionals (format' == "pyproject") [
        (if isBootstrapPackage then
          pypaBuildHook.override {
            inherit (python.pythonOnBuildForHost.pkgs.bootstrap) build;
            wheel = null;
          }
        else
          pypaBuildHook)
        (if isBootstrapPackage then
          pythonRuntimeDepsCheckHook.override {
            inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging;
          }
        else
          pythonRuntimeDepsCheckHook)
      ] ++ optionals (format' == "wheel") [ wheelUnpackHook ]
      ++ optionals (format' == "egg") [
        eggUnpackHook
        eggBuildHook
        eggInstallHook
      ] ++ optionals (format' != "other") [
        (if isBootstrapInstallPackage then
          pypaInstallHook.override {
            inherit (python.pythonOnBuildForHost.pkgs.bootstrap) installer;
          }
        else
          pypaInstallHook)
      ] ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
        # This is a test, however, it should be ran independent of the checkPhase and checkInputs
        pythonImportsCheckHook
      ] ++ optionals (python.pythonAtLeast "3.3") [
        # Optionally enforce PEP420 for python3
        pythonNamespacesHook
      ] ++ optionals withDistOutput [ pythonOutputDistHook ]
      ++ nativeBuildInputs ++ build-system;

    buildInputs =
      validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);

    propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs"
      (propagatedBuildInputs ++ dependencies ++ [
        # we propagate python even for packages transformed with 'toPythonApplication'
        # this pollutes the PATH but avoids rebuilds
        # see https://github.com/NixOS/nixpkgs/issues/170887 for more context
        python
      ]);

    inherit strictDeps;

    LANG = "${if python.stdenv.isDarwin then "en_US" else "C"}.UTF-8";

    # Python packages don't have a checkPhase, only an installCheckPhase
    doCheck = false;
    doInstallCheck = attrs.doCheck or true;
    nativeInstallCheckInputs = [ ] ++ optionals (format' == "setuptools") [
      # Longer-term we should get rid of this and require
      # users of this function to set the `installCheckPhase` or
      # pass in a hook that sets it.
      setuptoolsCheckHook
    ] ++ nativeCheckInputs;
    installCheckInputs = checkInputs;

    postFixup = optionalString (!dontWrapPythonPrograms) ''
      wrapPythonPrograms
    '' + attrs.postFixup or "";

    # Python packages built through cross-compilation are always for the host platform.
    disallowedReferences =
      optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform)
      [ python.pythonOnBuildForHost ];

    outputs = outputs ++ optional withDistOutput "dist";

    inherit passthru;

    meta = {
      # default to python's platforms
      platforms = python.meta.platforms;
      isBuildPythonPackage = python.meta.platforms;
    } // meta;
  } // optionalAttrs (attrs ? checkPhase) {
    # If given use the specified checkPhase, otherwise use the setup hook.
    # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
    installCheckPhase = attrs.checkPhase;
  } // optionalAttrs (disabledTestPaths != [ ]) {
    disabledTestPaths = escapeShellArgs disabledTestPaths;
  }));

in extendDerivation
(disabled -> throw "${name} not supported for interpreter ${python.executable}")
passthru self