From b9138b7c072f55dcfd7b8c0747c3c4291bd0b28e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 2 Dec 2023 17:11:10 +1300 Subject: [PATCH 1/4] mk-python-derivation: Add dependencies & optional-dependencies arguments Since https://github.com/NixOS/nixpkgs/pull/161835 we've had the concept of `passthru.optional-dependencies` for Python optional deps. Having to explicitly put optional-dependencies in the passthru attrset is a bit strange API-wise, even though it semantically makes sense. This change unifies the handling of non-optional & optional Python dependencies using the names established from PEP-621 (standardized pyproject.toml project metadata). --- doc/languages-frameworks/python.section.md | 59 +++++++++++-------- .../python/mk-python-derivation.nix | 40 +++++++++---- 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 0849aacdf166..9250961f581c 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -120,7 +120,7 @@ buildPythonPackage rec { setuptools-scm ]; - propagatedBuildInputs = [ + dependencies = [ attrs py setuptools @@ -214,9 +214,14 @@ because their behaviour is different: * `nativeCheckInputs ? []`: Dependencies needed for running the [`checkPhase`](#ssec-check-phase). These are added to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs) when [`doCheck = true`](#var-stdenv-doCheck). Items listed in `tests_require` go here. -* `propagatedBuildInputs ? []`: Aside from propagating dependencies, +* `dependencies ? []`: Aside from propagating dependencies, `buildPythonPackage` also injects code into and wraps executables with the paths included in this list. Items listed in `install_requires` go here. +* `optional-dependencies ? { }`: Optional feature flagged dependencies. Items listed in `extras_requires` go here. + +Aside from propagating dependencies, + `buildPythonPackage` also injects code into and wraps executables with the + paths included in this list. Items listed in `extras_requires` go here. ##### Overriding Python packages {#overriding-python-packages} @@ -303,9 +308,9 @@ python3Packages.buildPythonApplication rec { setuptools ]; - propagatedBuildInputs = with python3Packages; [ - tornado - python-daemon + dependencies = [ + python3Packages.tornado + python3Packages.python-daemon ]; meta = with lib; { @@ -977,13 +982,15 @@ that we introduced with the `let` expression. #### Handling dependencies {#handling-dependencies} -Our example, `toolz`, does not have any dependencies on other Python packages or -system libraries. According to the manual, [`buildPythonPackage`](#buildpythonpackage-function) uses the -arguments [`buildInputs`](#var-stdenv-buildInputs) and [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs) to specify dependencies. If -something is exclusively a build-time dependency, then the dependency should be -included in [`buildInputs`](#var-stdenv-buildInputs), but if it is (also) a runtime dependency, then it -should be added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). Test dependencies are considered -build-time dependencies and passed to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs). +Our example, `toolz`, does not have any dependencies on other Python packages or system libraries. +[`buildPythonPackage`](#buildpythonpackage-function) uses the the following arguments in the following circumstances: + +- `dependencies` - For Python runtime dependencies. +- `build-system` - For Python build-time requirements. +- [`buildInputs`](#var-stdenv-buildInputs) - For non-Python build-time requirements. +- [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) - For test dependencies + +Dependencies can belong to multiple arguments, for example if something is both a build time requirement & a runtime dependency. The following example shows which arguments are given to [`buildPythonPackage`](#buildpythonpackage-function) in order to build [`datashape`](https://github.com/blaze/datashape). @@ -1018,7 +1025,7 @@ buildPythonPackage rec { wheel ]; - propagatedBuildInputs = [ + dependencies = [ multipledispatch numpy python-dateutil @@ -1041,7 +1048,7 @@ buildPythonPackage rec { We can see several runtime dependencies, `numpy`, `multipledispatch`, and `python-dateutil`. Furthermore, we have [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) with `pytest`. `pytest` is a test runner and is only used during the [`checkPhase`](#ssec-check-phase) and is -therefore not added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). +therefore not added to `dependencies`. In the previous case we had only dependencies on other Python packages to consider. Occasionally you have also system libraries to consider. E.g., `lxml` provides @@ -1136,7 +1143,7 @@ buildPythonPackage rec { fftwLongDouble ]; - propagatedBuildInputs = [ + dependencies = [ numpy scipy ]; @@ -1459,9 +1466,7 @@ mode is activated. In the following example, we create a simple environment that has a Python 3.11 version of our package in it, as well as its dependencies and other packages we -like to have in the environment, all specified with [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). -Indeed, we can just add any package we like to have in our environment to -[`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). +like to have in the environment, all specified with `dependencies`. ```nix with import {}; @@ -1470,9 +1475,11 @@ with python311Packages; buildPythonPackage rec { name = "mypackage"; src = ./path/to/package/source; - propagatedBuildInputs = [ + dependencies = [ pytest numpy + ]; + propagatedBuildInputs = [ pkgs.libsndfile ]; } @@ -1903,8 +1910,8 @@ configure alternatives](#sec-overlays-alternatives-blas-lapack)". In a `setup.py` or `setup.cfg` it is common to declare dependencies: -* `setup_requires` corresponds to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs) -* `install_requires` corresponds to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs) +* `setup_requires` corresponds to `build-system` +* `install_requires` corresponds to `dependencies` * `tests_require` corresponds to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) ### How to enable interpreter optimizations? {#optimizations} @@ -1928,12 +1935,10 @@ in mypython Some packages define optional dependencies for additional features. With `setuptools` this is called `extras_require` and `flit` calls it -`extras-require`, while PEP 621 calls these `optional-dependencies`. A -method for supporting this is by declaring the extras of a package in its -`passthru`, e.g. in case of the package `dask` +`extras-require`, while PEP 621 calls these `optional-dependencies`. ```nix -passthru.optional-dependencies = { +optional-dependencies = { complete = [ distributed ]; }; ``` @@ -1941,11 +1946,13 @@ passthru.optional-dependencies = { and letting the package requiring the extra add the list to its dependencies ```nix -propagatedBuildInputs = [ +dependencies = [ ... ] ++ dask.optional-dependencies.complete; ``` +This method is using `passthru`, meaning that changing `optional-dependencies` of a package won't cause it to rebuild. + Note this method is preferred over adding parameters to builders, as that can result in packages depending on different variants and thereby causing collisions. diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 6944f70a4918..d06c2e3099ae 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -45,6 +45,11 @@ # C can import package A propagated by B , propagatedBuildInputs ? [] +# Python module dependencies. +# These are named after PEP-621. +, dependencies ? [] +, optional-dependencies ? {} + # DEPRECATED: use propagatedBuildInputs , pythonPath ? [] @@ -97,8 +102,6 @@ , meta ? {} -, passthru ? {} - , doCheck ? config.doCheckByDefault or false , disabledTestPaths ? [] @@ -193,10 +196,25 @@ let "setuptools" "wheel" ]; + passthru = + attrs.passthru or { } + // { + updateScript = let + filename = builtins.head (lib.splitString ":" self.meta.position); + in attrs.passthru.updateScript or [ update-python-libraries filename ]; + } + // lib.optionalAttrs (dependencies != []) { + inherit dependencies; + } + // lib.optionalAttrs (optional-dependencies != {}) { + inherit optional-dependencies; + }; + # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [ "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format" "disabledTestPaths" "outputs" "stdenv" + "dependencies" "optional-dependencies" ]) // { name = namePrefix + name_; @@ -260,7 +278,7 @@ let buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath); - propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ [ + propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ dependencies ++ [ # we propagate python even for packages transformed with 'toPythonApplication' # this pollutes the PATH but avoids rebuilds # see https://github.com/NixOS/nixpkgs/issues/170887 for more context @@ -292,6 +310,8 @@ let outputs = outputs ++ lib.optional withDistOutput "dist"; + inherit passthru; + meta = { # default to python's platforms platforms = python.meta.platforms; @@ -305,13 +325,7 @@ let disabledTestPaths = lib.escapeShellArgs disabledTestPaths; })); - passthru.updateScript = let - filename = builtins.head (lib.splitString ":" self.meta.position); - in attrs.passthru.updateScript or [ update-python-libraries filename ]; -in - if disabled then - throw "${name} not supported for interpreter ${python.executable}" -else - self.overrideAttrs (attrs: { - passthru = lib.recursiveUpdate passthru attrs.passthru; - }) +in lib.extendDerivation + (disabled -> throw "${name} not supported for interpreter ${python.executable}") + passthru + self From 4d0cca465476a1142c187198c9fe3fcd89aec6d1 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 2 Dec 2023 17:47:20 +1300 Subject: [PATCH 2/4] mk-python-derivation: Add build-system argument Much like the previous commit that adds dependencies & optional-dependencies this aligns PEP-517 build systems with how they are defined in PEP-518/PEP-621. The naming `build-system` (singular) is aligned with upstream Python standards. --- doc/languages-frameworks/python.section.md | 33 ++++++++++--------- .../python/mk-python-derivation.nix | 10 ++++-- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 9250961f581c..da06fe1d69b5 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -116,7 +116,7 @@ buildPythonPackage rec { rm testing/test_argcomplete.py ''; - nativeBuildInputs = [ + build-system = [ setuptools-scm ]; @@ -172,7 +172,7 @@ following are specific to `buildPythonPackage`: variable in wrapped programs. * `pyproject`: Whether the pyproject format should be used. When set to `true`, `pypaBuildHook` will be used, and you can add the required build dependencies - from `build-system.requires` to `nativeBuildInputs`. Note that the pyproject + from `build-system.requires` to `build-system`. Note that the pyproject format falls back to using `setuptools`, so you can use `pyproject = true` even if the package only has a `setup.py`. When set to `false`, you can use the existing [hooks](#setup-hooks0 or provide your own logic to build the @@ -206,8 +206,8 @@ build inputs (see "Specifying dependencies"). The following are of special interest for Python packages, either because these are primarily used, or because their behaviour is different: -* `nativeBuildInputs ? []`: Build-time only dependencies. Typically executables - as well as the items listed in `setup_requires`. +* `nativeBuildInputs ? []`: Build-time only dependencies. Typically executables. +* `build-system ? []`: Build-time only Python dependencies. Items listed in `build-system.requires`/`setup_requires`. * `buildInputs ? []`: Build and/or run-time dependencies that need to be compiled for the host machine. Typically non-Python libraries which are being linked. @@ -304,13 +304,14 @@ python3Packages.buildPythonApplication rec { hash = "sha256-Pe229rT0aHwA98s+nTHQMEFKZPo/yw6sot8MivFDvAw="; }; - nativeBuildInputs = with python3Packages; [ + build-system = with python3Packages; [ setuptools + wheel ]; - dependencies = [ - python3Packages.tornado - python3Packages.python-daemon + dependencies = with python3Packages; [ + tornado + python-daemon ]; meta = with lib; { @@ -467,11 +468,11 @@ are used in [`buildPythonPackage`](#buildpythonpackage-function). - `eggBuildHook` to skip building for eggs. - `eggInstallHook` to install eggs. - `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system - (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`. + (e.g. `setuptools` or `flit`) should still be added as `build-system`. - `pypaBuildHook` to build a wheel using [`pypa/build`](https://pypa-build.readthedocs.io/en/latest/index.html) and PEP 517/518. Note a build system (e.g. `setuptools` or `flit`) should still - be added as `nativeBuildInput`. + be added as `build-system`. - `pipInstallHook` to install wheels. - `pytestCheckHook` to run tests with `pytest`. See [example usage](#using-pytestcheckhook). - `pythonCatchConflictsHook` to check whether a Python package is not already existing. @@ -886,7 +887,7 @@ buildPythonPackage rec { hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA="; }; - nativeBuildInputs = [ + build-system = [ setuptools wheel ]; @@ -946,7 +947,7 @@ with import {}; hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA="; }; - nativeBuildInputs = [ + build-system = [ python311.pkgs.setuptools python311.pkgs.wheel ]; @@ -1020,7 +1021,7 @@ buildPythonPackage rec { hash = "sha256-FLLvdm1MllKrgTGC6Gb0k0deZeVYvtCCLji/B7uhong="; }; - nativeBuildInputs = [ + build-system = [ setuptools wheel ]; @@ -1075,7 +1076,7 @@ buildPythonPackage rec { hash = "sha256-s9NiusRxFydHzaNRMjjxFcvWxfi45jGb9ql6eJJyQJk="; }; - nativeBuildInputs = [ + build-system = [ setuptools wheel ]; @@ -1132,7 +1133,7 @@ buildPythonPackage rec { hash = "sha256-9ru2r6kwhUCaskiFoaPNuJCfCVoUL01J40byvRt4kHQ="; }; - nativeBuildInputs = [ + build-system = [ setuptools wheel ]; @@ -1526,7 +1527,7 @@ buildPythonPackage rec { hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA="; }; - nativeBuildInputs = [ + build-system = [ setuptools wheel ]; diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index d06c2e3099ae..074ccbf1bd23 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -50,6 +50,9 @@ , dependencies ? [] , optional-dependencies ? {} +# Python PEP-517 build systems. +, build-system ? [] + # DEPRECATED: use propagatedBuildInputs , pythonPath ? [] @@ -208,13 +211,16 @@ let } // lib.optionalAttrs (optional-dependencies != {}) { inherit optional-dependencies; + } + // lib.optionalAttrs (build-system != []) { + inherit build-system; }; # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [ "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format" "disabledTestPaths" "outputs" "stdenv" - "dependencies" "optional-dependencies" + "dependencies" "optional-dependencies" "build-system" ]) // { name = namePrefix + name_; @@ -274,7 +280,7 @@ let pythonNamespacesHook ] ++ lib.optionals withDistOutput [ pythonOutputDistHook - ] ++ nativeBuildInputs; + ] ++ nativeBuildInputs ++ build-system; buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath); From 7ba7e2dd65fe4cf8a8288461162de1761a520004 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 2 Dec 2023 17:50:22 +1300 Subject: [PATCH 3/4] python3.pkgs.requests: Use dependencies & optional-dependencies params --- pkgs/development/python-modules/requests/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/requests/default.nix b/pkgs/development/python-modules/requests/default.nix index 44733b2da0af..33749a992471 100644 --- a/pkgs/development/python-modules/requests/default.nix +++ b/pkgs/development/python-modules/requests/default.nix @@ -30,7 +30,7 @@ buildPythonPackage rec { hash = "sha256-lCxadY+Y15Dq7Ropy27vx/+w0c968Fw9J5Flbb1q0eE="; }; - propagatedBuildInputs = [ + dependencies = [ brotlicffi certifi charset-normalizer @@ -38,7 +38,7 @@ buildPythonPackage rec { urllib3 ]; - passthru.optional-dependencies = { + optional-dependencies = { security = []; socks = [ pysocks @@ -53,7 +53,7 @@ buildPythonPackage rec { pytest-xdist pytestCheckHook ] - ++ passthru.optional-dependencies.socks; + ++ optional-dependencies.socks; disabledTests = [ # Disable tests that require network access and use httpbin From 07a221b41f63cd4ec023d48104b21533686c1cc3 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 2 Dec 2023 17:50:57 +1300 Subject: [PATCH 4/4] python3.pkgs.pendulum: Use build-system & dependencies params --- pkgs/development/python-modules/pendulum/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/pendulum/default.nix b/pkgs/development/python-modules/pendulum/default.nix index cdc7ab035ed3..631494a1d9a3 100644 --- a/pkgs/development/python-modules/pendulum/default.nix +++ b/pkgs/development/python-modules/pendulum/default.nix @@ -20,8 +20,8 @@ buildPythonPackage rec { export HOME=$TMPDIR ''; - nativeBuildInputs = [ poetry-core ]; - propagatedBuildInputs = [ python-dateutil pytzdata ] + build-system = [ poetry-core ]; + dependencies = [ python-dateutil pytzdata ] ++ lib.optional (pythonOlder "3.5") typing ++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ];