Merge pull request #271597 from adisbladis/python-runtime-build-time-sep
python3.pkgs.buildPythonPackage: Separate runtime & build time dependencies
This commit is contained in:
Martin Weinelt
GitHub
commit
fa83add1b9
4 changed files with 84 additions and 56 deletions
|
|
@ -116,11 +116,11 @@ buildPythonPackage rec {
|
|||
rm testing/test_argcomplete.py
|
||||
'';
|
||||
|
||||
nativeBuildInputs = [
|
||||
build-system = [
|
||||
setuptools-scm
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
dependencies = [
|
||||
attrs
|
||||
py
|
||||
setuptools
|
||||
|
|
@ -172,7 +172,7 @@ following are specific to `buildPythonPackage`:
|
|||
variable in wrapped programs.
|
||||
* `pyproject`: Whether the pyproject format should be used. When set to `true`,
|
||||
`pypaBuildHook` will be used, and you can add the required build dependencies
|
||||
from `build-system.requires` to `nativeBuildInputs`. Note that the pyproject
|
||||
from `build-system.requires` to `build-system`. Note that the pyproject
|
||||
format falls back to using `setuptools`, so you can use `pyproject = true`
|
||||
even if the package only has a `setup.py`. When set to `false`, you can
|
||||
use the existing [hooks](#setup-hooks0 or provide your own logic to build the
|
||||
|
|
@ -206,17 +206,22 @@ build inputs (see "Specifying dependencies"). The following are of special
|
|||
interest for Python packages, either because these are primarily used, or
|
||||
because their behaviour is different:
|
||||
|
||||
* `nativeBuildInputs ? []`: Build-time only dependencies. Typically executables
|
||||
as well as the items listed in `setup_requires`.
|
||||
* `nativeBuildInputs ? []`: Build-time only dependencies. Typically executables.
|
||||
* `build-system ? []`: Build-time only Python dependencies. Items listed in `build-system.requires`/`setup_requires`.
|
||||
* `buildInputs ? []`: Build and/or run-time dependencies that need to be
|
||||
compiled for the host machine. Typically non-Python libraries which are being
|
||||
linked.
|
||||
* `nativeCheckInputs ? []`: Dependencies needed for running the [`checkPhase`](#ssec-check-phase). These
|
||||
are added to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs) when [`doCheck = true`](#var-stdenv-doCheck). Items listed in
|
||||
`tests_require` go here.
|
||||
* `propagatedBuildInputs ? []`: Aside from propagating dependencies,
|
||||
* `dependencies ? []`: Aside from propagating dependencies,
|
||||
`buildPythonPackage` also injects code into and wraps executables with the
|
||||
paths included in this list. Items listed in `install_requires` go here.
|
||||
* `optional-dependencies ? { }`: Optional feature flagged dependencies. Items listed in `extras_requires` go here.
|
||||
|
||||
Aside from propagating dependencies,
|
||||
`buildPythonPackage` also injects code into and wraps executables with the
|
||||
paths included in this list. Items listed in `extras_requires` go here.
|
||||
|
||||
##### Overriding Python packages {#overriding-python-packages}
|
||||
|
||||
|
|
@ -299,11 +304,12 @@ python3Packages.buildPythonApplication rec {
|
|||
hash = "sha256-Pe229rT0aHwA98s+nTHQMEFKZPo/yw6sot8MivFDvAw=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = with python3Packages; [
|
||||
build-system = with python3Packages; [
|
||||
setuptools
|
||||
wheel
|
||||
];
|
||||
|
||||
propagatedBuildInputs = with python3Packages; [
|
||||
dependencies = with python3Packages; [
|
||||
tornado
|
||||
python-daemon
|
||||
];
|
||||
|
|
@ -462,11 +468,11 @@ are used in [`buildPythonPackage`](#buildpythonpackage-function).
|
|||
- `eggBuildHook` to skip building for eggs.
|
||||
- `eggInstallHook` to install eggs.
|
||||
- `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system
|
||||
(e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`.
|
||||
(e.g. `setuptools` or `flit`) should still be added as `build-system`.
|
||||
- `pypaBuildHook` to build a wheel using
|
||||
[`pypa/build`](https://pypa-build.readthedocs.io/en/latest/index.html) and
|
||||
PEP 517/518. Note a build system (e.g. `setuptools` or `flit`) should still
|
||||
be added as `nativeBuildInput`.
|
||||
be added as `build-system`.
|
||||
- `pipInstallHook` to install wheels.
|
||||
- `pytestCheckHook` to run tests with `pytest`. See [example usage](#using-pytestcheckhook).
|
||||
- `pythonCatchConflictsHook` to check whether a Python package is not already existing.
|
||||
|
|
@ -881,7 +887,7 @@ buildPythonPackage rec {
|
|||
hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
];
|
||||
|
|
@ -941,7 +947,7 @@ with import <nixpkgs> {};
|
|||
hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
build-system = [
|
||||
python311.pkgs.setuptools
|
||||
python311.pkgs.wheel
|
||||
];
|
||||
|
|
@ -977,13 +983,15 @@ that we introduced with the `let` expression.
|
|||
|
||||
#### Handling dependencies {#handling-dependencies}
|
||||
|
||||
Our example, `toolz`, does not have any dependencies on other Python packages or
|
||||
system libraries. According to the manual, [`buildPythonPackage`](#buildpythonpackage-function) uses the
|
||||
arguments [`buildInputs`](#var-stdenv-buildInputs) and [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs) to specify dependencies. If
|
||||
something is exclusively a build-time dependency, then the dependency should be
|
||||
included in [`buildInputs`](#var-stdenv-buildInputs), but if it is (also) a runtime dependency, then it
|
||||
should be added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). Test dependencies are considered
|
||||
build-time dependencies and passed to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs).
|
||||
Our example, `toolz`, does not have any dependencies on other Python packages or system libraries.
|
||||
[`buildPythonPackage`](#buildpythonpackage-function) uses the the following arguments in the following circumstances:
|
||||
|
||||
- `dependencies` - For Python runtime dependencies.
|
||||
- `build-system` - For Python build-time requirements.
|
||||
- [`buildInputs`](#var-stdenv-buildInputs) - For non-Python build-time requirements.
|
||||
- [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) - For test dependencies
|
||||
|
||||
Dependencies can belong to multiple arguments, for example if something is both a build time requirement & a runtime dependency.
|
||||
|
||||
The following example shows which arguments are given to [`buildPythonPackage`](#buildpythonpackage-function) in
|
||||
order to build [`datashape`](https://github.com/blaze/datashape).
|
||||
|
|
@ -1013,12 +1021,12 @@ buildPythonPackage rec {
|
|||
hash = "sha256-FLLvdm1MllKrgTGC6Gb0k0deZeVYvtCCLji/B7uhong=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
dependencies = [
|
||||
multipledispatch
|
||||
numpy
|
||||
python-dateutil
|
||||
|
|
@ -1041,7 +1049,7 @@ buildPythonPackage rec {
|
|||
We can see several runtime dependencies, `numpy`, `multipledispatch`, and
|
||||
`python-dateutil`. Furthermore, we have [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) with `pytest`.
|
||||
`pytest` is a test runner and is only used during the [`checkPhase`](#ssec-check-phase) and is
|
||||
therefore not added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs).
|
||||
therefore not added to `dependencies`.
|
||||
|
||||
In the previous case we had only dependencies on other Python packages to consider.
|
||||
Occasionally you have also system libraries to consider. E.g., `lxml` provides
|
||||
|
|
@ -1068,7 +1076,7 @@ buildPythonPackage rec {
|
|||
hash = "sha256-s9NiusRxFydHzaNRMjjxFcvWxfi45jGb9ql6eJJyQJk=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
];
|
||||
|
|
@ -1125,7 +1133,7 @@ buildPythonPackage rec {
|
|||
hash = "sha256-9ru2r6kwhUCaskiFoaPNuJCfCVoUL01J40byvRt4kHQ=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
];
|
||||
|
|
@ -1136,7 +1144,7 @@ buildPythonPackage rec {
|
|||
fftwLongDouble
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
dependencies = [
|
||||
numpy
|
||||
scipy
|
||||
];
|
||||
|
|
@ -1459,9 +1467,7 @@ mode is activated.
|
|||
|
||||
In the following example, we create a simple environment that has a Python 3.11
|
||||
version of our package in it, as well as its dependencies and other packages we
|
||||
like to have in the environment, all specified with [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs).
|
||||
Indeed, we can just add any package we like to have in our environment to
|
||||
[`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs).
|
||||
like to have in the environment, all specified with `dependencies`.
|
||||
|
||||
```nix
|
||||
with import <nixpkgs> {};
|
||||
|
|
@ -1470,9 +1476,11 @@ with python311Packages;
|
|||
buildPythonPackage rec {
|
||||
name = "mypackage";
|
||||
src = ./path/to/package/source;
|
||||
propagatedBuildInputs = [
|
||||
dependencies = [
|
||||
pytest
|
||||
numpy
|
||||
];
|
||||
propagatedBuildInputs = [
|
||||
pkgs.libsndfile
|
||||
];
|
||||
}
|
||||
|
|
@ -1519,7 +1527,7 @@ buildPythonPackage rec {
|
|||
hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
];
|
||||
|
|
@ -1903,8 +1911,8 @@ configure alternatives](#sec-overlays-alternatives-blas-lapack)".
|
|||
|
||||
In a `setup.py` or `setup.cfg` it is common to declare dependencies:
|
||||
|
||||
* `setup_requires` corresponds to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs)
|
||||
* `install_requires` corresponds to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs)
|
||||
* `setup_requires` corresponds to `build-system`
|
||||
* `install_requires` corresponds to `dependencies`
|
||||
* `tests_require` corresponds to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs)
|
||||
|
||||
### How to enable interpreter optimizations? {#optimizations}
|
||||
|
|
@ -1928,12 +1936,10 @@ in mypython
|
|||
|
||||
Some packages define optional dependencies for additional features. With
|
||||
`setuptools` this is called `extras_require` and `flit` calls it
|
||||
`extras-require`, while PEP 621 calls these `optional-dependencies`. A
|
||||
method for supporting this is by declaring the extras of a package in its
|
||||
`passthru`, e.g. in case of the package `dask`
|
||||
`extras-require`, while PEP 621 calls these `optional-dependencies`.
|
||||
|
||||
```nix
|
||||
passthru.optional-dependencies = {
|
||||
optional-dependencies = {
|
||||
complete = [ distributed ];
|
||||
};
|
||||
```
|
||||
|
|
@ -1941,11 +1947,13 @@ passthru.optional-dependencies = {
|
|||
and letting the package requiring the extra add the list to its dependencies
|
||||
|
||||
```nix
|
||||
propagatedBuildInputs = [
|
||||
dependencies = [
|
||||
...
|
||||
] ++ dask.optional-dependencies.complete;
|
||||
```
|
||||
|
||||
This method is using `passthru`, meaning that changing `optional-dependencies` of a package won't cause it to rebuild.
|
||||
|
||||
Note this method is preferred over adding parameters to builders, as that can
|
||||
result in packages depending on different variants and thereby causing
|
||||
collisions.
|
||||
|
|
|
|||
|
|
@ -45,6 +45,14 @@
|
|||
# C can import package A propagated by B
|
||||
, propagatedBuildInputs ? []
|
||||
|
||||
# Python module dependencies.
|
||||
# These are named after PEP-621.
|
||||
, dependencies ? []
|
||||
, optional-dependencies ? {}
|
||||
|
||||
# Python PEP-517 build systems.
|
||||
, build-system ? []
|
||||
|
||||
# DEPRECATED: use propagatedBuildInputs
|
||||
, pythonPath ? []
|
||||
|
||||
|
|
@ -97,8 +105,6 @@
|
|||
|
||||
, meta ? {}
|
||||
|
||||
, passthru ? {}
|
||||
|
||||
, doCheck ? config.doCheckByDefault or false
|
||||
|
||||
, disabledTestPaths ? []
|
||||
|
|
@ -193,10 +199,28 @@ let
|
|||
"setuptools" "wheel"
|
||||
];
|
||||
|
||||
passthru =
|
||||
attrs.passthru or { }
|
||||
// {
|
||||
updateScript = let
|
||||
filename = builtins.head (lib.splitString ":" self.meta.position);
|
||||
in attrs.passthru.updateScript or [ update-python-libraries filename ];
|
||||
}
|
||||
// lib.optionalAttrs (dependencies != []) {
|
||||
inherit dependencies;
|
||||
}
|
||||
// lib.optionalAttrs (optional-dependencies != {}) {
|
||||
inherit optional-dependencies;
|
||||
}
|
||||
// lib.optionalAttrs (build-system != []) {
|
||||
inherit build-system;
|
||||
};
|
||||
|
||||
# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
|
||||
self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [
|
||||
"disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format"
|
||||
"disabledTestPaths" "outputs" "stdenv"
|
||||
"dependencies" "optional-dependencies" "build-system"
|
||||
]) // {
|
||||
|
||||
name = namePrefix + name_;
|
||||
|
|
@ -256,11 +280,11 @@ let
|
|||
pythonNamespacesHook
|
||||
] ++ lib.optionals withDistOutput [
|
||||
pythonOutputDistHook
|
||||
] ++ nativeBuildInputs;
|
||||
] ++ nativeBuildInputs ++ build-system;
|
||||
|
||||
buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);
|
||||
|
||||
propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ [
|
||||
propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ dependencies ++ [
|
||||
# we propagate python even for packages transformed with 'toPythonApplication'
|
||||
# this pollutes the PATH but avoids rebuilds
|
||||
# see https://github.com/NixOS/nixpkgs/issues/170887 for more context
|
||||
|
|
@ -292,6 +316,8 @@ let
|
|||
|
||||
outputs = outputs ++ lib.optional withDistOutput "dist";
|
||||
|
||||
inherit passthru;
|
||||
|
||||
meta = {
|
||||
# default to python's platforms
|
||||
platforms = python.meta.platforms;
|
||||
|
|
@ -305,13 +331,7 @@ let
|
|||
disabledTestPaths = lib.escapeShellArgs disabledTestPaths;
|
||||
}));
|
||||
|
||||
passthru.updateScript = let
|
||||
filename = builtins.head (lib.splitString ":" self.meta.position);
|
||||
in attrs.passthru.updateScript or [ update-python-libraries filename ];
|
||||
in
|
||||
if disabled then
|
||||
throw "${name} not supported for interpreter ${python.executable}"
|
||||
else
|
||||
self.overrideAttrs (attrs: {
|
||||
passthru = lib.recursiveUpdate passthru attrs.passthru;
|
||||
})
|
||||
in lib.extendDerivation
|
||||
(disabled -> throw "${name} not supported for interpreter ${python.executable}")
|
||||
passthru
|
||||
self
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ buildPythonPackage rec {
|
|||
export HOME=$TMPDIR
|
||||
'';
|
||||
|
||||
nativeBuildInputs = [ poetry-core ];
|
||||
propagatedBuildInputs = [ python-dateutil pytzdata ]
|
||||
build-system = [ poetry-core ];
|
||||
dependencies = [ python-dateutil pytzdata ]
|
||||
++ lib.optional (pythonOlder "3.5") typing
|
||||
++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ];
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ buildPythonPackage rec {
|
|||
hash = "sha256-lCxadY+Y15Dq7Ropy27vx/+w0c968Fw9J5Flbb1q0eE=";
|
||||
};
|
||||
|
||||
propagatedBuildInputs = [
|
||||
dependencies = [
|
||||
brotlicffi
|
||||
certifi
|
||||
charset-normalizer
|
||||
|
|
@ -38,7 +38,7 @@ buildPythonPackage rec {
|
|||
urllib3
|
||||
];
|
||||
|
||||
passthru.optional-dependencies = {
|
||||
optional-dependencies = {
|
||||
security = [];
|
||||
socks = [
|
||||
pysocks
|
||||
|
|
@ -53,7 +53,7 @@ buildPythonPackage rec {
|
|||
pytest-xdist
|
||||
pytestCheckHook
|
||||
]
|
||||
++ passthru.optional-dependencies.socks;
|
||||
++ optional-dependencies.socks;
|
||||
|
||||
disabledTests = [
|
||||
# Disable tests that require network access and use httpbin
|
||||
|
|
|
|||
Loading…
Reference in a new issue