From 6893201945af31336ce2e3ec886de773a1e48b6f Mon Sep 17 00:00:00 2001 From: figsoda Date: Fri, 8 Sep 2023 20:38:54 -0400 Subject: [PATCH 1/5] python310Packages.buildPythonPackage: introduce pyproject option --- .../python/mk-python-derivation.nix | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 8a75dbf43a74..e9c783116b60 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -82,6 +82,11 @@ # However, some packages do provide executables with extensions, and thus bytecode is generated. , removeBinBytecode ? true +# pyproject = true <-> format = "pyproject" +# pyproject = false <-> format = "other" +# https://github.com/NixOS/nixpkgs/issues/253154 +, pyproject ? null + # Several package formats are supported. # "setuptools" : Install a common setuptools/distutils based package. This builds a wheel. # "wheel" : Install from a pre-compiled wheel. @@ -89,7 +94,7 @@ # "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel. # "egg": Install a package from an egg. # "other" : Provide your own buildPhase and installPhase. -, format ? "setuptools" +, format ? null , meta ? {} @@ -101,10 +106,23 @@ , ... } @ attrs: +assert (pyproject != null) -> (format == null); + let inherit (python) stdenv; - withDistOutput = lib.elem format ["pyproject" "setuptools" "flit" "wheel"]; + format' = + if pyproject != null then + if pyproject then + "pyproject" + else + "other" + else if format != null then + format + else + "setuptools"; + + withDistOutput = lib.elem format' ["pyproject" "setuptools" "flit" "wheel"]; name_ = name; @@ -177,7 +195,7 @@ let # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [ - "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format" + "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format" "disabledTestPaths" "outputs" ]) // { @@ -202,11 +220,11 @@ let pythonRemoveBinBytecodeHook ] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [ unzip - ] ++ lib.optionals (format == "setuptools") [ + ] ++ lib.optionals (format' == "setuptools") [ setuptoolsBuildHook - ] ++ lib.optionals (format == "flit") [ + ] ++ lib.optionals (format' == "flit") [ flitBuildHook - ] ++ lib.optionals (format == "pyproject") [( + ] ++ lib.optionals (format' == "pyproject") [( if isBootstrapPackage then pypaBuildHook.override { inherit (python.pythonForBuild.pkgs.bootstrap) build; @@ -214,11 +232,11 @@ let } else pypaBuildHook - )] ++ lib.optionals (format == "wheel") [ + )] ++ lib.optionals (format' == "wheel") [ wheelUnpackHook - ] ++ lib.optionals (format == "egg") [ + ] ++ lib.optionals (format' == "egg") [ eggUnpackHook eggBuildHook eggInstallHook - ] ++ lib.optionals (format != "other") [( + ] ++ lib.optionals (format' != "other") [( if isBootstrapInstallPackage then pypaInstallHook.override { inherit (python.pythonForBuild.pkgs.bootstrap) installer; @@ -252,7 +270,7 @@ let doCheck = false; doInstallCheck = attrs.doCheck or true; nativeInstallCheckInputs = [ - ] ++ lib.optionals (format == "setuptools") [ + ] ++ lib.optionals (format' == "setuptools") [ # Longer-term we should get rid of this and require # users of this function to set the `installCheckPhase` or # pass in a hook that sets it. From 9e680e422f9ac7d8423295faee164624343b0ad5 Mon Sep 17 00:00:00 2001 From: figsoda Date: Fri, 8 Sep 2023 20:44:23 -0400 Subject: [PATCH 2/5] nix-update: use pyproject --- pkgs/tools/package-management/nix-update/default.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgs/tools/package-management/nix-update/default.nix b/pkgs/tools/package-management/nix-update/default.nix index 2dba8c0c1356..c0030fcf8a60 100644 --- a/pkgs/tools/package-management/nix-update/default.nix +++ b/pkgs/tools/package-management/nix-update/default.nix @@ -10,7 +10,7 @@ python3.pkgs.buildPythonApplication rec { pname = "nix-update"; version = "0.19.3"; - format = "setuptools"; + pyproject = true; src = fetchFromGitHub { owner = "Mic92"; @@ -19,6 +19,10 @@ python3.pkgs.buildPythonApplication rec { hash = "sha256-+WD+SV/L3TvksWBIg6jk+T0dUTNdp4VKONzdzVT+pac="; }; + nativeBuildInputs = [ + python3.pkgs.setuptools + ]; + makeWrapperArgs = [ "--prefix" "PATH" ":" (lib.makeBinPath [ nix nix-prefetch-git nixpkgs-fmt nixpkgs-review ]) ]; From de5c170b99fd184cabd9fb893e7d6c26a6b57081 Mon Sep 17 00:00:00 2001 From: figsoda Date: Fri, 8 Sep 2023 20:46:02 -0400 Subject: [PATCH 3/5] ruff-lsp: use `pyproject = true` --- pkgs/development/tools/language-servers/ruff-lsp/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/tools/language-servers/ruff-lsp/default.nix b/pkgs/development/tools/language-servers/ruff-lsp/default.nix index 4f4304017c4a..abd9bbb52c97 100644 --- a/pkgs/development/tools/language-servers/ruff-lsp/default.nix +++ b/pkgs/development/tools/language-servers/ruff-lsp/default.nix @@ -16,7 +16,7 @@ buildPythonPackage rec { pname = "ruff-lsp"; version = "0.0.39"; - format = "pyproject"; + pyproject = true; disabled = pythonOlder "3.7"; src = fetchFromGitHub { From e37dc2707f7a07cef1d6e35783e482ad9bd8f6d4 Mon Sep 17 00:00:00 2001 From: figsoda Date: Fri, 8 Sep 2023 20:47:31 -0400 Subject: [PATCH 4/5] seahub: use `pyproject = false` --- pkgs/applications/networking/seahub/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/networking/seahub/default.nix b/pkgs/applications/networking/seahub/default.nix index 9bd90ad928c9..089eabf93d65 100644 --- a/pkgs/applications/networking/seahub/default.nix +++ b/pkgs/applications/networking/seahub/default.nix @@ -22,7 +22,7 @@ in python.pkgs.buildPythonApplication rec { pname = "seahub"; version = "9.0.10"; - format = "other"; + pyproject = false; src = fetchFromGitHub { owner = "haiwen"; From 39c64e232328e09284c766d2b56f2d680e511d41 Mon Sep 17 00:00:00 2001 From: figsoda Date: Sat, 9 Sep 2023 10:53:22 -0400 Subject: [PATCH 5/5] doc/python: document `pyproject` and remove usages of `format` --- doc/languages-frameworks/python.section.md | 86 +++++++++++++++++----- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 2ced6d69edd5..daf0740bcde0 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -101,7 +101,7 @@ The following is an example: buildPythonPackage rec { pname = "pytest"; version = "3.3.1"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; @@ -167,12 +167,15 @@ following are specific to `buildPythonPackage`: * `dontWrapPythonPrograms ? false`: Skip wrapping of Python programs. * `permitUserSite ? false`: Skip setting the `PYTHONNOUSERSITE` environment variable in wrapped programs. -* `format ? "setuptools"`: Format of the source. Valid options are - `"setuptools"`, `"pyproject"`, `"flit"`, `"wheel"`, and `"other"`. - `"setuptools"` is for when the source has a `setup.py` and `setuptools` is - used to build a wheel, `flit`, in case `flit` should be used to build a wheel, - and `wheel` in case a wheel is provided. Use `other` when a custom - `buildPhase` and/or `installPhase` is needed. +* `pyproject`: Whether the pyproject format should be used. When set to `true`, + `pypaBuildHook` will be used, and you can add the required build dependencies + from `build-system.requires` to `nativeBuildInputs`. Note that the pyproject + format falls back to using `setuptools`, so you can use `pyproject = true` + even if the package only has a `setup.py`. When set to `false`, you can + use the existing [hooks](#setup-hooks0 or provide your own logic to build the + package. This can be useful for packages that don't support the pyproject + format. When unset, the legacy `setuptools` hooks are used for backwards + compatibility. * `makeWrapperArgs ? []`: A list of strings. Arguments to be passed to `makeWrapper`, which wraps generated binaries. By default, the arguments to `makeWrapper` set `PATH` and `PYTHONPATH` environment variables before calling @@ -286,20 +289,25 @@ specifying an interpreter version), like this: python3.pkgs.buildPythonApplication rec { pname = "luigi"; version = "2.7.9"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; hash = "sha256-Pe229rT0aHwA98s+nTHQMEFKZPo/yw6sot8MivFDvAw="; }; + nativeBuildInputs = [ + python3.pkgs.setuptools + python3.pkgs.wheel + ]; + propagatedBuildInputs = with python3.pkgs; [ tornado python-daemon ]; meta = with lib; { - ... + # ... }; } ``` @@ -858,18 +866,25 @@ building Python libraries is `buildPythonPackage`. Let's see how we can build th { lib , buildPythonPackage , fetchPypi +, setuptools +, wheel }: buildPythonPackage rec { pname = "toolz"; version = "0.10.0"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA="; }; + nativeBuildInputs = [ + setuptools + wheel + ]; + # has no tests doCheck = false; @@ -918,13 +933,18 @@ with import {}; my_toolz = python311.pkgs.buildPythonPackage rec { pname = "toolz"; version = "0.10.0"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA="; }; + nativeBuildInputs = [ + python311.pkgs.setuptools + python311.pkgs.wheel + ]; + # has no tests doCheck = false; @@ -972,6 +992,9 @@ order to build [`datashape`](https://github.com/blaze/datashape). , buildPythonPackage , fetchPypi +# build dependencies +, setuptools, wheel + # dependencies , numpy, multipledispatch, python-dateutil @@ -982,13 +1005,18 @@ order to build [`datashape`](https://github.com/blaze/datashape). buildPythonPackage rec { pname = "datashape"; version = "0.4.7"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; hash = "sha256-FLLvdm1MllKrgTGC6Gb0k0deZeVYvtCCLji/B7uhong="; }; + nativeBuildInputs = [ + setuptools + wheel + ]; + propagatedBuildInputs = [ multipledispatch numpy @@ -1023,6 +1051,8 @@ when building the bindings and are therefore added as `buildInputs`. { lib , buildPythonPackage , fetchPypi +, setuptools +, wheel , libxml2 , libxslt }: @@ -1030,13 +1060,18 @@ when building the bindings and are therefore added as `buildInputs`. buildPythonPackage rec { pname = "lxml"; version = "3.4.4"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; hash = "sha256-s9NiusRxFydHzaNRMjjxFcvWxfi45jGb9ql6eJJyQJk="; }; + nativeBuildInputs = [ + setuptools + wheel + ]; + buildInputs = [ libxml2 libxslt @@ -1067,6 +1102,10 @@ therefore we have to set `LDFLAGS` and `CFLAGS`. , buildPythonPackage , fetchPypi +# build dependencies +, setuptools +, wheel + # dependencies , fftw , fftwFloat @@ -1078,13 +1117,18 @@ therefore we have to set `LDFLAGS` and `CFLAGS`. buildPythonPackage rec { pname = "pyFFTW"; version = "0.9.2"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; hash = "sha256-9ru2r6kwhUCaskiFoaPNuJCfCVoUL01J40byvRt4kHQ="; }; + nativeBuildInputs = [ + setuptools + wheel + ]; + buildInputs = [ fftw fftwFloat @@ -1334,9 +1378,7 @@ instead of a dev dependency). Keep in mind that while the examples above are done with `requirements.txt`, `pythonRelaxDepsHook` works by modifying the resulting wheel file, so it should -work in any of the formats supported by `buildPythonPackage` currently, -with the exception of `other` (see `format` in -[`buildPythonPackage` parameters](#buildpythonpackage-parameters) for more details). +work with any of the existing [hooks](#setup-hooks). #### Using unittestCheckHook {#using-unittestcheckhook} @@ -1461,18 +1503,26 @@ We first create a function that builds `toolz` in `~/path/to/toolz/release.nix` ```nix { lib , buildPythonPackage +, fetchPypi +, setuptools +, wheel }: buildPythonPackage rec { pname = "toolz"; version = "0.10.0"; - format = "setuptools"; + pyproject = true; src = fetchPypi { inherit pname version; hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA="; }; + nativeBuildInputs = [ + setuptools + wheel + ]; + meta = with lib; { changelog = "https://github.com/pytoolz/toolz/releases/tag/${version}"; homepage = "https://github.com/pytoolz/toolz/";