Merge pull request #258 from chaoflow/python

Python recursivePthLoader in action

I've checked that nothing which passes in trunk: http://hydra.nixos.org/eval/810237 fails in python-rework: http://hydra.nixos.org/eval/810395

I can also now use buildEnv for python very comfortably. This is a massive improvement, thanks a lot!

pkgs/development/interpreters/python/2.6/default.nix

@@ -68,6 +68,8 @@ let
         rm -rf "$out/lib/python${majorVersion}/test"
         ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb
         ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb${majorVersion}
+        mv $out/share/man/man1/{python.1,python2.6.1}
+        ln -s $out/share/man/man1/{python2.6.1,python.1}
       '';
 
     passthru = {

pkgs/development/interpreters/python/2.7/default.nix

@@ -76,6 +76,7 @@ let
         rm -rf "$out/lib/python${majorVersion}/test"
         ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb
         ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb${majorVersion}
+        ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}
       '';
 
     passthru = {

pkgs/development/interpreters/python/wrapper.nix

@@ -1,12 +1,12 @@
 # Create a python that knows about additional python packages via
 # PYTHONPATH
 
-{stdenv, python, makeWrapper, extraLibs ? []}:
+{ stdenv, python, makeWrapper, recursivePthLoader, extraLibs ? [] }:
 
 stdenv.mkDerivation {
   name = "python-${python.version}-wrapper";
 
-  propagatedBuildInputs = [python makeWrapper] ++ extraLibs;
+  propagatedBuildInputs = extraLibs ++ [ python makeWrapper recursivePthLoader ];
 
   unpackPhase = "true";
   installPhase = ''

pkgs/development/python-modules/generic/default.nix

@@ -3,7 +3,7 @@
    (http://pypi.python.org/pypi/setuptools/), which represents a large
    number of Python packages nowadays.  */
 
-{ python, setuptools, wrapPython, lib, offlineDistutils, setuptoolsSite }:
+{ python, setuptools, wrapPython, lib, offlineDistutils, recursivePthLoader }:
 
 { name, namePrefix ? "python-"
 
@@ -48,16 +48,14 @@ python.stdenv.mkDerivation (attrs // {
 
   name = namePrefix + name;
 
+  # checkPhase after installPhase to run tests on installed packages
   phases = "unpackPhase patchPhase configurePhase buildPhase installPhase checkPhase fixupPhase distPhase";
 
   buildInputs = [ python wrapPython setuptools ] ++ buildInputs ++ pythonPath;
 
-  # setuptoolsSite is responsible for loading pth files
-  propagatedBuildInputs = propagatedBuildInputs ++ [ setuptoolsSite ];
+  propagatedBuildInputs = propagatedBuildInputs ++ [ recursivePthLoader ];
 
-  buildInputStrings = map toString buildInputs;
-
-  pythonPath = [ setuptools] ++ pythonPath;
+  pythonPath = [ setuptools ] ++ pythonPath;
 
   preConfigure = ''
     PYTHONPATH="${offlineDistutils}/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
@@ -83,9 +81,7 @@ python.stdenv.mkDerivation (attrs // {
     # Remove any site.py files generated by easy_install as these
     # cause collisions. If pth files are to be processed a
     # corresponding site.py needs to be included in the PYTHONPATH.
-    #
-    # leave them until we have a better solution: see #209
-    #rm -f "$out/lib/${python.libPrefix}"/site-packages/site.py*
+    rm -f "$out/lib/${python.libPrefix}"/site-packages/site.py*
 
     ${postInstall}
   '';
@@ -94,15 +90,6 @@ python.stdenv.mkDerivation (attrs // {
     ''
       wrapPythonPrograms
 
-      # If a user installs a Python package, she probably also wants its
-      # dependencies in the user environment (since Python modules don't
-      # have something like an RPATH, so the only way to find the
-      # dependencies is to have them in the PYTHONPATH variable).
-      if test -e $out/nix-support/propagated-build-inputs; then
-          ln -s $out/nix-support/propagated-build-inputs $out/nix-support/propagated-user-env-packages
-      fi
-
-      createBuildInputsPth build-inputs "$buildInputStrings"
       for inputsfile in propagated-build-inputs propagated-build-native-inputs; do
         if test -e $out/nix-support/$inputsfile; then
             createBuildInputsPth $inputsfile "$(cat $out/nix-support/$inputsfile)"

pkgs/development/python-modules/generic/wrap.sh

@@ -51,6 +51,9 @@ createBuildInputsPth() {
     local inputs="$2"
     if [ foo"$inputs" != foo ]; then
         for x in $inputs; do
+            if $(echo -n $x |grep -q python-recursive-pth-loader); then
+                continue
+            fi
             if test -d "$x"/lib/@libPrefix@/site-packages; then
                 echo $x/lib/@libPrefix@/site-packages \
                     >> "$out"/lib/@libPrefix@/site-packages/${name}-nix-python-$category.pth

pkgs/development/python-modules/pycairo/default.nix

@@ -1,4 +1,4 @@
-{stdenv, fetchurl, python, pkgconfig, cairo, x11}:
+{ stdenv, fetchurl, python, pkgconfig, cairo, x11 }:
 
 stdenv.mkDerivation {
   name = "pycairo-1.8.8";
@@ -7,5 +7,5 @@ stdenv.mkDerivation {
     sha256 = "0q18hd4ai4raljlvd76ylgi30kxpr2qq83ka6gzwh0ya8fcmjlig";
   };
 
-  buildInputs = [python pkgconfig cairo x11];
+  buildInputs = [ python pkgconfig cairo x11 ];
 }

pkgs/development/python-modules/pygobject/default.nix

@@ -12,16 +12,12 @@ stdenv.mkDerivation rec {
 
   buildInputs = [ python pkgconfig glib ];
 
+  # in a "normal" setup, pygobject and pygtk are installed into the
+  # same site-packages: we need a pth file for both. pygtk.py would be
+  # used to select a specific version, in our setup it should have no
+  # effect, but we leave it in case somebody expects and calls it.
   postInstall = ''
-    # All python code is installed into a "gtk-2.0" sub-directory. That
-    # sub-directory may be useful on systems which share several library
-    # versions in the same prefix, i.e. /usr/local, but on Nix that directory
-    # is useless. Furthermore, its existence makes it very hard to guess a
-    # proper $PYTHONPATH that allows "import gtk" to succeed.
-    cd $(toPythonPath $out)/gtk-2.0
-    for n in *; do
-      ln -s "gtk-2.0/$n" "../$n"
-    done
+    mv $out/lib/${python.libPrefix}/site-packages/{pygtk.pth,${name}.pth}
   '';
 
   meta = {

pkgs/development/python-modules/pygtk/default.nix

@@ -1,7 +1,7 @@
-{ stdenv, fetchurl, makeWrapper, python, pkgconfig, glib, gtk, pygobject, pycairo
-, libglade ? null }:
+{ stdenv, fetchurl, python, pkgconfig, glib, gtk, pygobject, pycairo
+, buildPythonPackage, libglade ? null }:
 
-stdenv.mkDerivation rec {
+buildPythonPackage rec {
   name = "pygtk-2.22.0";
 
   src = fetchurl {
@@ -10,26 +10,35 @@ stdenv.mkDerivation rec {
   };
 
   buildInputs =
-    [ makeWrapper python pkgconfig glib gtk ]
+    [ pkgconfig glib gtk ]
     ++ stdenv.lib.optional (libglade != null) libglade;
 
   propagatedBuildInputs = [ pygobject pycairo ];
 
+  installCommand = "make install";
+  checkPhase = stdenv.lib.optionalString (libglade == null)
+    ''
+      sed -i -e "s/glade = importModule('gtk.glade', buildDir)//" \
+             tests/common.py
+      sed -i -e "s/, glade$//" \
+             -e "s/.*testGlade.*//" \
+             -e "s/.*(glade.*//" \
+             tests/test_api.py
+    '' + ''
+      sed -i -e "s/sys.path.insert(0, os.path.join(buildDir, 'gtk'))//" \
+             -e "s/sys.path.insert(0, buildDir)//" \
+             tests/common.py
+      make check
+    '';
+  # XXX: TypeError: Unsupported type: <class 'gtk._gtk.WindowType'>
+  # The check phase was not executed in the previous
+  # non-buildPythonPackage setup - not sure why not.
+  doCheck = false;
+
   postInstall = ''
     rm $out/bin/pygtk-codegen-2.0
     ln -s ${pygobject}/bin/pygobject-codegen-2.0  $out/bin/pygtk-codegen-2.0
-
-    # All python code is installed into a "gtk-2.0" sub-directory. That
-    # sub-directory may be useful on systems which share several library
-    # versions in the same prefix, i.e. /usr/local, but on Nix that directory
-    # is useless. Furthermore, its existence makes it very hard to guess a
-    # proper $PYTHONPATH that allows "import gtk" to succeed.
-    cd $(toPythonPath $out)/gtk-2.0
-    for n in *; do
-      ln -s "gtk-2.0/$n" "../$n"
-    done
-
-    wrapProgram $out/bin/pygtk-demo --prefix PYTHONPATH ":" \
-        $(toPythonPath "${pygobject} ${pycairo} $out")
+    ln -s ${pygobject}/lib/${python.libPrefix}/site-packages/${pygobject.name}.pth \
+                  $out/lib/${python.libPrefix}/site-packages/${name}.pth
   '';
 }

pkgs/development/python-modules/setuptools/default.nix

@@ -1,7 +1,7 @@
 { stdenv, fetchurl, python, wrapPython }:
 
 stdenv.mkDerivation rec {
-  name = "setuptools-" + version;
+  name = "python-setuptools-" + version;
 
   version = "0.6c11";
 

pkgs/top-level/all-packages.nix

@@ -2834,25 +2834,23 @@ let
 
   pure = callPackage ../development/interpreters/pure {};
 
-  python = python27;
   python3 = python32;
-
-  python26 = callPackage ../development/interpreters/python/2.6 { };
-
-  python27 = callPackage ../development/interpreters/python/2.7 { };
-
   python32 = callPackage ../development/interpreters/python/3.2 { };
 
-  pythonFull = python27Full;
+  python = python27;
+  python26 = callPackage ../development/interpreters/python/2.6 { };
+  python27 = callPackage ../development/interpreters/python/2.7 { };
 
+  pythonFull = python27Full;
   python26Full = callPackage ../development/interpreters/python/wrapper.nix {
     extraLibs = lib.attrValues python26.modules;
     python = python26;
+    inherit (python26Packages) recursivePthLoader;
   };
-
   python27Full = callPackage ../development/interpreters/python/wrapper.nix {
     extraLibs = lib.attrValues python27.modules;
     python = python27;
+    inherit (python27Packages) recursivePthLoader;
   };
 
   pythonhomeWrapper = callPackage ../development/interpreters/python/pythonhome-wrapper.nix { };
@@ -5293,6 +5291,10 @@ let
 
   pythonPackages = python27Packages;
 
+  # `nix-env -i python-nose` installs for 2.7, the default python.
+  # Therefore we do not recurse into attributes here, in contrast to
+  # python27Packages. `nix-env -iA python26Packages.nose` works
+  # regardless.
   python26Packages = import ./python-packages.nix {
     inherit pkgs;
     python = python26;
@@ -5309,13 +5311,13 @@ let
 
   numeric = callPackage ../development/python-modules/numeric { };
 
-  pil = python27Packages.pil;
+  pil = pythonPackages.pil;
 
   psyco = callPackage ../development/python-modules/psyco { };
 
-  pycairo = callPackage ../development/python-modules/pycairo { };
+  pycairo = pythonPackages.pycairo;
 
-  pycrypto = python27Packages.pycrypto;
+  pycrypto = pythonPackages.pycrypto;
 
   pycups = callPackage ../development/python-modules/pycups { };
 
@@ -5323,13 +5325,11 @@ let
 
   pygame = callPackage ../development/python-modules/pygame { };
 
-  pygobject = callPackage ../development/python-modules/pygobject { };
+  pygobject = pythonPackages.pygobject;
 
-  pygtk = callPackage ../development/python-modules/pygtk { };
+  pygtk = pythonPackages.pygtk;
 
-  pyGtkGlade = callPackage ../development/python-modules/pygtk {
-    inherit (gnome) libglade;
-  };
+  pyGtkGlade = pythonPackages.pyGtkGlade;
 
   pyopenssl = builderDefsPackage (import ../development/python-modules/pyopenssl) {
     inherit python openssl;

pkgs/top-level/python-packages.nix

@@ -3,22 +3,28 @@
 let pythonPackages = python.modules // rec {
 
   inherit python;
-
   inherit (pkgs) fetchurl fetchsvn fetchgit stdenv;
 
+  # helpers
 
   buildPythonPackage = import ../development/python-modules/generic {
     inherit (pkgs) lib;
-    inherit python wrapPython setuptools setuptoolsSite offlineDistutils;
+    inherit python wrapPython setuptools recursivePthLoader offlineDistutils;
   };
 
+  wrapPython = pkgs.makeSetupHook
+    { deps = pkgs.makeWrapper;
+      substitutions.libPrefix = python.libPrefix;
+    }
+    ../development/python-modules/generic/wrap.sh;
+
+  # specials
 
   recursivePthLoader = import ../development/python-modules/recursive-pth-loader {
     inherit (pkgs) stdenv;
     inherit python;
   };
 
-
   setuptools = import ../development/python-modules/setuptools {
     inherit (pkgs) stdenv fetchurl;
     inherit python wrapPython;
@@ -34,6 +40,8 @@ let pythonPackages = python.modules // rec {
     inherit python;
   };
 
+  # packages defined elsewhere
+
   ipython = import ../shells/ipython {
     inherit (pkgs) stdenv fetchurl;
     inherit buildPythonPackage pythonPackages;
@@ -44,17 +52,37 @@ let pythonPackages = python.modules // rec {
     inherit python buildPythonPackage;
   };
 
+  pycairo = import ../development/python-modules/pycairo {
+    inherit (pkgs) stdenv fetchurl pkgconfig cairo x11;
+    inherit python;
+  };
+
   pycrypto = import ../development/python-modules/pycrypto {
     inherit (pkgs) fetchurl stdenv gmp;
     inherit python buildPythonPackage;
   };
 
-  wrapPython = pkgs.makeSetupHook
-    { deps = pkgs.makeWrapper;
-      substitutions.libPrefix = python.libPrefix;
-    }
-    ../development/python-modules/generic/wrap.sh;
+  pygobject = import ../development/python-modules/pygobject {
+    inherit (pkgs) stdenv fetchurl pkgconfig glib;
+    inherit python;
+  };
 
+  pygtk = import ../development/python-modules/pygtk {
+    inherit (pkgs) fetchurl stdenv pkgconfig glib gtk;
+    inherit python buildPythonPackage pygobject pycairo;
+  };
+
+  # XXX: how can we get an override here?
+  #pyGtkGlade = pygtk.override {
+  #  inherit (pkgs.gnome) libglade;
+  #};
+  pyGtkGlade = import ../development/python-modules/pygtk {
+    inherit (pkgs) fetchurl stdenv pkgconfig glib gtk;
+    inherit (pkgs.gnome) libglade;
+    inherit python buildPythonPackage pygobject pycairo;
+  };
+
+  # packages defined here
 
   afew = buildPythonPackage rec {
     rev = "6bb3915636aaf86f046a017ffffd9a4ef395e199";