Merge pull request #86843 from fare-patches/fare

Enhanced support for Gerbil, Gambit

pkgs/development/compilers/gambit/bootstrap.nix

@@ -1,6 +1,10 @@
-{ stdenv, fetchurl, autoconf, gcc, coreutils, ... }:
+# This derivation is a reduced-functionality variant of Gambit stable,
+# used to compile the full version of Gambit stable *and* unstable.
 
-stdenv.mkDerivation {
+{ gccStdenv, lib, fetchurl, autoconf, gcc, coreutils, gambit-support, ... }:
+# As explained in build.nix, GCC compiles Gambit 10x faster than Clang, for code 3x better
+
+gccStdenv.mkDerivation {
   pname = "gambit-bootstrap";
   version = "4.9.3";
 
@@ -16,29 +20,25 @@ stdenv.mkDerivation {
            CPP=${gcc}/bin/cpp CXXCPP=${gcc}/bin/cpp LD=${gcc}/bin/ld \
            XMKMF=${coreutils}/bin/false
     unset CFLAGS LDFLAGS LIBS CPPFLAGS CXXFLAGS
-    ./configure --prefix=$out
+    ./configure --prefix=$out/gambit
   '';
 
   buildPhase = ''
     # Copy the (configured) sources now, not later, so we don't have to filter out
     # all the intermediate build products.
-    mkdir -p $out ; cp -rp . $out/
+    mkdir -p $out/gambit ; cp -rp . $out/gambit/
 
     # build the gsc-boot* compiler
-    make bootstrap
+    make -j$NIX_BUILD_CORES bootstrap
   '';
 
   installPhase = ''
-    cp -fa ./ $out/
+    cp -fa ./ $out/gambit/
   '';
 
   forceShare = [ "info" ];
 
-  meta = {
+  meta = gambit-support.meta // {
     description = "Optimizing Scheme to C compiler, bootstrap step";
-    homepage    = "http://gambitscheme.org";
-    license     = stdenv.lib.licenses.lgpl2;
-    platforms   = stdenv.lib.platforms.unix;
-    maintainers = with stdenv.lib.maintainers; [ thoughtpolice raskin fare ];
   };
 }

pkgs/development/compilers/gambit/build.nix

@@ -1,11 +1,13 @@
-{ stdenv, git, openssl, autoconf, pkgs, makeStaticLibraries, version, gcc, src, coreutils }:
+{ gccStdenv, lib, git, openssl, autoconf, pkgs, makeStaticLibraries, gcc, coreutils, gnused, gnugrep,
+  src, version, git-version,
+  gambit-support, optimizationSetting ? "-O1", gambit-params ? pkgs.gambit-support.stable-params }:
 
 # Note that according to a benchmark run by Marc Feeley on May 2018,
 # clang is 10x (with default settings) to 15% (with -O2) slower than GCC at compiling
 # Gambit output, producing code that is 3x slower. IIRC the benchmarks from Gambit@30,
 # the numbers were still heavily in favor of GCC in October 2019.
 # Thus we use GCC over clang, even on macOS.
-
+#
 # Also note that I (fare) just ran benchmarks from https://github.com/ecraven/r7rs-benchmarks
 # with Gambit 4.9.3 with -O1 vs -O2 vs -Os on Feb 2020. Which wins depends on the benchmark.
 # The fight is unclear between -O1 and -O2, where -O1 wins more often, by up to 17%,
@@ -13,29 +15,34 @@
 # However, -Os seems more consistent in winning slightly against both -O1 and -O2,
 # and is overall 15% faster than -O2. As for compile times, -O1 is fastest,
 # -Os is about 29%-33% slower than -O1, while -O2 is about 40%-50% slower than -O1.
-# Overall, -Os seems like the best choice, and that's what we now use.
+#
+# Overall, -Os seems like the best choice, but I care more about compile-time,
+# so I stick with -O1 (in the defaults above), which is also the default for Gambit.
+
+gccStdenv.mkDerivation rec {
 
-stdenv.mkDerivation rec {
   pname = "gambit";
-  inherit version;
-  inherit src;
-
-  bootstrap = import ./bootstrap.nix ( pkgs );
+  inherit src version git-version;
+  bootstrap = gambit-support.gambit-bootstrap;
 
   # TODO: if/when we can get all the library packages we depend on to have static versions,
   # we could use something like (makeStaticLibraries openssl) to enable creation
   # of statically linked binaries by gsc.
   buildInputs = [ git autoconf bootstrap openssl ];
 
+  # TODO: patch gambit's source so it has the full path to sed, grep, fgrep? Is there more?
+  # Or wrap relevant programs to add a suitable PATH ?
+  #runtimeDeps = [ gnused gnugrep ];
+
   configureFlags = [
     "--enable-single-host"
-    "--enable-c-opt=-Os"
+    "--enable-c-opt=${optimizationSetting}"
     "--enable-gcc-opts"
     "--enable-shared"
     "--enable-absolute-shared-libs" # Yes, NixOS will want an absolute path, and fix it.
     "--enable-poll"
     "--enable-openssl"
-    "--enable-default-runtime-options=f8,-8,t8" # Default to UTF-8 for source and all I/O
+    "--enable-default-runtime-options=${gambit-params.defaultRuntimeOptions}"
     # "--enable-debug" # Nope: enables plenty of good stuff, but also the costly console.log
     # "--enable-multiple-versions" # Nope, NixOS already does version multiplexing
     # "--enable-guide"
@@ -53,11 +60,17 @@ stdenv.mkDerivation rec {
   ];
 
   configurePhase = ''
-    export CC=${gcc}/bin/gcc CXX=${gcc}/bin/g++ \
-           CPP=${gcc}/bin/cpp CXXCPP=${gcc}/bin/cpp LD=${gcc}/bin/ld \
+    export CC=${gcc}/bin/gcc \
+           CXX=${gcc}/bin/g++ \
+           CPP=${gcc}/bin/cpp \
+           CXXCPP=${gcc}/bin/cpp \
+           LD=${gcc}/bin/ld \
            XMKMF=${coreutils}/bin/false
     unset CFLAGS LDFLAGS LIBS CPPFLAGS CXXFLAGS
-    ./configure --prefix=$out ${builtins.concatStringsSep " " configureFlags}
+
+    ${gambit-params.fix-stamp git-version}
+
+    ./configure --prefix=$out/gambit ${builtins.concatStringsSep " " configureFlags}
 
     # OS-specific paths are hardcoded in ./configure
     substituteInPlace config.status \
@@ -69,28 +82,26 @@ stdenv.mkDerivation rec {
   buildPhase = ''
     # Make bootstrap compiler, from release bootstrap
     mkdir -p boot &&
-    cp -rp ${bootstrap}/. boot/. &&
+    cp -rp ${bootstrap}/gambit/. boot/. &&
     chmod -R u+w boot &&
     cd boot &&
-    cp ../gsc/makefile.in ../gsc/*.scm gsc && # */
+    cp ../gsc/makefile.in ../gsc/*.scm gsc/ && # */
     ./configure &&
-    for i in lib gsi gsc ; do (cd $i ; make ) ; done &&
+    for i in lib gsi gsc ; do (cd $i ; make -j$NIX_BUILD_CORES) ; done &&
     cd .. &&
     cp boot/gsc/gsc gsc-boot &&
 
     # Now use the bootstrap compiler to build the real thing!
-    make -j2 from-scratch
+    make -j$NIX_BUILD_CORES from-scratch
+  '';
+
+  postInstall = ''
+    mkdir $out/bin
+    cd $out/bin
+    ln -s ../gambit/bin/* .
   '';
 
   doCheck = true;
 
-  meta = {
-    description = "Optimizing Scheme to C compiler";
-    homepage    = "http://gambitscheme.org";
-    license     = stdenv.lib.licenses.lgpl2;
-    # NB regarding platforms: only actually tested on Linux, *should* work everywhere,
-    # but *might* need adaptation e.g. on macOS.
-    platforms   = stdenv.lib.platforms.unix;
-    maintainers = with stdenv.lib.maintainers; [ thoughtpolice raskin fare ];
-  };
+  meta = gambit-support.meta;
 }

pkgs/development/compilers/gambit/default.nix

@@ -1,10 +1,10 @@
-{ stdenv, callPackage, fetchurl }:
+{ callPackage, fetchurl }:
 
-callPackage ./build.nix {
+callPackage ./build.nix rec {
   version = "4.9.3";
+  git-version = version;
   src = fetchurl {
     url = "http://www.iro.umontreal.ca/~gambit/download/gambit/v4.9/source/gambit-v4_9_3.tgz";
     sha256 = "1p6172vhcrlpjgia6hsks1w4fl8rdyjf9xjh14wxfkv7dnx8a5hk";
   };
-  inherit stdenv;
 }

pkgs/development/compilers/gambit/gambit-support.nix

@@ -0,0 +1,33 @@
+{ pkgs, lib }:
+
+rec {
+  stable-params = {
+    defaultRuntimeOptions = "f8,-8,t8";
+    buildRuntimeOptions = "f8,-8,t8";
+    fix-stamp = git-version : "";
+  };
+
+  unstable-params = {
+    defaultRuntimeOptions = "iL,fL,-L,tL";
+    buildRuntimeOptions = "i8,f8,-8,t8";
+    fix-stamp = git-version : ''
+      substituteInPlace configure \
+        --replace "$(grep '^PACKAGE_VERSION=.*$' configure)" 'PACKAGE_VERSION="v${git-version}"' \
+        --replace "$(grep '^PACKAGE_STRING=.*$' configure)" 'PACKAGE_STRING="Gambit v${git-version}"' ;
+    '';
+  };
+
+  export-gambopt = params : "export GAMBOPT=${params.buildRuntimeOptions} ;";
+
+  gambit-bootstrap = import ./bootstrap.nix ( pkgs );
+
+  meta = {
+    description = "Optimizing Scheme to C compiler";
+    homepage    = "http://gambitscheme.org";
+    license     = lib.licenses.lgpl21; # dual, also asl20
+    # NB regarding platforms: continuously tested on Linux,
+    # tested on macOS once in a while, *should* work everywhere.
+    platforms   = lib.platforms.unix;
+    maintainers = with lib.maintainers; [ thoughtpolice raskin fare ];
+  };
+}

pkgs/development/compilers/gambit/unstable.nix

@@ -1,13 +1,13 @@
-{ stdenv, callPackage, fetchFromGitHub }:
+{ callPackage, fetchFromGitHub, gambit-support }:
 
 callPackage ./build.nix {
-  version = "unstable-2020-02-24";
-# git-version = "4.9.3-979-gc69e9f70";
+  version = "unstable-2020-05-15";
+  git-version = "4.9.3-1109-g3c4d40de";
   src = fetchFromGitHub {
     owner = "feeley";
     repo = "gambit";
-    rev = "c69e9f70dfdc6545353b135a5d5e2f9234f1e1cc";
-    sha256 = "1f69n7yzzdv3wpnjlrbck38xpa8115vbady43mc544l39ckklr0k";
+    rev = "3c4d40de908ae03ca0e3d854edc2234ef401b36c";
+    sha256 = "1c9a6rys2kiiqb79gvw29nv3dwwk6hmi1q4jk1whcx7mds7q5dvr";
   };
-  inherit stdenv;
+  gambit-params = gambit-support.unstable-params;
 }

pkgs/development/compilers/gerbil/build.nix

@@ -1,34 +1,34 @@
-{ stdenv, makeStaticLibraries,
-  coreutils, rsync, bash,
+{ pkgs, gccStdenv, lib, coreutils, bash, # makeStaticLibraries,
   openssl, zlib, sqlite, libxml2, libyaml, libmysqlclient, lmdb, leveldb, postgresql,
-  version, git-version, gambit, src }:
+  version, git-version,
+  gambit-support,
+  gambit ? pkgs.gambit, gambit-params ? pkgs.gambit-support.stable-params, src }:
+
+# We use Gambit, that works 10x better with GCC than Clang. See ../gambit/build.nix
+let stdenv = gccStdenv; in
 
 stdenv.mkDerivation rec {
   pname = "gerbil";
   inherit version;
   inherit src;
 
-  # Use makeStaticLibraries to enable creation of statically linked binaries
   buildInputs_libraries = [ openssl zlib sqlite libxml2 libyaml libmysqlclient lmdb leveldb postgresql ];
-  buildInputs_staticLibraries = map makeStaticLibraries buildInputs_libraries;
 
-  buildInputs = [ gambit rsync bash ]
-    ++ buildInputs_libraries ++ buildInputs_staticLibraries;
+  # TODO: either fix all of Gerbil's dependencies to provide static libraries,
+  # or give up and delete all tentative support for static libraries.
+  #buildInputs_staticLibraries = map makeStaticLibraries buildInputs_libraries;
+
+  buildInputs = [ gambit ]
+    ++ buildInputs_libraries; # ++ buildInputs_staticLibraries;
 
   NIX_CFLAGS_COMPILE = "-I${libmysqlclient}/include/mysql -L${libmysqlclient}/lib/mysql";
 
   postPatch = ''
-    echo '(define (gerbil-version-string) "v${git-version}")' > src/gerbil/runtime/gx-version.scm
-
-    patchShebangs .
-
+    echo '(define (gerbil-version-string) "v${git-version}")' > src/gerbil/runtime/gx-version.scm ;
+    patchShebangs . ;
     grep -Fl '#!/usr/bin/env' `find . -type f -executable` | while read f ; do
-      substituteInPlace "$f" --replace '#!/usr/bin/env' '#!${coreutils}/bin/env'
-    done
-    grep -Fl '"gsc"' `find . -type f -name '*.s*'` | while read f ; do
-      substituteInPlace "$f" --replace '"gsc"' '"${gambit}/bin/gsc"'
-    done
-    substituteInPlace "etc/gerbil.el" --replace '"gxc"' "\"$out/bin/gxc\""
+      substituteInPlace "$f" --replace '#!/usr/bin/env' '#!${coreutils}/bin/env' ;
+    done ;
 '';
 
 ## TODO: make static compilation work.
@@ -47,22 +47,31 @@ stdenv.mkDerivation rec {
 # LEVELDB=${makeStaticLibraries leveldb}/lib/libleveldb.a
 # EOF
 
+  configurePhase = ''
+    (cd src && ./configure \
+      --prefix=$out/gerbil \
+      --with-gambit=${gambit}/gambit \
+      --enable-libxml \
+      --enable-libyaml \
+      --enable-zlib \
+      --enable-sqlite \
+      --enable-mysql \
+      --enable-lmdb \
+      --enable-leveldb)
+  '';
+
   buildPhase = ''
     runHook preBuild
 
-    # Enable all optional libraries
-    substituteInPlace "src/std/build-features.ss" --replace '#f' '#t'
-
-    # Enable autodetection of a default GERBIL_HOME
-    for i in src/gerbil/boot/gx-init-exe.scm src/gerbil/boot/gx-init.scm ; do
-      substituteInPlace "$i" --replace '(getenv "GERBIL_HOME" #f)' "(getenv \"GERBIL_HOME\" \"$out\")"
-    done
-    for i in src/gerbil/boot/gxi-init.scm src/gerbil/compiler/driver.ss src/gerbil/runtime/gx-gambc.scm src/std/build.ss src/tools/build.ss ; do
-      substituteInPlace "$i" --replace '(getenv "GERBIL_HOME")' "(getenv \"GERBIL_HOME\" \"$out\")"
-    done
-
     # gxprof testing uses $HOME/.cache/gerbil/gxc
-    export HOME=$$PWD
+    export HOME=$PWD
+    export GERBIL_BUILD_CORES=$NIX_BUILD_CORES
+    export GERBIL_GXC=$PWD/bin/gxc
+    export GERBIL_BASE=$PWD
+    export GERBIL_HOME=$PWD
+    export GERBIL_PATH=$PWD/lib
+    export PATH=$PWD/bin:$PATH
+    ${gambit-support.export-gambopt gambit-params}
 
     # Build, replacing make by build.sh
     ( cd src && sh build.sh )
@@ -72,19 +81,9 @@ stdenv.mkDerivation rec {
 
   installPhase = ''
     runHook preInstall
-    mkdir -p $out/
-    cp -fa bin lib etc doc $out/
-
-    cat > $out/bin/gxi <<EOF
-#!${bash}/bin/bash -e
-export GERBIL_HOME=$out
-case "\$1" in -:*) GSIOPTIONS=\$1 ; shift ;; esac
-if [[ \$# = 0 ]] ; then
-  exec ${gambit}/bin/gsi \$GSIOPTIONS \$GERBIL_HOME/lib/gxi-init \$GERBIL_HOME/lib/gxi-interactive -
-else
-  exec ${gambit}/bin/gsi \$GSIOPTIONS \$GERBIL_HOME/lib/gxi-init "\$@"
-fi
-EOF
+    mkdir -p $out/gerbil $out/bin
+    (cd src; ./install)
+    (cd $out/bin ; ln -s ../gerbil/bin/* .)
     runHook postInstall
   '';
 
@@ -93,10 +92,10 @@ EOF
   meta = {
     description = "Gerbil Scheme";
     homepage    = "https://github.com/vyzo/gerbil";
-    license     = stdenv.lib.licenses.lgpl2;
+    license     = lib.licenses.lgpl21; # also asl20, like Gambit
     # NB regarding platforms: regularly tested on Linux, only occasionally on macOS.
     # Please report success and/or failure to fare.
-    platforms   = stdenv.lib.platforms.unix;
-    maintainers = with stdenv.lib.maintainers; [ fare ];
+    platforms   = lib.platforms.unix;
+    maintainers = with lib.maintainers; [ fare ];
   };
 }

pkgs/development/compilers/gerbil/default.nix

@@ -1,14 +1,12 @@
-{ stdenv, callPackage, fetchFromGitHub, gambit }:
+{ callPackage, fetchFromGitHub }:
 
 callPackage ./build.nix rec {
-  version = "0.15.1";
-  git-version = "0.15.1";
-  inherit gambit;
+  version = "0.16";
+  git-version = version;
   src = fetchFromGitHub {
     owner = "vyzo";
     repo = "gerbil";
     rev = "v${version}";
-    sha256 = "0qpqms66hz41wwhxb1z0fnzj96ivkm7qi9h9d7lhlr3fsxm1kp1n";
+    sha256 = "0vng0kxpnwsg8jbjdpyn4sdww36jz7zfpfbzayg9sdpz6bjxjy0f";
   };
-  inherit stdenv;
 }

pkgs/development/compilers/gerbil/gerbil-support.nix

@@ -0,0 +1,64 @@
+{ pkgs, gccStdenv, callPackage, fetchFromGitHub }:
+# See ../gambit/build.nix regarding gccStdenv
+
+rec {
+  # Gerbil libraries
+  gerbilPackages-unstable = {
+    gerbil-utils = callPackage ./gerbil-utils.nix { };
+  };
+
+  # Use this function in any package that uses Gerbil libraries, to define the GERBIL_LOADPATH.
+  gerbilLoadPath =
+    gerbilInputs : builtins.concatStringsSep ":" (map (x : x + "/gerbil/lib") gerbilInputs);
+
+  # Use this function to create a Gerbil library. See gerbil-utils as an example.
+  gerbilPackage = {
+    pname, version, src, meta, package,
+    git-version ? "", version-path ? "config/version.ss",
+    gerbil ? pkgs.gerbil-unstable,
+    gambit-params ? pkgs.gambit-support.stable-params,
+    gerbilInputs ? [],
+    buildInputs ? [],
+    softwareName ? "" } :
+    let buildInputs_ = buildInputs; in
+    gccStdenv.mkDerivation rec {
+      inherit src meta pname version;
+      buildInputs = [ gerbil ] ++ gerbilInputs ++ buildInputs_;
+      postPatch = ''
+        set -e ;
+        if [ -n "${version-path}" ] ; then
+          echo '(import :clan/utils/version)\n(register-software "${softwareName}" "${git-version}")\n' > "${version-path}"
+        fi
+        patchShebangs . ;
+      '';
+
+      postConfigure = ''
+        export GERBIL_BUILD_CORES=$NIX_BUILD_CORES
+        export GERBIL_PATH=$PWD/.build
+        export GERBIL_LOADPATH=${gerbilLoadPath gerbilInputs}
+        ${pkgs.gambit-support.export-gambopt gambit-params}
+      '';
+
+      buildPhase = ''
+        runHook preBuild
+        ./build.ss
+        runHook postBuild
+      '';
+
+      installPhase = ''
+        runHook preInstall
+        mkdir -p $out/gerbil/lib
+        cp -fa .build/lib $out/gerbil/
+        bins=(.build/bin/*)
+        if [ 0 -lt ''${#bins} ] ; then
+          cp -fa .build/bin $out/gerbil/
+          mkdir $out/bin
+          cd $out/bin
+          ln -s ../gerbil/bin/* .
+        fi
+        runHook postInstall
+      '';
+
+      dontFixup = true;
+    };
+}

pkgs/development/compilers/gerbil/gerbil-utils.nix

@@ -0,0 +1,24 @@
+{ lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
+
+gerbil-support.gerbilPackage {
+  pname = "gerbil-utils";
+  version = "unstable-2020-05-17";
+  git-version = "33ef720";
+  package = "clan";
+  gerbil = gerbil-unstable;
+  gambit-params = gambit-support.unstable-params;
+  version-path = "";
+  src = fetchFromGitHub {
+    owner = "fare";
+    repo = "gerbil-utils";
+    rev = "33ef720799ba98dc9eec773c662f070af4bac016";
+    sha256 = "0dsb97magbxzjqqfzwq4qwf7i80llv0s1dsy9nkzkvkq8drxlmqf";
+  };
+  meta = {
+    description = "Gerbil Clan: Community curated Collection of Common Utilities";
+    homepage    = "https://github.com/fare/gerbil-utils";
+    license     = lib.licenses.lgpl21;
+    platforms   = lib.platforms.unix;
+    maintainers = with lib.maintainers; [ fare ];
+  };
+}

pkgs/development/compilers/gerbil/unstable.nix

@@ -1,15 +1,15 @@
-{ stdenv, callPackage, fetchFromGitHub, gambit, gambit-unstable }:
+{ callPackage, fetchFromGitHub, gambit-unstable, gambit-support }:
 
-callPackage ./build.nix {
-  version = "unstable-2020-02-27";
-  git-version = "0.16-DEV-493-g1ffb74db";
-  #gambit = gambit-unstable;
-  gambit = gambit;
+callPackage ./build.nix rec {
+  version = "unstable-2020-05-17";
+  git-version = "0.16-1-g36a31050";
   src = fetchFromGitHub {
     owner = "vyzo";
     repo = "gerbil";
-    rev = "1ffb74db5ffd49b4bad751586cef5e619c891d41";
-    sha256 = "1szmdp8lvy5gpcwn5bpa7x383m6vywl35xa7hz9a5vs1rq4w2097";
+    rev = "36a31050f6c80e7e1a49dfae96a57b2ad0260698";
+    sha256 = "0k3fypam9qx110sjxgzxa1mdf5b631w16s9p5v37cb8ll26vqfiv";
   };
-  inherit stdenv;
+  inherit gambit-support;
+  gambit = gambit-unstable;
+  gambit-params = gambit-support.unstable-params;
 }

pkgs/top-level/all-packages.nix

@@ -8202,10 +8202,13 @@ in
 
   fpc = callPackage ../development/compilers/fpc { };
 
-  gambit = callPackage ../development/compilers/gambit { stdenv = gccStdenv; };
-  gambit-unstable = callPackage ../development/compilers/gambit/unstable.nix { stdenv = gccStdenv; };
-  gerbil = callPackage ../development/compilers/gerbil { stdenv = gccStdenv; };
-  gerbil-unstable = callPackage ../development/compilers/gerbil/unstable.nix { stdenv = gccStdenv; };
+  gambit = callPackage ../development/compilers/gambit { };
+  gambit-unstable = callPackage ../development/compilers/gambit/unstable.nix { };
+  gambit-support = callPackage ../development/compilers/gambit/gambit-support.nix { };
+  gerbil = callPackage ../development/compilers/gerbil { };
+  gerbil-unstable = callPackage ../development/compilers/gerbil/unstable.nix { };
+  gerbil-support = callPackage ../development/compilers/gerbil/gerbil-support.nix { };
+  gerbilPackages-unstable = gerbil-support.gerbilPackages-unstable; # NB: don't recurseIntoAttrs for (unstable!) libraries
 
   gccFun = callPackage (if stdenv.targetPlatform.isVc4 then ../development/compilers/gcc/6 else ../development/compilers/gcc/9);
   gcc = if stdenv.targetPlatform.isVc4 then gcc6 else gcc9;