foundationdb: init 6.1.6pre4898_26fbbbf, cmake build

This adds a new build of FoundationDB 6.1, using the new, much improved
with CMake build system with fewer patches and rough edges.

Signed-off-by: Austin Seipp <aseipp@pobox.com>

pkgs/servers/foundationdb/cmake.nix

@@ -0,0 +1,129 @@
+# This builder is for FoundationDB CMake build system.
+
+{ lib, fetchurl, fetchpatch, fetchFromGitHub
+, cmake, ninja, boost, python3, openjdk, mono, libressl
+
+, gccStdenv, llvmPackages
+, useClang ? true
+, ...
+}:
+
+let
+  stdenv = if useClang then llvmPackages.libcxxStdenv else gccStdenv;
+
+  tests = with builtins;
+    builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt);
+
+  makeFdb =
+    { version
+    , branch
+    , sha256
+    , rev ? "refs/tags/${version}"
+    , officialRelease ? true
+    , patches ? []
+    }: stdenv.mkDerivation rec {
+        name = "foundationdb-${version}";
+        inherit version;
+
+        src = fetchFromGitHub {
+          owner = "apple";
+          repo  = "foundationdb";
+          inherit rev sha256;
+        };
+
+        buildInputs = [ libressl boost ];
+        nativeBuildInputs = [ cmake ninja python3 openjdk mono ]
+          ++ lib.optional useClang [ llvmPackages.lld ];
+
+        separateDebugInfo = true;
+        enableParallelBuilding = true;
+        dontFixCmake = true;
+
+        cmakeFlags =
+          [ "-DCMAKE_BUILD_TYPE=Release"
+            "-DLIBRESSL_USE_STATIC_LIBS=FALSE"
+
+            # CMake can't find these easily for some reason?
+            "-DLIBRESSL_INCLUDE_DIR=${libressl.dev}"
+            "-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so"
+            "-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so"
+            "-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so"
+
+            # LTO brings up overall build time, but results in much smaller
+            # binaries for all users and the cache.
+            #
+            # TODO FIXME: bugs :(
+            (lib.optionalString (!useClang) "-DUSE_LTO=ON")
+
+            # Gold helps alleviate the link time, especially when LTO is
+            # enabled. But even then, it still takes a majority of the time.
+            # Same with LLD when Clang is available.
+            (lib.optionalString useClang    "-DUSE_LD=LLD")
+            (lib.optionalString (!useClang) "-DUSE_LD=GOLD")
+          ]
+          ++ lib.optional officialRelease [ "FDB_RELEASE=1" ];
+
+        inherit patches;
+        postPatch = ''
+          for x in bindings/c/CMakeLists.txt fdbserver/CMakeLists.txt fdbmonitor/CMakeLists.txt fdbbackup/CMakeLists.txt fdbcli/CMakeLists.txt; do 
+            substituteInPlace $x --replace 'fdb_install' 'install'
+          done
+        '';
+
+        # the install phase for cmake is pretty wonky right now since it's not designed to
+        # coherently install packages as most linux distros expect -- it's designed to build
+        # packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to
+        # cmake upstream to fix this, and if we do, i think most of this can go away.
+        postInstall = ''
+          mv $out/sbin/fdbserver $out/bin/fdbserver
+          rm -rf \
+            $out/lib/systemd $out/Library $out/usr $out/sbin \
+            $out/var $out/log $out/etc
+
+          mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor
+
+          rm -rf $out/lib/foundationdb/
+          mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent
+
+          mkdir $out/include/foundationdb && \
+            mv $out/include/*.h $out/include/*.options $out/include/foundationdb
+
+          # move results into multi outputs
+          mkdir -p $dev $lib
+          mv $out/include $dev/include
+          mv $out/lib $lib/lib
+
+          # python bindings
+          # NB: use the original setup.py.in, so we can substitute VERSION correctly
+          cp ../LICENSE ./bindings/python
+          substitute ../bindings/python/setup.py.in ./bindings/python/setup.py \
+            --replace 'VERSION' "${version}"
+          rm -f ./bindings/python/setup.py.* ./bindings/python/CMakeLists.txt
+          rm -f ./bindings/python/fdb/*.pth # remove useless files
+          rm -f ./bindings/python/*.rst ./bindings/python/*.mk
+
+          cp -R ./bindings/python/                          tmp-pythonsrc/
+          tar -zcf $pythonsrc --transform s/tmp-pythonsrc/python-foundationdb/ ./tmp-pythonsrc/
+
+          # java bindings
+          mkdir -p $lib/share/java
+          mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar
+
+          # include the tests
+          mkdir -p $out/share/test
+          (cd ../tests && for x in ${tests}; do
+            cp --parents $x $out/share/test
+          done)
+        '';
+
+        outputs = [ "out" "dev" "lib" "pythonsrc" ];
+
+        meta = with stdenv.lib; {
+          description = "Open source, distributed, transactional key-value store";
+          homepage    = https://www.foundationdb.org;
+          license     = licenses.asl20;
+          platforms   = [ "x86_64-linux" ];
+          maintainers = with maintainers; [ thoughtpolice ];
+       };
+    };
+in makeFdb

pkgs/servers/foundationdb/default.nix

@@ -1,12 +1,16 @@
-{ stdenv49
+{ stdenv, stdenv49, gcc9Stdenv, llvmPackages_8
 , lib, fetchurl, fetchpatch, fetchFromGitHub
 
-, which, findutils, m4, gawk
-, python, openjdk, mono, libressl
+, cmake, ninja, which, findutils, m4, gawk
+, python, python3, openjdk, mono, libressl, boost
 }@args:
 
 let
   vsmakeBuild = import ./vsmake.nix args;
+  cmakeBuild = import ./cmake.nix (args // {
+    gccStdenv    = gcc9Stdenv;
+    llvmPackages = llvmPackages_8;
+  });
 
   python3-six-patch = fetchpatch {
     name   = "update-python-six.patch";
@@ -21,6 +25,10 @@ let
   };
 
 in with builtins; {
+
+  # Older versions use the bespoke 'vsmake' build system
+  # ------------------------------------------------------
+
   foundationdb51 = vsmakeBuild rec {
     version = "5.1.7";
     branch  = "release-5.1";
@@ -56,4 +64,21 @@ in with builtins; {
       ./patches/ldflags-6.0.patch
     ];
   };
+
+  # 6.1 and later versions should always use CMake
+  # ------------------------------------------------------
+
+  foundationdb61 = cmakeBuild rec {
+    version = "6.1.6pre4898_${substring 0 7 rev}";
+    branch  = "release-6.1";
+    rev     = "26fbbbf798971b2b9ecb882a8af766fa36734f53";
+    sha256  = "1q1a1j8h0qlh67khcds0dg416myvjbp6gfm6s4sk8d60zfzny7wb";
+    officialRelease = false;
+
+    patches = [
+      ./patches/clang-libcxx.patch
+      ./patches/suppress-clang-warnings.patch
+    ];
+  };
+
 }

pkgs/servers/foundationdb/patches/clang-libcxx.patch

@@ -0,0 +1,52 @@
+commit 7ed4745a092a203f92fc37ab5894e92117db0c94
+Author: Austin Seipp <aseipp@pobox.com>
+Date:   Sat May 4 15:23:35 2019 -0500
+
+    flow: fix a build failure with Clang/libcxx on Linux
+    
+    11bd7d7da introduced a hack on Linux to work around a missing symbol in
+    libstdc++'s _pic library on Ubuntu. Unfortunately, this causes the build
+    to fail when using Clang, as it doesn't believe this symbol is part of
+    its headers in c++11 mode.
+    
+    Unfortunately there's no good way to distinguish libcxx from libstdc++
+    with the preprocessor, so we merely gate it by only checking for clang,
+    iff we are on Linux.
+    
+    With this change, Clang 8.x can build FoundationDB on Linux using libcxx
+    as the standard C++ library.
+    
+    Signed-off-by: Austin Seipp <aseipp@pobox.com>
+
+diff --git a/flow/Platform.cpp b/flow/Platform.cpp
+index 3d3f1ac0..9f21dfd4 100644
+--- a/flow/Platform.cpp
++++ b/flow/Platform.cpp
+@@ -2841,13 +2841,26 @@ void setupSlowTaskProfiler() {
+ #endif
+ }
+ 
+-#ifdef __linux__
++#if defined(__linux__) && !defined(__clang__)
+ // There's no good place to put this, so it's here.
+ // Ubuntu's packaging of libstdc++_pic offers different symbols than libstdc++.  Go figure.
+ // Notably, it's missing a definition of std::istream::ignore(long), which causes compilation errors
+ // in the bindings.  Thus, we provide weak versions of their definitions, so that if the
+ // linked-against libstdc++ is missing their definitions, we'll be able to use the provided
+ // ignore(long, int) version.
++//
++// Note that this hack is DISABLED when we use Clang. It is only needed when we statically link
++// to the _pic libraries, but only official FDB Linux binaries are built this way using GCC. If we
++// don't use the _pic libraries, then this hack is entirely unneeded -- likely the case when using
++// Clang on Linux.
++//
++// Doing this allows us to use LLVM's libc++ with Clang on Linux -- otherwise, providing
++// a weak symbol definition for an internal (non-public) class member fails (due to that member
++// being non-existant on libc++.) See upstream GitHub issue #1533 for more information.
++//
++// TODO FIXME: Obliterate this when the official build environment is upgraded beyond Ubuntu 14.04.
++// (This problem should be fixed in later LTS releases.)
++
+ #include <istream>
+ namespace std {
+ typedef basic_istream<char, std::char_traits<char>> char_basic_istream;

pkgs/servers/foundationdb/patches/suppress-clang-warnings.patch

@@ -0,0 +1,34 @@
+commit 8076537a52bb026941f13f5542395aac69ef0825
+Author: Austin Seipp <aseipp@pobox.com>
+Date:   Sat May 4 17:34:51 2019 -0500
+
+    cmake: add workarounds for NixOS-specific deficiencies [NixOS]
+    
+    The NixOS debug builder hook adds '-Wa,--compress-debug-sections' to the
+    link flags (it actually adds it to the compiler flags, but the compiler
+    is used for linking, so...). This makes the compiler angry when -Werror
+    is passed, because it's unused at link-time (-Wa applies to the
+    assembler). Suppress this warning with -Wno-unused-command-line-argument
+    
+    NB: we *could* use -Wno-error=unused-command-line-argument, but that
+    still results in warnings anyway, just not fatal ones. We'd like to
+    remove them all for the sake of the build output.
+    
+    Signed-off-by: Austin Seipp <aseipp@pobox.com>
+
+diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
+index 03af9c10..7d059375 100644
+--- a/cmake/ConfigureCompiler.cmake
++++ b/cmake/ConfigureCompiler.cmake
+@@ -119,6 +119,11 @@ else()
+   else()
+     add_compile_options(-Werror)
+   endif()
++  if (CLANG)
++    # aseipp: NixOS hack
++    add_compile_options(-Wno-unused-command-line-argument)
++    add_link_options(-Wno-unused-command-line-argument)
++  endif()
+   add_compile_options($<$<BOOL:${GCC}>:-Wno-pragmas>)
+   add_compile_options(-Wno-error=format
+     -Wunused-variable

pkgs/servers/foundationdb/test-list.txt

@@ -0,0 +1,80 @@
+fast/AtomicBackupCorrectness.txt
+fast/AtomicBackupToDBCorrectness.txt
+fast/AtomicOps.txt
+fast/AtomicOpsApiCorrectness.txt
+fast/BackupCorrectness.txt
+fast/BackupCorrectnessClean.txt
+fast/BackupToDBCorrectness.txt
+fast/BackupToDBCorrectnessClean.txt
+fast/CloggedSideband.txt
+fast/ConstrainedRandomSelector.txt
+fast/CycleAndLock.txt
+fast/CycleTest.txt
+fast/FuzzApiCorrectness.txt
+fast/FuzzApiCorrectnessClean.txt
+fast/IncrementTest.txt
+fast/InventoryTestAlmostReadOnly.txt
+fast/InventoryTestSomeWrites.txt
+fast/KillRegionCycle.txt
+fast/LongStackWriteDuringRead.txt
+fast/LowLatency.txt
+fast/MemoryLifetime.txt
+fast/MoveKeysCycle.txt
+fast/RandomSelector.txt
+fast/RandomUnitTests.txt
+fast/SelectorCorrectness.txt
+fast/Sideband.txt
+fast/SidebandWithStatus.txt
+fast/SwizzledRollbackSideband.txt
+fast/SystemRebootTestCycle.txt
+fast/TaskBucketCorrectness.txt
+fast/TimeKeeperCorrectness.txt
+fast/Unreadable.txt
+fast/VersionStamp.txt
+fast/Watches.txt
+fast/WriteDuringRead.txt
+fast/WriteDuringReadClean.txt
+rare/CheckRelocation.txt
+rare/ClogUnclog.txt
+rare/CloggedCycleWithKills.txt
+rare/ConflictRangeCheck.txt
+rare/ConflictRangeRYOWCheck.txt
+rare/CycleRollbackClogged.txt
+rare/CycleWithKills.txt
+rare/FuzzTest.txt
+rare/InventoryTestHeavyWrites.txt
+rare/LargeApiCorrectness.txt
+rare/LargeApiCorrectnessStatus.txt
+rare/RYWDisable.txt
+rare/RandomReadWriteTest.txt
+rare/SwizzledLargeApiCorrectness.txt
+slow/ApiCorrectness.txt
+slow/ApiCorrectnessAtomicRestore.txt
+slow/ApiCorrectnessSwitchover.txt
+slow/ClogWithRollbacks.txt
+slow/CloggedCycleTest.txt
+slow/CloggedStorefront.txt
+slow/CommitBug.txt
+slow/ConfigureTest.txt
+slow/CycleRollbackPlain.txt
+slow/DDBalanceAndRemove.txt
+slow/DDBalanceAndRemoveStatus.txt
+slow/FastTriggeredWatches.txt
+slow/LowLatencyWithFailures.txt
+slow/MoveKeysClean.txt
+slow/MoveKeysSideband.txt
+slow/RyowCorrectness.txt
+slow/Serializability.txt
+slow/SharedBackupCorrectness.txt
+slow/SharedBackupToDBCorrectness.txt
+slow/StorefrontTest.txt
+slow/SwizzledApiCorrectness.txt
+slow/SwizzledCycleTest.txt
+slow/SwizzledDdBalance.txt
+slow/SwizzledRollbackTimeLapse.txt
+slow/SwizzledRollbackTimeLapseIncrement.txt
+slow/VersionStampBackupToDB.txt
+slow/VersionStampSwitchover.txt
+slow/WriteDuringReadAtomicRestore.txt
+slow/WriteDuringReadSwitchover.txt
+slow/ddbalance.txt

pkgs/servers/foundationdb/vsmake.nix

@@ -5,6 +5,7 @@
 
 , which, findutils, m4, gawk
 , python, openjdk, mono, libressl
+, ...
 }:
 
 let

pkgs/top-level/all-packages.nix

@@ -3026,7 +3026,9 @@ in
   inherit (fdbPackages)
     foundationdb51
     foundationdb52
-    foundationdb60;
+    foundationdb60
+    foundationdb61
+  ;
 
   foundationdb = foundationdb60;
 

pkgs/top-level/python-packages.nix

@@ -5528,6 +5528,7 @@ in {
   foundationdb51 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb51; };
   foundationdb52 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb52; };
   foundationdb60 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb60; };
+  foundationdb61 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb61; };
 
   libtorrentRasterbar = (toPythonModule (pkgs.libtorrentRasterbar.override {
     inherit python;