foundationdb: add 7.1.26

pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json

@@ -5767,6 +5767,9 @@
   "foundationdb61": [
     "setuptools"
   ],
+  "foundationdb71": [
+    "setuptools"
+  ],
   "fountains": [
     "setuptools"
   ],

pkgs/servers/foundationdb/cmake.nix

@@ -1,8 +1,8 @@
 # This builder is for FoundationDB CMake build system.
 
 { lib, fetchFromGitHub
-, cmake, ninja, boost, python3, openjdk, mono, libressl
+, cmake, ninja, python3, openjdk, mono, pkg-config
-, pkg-config
+, msgpack, toml11
 
 , gccStdenv, llvmPackages
 , useClang ? false
@@ -20,6 +20,8 @@ let
     , rev ? "refs/tags/${version}"
     , officialRelease ? true
     , patches ? []
+    , boost
+    , ssl
     }: stdenv.mkDerivation {
         pname = "foundationdb";
         inherit version;
@@ -30,7 +32,9 @@ let
           inherit rev sha256;
         };
 
-        buildInputs = [ libressl boost ];
+        buildInputs = [ ssl boost ]
+          ++ lib.optionals (lib.versionAtLeast version "7.1.0") [ msgpack toml11 ];
+
         nativeBuildInputs = [ pkg-config cmake ninja python3 openjdk mono ]
           ++ lib.optionals useClang [ llvmPackages.lld ];
 
@@ -40,12 +44,16 @@ let
         cmakeFlags =
           [ (lib.optionalString officialRelease "-DFDB_RELEASE=TRUE")
 
-            # FIXME: why can't libressl be found automatically?
+            # Disable CMake warnings for project developers.
-            "-DLIBRESSL_USE_STATIC_LIBS=FALSE"
+            "-Wno-dev"
-            "-DLIBRESSL_INCLUDE_DIR=${libressl.dev}"
+
-            "-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so"
+            # CMake Error at fdbserver/CMakeLists.txt:332 (find_library):
-            "-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so"
+            # >   Could not find lz4_STATIC_LIBRARIES using the following names: liblz4.a
-            "-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so"
+            "-DSSD_ROCKSDB_EXPERIMENTAL=FALSE"
+
+            # FoundationDB's CMake is hardcoded to pull in jemalloc as an external
+            # project at build time.
+            "-DUSE_JEMALLOC=FALSE"
 
             # LTO brings up overall build time, but results in much smaller
             # binaries for all users and the cache.
@@ -56,6 +64,18 @@ let
             # Same with LLD when Clang is available.
             (lib.optionalString useClang    "-DUSE_LD=LLD")
             (lib.optionalString (!useClang) "-DUSE_LD=GOLD")
+          ] ++ lib.optionals (lib.versionOlder version "7.0.0")
+          [ # FIXME: why can't libressl be found automatically?
+            "-DLIBRESSL_USE_STATIC_LIBS=FALSE"
+            "-DLIBRESSL_INCLUDE_DIR=${ssl.dev}"
+            "-DLIBRESSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so"
+            "-DLIBRESSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so"
+            "-DLIBRESSL_TLS_LIBRARY=${ssl.out}/lib/libtls.so"
+          ] ++ lib.optionals (lib.versionAtLeast version "7.1.0" && lib.versionOlder version "7.2.0")
+          [ # FIXME: why can't openssl be found automatically?
+            "-DOPENSSL_USE_STATIC_LIBS=FALSE"
+            "-DOPENSSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so"
+            "-DOPENSSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so"
           ];
 
         env.NIX_CFLAGS_COMPILE = toString [
@@ -69,19 +89,20 @@ let
         # coherently install packages as most linux distros expect -- it's designed to build
         # packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to
         # cmake upstream to fix this, and if we do, i think most of this can go away.
-        postInstall = ''
+        postInstall = lib.optionalString (lib.versionOlder version "7.0.0") ''
-          mv $out/sbin/fdbserver $out/bin/fdbserver
-          rm -rf \
-            $out/lib/systemd $out/Library $out/usr $out/sbin \
-            $out/var $out/log $out/etc
-
           mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor
-
-          rm -rf $out/lib/foundationdb/
           mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent
-
+          rm -rf $out/Library
+          rm -rf $out/lib/foundationdb/
           mkdir $out/include/foundationdb && \
             mv $out/include/*.h $out/include/*.options $out/include/foundationdb
+        '' + lib.optionalString (lib.versionAtLeast version "7.0.0") ''
+          mv $out/sbin/fdbmonitor $out/bin/fdbmonitor
+          mkdir $out/libexec && mv $out/usr/lib/foundationdb/backup_agent/backup_agent $out/libexec/backup_agent
+        '' + ''
+          mv $out/sbin/fdbserver $out/bin/fdbserver
+
+          rm -rf $out/etc $out/lib/foundationdb $out/lib/systemd $out/log $out/sbin $out/usr $out/var
 
           # move results into multi outputs
           mkdir -p $dev $lib
@@ -103,12 +124,6 @@ let
           # java bindings
           mkdir -p $lib/share/java
           mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar
-
-          # include the tests
-          mkdir -p $out/share/test
-          (cd ../tests && for x in ${tests}; do
-            cp --parents $x $out/share/test
-          done)
         '';
 
         outputs = [ "out" "dev" "lib" "pythonsrc" ];

pkgs/servers/foundationdb/default.nix

@@ -2,17 +2,13 @@
 , lib, fetchurl, fetchpatch, fetchFromGitHub
 
 , cmake, ninja, which, findutils, m4, gawk
-, python2, python3, openjdk, mono, libressl, boost168
+, python2, python3, openjdk, mono, libressl, openssl, boost168, boost178
-, pkg-config
+, pkg-config, msgpack, toml11
 }@args:
 
 let
   vsmakeBuild = import ./vsmake.nix args;
-  cmakeBuild = import ./cmake.nix (args // {
+  cmakeBuild = import ./cmake.nix args;
-    gccStdenv    = gccStdenv;
-    llvmPackages = llvmPackages;
-    boost        = boost168;
-  });
 
   python3-six-patch = fetchpatch {
     name   = "update-python-six.patch";
@@ -81,6 +77,8 @@ in with builtins; {
   foundationdb61 = cmakeBuild {
     version = "6.1.13";
     sha256  = "10vd694dcnh2pp91mri1m80kfbwjanhiy50c53c5ncqfa6pwvk00";
+    boost   = boost168;
+    ssl     = libressl;
 
     patches = [
       ./patches/clang-libcxx.patch
@@ -90,4 +88,17 @@ in with builtins; {
     ];
   };
 
+  foundationdb71 = cmakeBuild {
+    version = "7.1.26";
+    sha256  = "sha256-IVUFC2Z/nJAeKr/TtEiHAo+1HUeZuSZ2birwJtiYZx0=";
+    boost   = boost178;
+    ssl     = openssl;
+
+    patches = [
+      ./patches/disable-flowbench.patch
+      ./patches/don-t-run-tests-requiring-doctest.patch
+      ./patches/don-t-use-static-boost-libs.patch
+      ./patches/fix-open-with-O_CREAT.patch
+    ];
+  };
 }

pkgs/servers/foundationdb/patches/disable-flowbench.patch

@@ -0,0 +1,15 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 185b721eb..6752ff32d 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -185,7 +185,6 @@ endif()
+ add_subdirectory(fdbbackup)
+ add_subdirectory(contrib)
+ add_subdirectory(tests)
+-add_subdirectory(flowbench EXCLUDE_FROM_ALL)
+ if(WITH_PYTHON AND WITH_C_BINDING)
+   add_subdirectory(bindings)
+ endif()
+-- 
+2.38.1
+

pkgs/servers/foundationdb/patches/don-t-run-tests-requiring-doctest.patch

@@ -0,0 +1,128 @@
+From 10c502fd36df24f1fdbdeff446982ff5247ba20e Mon Sep 17 00:00:00 2001
+From: Jente Hidskes Ankarberg <jente@griffin.sh>
+Date: Thu, 9 Feb 2023 12:40:21 +0100
+Subject: [PATCH] Don't run tests requiring doctest
+
+Doctest is unconditionally pulled in as an external project, which we can't do
+---
+ bindings/c/CMakeLists.txt | 59 ---------------------------------------
+ 1 file changed, 59 deletions(-)
+
+diff --git bindings/c/CMakeLists.txt bindings/c/CMakeLists.txt
+index b1a187b99..25b626819 100644
+--- a/bindings/c/CMakeLists.txt
++++ b/bindings/c/CMakeLists.txt
+@@ -84,7 +84,6 @@ if(NOT WIN32)
+     test/mako/mako.h
+     test/mako/utils.c
+     test/mako/utils.h)
+-  add_subdirectory(test/unit/third_party)
+   find_package(Threads REQUIRED)
+   set(UNIT_TEST_SRCS
+     test/unit/unit_tests.cpp
+@@ -93,10 +92,6 @@ if(NOT WIN32)
+ 
+   set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp)
+   set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp)
+-  set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS
+-    test/unit/disconnected_timeout_tests.cpp
+-    test/unit/fdb_api.cpp
+-    test/unit/fdb_api.hpp)
+ 
+   set(API_TESTER_SRCS
+     test/apitester/fdb_c_api_tester.cpp
+@@ -128,11 +123,7 @@ if(NOT WIN32)
+     add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h)
+     add_library(fdb_c_client_memory_test OBJECT test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)
+     add_library(mako OBJECT ${MAKO_SRCS})
+-    add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp)
+-    add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
+-    add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})
+     add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
+-    add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
+     add_library(fdb_c_api_tester OBJECT ${API_TESTER_SRCS})
+   else()
+     add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
+@@ -140,11 +131,7 @@ if(NOT WIN32)
+     add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h)
+     add_executable(fdb_c_client_memory_test test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)
+     add_executable(mako ${MAKO_SRCS})
+-    add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp)
+-    add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
+-    add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS})
+     add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
+-    add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
+     add_executable(fdb_c_api_tester ${API_TESTER_SRCS})
+     strip_debug_symbols(fdb_c_performance_test)
+     strip_debug_symbols(fdb_c_ryw_benchmark)
+@@ -155,20 +142,7 @@ if(NOT WIN32)
+   target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c Threads::Threads)
+   target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c Threads::Threads)
+   target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)
+-
+-  add_dependencies(fdb_c_setup_tests doctest)
+-  add_dependencies(fdb_c_unit_tests doctest)
+-  add_dependencies(fdb_c_unit_tests_version_510 doctest)
+-  add_dependencies(disconnected_timeout_unit_tests doctest)
+-  target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
+-  target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
+-  target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})
+-  target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
+-  target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
+-  target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads fdbclient)
+-  target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)
+   target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads flow)
+-  target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)
+ 
+ if(USE_SANITIZER)
+   target_link_libraries(fdb_c_api_tester PRIVATE fdb_c toml11_target Threads::Threads fmt::fmt boost_asan)
+@@ -203,46 +177,13 @@ endif()
+     DEPENDS fdb_c
+     COMMENT "Copy libfdb_c to use as external client for test")
+   add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so)
+-  add_dependencies(fdb_c_unit_tests external_client)
+-  add_dependencies(disconnected_timeout_unit_tests external_client)
+   add_dependencies(fdb_c_api_tester external_client)
+ 
+-  add_fdbclient_test(
+-    NAME fdb_c_setup_tests
+-    COMMAND $<TARGET_FILE:fdb_c_setup_tests>)
+-  add_fdbclient_test(
+-    NAME fdb_c_unit_tests
+-    COMMAND $<TARGET_FILE:fdb_c_unit_tests>
+-            @CLUSTER_FILE@
+-            fdb)
+-  add_fdbclient_test(
+-    NAME fdb_c_unit_tests_version_510
+-    COMMAND $<TARGET_FILE:fdb_c_unit_tests_version_510>
+-            @CLUSTER_FILE@
+-            fdb)
+   add_fdbclient_test(
+     NAME trace_partial_file_suffix_test
+     COMMAND $<TARGET_FILE:trace_partial_file_suffix_test>
+             @CLUSTER_FILE@
+             fdb)
+-  add_fdbclient_test(
+-    NAME fdb_c_external_client_unit_tests
+-    COMMAND $<TARGET_FILE:fdb_c_unit_tests>
+-            @CLUSTER_FILE@
+-            fdb
+-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+-            )
+-  add_unavailable_fdbclient_test(
+-    NAME disconnected_timeout_unit_tests
+-    COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
+-            @CLUSTER_FILE@
+-            )
+-  add_unavailable_fdbclient_test(
+-    NAME disconnected_timeout_external_client_unit_tests
+-    COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
+-            @CLUSTER_FILE@
+-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+-            )
+   add_fdbclient_test(
+     NAME fdb_c_api_tests
+     DISABLE_LOG_DUMP
+-- 
+2.37.1 (Apple Git-137.1)
+

pkgs/servers/foundationdb/patches/don-t-use-static-boost-libs.patch

@@ -0,0 +1,27 @@
+From 1a217164f8086137ce175da09329745d5ea63027 Mon Sep 17 00:00:00 2001
+From: Jente Hidskes Ankarberg <jente@griffin.sh>
+Date: Tue, 7 Feb 2023 17:17:16 +0100
+Subject: Don't use static Boost libs
+
+We cannot override this in our CMake flags, hence we have to patch it in the source.
+
+---
+ cmake/CompileBoost.cmake | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git cmake/CompileBoost.cmake cmake/CompileBoost.cmake
+index 3bc47c776..62b448421 100644
+--- a/cmake/CompileBoost.cmake
++++ b/cmake/CompileBoost.cmake
+@@ -85,7 +85,7 @@ if(USE_SANITIZER)
+ endif()
+ 
+ # since boost 1.72 boost installs cmake configs. We will enforce config mode
+-set(Boost_USE_STATIC_LIBS ON)
++set(Boost_USE_STATIC_LIBS OFF)
+ 
+ # Clang and Gcc will have different name mangling to std::call_once, etc.
+ if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$")
+-- 
+2.37.1 (Apple Git-137.1)
+

pkgs/servers/foundationdb/patches/fix-open-with-O_CREAT.patch

@@ -0,0 +1,25 @@
+From 533d064ce028a7ebd0ef3a845cb69f10ca04b09f Mon Sep 17 00:00:00 2001
+From: Jente Hidskes Ankarberg <jente@griffin.sh>
+Date: Fri, 17 Feb 2023 00:07:20 +0100
+Subject: [PATCH 2/2] fix open with O_CREAT
+
+---
+ fdbbackup/FileDecoder.actor.cpp | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git fdbbackup/FileDecoder.actor.cpp fdbbackup/FileDecoder.actor.cpp
+index 71f693259..547147ea0 100644
+--- a/fdbbackup/FileDecoder.actor.cpp
++++ b/fdbbackup/FileDecoder.actor.cpp
+@@ -433,7 +433,7 @@ public:
+ 					platform::createDirectory(path);
+ 				}
+ 			}
+-			self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
++			self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600);
+ 			if (self->lfd == -1) {
+ 				TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName);
+ 				throw platform_error();
+-- 
+2.38.1
+

pkgs/top-level/all-packages.nix

@@ -7612,6 +7612,7 @@ with pkgs;
     foundationdb52
     foundationdb60
     foundationdb61
+    foundationdb71
   ;
 
   foundationdb = foundationdb61;

pkgs/top-level/python-packages.nix

@@ -3666,6 +3666,7 @@ self: super: with self; {
   foundationdb52 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb52; };
   foundationdb60 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb60; };
   foundationdb61 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb61; };
+  foundationdb71 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb71; };
 
   fountains = callPackage ../development/python-modules/fountains { };