From 6fc2ba036ab77c647ea897b2aa26d38d540c4046 Mon Sep 17 00:00:00 2001 From: Jente Hidskes Ankarberg Date: Wed, 8 Feb 2023 13:57:07 +0100 Subject: [PATCH] foundationdb: add 7.1.26 --- .../poetry2nix/overrides/build-systems.json | 3 + pkgs/servers/foundationdb/cmake.nix | 63 +++++---- pkgs/servers/foundationdb/default.nix | 25 +++- .../patches/disable-flowbench.patch | 15 ++ .../don-t-run-tests-requiring-doctest.patch | 128 ++++++++++++++++++ .../patches/don-t-use-static-boost-libs.patch | 27 ++++ .../patches/fix-open-with-O_CREAT.patch | 25 ++++ pkgs/top-level/all-packages.nix | 1 + pkgs/top-level/python-packages.nix | 1 + 9 files changed, 257 insertions(+), 31 deletions(-) create mode 100644 pkgs/servers/foundationdb/patches/disable-flowbench.patch create mode 100644 pkgs/servers/foundationdb/patches/don-t-run-tests-requiring-doctest.patch create mode 100644 pkgs/servers/foundationdb/patches/don-t-use-static-boost-libs.patch create mode 100644 pkgs/servers/foundationdb/patches/fix-open-with-O_CREAT.patch diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json index 82908ddaf332..d2f289e227fb 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json @@ -5767,6 +5767,9 @@ "foundationdb61": [ "setuptools" ], + "foundationdb71": [ + "setuptools" + ], "fountains": [ "setuptools" ], diff --git a/pkgs/servers/foundationdb/cmake.nix b/pkgs/servers/foundationdb/cmake.nix index c989f6668989..dcaa9f3e6f8e 100644 --- a/pkgs/servers/foundationdb/cmake.nix +++ b/pkgs/servers/foundationdb/cmake.nix @@ -1,8 +1,8 @@ # This builder is for FoundationDB CMake build system. { lib, fetchFromGitHub -, cmake, ninja, boost, python3, openjdk, mono, libressl -, pkg-config +, cmake, ninja, python3, openjdk, mono, pkg-config +, msgpack, toml11 , gccStdenv, llvmPackages , useClang ? false @@ -20,6 +20,8 @@ let , rev ? "refs/tags/${version}" , officialRelease ? true , patches ? [] + , boost + , ssl }: stdenv.mkDerivation { pname = "foundationdb"; inherit version; @@ -30,7 +32,9 @@ let inherit rev sha256; }; - buildInputs = [ libressl boost ]; + buildInputs = [ ssl boost ] + ++ lib.optionals (lib.versionAtLeast version "7.1.0") [ msgpack toml11 ]; + nativeBuildInputs = [ pkg-config cmake ninja python3 openjdk mono ] ++ lib.optionals useClang [ llvmPackages.lld ]; @@ -40,12 +44,16 @@ let cmakeFlags = [ (lib.optionalString officialRelease "-DFDB_RELEASE=TRUE") - # FIXME: why can't libressl be found automatically? - "-DLIBRESSL_USE_STATIC_LIBS=FALSE" - "-DLIBRESSL_INCLUDE_DIR=${libressl.dev}" - "-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so" - "-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so" - "-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so" + # Disable CMake warnings for project developers. + "-Wno-dev" + + # CMake Error at fdbserver/CMakeLists.txt:332 (find_library): + # > Could not find lz4_STATIC_LIBRARIES using the following names: liblz4.a + "-DSSD_ROCKSDB_EXPERIMENTAL=FALSE" + + # FoundationDB's CMake is hardcoded to pull in jemalloc as an external + # project at build time. + "-DUSE_JEMALLOC=FALSE" # LTO brings up overall build time, but results in much smaller # binaries for all users and the cache. @@ -56,6 +64,18 @@ let # Same with LLD when Clang is available. (lib.optionalString useClang "-DUSE_LD=LLD") (lib.optionalString (!useClang) "-DUSE_LD=GOLD") + ] ++ lib.optionals (lib.versionOlder version "7.0.0") + [ # FIXME: why can't libressl be found automatically? + "-DLIBRESSL_USE_STATIC_LIBS=FALSE" + "-DLIBRESSL_INCLUDE_DIR=${ssl.dev}" + "-DLIBRESSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so" + "-DLIBRESSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so" + "-DLIBRESSL_TLS_LIBRARY=${ssl.out}/lib/libtls.so" + ] ++ lib.optionals (lib.versionAtLeast version "7.1.0" && lib.versionOlder version "7.2.0") + [ # FIXME: why can't openssl be found automatically? + "-DOPENSSL_USE_STATIC_LIBS=FALSE" + "-DOPENSSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so" + "-DOPENSSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so" ]; env.NIX_CFLAGS_COMPILE = toString [ @@ -69,19 +89,20 @@ let # coherently install packages as most linux distros expect -- it's designed to build # packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to # cmake upstream to fix this, and if we do, i think most of this can go away. - postInstall = '' - mv $out/sbin/fdbserver $out/bin/fdbserver - rm -rf \ - $out/lib/systemd $out/Library $out/usr $out/sbin \ - $out/var $out/log $out/etc - + postInstall = lib.optionalString (lib.versionOlder version "7.0.0") '' mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor - - rm -rf $out/lib/foundationdb/ mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent - + rm -rf $out/Library + rm -rf $out/lib/foundationdb/ mkdir $out/include/foundationdb && \ mv $out/include/*.h $out/include/*.options $out/include/foundationdb + '' + lib.optionalString (lib.versionAtLeast version "7.0.0") '' + mv $out/sbin/fdbmonitor $out/bin/fdbmonitor + mkdir $out/libexec && mv $out/usr/lib/foundationdb/backup_agent/backup_agent $out/libexec/backup_agent + '' + '' + mv $out/sbin/fdbserver $out/bin/fdbserver + + rm -rf $out/etc $out/lib/foundationdb $out/lib/systemd $out/log $out/sbin $out/usr $out/var # move results into multi outputs mkdir -p $dev $lib @@ -103,12 +124,6 @@ let # java bindings mkdir -p $lib/share/java mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar - - # include the tests - mkdir -p $out/share/test - (cd ../tests && for x in ${tests}; do - cp --parents $x $out/share/test - done) ''; outputs = [ "out" "dev" "lib" "pythonsrc" ]; diff --git a/pkgs/servers/foundationdb/default.nix b/pkgs/servers/foundationdb/default.nix index 383efee182c3..9a6faac0562b 100644 --- a/pkgs/servers/foundationdb/default.nix +++ b/pkgs/servers/foundationdb/default.nix @@ -2,17 +2,13 @@ , lib, fetchurl, fetchpatch, fetchFromGitHub , cmake, ninja, which, findutils, m4, gawk -, python2, python3, openjdk, mono, libressl, boost168 -, pkg-config +, python2, python3, openjdk, mono, libressl, openssl, boost168, boost178 +, pkg-config, msgpack, toml11 }@args: let vsmakeBuild = import ./vsmake.nix args; - cmakeBuild = import ./cmake.nix (args // { - gccStdenv = gccStdenv; - llvmPackages = llvmPackages; - boost = boost168; - }); + cmakeBuild = import ./cmake.nix args; python3-six-patch = fetchpatch { name = "update-python-six.patch"; @@ -81,6 +77,8 @@ in with builtins; { foundationdb61 = cmakeBuild { version = "6.1.13"; sha256 = "10vd694dcnh2pp91mri1m80kfbwjanhiy50c53c5ncqfa6pwvk00"; + boost = boost168; + ssl = libressl; patches = [ ./patches/clang-libcxx.patch @@ -90,4 +88,17 @@ in with builtins; { ]; }; + foundationdb71 = cmakeBuild { + version = "7.1.26"; + sha256 = "sha256-IVUFC2Z/nJAeKr/TtEiHAo+1HUeZuSZ2birwJtiYZx0="; + boost = boost178; + ssl = openssl; + + patches = [ + ./patches/disable-flowbench.patch + ./patches/don-t-run-tests-requiring-doctest.patch + ./patches/don-t-use-static-boost-libs.patch + ./patches/fix-open-with-O_CREAT.patch + ]; + }; } diff --git a/pkgs/servers/foundationdb/patches/disable-flowbench.patch b/pkgs/servers/foundationdb/patches/disable-flowbench.patch new file mode 100644 index 000000000000..a604bf94f322 --- /dev/null +++ b/pkgs/servers/foundationdb/patches/disable-flowbench.patch @@ -0,0 +1,15 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 185b721eb..6752ff32d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -185,7 +185,6 @@ endif() + add_subdirectory(fdbbackup) + add_subdirectory(contrib) + add_subdirectory(tests) +-add_subdirectory(flowbench EXCLUDE_FROM_ALL) + if(WITH_PYTHON AND WITH_C_BINDING) + add_subdirectory(bindings) + endif() +-- +2.38.1 + diff --git a/pkgs/servers/foundationdb/patches/don-t-run-tests-requiring-doctest.patch b/pkgs/servers/foundationdb/patches/don-t-run-tests-requiring-doctest.patch new file mode 100644 index 000000000000..702245bd52ff --- /dev/null +++ b/pkgs/servers/foundationdb/patches/don-t-run-tests-requiring-doctest.patch @@ -0,0 +1,128 @@ +From 10c502fd36df24f1fdbdeff446982ff5247ba20e Mon Sep 17 00:00:00 2001 +From: Jente Hidskes Ankarberg +Date: Thu, 9 Feb 2023 12:40:21 +0100 +Subject: [PATCH] Don't run tests requiring doctest + +Doctest is unconditionally pulled in as an external project, which we can't do +--- + bindings/c/CMakeLists.txt | 59 --------------------------------------- + 1 file changed, 59 deletions(-) + +diff --git bindings/c/CMakeLists.txt bindings/c/CMakeLists.txt +index b1a187b99..25b626819 100644 +--- a/bindings/c/CMakeLists.txt ++++ b/bindings/c/CMakeLists.txt +@@ -84,7 +84,6 @@ if(NOT WIN32) + test/mako/mako.h + test/mako/utils.c + test/mako/utils.h) +- add_subdirectory(test/unit/third_party) + find_package(Threads REQUIRED) + set(UNIT_TEST_SRCS + test/unit/unit_tests.cpp +@@ -93,10 +92,6 @@ if(NOT WIN32) + + set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp) + set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp) +- set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS +- test/unit/disconnected_timeout_tests.cpp +- test/unit/fdb_api.cpp +- test/unit/fdb_api.hpp) + + set(API_TESTER_SRCS + test/apitester/fdb_c_api_tester.cpp +@@ -128,11 +123,7 @@ if(NOT WIN32) + add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h) + add_library(fdb_c_client_memory_test OBJECT test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp) + add_library(mako OBJECT ${MAKO_SRCS}) +- add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp) +- add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS}) +- add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS}) + add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS}) +- add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS}) + add_library(fdb_c_api_tester OBJECT ${API_TESTER_SRCS}) + else() + add_executable(fdb_c_performance_test test/performance_test.c test/test.h) +@@ -140,11 +131,7 @@ if(NOT WIN32) + add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h) + add_executable(fdb_c_client_memory_test test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp) + add_executable(mako ${MAKO_SRCS}) +- add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp) +- add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS}) +- add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS}) + add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS}) +- add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS}) + add_executable(fdb_c_api_tester ${API_TESTER_SRCS}) + strip_debug_symbols(fdb_c_performance_test) + strip_debug_symbols(fdb_c_ryw_benchmark) +@@ -155,20 +142,7 @@ if(NOT WIN32) + target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c Threads::Threads) + target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c Threads::Threads) + target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads) +- +- add_dependencies(fdb_c_setup_tests doctest) +- add_dependencies(fdb_c_unit_tests doctest) +- add_dependencies(fdb_c_unit_tests_version_510 doctest) +- add_dependencies(disconnected_timeout_unit_tests doctest) +- target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) +- target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) +- target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR}) +- target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) +- target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads) +- target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads fdbclient) +- target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads) + target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads flow) +- target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads) + + if(USE_SANITIZER) + target_link_libraries(fdb_c_api_tester PRIVATE fdb_c toml11_target Threads::Threads fmt::fmt boost_asan) +@@ -203,46 +177,13 @@ endif() + DEPENDS fdb_c + COMMENT "Copy libfdb_c to use as external client for test") + add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so) +- add_dependencies(fdb_c_unit_tests external_client) +- add_dependencies(disconnected_timeout_unit_tests external_client) + add_dependencies(fdb_c_api_tester external_client) + +- add_fdbclient_test( +- NAME fdb_c_setup_tests +- COMMAND $) +- add_fdbclient_test( +- NAME fdb_c_unit_tests +- COMMAND $ +- @CLUSTER_FILE@ +- fdb) +- add_fdbclient_test( +- NAME fdb_c_unit_tests_version_510 +- COMMAND $ +- @CLUSTER_FILE@ +- fdb) + add_fdbclient_test( + NAME trace_partial_file_suffix_test + COMMAND $ + @CLUSTER_FILE@ + fdb) +- add_fdbclient_test( +- NAME fdb_c_external_client_unit_tests +- COMMAND $ +- @CLUSTER_FILE@ +- fdb +- ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so +- ) +- add_unavailable_fdbclient_test( +- NAME disconnected_timeout_unit_tests +- COMMAND $ +- @CLUSTER_FILE@ +- ) +- add_unavailable_fdbclient_test( +- NAME disconnected_timeout_external_client_unit_tests +- COMMAND $ +- @CLUSTER_FILE@ +- ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so +- ) + add_fdbclient_test( + NAME fdb_c_api_tests + DISABLE_LOG_DUMP +-- +2.37.1 (Apple Git-137.1) + diff --git a/pkgs/servers/foundationdb/patches/don-t-use-static-boost-libs.patch b/pkgs/servers/foundationdb/patches/don-t-use-static-boost-libs.patch new file mode 100644 index 000000000000..d26df9004e86 --- /dev/null +++ b/pkgs/servers/foundationdb/patches/don-t-use-static-boost-libs.patch @@ -0,0 +1,27 @@ +From 1a217164f8086137ce175da09329745d5ea63027 Mon Sep 17 00:00:00 2001 +From: Jente Hidskes Ankarberg +Date: Tue, 7 Feb 2023 17:17:16 +0100 +Subject: Don't use static Boost libs + +We cannot override this in our CMake flags, hence we have to patch it in the source. + +--- + cmake/CompileBoost.cmake | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git cmake/CompileBoost.cmake cmake/CompileBoost.cmake +index 3bc47c776..62b448421 100644 +--- a/cmake/CompileBoost.cmake ++++ b/cmake/CompileBoost.cmake +@@ -85,7 +85,7 @@ if(USE_SANITIZER) + endif() + + # since boost 1.72 boost installs cmake configs. We will enforce config mode +-set(Boost_USE_STATIC_LIBS ON) ++set(Boost_USE_STATIC_LIBS OFF) + + # Clang and Gcc will have different name mangling to std::call_once, etc. + if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$") +-- +2.37.1 (Apple Git-137.1) + diff --git a/pkgs/servers/foundationdb/patches/fix-open-with-O_CREAT.patch b/pkgs/servers/foundationdb/patches/fix-open-with-O_CREAT.patch new file mode 100644 index 000000000000..23a642cb28fd --- /dev/null +++ b/pkgs/servers/foundationdb/patches/fix-open-with-O_CREAT.patch @@ -0,0 +1,25 @@ +From 533d064ce028a7ebd0ef3a845cb69f10ca04b09f Mon Sep 17 00:00:00 2001 +From: Jente Hidskes Ankarberg +Date: Fri, 17 Feb 2023 00:07:20 +0100 +Subject: [PATCH 2/2] fix open with O_CREAT + +--- + fdbbackup/FileDecoder.actor.cpp | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git fdbbackup/FileDecoder.actor.cpp fdbbackup/FileDecoder.actor.cpp +index 71f693259..547147ea0 100644 +--- a/fdbbackup/FileDecoder.actor.cpp ++++ b/fdbbackup/FileDecoder.actor.cpp +@@ -433,7 +433,7 @@ public: + platform::createDirectory(path); + } + } +- self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC); ++ self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (self->lfd == -1) { + TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName); + throw platform_error(); +-- +2.38.1 + diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index b2b0d8f8ad18..e26190b07cbe 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7612,6 +7612,7 @@ with pkgs; foundationdb52 foundationdb60 foundationdb61 + foundationdb71 ; foundationdb = foundationdb61; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 12a65e9a4c9c..ff461e76fb57 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -3666,6 +3666,7 @@ self: super: with self; { foundationdb52 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb52; }; foundationdb60 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb60; }; foundationdb61 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb61; }; + foundationdb71 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb71; }; fountains = callPackage ../development/python-modules/fountains { };