Merge pull request #54400 from veprbl/pr/arrow_0.12.0

arrow-cpp, pythonPackages.pyarrow: 0.11.0 -> 0.12.0

pkgs/development/libraries/arrow-cpp/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, symlinkJoin, fetchurl, fetchFromGitHub, boost, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl, python, rapidjson, snappy, thrift, which, zlib, zstd }:
+{ stdenv, symlinkJoin, fetchurl, fetchFromGitHub, autoconf, boost, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl, python, rapidjson, snappy, thrift, which, zlib, zstd }:
 
 let
   parquet-testing = fetchFromGitHub {
@@ -11,27 +11,21 @@ in
 
 stdenv.mkDerivation rec {
   name = "arrow-cpp-${version}";
-  version = "0.11.0";
+  version = "0.12.0";
 
   src = fetchurl {
     url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
-    sha256 = "0pc5pqr0dbnx8s1ji102dhw9bbrsq3ml4ac3mmi2022yfyizlf0q";
+    sha256 = "163s4i2cywq95jgrxbaq48qwmww0ibkq61k1aad4w9z9vpjfgnil";
   };
 
   sourceRoot = "apache-arrow-${version}/cpp";
 
   patches = [
-    # fix ARROW-3467
-    ./double-conversion_cmake.patch
-
     # patch to fix python-test
     ./darwin.patch
-
-    # facebook/zstd#1385
-    ./zstd136.patch
     ];
 
-  nativeBuildInputs = [ cmake ];
+  nativeBuildInputs = [ cmake autoconf /* for vendored jemalloc */ ];
   buildInputs = [ boost double-conversion glog python.pkgs.python python.pkgs.numpy ];
 
   preConfigure = ''
@@ -58,6 +52,7 @@ stdenv.mkDerivation rec {
   ZSTD_HOME = zstd;
 
   cmakeFlags = [
+    "-DARROW_BUILD_TESTS=ON"
     "-DARROW_PYTHON=ON"
     "-DARROW_PARQUET=ON"
   ];

pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch

@@ -1,43 +0,0 @@
-diff --git a/CMakeLists.txt b/cpp/CMakeLists.txt
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -615,7 +615,7 @@ endif(UNIX)
- set(ARROW_LINK_LIBS)
- 
- # Libraries to link statically with libarrow.so
--set(ARROW_STATIC_LINK_LIBS double-conversion)
-+set(ARROW_STATIC_LINK_LIBS ${DOUBLE_CONVERSION_TARGET})
- 
- if (ARROW_WITH_BROTLI)
-   SET(ARROW_STATIC_LINK_LIBS
-@@ -694,7 +694,7 @@ else ()
-   set(ARROW_MIN_TEST_LIBS
-     arrow_shared
-     ${ARROW_LINK_LIBS}
--    double-conversion
-+    ${DOUBLE_CONVERSION_TARGET}
-     ${BOOST_SYSTEM_LIBRARY}
-     ${BOOST_FILESYSTEM_LIBRARY}
-     ${BOOST_REGEX_LIBRARY}
-diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
---- a/cmake_modules/ThirdpartyToolchain.cmake
-+++ b/cmake_modules/ThirdpartyToolchain.cmake
-@@ -469,14 +469,16 @@ if("${DOUBLE_CONVERSION_HOME}" STREQUAL "")
-   set(DOUBLE_CONVERSION_VENDORED 1)
- else()
-   find_package(double-conversion REQUIRED)
-+  set(DOUBLE_CONVERSION_TARGET double-conversion::double-conversion)
-   set(DOUBLE_CONVERSION_VENDORED 0)
- endif()
- 
- include_directories(SYSTEM ${DOUBLE_CONVERSION_INCLUDE_DIR})
--ADD_THIRDPARTY_LIB(double-conversion
--  STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
- 
- if (DOUBLE_CONVERSION_VENDORED)
-+  ADD_THIRDPARTY_LIB(double-conversion
-+    STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
-+  set(DOUBLE_CONVERSION_TARGET double-conversion)
-   add_dependencies(arrow_dependencies double-conversion_ep)
- endif()
- 

pkgs/development/libraries/arrow-cpp/zstd136.patch

@@ -1,17 +0,0 @@
---- a/src/arrow/util/compression_zstd.cc
-+++ b/src/arrow/util/compression_zstd.cc
-@@ -35,8 +35,13 @@ namespace util {
- 
- Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-                              uint8_t* output_buffer) {
-+  void *safe_output_buffer = static_cast<void*>(output_buffer);
-+  int dummy {};
-+  if ((output_len == 0) && (output_buffer == NULL)) {
-+    safe_output_buffer = static_cast<void*>(&dummy);
-+  }
-   int64_t decompressed_size =
--      ZSTD_decompress(output_buffer, static_cast<size_t>(output_len), input,
-+      ZSTD_decompress(safe_output_buffer, static_cast<size_t>(output_len), input,
-                       static_cast<size_t>(input_len));
-   if (decompressed_size != output_len) {
-     return Status::IOError("Corrupt ZSTD compressed data.");

pkgs/development/python-modules/pyarrow/default.nix

@@ -1,4 +1,4 @@
-{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pkgconfig, setuptools_scm, six }:
+{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, hypothesis, numpy, pandas, pytest, pkgconfig, setuptools_scm, six }:
 
 let
   _arrow-cpp = arrow-cpp.override { inherit python; };
@@ -13,10 +13,15 @@ buildPythonPackage rec {
 
   nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ];
   propagatedBuildInputs = [ numpy six ] ++ lib.optionals (!isPy3k) [ futures ];
-  checkInputs = [ pandas pytest ];
+  checkInputs = [ hypothesis pandas pytest ];
 
   PYARROW_BUILD_TYPE = "release";
-  PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib";
+  PYARROW_CMAKE_OPTIONS = [
+    "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib"
+
+    # for some reason cmake won't set -std=c++11 for clang
+    "-DPYARROW_CXXFLAGS=-std=c++11"
+  ];
 
   preCheck = ''
     rm pyarrow/tests/test_jvm.py