diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index b173c90c7f3b..342e8f461b57 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -189,9 +189,9 @@ in grocy = handleTest ./grocy.nix {}; grub = handleTest ./grub.nix {}; gvisor = handleTest ./gvisor.nix {}; - hadoop.all = handleTestOn [ "x86_64-linux" ] ./hadoop/hadoop.nix {}; - hadoop.hdfs = handleTestOn [ "x86_64-linux" ] ./hadoop/hdfs.nix {}; - hadoop.yarn = handleTestOn [ "x86_64-linux" ] ./hadoop/yarn.nix {}; + hadoop.all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hadoop.nix {}; + hadoop.hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hdfs.nix {}; + hadoop.yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/yarn.nix {}; haka = handleTest ./haka.nix {}; haproxy = handleTest ./haproxy.nix {}; hardened = handleTest ./hardened.nix {}; diff --git a/pkgs/applications/networking/cluster/hadoop/default.nix b/pkgs/applications/networking/cluster/hadoop/default.nix index 0e8e652fbb30..6a48cc8ada89 100644 --- a/pkgs/applications/networking/cluster/hadoop/default.nix +++ b/pkgs/applications/networking/cluster/hadoop/default.nix @@ -1,23 +1,38 @@ -{ lib, stdenv, fetchurl, makeWrapper, autoPatchelfHook -, jdk8_headless, jdk11_headless -, bash, coreutils, which -, bzip2, cyrus_sasl , protobuf3_7, snappy, zlib, zstd +{ lib +, stdenv +, fetchurl +, makeWrapper +, autoPatchelfHook +, jdk8_headless +, jdk11_headless +, bash +, coreutils +, which +, bzip2 +, cyrus_sasl +, protobuf3_7 +, snappy +, zlib +, zstd , openssl }: with lib; +assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; + let - common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl, nativeLibs ? [ ], libPatches ? "" }: + common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "" }: stdenv.mkDerivation rec { inherit pname version jdk libPatches untarDir openssl; src = fetchurl { - url = "mirror://apache/hadoop/common/hadoop-${version}/hadoop-${version}.tar.gz"; - inherit sha256; + url = "mirror://apache/hadoop/common/hadoop-${version}/hadoop-${version}" + optionalString stdenv.isAarch64 "-aarch64" + ".tar.gz"; + sha256 = sha256.${stdenv.system}; }; + doCheck = true; nativeBuildInputs = [ makeWrapper ] - ++ optional (nativeLibs != [] || libPatches != "") [ autoPatchelfHook ]; + ++ optional (stdenv.isLinux && (nativeLibs != [ ] || libPatches != "")) [ autoPatchelfHook ]; buildInputs = [ openssl ] ++ nativeLibs; installPhase = '' @@ -51,46 +66,49 @@ let computers, each of which may be prone to failures. ''; maintainers = with maintainers; [ volth illustris ]; - platforms = [ "x86_64-linux" ]; + platforms = attrNames sha256; }; - }; in { # Different version of hadoop support different java runtime versions # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions - hadoop_3_3 = common rec { - pname = "hadoop"; - version = "3.3.1"; - sha256 = "1b3v16ihysqaxw8za1r5jlnphy8dwhivdx2d0z64309w57ihlxxd"; - untarDir = "${pname}-${version}"; - jdk = jdk11_headless; - inherit openssl; - # TODO: Package and add Intel Storage Acceleration Library - nativeLibs = [ stdenv.cc.cc.lib protobuf3_7 zlib snappy ]; - libPatches = '' - ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${untarDir}/lib/native/libsasl2.so.2 - ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${untarDir}/lib/native/ - ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${untarDir}/lib/native/ - ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${untarDir}/lib/native/ - ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/ - patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0 - ''; - }; + hadoop_3_3 = + common + (rec { + pname = "hadoop"; + version = "3.3.1"; + untarDir = "${pname}-${version}"; + sha256 = rec { + x86_64-linux = "1b3v16ihysqaxw8za1r5jlnphy8dwhivdx2d0z64309w57ihlxxd"; + x86_64-darwin = x86_64-linux; + aarch64-linux = "00ln18vpi07jq2slk3kplyhcj8ad41n0yl880q5cihilk7daclxz"; + aarch64-darwin = aarch64-linux; + }; + + inherit openssl; + nativeLibs = [ stdenv.cc.cc.lib protobuf3_7 zlib snappy ]; + libPatches = '' + ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${untarDir}/lib/native/libsasl2.so.2 + ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${untarDir}/lib/native/ + ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${untarDir}/lib/native/ + ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${untarDir}/lib/native/ + ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/ + '' + optionalString stdenv.isLinux "patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0"; + jdk = jdk11_headless; + }); hadoop_3_2 = common rec { pname = "hadoop"; version = "3.2.2"; - sha256 = "1hxq297cqvkfgz2yfdiwa3l28g44i2abv5921k2d6b4pqd33prwp"; + sha256.x86_64-linux = "1hxq297cqvkfgz2yfdiwa3l28g44i2abv5921k2d6b4pqd33prwp"; jdk = jdk8_headless; # not using native libs because of broken openssl_1_0_2 dependency # can be manually overriden - openssl = null; }; hadoop2 = common rec { pname = "hadoop"; version = "2.10.1"; - sha256 = "1w31x4bk9f2swnx8qxx0cgwfg8vbpm6cy5lvfnbbpl3rsjhmyg97"; + sha256.x86_64-linux = "1w31x4bk9f2swnx8qxx0cgwfg8vbpm6cy5lvfnbbpl3rsjhmyg97"; jdk = jdk8_headless; - openssl = null; }; }