From b31083035992f8090f20b6f321420c1caa08c1b6 Mon Sep 17 00:00:00 2001 From: annalee <150648636+a-n-n-a-l-e-e@users.noreply.github.com> Date: Mon, 11 Mar 2024 23:02:27 +0000 Subject: [PATCH] stdenv.darwin.make-boostrap-tools: update tools and unpin LLVM11 - unpin LLVM11. fix discrepancy with freshBootstrapTools and the tools built on hydra. pinning the stdenv for the hydra build doesn't pin the tools as the included packages are able to change. - remove unused LLVM tools & libs which reduces the uncompressed and compressed file sizes by more than 1/2. compressed tarball is now 40M and uncompressed is around 200M - add @loader_path/. to dylibs that reference other libs in the archive. this is needed for libraries with re-exports. - validate shared objects with @rpath references contain the reference in lib - add a test to verify that the @loader_path/ works for libc++ as it re-exports libc++abi --- pkgs/stdenv/darwin/make-bootstrap-tools.nix | 251 ++++++++++-------- ...tools-next.sh => patch-bootstrap-tools.sh} | 7 +- 2 files changed, 149 insertions(+), 109 deletions(-) rename pkgs/stdenv/darwin/{patch-bootstrap-tools-next.sh => patch-bootstrap-tools.sh} (91%) diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index d00e6d939259..7725665ae436 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -16,30 +16,10 @@ let cross = if crossSystem != null in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap); rec { - coreutils_ = (coreutils.override (args: { - # We want coreutils without ACL support. - aclSupport = false; - # Cannot use a single binary build, or it gets dynamically linked against gmp. - singleBinary = false; - })).overrideAttrs (oa: { - # Increase header size to be able to inject extra RPATHs. Otherwise - # x86_64-darwin build fails as: - # https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv - NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names"; - }); - - cctools_ = darwin.cctools; - - # Avoid debugging larger changes for now. - bzip2_ = bzip2.override (args: { enableStatic = true; enableShared = false; }); - - # Avoid messing with libkrb5 and libnghttp2. - curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; }); - build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; - nativeBuildInputs = [ nukeReferences dumpnar ]; + nativeBuildInputs = [ dumpnar nukeReferences ]; buildCommand = let inherit (lib) @@ -47,6 +27,35 @@ rec { getDev getLib ; + + coreutils_ = (coreutils.override (args: { + # We want coreutils without ACL support. + aclSupport = false; + # Cannot use a single binary build, or it gets dynamically linked against gmp. + singleBinary = false; + })).overrideAttrs (oa: { + # Increase header size to be able to inject extra RPATHs. Otherwise + # x86_64-darwin build fails as: + # https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv + NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names"; + }); + + cctools_ = darwin.cctools; + + # Avoid messing with libkrb5 and libnghttp2. + curl_ = curlMinimal.override (args: { + gssSupport = false; + http2Support = false; + scpSupport = false; + }); + + gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: { + configureFlags = [ + "--disable-nls" + ] ++ old.configureFlags or []; + }); + + xz_ = xz.override { enableStatic = true; }; in '' mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin @@ -61,7 +70,6 @@ rec { cp -rL ${getDev darwin.Libsystem}/include $out chmod -R u+w $out/include - cp -rL ${getDev darwin.ICU}/include* $out/include cp -rL ${getDev libiconv}/include/* $out/include cp -rL ${getDev gnugrep.pcre2}/include/* $out/include mv $out/include $out/include-Libsystem @@ -81,7 +89,7 @@ rec { cp -d ${getBin gawk}/bin/awk $out/bin cp ${getBin gnutar}/bin/tar $out/bin cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip - cp ${getBin bzip2_}/bin/bzip2 $out/bin + cp ${getBin bzip2}/bin/bzip2 $out/bin ln -s bzip2 $out/bin/bunzip2 cp -d ${getBin gnumake}/bin/* $out/bin cp -d ${getBin patch}/bin/* $out/bin @@ -92,7 +100,6 @@ rec { # because I can't be bothered to make it partially static cp ${getBin curl_}/bin/curl $out/bin cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib - cp -d ${getLib libssh2}/lib/libssh*.dylib $out/lib cp -d ${getLib openssl}/lib/*.dylib $out/lib cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib @@ -103,17 +110,14 @@ rec { cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib # Copy what we need of clang - cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang* $out/bin - cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/* $out/lib + cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin + cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib + cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib - '' - # libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11 - + lib.optionalString (llvmPackages ? libcxxabi) '' - cp -d ${getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib - '' + '' - cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin - cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib + mkdir -p $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib @@ -133,7 +137,6 @@ rec { cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin - cp -d ${getLib darwin.ICU}/lib/libicu*.dylib $out/lib cp -d ${getLib zlib}/lib/libz.* $out/lib cp -d ${getLib gmpxx}/lib/libgmp*.* $out/lib cp -d ${getLib xz}/lib/liblzma*.* $out/lib @@ -145,63 +148,103 @@ rec { cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib - cp -rd ${getLib pkgs.darwin.CF}/Library $out - ${lib.optionalString stdenv.targetPlatform.isAarch64 '' - cp -rd ${getLib pkgs.darwin.libobjc}/lib/* $out/lib/ - ''} + # tools needed to unpack bootstrap archive. they should not contain any + # external references. we will process them like the other tools but + # perform some additional checks and will not pack them into the archive. + mkdir -p unpack/bin + cp ${getBin bash}/bin/bash unpack/bin + ln -s bash unpack/bin/sh + cp ${getBin coreutils_}/bin/mkdir unpack/bin + cp ${getBin gnutar_}/bin/tar unpack/bin + cp ${getBin xz_}/bin/xz unpack/bin - chmod -R u+w $out + # + # All files copied. Perform processing to update references to point into + # the archive + # - nuke-refs $out/bin/* + chmod -R u+w $out unpack + # - change nix store library paths to use @rpath/library + # - if needed add an rpath containing lib/ + # - strip executable rpathify() { - local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true - local newlib + local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true) + local lib rpath for lib in $libs; do ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" done + + case "$(dirname "$1")" in + */bin) + # Strip executables even further + ${stdenv.cc.targetPrefix}strip "$i" + rpath='@executable_path/../lib' + ;; + */lib) + # the '/.' suffix is required + rpath='@loader_path/.' + ;; + */lib/darwin) + rpath='@loader_path/..' + ;; + *) + echo unkown executable $1 >&2 + exit 1 + ;; + esac + + # if shared object contains references add an rpath to lib/ + if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then + ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1" + fi } - # Strip executables even further - for i in $out/bin/*; do + # check that linked library paths exist in $out/lib + # must be run after rpathify is performed + checkDeps() { + local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true) + local lib + for lib in $deps; do + if [[ ! -e $out/''${lib/@rpath/lib} ]]; then + echo "error: $1 missing lib for $lib" >&2 + exit 1 + fi + done + } + + for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do if [[ ! -L $i ]] && isMachO "$i"; then - chmod +w $i - ${stdenv.cc.targetPrefix}strip $i || true + rpathify "$i" + checkDeps "$i" fi done - for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do - if [[ ! -L "$i" ]]; then - rpathify $i - fi - done - - for i in $out/bin/*; do - if [[ ! -L "$i" ]] && isMachO "$i"; then - ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i - fi - done - - ${if stdenv.targetPlatform.isx86_64 then '' - rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation - '' else '' - sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \ - $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd - ''} - + nuke-refs $out/bin/* nuke-refs $out/lib/* - nuke-refs $out/lib/system/* nuke-refs $out/lib/darwin/* - ${lib.optionalString stdenv.targetPlatform.isx86_64 '' - nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation - ''} + nuke-refs $out/lib/system/* + nuke-refs unpack/bin/* mkdir $out/.pack mv $out/* $out/.pack mv $out/.pack $out/pack + # validate that tools contain no references into the archive + for tool in unpack/bin/*; do + deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true) + if [[ -n "$deps" ]]; then + printf "error: $tool is not self contained\n$deps\n" >&2 + exit 1 + fi + done + mkdir $out/on-server - dumpnar $out/pack | ${getBin xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz + cp -r unpack $out + + XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \ + --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . + dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz ''; allowedReferences = []; @@ -211,44 +254,42 @@ rec { }; }; - dist = stdenv.mkDerivation { - name = "stdenv-bootstrap-tools"; - - buildCommand = '' - mkdir -p $out/nix-support - echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products - ''; - }; + dist = runCommand "stdenv-bootstrap-tools" {} '' + mkdir -p $out/nix-support + echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products + echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products + ''; bootstrapFiles = { - tools = "${build}/pack"; + bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz"; + unpack = runCommand "unpack" { allowedReferences = []; } '' + cp -r ${build}/unpack $out + ''; }; bootstrapTools = derivation { inherit (stdenv.hostPlatform) system; name = "bootstrap-tools"; - builder = "${bootstrapFiles.tools}/bin/bash"; + builder = "${bootstrapFiles.unpack}/bin/bash"; - # This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly, - # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the - # patch-bootstrap-tools.sh for the next round of bootstrap tools. - args = [ ./patch-bootstrap-tools-next.sh ]; - - inherit (bootstrapFiles) tools; + args = [ ./patch-bootstrap-tools.sh bootstrapFiles.bootstrapTools ]; + PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ]; allowedReferences = [ "out" ]; }; - test = stdenv.mkDerivation { - name = "test"; - - realBuilder = "${bootstrapTools}/bin/bash"; - + test = derivation { + name = "test-bootstrap-tools"; + inherit (stdenv.hostPlatform) system; + builder = "${bootstrapTools}/bin/bash"; + args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ]; + PATH = lib.makeBinPath [ bootstrapTools ]; tools = bootstrapTools; + "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion; + + # Create a pure environment where we use just what's in the bootstrap tools. buildCommand = '' - # Create a pure environment where we use just what's in the bootstrap tools. - export PATH=$tools/bin ls -l mkdir $out @@ -275,16 +316,15 @@ rec { ${stdenv.cc.libc_dev}/lib/system \ libSystem-boot - substituteInPlace libSystem-boot/libSystem.B.tbd \ - --replace "/usr/lib/system/" "$PWD/libSystem-boot/system/" + sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd # End of bootstrap libSystem export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot" export CPP="clang -E $flags" - export CC="clang $flags -rpath $tools/lib" - export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib" + export CC="clang $flags" + export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1" # NOTE: These tests do a separate 'install' step (using cp), because # having clang write directly to the final location apparently will make @@ -301,22 +341,23 @@ rec { cp hello1 $out/bin/ $out/bin/hello1 - echo '#include ' >> hello2.c - echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c - $CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c - cp hello2 $out/bin/ - $out/bin/hello2 - echo '#include ' >> hello3.cc echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc $CXX -v -o hello3 hello3.cc cp hello3 $out/bin/ $out/bin/hello3 + # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked. + # using -Wl,-flat_namespace is required to generate an error + mkdir libtest/ + ln -s $tools/lib/libc++.dylib libtest/ + clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \ + --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc + tar xvf ${hello.src} cd hello-* - # stdenv bootstrap tools ship a broken libiconv.dylib https://github.com/NixOS/nixpkgs/issues/158331 - am_cv_func_iconv=no ./configure --prefix=$out + # hello configure detects -liconv is needed but doesn't add to the link step + LDFLAGS=-liconv ./configure --prefix=$out make make install $out/bin/hello diff --git a/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh b/pkgs/stdenv/darwin/patch-bootstrap-tools.sh similarity index 91% rename from pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh rename to pkgs/stdenv/darwin/patch-bootstrap-tools.sh index a5b9edff7cd5..555c340a7e75 100644 --- a/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh +++ b/pkgs/stdenv/darwin/patch-bootstrap-tools.sh @@ -1,9 +1,8 @@ set -euo pipefail -export PATH=$tools/bin - -cp -R $tools $out -chmod -R u+w $out +echo Unpacking the bootstrap tools... >&2 +mkdir $out +tar xf "$1" -C $out updateInstallName() { local path="$1"