stdenv.darwin.make-boostrap-tools: update tools and unpin LLVM11
- unpin LLVM11. fix discrepancy with freshBootstrapTools and the tools built on hydra. pinning the stdenv for the hydra build doesn't pin the tools as the included packages are able to change. - remove unused LLVM tools & libs which reduces the uncompressed and compressed file sizes by more than 1/2. compressed tarball is now 40M and uncompressed is around 200M - add @loader_path/. to dylibs that reference other libs in the archive. this is needed for libraries with re-exports. - validate shared objects with @rpath references contain the reference in lib - add a test to verify that the @loader_path/ works for libc++ as it re-exports libc++abi
This commit is contained in:
annalee
parent
dcc63e54a4
commit
b310830359
2 changed files with 149 additions and 109 deletions
|
|
@ -16,30 +16,10 @@ let cross = if crossSystem != null
|
|||
in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap);
|
||||
|
||||
rec {
|
||||
coreutils_ = (coreutils.override (args: {
|
||||
# We want coreutils without ACL support.
|
||||
aclSupport = false;
|
||||
# Cannot use a single binary build, or it gets dynamically linked against gmp.
|
||||
singleBinary = false;
|
||||
})).overrideAttrs (oa: {
|
||||
# Increase header size to be able to inject extra RPATHs. Otherwise
|
||||
# x86_64-darwin build fails as:
|
||||
# https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv
|
||||
NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names";
|
||||
});
|
||||
|
||||
cctools_ = darwin.cctools;
|
||||
|
||||
# Avoid debugging larger changes for now.
|
||||
bzip2_ = bzip2.override (args: { enableStatic = true; enableShared = false; });
|
||||
|
||||
# Avoid messing with libkrb5 and libnghttp2.
|
||||
curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; });
|
||||
|
||||
build = stdenv.mkDerivation {
|
||||
name = "stdenv-bootstrap-tools";
|
||||
|
||||
nativeBuildInputs = [ nukeReferences dumpnar ];
|
||||
nativeBuildInputs = [ dumpnar nukeReferences ];
|
||||
|
||||
buildCommand = let
|
||||
inherit (lib)
|
||||
|
|
@ -47,6 +27,35 @@ rec {
|
|||
getDev
|
||||
getLib
|
||||
;
|
||||
|
||||
coreutils_ = (coreutils.override (args: {
|
||||
# We want coreutils without ACL support.
|
||||
aclSupport = false;
|
||||
# Cannot use a single binary build, or it gets dynamically linked against gmp.
|
||||
singleBinary = false;
|
||||
})).overrideAttrs (oa: {
|
||||
# Increase header size to be able to inject extra RPATHs. Otherwise
|
||||
# x86_64-darwin build fails as:
|
||||
# https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv
|
||||
NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names";
|
||||
});
|
||||
|
||||
cctools_ = darwin.cctools;
|
||||
|
||||
# Avoid messing with libkrb5 and libnghttp2.
|
||||
curl_ = curlMinimal.override (args: {
|
||||
gssSupport = false;
|
||||
http2Support = false;
|
||||
scpSupport = false;
|
||||
});
|
||||
|
||||
gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: {
|
||||
configureFlags = [
|
||||
"--disable-nls"
|
||||
] ++ old.configureFlags or [];
|
||||
});
|
||||
|
||||
xz_ = xz.override { enableStatic = true; };
|
||||
in
|
||||
''
|
||||
mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin
|
||||
|
|
@ -61,7 +70,6 @@ rec {
|
|||
|
||||
cp -rL ${getDev darwin.Libsystem}/include $out
|
||||
chmod -R u+w $out/include
|
||||
cp -rL ${getDev darwin.ICU}/include* $out/include
|
||||
cp -rL ${getDev libiconv}/include/* $out/include
|
||||
cp -rL ${getDev gnugrep.pcre2}/include/* $out/include
|
||||
mv $out/include $out/include-Libsystem
|
||||
|
|
@ -81,7 +89,7 @@ rec {
|
|||
cp -d ${getBin gawk}/bin/awk $out/bin
|
||||
cp ${getBin gnutar}/bin/tar $out/bin
|
||||
cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip
|
||||
cp ${getBin bzip2_}/bin/bzip2 $out/bin
|
||||
cp ${getBin bzip2}/bin/bzip2 $out/bin
|
||||
ln -s bzip2 $out/bin/bunzip2
|
||||
cp -d ${getBin gnumake}/bin/* $out/bin
|
||||
cp -d ${getBin patch}/bin/* $out/bin
|
||||
|
|
@ -92,7 +100,6 @@ rec {
|
|||
# because I can't be bothered to make it partially static
|
||||
cp ${getBin curl_}/bin/curl $out/bin
|
||||
cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib
|
||||
cp -d ${getLib libssh2}/lib/libssh*.dylib $out/lib
|
||||
cp -d ${getLib openssl}/lib/*.dylib $out/lib
|
||||
|
||||
cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib
|
||||
|
|
@ -103,17 +110,14 @@ rec {
|
|||
cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib
|
||||
|
||||
# Copy what we need of clang
|
||||
cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang* $out/bin
|
||||
cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/* $out/lib
|
||||
cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin
|
||||
cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib
|
||||
cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib
|
||||
|
||||
cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib
|
||||
''
|
||||
# libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11
|
||||
+ lib.optionalString (llvmPackages ? libcxxabi) ''
|
||||
cp -d ${getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib
|
||||
'' + ''
|
||||
cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin
|
||||
cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib
|
||||
mkdir -p $out/lib/darwin
|
||||
cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin
|
||||
cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib
|
||||
cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib
|
||||
cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib
|
||||
|
||||
|
|
@ -133,7 +137,6 @@ rec {
|
|||
cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin
|
||||
cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin
|
||||
|
||||
cp -d ${getLib darwin.ICU}/lib/libicu*.dylib $out/lib
|
||||
cp -d ${getLib zlib}/lib/libz.* $out/lib
|
||||
cp -d ${getLib gmpxx}/lib/libgmp*.* $out/lib
|
||||
cp -d ${getLib xz}/lib/liblzma*.* $out/lib
|
||||
|
|
@ -145,63 +148,103 @@ rec {
|
|||
|
||||
cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib
|
||||
|
||||
cp -rd ${getLib pkgs.darwin.CF}/Library $out
|
||||
${lib.optionalString stdenv.targetPlatform.isAarch64 ''
|
||||
cp -rd ${getLib pkgs.darwin.libobjc}/lib/* $out/lib/
|
||||
''}
|
||||
# tools needed to unpack bootstrap archive. they should not contain any
|
||||
# external references. we will process them like the other tools but
|
||||
# perform some additional checks and will not pack them into the archive.
|
||||
mkdir -p unpack/bin
|
||||
cp ${getBin bash}/bin/bash unpack/bin
|
||||
ln -s bash unpack/bin/sh
|
||||
cp ${getBin coreutils_}/bin/mkdir unpack/bin
|
||||
cp ${getBin gnutar_}/bin/tar unpack/bin
|
||||
cp ${getBin xz_}/bin/xz unpack/bin
|
||||
|
||||
chmod -R u+w $out
|
||||
#
|
||||
# All files copied. Perform processing to update references to point into
|
||||
# the archive
|
||||
#
|
||||
|
||||
nuke-refs $out/bin/*
|
||||
chmod -R u+w $out unpack
|
||||
|
||||
# - change nix store library paths to use @rpath/library
|
||||
# - if needed add an rpath containing lib/
|
||||
# - strip executable
|
||||
rpathify() {
|
||||
local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true
|
||||
local newlib
|
||||
local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true)
|
||||
local lib rpath
|
||||
for lib in $libs; do
|
||||
${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1"
|
||||
done
|
||||
|
||||
case "$(dirname "$1")" in
|
||||
*/bin)
|
||||
# Strip executables even further
|
||||
${stdenv.cc.targetPrefix}strip "$i"
|
||||
rpath='@executable_path/../lib'
|
||||
;;
|
||||
*/lib)
|
||||
# the '/.' suffix is required
|
||||
rpath='@loader_path/.'
|
||||
;;
|
||||
*/lib/darwin)
|
||||
rpath='@loader_path/..'
|
||||
;;
|
||||
*)
|
||||
echo unkown executable $1 >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# if shared object contains references add an rpath to lib/
|
||||
if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then
|
||||
${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Strip executables even further
|
||||
for i in $out/bin/*; do
|
||||
# check that linked library paths exist in $out/lib
|
||||
# must be run after rpathify is performed
|
||||
checkDeps() {
|
||||
local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true)
|
||||
local lib
|
||||
for lib in $deps; do
|
||||
if [[ ! -e $out/''${lib/@rpath/lib} ]]; then
|
||||
echo "error: $1 missing lib for $lib" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do
|
||||
if [[ ! -L $i ]] && isMachO "$i"; then
|
||||
chmod +w $i
|
||||
${stdenv.cc.targetPrefix}strip $i || true
|
||||
rpathify "$i"
|
||||
checkDeps "$i"
|
||||
fi
|
||||
done
|
||||
|
||||
for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do
|
||||
if [[ ! -L "$i" ]]; then
|
||||
rpathify $i
|
||||
fi
|
||||
done
|
||||
|
||||
for i in $out/bin/*; do
|
||||
if [[ ! -L "$i" ]] && isMachO "$i"; then
|
||||
${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i
|
||||
fi
|
||||
done
|
||||
|
||||
${if stdenv.targetPlatform.isx86_64 then ''
|
||||
rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation
|
||||
'' else ''
|
||||
sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \
|
||||
$out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd
|
||||
''}
|
||||
|
||||
nuke-refs $out/bin/*
|
||||
nuke-refs $out/lib/*
|
||||
nuke-refs $out/lib/system/*
|
||||
nuke-refs $out/lib/darwin/*
|
||||
${lib.optionalString stdenv.targetPlatform.isx86_64 ''
|
||||
nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation
|
||||
''}
|
||||
nuke-refs $out/lib/system/*
|
||||
nuke-refs unpack/bin/*
|
||||
|
||||
mkdir $out/.pack
|
||||
mv $out/* $out/.pack
|
||||
mv $out/.pack $out/pack
|
||||
|
||||
# validate that tools contain no references into the archive
|
||||
for tool in unpack/bin/*; do
|
||||
deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true)
|
||||
if [[ -n "$deps" ]]; then
|
||||
printf "error: $tool is not self contained\n$deps\n" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
mkdir $out/on-server
|
||||
dumpnar $out/pack | ${getBin xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz
|
||||
cp -r unpack $out
|
||||
|
||||
XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \
|
||||
--hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack .
|
||||
dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz
|
||||
'';
|
||||
|
||||
allowedReferences = [];
|
||||
|
|
@ -211,44 +254,42 @@ rec {
|
|||
};
|
||||
};
|
||||
|
||||
dist = stdenv.mkDerivation {
|
||||
name = "stdenv-bootstrap-tools";
|
||||
|
||||
buildCommand = ''
|
||||
mkdir -p $out/nix-support
|
||||
echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
};
|
||||
dist = runCommand "stdenv-bootstrap-tools" {} ''
|
||||
mkdir -p $out/nix-support
|
||||
echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products
|
||||
echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
|
||||
bootstrapFiles = {
|
||||
tools = "${build}/pack";
|
||||
bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz";
|
||||
unpack = runCommand "unpack" { allowedReferences = []; } ''
|
||||
cp -r ${build}/unpack $out
|
||||
'';
|
||||
};
|
||||
|
||||
bootstrapTools = derivation {
|
||||
inherit (stdenv.hostPlatform) system;
|
||||
|
||||
name = "bootstrap-tools";
|
||||
builder = "${bootstrapFiles.tools}/bin/bash";
|
||||
builder = "${bootstrapFiles.unpack}/bin/bash";
|
||||
|
||||
# This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly,
|
||||
# we can't make any changes to it due to our testing stdenv depending on it. Think of this as the
|
||||
# patch-bootstrap-tools.sh for the next round of bootstrap tools.
|
||||
args = [ ./patch-bootstrap-tools-next.sh ];
|
||||
|
||||
inherit (bootstrapFiles) tools;
|
||||
args = [ ./patch-bootstrap-tools.sh bootstrapFiles.bootstrapTools ];
|
||||
PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ];
|
||||
|
||||
allowedReferences = [ "out" ];
|
||||
};
|
||||
|
||||
test = stdenv.mkDerivation {
|
||||
name = "test";
|
||||
|
||||
realBuilder = "${bootstrapTools}/bin/bash";
|
||||
|
||||
test = derivation {
|
||||
name = "test-bootstrap-tools";
|
||||
inherit (stdenv.hostPlatform) system;
|
||||
builder = "${bootstrapTools}/bin/bash";
|
||||
args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ];
|
||||
PATH = lib.makeBinPath [ bootstrapTools ];
|
||||
tools = bootstrapTools;
|
||||
"${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion;
|
||||
|
||||
# Create a pure environment where we use just what's in the bootstrap tools.
|
||||
buildCommand = ''
|
||||
# Create a pure environment where we use just what's in the bootstrap tools.
|
||||
export PATH=$tools/bin
|
||||
|
||||
ls -l
|
||||
mkdir $out
|
||||
|
|
@ -275,16 +316,15 @@ rec {
|
|||
${stdenv.cc.libc_dev}/lib/system \
|
||||
libSystem-boot
|
||||
|
||||
substituteInPlace libSystem-boot/libSystem.B.tbd \
|
||||
--replace "/usr/lib/system/" "$PWD/libSystem-boot/system/"
|
||||
sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd
|
||||
ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd
|
||||
# End of bootstrap libSystem
|
||||
|
||||
export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot"
|
||||
|
||||
export CPP="clang -E $flags"
|
||||
export CC="clang $flags -rpath $tools/lib"
|
||||
export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib"
|
||||
export CC="clang $flags"
|
||||
export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1"
|
||||
|
||||
# NOTE: These tests do a separate 'install' step (using cp), because
|
||||
# having clang write directly to the final location apparently will make
|
||||
|
|
@ -301,22 +341,23 @@ rec {
|
|||
cp hello1 $out/bin/
|
||||
$out/bin/hello1
|
||||
|
||||
echo '#include <CoreFoundation/CoreFoundation.h>' >> hello2.c
|
||||
echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c
|
||||
$CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c
|
||||
cp hello2 $out/bin/
|
||||
$out/bin/hello2
|
||||
|
||||
echo '#include <iostream>' >> hello3.cc
|
||||
echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc
|
||||
$CXX -v -o hello3 hello3.cc
|
||||
cp hello3 $out/bin/
|
||||
$out/bin/hello3
|
||||
|
||||
# test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked.
|
||||
# using -Wl,-flat_namespace is required to generate an error
|
||||
mkdir libtest/
|
||||
ln -s $tools/lib/libc++.dylib libtest/
|
||||
clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \
|
||||
--sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc
|
||||
|
||||
tar xvf ${hello.src}
|
||||
cd hello-*
|
||||
# stdenv bootstrap tools ship a broken libiconv.dylib https://github.com/NixOS/nixpkgs/issues/158331
|
||||
am_cv_func_iconv=no ./configure --prefix=$out
|
||||
# hello configure detects -liconv is needed but doesn't add to the link step
|
||||
LDFLAGS=-liconv ./configure --prefix=$out
|
||||
make
|
||||
make install
|
||||
$out/bin/hello
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
set -euo pipefail
|
||||
|
||||
export PATH=$tools/bin
|
||||
|
||||
cp -R $tools $out
|
||||
chmod -R u+w $out
|
||||
echo Unpacking the bootstrap tools... >&2
|
||||
mkdir $out
|
||||
tar xf "$1" -C $out
|
||||
|
||||
updateInstallName() {
|
||||
local path="$1"
|
||||
Loading…
Reference in a new issue