cctools-llvm: init at 11.1.0-973.0.1

cctools-llvm is a replacement for cctools that replaces as much of cctools with equivalents from LLVM that it can reasonably do. This was motivated by wanting to reduce dependencies on cctools, which are updated infrequently by upstream.

To provide a motivating example, the version of `strip` included in cctools cannot properly strip the archives in compiler-rt in LLVM 15. Paths are left to bootstrap tools, resulting in failed requisites checks in the final stdenv build. Since `strip` needs replaced, the opportunity was taken to replace other provided they are functional replacements.

Note: This has to be done in cctools (or some equivalent) because some derivations (noteably LLVM) use the bintools of the stdenv directly instead of going through the wrapper.

The following tools from LLVM are not used in this derivation:

* LLD - not fully compatible with ld64 yet and potentially too big of a change;
* libtool - not a drop-in replacement yet because it does not support linker passthrough, which is needed by xcbuild;
* lipo - crashes when running the LLVM test suite;
* install_name_tool - fails when trying to build swift-corefoundation; and.
* randlib - not completely a drop-in replacement, so leaving it out for now.

If other incompatabilities are found, the tools can be reverted or made conditional. For example, cctools `strip` is preferred on older versions of LLVM (which lack the compiler-rt issue) or when cctools itself is a new enough version because `llvm-strip` on LLVM 11 produces files that older verions of `codesign_allocate` cannot process correctly.

One final caveat/note: Some tools are not duplicated or linked from cctools-port. The names of the tools and which ones were linked was determined based on what is provided upstream in Xcode and is installed on macOS system.

pkgs/os-specific/darwin/cctools/llvm.nix

@@ -0,0 +1,104 @@
+# Create a cctools-compatible bintools that uses equivalent tools from LLVM in place of the ones
+# from cctools when possible.
+
+{ lib, stdenv, makeWrapper, cctools-port, llvmPackages, enableManpages ? true }:
+
+let
+  cctoolsVersion = lib.getVersion cctools-port;
+  llvmVersion = llvmPackages.release_version;
+
+  # `bitcode_strip` is not available until LLVM 12.
+  useLLVMBitcodeStrip = lib.versionAtLeast llvmVersion "12";
+
+  # A compatible implementation of `otool` was not added until LLVM 13.
+  useLLVMOtool = lib.versionAtLeast llvmVersion "13";
+
+  # Older versions of `strip` cause problems for the version of `codesign_allocate` available in
+  # the version of cctools in nixpkgs. The version of `codesign_allocate` in cctools-1005.2 does
+  # not appear to have issues, but the source is not available yet (as of June 2023).
+  useLLVMStrip = lib.versionAtLeast llvmVersion "15" || lib.versionAtLeast cctoolsVersion "1005.2";
+
+  llvm_bins = [
+    "dwarfdump"
+    "nm"
+    "objdump"
+    "size"
+    "strings"
+  ]
+  ++ lib.optional useLLVMBitcodeStrip "bitcode-strip"
+  ++ lib.optional useLLVMOtool "otool"
+  ++ lib.optional useLLVMStrip "strip";
+
+  # Only include the tools that LLVM doesn’t provide and that are present normally on Darwin.
+  # The only exceptions are the following tools, which should be reevaluated when LLVM is bumped.
+  # - install_name_tool (llvm-objcopy): unrecognized linker commands when building open source CF;
+  # - libtool (llvm-libtool-darwin): not fully compatible when used with xcbuild; and
+  # - lipo (llvm-lipo): crashes when running the LLVM test suite.
+  cctools_bins = [
+    "cmpdylib"
+    "codesign_allocate"
+    "ctf_insert"
+    "install_name_tool"
+    "ld"
+    "libtool"
+    "lipo"
+    "nmedit"
+    "pagestuff"
+    "ranlib"
+    "segedit"
+    "vtool"
+  ]
+  ++ lib.optional (!useLLVMBitcodeStrip) "bitcode_strip"
+  ++ lib.optional (!useLLVMOtool) "otool"
+  ++ lib.optional (!useLLVMStrip) "strip";
+
+  inherit (stdenv.cc) targetPrefix;
+
+  linkManPages = pkg: source: target: lib.optionalString enableManpages ''
+    sourcePath=${pkg}/share/man/man1/${source}.1.gz
+    targetPath=$man/share/man/man1/${target}.1.gz
+
+    if [ -f "$sourcePath" ]; then
+      mkdir -p "$(dirname "$targetPath")"
+      ln -s "$sourcePath" "$targetPath"
+    fi
+  '';
+in
+stdenv.mkDerivation {
+  pname = "cctools-llvm";
+  version = "${llvmVersion}-${cctoolsVersion}";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  # The `man` output has to be included unconditionally because darwin.binutils expects it.
+  outputs = [ "out" "dev" "man" ];
+
+  buildCommand = ''
+    mkdir -p "$out/bin" "$man"
+    ln -s ${lib.getDev cctools-port} "$dev"
+
+    # Use the clang-integrated assembler instead of using `as` from cctools.
+    makeWrapper "${lib.getBin llvmPackages.clang-unwrapped}/bin/clang" "$out/bin/${targetPrefix}as" \
+      --add-flags "-x assembler -integrated-as -c"
+
+    ln -s "${lib.getBin llvmPackages.bintools-unwrapped}/bin/llvm-ar" "$out/bin/${targetPrefix}ar"
+    ${linkManPages llvmPackages.llvm-manpages "llvm-ar" "ar"}
+
+    for tool in ${toString llvm_bins}; do
+      cctoolsTool=''${tool/-/_}
+      ln -s "${lib.getBin llvmPackages.llvm}/bin/llvm-$tool" "$out/bin/${targetPrefix}$cctoolsTool"
+      ${linkManPages llvmPackages.llvm-manpages "llvm-$tool" "$cctoolsTool"}
+    done
+
+    for tool in ${toString cctools_bins}; do
+      ln -s "${lib.getBin cctools-port}/bin/${targetPrefix}$tool" "$out/bin/${targetPrefix}$tool"
+      ${linkManPages (lib.getMan cctools-port) "$tool" "$tool"}
+    done
+
+    ${linkManPages (lib.getMan cctools-port) "ld64" "ld64"}
+    ${lib.optionalString (!useLLVMOtool)  # The actual man page for otool in cctools is llvm-otool
+      linkManPages (lib.getMan cctools-port) "llvm-otool" "llvm-otool"}
+  '';
+
+  passthru = { inherit targetPrefix; };
+}

pkgs/stdenv/darwin/default.nix

@@ -651,7 +651,7 @@ rec {
         darwin = super.darwin.overrideScope (_: _: {
           inherit (darwin) dyld ICU Libsystem Csu libiconv rewrite-tbd;
         } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) {
-          inherit (darwin) binutils binutils-unwrapped cctools;
+          inherit (darwin) binutils binutils-unwrapped cctools-port;
         });
       } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) {
         inherit llvm;

pkgs/top-level/darwin-packages.nix

@@ -100,11 +100,17 @@ impure-cmds // appleSourcePackages // chooseLibs // {
     bintools = self.binutils-unwrapped;
   };
 
-  cctools = callPackage ../os-specific/darwin/cctools/port.nix {
+  cctools = self.cctools-port;
+
+  cctools-apple = callPackage ../os-specific/darwin/cctools/apple.nix {
     stdenv = if stdenv.isDarwin then stdenv else pkgs.libcxxStdenv;
   };
 
-  cctools-apple = callPackage ../os-specific/darwin/cctools/apple.nix {
+  cctools-llvm = callPackage ../os-specific/darwin/cctools/llvm.nix {
+    stdenv = if stdenv.isDarwin then stdenv else pkgs.libcxxStdenv;
+  };
+
+  cctools-port = callPackage ../os-specific/darwin/cctools/port.nix {
     stdenv = if stdenv.isDarwin then stdenv else pkgs.libcxxStdenv;
   };