Merge pull request #56555 from matthewbauer/wasm

Initial WebAssembly/WASI cross-compilation support
This commit is contained in:
Matthew Bauer 2019-04-23 22:44:33 -04:00 committed by GitHub
commit 7488a367af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 1678 additions and 31 deletions

View file

@ -30,6 +30,7 @@ rec {
libc =
/**/ if final.isDarwin then "libSystem"
else if final.isMinGW then "msvcrt"
else if final.isWasi then "wasilibc"
else if final.isMusl then "musl"
else if final.isUClibc then "uclibc"
else if final.isAndroid then "bionic"
@ -62,7 +63,7 @@ rec {
"netbsd" = "NetBSD";
"freebsd" = "FreeBSD";
"openbsd" = "OpenBSD";
"wasm" = "Wasm";
"wasi" = "Wasi";
}.${final.parsed.kernel.name} or null;
# uname -p
@ -114,8 +115,8 @@ rec {
then "${wine}/bin/${wine-name}"
else if final.isLinux && pkgs.stdenv.hostPlatform.isLinux
then "${qemu-user}/bin/qemu-${final.qemuArch}"
else if final.isWasm
then "${pkgs.v8}/bin/d8"
else if final.isWasi
then "${pkgs.wasmtime}/bin/wasmtime"
else throw "Don't know how to run ${final.config} executables.";
} // mapAttrs (n: v: v final.parsed) inspect.predicates

View file

@ -17,6 +17,8 @@ let
"x86_64-netbsd" "x86_64-openbsd" "x86_64-solaris"
"x86_64-windows" "i686-windows"
"wasm64-wasi" "wasm32-wasi"
];
allParsed = map parse.mkSystemFromString all;
@ -45,6 +47,7 @@ in rec {
netbsd = filterDoubles predicates.isNetBSD;
openbsd = filterDoubles predicates.isOpenBSD;
unix = filterDoubles predicates.isUnix;
wasi = filterDoubles predicates.isWasi;
windows = filterDoubles predicates.isWindows;
mesaPlatforms = ["i686-linux" "x86_64-linux" "x86_64-darwin" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "armv7a-linux" "aarch64-linux" "powerpc64le-linux"];

View file

@ -116,7 +116,7 @@ rec {
config = "aarch64-none-elf";
libc = "newlib";
};
aarch64be-embedded = {
config = "aarch64_be-none-elf";
libc = "newlib";
@ -126,7 +126,7 @@ rec {
config = "powerpc-none-eabi";
libc = "newlib";
};
ppcle-embedded = {
config = "powerpcle-none-eabi";
libc = "newlib";
@ -211,4 +211,14 @@ rec {
config = "x86_64-unknown-netbsd";
libc = "nblibc";
};
#
# WASM
#
wasi32 = {
config = "wasm32-unknown-wasi";
useLLVM = true;
};
}

View file

@ -32,6 +32,7 @@ in rec {
openbsd = [ patterns.isOpenBSD ];
unix = patterns.isUnix; # Actually a list
windows = [ patterns.isWindows ];
wasi = [ patterns.isWasi ];
inherit (lib.systems.doubles) mesaPlatforms;
}

View file

@ -43,6 +43,7 @@ rec {
isWindows = { kernel = kernels.windows; };
isCygwin = { kernel = kernels.windows; abi = abis.cygnus; };
isMinGW = { kernel = kernels.windows; abi = abis.gnu; };
isWasi = { kernel = kernels.wasi; };
isAndroid = [ { abi = abis.android; } { abi = abis.androideabi; } ];
isMusl = with abis; map (a: { abi = a; }) [ musl musleabi musleabihf ];

View file

@ -226,6 +226,7 @@ rec {
elf = {};
macho = {};
pe = {};
wasm = {};
unknown = {};
};
@ -268,6 +269,7 @@ rec {
none = { execFormat = unknown; families = { }; };
openbsd = { execFormat = elf; families = { inherit bsd; }; };
solaris = { execFormat = elf; families = { }; };
wasi = { execFormat = wasm; families = { }; };
windows = { execFormat = pe; families = { }; };
} // { # aliases
# 'darwin' is the kernel for all of them. We choose macOS by default.
@ -376,6 +378,8 @@ rec {
then { cpu = elemAt l 0; kernel = elemAt l 1; abi = elemAt l 2; }
else if (elemAt l 2 == "mingw32") # autotools breaks on -gnu for window
then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = "windows"; }
else if (elemAt l 2 == "wasi")
then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = "wasi"; }
else if hasPrefix "netbsd" (elemAt l 2)
then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; }
else if (elem (elemAt l 2) ["eabi" "eabihf" "elf"])

View file

@ -191,7 +191,8 @@ stdenv.mkDerivation {
else if targetPlatform.isAvr then "avr"
else if targetPlatform.isAlpha then "alpha"
else throw "unknown emulation for platform: ${targetPlatform.config}";
in targetPlatform.platform.bfdEmulation or (fmt + sep + arch);
in if targetPlatform.useLLVM or false then ""
else targetPlatform.platform.bfdEmulation or (fmt + sep + arch);
strictDeps = true;
depsTargetTargetPropagated = extraPackages;

View file

@ -347,6 +347,10 @@ stdenv.mkDerivation {
hardening_unsupported_flags+=" stackprotector fortify"
''
+ optionalString targetPlatform.isWasm ''
hardening_unsupported_flags+=" stackprotector fortify pie pic"
''
+ optionalString (libc != null && targetPlatform.isAvr) ''
for isa in avr5 avr3 avr4 avr6 avr25 avr31 avr35 avr51 avrxmega2 avrxmega4 avrxmega5 avrxmega6 avrxmega7 tiny-stack; do
echo "-B${getLib libc}/avr/lib/$isa" >> $out/nix-support/libc-cflags

View file

@ -19,6 +19,8 @@ stdenv.mkDerivation rec {
"-DCOMPILER_RT_BUILD_LIBFUZZER=OFF"
"-DCOMPILER_RT_BUILD_PROFILE=OFF"
"-DCOMPILER_RT_BAREMETAL_BUILD=ON"
#https://stackoverflow.com/questions/53633705/cmake-the-c-compiler-is-not-able-to-compile-a-simple-test-program
"-DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY"
"-DCMAKE_SIZEOF_VOID_P=${toString (stdenv.hostPlatform.parsed.cpu.bits / 8)}"
] ++ stdenv.lib.optionals stdenv.hostPlatform.isMusl [
"-DCOMPILER_RT_BUILD_SANITIZERS=OFF"
@ -53,7 +55,7 @@ stdenv.mkDerivation rec {
'';
# Hack around weird upsream RPATH bug
postInstall = stdenv.lib.optionalString stdenv.isDarwin ''
postInstall = stdenv.lib.optionalString (stdenv.hostPlatform.isDarwin || stdenv.hostPlatform.isWasm) ''
ln -s "$out/lib"/*/* "$out/lib"
'' + stdenv.lib.optionalString (stdenv.hostPlatform.useLLVM or false) ''
ln -s $out/lib/*/clang_rt.crtbegin-*.o $out/lib/crtbegin.o

View file

@ -97,13 +97,17 @@ let
targetLlvmLibraries.libcxx
targetLlvmLibraries.libcxxabi
targetLlvmLibraries.compiler-rt
] ++ stdenv.lib.optionals (!stdenv.targetPlatform.isWasm) [
targetLlvmLibraries.libunwind
];
extraBuildCommands = ''
echo "-target ${stdenv.targetPlatform.config}" >> $out/nix-support/cc-cflags
echo "-rtlib=compiler-rt -Wno-unused-command-line-argument" >> $out/nix-support/cc-cflags
echo "-B${targetLlvmLibraries.compiler-rt}/lib" >> $out/nix-support/cc-cflags
'' + stdenv.lib.optionalString (!stdenv.targetPlatform.isWasm) ''
echo "--unwindlib=libunwind" >> $out/nix-support/cc-cflags
'' + stdenv.lib.optionalString stdenv.targetPlatform.isWasm ''
echo "-fno-exceptions" >> $out/nix-support/cc-cflags
'' + mkExtraBuildCommands cc;
};

View file

@ -1,4 +1,5 @@
{ lib, stdenv, fetch, cmake, python, libcxxabi, fixDarwinDylibNames, version }:
{ lib, stdenv, fetch, cmake, python, libcxxabi, fixDarwinDylibNames, version
, enableShared ? true }:
stdenv.mkDerivation rec {
name = "libc++-${version}";
@ -22,7 +23,8 @@ stdenv.mkDerivation rec {
'' + lib.optionalString stdenv.hostPlatform.isMusl ''
patchShebangs utils/cat_files.py
'';
nativeBuildInputs = [ cmake ] ++ stdenv.lib.optional stdenv.hostPlatform.isMusl python;
nativeBuildInputs = [ cmake ]
++ stdenv.lib.optional (stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWasi) python;
buildInputs = [ libcxxabi ] ++ lib.optional stdenv.isDarwin fixDarwinDylibNames;
@ -30,8 +32,13 @@ stdenv.mkDerivation rec {
"-DLIBCXX_LIBCXXABI_LIB_PATH=${libcxxabi}/lib"
"-DLIBCXX_LIBCPPABI_VERSION=2"
"-DLIBCXX_CXX_ABI=libcxxabi"
] ++ stdenv.lib.optional stdenv.hostPlatform.isMusl "-DLIBCXX_HAS_MUSL_LIBC=1"
++ stdenv.lib.optional (stdenv.hostPlatform.useLLVM or false) "-DLIBCXX_USE_COMPILER_RT=ON";
] ++ stdenv.lib.optional (stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWasi) "-DLIBCXX_HAS_MUSL_LIBC=1"
++ stdenv.lib.optional (stdenv.hostPlatform.useLLVM or false) "-DLIBCXX_USE_COMPILER_RT=ON"
++ stdenv.lib.optional stdenv.hostPlatform.isWasm [
"-DLIBCXX_ENABLE_THREADS=OFF"
"-DLIBCXX_ENABLE_FILESYSTEM=OFF"
"-DLIBCXX_ENABLE_EXCEPTIONS=OFF"
] ++ stdenv.lib.optional (!enableShared) "-DLIBCXX_ENABLE_SHARED=OFF";
enableParallelBuilding = true;
@ -46,6 +53,6 @@ stdenv.mkDerivation rec {
homepage = http://libcxx.llvm.org/;
description = "A new implementation of the C++ standard library, targeting C++11";
license = with stdenv.lib.licenses; [ ncsa mit ];
platforms = stdenv.lib.platforms.unix;
platforms = stdenv.lib.platforms.all;
};
}

View file

@ -1,4 +1,5 @@
{ stdenv, cmake, fetch, libcxx, libunwind, llvm, version }:
{ stdenv, cmake, fetch, libcxx, libunwind, llvm, version
, enableShared ? true }:
stdenv.mkDerivation {
name = "libc++abi-${version}";
@ -6,13 +7,20 @@ stdenv.mkDerivation {
src = fetch "libcxxabi" "1k875f977ybdkpdnr9105wa6hccy9qvpd9xd42n75h7p56bdxmn2";
nativeBuildInputs = [ cmake ];
buildInputs = stdenv.lib.optional (!stdenv.isDarwin && !stdenv.isFreeBSD) libunwind;
buildInputs = stdenv.lib.optional (!stdenv.isDarwin && !stdenv.isFreeBSD && !stdenv.hostPlatform.isWasm) libunwind;
cmakeFlags = stdenv.lib.optionals (stdenv.hostPlatform.useLLVM or false) [
"-DLLVM_ENABLE_LIBCXX=ON"
"-DLIBCXXABI_USE_LLVM_UNWINDER=ON"
] ++ stdenv.lib.optionals stdenv.hostPlatform.isWasm [
"-DLIBCXXABI_ENABLE_THREADS=OFF"
"-DLIBCXXABI_ENABLE_EXCEPTIONS=OFF"
] ++ stdenv.lib.optionals (!enableShared) [
"-DLIBCXXABI_ENABLE_SHARED=OFF"
];
patches = [ ./libcxxabi-no-threads.patch ];
postUnpack = ''
unpackFile ${libcxx.src}
unpackFile ${llvm.src}
@ -21,6 +29,8 @@ stdenv.mkDerivation {
export TRIPLE=x86_64-apple-darwin
'' + stdenv.lib.optionalString stdenv.hostPlatform.isMusl ''
patch -p1 -d $(ls -d libcxx-*) -i ${../libcxx-0001-musl-hacks.patch}
'' + stdenv.lib.optionalString stdenv.hostPlatform.isWasm ''
patch -p1 -d $(ls -d llvm-*) -i ${./libcxxabi-wasm.patch}
'';
installPhase = if stdenv.isDarwin
@ -39,8 +49,9 @@ stdenv.mkDerivation {
else ''
install -d -m 755 $out/include $out/lib
install -m 644 lib/libc++abi.a $out/lib
install -m 644 lib/libc++abi.so.1.0 $out/lib
install -m 644 ../include/cxxabi.h $out/include
'' + stdenv.lib.optionalString enableShared ''
install -m 644 lib/libc++abi.so.1.0 $out/lib
ln -s libc++abi.so.1.0 $out/lib/libc++abi.so
ln -s libc++abi.so.1.0 $out/lib/libc++abi.so.1
'';
@ -50,6 +61,6 @@ stdenv.mkDerivation {
description = "A new implementation of low level support for a standard C++ library";
license = with stdenv.lib.licenses; [ ncsa mit ];
maintainers = with stdenv.lib.maintainers; [ vlstill ];
platforms = stdenv.lib.platforms.unix;
platforms = stdenv.lib.platforms.all;
};
}

View file

@ -0,0 +1,12 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4138acf..41b4763 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -362,6 +362,7 @@ if (NOT LIBCXXABI_ENABLE_THREADS)
" is also set to ON.")
endif()
add_definitions(-D_LIBCXXABI_HAS_NO_THREADS)
+ add_definitions(-D_LIBCPP_HAS_NO_THREADS)
endif()
if (LIBCXXABI_HAS_EXTERNAL_THREAD_API)

View file

@ -0,0 +1,16 @@
diff --git a/cmake/modules/HandleLLVMOptions.cmake b/cmake/modules/HandleLLVMOptions.cmake
index 15497d405e0..33f7f18193a 100644
--- a/cmake/modules/HandleLLVMOptions.cmake
+++ b/cmake/modules/HandleLLVMOptions.cmake
@@ -127,7 +127,10 @@ else(WIN32)
set(LLVM_HAVE_LINK_VERSION_SCRIPT 1)
endif()
else(FUCHSIA OR UNIX)
- MESSAGE(SEND_ERROR "Unable to determine platform")
+ if(${CMAKE_SYSTEM_NAME} MATCHES "Wasi")
+ else()
+ MESSAGE(SEND_ERROR "Unable to determine platform")
+ endif()
endif(FUCHSIA OR UNIX)
endif(WIN32)

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,31 @@
{ rustPlatform, fetchFromGitHub, lib, python, cmake, llvmPackages, clang }:
rustPlatform.buildRustPackage rec {
name = "wasmtime-${version}";
version = "0.1.0";
src = fetchFromGitHub {
owner = "CraneStation";
repo = "wasmtime";
rev = "07a6ca8f4e1136ecd9f4af8d1f03a01aade60407";
sha256 = "1cq6nz90kaf023mcyblca90bpvbzhq8xjq01laa28v7r50lagcn5";
fetchSubmodules = true;
};
cargoSha256 = "17k8n5xar4pvvi4prhm6c51vlim9xqwkkhysbnss299mm3fyh36h";
cargoPatches = [ ./cargo-lock.patch ];
nativeBuildInputs = [ python cmake clang ];
buildInputs = [ llvmPackages.libclang ];
LIBCLANG_PATH = "${llvmPackages.libclang}/lib";
meta = with lib; {
description = "Standalone JIT-style runtime for WebAsssembly, using Cranelift";
homepage = https://github.com/CraneStation/wasmtime;
license = licenses.asl20;
maintainers = [ maintainers.matthewbauer ];
platforms = platforms.unix;
};
}

View file

@ -1,4 +1,5 @@
{ stdenv, fetchurl, m4, cxx ? !stdenv.hostPlatform.useAndroidPrebuilt
{ stdenv, fetchurl, m4
, cxx ? !stdenv.hostPlatform.useAndroidPrebuilt && !stdenv.hostPlatform.isWasm
, buildPackages
, withStatic ? false }:

View file

@ -1,21 +1,21 @@
{ stdenv, fetchurl }:
let
rev = "b75cdc942a6172f63b34faf642b8c797239f6776";
rev = "a8d79c3130da83c7cacd6fee31b9acc53799c406";
# Don't use fetchgit as this is needed during Aarch64 bootstrapping
configGuess = fetchurl {
url = "https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=${rev}";
sha256 = "1bb8z1wzjs81p9qrvji4bc2a8zyxjinz90k8xq7sxxdp6zrmq1sv";
sha256 = "0qbq49gr2cmf4gzrjvrmpwxxgzl3vap1xm902xa8pkcqdvriq0qw";
};
configSub = fetchurl {
url = "https://git.savannah.gnu.org/cgit/config.git/plain/config.sub?id=${rev}";
sha256 = "00dn5i2cp4iqap5vr368r5ifrgcjfq5pr97i4dkkdbha1han5hsc";
sha256 = "0i699axqfkxk9mgv1hlms5r44pf0s642yz75ajjjpwzhw4d5pnv4";
};
in
stdenv.mkDerivation rec {
name = "gnu-config-${version}";
version = "2016-12-31";
version = "2019-04-15";
buildCommand = ''
mkdir -p $out

View file

@ -0,0 +1,29 @@
{ stdenv, fetchFromGitHub, lib }:
stdenv.mkDerivation {
name = "wasilibc-20190413";
src = fetchFromGitHub {
owner = "CraneStation";
repo = "wasi-sysroot";
rev = "079d7bda78bc0ad8f69c1594444b54786545ce57";
sha256 = "09s906bc9485wzkgibnpfh0mii7jkldzr1a6g8k7ch0si8rshi5r";
};
makeFlags = [
"WASM_CC=${stdenv.cc.targetPrefix}cc"
"WASM_NM=${stdenv.cc.targetPrefix}nm"
"WASM_AR=${stdenv.cc.targetPrefix}ar"
"INSTALL_DIR=${placeholder "out"}"
];
postInstall = ''
mv $out/lib/*/* $out/lib
ln -s $out/share/wasm32-wasi/undefined-symbols.txt $out/lib/wasi.imports
'';
meta = {
description = "WASI libc implementation for WebAssembly";
homepage = "https://wasi.dev";
platforms = lib.platforms.wasi;
maintainers = [ lib.maintainers.matthewbauer ];
};
}

View file

@ -37,7 +37,8 @@ in lib.init bootStages ++ [
# Run Packages
(buildPackages: {
inherit config;
overlays = overlays ++ crossOverlays;
overlays = overlays ++ crossOverlays
++ (if crossSystem.isWasm then [(import ../../top-level/static.nix)] else []);
selfBuild = false;
stdenv = buildPackages.stdenv.override (old: rec {
buildPlatform = localSystem;
@ -63,7 +64,7 @@ in lib.init bootStages ++ [
(hostPlatform.isLinux && !buildPlatform.isLinux)
[ buildPackages.patchelf ]
++ lib.optional
(let f = p: !p.isx86 || p.libc == "musl"; in f hostPlatform && !(f buildPlatform))
(let f = p: !p.isx86 || p.libc == "musl" || p.libc == "wasilibc"; in f hostPlatform && !(f buildPlatform))
buildPackages.updateAutotoolsGnuConfigScriptsHook
# without proper `file` command, libtool sometimes fails
# to recognize 64-bit DLLs

View file

@ -88,7 +88,7 @@ let
# there (yet?) so it goes here until then.
preHook = preHook+ lib.optionalString buildPlatform.isDarwin ''
export NIX_BUILD_DONT_SET_RPATH=1
'' + lib.optionalString hostPlatform.isDarwin ''
'' + lib.optionalString (hostPlatform.isDarwin || (hostPlatform.parsed.kernel.execFormat != lib.systems.parse.execFormats.elf && hostPlatform.parsed.kernel.execFormat != lib.systems.parse.execFormats.macho)) ''
export NIX_DONT_SET_RPATH=1
export NIX_NO_SELF_RPATH=1
''

View file

@ -93,8 +93,7 @@ let
};
in (lib.mapAttrs (_: mapMultiPlatformTest builtins.id) tests)
// (lib.mapAttrs' (name: test: {
name = "${name}-llvm";
value = mapMultiPlatformTest (system: system // {useLLVM = true;}) test;
}) tests)
in {
gcc = (lib.mapAttrs (_: mapMultiPlatformTest (system: system // {useLLVM = false;})) tests);
llvm = (lib.mapAttrs (_: mapMultiPlatformTest (system: system // {useLLVM = true;})) tests);
}

View file

@ -10311,10 +10311,15 @@ in
else if stdenv.targetPlatform.useiOSPrebuilt then targetPackages.darwin.iosSdkPkgs.libraries or darwin.iosSdkPkgs.libraries
else if name == "libSystem" then targetPackages.darwin.xcode
else if name == "nblibc" then targetPackages.netbsdCross.libc
else throw "Unknown libc";
else if name == "wasilibc" then targetPackages.wasilibc or wasilibc
else throw "Unknown libc ${name}";
libcCross = assert stdenv.targetPlatform != stdenv.buildPlatform; libcCrossChooser stdenv.targetPlatform.libc;
wasilibc = callPackages ../development/libraries/wasilibc {
stdenv = crossLibcStdenv;
};
# Only supported on Linux, using glibc
glibcLocales = if stdenv.hostPlatform.libc == "glibc" then callPackage ../development/libraries/glibc/locales.nix { } else null;
@ -11397,7 +11402,7 @@ in
# We also provide `libiconvReal`, which will always be a standalone libiconv,
# just in case you want it regardless of platform.
libiconv =
if (stdenv.hostPlatform.libc == "glibc" || stdenv.hostPlatform.libc == "musl")
if lib.elem stdenv.hostPlatform.libc ["glibc" "musl" "wasilibc"]
then glibcIconv (if stdenv.hostPlatform != stdenv.buildPlatform
then libcCross
else stdenv.cc.libc)
@ -23807,4 +23812,6 @@ in
stdenv = crossLibcStdenv;
};
wasmtime = callPackage ../development/interpreters/wasmtime {};
}

View file

@ -54,6 +54,13 @@ let
windows.mingw_w64_pthreads = nativePlatforms;
};
wasiCommon = {
gmp = nativePlatforms;
boehmgc = nativePlatforms;
hello = nativePlatforms;
zlib = nativePlatforms;
};
darwinCommon = {
buildPackages.binutils = darwin;
};
@ -140,6 +147,8 @@ in
android64 = mapTestOnCross lib.systems.examples.aarch64-android-prebuilt (linuxCommon // {
});
wasi32 = mapTestOnCross lib.systems.examples.wasi32 wasiCommon;
msp430 = mapTestOnCross lib.systems.examples.msp430 embedded;
avr = mapTestOnCross lib.systems.examples.avr embedded;
arm-embedded = mapTestOnCross lib.systems.examples.arm-embedded embedded;

View file

@ -148,4 +148,16 @@ in {
};
};
llvmPackages_8 = super.llvmPackages_8 // {
libraries = super.llvmPackages_8.libraries // rec {
libcxxabi = super.llvmPackages_8.libraries.libcxxabi.override {
enableShared = false;
};
libcxx = super.llvmPackages_8.libraries.libcxx.override {
enableShared = false;
inherit libcxxabi;
};
};
};
}