build: Use proton sdk docker image for building.

Makefile

@@ -42,7 +42,7 @@ endif
 
 protonsdk_version := 0.20210126.1-0
 CONFIGURE_CMD := ../proton/configure.sh \
-	--steam-runtime-image=steam-proton-dev \
+	--steam-runtime-image=registry.gitlab.steamos.cloud/proton/soldier/sdk:$(protonsdk_version) \
 	--build-name="$(_build_name)"
 
 # make doesn't handle spaces well... replace them with underscores in paths

Vagrantfile

@@ -77,15 +77,7 @@ Vagrant.configure(2) do |config|
       #install host build-time dependencies
       apt-get update
       apt-get install -y ccache texinfo gpgv2 gnupg2 git docker-ce docker-ce-cli containerd.io \
-          fontforge-nox python-debian python-pip meson libmpc-dev libmpc-dev:i386 \
+          fontforge-nox python-debian python-pip
-          gcc g++ gcc-i686-linux-gnu g++-i686-linux-gnu binutils-i686-linux-gnu \
-          gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 \
-          g++-mingw-w64-i686 g++-mingw-w64-x86-64
-
-      update-alternatives --set x86_64-w64-mingw32-gcc `which x86_64-w64-mingw32-gcc-posix`
-      update-alternatives --set x86_64-w64-mingw32-g++ `which x86_64-w64-mingw32-g++-posix`
-      update-alternatives --set i686-w64-mingw32-gcc `which i686-w64-mingw32-gcc-posix`
-      update-alternatives --set i686-w64-mingw32-g++ `which i686-w64-mingw32-g++-posix`
 
       #install adobe font devkit to build source san hans
       pip install afdko
@@ -93,18 +85,11 @@ Vagrant.configure(2) do |config|
       #allow vagrant user to run docker
       adduser vagrant docker
 
-      #add steamrt docker
-      docker pull registry.gitlab.steamos.cloud/steamrt/soldier/sdk
-      docker image tag registry.gitlab.steamos.cloud/steamrt/soldier/sdk steam-proton-dev
-
       #allow user to run stuff in steamrt
       sysctl kernel.unprivileged_userns_clone=1
       mkdir -p /etc/sysctl.d/
       echo kernel.unprivileged_userns_clone=1 > /etc/sysctl.d/docker_user.conf
 
-      # the script below will set up the steam-runtime docker containers
-      sudo -u vagrant /home/vagrant/proton/vagrant-user-setup.sh
-
       #ensure we use only the mingw-w64 that we built
       apt-get remove -y '*mingw-w64*'
     SHELL

build-mingw-w64.sh

@@ -1,229 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# We need two cross-compilers to build Proton:
-#
-# 64-bit cross-compiler:
-#  Build (where the compiler is built): 64-bit linux (our VM)
-#  Host (where the compiler is run):    64-bit linux (64-bit Steam runtime)
-#  Target (what the compiler outputs):  64-bit win32 (PE files to be run)
-#
-# 32-bit cross-compiler:
-#  Build (where the compiler is built): 64-bit linux (our VM)
-#  Host (where the compiler is run):    64-bit linux (64-bit Steam runtime)
-#  Target (what the compiler outputs):  32-bit win32 (PE files to be run)
-
-if [ -z "$1" ]; then
-    echo "Makes a local build of mingw-w64 in this directory and installs it to the given path."
-    echo ""
-    echo "Note: Requires a system mingw-w64 compiler to be present already on your build machine, for us to bootstrap with."
-    echo ""
-    echo "usage:"
-    echo -e "\t$0 <installation path e.g. \$HOME/.local>"
-    exit 1
-fi
-
-if [ -z "$MAKEFLAGS" ]; then
-    JOBS=-j$(($(nproc) - 1))
-fi
-
-DST_DIR="$1"
-
-BINUTILS_VER=2.35
-BINUTILS_SRCTARBALL=binutils-$BINUTILS_VER.tar.xz
-BINUTILS_URL="https://ftp.gnu.org/gnu/binutils/$BINUTILS_SRCTARBALL"
-BINUTILS_SRCDIR=binutils-$BINUTILS_VER
-
-GCC_VER=9.3.0
-GCC_SRCTARBALL=gcc-$GCC_VER.tar.xz
-GCC_URL="https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VER/$GCC_SRCTARBALL"
-GCC_SRCDIR=gcc-$GCC_VER
-
-MINGW_W64_GITVER=v8.0.0
-MINGW_W64_GITURL="git://git.code.sf.net/p/mingw-w64/mingw-w64"
-MINGW_W64_SRCDIR=mingw-w64-git
-
-ISL_VER=0.22.1
-ISL_SRCTARBALL=isl-$ISL_VER.tar.bz2
-ISL_URL="http://isl.gforge.inria.fr/isl-$ISL_VER.tar.bz2"
-ISL_SRCDIR=isl-$ISL_VER
-
-function setup_src {
-    if [ ! -e "$BINUTILS_SRCTARBALL" ]; then
-        wget -O "$BINUTILS_SRCTARBALL" "$BINUTILS_URL"
-    fi
-
-    if [ ! -e "$BINUTILS_SRCDIR" ]; then
-        tar -xf "$BINUTILS_SRCTARBALL"
-    fi
-
-    if [ ! -e "$ISL_SRCTARBALL" ]; then
-        wget -O "$ISL_SRCTARBALL" "$ISL_URL"
-    fi
-
-    if [ ! -e "$ISL_SRCDIR" ]; then
-        tar -xf "$ISL_SRCTARBALL"
-    fi
-
-    if [ ! -e "$GCC_SRCTARBALL" ]; then
-        wget -O "$GCC_SRCTARBALL" "$GCC_URL"
-    fi
-
-    if [ ! -e "$GCC_SRCDIR" ]; then
-        tar -xf "$GCC_SRCTARBALL"
-        ln -s ../$ISL_SRCDIR $GCC_SRCDIR/isl
-        for f in $(dirname $0)/mingw-w64-patches/gcc-*; do
-            patch -d "$GCC_SRCDIR" -p1 < $f
-        done
-    fi
-
-    if [ ! -e "$MINGW_W64_SRCDIR" ]; then
-        #shallow clone, we just want this one version
-        git clone -b $MINGW_W64_GITVER --depth 1 \
-            "$MINGW_W64_GITURL" "$MINGW_W64_SRCDIR"
-    fi
-}
-
-function build_arch {
-    BUILD_ARCH=$(gcc -dumpmachine) #machine which is building the compiler
-    HOST_ARCH=$1 #machine which will run the compiler
-    WIN32_TARGET_ARCH=$2 #machine which we are building for
-    NEWPATH=$DST_DIR/bin:$PATH
-
-    mkdir -p build-$WIN32_TARGET_ARCH/
-    pushd build-$WIN32_TARGET_ARCH/
-
-        mkdir -p binutils/
-        pushd binutils/
-            if [ ! -e Makefile ]; then
-                ../../$BINUTILS_SRCDIR/configure \
-                    --prefix=$DST_DIR/ \
-                    --build=$BUILD_ARCH \
-                    --host=$HOST_ARCH \
-                    --target=$WIN32_TARGET_ARCH \
-                    --enable-lto \
-                    --enable-plugins \
-                    --enable-deterministic-archives \
-                    --disable-multilib \
-                    --disable-nls \
-                    --disable-werror \
-                    $BINUTILS_EXTRA_CONFIGURE
-            fi
-            make $JOBS configure-host
-            make $JOBS LDFLAGS=-all-static
-            make $JOBS install
-        popd
-
-        mkdir -p mingw-w64-headers/
-        pushd mingw-w64-headers/
-            if [ ! -e Makefile ]; then
-                PATH=$NEWPATH:$PATH ../../$MINGW_W64_SRCDIR/mingw-w64-headers/configure \
-                    --prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \
-                    --host=$WIN32_TARGET_ARCH \
-                    --enable-sdk=all \
-                    --enable-secure-api \
-                    $MINGW_W64_HEADERS_EXTRA_CONFIGURE
-            fi
-            PATH=$NEWPATH:$PATH make $JOBS install
-        popd
-
-        mkdir -p gcc/
-        pushd gcc/
-            if [ ! -e Makefile ]; then
-                #arguments mostly taken from Arch AUR mingw-w64-gcc PKGBUILD,
-                #except "--disable-dw2-exceptions" swapped for "--disable-sjlj-exceptions --with-dwarf2"
-                #for performance reasons on 32-bit
-                LDFLAGS=-static PATH=$NEWPATH:$PATH ../../$GCC_SRCDIR/configure \
-                    --prefix=$DST_DIR/ \
-                    --build=$BUILD_ARCH \
-                    --host=$HOST_ARCH \
-                    --target=$WIN32_TARGET_ARCH \
-                    --enable-static \
-                    --enable-shared \
-                    --enable-languages=c,lto,c++ \
-                    --disable-multilib \
-                    --enable-threads=posix \
-                    --enable-fully-dynamic-string \
-                    --enable-libstdcxx-time=yes \
-                    --enable-libstdcxx-filesystem-ts=yes \
-                    --enable-cloog-backend=isl \
-                    --enable-libgomp \
-                    --enable-lto \
-                    --disable-sjlj-exceptions \
-                    --with-dwarf2 \
-                    $GCC_EXTRA_CONFIGURE
-            fi
-            PATH=$NEWPATH make $JOBS all-gcc
-            PATH=$NEWPATH make $JOBS install-gcc
-        popd
-
-        mkdir -p mingw-w64-crt/
-        pushd mingw-w64-crt/
-            if [ ! -e Makefile ]; then
-                PATH=$NEWPATH ../../$MINGW_W64_SRCDIR/mingw-w64-crt/configure \
-                    --prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \
-                    --host=$WIN32_TARGET_ARCH \
-                    --enable-wildcard \
-                    $MINGW_W64_CRT_EXTRA_CONFIGURE
-            fi
-            PATH=$NEWPATH make $JOBS
-            PATH=$NEWPATH make $JOBS install
-        popd
-
-        mkdir -p mingw-w64-winpthreads/
-        pushd mingw-w64-winpthreads/
-            if [ ! -e Makefile ]; then
-                PATH=$NEWPATH ../../$MINGW_W64_SRCDIR/mingw-w64-libraries/winpthreads/configure \
-                    --prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \
-                    --host=$WIN32_TARGET_ARCH \
-                    --enable-static \
-                    --enable-shared \
-                    $MINGW_W64_WINPTHREADS_EXTRA_CONFIGURE
-            fi
-            PATH=$NEWPATH make $JOBS
-            PATH=$NEWPATH make $JOBS install
-        popd
-
-        pushd gcc/
-            #next step requires libgcc in default library location, but
-            #"canadian" build doesn't handle that(?), so install it explicitly
-            PATH=$NEWPATH make configure-target-libgcc
-            PATH=$NEWPATH make -C $WIN32_TARGET_ARCH/libgcc $JOBS
-            PATH=$NEWPATH make -C $WIN32_TARGET_ARCH/libgcc $JOBS install
-
-            #install libstdc++ and other stuff
-            PATH=$NEWPATH make $JOBS
-            PATH=$NEWPATH make $JOBS install
-
-            #libstdc++ requires that libstdc++ is installed in order to find gettimeofday(???)
-            #so, rebuild libstdc++ after installing it above
-            PATH=$NEWPATH make $JOBS -C $WIN32_TARGET_ARCH/libstdc++-v3/ distclean
-            PATH=$NEWPATH make $JOBS
-            PATH=$NEWPATH make $JOBS install
-        popd
-
-        mkdir -p mingw-w64-tools/widl
-        pushd mingw-w64-tools/widl/
-            if [ ! -e Makefile ]; then
-                PATH=$NEWPATH ../../../$MINGW_W64_SRCDIR/mingw-w64-tools/widl/configure \
-                    --prefix=$DST_DIR/ \
-                    --target=$WIN32_TARGET_ARCH \
-                    --program-prefix="${WIN32_TARGET_ARCH}-"
-            fi
-            PATH=$NEWPATH make $JOBS
-            PATH=$NEWPATH make $JOBS install
-        popd
-
-    popd
-}
-
-setup_src
-
-mkdir -p $DST_DIR
-
-MINGW_W64_CRT_EXTRA_CONFIGURE="--disable-lib64 --enable-lib32" build_arch x86_64-linux-gnu i686-w64-mingw32
-
-MINGW_W64_CRT_EXTRA_CONFIGURE="--disable-lib32 --enable-lib64" build_arch x86_64-linux-gnu x86_64-w64-mingw32
-
-echo "Done!"

build/makefile_base.mak

@@ -40,20 +40,12 @@ else
 	DOCKER_CCACHE_FLAG = -e CCACHE_DISABLE=1
 endif
 
-ifneq ($(filter default undefined,$(origin CC)),)
+CC := $(CCACHE_BIN) x86_64-linux-gnu-gcc
-	CC = $(CCACHE_BIN) gcc
+CXX := $(CCACHE_BIN) x86_64-linux-gnu-g++
-endif
+CROSSCC32 := $(CCACHE_BIN) i686-w64-mingw32-gcc
-ifneq ($(filter default undefined,$(origin CXX)),)
+CROSSCC64 := $(CCACHE_BIN) x86_64-w64-mingw32-gcc
-	CXX = $(CCACHE_BIN) g++
+CROSSCXX32 := $(CCACHE_BIN) i686-w64-mingw32-g++
-endif
+CROSSCXX64 := $(CCACHE_BIN) x86_64-w64-mingw32-g++
-ifneq ($(filter default undefined,$(origin CROSSCC)),)
-	CROSSCC32 = $(CCACHE_BIN) i686-w64-mingw32-gcc
-	CROSSCC64 = $(CCACHE_BIN) x86_64-w64-mingw32-gcc
-endif
-ifneq ($(filter default undefined,$(origin CROSSCXX)),)
-	CROSSCXX32 = $(CCACHE_BIN) i686-w64-mingw32-g++
-	CROSSCXX64 = $(CCACHE_BIN) x86_64-w64-mingw32-g++
-endif
 
 export CC
 export CXX
@@ -63,8 +55,8 @@ ifeq ($(ENABLE_CCACHE),1)
 else
 endif
 
-CC32 := $(CC) -m32 -mstackrealign
+CC32 := $(CCACHE_BIN) i686-linux-gnu-gcc -mstackrealign
-CXX32 := $(CXX) -m32 -mstackrealign
+CXX32 := $(CCACHE_BIN) i686-linux-gnu-g++ -mstackrealign
 PKG_CONFIG32 := i686-linux-gnu-pkg-config
 
 cc-option = $(shell if test -z "`echo 'void*p=1;' | \
@@ -1527,6 +1519,7 @@ vkd3d-proton: vkd3d32 vkd3d64
 mediaconv32: SHELL = $(CONTAINER_SHELL)
 mediaconv32: $(MAKEFILE_DEP) gstreamer32 | $(MEDIACONV_OBJ32)
 	cd $(abspath $(MEDIACONV)) && \
+		CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER="i686-linux-gnu-gcc" \
 		PKG_CONFIG_ALLOW_CROSS=1 \
 		PKG_CONFIG_PATH=$(abspath $(TOOLS_DIR32))/lib/pkgconfig \
 		cargo build --target i686-unknown-linux-gnu --target-dir $(abspath $(MEDIACONV_OBJ32)) $(CARGO_BUILD_ARG)

docker/Makefile

@@ -2,7 +2,7 @@ STEAMRT_VERSION = 0.20210126.1
 STEAMRT_URLBASE = registry.gitlab.steamos.cloud
 
 PROTONSDK_URLBASE = $(STEAMRT_URLBASE)/proton/soldier/sdk
-PROTONSDK_VERSION = $(STEAMRT_VERSION)-0
+PROTONSDK_VERSION = $(STEAMRT_VERSION)-0-dev
 
 # this is just for building toolchain, as we do static builds it should
 # not have any impact on the end result, but changing it will invalidate

docker/README.md

@@ -0,0 +1,81 @@
+Proton (soldier) SDK
+====================
+
+These are the build rules that are used to create docker images to build
+Proton. The automated creation of the official images lives in
+<https://gitlab.steamos.cloud/proton/soldier/sdk>, but this can be
+used to create local images too.
+
+Local usage
+-----------
+
+The `protonsdk_version` make variable will override which image is used
+to build Proton, but may not trigger a full rebuild, so building from
+scratch may be necessary.
+
+Building Proton with a locally build docker image, instead of using the
+official images, can be done by using `protonsdk_version=local`. This
+may be used for instance to test changes to the docker image recipes.
+
+Or, it is also possible to build the docker images first by invoking
+`make protonsdk` and it will tag the images with the `protonsdk_version`
+variable value.
+
+Official images
+---------------
+
+To update the official Proton SDK images:
+
+1) Update the image build rules, `STEAMRT_VERSION` and
+   `PROTONSDK_VERSION` version numbers in this folder, test locally,
+   commit and push the changes.
+
+2) Update `.gitlab-ci.yml` in the
+   [Proton SDK](https://gitlab.steamos.cloud/proton/soldier/sdk)
+   repository to point to the new commit, commit and push to trigger a
+   new build of "-dev" images.
+
+3) Once the images are satifying, tag the version in Proton SDK
+   repository and push the tag, this will trigger a new build of the
+   images and version them with the same tag as the Git tag.
+
+4) Once the images have been published, update the default
+   `protonsdk_version` version number in `Makefile` to use the newly
+   built images by default.
+
+Any change or addition to GPL-ed source first requires to update or add
+the corresponding source to <https://repo.steampowered.com/proton-sdk>.
+The `SOURCES_URLBASE` variable must be used to download the sources
+from there, and its sha256 must be added to validate the sources in the
+same way the existing code does.
+
+Technical details
+-----------------
+
+The images are built incrementally, with intermediate images created
+for each component of the toolchain, then assembled together in a single
+`proton` image.
+
+The reason behind this is to optimize the use of docker cache, so that
+components that didn't change do not require to be rebuilt every time,
+saving a lot of time in the build process, when only small additions are
+made.
+
+It also lets us build several components of the toolchain separately,
+in parallel. This is especially the case when building the images on
+the <https://gitlab.steamos.cloud> CI.
+
+Because the `ARG` parameter in Dockerfile doesn't behave nicely with
+docker caches it is avoided, and `Dockerfile.in` are used with variable
+replacements done using `sed`.
+
+The <https://gitlab.steamos.cloud> CI uses Kaniko instead of Docker,
+with a bit of script conversion to generate commands usable there. More
+details are available in
+[Proton SDK](https://gitlab.steamos.cloud/proton/soldier/sdk).
+
+The `build-base` images are there to create a common ground to build the
+other elements of the toolchain. They are based on fairly recent (more
+than what `steamrt` provides), but they are only used temporarily to
+build a static version of the Binutils, MinGW and GCC binaries, which
+are then copied over the `steamrt` base image.

vagrant-user-setup.sh

@@ -1,33 +0,0 @@
-#!/bin/sh
-
-set -e
-
-#build and install recent mingw-w64
-if [ ! -e "$HOME/.local/bin/x86_64-w64-mingw32-gcc" ]; then
-    mkdir -p $HOME/mingw-w64-build/
-    (cd $HOME/mingw-w64-build && $HOME/proton/build-mingw-w64.sh "$HOME/.local/")
-    #clean up the build tree, this takes up like 6GB
-    rm -rf $HOME/mingw-w64-build/
-fi
-
-#install rust with x86_64 and i686 targets
-RUST_VERSION="1.44.1"
-RUST_CARGO_HOME=$HOME/rust/cargo
-RUST_RUSTUP_HOME=$HOME/rust/rustup
-if [ ! -e "$RUST_CARGO_HOME" ]; then
-
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > $HOME/rustup-install.sh
-
-    #64-bit toolchain
-    CARGO_HOME=$RUST_CARGO_HOME RUSTUP_HOME=$RUST_RUSTUP_HOME \
-        sh $HOME/rustup-install.sh -y --no-modify-path \
-        --default-host "x86_64-unknown-linux-gnu" \
-        --default-toolchain "$RUST_VERSION-x86_64-unknown-linux-gnu" \
-        -t "i686-unknown-linux-gnu"
-
-    cat > $HOME/.local/bin/cargo <<EOF
-PATH=$RUST_CARGO_HOME/bin/:\$PATH CARGO_HOME=$RUST_CARGO_HOME RUSTUP_HOME=$RUST_RUSTUP_HOME cargo \$@
-EOF
-
-    chmod 755 $HOME/.local/bin/cargo
-fi