suyu/Yucom
25
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

build: Use proton sdk docker image for building.

Browse source
This commit is contained in:
Rémi Bernon 2021-03-10 18:38:53 +01:00 committed by Andrew Eikum
parent 6eba98ad23
commit 68bd1c20e8
7 changed files with 93 additions and 296 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -42,7 +42,7 @@ endif
protonsdk_version := 0.20210126.1-0
CONFIGURE_CMD := ../proton/configure.sh \
--steam-runtime-image=steam-proton-dev \
--steam-runtime-image=registry.gitlab.steamos.cloud/proton/soldier/sdk:$(protonsdk_version) \
--build-name="$(_build_name)"
# make doesn't handle spaces well... replace them with underscores in paths

17
Vagrantfile vendored
View file

@ -77,15 +77,7 @@ Vagrant.configure(2) do |config|
#install host build-time dependencies
apt-get update
apt-get install -y ccache texinfo gpgv2 gnupg2 git docker-ce docker-ce-cli containerd.io \
fontforge-nox python-debian python-pip meson libmpc-dev libmpc-dev:i386 \
gcc g++ gcc-i686-linux-gnu g++-i686-linux-gnu binutils-i686-linux-gnu \
gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 \
g++-mingw-w64-i686 g++-mingw-w64-x86-64
update-alternatives --set x86_64-w64-mingw32-gcc `which x86_64-w64-mingw32-gcc-posix`
update-alternatives --set x86_64-w64-mingw32-g++ `which x86_64-w64-mingw32-g++-posix`
update-alternatives --set i686-w64-mingw32-gcc `which i686-w64-mingw32-gcc-posix`
update-alternatives --set i686-w64-mingw32-g++ `which i686-w64-mingw32-g++-posix`
fontforge-nox python-debian python-pip
#install adobe font devkit to build source san hans
pip install afdko
@ -93,18 +85,11 @@ Vagrant.configure(2) do |config|
#allow vagrant user to run docker
adduser vagrant docker
#add steamrt docker
docker pull registry.gitlab.steamos.cloud/steamrt/soldier/sdk
docker image tag registry.gitlab.steamos.cloud/steamrt/soldier/sdk steam-proton-dev
#allow user to run stuff in steamrt
sysctl kernel.unprivileged_userns_clone=1
mkdir -p /etc/sysctl.d/
echo kernel.unprivileged_userns_clone=1 > /etc/sysctl.d/docker_user.conf
# the script below will set up the steam-runtime docker containers
sudo -u vagrant /home/vagrant/proton/vagrant-user-setup.sh
#ensure we use only the mingw-w64 that we built
apt-get remove -y '*mingw-w64*'
SHELL

229
build-mingw-w64.sh
View file

@ -1,229 +0,0 @@
#!/bin/bash
set -e
# We need two cross-compilers to build Proton:
#
# 64-bit cross-compiler:
# Build (where the compiler is built): 64-bit linux (our VM)
# Host (where the compiler is run): 64-bit linux (64-bit Steam runtime)
# Target (what the compiler outputs): 64-bit win32 (PE files to be run)
#
# 32-bit cross-compiler:
# Build (where the compiler is built): 64-bit linux (our VM)
# Host (where the compiler is run): 64-bit linux (64-bit Steam runtime)
# Target (what the compiler outputs): 32-bit win32 (PE files to be run)
if [ -z "$1" ]; then
echo "Makes a local build of mingw-w64 in this directory and installs it to the given path."
echo ""
echo "Note: Requires a system mingw-w64 compiler to be present already on your build machine, for us to bootstrap with."
echo ""
echo "usage:"
echo -e "\t$0 <installation path e.g. \$HOME/.local>"
exit 1
fi
if [ -z "$MAKEFLAGS" ]; then
JOBS=-j$(($(nproc) - 1))
fi
DST_DIR="$1"
BINUTILS_VER=2.35
BINUTILS_SRCTARBALL=binutils-$BINUTILS_VER.tar.xz
BINUTILS_URL="https://ftp.gnu.org/gnu/binutils/$BINUTILS_SRCTARBALL"
BINUTILS_SRCDIR=binutils-$BINUTILS_VER
GCC_VER=9.3.0
GCC_SRCTARBALL=gcc-$GCC_VER.tar.xz
GCC_URL="https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VER/$GCC_SRCTARBALL"
GCC_SRCDIR=gcc-$GCC_VER
MINGW_W64_GITVER=v8.0.0
MINGW_W64_GITURL="git://git.code.sf.net/p/mingw-w64/mingw-w64"
MINGW_W64_SRCDIR=mingw-w64-git
ISL_VER=0.22.1
ISL_SRCTARBALL=isl-$ISL_VER.tar.bz2
ISL_URL="http://isl.gforge.inria.fr/isl-$ISL_VER.tar.bz2"
ISL_SRCDIR=isl-$ISL_VER
function setup_src {
if [ ! -e "$BINUTILS_SRCTARBALL" ]; then
wget -O "$BINUTILS_SRCTARBALL" "$BINUTILS_URL"
fi
if [ ! -e "$BINUTILS_SRCDIR" ]; then
tar -xf "$BINUTILS_SRCTARBALL"
fi
if [ ! -e "$ISL_SRCTARBALL" ]; then
wget -O "$ISL_SRCTARBALL" "$ISL_URL"
fi
if [ ! -e "$ISL_SRCDIR" ]; then
tar -xf "$ISL_SRCTARBALL"
fi
if [ ! -e "$GCC_SRCTARBALL" ]; then
wget -O "$GCC_SRCTARBALL" "$GCC_URL"
fi
if [ ! -e "$GCC_SRCDIR" ]; then
tar -xf "$GCC_SRCTARBALL"
ln -s ../$ISL_SRCDIR $GCC_SRCDIR/isl
for f in $(dirname $0)/mingw-w64-patches/gcc-*; do
patch -d "$GCC_SRCDIR" -p1 < $f
done
fi
if [ ! -e "$MINGW_W64_SRCDIR" ]; then
#shallow clone, we just want this one version
git clone -b $MINGW_W64_GITVER --depth 1 \
"$MINGW_W64_GITURL" "$MINGW_W64_SRCDIR"
fi
}
function build_arch {
BUILD_ARCH=$(gcc -dumpmachine) #machine which is building the compiler
HOST_ARCH=$1 #machine which will run the compiler
WIN32_TARGET_ARCH=$2 #machine which we are building for
NEWPATH=$DST_DIR/bin:$PATH
mkdir -p build-$WIN32_TARGET_ARCH/
pushd build-$WIN32_TARGET_ARCH/
mkdir -p binutils/
pushd binutils/
if [ ! -e Makefile ]; then
../../$BINUTILS_SRCDIR/configure \
--prefix=$DST_DIR/ \
--build=$BUILD_ARCH \
--host=$HOST_ARCH \
--target=$WIN32_TARGET_ARCH \
--enable-lto \
--enable-plugins \
--enable-deterministic-archives \
--disable-multilib \
--disable-nls \
--disable-werror \
$BINUTILS_EXTRA_CONFIGURE
fi
make $JOBS configure-host
make $JOBS LDFLAGS=-all-static
make $JOBS install
popd
mkdir -p mingw-w64-headers/
pushd mingw-w64-headers/
if [ ! -e Makefile ]; then
PATH=$NEWPATH:$PATH ../../$MINGW_W64_SRCDIR/mingw-w64-headers/configure \
--prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \
--host=$WIN32_TARGET_ARCH \
--enable-sdk=all \
--enable-secure-api \
$MINGW_W64_HEADERS_EXTRA_CONFIGURE
fi
PATH=$NEWPATH:$PATH make $JOBS install
popd
mkdir -p gcc/
pushd gcc/
if [ ! -e Makefile ]; then
#arguments mostly taken from Arch AUR mingw-w64-gcc PKGBUILD,
#except "--disable-dw2-exceptions" swapped for "--disable-sjlj-exceptions --with-dwarf2"
#for performance reasons on 32-bit
LDFLAGS=-static PATH=$NEWPATH:$PATH ../../$GCC_SRCDIR/configure \
--prefix=$DST_DIR/ \
--build=$BUILD_ARCH \
--host=$HOST_ARCH \
--target=$WIN32_TARGET_ARCH \
--enable-static \
--enable-shared \
--enable-languages=c,lto,c++ \
--disable-multilib \
--enable-threads=posix \
--enable-fully-dynamic-string \
--enable-libstdcxx-time=yes \
--enable-libstdcxx-filesystem-ts=yes \
--enable-cloog-backend=isl \
--enable-libgomp \
--enable-lto \
--disable-sjlj-exceptions \
--with-dwarf2 \
$GCC_EXTRA_CONFIGURE
fi
PATH=$NEWPATH make $JOBS all-gcc
PATH=$NEWPATH make $JOBS install-gcc
popd
mkdir -p mingw-w64-crt/
pushd mingw-w64-crt/
if [ ! -e Makefile ]; then
PATH=$NEWPATH ../../$MINGW_W64_SRCDIR/mingw-w64-crt/configure \
--prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \
--host=$WIN32_TARGET_ARCH \
--enable-wildcard \
$MINGW_W64_CRT_EXTRA_CONFIGURE
fi
PATH=$NEWPATH make $JOBS
PATH=$NEWPATH make $JOBS install
popd
mkdir -p mingw-w64-winpthreads/
pushd mingw-w64-winpthreads/
if [ ! -e Makefile ]; then
PATH=$NEWPATH ../../$MINGW_W64_SRCDIR/mingw-w64-libraries/winpthreads/configure \
--prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \
--host=$WIN32_TARGET_ARCH \
--enable-static \
--enable-shared \
$MINGW_W64_WINPTHREADS_EXTRA_CONFIGURE
fi
PATH=$NEWPATH make $JOBS
PATH=$NEWPATH make $JOBS install
popd
pushd gcc/
#next step requires libgcc in default library location, but
#"canadian" build doesn't handle that(?), so install it explicitly
PATH=$NEWPATH make configure-target-libgcc
PATH=$NEWPATH make -C $WIN32_TARGET_ARCH/libgcc $JOBS
PATH=$NEWPATH make -C $WIN32_TARGET_ARCH/libgcc $JOBS install
#install libstdc++ and other stuff
PATH=$NEWPATH make $JOBS
PATH=$NEWPATH make $JOBS install
#libstdc++ requires that libstdc++ is installed in order to find gettimeofday(???)
#so, rebuild libstdc++ after installing it above
PATH=$NEWPATH make $JOBS -C $WIN32_TARGET_ARCH/libstdc++-v3/ distclean
PATH=$NEWPATH make $JOBS
PATH=$NEWPATH make $JOBS install
popd
mkdir -p mingw-w64-tools/widl
pushd mingw-w64-tools/widl/
if [ ! -e Makefile ]; then
PATH=$NEWPATH ../../../$MINGW_W64_SRCDIR/mingw-w64-tools/widl/configure \
--prefix=$DST_DIR/ \
--target=$WIN32_TARGET_ARCH \
--program-prefix="${WIN32_TARGET_ARCH}-"
fi
PATH=$NEWPATH make $JOBS
PATH=$NEWPATH make $JOBS install
popd
popd
}
setup_src
mkdir -p $DST_DIR
MINGW_W64_CRT_EXTRA_CONFIGURE="--disable-lib64 --enable-lib32" build_arch x86_64-linux-gnu i686-w64-mingw32
MINGW_W64_CRT_EXTRA_CONFIGURE="--disable-lib32 --enable-lib64" build_arch x86_64-linux-gnu x86_64-w64-mingw32
echo "Done!"

25
build/makefile_base.mak
View file

@ -40,20 +40,12 @@ else
DOCKER_CCACHE_FLAG = -e CCACHE_DISABLE=1
endif
ifneq ($(filter default undefined,$(origin CC)),)
CC = $(CCACHE_BIN) gcc
endif
ifneq ($(filter default undefined,$(origin CXX)),)
CXX = $(CCACHE_BIN) g++
endif
ifneq ($(filter default undefined,$(origin CROSSCC)),)
CROSSCC32 = $(CCACHE_BIN) i686-w64-mingw32-gcc
CROSSCC64 = $(CCACHE_BIN) x86_64-w64-mingw32-gcc
endif
ifneq ($(filter default undefined,$(origin CROSSCXX)),)
CROSSCXX32 = $(CCACHE_BIN) i686-w64-mingw32-g++
CROSSCXX64 = $(CCACHE_BIN) x86_64-w64-mingw32-g++
endif
CC := $(CCACHE_BIN) x86_64-linux-gnu-gcc
CXX := $(CCACHE_BIN) x86_64-linux-gnu-g++
CROSSCC32 := $(CCACHE_BIN) i686-w64-mingw32-gcc
CROSSCC64 := $(CCACHE_BIN) x86_64-w64-mingw32-gcc
CROSSCXX32 := $(CCACHE_BIN) i686-w64-mingw32-g++
CROSSCXX64 := $(CCACHE_BIN) x86_64-w64-mingw32-g++
export CC
export CXX
@ -63,8 +55,8 @@ ifeq ($(ENABLE_CCACHE),1)
else
endif
CC32 := $(CC) -m32 -mstackrealign
CXX32 := $(CXX) -m32 -mstackrealign
CC32 := $(CCACHE_BIN) i686-linux-gnu-gcc -mstackrealign
CXX32 := $(CCACHE_BIN) i686-linux-gnu-g++ -mstackrealign
PKG_CONFIG32 := i686-linux-gnu-pkg-config
cc-option = $(shell if test -z "`echo 'void*p=1;' | \
@ -1527,6 +1519,7 @@ vkd3d-proton: vkd3d32 vkd3d64
mediaconv32: SHELL = $(CONTAINER_SHELL)
mediaconv32: $(MAKEFILE_DEP) gstreamer32 | $(MEDIACONV_OBJ32)
cd $(abspath $(MEDIACONV)) && \
CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER="i686-linux-gnu-gcc" \
PKG_CONFIG_ALLOW_CROSS=1 \
PKG_CONFIG_PATH=$(abspath $(TOOLS_DIR32))/lib/pkgconfig \
cargo build --target i686-unknown-linux-gnu --target-dir $(abspath $(MEDIACONV_OBJ32)) $(CARGO_BUILD_ARG)

2
docker/Makefile
View file

@ -2,7 +2,7 @@ STEAMRT_VERSION = 0.20210126.1
STEAMRT_URLBASE = registry.gitlab.steamos.cloud
PROTONSDK_URLBASE = $(STEAMRT_URLBASE)/proton/soldier/sdk
PROTONSDK_VERSION = $(STEAMRT_VERSION)-0
PROTONSDK_VERSION = $(STEAMRT_VERSION)-0-dev
# this is just for building toolchain, as we do static builds it should
# not have any impact on the end result, but changing it will invalidate

81
docker/README.md Normal file
View file

@ -0,0 +1,81 @@
Proton (soldier) SDK
====================
These are the build rules that are used to create docker images to build
Proton. The automated creation of the official images lives in
<https://gitlab.steamos.cloud/proton/soldier/sdk>, but this can be
used to create local images too.
Local usage
-----------
The `protonsdk_version` make variable will override which image is used
to build Proton, but may not trigger a full rebuild, so building from
scratch may be necessary.
Building Proton with a locally build docker image, instead of using the
official images, can be done by using `protonsdk_version=local`. This
may be used for instance to test changes to the docker image recipes.
Or, it is also possible to build the docker images first by invoking
`make protonsdk` and it will tag the images with the `protonsdk_version`
variable value.
Official images
---------------
To update the official Proton SDK images:
1) Update the image build rules, `STEAMRT_VERSION` and
`PROTONSDK_VERSION` version numbers in this folder, test locally,
commit and push the changes.
2) Update `.gitlab-ci.yml` in the
[Proton SDK](https://gitlab.steamos.cloud/proton/soldier/sdk)
repository to point to the new commit, commit and push to trigger a
new build of "-dev" images.
3) Once the images are satifying, tag the version in Proton SDK
repository and push the tag, this will trigger a new build of the
images and version them with the same tag as the Git tag.
4) Once the images have been published, update the default
`protonsdk_version` version number in `Makefile` to use the newly
built images by default.
Any change or addition to GPL-ed source first requires to update or add
the corresponding source to <https://repo.steampowered.com/proton-sdk>.
The `SOURCES_URLBASE` variable must be used to download the sources
from there, and its sha256 must be added to validate the sources in the
same way the existing code does.
Technical details
-----------------
The images are built incrementally, with intermediate images created
for each component of the toolchain, then assembled together in a single
`proton` image.
The reason behind this is to optimize the use of docker cache, so that
components that didn't change do not require to be rebuilt every time,
saving a lot of time in the build process, when only small additions are
made.
It also lets us build several components of the toolchain separately,
in parallel. This is especially the case when building the images on
the <https://gitlab.steamos.cloud> CI.
Because the `ARG` parameter in Dockerfile doesn't behave nicely with
docker caches it is avoided, and `Dockerfile.in` are used with variable
replacements done using `sed`.
The <https://gitlab.steamos.cloud> CI uses Kaniko instead of Docker,
with a bit of script conversion to generate commands usable there. More
details are available in
[Proton SDK](https://gitlab.steamos.cloud/proton/soldier/sdk).
The `build-base` images are there to create a common ground to build the
other elements of the toolchain. They are based on fairly recent (more
than what `steamrt` provides), but they are only used temporarily to
build a static version of the Binutils, MinGW and GCC binaries, which
are then copied over the `steamrt` base image.

33
vagrant-user-setup.sh
View file

@ -1,33 +0,0 @@
#!/bin/sh
set -e
#build and install recent mingw-w64
if [ ! -e "$HOME/.local/bin/x86_64-w64-mingw32-gcc" ]; then
mkdir -p $HOME/mingw-w64-build/
(cd $HOME/mingw-w64-build && $HOME/proton/build-mingw-w64.sh "$HOME/.local/")
#clean up the build tree, this takes up like 6GB
rm -rf $HOME/mingw-w64-build/
fi
#install rust with x86_64 and i686 targets
RUST_VERSION="1.44.1"
RUST_CARGO_HOME=$HOME/rust/cargo
RUST_RUSTUP_HOME=$HOME/rust/rustup
if [ ! -e "$RUST_CARGO_HOME" ]; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > $HOME/rustup-install.sh
#64-bit toolchain
CARGO_HOME=$RUST_CARGO_HOME RUSTUP_HOME=$RUST_RUSTUP_HOME \
sh $HOME/rustup-install.sh -y --no-modify-path \
--default-host "x86_64-unknown-linux-gnu" \
--default-toolchain "$RUST_VERSION-x86_64-unknown-linux-gnu" \
-t "i686-unknown-linux-gnu"
cat > $HOME/.local/bin/cargo <<EOF
PATH=$RUST_CARGO_HOME/bin/:\$PATH CARGO_HOME=$RUST_CARGO_HOME RUSTUP_HOME=$RUST_RUSTUP_HOME cargo \$@
EOF
chmod 755 $HOME/.local/bin/cargo
fi