Refactor nixos-install to separate out filesystem build logic
The key distinction I'm drawing is that there's a component that deals with the store of the machine being built, and another component for the store building it. The inner part of it assumes nothing from the builder (doesn't need chroot or root powers) so it can run comfortably inside a Nix build, as well as nixos-rebuild. I have some upcoming work that will use that to significantly speed up and streamline image builds for NixOS, especially on virtualized hosts like EC2, but it's also a reasonable speedup on native hosts.
This commit is contained in:
Dan Peebles
parent
91d2dc00bb
commit
d990aa7163
5 changed files with 164 additions and 130 deletions
|
|
@ -87,38 +87,6 @@ if ! test -e "$mountPoint"; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# Mount some stuff in the target root directory.
|
||||
mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
|
||||
mkdir -m 01777 -p $mountPoint/tmp
|
||||
mkdir -m 0755 -p $mountPoint/tmp/root
|
||||
mkdir -m 0755 -p $mountPoint/var
|
||||
mkdir -m 0700 -p $mountPoint/root
|
||||
mount --rbind /dev $mountPoint/dev
|
||||
mount --rbind /proc $mountPoint/proc
|
||||
mount --rbind /sys $mountPoint/sys
|
||||
mount --rbind / $mountPoint/tmp/root
|
||||
mount -t tmpfs -o "mode=0755" none $mountPoint/run
|
||||
rm -rf $mountPoint/var/run
|
||||
ln -s /run $mountPoint/var/run
|
||||
for f in /etc/resolv.conf /etc/hosts; do rm -f $mountPoint/$f; [ -f "$f" ] && cp -Lf $f $mountPoint/etc/; done
|
||||
for f in /etc/passwd /etc/group; do touch $mountPoint/$f; [ -f "$f" ] && mount --rbind -o ro $f $mountPoint/$f; done
|
||||
|
||||
cp -Lf "@cacert@" "$mountPoint/tmp/ca-cert.crt"
|
||||
export SSL_CERT_FILE=/tmp/ca-cert.crt
|
||||
# For Nix 1.7
|
||||
export CURL_CA_BUNDLE=/tmp/ca-cert.crt
|
||||
|
||||
if [ -n "$runChroot" ]; then
|
||||
if ! [ -L $mountPoint/nix/var/nix/profiles/system ]; then
|
||||
echo "$0: installation not finished; cannot chroot into installation directory"
|
||||
exit 1
|
||||
fi
|
||||
ln -s /nix/var/nix/profiles/system $mountPoint/run/current-system
|
||||
exec chroot $mountPoint "${chrootCommand[@]}"
|
||||
fi
|
||||
|
||||
|
||||
# Get the path of the NixOS configuration file.
|
||||
if test -z "$NIXOS_CONFIG"; then
|
||||
NIXOS_CONFIG=/etc/nixos/configuration.nix
|
||||
|
|
@ -130,121 +98,60 @@ if [ ! -e "$mountPoint/$NIXOS_CONFIG" ] && [ -z "$closure" ]; then
|
|||
fi
|
||||
|
||||
|
||||
# Create the necessary Nix directories on the target device, if they
|
||||
# don't already exist.
|
||||
mkdir -m 0755 -p \
|
||||
$mountPoint/nix/var/nix/gcroots \
|
||||
$mountPoint/nix/var/nix/temproots \
|
||||
$mountPoint/nix/var/nix/userpool \
|
||||
$mountPoint/nix/var/nix/profiles \
|
||||
$mountPoint/nix/var/nix/db \
|
||||
$mountPoint/nix/var/log/nix/drvs
|
||||
|
||||
mkdir -m 1775 -p $mountPoint/nix/store
|
||||
chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
|
||||
|
||||
|
||||
# There is no daemon in the chroot.
|
||||
unset NIX_REMOTE
|
||||
|
||||
|
||||
# We don't have locale-archive in the chroot, so clear $LANG.
|
||||
export LANG=
|
||||
export LC_ALL=
|
||||
export LC_TIME=
|
||||
|
||||
|
||||
# Builds will use users that are members of this group
|
||||
extraBuildFlags+=(--option "build-users-group" "$buildUsersGroup")
|
||||
|
||||
|
||||
# Inherit binary caches from the host
|
||||
# TODO: will this still work with Nix 1.12 now that it has no perl? Probably not...
|
||||
binary_caches="$(@perl@/bin/perl -I @nix@/lib/perl5/site_perl/*/* -e 'use Nix::Config; Nix::Config::readConfig; print $Nix::Config::config{"binary-caches"};')"
|
||||
extraBuildFlags+=(--option "binary-caches" "$binary_caches")
|
||||
|
||||
nixpkgs="$(readlink -f "$(nix-instantiate --find-file nixpkgs)")"
|
||||
export NIX_PATH="nixpkgs=$nixpkgs:nixos-config=$mountPoint/$NIXOS_CONFIG"
|
||||
unset NIXOS_CONFIG
|
||||
|
||||
# Copy Nix to the Nix store on the target device, unless it's already there.
|
||||
if ! NIX_DB_DIR=$mountPoint/nix/var/nix/db nix-store --check-validity @nix@ 2> /dev/null; then
|
||||
echo "copying Nix to $mountPoint...."
|
||||
for i in $(@perl@/bin/perl @pathsFromGraph@ @nixClosure@); do
|
||||
echo " $i"
|
||||
chattr -R -i $mountPoint/$i 2> /dev/null || true # clear immutable bit
|
||||
@rsync@/bin/rsync -a $i $mountPoint/nix/store/
|
||||
done
|
||||
|
||||
# Register the paths in the Nix closure as valid. This is necessary
|
||||
# to prevent them from being deleted the first time we install
|
||||
# something. (I.e., Nix will see that, e.g., the glibc path is not
|
||||
# valid, delete it to get it out of the way, but as a result nothing
|
||||
# will work anymore.)
|
||||
chroot $mountPoint @nix@/bin/nix-store --register-validity < @nixClosure@
|
||||
fi
|
||||
# TODO: do I need to set NIX_SUBSTITUTERS here or is the --option binary-caches above enough?
|
||||
|
||||
|
||||
# Create the required /bin/sh symlink; otherwise lots of things
|
||||
# (notably the system() function) won't work.
|
||||
mkdir -m 0755 -p $mountPoint/bin
|
||||
# !!! assuming that @shell@ is in the closure
|
||||
ln -sf @shell@ $mountPoint/bin/sh
|
||||
# A place to drop temporary closures
|
||||
trap "rm -rf $tmpdir" EXIT
|
||||
tmpdir="$(mktemp -d)"
|
||||
|
||||
# Build a closure (on the host; we then copy it into the guest)
|
||||
function closure() {
|
||||
nix-build "${extraBuildFlags[@]}" --no-out-link -E "with import <nixpkgs> {}; runCommand \"closure\" { exportReferencesGraph = [ \"x\" (buildEnv { name = \"env\"; paths = [ ($1) stdenv ]; }) ]; } \"cp x \$out\""
|
||||
}
|
||||
|
||||
# Build hooks likely won't function correctly in the minimal chroot; just disable them.
|
||||
unset NIX_BUILD_HOOK
|
||||
|
||||
# Make the build below copy paths from the CD if possible. Note that
|
||||
# /tmp/root in the chroot is the root of the CD.
|
||||
export NIX_OTHER_STORES=/tmp/root/nix:$NIX_OTHER_STORES
|
||||
|
||||
p=@nix@/libexec/nix/substituters
|
||||
export NIX_SUBSTITUTERS=$p/copy-from-other-stores.pl:$p/download-from-binary-cache.pl
|
||||
|
||||
system_closure="$tmpdir/system.closure"
|
||||
|
||||
if [ -z "$closure" ]; then
|
||||
# Get the absolute path to the NixOS/Nixpkgs sources.
|
||||
nixpkgs="$(readlink -f $(nix-instantiate --find-file nixpkgs))"
|
||||
|
||||
nixEnvAction="-f <nixpkgs/nixos> --set -A system"
|
||||
expr="(import <nixpkgs/nixos> {}).system"
|
||||
system_root="$(nix-build -E "$expr")"
|
||||
system_closure="$(closure "$expr")"
|
||||
else
|
||||
nixpkgs=""
|
||||
nixEnvAction="--set $closure"
|
||||
system_root=$closure
|
||||
# Create a temporary file ending in .closure (so nixos-prepare-root knows to --import it) to transport the store closure
|
||||
# to the filesytem we're preparing. Also delete it on exit!
|
||||
nix-store --export $(nix-store -qR $closure) > $system_closure
|
||||
fi
|
||||
|
||||
# Build the specified Nix expression in the target store and install
|
||||
# it into the system configuration profile.
|
||||
echo "building the system configuration..."
|
||||
NIX_PATH="nixpkgs=/tmp/root/$nixpkgs:nixos-config=$NIXOS_CONFIG" NIXOS_CONFIG= \
|
||||
chroot $mountPoint @nix@/bin/nix-env \
|
||||
"${extraBuildFlags[@]}" -p /nix/var/nix/profiles/system $nixEnvAction
|
||||
channel_root="$(nix-env -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")"
|
||||
channel_closure="$tmpdir/channel.closure"
|
||||
nix-store --export $channel_root > $channel_closure
|
||||
|
||||
# Populate the target root directory with the basics
|
||||
@prepare_root@/bin/nixos-prepare-root $mountPoint $channel_root $system_root @nixClosure@ $system_closure $channel_closure
|
||||
|
||||
# Copy the NixOS/Nixpkgs sources to the target as the initial contents
|
||||
# of the NixOS channel.
|
||||
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
|
||||
mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
|
||||
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
|
||||
srcs=$(nix-env "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")
|
||||
if [ -z "$noChannelCopy" ] && [ -n "$srcs" ]; then
|
||||
echo "copying NixOS/Nixpkgs sources..."
|
||||
chroot $mountPoint @nix@/bin/nix-env \
|
||||
"${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -i "$srcs" --quiet
|
||||
fi
|
||||
mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
|
||||
ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
|
||||
|
||||
|
||||
# Get rid of the /etc bind mounts.
|
||||
for f in /etc/passwd /etc/group; do [ -f "$f" ] && umount $mountPoint/$f; done
|
||||
# nixos-prepare-root doesn't currently do anything with file ownership, so we set it up here instead
|
||||
chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
|
||||
|
||||
mount --rbind /dev $mountPoint/dev
|
||||
mount --rbind /proc $mountPoint/proc
|
||||
mount --rbind /sys $mountPoint/sys
|
||||
|
||||
# Grub needs an mtab.
|
||||
ln -sfn /proc/mounts $mountPoint/etc/mtab
|
||||
|
||||
|
||||
# Mark the target as a NixOS installation, otherwise
|
||||
# switch-to-configuration will chicken out.
|
||||
touch $mountPoint/etc/NIXOS
|
||||
|
||||
|
||||
# Switch to the new system configuration. This will install Grub with
|
||||
# a menu default pointing at the kernel/initrd/etc of the new
|
||||
# configuration.
|
||||
|
|
|
|||
105
nixos/modules/installer/tools/nixos-prepare-root.sh
Normal file
105
nixos/modules/installer/tools/nixos-prepare-root.sh
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
#! @shell@
|
||||
|
||||
# This script's goal is to perform all "static" setup of a filesystem structure from pre-built store paths. Everything
|
||||
# in here should run in a non-root context and inside a Nix builder. It's designed primarily to be called from image-
|
||||
# building scripts and from nixos-install, but because it makes very few assumptions about the context in which it runs,
|
||||
# it could be useful in other contexts as well.
|
||||
#
|
||||
# Current behavior:
|
||||
# - set up basic filesystem structure
|
||||
# - make Nix store etc.
|
||||
# - copy Nix, system, channel, and misceallaneous closures to target Nix store
|
||||
# - register validity of all paths in the target store
|
||||
# - set up channel and system profiles
|
||||
|
||||
# Ensure a consistent umask.
|
||||
umask 0022
|
||||
|
||||
set -e
|
||||
|
||||
mountPoint="$1"
|
||||
channel="$2"
|
||||
system="$3"
|
||||
shift 3
|
||||
closures="$@"
|
||||
|
||||
PATH="@coreutils@/bin:@nix@/bin:@perl@/bin:@utillinux@/bin:@rsync@/bin"
|
||||
|
||||
if ! test -e "$mountPoint"; then
|
||||
echo "mount point $mountPoint doesn't exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create a few of the standard directories in the target root directory.
|
||||
mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
|
||||
mkdir -m 01777 -p $mountPoint/tmp
|
||||
mkdir -m 0755 -p $mountPoint/tmp/root
|
||||
mkdir -m 0755 -p $mountPoint/var
|
||||
mkdir -m 0700 -p $mountPoint/root
|
||||
|
||||
ln -s /run $mountPoint/var/run
|
||||
|
||||
# Create the necessary Nix directories on the target device
|
||||
mkdir -m 0755 -p \
|
||||
$mountPoint/nix/var/nix/gcroots \
|
||||
$mountPoint/nix/var/nix/temproots \
|
||||
$mountPoint/nix/var/nix/userpool \
|
||||
$mountPoint/nix/var/nix/profiles \
|
||||
$mountPoint/nix/var/nix/db \
|
||||
$mountPoint/nix/var/log/nix/drvs
|
||||
|
||||
mkdir -m 1775 -p $mountPoint/nix/store
|
||||
|
||||
# All Nix operations below should operate on our target store, not /nix/store.
|
||||
# N.B: this relies on Nix 1.12 or higher
|
||||
export NIX_REMOTE=local?root=$mountPoint
|
||||
|
||||
# Copy our closures to the Nix store on the target mount point, unless they're already there.
|
||||
for i in $closures; do
|
||||
# We support closures both in the format produced by `nix-store --export` and by `exportReferencesGraph`,
|
||||
# mostly because there doesn't seem to be a single format that can be produced outside of a nix build and
|
||||
# inside one. See https://github.com/NixOS/nix/issues/1242 for more discussion.
|
||||
if [[ "$i" =~ \.closure$ ]]; then
|
||||
echo "importing serialized closure $i to $mountPoint..."
|
||||
nix-store --import < $i
|
||||
else
|
||||
# There has to be a better way to do this, right?
|
||||
echo "copying closure $i to $mountPoint..."
|
||||
for j in $(perl @pathsFromGraph@ $i); do
|
||||
echo " $j... "
|
||||
rsync -a $j $mountPoint/nix/store/
|
||||
done
|
||||
|
||||
nix-store --register-validity < $i
|
||||
fi
|
||||
done
|
||||
|
||||
# Create the required /bin/sh symlink; otherwise lots of things
|
||||
# (notably the system() function) won't work.
|
||||
if [ ! -x $mountPoint/@shell@ ]; then
|
||||
echo "Error: @shell@ wasn't included in the closure" >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -m 0755 -p $mountPoint/bin
|
||||
ln -sf @shell@ $mountPoint/bin/sh
|
||||
|
||||
echo "setting the system closure to '$system'..."
|
||||
nix-env "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/system --set "$system"
|
||||
|
||||
ln -sfn /nix/var/nix/profiles/system $mountPoint/run/current-system
|
||||
|
||||
# Copy the NixOS/Nixpkgs sources to the target as the initial contents of the NixOS channel.
|
||||
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
|
||||
mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
|
||||
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
|
||||
|
||||
if [ -z "$noChannelCopy" ] && [ -n "$channel" ]; then
|
||||
echo "copying channel..."
|
||||
nix-env --option build-use-substitutes false "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/per-user/root/channels --set "$channel" --quiet
|
||||
fi
|
||||
mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
|
||||
ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
|
||||
|
||||
# Mark the target as a NixOS installation, otherwise switch-to-configuration will chicken out.
|
||||
touch $mountPoint/etc/NIXOS
|
||||
|
||||
|
|
@ -4,7 +4,6 @@
|
|||
{ config, pkgs, modulesPath, ... }:
|
||||
|
||||
let
|
||||
|
||||
cfg = config.installer;
|
||||
|
||||
makeProg = args: pkgs.substituteAll (args // {
|
||||
|
|
@ -17,6 +16,14 @@ let
|
|||
src = ./nixos-build-vms/nixos-build-vms.sh;
|
||||
};
|
||||
|
||||
nixos-prepare-root = makeProg {
|
||||
name = "nixos-prepare-root";
|
||||
src = ./nixos-prepare-root.sh;
|
||||
|
||||
nix = pkgs.nixUnstable;
|
||||
inherit (pkgs) perl pathsFromGraph rsync utillinux coreutils;
|
||||
};
|
||||
|
||||
nixos-install = makeProg {
|
||||
name = "nixos-install";
|
||||
src = ./nixos-install.sh;
|
||||
|
|
@ -26,6 +33,7 @@ let
|
|||
cacert = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
||||
root_uid = config.ids.uids.root;
|
||||
nixbld_gid = config.ids.gids.nixbld;
|
||||
prepare_root = nixos-prepare-root;
|
||||
|
||||
nixClosure = pkgs.runCommand "closure"
|
||||
{ exportReferencesGraph = ["refs" config.nix.package.out]; }
|
||||
|
|
@ -69,6 +77,7 @@ in
|
|||
|
||||
environment.systemPackages =
|
||||
[ nixos-build-vms
|
||||
nixos-prepare-root
|
||||
nixos-install
|
||||
nixos-rebuild
|
||||
nixos-generate-config
|
||||
|
|
@ -77,7 +86,7 @@ in
|
|||
];
|
||||
|
||||
system.build = {
|
||||
inherit nixos-install nixos-generate-config nixos-option nixos-rebuild;
|
||||
inherit nixos-install nixos-prepare-root nixos-generate-config nixos-option nixos-rebuild;
|
||||
};
|
||||
|
||||
};
|
||||
|
|
|
|||
|
|
@ -34,6 +34,12 @@ let
|
|||
boot.loader.systemd-boot.enable = true;
|
||||
''}
|
||||
|
||||
users.extraUsers.alice = {
|
||||
isNormalUser = true;
|
||||
home = "/home/alice";
|
||||
description = "Alice Foobar";
|
||||
};
|
||||
|
||||
hardware.enableAllFirmware = lib.mkForce false;
|
||||
|
||||
${replaceChars ["\n"] ["\n "] extraConfig}
|
||||
|
|
@ -96,7 +102,7 @@ let
|
|||
$machine->shutdown;
|
||||
|
||||
# Now see if we can boot the installation.
|
||||
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
|
||||
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "boot-after-install" });
|
||||
|
||||
# For example to enter LUKS passphrase.
|
||||
${preBootCommands}
|
||||
|
|
@ -118,11 +124,17 @@ let
|
|||
$machine->waitForUnit("swap.target");
|
||||
$machine->succeed("cat /proc/swaps | grep -q /dev");
|
||||
|
||||
# Check that the store is in good shape
|
||||
$machine->succeed("nix-store --verify --check-contents >&2");
|
||||
|
||||
# Check whether the channel works.
|
||||
$machine->succeed("nix-env -iA nixos.procps >&2");
|
||||
$machine->succeed("type -tP ps | tee /dev/stderr") =~ /.nix-profile/
|
||||
or die "nix-env failed";
|
||||
|
||||
# Check that the daemon works, and that non-root users can run builds (this will build a new profile generation through the daemon)
|
||||
$machine->succeed("su alice -l -c 'nix-env -iA nixos.procps' >&2");
|
||||
|
||||
# We need to a writable nix-store on next boot.
|
||||
$machine->copyFileFromHost(
|
||||
"${ makeConfig { inherit bootLoader grubVersion grubDevice grubIdentifier extraConfig; forceGrubReinstallCount = 1; } }",
|
||||
|
|
@ -139,7 +151,7 @@ let
|
|||
$machine->shutdown;
|
||||
|
||||
# Check whether a writable store build works
|
||||
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
|
||||
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "rebuild-switch" });
|
||||
${preBootCommands}
|
||||
$machine->waitForUnit("multi-user.target");
|
||||
$machine->copyFileFromHost(
|
||||
|
|
@ -150,7 +162,7 @@ let
|
|||
|
||||
# And just to be sure, check that the machine still boots after
|
||||
# "nixos-rebuild switch".
|
||||
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
|
||||
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", "boot-after-rebuild-switch" });
|
||||
${preBootCommands}
|
||||
$machine->waitForUnit("network.target");
|
||||
$machine->shutdown;
|
||||
|
|
|
|||
|
|
@ -131,12 +131,13 @@ in rec {
|
|||
sha256 = "69e0f398affec2a14c47b46fec712906429c85312d5483be43e4c34da4f63f67";
|
||||
};
|
||||
|
||||
# 1.11.8 doesn't yet have the patch to work on LLVM 4, so we patch it for now. Take this out once
|
||||
# we move to a higher version. I'd pull the specific patch from upstream but it doesn't apply cleanly.
|
||||
# Until 1.11.9 is released, we do this :)
|
||||
patchPhase = ''
|
||||
substituteInPlace src/libexpr/json-to-value.cc \
|
||||
--replace 'std::less<Symbol>, gc_allocator<Value *>' \
|
||||
'std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> >'
|
||||
|
||||
sed -i '/if (settings.readOnlyMode) {/a curSchema = getSchema();' src/libstore/local-store.cc
|
||||
'';
|
||||
}) // { perl-bindings = nixStable; };
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue