From 70c5a7806256518c9e63aceacba881da212a9125 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Thu, 12 Dec 2019 23:14:04 -0800 Subject: [PATCH] nixos: compress make-ext4-fs with zstd --- nixos/lib/make-ext4-fs.nix | 44 ++++++++++++--------- nixos/modules/installer/cd-dvd/sd-image.nix | 13 ++++-- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/nixos/lib/make-ext4-fs.nix b/nixos/lib/make-ext4-fs.nix index 932adcd97967..f46d3990c06b 100644 --- a/nixos/lib/make-ext4-fs.nix +++ b/nixos/lib/make-ext4-fs.nix @@ -4,8 +4,11 @@ # generated image is sized to only fit its contents, with the expectation # that a script resizes the filesystem at boot time. { pkgs +, lib # List of derivations to be included , storePaths +# Whether or not to compress the resulting image with zstd +, compressImage ? false, zstd # Shell commands to populate the ./files directory. # All files in that directory are copied to the root of the FS. , populateImageCommands ? "" @@ -20,18 +23,20 @@ let sdClosureInfo = pkgs.buildPackages.closureInfo { rootPaths = storePaths; }; in - pkgs.stdenv.mkDerivation { - name = "ext4-fs.img"; + name = "ext4-fs.img${lib.optionalString compressImage ".zst"}"; - nativeBuildInputs = [e2fsprogs.bin libfaketime perl lkl]; + nativeBuildInputs = [ e2fsprogs.bin libfaketime perl lkl ] + ++ lib.optional compressImage zstd; buildCommand = '' + ${if compressImage then "img=temp.img" else "img=$out"} ( mkdir -p ./files ${populateImageCommands} ) + # Add the closures of the top-level store objects. storePaths=$(cat ${sdClosureInfo}/store-paths) @@ -42,28 +47,26 @@ pkgs.stdenv.mkDerivation { bytes=$((2 * 4096 * $numInodes + 4096 * $numDataBlocks)) echo "Creating an EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks)" - truncate -s $bytes $out - faketime -f "1970-01-01 00:00:01" mkfs.ext4 -L ${volumeLabel} -U ${uuid} $out + truncate -s $bytes $img + faketime -f "1970-01-01 00:00:01" mkfs.ext4 -L ${volumeLabel} -U ${uuid} $img # Also include a manifest of the closures in a format suitable for nix-store --load-db. cp ${sdClosureInfo}/registration nix-path-registration - cptofs -t ext4 -i $out nix-path-registration / + cptofs -t ext4 -i $img nix-path-registration / # Create nix/store before copying paths faketime -f "1970-01-01 00:00:01" mkdir -p nix/store - cptofs -t ext4 -i $out nix / + cptofs -t ext4 -i $img nix / echo "copying store paths to image..." - cptofs -t ext4 -i $out $storePaths /nix/store/ + cptofs -t ext4 -i $img $storePaths /nix/store/ - ( echo "copying files to image..." - cd ./files - cptofs -t ext4 -i $out ./* / - ) + cptofs -t ext4 -i $img ./files/* / + # I have ended up with corrupted images sometimes, I suspect that happens when the build machine's disk gets full during the build. - if ! fsck.ext4 -n -f $out; then + if ! fsck.ext4 -n -f $img; then echo "--- Fsck failed for EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks) ---" cat errorlog return 1 @@ -71,9 +74,9 @@ pkgs.stdenv.mkDerivation { ( # Resizes **snugly** to its actual limits (or closer to) - free=$(dumpe2fs $out | grep '^Free blocks:') - blocksize=$(dumpe2fs $out | grep '^Block size:') - blocks=$(dumpe2fs $out | grep '^Block count:') + free=$(dumpe2fs $img | grep '^Free blocks:') + blocksize=$(dumpe2fs $img | grep '^Block size:') + blocks=$(dumpe2fs $img | grep '^Block count:') blocks=$((''${blocks##*:})) # format the number. blocksize=$((''${blocksize##*:})) # format the number. # System can't boot with 0 blocks free. @@ -82,10 +85,15 @@ pkgs.stdenv.mkDerivation { size=$(( blocks - ''${free##*:} + fudge )) echo "Resizing from $blocks blocks to $size blocks. (~ $((size*blocksize/1024/1024))MiB)" - EXT2FS_NO_MTAB_OK=yes resize2fs $out -f $size + EXT2FS_NO_MTAB_OK=yes resize2fs $img -f $size ) # And a final fsck, because of the previous truncating. - fsck.ext4 -n -f $out + fsck.ext4 -n -f $img + + if [ ${builtins.toString compressImage} ]; then + echo "Compressing image" + zstd -v --no-progress ./$img -o $out + fi ''; } diff --git a/nixos/modules/installer/cd-dvd/sd-image.nix b/nixos/modules/installer/cd-dvd/sd-image.nix index 7865b767f0b7..901c60befb6c 100644 --- a/nixos/modules/installer/cd-dvd/sd-image.nix +++ b/nixos/modules/installer/cd-dvd/sd-image.nix @@ -18,6 +18,7 @@ with lib; let rootfsImage = pkgs.callPackage ../../../lib/make-ext4-fs.nix ({ inherit (config.sdImage) storePaths; + compressImage = true; populateImageCommands = config.sdImage.populateRootCommands; volumeLabel = "NIXOS_SD"; } // optionalAttrs (config.sdImage.rootPartitionUUID != null) { @@ -128,10 +129,11 @@ in sdImage.storePaths = [ config.system.build.toplevel ]; - system.build.sdImage = pkgs.callPackage ({ stdenv, dosfstools, e2fsprogs, mtools, libfaketime, utillinux, bzip2 }: stdenv.mkDerivation { + system.build.sdImage = pkgs.callPackage ({ stdenv, dosfstools, e2fsprogs, + mtools, libfaketime, utillinux, bzip2, zstd }: stdenv.mkDerivation { name = config.sdImage.imageName; - nativeBuildInputs = [ dosfstools e2fsprogs mtools libfaketime utillinux bzip2 ]; + nativeBuildInputs = [ dosfstools e2fsprogs mtools libfaketime utillinux bzip2 zstd ]; inherit (config.sdImage) compressImage; @@ -146,11 +148,14 @@ in echo "file sd-image $img" >> $out/nix-support/hydra-build-products fi + echo "Decompressing rootfs image" + zstd -d --no-progress "${rootfsImage}" -o ./root-fs.img + # Gap in front of the first partition, in MiB gap=8 # Create the image file sized to fit /boot/firmware and /, plus slack for the gap. - rootSizeBlocks=$(du -B 512 --apparent-size ${rootfsImage} | awk '{ print $1 }') + rootSizeBlocks=$(du -B 512 --apparent-size ./root-fs.img | awk '{ print $1 }') firmwareSizeBlocks=$((${toString config.sdImage.firmwareSize} * 1024 * 1024 / 512)) imageSize=$((rootSizeBlocks * 512 + firmwareSizeBlocks * 512 + gap * 1024 * 1024)) truncate -s $imageSize $img @@ -168,7 +173,7 @@ in # Copy the rootfs into the SD image eval $(partx $img -o START,SECTORS --nr 2 --pairs) - dd conv=notrunc if=${rootfsImage} of=$img seek=$START count=$SECTORS + dd conv=notrunc if=./root-fs.img of=$img seek=$START count=$SECTORS # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img eval $(partx $img -o START,SECTORS --nr 1 --pairs)