365 lines
10 KiB
Nix
365 lines
10 KiB
Nix
{ stdenv, lib, callPackage, runCommand, writeReferencesToFile, writeText, vmTools, writeScript
|
|
, docker, shadow, utillinux, coreutils, jshon, e2fsprogs, goPackages }:
|
|
|
|
# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future.
|
|
|
|
rec {
|
|
|
|
pullImage = callPackage ./pull.nix {};
|
|
|
|
# We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash.
|
|
# And we cannot untar it, because then we cannot preserve permissions ecc.
|
|
tarsum = runCommand "tarsum" {
|
|
buildInputs = [ goPackages.go ];
|
|
} ''
|
|
mkdir tarsum
|
|
cd tarsum
|
|
|
|
cp ${./tarsum.go} tarsum.go
|
|
export GOPATH=$(pwd)
|
|
mkdir src
|
|
ln -sT ${docker.src}/pkg/tarsum src/tarsum
|
|
go build
|
|
|
|
cp tarsum $out
|
|
'';
|
|
|
|
# buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM
|
|
mergeDrvs = { drvs, onlyDeps ? false }:
|
|
runCommand "merge-drvs" {
|
|
inherit drvs onlyDeps;
|
|
} ''
|
|
if [ -n "$onlyDeps" ]; then
|
|
echo $drvs > $out
|
|
exit 0
|
|
fi
|
|
|
|
mkdir $out
|
|
for drv in $drvs; do
|
|
echo Merging $drv
|
|
if [ -d "$drv" ]; then
|
|
cp -drf --preserve=mode -f $drv/* $out/
|
|
else
|
|
tar -C $out -xpf $drv || true
|
|
fi
|
|
done
|
|
'';
|
|
|
|
mkTarball = { name ? "docker-tar", drv, onlyDeps ? false }:
|
|
runCommand "${name}.tar.gz" rec {
|
|
inherit drv onlyDeps;
|
|
|
|
drvClosure = writeReferencesToFile drv;
|
|
|
|
} ''
|
|
while read dep; do
|
|
echo Copying $dep
|
|
dir="$(dirname "$dep")"
|
|
mkdir -p "rootfs/$dir"
|
|
cp -drf --preserve=mode $dep "rootfs/$dir/"
|
|
done < "$drvClosure"
|
|
|
|
if [ -z "$onlyDeps" ]; then
|
|
cp -drf --preserve=mode $drv/* rootfs/
|
|
fi
|
|
|
|
tar -C rootfs/ -cpzf $out .
|
|
'';
|
|
|
|
shellScript = text:
|
|
writeScript "script.sh" ''
|
|
#!${stdenv.shell}
|
|
set -e
|
|
export PATH=${coreutils}/bin:/bin
|
|
|
|
${text}
|
|
'';
|
|
|
|
shadowSetup = ''
|
|
export PATH=${shadow}/bin:$PATH
|
|
mkdir -p /etc/pam.d
|
|
if [ ! -f /etc/passwd ]; then
|
|
echo "root:x:0:0::/root:/bin/sh" > /etc/passwd
|
|
echo "root:!x:::::::" > /etc/shadow
|
|
fi
|
|
if [ ! -f /etc/group ]; then
|
|
echo "root:x:0:" > /etc/group
|
|
echo "root:x::" > /etc/gshadow
|
|
fi
|
|
if [ ! -f /etc/pam.d/other ]; then
|
|
cat > /etc/pam.d/other <<EOF
|
|
account sufficient pam_unix.so
|
|
auth sufficient pam_rootok.so
|
|
password requisite pam_unix.so nullok sha512
|
|
session required pam_unix.so
|
|
EOF
|
|
fi
|
|
if [ ! -f /etc/login.defs ]; then
|
|
touch /etc/login.defs
|
|
fi
|
|
'';
|
|
|
|
# Append to tar instead of unpacking
|
|
mergeTarballs = tarballs:
|
|
runCommand "merge-tars" { inherit tarballs; } ''
|
|
mkdir tmp
|
|
for tb in $tarballs; do
|
|
tar -C tmp -xkpf $tb
|
|
done
|
|
tar -C tmp -cpzf $out .
|
|
'';
|
|
|
|
runWithOverlay = { name , fromImage ? null, fromImageName ? null, fromImageTag ? null
|
|
, diskSize ? 1024, preMount ? "", postMount ? "", postUmount ? "" }:
|
|
vmTools.runInLinuxVM (
|
|
runCommand name {
|
|
preVM = vmTools.createEmptyImage { size = diskSize; fullName = "docker-run-disk"; };
|
|
|
|
inherit fromImage fromImageName fromImageTag;
|
|
|
|
buildInputs = [ utillinux e2fsprogs jshon ];
|
|
} ''
|
|
rm -rf $out
|
|
|
|
mkdir disk
|
|
mkfs /dev/${vmTools.hd}
|
|
mount /dev/${vmTools.hd} disk
|
|
cd disk
|
|
|
|
if [ -n "$fromImage" ]; then
|
|
echo Unpacking base image
|
|
mkdir image
|
|
tar -C image -xpf "$fromImage"
|
|
|
|
if [ -z "$fromImageName" ]; then
|
|
fromImageName=$(jshon -k < image/repositories|head -n1)
|
|
fi
|
|
if [ -z "$fromImageTag" ]; then
|
|
fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1)
|
|
fi
|
|
parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories)
|
|
fi
|
|
|
|
lowerdir=""
|
|
while [ -n "$parentID" ]; do
|
|
echo Unpacking layer $parentID
|
|
mkdir -p image/$parentID/layer
|
|
tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
|
|
rm image/$parentID/layer.tar
|
|
|
|
find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
|
|
|
|
lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
|
|
parentID=$(cat image/$parentID/json|(jshon -e parent -u 2>/dev/null || true))
|
|
done
|
|
|
|
mkdir work
|
|
mkdir layer
|
|
mkdir mnt
|
|
|
|
${preMount}
|
|
|
|
if [ -n "$lowerdir" ]; then
|
|
mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
|
|
else
|
|
mount --bind layer mnt
|
|
fi
|
|
|
|
${postMount}
|
|
|
|
umount mnt
|
|
|
|
pushd layer
|
|
find . -type c -exec bash -c 'name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"' \;
|
|
popd
|
|
|
|
${postUmount}
|
|
'');
|
|
|
|
exportImage = { name ? fromImage.name, fromImage, fromImageName ? null, fromImageTag ? null, diskSize ? 1024 }:
|
|
runWithOverlay {
|
|
inherit name fromImage fromImageName fromImageTag diskSize;
|
|
|
|
postMount = ''
|
|
echo Packing raw image
|
|
tar -C mnt -czf $out .
|
|
'';
|
|
};
|
|
|
|
mkPureLayer = { baseJson, contents ? null, extraCommands ? "" }:
|
|
runCommand "docker-layer" {
|
|
inherit baseJson contents extraCommands;
|
|
|
|
buildInputs = [ jshon ];
|
|
} ''
|
|
mkdir layer
|
|
if [ -n "$contents" ]; then
|
|
echo Adding contents
|
|
for c in $contents; do
|
|
cp -drf $c/* layer/
|
|
chmod -R ug+w layer/
|
|
done
|
|
fi
|
|
|
|
pushd layer
|
|
${extraCommands}
|
|
popd
|
|
|
|
echo Packing layer
|
|
mkdir $out
|
|
tar -C layer -cf $out/layer.tar .
|
|
ts=$(${tarsum} < $out/layer.tar)
|
|
cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
|
|
echo -n "1.0" > $out/VERSION
|
|
'';
|
|
|
|
mkRootLayer = { runAsRoot, baseJson, fromImage ? null, fromImageName ? null, fromImageTag ? null
|
|
, diskSize ? 1024, contents ? null, extraCommands ? "" }:
|
|
let runAsRootScript = writeScript "run-as-root.sh" runAsRoot;
|
|
in runWithOverlay {
|
|
name = "docker-layer";
|
|
|
|
inherit fromImage fromImageName fromImageTag diskSize;
|
|
|
|
preMount = lib.optionalString (contents != null) ''
|
|
echo Adding contents
|
|
for c in ${builtins.toString contents}; do
|
|
cp -drf $c/* layer/
|
|
chmod -R ug+w layer/
|
|
done
|
|
'';
|
|
|
|
postMount = ''
|
|
mkdir -p mnt/{dev,proc,sys,nix/store}
|
|
mount --rbind /dev mnt/dev
|
|
mount --rbind /sys mnt/sys
|
|
mount --rbind /nix/store mnt/nix/store
|
|
|
|
unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript}
|
|
umount -R mnt/dev mnt/sys mnt/nix/store
|
|
rmdir --ignore-fail-on-non-empty mnt/dev mnt/proc mnt/sys mnt/nix/store mnt/nix
|
|
'';
|
|
|
|
postUmount = ''
|
|
pushd layer
|
|
${extraCommands}
|
|
popd
|
|
|
|
echo Packing layer
|
|
mkdir $out
|
|
tar -C layer -cf $out/layer.tar .
|
|
ts=$(${tarsum} < $out/layer.tar)
|
|
cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
|
|
echo -n "1.0" > $out/VERSION
|
|
'';
|
|
};
|
|
|
|
# 1. extract the base image
|
|
# 2. create the layer
|
|
# 3. add layer deps to the layer itself, diffing with the base image
|
|
# 4. compute the layer id
|
|
# 5. put the layer in the image
|
|
# 6. repack the image
|
|
buildImage = args@{ name, tag ? "latest"
|
|
, fromImage ? null, fromImageName ? null, fromImageTag ? null
|
|
, contents ? null, tarballs ? [], config ? null
|
|
, runAsRoot ? null, diskSize ? 1024, extraCommands ? "" }:
|
|
|
|
let
|
|
|
|
baseJson = writeText "${name}-config.json" (builtins.toJSON {
|
|
created = "1970-01-01T00:00:01Z";
|
|
architecture = "amd64";
|
|
os = "linux";
|
|
config = config;
|
|
});
|
|
|
|
layer = (if runAsRoot == null
|
|
then mkPureLayer { inherit baseJson contents extraCommands; }
|
|
else mkRootLayer { inherit baseJson fromImage fromImageName fromImageTag contents runAsRoot diskSize extraCommands; });
|
|
depsTarball = mkTarball { name = "${name}-deps";
|
|
drv = layer;
|
|
onlyDeps = true; };
|
|
|
|
result = runCommand "${name}.tar.gz" {
|
|
buildInputs = [ jshon ];
|
|
|
|
imageName = name;
|
|
imageTag = tag;
|
|
inherit fromImage baseJson;
|
|
|
|
mergedTarball = if tarballs == [] then depsTarball else mergeTarballs ([ depsTarball ] ++ tarballs);
|
|
|
|
passthru = {
|
|
buildArgs = args;
|
|
};
|
|
} ''
|
|
mkdir image
|
|
touch baseFiles
|
|
if [ -n "$fromImage" ]; then
|
|
echo Unpacking base image
|
|
tar -C image -xpf "$fromImage"
|
|
|
|
if [ -z "$fromImageName" ]; then
|
|
fromImageName=$(jshon -k < image/repositories|head -n1)
|
|
fi
|
|
if [ -z "$fromImageTag" ]; then
|
|
fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1)
|
|
fi
|
|
parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories)
|
|
|
|
for l in image/*/layer.tar; do
|
|
tar -tf $l >> baseFiles
|
|
done
|
|
fi
|
|
|
|
chmod -R ug+rw image
|
|
|
|
mkdir temp
|
|
cp ${layer}/* temp/
|
|
chmod ug+w temp/*
|
|
|
|
echo Adding dependencies
|
|
tar -tf temp/layer.tar >> baseFiles
|
|
tar -tf "$mergedTarball" | grep -v ${layer} > layerFiles
|
|
if [ "$(wc -l layerFiles|cut -d ' ' -f 1)" -gt 3 ]; then
|
|
sed -i -e 's|^[\./]\+||' baseFiles layerFiles
|
|
comm <(sort -n baseFiles|uniq) <(sort -n layerFiles|uniq) -1 -3 > newFiles
|
|
mkdir deps
|
|
pushd deps
|
|
tar -xpf "$mergedTarball" --no-recursion --files-from ../newFiles 2>/dev/null || true
|
|
tar -rf ../temp/layer.tar --no-recursion --files-from ../newFiles 2>/dev/null || true
|
|
popd
|
|
else
|
|
echo No new deps, no diffing needed
|
|
fi
|
|
|
|
echo Adding meta
|
|
|
|
if [ -n "$parentID" ]; then
|
|
cat temp/json | jshon -s "$parentID" -i parent > tmpjson
|
|
mv tmpjson temp/json
|
|
fi
|
|
|
|
layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
|
|
size=$(stat --printf="%s" temp/layer.tar)
|
|
cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
|
|
mv tmpjson temp/json
|
|
|
|
mv temp image/$layerID
|
|
|
|
jshon -n object \
|
|
-n object -s "$layerID" -i "$imageTag" \
|
|
-i "$imageName" > image/repositories
|
|
|
|
chmod -R a-w image
|
|
|
|
echo Cooking the image
|
|
tar -C image -czf $out .
|
|
'';
|
|
|
|
in
|
|
|
|
result;
|
|
|
|
}
|