dockerTools.buildImage: support using a layered image in fromImage

Docker images used to be, essentially, a linked list of layers. Each
layer would have a tarball and a json document pointing to its parent,
and the image pointed to the top layer:

    imageA  ----> layerA
                    |
                    v
                  layerB
                    |
                    v
                  layerC

The current image spec changed this format to where the Image defined
the order and set of layers:

    imageA  ---> layerA
            |--> layerB
            `--> layerC

For backwards compatibility, docker produces images which follow both
specs: layers point to parents, and images also point to the entire
list:

    imageA  ---> layerA
            |      |
            |      v
            |--> layerB
            |      |
            |      v
            `--> layerC

This is nice for tooling which supported the older version and never
updated to support the newer format.

Our `buildImage` code only supported the old version, so in order for
`buildImage` to properly generate an image based on another image
with `fromImage`, the parent image's layers must fully support the old
mechanism.

This is not a problem in general, but is a problem with
`buildLayeredImage`.

`buildLayeredImage` creates images with newer image spec, because
individual store paths don't have a guaranteed parent layer. Including
a specific parent ID in the layer's json makes the output less likely
to cache hit when published or pulled.

This means until now, `buildLayeredImage` could not be the input to
`buildImage`.

The changes in this PR change `buildImage` to only use the layer's
manifest when locating parent IDs. This does break buildImage on
extremely old Docker images, though I do wonder how many of these
exist.

This work has been sponsored by Target.

nixos/tests/docker-tools.nix

@@ -62,5 +62,9 @@ import ./make-test.nix ({ pkgs, ... }: {
       # Ensure Layered Docker images work
       $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-image}'");
       $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-image.imageName}");
+
+      # Ensure building an image on top of a layered Docker images work
+      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-on-top}'");
+      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-on-top.imageName}");
     '';
 })

pkgs/build-support/docker/default.nix

@@ -188,22 +188,27 @@ rec {
         # Use the name and tag to get the parent ID field.
         parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
                    < image/repositories)
+
+        cat ./image/manifest.json  | jq -r '.[0].Layers | .[]' > layer-list
+      else
+        touch layer-list
       fi
 
       # Unpack all of the parent layers into the image.
       lowerdir=""
-      while [[ -n "$parentID" ]]; do
-        echo "Unpacking layer $parentID"
-        mkdir -p image/$parentID/layer
-        tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
-        rm image/$parentID/layer.tar
+      extractionID=0
+      for layerTar in $(cat layer-list); do
+        echo "Unpacking layer $layerTar"
+        extractionID=$((extractionID + 1))
 
-        find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
+        mkdir -p image/$extractionID/layer
+        tar -C image/$extractionID/layer -xpf $layerTar
+        rm $layerTar
+
+        find image/$extractionID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
 
         # Get the next lower directory and continue the loop.
-        lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
-        parentID=$(cat image/$parentID/json \
-                  | (jshon -e parent -u 2>/dev/null || true))
+        lowerdir=$lowerdir''${lowerdir:+:}image/$extractionID/layer
       done
 
       mkdir work
@@ -673,6 +678,9 @@ rec {
         if [[ -n "$fromImage" ]]; then
           echo "Unpacking base image..."
           tar -C image -xpf "$fromImage"
+
+          cat ./image/manifest.json  | jq -r '.[0].Layers | .[]' > layer-list
+
           # Do not import the base image configuration and manifest
           chmod a+w image image/*.json
           rm -f image/*.json
@@ -690,6 +698,8 @@ rec {
           for l in image/*/layer.tar; do
             ls_tar $l >> baseFiles
           done
+        else
+          touch layer-list
         fi
 
         chmod -R ug+rw image
@@ -742,17 +752,23 @@ rec {
         # Use the temp folder we've been working on to create a new image.
         mv temp image/$layerID
 
+        # Add the new layer ID to the beginning of the layer list
+        (
+          # originally this used `sed -i "1i$layerID" layer-list`, but
+          # would fail if layer-list was completely empty.
+          echo "$layerID/layer.tar"
+          cat layer-list
+        ) | ${pkgs.moreutils}/bin/sponge layer-list
+
         # Create image json and image manifest
         imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
         manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
-        currentID=$layerID
-        while [[ -n "$currentID" ]]; do
-          layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
+
+        for layerTar in $(cat ./layer-list); do
+          layerChecksum=$(sha256sum image/$layerTar | cut -d ' ' -f1)
           imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"$(jq -r .created ${baseJson})\"}] + .")
           imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
-          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
-
-          currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
+          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$layerTar\"] + .")
         done
 
         imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)

pkgs/build-support/docker/examples.nix

@@ -156,5 +156,24 @@ rec {
     name = "layered-image";
     tag = "latest";
     config.Cmd = [ "${pkgs.hello}/bin/hello" ];
+    contents = [ pkgs.hello pkgs.bash pkgs.coreutils ];
+  };
+
+  # 11. Create an image on top of a layered image
+  layered-on-top = pkgs.dockerTools.buildImage {
+    name = "layered-on-top";
+    tag = "latest";
+    fromImage = layered-image;
+    extraCommands = ''
+      mkdir ./example-output
+      chmod 777 ./example-output
+    '';
+    config = {
+      Env = [ "PATH=${pkgs.coreutils}/bin/" ];
+      WorkingDir = "/example-output";
+      Cmd = [
+        "${pkgs.bash}/bin/bash" "-c" "echo hello > foo; cat foo"
+      ];
+    };
   };
 }