Merge pull request #256226 from ElvishJerricco/systemd-stage-1-testing-backdoor

nixos/doc/manual/release-notes/rl-2311.section.md

@@ -556,3 +556,5 @@ The module update takes care of the new config syntax and the data itself (user
 - `teleport` has been upgraded from major version 12 to major version 14. Please see upstream [upgrade instructions](https://goteleport.com/docs/management/operations/upgrading/) and release notes for versions [13](https://goteleport.com/docs/changelog/#1300-050823) and [14](https://goteleport.com/docs/changelog/#1400-092023). Note that Teleport does not officially support upgrades across more than one major version at a time. If you're running Teleport server components, it is recommended to first upgrade to an intermediate 13.x version by setting `services.teleport.package = pkgs.teleport_13`. Afterwards, this option can be removed to upgrade to the default version (14).
 
 - The Linux kernel module `msr` (see [`msr(4)`](https://man7.org/linux/man-pages/man4/msr.4.html)), which provides an interface to read and write the model-specific registers (MSRs) of an x86 CPU, can now be configured via `hardware.cpu.x86.msr`.
+
+- There is a new NixOS option when writing NixOS tests `testing.initrdBackdoor`, that enables `backdoor.service` in initrd. Requires `boot.initrd.systemd.enable` to be enabled. Boot will pause in stage 1 at `initrd.target`, and will listen for commands from the `Machine` python interface, just like stage 2 normally does. This enables commands to be sent to test and debug stage 1. Use `machine.switch_root()` to leave stage 1 and proceed to stage 2.

nixos/lib/test-driver/test_driver/machine.py

@@ -1278,3 +1278,19 @@ class Machine:
     def run_callbacks(self) -> None:
         for callback in self.callbacks:
             callback()
+
+    def switch_root(self) -> None:
+        """
+        Transition from stage 1 to stage 2. This requires the
+        machine to be configured with `testing.initrdBackdoor = true`
+        and `boot.initrd.systemd.enable = true`.
+        """
+        self.wait_for_unit("initrd.target")
+        self.execute(
+            "systemctl isolate --no-block initrd-switch-root.target 2>/dev/null >/dev/null",
+            check_return=False,
+            check_output=False,
+        )
+        self.wait_for_console_text(r"systemd\[1\]:.*Switching root\.")
+        self.connected = False
+        self.connect()

nixos/modules/testing/test-instrumentation.nix

@@ -6,15 +6,15 @@
 with lib;
 
 let
+  cfg = config.testing;
+
   qemu-common = import ../../lib/qemu-common.nix { inherit lib pkgs; };
-in
 
-{
-
-  config = {
-
-    systemd.services.backdoor =
-      { wantedBy = [ "multi-user.target" ];
+  backdoorService = {
+    wantedBy = [ "sysinit.target" ];
+    unitConfig.DefaultDependencies = false;
+    conflicts = [ "shutdown.target" "initrd-switch-root.target" ];
+    before = [ "shutdown.target" "initrd-switch-root.target" ];
     requires = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ];
     after = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ];
     script =
@@ -23,7 +23,9 @@ in
         export HOME=/root
         export DISPLAY=:0.0
 
+        if [[ -e /etc/profile ]]; then
             source /etc/profile
+        fi
 
         # Don't use a pager when executing backdoor
         # actions. Because we use a tty, commands like systemctl
@@ -50,6 +52,64 @@ in
       serviceConfig.KillSignal = "SIGHUP";
   };
 
+in
+
+{
+
+  options.testing = {
+
+    initrdBackdoor = lib.mkEnableOption (lib.mdDoc ''
+      enable backdoor.service in initrd. Requires
+      boot.initrd.systemd.enable to be enabled. Boot will pause in
+      stage 1 at initrd.target, and will listen for commands from the
+      Machine python interface, just like stage 2 normally does. This
+      enables commands to be sent to test and debug stage 1. Use
+      machine.switch_root() to leave stage 1 and proceed to stage 2.
+    '');
+
+  };
+
+  config = {
+
+    assertions = [
+      {
+        assertion = cfg.initrdBackdoor -> config.boot.initrd.systemd.enable;
+        message = ''
+          testing.initrdBackdoor requires boot.initrd.systemd.enable to be enabled.
+        '';
+      }
+    ];
+
+    systemd.services.backdoor = backdoorService;
+
+    boot.initrd.systemd = lib.mkMerge [
+      {
+        contents."/etc/systemd/journald.conf".text = ''
+          [Journal]
+          ForwardToConsole=yes
+          MaxLevelConsole=debug
+        '';
+
+        extraConfig = config.systemd.extraConfig;
+      }
+
+      (lib.mkIf cfg.initrdBackdoor {
+        # Implemented in machine.switch_root(). Suppress the unit by
+        # making it a noop without removing it, which would break
+        # initrd-parse-etc.service
+        services.initrd-cleanup.serviceConfig.ExecStart = [
+          # Reset
+          ""
+          # noop
+          "/bin/true"
+        ];
+
+        services.backdoor = backdoorService;
+
+        contents."/usr/bin/env".source = "${pkgs.coreutils}/bin/env";
+      })
+    ];
+
     # Prevent agetty from being instantiated on the serial device, since it
     # interferes with the backdoor (writes to it will randomly fail
     # with EIO).  Likewise for hvc0.
@@ -104,12 +164,6 @@ in
         MaxLevelConsole=debug
       '';
 
-    boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
-      [Journal]
-      ForwardToConsole=yes
-      MaxLevelConsole=debug
-    '';
-
     systemd.extraConfig = ''
       # Don't clobber the console with duplicate systemd messages.
       ShowStatus=no
@@ -123,8 +177,6 @@ in
       DefaultDeviceTimeoutSec=300
     '';
 
-    boot.initrd.systemd.extraConfig = config.systemd.extraConfig;
-
     boot.consoleLogLevel = 7;
 
     # Prevent tests from accessing the Internet.

nixos/tests/systemd-initrd-modprobe.nix

@@ -2,6 +2,7 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: {
   name = "systemd-initrd-modprobe";
 
   nodes.machine = { pkgs, ... }: {
+    testing.initrdBackdoor = true;
     boot.initrd.systemd.enable = true;
     boot.initrd.kernelModules = [ "loop" ]; # Load module in initrd.
     boot.extraModprobeConfig = ''
@@ -10,6 +11,12 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: {
   };
 
   testScript = ''
+    machine.wait_for_unit("initrd.target")
+    max_loop = machine.succeed("cat /sys/module/loop/parameters/max_loop")
+    assert int(max_loop) == 42, "Parameter should be respected for initrd kernel modules"
+
+    # Make sure it sticks in stage 2
+    machine.switch_root()
     machine.wait_for_unit("multi-user.target")
     max_loop = machine.succeed("cat /sys/module/loop/parameters/max_loop")
     assert int(max_loop) == 42, "Parameter should be respected for initrd kernel modules"

nixos/tests/systemd-initrd-networkd-ssh.nix

@@ -4,34 +4,16 @@ import ./make-test-python.nix ({ lib, ... }: {
 
   nodes = {
     server = { config, pkgs, ... }: {
-      environment.systemPackages = [ pkgs.cryptsetup ];
-      boot.loader.systemd-boot.enable = true;
-      boot.loader.timeout = 0;
-      virtualisation = {
-        emptyDiskImages = [ 4096 ];
-        useBootLoader = true;
-        # Booting off the encrypted disk requires an available init script from
-        # the Nix store
-        mountHostNixStore = true;
-        useEFIBoot = true;
-      };
-
-      specialisation.encrypted-root.configuration = {
-        virtualisation.rootDevice = "/dev/mapper/root";
-        virtualisation.fileSystems."/".autoFormat = true;
-        boot.initrd.luks.devices = lib.mkVMOverride {
-          root.device = "/dev/vdb";
-        };
+      testing.initrdBackdoor = true;
       boot.initrd.systemd.enable = true;
+      boot.initrd.systemd.contents."/etc/msg".text = "foo";
       boot.initrd.network = {
         enable = true;
         ssh = {
           enable = true;
           authorizedKeys = [ (lib.readFile ./initrd-network-ssh/id_ed25519.pub) ];
           port = 22;
-            # Terrible hack so it works with useBootLoader
-            hostKeys = [ { outPath = "${./initrd-network-ssh/ssh_host_ed25519_key}"; } ];
-          };
+          hostKeys = [ ./initrd-network-ssh/ssh_host_ed25519_key ];
         };
       };
     };
@@ -63,24 +45,16 @@ import ./make-test-python.nix ({ lib, ... }: {
         status, _ = client.execute("nc -z server 22")
         return status == 0
 
-    server.wait_for_unit("multi-user.target")
-    server.succeed(
-        "echo somepass | cryptsetup luksFormat --type=luks2 /dev/vdb",
-        "bootctl set-default nixos-generation-1-specialisation-encrypted-root.conf",
-        "sync",
-    )
-    server.shutdown()
-    server.start()
-
     client.wait_for_unit("network.target")
     with client.nested("waiting for SSH server to come up"):
         retry(ssh_is_up)
 
-    client.succeed(
-        "echo somepass | ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'systemd-tty-ask-password-agent' & exit"
+    msg = client.succeed(
+        "ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'cat /etc/msg'"
     )
+    assert "foo" in msg
 
+    server.switch_root()
     server.wait_for_unit("multi-user.target")
-    server.succeed("mount | grep '/dev/mapper/root on /'")
   '';
 })

nixos/tests/systemd-initrd-networkd.nix

@@ -1,14 +1,36 @@
-import ./make-test-python.nix ({ pkgs, lib, ... }: {
-  name = "systemd-initrd-network";
-  meta.maintainers = [ lib.maintainers.elvishjerricco ];
+{ system ? builtins.currentSystem
+, config ? {}
+, pkgs ? import ../.. { inherit system config; }
+, lib ? pkgs.lib
+}:
 
-  nodes = let
-    mkFlushTest = flush: script: { ... }: {
-      boot.initrd.systemd.enable = true;
-      boot.initrd.network = {
+with import ../lib/testing-python.nix { inherit system pkgs; };
+
+let
+  inherit (lib.maintainers) elvishjerricco;
+
+  common = {
+    boot.initrd.systemd = {
       enable = true;
-        flushBeforeStage2 = flush;
+      network.wait-online.timeout = 10;
+      network.wait-online.anyInterface = true;
+      targets.network-online.requiredBy = [ "initrd.target" ];
+      services.systemd-networkd-wait-online.requiredBy =
+        [ "network-online.target" ];
+      initrdBin = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ];
     };
+    testing.initrdBackdoor = true;
+    boot.initrd.network.enable = true;
+  };
+
+  mkFlushTest = flush: script: makeTest {
+    name = "systemd-initrd-network-${lib.optionalString (!flush) "no-"}flush";
+    meta.maintainers = [ elvishjerricco ];
+
+    nodes.machine = {
+      imports = [ common ];
+
+      boot.initrd.network.flushBeforeStage2 = flush;
       systemd.services.check-flush = {
         requiredBy = ["multi-user.target"];
         before = ["network-pre.target" "multi-user.target"];
@@ -19,32 +41,40 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
         inherit script;
       };
     };
-  in {
-    basic = { ... }: {
-      boot.initrd.network.enable = true;
 
-      boot.initrd.systemd = {
-        enable = true;
-        # Enable network-online to fail the test in case of timeout
-        network.wait-online.timeout = 10;
-        network.wait-online.anyInterface = true;
-        targets.network-online.requiredBy = [ "initrd.target" ];
-        services.systemd-networkd-wait-online.requiredBy =
-          [ "network-online.target" ];
+    testScript = ''
+      machine.wait_for_unit("network-online.target")
+      machine.succeed(
+          "ip addr | grep 10.0.2.15",
+          "ping -c1 10.0.2.2",
+      )
+      machine.switch_root()
 
-          initrdBin = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ];
-          services.check = {
-            requiredBy = [ "initrd.target" ];
-            before = [ "initrd.target" ];
-            after = [ "network-online.target" ];
-            serviceConfig.Type = "oneshot";
-            path = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ];
-            script = ''
-              ip addr | grep 10.0.2.15 || exit 1
-              ping -c1 10.0.2.2 || exit 1
+      machine.wait_for_unit("multi-user.target")
     '';
   };
-      };
+
+in {
+  basic = makeTest {
+    name = "systemd-initrd-network";
+    meta.maintainers = [ elvishjerricco ];
+
+    nodes.machine = common;
+
+    testScript = ''
+      machine.wait_for_unit("network-online.target")
+      machine.succeed(
+          "ip addr | grep 10.0.2.15",
+          "ping -c1 10.0.2.2",
+      )
+      machine.switch_root()
+
+      # Make sure the systemd-network user was set correctly in initrd
+      machine.wait_for_unit("multi-user.target")
+      machine.succeed("[ $(stat -c '%U,%G' /run/systemd/netif/links) = systemd-network,systemd-network ]")
+      machine.succeed("ip addr show >&2")
+      machine.succeed("ip route show >&2")
+    '';
   };
 
   doFlush = mkFlushTest true ''
@@ -60,16 +90,4 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
       exit 1
     fi
   '';
-  };
-
-  testScript = ''
-    start_all()
-    basic.wait_for_unit("multi-user.target")
-    doFlush.wait_for_unit("multi-user.target")
-    dontFlush.wait_for_unit("multi-user.target")
-    # Make sure the systemd-network user was set correctly in initrd
-    basic.succeed("[ $(stat -c '%U,%G' /run/systemd/netif/links) = systemd-network,systemd-network ]")
-    basic.succeed("ip addr show >&2")
-    basic.succeed("ip route show >&2")
-  '';
-})
+}

nixos/tests/systemd-initrd-simple.nix

@@ -2,16 +2,19 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: {
   name = "systemd-initrd-simple";
 
   nodes.machine = { pkgs, ... }: {
-    boot.initrd.systemd = {
-      enable = true;
-      emergencyAccess = true;
-    };
+    testing.initrdBackdoor = true;
+    boot.initrd.systemd.enable = true;
     virtualisation.fileSystems."/".autoResize = true;
   };
 
   testScript = ''
     import subprocess
 
+    with subtest("testing initrd backdoor"):
+        machine.wait_for_unit("initrd.target")
+        machine.succeed("systemctl status initrd-fs.target")
+        machine.switch_root()
+
     with subtest("handover to stage-2 systemd works"):
         machine.wait_for_unit("multi-user.target")
         machine.succeed("systemd-analyze | grep -q '(initrd)'")  # direct handover
@@ -37,6 +40,7 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: {
         subprocess.check_call(["qemu-img", "resize", "vm-state-machine/machine.qcow2", "+1G"])
 
         machine.start()
+        machine.switch_root()
         newAvail = machine.succeed("df --output=avail / | sed 1d")
 
         assert int(oldAvail) < int(newAvail), "File system did not grow"