Merge pull request #271067 from nikstur/sysinit-reactivation

nixos/switch-to-configuration: add sysinit-reactivation.target

nixos/doc/manual/development/unit-handling.section.md

@@ -63,3 +63,42 @@ checks:
     is **restart**ed with the others. If it is set, both the service and the
     socket are **stop**ped and the socket is **start**ed, leaving socket
     activation to start the service when it's needed.
+
+## Sysinit reactivation {#sec-sysinit-reactivation}
+
+[`sysinit.target`](https://www.freedesktop.org/software/systemd/man/latest/systemd.special.html#sysinit.target)
+is a systemd target that encodes system initialization (i.e. early startup). A
+few units that need to run very early in the bootup process are ordered to
+finish before this target is reached. Probably the most notable one of these is
+`systemd-tmpfiles-setup.service`. We will refer to these units as "sysinit
+units".
+
+"Normal" systemd units, by default, are ordered AFTER `sysinit.target`. In
+other words, these "normal" units expect all services ordered before
+`sysinit.target` to have finished without explicity declaring this dependency
+relationship for each dependency. See the [systemd
+bootup](https://www.freedesktop.org/software/systemd/man/latest/bootup.html)
+for more details on the bootup process.
+
+When restarting both a unit ordered before `sysinit.target` as well as one
+after, this presents a problem because they would be started at the same time
+as they do not explicitly declare their dependency relations.
+
+To solve this, NixOS has an artificial `sysinit-reactivation.target` which
+allows you to ensure that services ordered before `sysinit.target` are
+restarted correctly. This applies both to the ordering between these sysinit
+services as well as ensuring that sysinit units are restarted before "normal"
+units.
+
+To make an existing sysinit service restart correctly during system switch, you
+have to declare:
+
+```nix
+systemd.services.my-sysinit = {
+  requiredBy = [ "sysinit-reactivation.target" ];
+  before = [ "sysinit-reactivation.target" ];
+  restartTriggers = [ config.environment.etc."my-sysinit.d".source ];
+};
+```
+
+You need to configure appropriate `restartTriggers` specific to your service.

nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md

@@ -37,7 +37,7 @@ of actions is always the same:
 - Forget about the failed state of units (`systemctl reset-failed`)
 - Reload systemd (`systemctl daemon-reload`)
 - Reload systemd user instances (`systemctl --user daemon-reload`)
-- Set up tmpfiles (`systemd-tmpfiles --create`)
+- Reactivate sysinit (`systemctl restart sysinit-reactivation.target`)
 - Reload units (`systemctl reload`)
 - Restart units (`systemctl restart`)
 - Start units (`systemctl start`)

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -114,6 +114,11 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m
 
 - The executable file names for `firefox-devedition`, `firefox-beta`, `firefox-esr` now matches their package names, which is consistent with the `firefox-*-bin` packages. The desktop entries are also updated so that you can have multiple editions of firefox in your app launcher.
 
+- switch-to-configuration does not directly call systemd-tmpfiles anymore.
+  Instead, the new artificial sysinit-reactivation.target is introduced which
+  allows to restart multiple services that are ordered before sysinit.target
+  and respect the ordering between the services.
+
 - The `systemd.oomd` module behavior is changed as:
 
   - Raise ManagedOOMMemoryPressureLimit from 50% to 80%. This should make systemd-oomd kill things less often, and fix issues like [this](https://pagure.io/fedora-workstation/issue/358).

nixos/modules/system/activation/switch-to-configuration.pl

@@ -889,9 +889,15 @@ while (my $f = <$list_active_users>) {
 
 close($list_active_users) || die("Unable to close the file handle to loginctl");
 
-# Set the new tmpfiles
-print STDERR "setting up tmpfiles\n";
-system("$new_systemd/bin/systemd-tmpfiles", "--create", "--remove", "--exclude-prefix=/dev") == 0 or $res = 3;
+# Restart sysinit-reactivation.target.
+# This target only exists to restart services ordered before sysinit.target. We
+# cannot use X-StopOnReconfiguration to restart sysinit.target because then ALL
+# services of the system would be restarted since all normal services have a
+# default dependency on sysinit.target. sysinit-reactivation.target ensures
+# that services ordered BEFORE sysinit.target get re-started in the correct
+# order. Ordering between these services is respected.
+print STDERR "restarting sysinit-reactivation.target\n";
+system("$new_systemd/bin/systemctl", "restart", "sysinit-reactivation.target") == 0 or $res = 4;
 
 # Before reloading we need to ensure that the units are still active. They may have been
 # deactivated because one of their requirements got stopped. If they are inactive

nixos/modules/system/boot/systemd.nix

@@ -569,6 +569,13 @@ in
         unitConfig.X-StopOnReconfiguration = true;
       };
 
+    # This target only exists so that services ordered before sysinit.target
+    # are restarted in the correct order, notably BEFORE the other services,
+    # when switching configurations.
+    systemd.targets.sysinit-reactivation = {
+      description = "Reactivate sysinit units";
+    };
+
     systemd.units =
          mapAttrs' (n: v: nameValuePair "${n}.path"    (pathToUnit    n v)) cfg.paths
       // mapAttrs' (n: v: nameValuePair "${n}.service" (serviceToUnit n v)) cfg.services

nixos/modules/system/boot/systemd/tmpfiles.nix

@@ -150,6 +150,41 @@ in
       "systemd-tmpfiles-setup.service"
     ];
 
+    # Allow systemd-tmpfiles to be restarted by switch-to-configuration. This
+    # service is not pulled into the normal boot process. It only exists for
+    # switch-to-configuration.
+    #
+    # This needs to be a separate unit because it does not execute
+    # systemd-tmpfiles with `--boot` as that is supposed to only be executed
+    # once at boot time.
+    #
+    # Keep this aligned with the upstream `systemd-tmpfiles-setup.service` unit.
+    systemd.services."systemd-tmpfiles-resetup" = {
+      description = "Re-setup tmpfiles on a system that is already running.";
+
+      requiredBy = [ "sysinit-reactivation.target" ];
+      after = [ "local-fs.target" "systemd-sysusers.service" "systemd-journald.service" ];
+      before = [ "sysinit-reactivation.target" "shutdown.target" ];
+      conflicts = [ "shutdown.target" ];
+      restartTriggers = [ config.environment.etc."tmpfiles.d".source ];
+
+      unitConfig.DefaultDependencies = false;
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "systemd-tmpfiles --create --remove --exclude-prefix=/dev";
+        SuccessExitStatus = "DATAERR CANTCREAT";
+        ImportCredential = [
+          "tmpfiles.*"
+          "loging.motd"
+          "login.issue"
+          "network.hosts"
+          "ssh.authorized_keys.root"
+        ];
+      };
+    };
+
     environment.etc = {
       "tmpfiles.d".source = (pkgs.symlinkJoin {
         name = "tmpfiles.d";

nixos/tests/all-tests.nix

@@ -819,6 +819,7 @@ in {
   syncthing-init = handleTest ./syncthing-init.nix {};
   syncthing-many-devices = handleTest ./syncthing-many-devices.nix {};
   syncthing-relay = handleTest ./syncthing-relay.nix {};
+  sysinit-reactivation = runTest ./sysinit-reactivation.nix;
   systemd = handleTest ./systemd.nix {};
   systemd-analyze = handleTest ./systemd-analyze.nix {};
   systemd-binfmt = handleTestOn ["x86_64-linux"] ./systemd-binfmt.nix {};

nixos/tests/sysinit-reactivation.nix

@@ -0,0 +1,107 @@
+# This runs to two scenarios but in one tests:
+# - A post-sysinit service needs to be restarted AFTER tmpfiles was restarted.
+# - A service needs to be restarted BEFORE tmpfiles is restarted
+
+{ lib, ... }:
+
+let
+  makeGeneration = generation: {
+    "${generation}".configuration = {
+      systemd.services.pre-sysinit-before-tmpfiles.environment.USER =
+        lib.mkForce "${generation}-tmpfiles-user";
+
+      systemd.services.pre-sysinit-after-tmpfiles.environment = {
+        NEEDED_PATH = lib.mkForce "/run/${generation}-needed-by-pre-sysinit-after-tmpfiles";
+        PATH_TO_CREATE = lib.mkForce "/run/${generation}-needed-by-post-sysinit";
+      };
+
+      systemd.services.post-sysinit.environment = {
+        NEEDED_PATH = lib.mkForce "/run/${generation}-needed-by-post-sysinit";
+        PATH_TO_CREATE = lib.mkForce "/run/${generation}-created-by-post-sysinit";
+      };
+
+      systemd.tmpfiles.settings.test = lib.mkForce {
+        "/run/${generation}-needed-by-pre-sysinit-after-tmpfiles".f.user =
+          "${generation}-tmpfiles-user";
+      };
+    };
+  };
+in
+
+{
+
+  name = "sysinit-reactivation";
+
+  meta.maintainers = with lib.maintainers; [ nikstur ];
+
+  nodes.machine = { config, lib, pkgs, ... }: {
+    systemd.services.pre-sysinit-before-tmpfiles = {
+      wantedBy = [ "sysinit.target" ];
+      requiredBy = [ "sysinit-reactivation.target" ];
+      before = [ "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ];
+      unitConfig.DefaultDependencies = false;
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      environment.USER = "tmpfiles-user";
+      script = "${pkgs.shadow}/bin/useradd $USER";
+    };
+
+    systemd.services.pre-sysinit-after-tmpfiles = {
+      wantedBy = [ "sysinit.target" ];
+      requiredBy = [ "sysinit-reactivation.target" ];
+      after = [ "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ];
+      unitConfig.DefaultDependencies = false;
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      environment = {
+        NEEDED_PATH = "/run/needed-by-pre-sysinit-after-tmpfiles";
+        PATH_TO_CREATE = "/run/needed-by-post-sysinit";
+      };
+      script = ''
+        if [[ -e $NEEDED_PATH ]]; then
+          touch $PATH_TO_CREATE
+        fi
+      '';
+    };
+
+    systemd.services.post-sysinit = {
+      wantedBy = [ "default.target" ];
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      environment = {
+        NEEDED_PATH = "/run/needed-by-post-sysinit";
+        PATH_TO_CREATE = "/run/created-by-post-sysinit";
+      };
+      script = ''
+        if [[ -e $NEEDED_PATH ]]; then
+          touch $PATH_TO_CREATE
+        fi
+      '';
+    };
+
+    systemd.tmpfiles.settings.test = {
+      "/run/needed-by-pre-sysinit-after-tmpfiles".f.user =
+        "tmpfiles-user";
+    };
+
+    specialisation = lib.mkMerge [
+      (makeGeneration "second")
+      (makeGeneration "third")
+    ];
+  };
+
+  testScript = { nodes, ... }: ''
+    def switch(generation):
+      toplevel = "${nodes.machine.system.build.toplevel}";
+      machine.succeed(f"{toplevel}/specialisation/{generation}/bin/switch-to-configuration switch")
+
+    machine.wait_for_unit("default.target")
+    machine.succeed("test -e /run/created-by-post-sysinit")
+
+    switch("second")
+    machine.succeed("test -e /run/second-created-by-post-sysinit")
+
+    switch("third")
+    machine.succeed("test -e /run/third-created-by-post-sysinit")
+  '';
+}