Merge pull request #185231 from moduon/k3s-ha

nixos/k3s: streamline HA setup

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -436,6 +436,12 @@
           due to upstream dropping support.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <literal>k3s</literal> supports <literal>clusterInit</literal>
+          option, and it is enabled by default, for servers.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-22.11-notable-changes">

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -152,6 +152,8 @@ Use `configure.packages` instead.
 
 - `k3s` no longer supports docker as runtime due to upstream dropping support.
 
+- `k3s` supports `clusterInit` option, and it is enabled by default, for servers.
+
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
 
 ## Other Notable Changes {#sec-release-22.11-notable-changes}

nixos/modules/services/cluster/k3s/default.nix

@@ -25,7 +25,17 @@ in
     role = mkOption {
       description = lib.mdDoc ''
         Whether k3s should run as a server or agent.
-        Note that the server, by default, also runs as an agent.
+
+        If it's a server:
+
+        - By default it also runs workloads as an agent.
+        - Starts by default as a standalone server using an embedded sqlite datastore.
+        - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
+        - Configure `serverAddr` to join an already-initialized HA cluster.
+
+        If it's an agent:
+
+        - `serverAddr` is required.
       '';
       default = "server";
       type = types.enum [ "server" "agent" ];
@@ -33,15 +43,44 @@ in
 
     serverAddr = mkOption {
       type = types.str;
-      description = lib.mdDoc "The k3s server to connect to. This option only makes sense for an agent.";
+      description = lib.mdDoc ''
+        The k3s server to connect to.
+
+        Servers and agents need to communicate each other. Read
+        [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
+        to know how to configure the firewall.
+      '';
       example = "https://10.0.0.10:6443";
       default = "";
     };
 
+    clusterInit = mkOption {
+      type = types.bool;
+      default = false;
+      description = lib.mdDoc ''
+        Initialize HA cluster using an embedded etcd datastore.
+
+        If this option is `false` and `role` is `server`
+
+        On a server that was using the default embedded sqlite backend,
+        enabling this option will migrate to an embedded etcd DB.
+
+        If an HA cluster using the embedded etcd datastore was already initialized,
+        this option has no effect.
+
+        This option only makes sense in a server that is not connecting to another server.
+
+        If you are configuring an HA cluster with an embedded etcd,
+        the 1st server must have `clusterInit = true`
+        and other servers must connect to it using `serverAddr`.
+      '';
+    };
+
     token = mkOption {
       type = types.str;
       description = lib.mdDoc ''
-        The k3s token to use when connecting to the server. This option only makes sense for an agent.
+        The k3s token to use when connecting to a server.
+
         WARNING: This option will expose store your token unencrypted world-readable in the nix store.
         If this is undesired use the tokenFile option instead.
       '';
@@ -50,7 +89,7 @@ in
 
     tokenFile = mkOption {
       type = types.nullOr types.path;
-      description = lib.mdDoc "File path containing k3s token to use when connecting to the server. This option only makes sense for an agent.";
+      description = lib.mdDoc "File path containing k3s token to use when connecting to the server.";
       default = null;
     };
 
@@ -86,6 +125,14 @@ in
         assertion = cfg.role == "agent" -> cfg.configPath != null || cfg.tokenFile != null || cfg.token != "";
         message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
       }
+      {
+        assertion = cfg.role == "agent" -> !cfg.disableAgent;
+        message = "disableAgent must be false if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !cfg.clusterInit;
+        message = "clusterInit must be false if role is 'agent'";
+      }
     ];
 
     environment.systemPackages = [ config.services.k3s.package ];
@@ -111,6 +158,7 @@ in
           [
             "${cfg.package}/bin/k3s ${cfg.role}"
           ]
+          ++ (optional cfg.clusterInit "--cluster-init")
           ++ (optional cfg.disableAgent "--disable-agent")
           ++ (optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
           ++ (optional (cfg.token != "") "--token ${cfg.token}")

nixos/tests/k3s/multi-node.nix

@@ -53,9 +53,10 @@ import ../make-test-python.nix ({ pkgs, ... }:
           enable = true;
           role = "server";
           package = pkgs.k3s;
+          clusterInit = true;
           extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local --node-ip 192.168.1.1";
         };
-        networking.firewall.allowedTCPPorts = [ 6443 ];
+        networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ];
         networking.firewall.allowedUDPPorts = [ 8472 ];
         networking.firewall.trustedInterfaces = [ "flannel.1" ];
         networking.useDHCP = false;
@@ -65,6 +66,28 @@ import ../make-test-python.nix ({ pkgs, ... }:
         ];
       };
 
+      server2 = { pkgs, ... }: {
+        environment.systemPackages = with pkgs; [ gzip jq ];
+        virtualisation.memorySize = 1536;
+        virtualisation.diskSize = 4096;
+
+        services.k3s = {
+          inherit tokenFile;
+          enable = true;
+          serverAddr = "https://192.168.1.1:6443";
+          clusterInit = false;
+          extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local --node-ip 192.168.1.3";
+        };
+        networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ];
+        networking.firewall.allowedUDPPorts = [ 8472 ];
+        networking.firewall.trustedInterfaces = [ "flannel.1" ];
+        networking.useDHCP = false;
+        networking.defaultGateway = "192.168.1.3";
+        networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
+          { address = "192.168.1.3"; prefixLength = 24; }
+        ];
+      };
+
       agent = { pkgs, ... }: {
         virtualisation.memorySize = 1024;
         virtualisation.diskSize = 2048;
@@ -72,7 +95,7 @@ import ../make-test-python.nix ({ pkgs, ... }:
           inherit tokenFile;
           enable = true;
           role = "agent";
-          serverAddr = "https://192.168.1.1:6443";
+          serverAddr = "https://192.168.1.3:6443";
           extraFlags = "--pause-image test.local/pause:local --node-ip 192.168.1.2";
         };
         networking.firewall.allowedTCPPorts = [ 6443 ];
@@ -91,9 +114,9 @@ import ../make-test-python.nix ({ pkgs, ... }:
     };
 
     testScript = ''
-      start_all()
-      machines = [server, agent]
+      machines = [server, server2, agent]
       for m in machines:
+          m.start()
           m.wait_for_unit("k3s")
 
       # wait for the agent to show up