Merge pull request #109768 from cpcloud/nomad-datadir-cleanup
nixos/nomad: enforce specific data_dir semantics
This commit is contained in:
Bernardo Meurer
GitHub
commit
105b9eb1b8
2 changed files with 99 additions and 31 deletions
|
|
@ -66,6 +66,20 @@ in
|
|||
description = ''
|
||||
Configuration for Nomad. See the <link xlink:href="https://www.nomadproject.io/docs/configuration">documentation</link>
|
||||
for supported values.
|
||||
|
||||
Notes about <literal>data_dir</literal>:
|
||||
|
||||
If <literal>data_dir</literal> is set to a value other than the
|
||||
default value of <literal>"/var/lib/nomad"</literal> it is the Nomad
|
||||
cluster manager's responsibility to make sure that this directory
|
||||
exists and has the appropriate permissions.
|
||||
|
||||
Additionally, if <literal>dropPrivileges</literal> is
|
||||
<literal>true</literal> then <literal>data_dir</literal>
|
||||
<emphasis>cannot</emphasis> be customized. Setting
|
||||
<literal>dropPrivileges</literal> to <literal>true</literal> enables
|
||||
the <literal>DynamicUser</literal> feature of systemd which directly
|
||||
manages and operates on <literal>StateDirectory</literal>.
|
||||
'';
|
||||
example = literalExample ''
|
||||
{
|
||||
|
|
@ -109,25 +123,28 @@ in
|
|||
iptables
|
||||
]);
|
||||
|
||||
serviceConfig = {
|
||||
DynamicUser = cfg.dropPrivileges;
|
||||
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
|
||||
ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" +
|
||||
concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths;
|
||||
KillMode = "process";
|
||||
KillSignal = "SIGINT";
|
||||
LimitNOFILE = 65536;
|
||||
LimitNPROC = "infinity";
|
||||
OOMScoreAdjust = -1000;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 2;
|
||||
# Agrees with the default `data_dir = "/var/lib/nomad"` in `settings` above.
|
||||
StateDirectory = "nomad";
|
||||
TasksMax = "infinity";
|
||||
User = optionalString cfg.dropPrivileges "nomad";
|
||||
} // (optionalAttrs cfg.enableDocker {
|
||||
SupplementaryGroups = "docker"; # space-separated string
|
||||
});
|
||||
serviceConfig = mkMerge [
|
||||
{
|
||||
DynamicUser = cfg.dropPrivileges;
|
||||
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
|
||||
ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" +
|
||||
concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths;
|
||||
KillMode = "process";
|
||||
KillSignal = "SIGINT";
|
||||
LimitNOFILE = 65536;
|
||||
LimitNPROC = "infinity";
|
||||
OOMScoreAdjust = -1000;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 2;
|
||||
TasksMax = "infinity";
|
||||
}
|
||||
(mkIf cfg.enableDocker {
|
||||
SupplementaryGroups = "docker"; # space-separated string
|
||||
})
|
||||
(mkIf (cfg.settings.data_dir == "/var/lib/nomad") {
|
||||
StateDirectory = "nomad";
|
||||
})
|
||||
];
|
||||
|
||||
unitConfig = {
|
||||
StartLimitIntervalSec = 10;
|
||||
|
|
@ -135,6 +152,13 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.dropPrivileges -> cfg.settings.data_dir == "/var/lib/nomad";
|
||||
message = "settings.data_dir must be equal to \"/var/lib/nomad\" if dropPrivileges is true";
|
||||
}
|
||||
];
|
||||
|
||||
# Docker support requires the Docker daemon to be running.
|
||||
virtualisation.docker.enable = mkIf cfg.enableDocker true;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import ./make-test-python.nix (
|
|||
{ lib, ... }: {
|
||||
name = "nomad";
|
||||
nodes = {
|
||||
server = { pkgs, lib, ... }: {
|
||||
default_server = { pkgs, lib, ... }: {
|
||||
networking = {
|
||||
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{
|
||||
address = "192.168.1.1";
|
||||
|
|
@ -30,24 +30,68 @@ import ./make-test-python.nix (
|
|||
enableDocker = false;
|
||||
};
|
||||
};
|
||||
|
||||
custom_state_dir_server = { pkgs, lib, ... }: {
|
||||
networking = {
|
||||
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{
|
||||
address = "192.168.1.1";
|
||||
prefixLength = 16;
|
||||
}];
|
||||
};
|
||||
|
||||
environment.etc."nomad.custom.json".source =
|
||||
(pkgs.formats.json { }).generate "nomad.custom.json" {
|
||||
region = "universe";
|
||||
datacenter = "earth";
|
||||
};
|
||||
|
||||
services.nomad = {
|
||||
enable = true;
|
||||
dropPrivileges = false;
|
||||
|
||||
settings = {
|
||||
data_dir = "/nomad/data/dir";
|
||||
server = {
|
||||
enabled = true;
|
||||
bootstrap_expect = 1;
|
||||
};
|
||||
};
|
||||
|
||||
extraSettingsPaths = [ "/etc/nomad.custom.json" ];
|
||||
enableDocker = false;
|
||||
};
|
||||
|
||||
systemd.services.nomad.serviceConfig.ExecStartPre = "${pkgs.writeShellScript "mk_data_dir" ''
|
||||
set -euxo pipefail
|
||||
|
||||
${pkgs.coreutils}/bin/mkdir -p /nomad/data/dir
|
||||
''}";
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
server.wait_for_unit("nomad.service")
|
||||
def test_nomad_server(server):
|
||||
server.wait_for_unit("nomad.service")
|
||||
|
||||
# wait for healthy server
|
||||
server.wait_until_succeeds(
|
||||
"[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]"
|
||||
)
|
||||
# wait for healthy server
|
||||
server.wait_until_succeeds(
|
||||
"[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]"
|
||||
)
|
||||
|
||||
# wait for server liveness
|
||||
server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]")
|
||||
# wait for server liveness
|
||||
server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]")
|
||||
|
||||
# check the region
|
||||
server.succeed("nomad server members | grep -o universe")
|
||||
# check the region
|
||||
server.succeed("nomad server members | grep -o universe")
|
||||
|
||||
# check the datacenter
|
||||
server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]")
|
||||
# check the datacenter
|
||||
server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]")
|
||||
|
||||
|
||||
servers = [default_server, custom_state_dir_server]
|
||||
|
||||
for server in servers:
|
||||
test_nomad_server(server)
|
||||
'';
|
||||
}
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue