4a9d9928dc
The `nix.*` options, apart from options for setting up the daemon itself, currently provide a lot of setting mappings for the Nix daemon configuration. The scope of the mapping yields convience, but the line where an option is considered essential is blurry. For instance, the `extra-sandbox-paths` mapping is provided without its primary consumer, and the corresponding `sandbox-paths` option is also not mapped. The current system increases the maintenance burden as maintainers have to closely follow upstream changes. In this case, there are two state versions of Nix which have to be maintained collectively, with different options avaliable. This commit aims to following the standard outlined in RFC 42[1] to implement a structural setting pattern. The Nix configuration is encoded at its core as key-value pairs which maps nicely to attribute sets, making it feasible to express in the Nix language itself. Some existing options are kept such as `buildMachines` and `registry` which present a simplified interface to managing the respective settings. The interface is exposed as `nix.settings`. Legacy configurations are mapped to their corresponding options under `nix.settings` for backwards compatibility. Various options settings in other nixos modules and relevant tests have been updated to use structural setting for consistency. The generation and validation of the configration file has been modified to use `writeTextFile` instead of `runCommand` for clarity. Note that validation is now mandatory as strict checking of options has been pushed down to the derivation level due to freeformType consuming unmatched options. Furthermore, validation can not occur when cross-compiling due to current limitations. A new option `publicHostKey` was added to the `buildMachines` submodule corresponding to the base64 encoded public host key settings exposed in the builder syntax. The build machine generation was subsequently rewritten to use `concatStringsSep` for better performance by grouping concatenations. [1] - https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md
267 lines
8.9 KiB
Nix
267 lines
8.9 KiB
Nix
import ./make-test-python.nix (
|
|
{ pkgs, lib, ... }:
|
|
let
|
|
initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
|
|
targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
|
|
in
|
|
{
|
|
name = "iscsi";
|
|
meta = {
|
|
maintainers = pkgs.lib.teams.deshaw.members;
|
|
};
|
|
|
|
nodes = {
|
|
target = { config, pkgs, lib, ... }: {
|
|
virtualisation.vlans = [ 1 2 ];
|
|
services.target = {
|
|
enable = true;
|
|
config = {
|
|
fabric_modules = [ ];
|
|
storage_objects = [
|
|
{
|
|
dev = "/dev/vdb";
|
|
name = "test";
|
|
plugin = "block";
|
|
write_back = true;
|
|
wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
|
|
}
|
|
];
|
|
targets = [
|
|
{
|
|
fabric = "iscsi";
|
|
tpgs = [
|
|
{
|
|
enable = true;
|
|
attributes = {
|
|
authentication = 0;
|
|
generate_node_acls = 1;
|
|
};
|
|
luns = [
|
|
{
|
|
alias = "94dfe06967";
|
|
alua_tg_pt_gp_name = "default_tg_pt_gp";
|
|
index = 0;
|
|
storage_object = "/backstores/block/test";
|
|
}
|
|
];
|
|
node_acls = [
|
|
{
|
|
mapped_luns = [
|
|
{
|
|
alias = "d42f5bdf8a";
|
|
index = 0;
|
|
tpg_lun = 0;
|
|
write_protect = false;
|
|
}
|
|
];
|
|
node_wwn = initiatorName;
|
|
}
|
|
];
|
|
portals = [
|
|
{
|
|
ip_address = "0.0.0.0";
|
|
iser = false;
|
|
offload = false;
|
|
port = 3260;
|
|
}
|
|
];
|
|
tag = 1;
|
|
}
|
|
];
|
|
wwn = targetName;
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
networking.firewall.allowedTCPPorts = [ 3260 ];
|
|
networking.firewall.allowedUDPPorts = [ 3260 ];
|
|
|
|
virtualisation.memorySize = 2048;
|
|
virtualisation.emptyDiskImages = [ 2048 ];
|
|
};
|
|
|
|
initiatorAuto = { nodes, config, pkgs, ... }: {
|
|
virtualisation.vlans = [ 1 2 ];
|
|
|
|
services.multipath = {
|
|
enable = true;
|
|
defaults = ''
|
|
find_multipaths yes
|
|
user_friendly_names yes
|
|
'';
|
|
pathGroups = [
|
|
{
|
|
alias = 123456;
|
|
wwid = "3600140592b17c3f6b404168b082ceeb7";
|
|
}
|
|
];
|
|
};
|
|
|
|
services.openiscsi = {
|
|
enable = true;
|
|
enableAutoLoginOut = true;
|
|
discoverPortal = "target";
|
|
name = initiatorName;
|
|
};
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
xfsprogs
|
|
];
|
|
|
|
environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
|
|
|
|
nix.settings = {
|
|
substituters = lib.mkForce [ ];
|
|
hashed-mirrors = null;
|
|
connect-timeout = 1;
|
|
};
|
|
};
|
|
|
|
initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
|
|
boot.initrd.network.enable = true;
|
|
boot.loader.grub.enable = false;
|
|
|
|
boot.kernelParams = lib.mkOverride 5 (
|
|
[
|
|
"boot.shell_on_fail"
|
|
"console=tty1"
|
|
"ip=192.168.1.1:::255.255.255.0::ens9:none"
|
|
"ip=192.168.2.1:::255.255.255.0::ens10:none"
|
|
]
|
|
);
|
|
|
|
# defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
|
|
virtualisation.writableStore = false;
|
|
virtualisation.vlans = [ 1 2 ];
|
|
|
|
services.multipath = {
|
|
enable = true;
|
|
defaults = ''
|
|
find_multipaths yes
|
|
user_friendly_names yes
|
|
'';
|
|
pathGroups = [
|
|
{
|
|
alias = 123456;
|
|
wwid = "3600140592b17c3f6b404168b082ceeb7";
|
|
}
|
|
];
|
|
};
|
|
|
|
fileSystems = lib.mkOverride 5 {
|
|
"/" = {
|
|
fsType = "xfs";
|
|
device = "/dev/mapper/123456";
|
|
options = [ "_netdev" ];
|
|
};
|
|
};
|
|
|
|
boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
|
|
|
|
boot.iscsi-initiator = {
|
|
discoverPortal = "target";
|
|
name = initiatorName;
|
|
target = targetName;
|
|
extraIscsiCommands = ''
|
|
iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
|
|
'';
|
|
};
|
|
};
|
|
|
|
};
|
|
|
|
testScript = { nodes, ... }: ''
|
|
target.start()
|
|
target.wait_for_unit("iscsi-target.service")
|
|
|
|
initiatorAuto.start()
|
|
|
|
initiatorAuto.wait_for_unit("iscsid.service")
|
|
initiatorAuto.wait_for_unit("iscsi.service")
|
|
initiatorAuto.get_unit_info("iscsi")
|
|
|
|
# Expecting this to fail since we should already know about 192.168.1.3
|
|
initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
|
|
# Expecting this to succeed since we don't yet know about 192.168.2.3
|
|
initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
|
|
|
|
# /dev/sda is provided by iscsi on target
|
|
initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
|
|
|
|
initiatorAuto.succeed("mkfs.xfs /dev/sda")
|
|
initiatorAuto.succeed("mkdir /mnt")
|
|
|
|
# Start by verifying /dev/sda and /dev/sdb are both the same disk
|
|
initiatorAuto.succeed("mount /dev/sda /mnt")
|
|
initiatorAuto.succeed("touch /mnt/hi")
|
|
initiatorAuto.succeed("umount /mnt")
|
|
|
|
initiatorAuto.succeed("mount /dev/sdb /mnt")
|
|
initiatorAuto.succeed("test -e /mnt/hi")
|
|
initiatorAuto.succeed("umount /mnt")
|
|
|
|
initiatorAuto.succeed("systemctl restart multipathd")
|
|
initiatorAuto.succeed("multipath -ll | systemd-cat")
|
|
|
|
# Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
|
|
initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
|
|
initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
|
|
initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
|
|
initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
|
|
initiatorAuto.succeed(
|
|
"nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
|
|
)
|
|
initiatorAuto.succeed("umount /mnt")
|
|
initiatorAuto.shutdown()
|
|
|
|
initiatorRootDisk.start()
|
|
initiatorRootDisk.wait_for_unit("multi-user.target")
|
|
initiatorRootDisk.wait_for_unit("iscsid")
|
|
|
|
# Log in over both nodes
|
|
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
|
|
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
|
|
initiatorRootDisk.succeed("systemctl restart multipathd")
|
|
initiatorRootDisk.succeed("multipath -ll | systemd-cat")
|
|
|
|
# Verify we can write and sync the root disk
|
|
initiatorRootDisk.succeed("mkdir /scratch")
|
|
initiatorRootDisk.succeed("touch /scratch/both-up")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
|
|
# Verify we can write to the root with ens9 (sda, 192.168.1.3) down
|
|
initiatorRootDisk.succeed("ip link set ens9 down")
|
|
initiatorRootDisk.succeed("touch /scratch/ens9-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
initiatorRootDisk.succeed("ip link set ens9 up")
|
|
|
|
# todo: better way to wait until multipath notices the link is back
|
|
initiatorRootDisk.succeed("sleep 5")
|
|
initiatorRootDisk.succeed("touch /scratch/both-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
|
|
# Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
|
|
initiatorRootDisk.succeed("ip link set ens10 down")
|
|
initiatorRootDisk.succeed("touch /scratch/ens10-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
initiatorRootDisk.succeed("ip link set ens10 up")
|
|
initiatorRootDisk.succeed("touch /scratch/ens10-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
|
|
initiatorRootDisk.succeed("ip link set ens9 up")
|
|
initiatorRootDisk.succeed("ip link set ens10 up")
|
|
initiatorRootDisk.shutdown()
|
|
|
|
# Verify we can boot with the target's eth1 down, forcing
|
|
# it to multipath via the second link
|
|
target.succeed("ip link set eth1 down")
|
|
initiatorRootDisk.start()
|
|
initiatorRootDisk.wait_for_unit("multi-user.target")
|
|
initiatorRootDisk.wait_for_unit("iscsid")
|
|
initiatorRootDisk.succeed("test -e /scratch/both-up")
|
|
'';
|
|
}
|
|
)
|
|
|
|
|