nixpkgs/nixos/tests/iscsi-multipath-root.nix

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

268 lines
8.9 KiB
Nix
Raw Normal View History

import ./make-test-python.nix (
{ pkgs, lib, ... }:
let
initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
in
{
name = "iscsi";
meta = {
maintainers = pkgs.lib.teams.deshaw.members;
};
nodes = {
target = { config, pkgs, lib, ... }: {
virtualisation.vlans = [ 1 2 ];
services.target = {
enable = true;
config = {
fabric_modules = [ ];
storage_objects = [
{
dev = "/dev/vdb";
name = "test";
plugin = "block";
write_back = true;
wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
}
];
targets = [
{
fabric = "iscsi";
tpgs = [
{
enable = true;
attributes = {
authentication = 0;
generate_node_acls = 1;
};
luns = [
{
alias = "94dfe06967";
alua_tg_pt_gp_name = "default_tg_pt_gp";
index = 0;
storage_object = "/backstores/block/test";
}
];
node_acls = [
{
mapped_luns = [
{
alias = "d42f5bdf8a";
index = 0;
tpg_lun = 0;
write_protect = false;
}
];
node_wwn = initiatorName;
}
];
portals = [
{
ip_address = "0.0.0.0";
iser = false;
offload = false;
port = 3260;
}
];
tag = 1;
}
];
wwn = targetName;
}
];
};
};
networking.firewall.allowedTCPPorts = [ 3260 ];
networking.firewall.allowedUDPPorts = [ 3260 ];
virtualisation.memorySize = 2048;
virtualisation.emptyDiskImages = [ 2048 ];
};
initiatorAuto = { nodes, config, pkgs, ... }: {
virtualisation.vlans = [ 1 2 ];
services.multipath = {
enable = true;
defaults = ''
find_multipaths yes
user_friendly_names yes
'';
pathGroups = [
{
alias = 123456;
wwid = "3600140592b17c3f6b404168b082ceeb7";
}
];
};
services.openiscsi = {
enable = true;
enableAutoLoginOut = true;
discoverPortal = "target";
name = initiatorName;
};
environment.systemPackages = with pkgs; [
xfsprogs
];
environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
nixos/nix-daemon: use structural settings The `nix.*` options, apart from options for setting up the daemon itself, currently provide a lot of setting mappings for the Nix daemon configuration. The scope of the mapping yields convience, but the line where an option is considered essential is blurry. For instance, the `extra-sandbox-paths` mapping is provided without its primary consumer, and the corresponding `sandbox-paths` option is also not mapped. The current system increases the maintenance burden as maintainers have to closely follow upstream changes. In this case, there are two state versions of Nix which have to be maintained collectively, with different options avaliable. This commit aims to following the standard outlined in RFC 42[1] to implement a structural setting pattern. The Nix configuration is encoded at its core as key-value pairs which maps nicely to attribute sets, making it feasible to express in the Nix language itself. Some existing options are kept such as `buildMachines` and `registry` which present a simplified interface to managing the respective settings. The interface is exposed as `nix.settings`. Legacy configurations are mapped to their corresponding options under `nix.settings` for backwards compatibility. Various options settings in other nixos modules and relevant tests have been updated to use structural setting for consistency. The generation and validation of the configration file has been modified to use `writeTextFile` instead of `runCommand` for clarity. Note that validation is now mandatory as strict checking of options has been pushed down to the derivation level due to freeformType consuming unmatched options. Furthermore, validation can not occur when cross-compiling due to current limitations. A new option `publicHostKey` was added to the `buildMachines` submodule corresponding to the base64 encoded public host key settings exposed in the builder syntax. The build machine generation was subsequently rewritten to use `concatStringsSep` for better performance by grouping concatenations. [1] - https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md
2021-11-19 22:36:26 +00:00
nix.settings = {
substituters = lib.mkForce [ ];
hashed-mirrors = null;
connect-timeout = 1;
};
};
initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
boot.initrd.network.enable = true;
boot.loader.grub.enable = false;
boot.kernelParams = lib.mkOverride 5 (
[
"boot.shell_on_fail"
"console=tty1"
"ip=192.168.1.1:::255.255.255.0::ens9:none"
"ip=192.168.2.1:::255.255.255.0::ens10:none"
]
);
# defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
virtualisation.writableStore = false;
virtualisation.vlans = [ 1 2 ];
services.multipath = {
enable = true;
defaults = ''
find_multipaths yes
user_friendly_names yes
'';
pathGroups = [
{
alias = 123456;
wwid = "3600140592b17c3f6b404168b082ceeb7";
}
];
};
fileSystems = lib.mkOverride 5 {
"/" = {
fsType = "xfs";
device = "/dev/mapper/123456";
options = [ "_netdev" ];
};
};
boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
boot.iscsi-initiator = {
discoverPortal = "target";
name = initiatorName;
target = targetName;
extraIscsiCommands = ''
iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
'';
};
};
};
testScript = { nodes, ... }: ''
target.start()
target.wait_for_unit("iscsi-target.service")
initiatorAuto.start()
initiatorAuto.wait_for_unit("iscsid.service")
initiatorAuto.wait_for_unit("iscsi.service")
initiatorAuto.get_unit_info("iscsi")
# Expecting this to fail since we should already know about 192.168.1.3
initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
# Expecting this to succeed since we don't yet know about 192.168.2.3
initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
# /dev/sda is provided by iscsi on target
initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
initiatorAuto.succeed("mkfs.xfs /dev/sda")
initiatorAuto.succeed("mkdir /mnt")
# Start by verifying /dev/sda and /dev/sdb are both the same disk
initiatorAuto.succeed("mount /dev/sda /mnt")
initiatorAuto.succeed("touch /mnt/hi")
initiatorAuto.succeed("umount /mnt")
initiatorAuto.succeed("mount /dev/sdb /mnt")
initiatorAuto.succeed("test -e /mnt/hi")
initiatorAuto.succeed("umount /mnt")
initiatorAuto.succeed("systemctl restart multipathd")
initiatorAuto.succeed("multipath -ll | systemd-cat")
# Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
initiatorAuto.succeed(
"nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
)
initiatorAuto.succeed("umount /mnt")
initiatorAuto.shutdown()
initiatorRootDisk.start()
initiatorRootDisk.wait_for_unit("multi-user.target")
initiatorRootDisk.wait_for_unit("iscsid")
# Log in over both nodes
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
initiatorRootDisk.succeed("systemctl restart multipathd")
initiatorRootDisk.succeed("multipath -ll | systemd-cat")
# Verify we can write and sync the root disk
initiatorRootDisk.succeed("mkdir /scratch")
initiatorRootDisk.succeed("touch /scratch/both-up")
initiatorRootDisk.succeed("sync /scratch")
# Verify we can write to the root with ens9 (sda, 192.168.1.3) down
initiatorRootDisk.succeed("ip link set ens9 down")
initiatorRootDisk.succeed("touch /scratch/ens9-down")
initiatorRootDisk.succeed("sync /scratch")
initiatorRootDisk.succeed("ip link set ens9 up")
# todo: better way to wait until multipath notices the link is back
initiatorRootDisk.succeed("sleep 5")
initiatorRootDisk.succeed("touch /scratch/both-down")
initiatorRootDisk.succeed("sync /scratch")
# Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
initiatorRootDisk.succeed("ip link set ens10 down")
initiatorRootDisk.succeed("touch /scratch/ens10-down")
initiatorRootDisk.succeed("sync /scratch")
initiatorRootDisk.succeed("ip link set ens10 up")
initiatorRootDisk.succeed("touch /scratch/ens10-down")
initiatorRootDisk.succeed("sync /scratch")
initiatorRootDisk.succeed("ip link set ens9 up")
initiatorRootDisk.succeed("ip link set ens10 up")
initiatorRootDisk.shutdown()
# Verify we can boot with the target's eth1 down, forcing
# it to multipath via the second link
target.succeed("ip link set eth1 down")
initiatorRootDisk.start()
initiatorRootDisk.wait_for_unit("multi-user.target")
initiatorRootDisk.wait_for_unit("iscsid")
initiatorRootDisk.succeed("test -e /scratch/both-up")
'';
}
)