nixos/ceph: use ConditionPathExists to delay ceph daemon start
This prevents services to be started before they're initialized, and renders the `systemd.targets.ceph.wantedBy = lib.mkForce [];` hack in the vm tests obsolete - The config now starts up ceph after a reboot, too. Let's take advantage of that, crash all VMs, and boot them up again.
This commit is contained in:
parent
67e0777f62
commit
ffd0060869
|
@ -14,7 +14,9 @@ let
|
||||||
{ "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
|
{ "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
|
||||||
daemonIds));
|
daemonIds));
|
||||||
|
|
||||||
makeService = (daemonType: daemonId: clusterName: ceph: {
|
makeService = (daemonType: daemonId: clusterName: ceph:
|
||||||
|
let
|
||||||
|
stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in {
|
||||||
enable = true;
|
enable = true;
|
||||||
description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
|
description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
|
||||||
after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
|
after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
|
||||||
|
@ -24,6 +26,9 @@ let
|
||||||
|
|
||||||
path = [ pkgs.getopt ];
|
path = [ pkgs.getopt ];
|
||||||
|
|
||||||
|
# Don't start services that are not yet initialized
|
||||||
|
unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
|
||||||
|
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
LimitNOFILE = 1048576;
|
LimitNOFILE = 1048576;
|
||||||
LimitNPROC = 1048576;
|
LimitNPROC = 1048576;
|
||||||
|
@ -36,7 +41,7 @@ let
|
||||||
Restart = "on-failure";
|
Restart = "on-failure";
|
||||||
StartLimitBurst = "5";
|
StartLimitBurst = "5";
|
||||||
StartLimitInterval = "30min";
|
StartLimitInterval = "30min";
|
||||||
StateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}";
|
StateDirectory = stateDirectory;
|
||||||
User = "ceph";
|
User = "ceph";
|
||||||
Group = if daemonType == "osd" then "disk" else "ceph";
|
Group = if daemonType == "osd" then "disk" else "ceph";
|
||||||
ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
|
ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
|
||||||
|
@ -60,6 +65,7 @@ let
|
||||||
partOf = [ "ceph.target" ];
|
partOf = [ "ceph.target" ];
|
||||||
wantedBy = [ "ceph.target" ];
|
wantedBy = [ "ceph.target" ];
|
||||||
before = [ "ceph.target" ];
|
before = [ "ceph.target" ];
|
||||||
|
unitConfig.StopWhenUnneeded = true;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
@ -389,9 +395,12 @@ in
|
||||||
|
|
||||||
systemd.targets = let
|
systemd.targets = let
|
||||||
targets = [
|
targets = [
|
||||||
{ ceph = { description = "Ceph target allowing to start/stop all ceph service instances at once";
|
{ ceph = {
|
||||||
wantedBy = [ "multi-user.target" ]; }; }
|
description = "Ceph target allowing to start/stop all ceph service instances at once";
|
||||||
] ++ optional cfg.mon.enable (makeTarget "mon")
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
unitConfig.StopWhenUnneeded = true;
|
||||||
|
}; } ]
|
||||||
|
++ optional cfg.mon.enable (makeTarget "mon")
|
||||||
++ optional cfg.mds.enable (makeTarget "mds")
|
++ optional cfg.mds.enable (makeTarget "mds")
|
||||||
++ optional cfg.osd.enable (makeTarget "osd")
|
++ optional cfg.osd.enable (makeTarget "osd")
|
||||||
++ optional cfg.rgw.enable (makeTarget "rgw")
|
++ optional cfg.rgw.enable (makeTarget "rgw")
|
||||||
|
|
|
@ -49,9 +49,6 @@ let
|
||||||
boot.kernelModules = [ "xfs" ];
|
boot.kernelModules = [ "xfs" ];
|
||||||
|
|
||||||
services.ceph = cephConfig;
|
services.ceph = cephConfig;
|
||||||
|
|
||||||
# So that we don't have to battle systemd when bootstraping
|
|
||||||
systemd.targets.ceph.wantedBy = lib.mkForce [];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
networkMonA = {
|
networkMonA = {
|
||||||
|
@ -191,22 +188,17 @@ let
|
||||||
"ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it"
|
"ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it"
|
||||||
);
|
);
|
||||||
|
|
||||||
# As we disable the target in the config, we still want to test that it works as intended
|
# Shut down ceph on all machines in a very unpolite way
|
||||||
$osd0->mustSucceed("systemctl stop ceph-osd-${cfg.osd0.name}");
|
$monA->crash;
|
||||||
$osd1->mustSucceed("systemctl stop ceph-osd-${cfg.osd1.name}");
|
$osd0->crash;
|
||||||
$monA->mustSucceed(
|
$osd1->crash;
|
||||||
"systemctl stop ceph-mgr-${cfg.monA.name}",
|
|
||||||
"systemctl stop ceph-mon-${cfg.monA.name}"
|
|
||||||
);
|
|
||||||
|
|
||||||
$monA->succeed("systemctl start ceph.target");
|
# Start it up
|
||||||
$monA->waitForUnit("ceph-mon-${cfg.monA.name}");
|
$osd0->start;
|
||||||
$monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
|
$osd1->start;
|
||||||
$osd0->succeed("systemctl start ceph.target");
|
$monA->start;
|
||||||
$osd0->waitForUnit("ceph-osd-${cfg.osd0.name}");
|
|
||||||
$osd1->succeed("systemctl start ceph.target");
|
|
||||||
$osd1->waitForUnit("ceph-osd-${cfg.osd1.name}");
|
|
||||||
|
|
||||||
|
# Ensure the cluster comes back up again
|
||||||
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
|
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
|
||||||
$monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
|
$monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
|
||||||
$monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
|
$monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
|
||||||
|
|
|
@ -46,9 +46,6 @@ let
|
||||||
boot.kernelModules = [ "xfs" ];
|
boot.kernelModules = [ "xfs" ];
|
||||||
|
|
||||||
services.ceph = cephConfig;
|
services.ceph = cephConfig;
|
||||||
|
|
||||||
# So that we don't have to battle systemd when bootstraping
|
|
||||||
systemd.targets.ceph.wantedBy = lib.mkForce [];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
networkMonA = {
|
networkMonA = {
|
||||||
|
@ -151,20 +148,17 @@ let
|
||||||
"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it"
|
"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it"
|
||||||
);
|
);
|
||||||
|
|
||||||
# As we disable the target in the config, we still want to test that it works as intended
|
# Shut down ceph by stopping ceph.target.
|
||||||
$monA->mustSucceed(
|
$monA->mustSucceed("systemctl stop ceph.target");
|
||||||
"systemctl stop ceph-osd-${cfg.osd0.name}",
|
|
||||||
"systemctl stop ceph-osd-${cfg.osd1.name}",
|
|
||||||
"systemctl stop ceph-mgr-${cfg.monA.name}",
|
|
||||||
"systemctl stop ceph-mon-${cfg.monA.name}"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
# Start it up
|
||||||
$monA->succeed("systemctl start ceph.target");
|
$monA->succeed("systemctl start ceph.target");
|
||||||
$monA->waitForUnit("ceph-mon-${cfg.monA.name}");
|
$monA->waitForUnit("ceph-mon-${cfg.monA.name}");
|
||||||
$monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
|
$monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
|
||||||
$monA->waitForUnit("ceph-osd-${cfg.osd0.name}");
|
$monA->waitForUnit("ceph-osd-${cfg.osd0.name}");
|
||||||
$monA->waitForUnit("ceph-osd-${cfg.osd1.name}");
|
$monA->waitForUnit("ceph-osd-${cfg.osd1.name}");
|
||||||
|
|
||||||
|
# Ensure the cluster comes back up again
|
||||||
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
|
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
|
||||||
$monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
|
$monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
|
||||||
$monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
|
$monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
|
||||||
|
|
Loading…
Reference in a new issue