nixos/ceph: run unprivileged, use StateDirectory and tmpfiles, don't pass extraServiceConfig
Don't pass user and group to ceph, and rely on it to drop ceps, but let systemd handle running it as the appropriate user. This also inlines the extraServiceConfig into the makeService function, as we have conditionals depending on daemonType there anyways. Use StateDirectory to create directories in /var/lib/ceph/${daemonType}/${clusterName}-${daemonId}. There previously was a condition on daemonType being one of mds,mon,rgw or mgr. We only instantiate makeServices with these types, and "osd" was special. In the osd case, test examples suggest it'd be in something like /var/lib/ceph/osd/ceph-${cfg.osd0.name} - so it's not special at all, but exactly like the pattern for the others. During initialization, we also need these folders, before the unit is started up. Move the mkdir -p commands in the vm tests to the line immediately before they're required.
This commit is contained in:
parent
64c9c08302
commit
67e0777f62
|
@ -9,12 +9,12 @@ let
|
||||||
expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
|
expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
|
||||||
expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
|
expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
|
||||||
|
|
||||||
makeServices = (daemonType: daemonIds: extraServiceConfig:
|
makeServices = (daemonType: daemonIds:
|
||||||
mkMerge (map (daemonId:
|
mkMerge (map (daemonId:
|
||||||
{ "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; })
|
{ "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
|
||||||
daemonIds));
|
daemonIds));
|
||||||
|
|
||||||
makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
|
makeService = (daemonType: daemonId: clusterName: ceph: {
|
||||||
enable = true;
|
enable = true;
|
||||||
description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
|
description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
|
||||||
after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
|
after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
|
||||||
|
@ -22,6 +22,8 @@ let
|
||||||
partOf = [ "ceph-${daemonType}.target" ];
|
partOf = [ "ceph-${daemonType}.target" ];
|
||||||
wantedBy = [ "ceph-${daemonType}.target" ];
|
wantedBy = [ "ceph-${daemonType}.target" ];
|
||||||
|
|
||||||
|
path = [ pkgs.getopt ];
|
||||||
|
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
LimitNOFILE = 1048576;
|
LimitNOFILE = 1048576;
|
||||||
LimitNPROC = 1048576;
|
LimitNPROC = 1048576;
|
||||||
|
@ -34,22 +36,22 @@ let
|
||||||
Restart = "on-failure";
|
Restart = "on-failure";
|
||||||
StartLimitBurst = "5";
|
StartLimitBurst = "5";
|
||||||
StartLimitInterval = "30min";
|
StartLimitInterval = "30min";
|
||||||
|
StateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}";
|
||||||
|
User = "ceph";
|
||||||
|
Group = if daemonType == "osd" then "disk" else "ceph";
|
||||||
ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
|
ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
|
||||||
-f --cluster ${clusterName} --id ${daemonId} --setuser ceph \
|
-f --cluster ${clusterName} --id ${daemonId}'';
|
||||||
--setgroup ${if daemonType == "osd" then "disk" else "ceph"}'';
|
} // optionalAttrs (daemonType == "osd") {
|
||||||
} // extraServiceConfig
|
ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}'';
|
||||||
// optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \
|
StartLimitBurst = "30";
|
||||||
--id ${daemonId} --cluster ${clusterName}''; };
|
RestartSec = "20s";
|
||||||
} // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) {
|
PrivateDevices = "no"; # osd needs disk access
|
||||||
preStart = ''
|
} // optionalAttrs ( daemonType == "mon") {
|
||||||
daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"
|
RestartSec = "10";
|
||||||
if [ ! -d $daemonPath ]; then
|
} // optionalAttrs (lib.elem daemonType ["mgr" "mds"]) {
|
||||||
mkdir -m 755 -p $daemonPath
|
StartLimitBurst = "3";
|
||||||
chown -R ceph:ceph $daemonPath
|
};
|
||||||
fi
|
});
|
||||||
'';
|
|
||||||
} // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; }
|
|
||||||
);
|
|
||||||
|
|
||||||
makeTarget = (daemonType:
|
makeTarget = (daemonType:
|
||||||
{
|
{
|
||||||
|
@ -377,14 +379,11 @@ in
|
||||||
|
|
||||||
systemd.services = let
|
systemd.services = let
|
||||||
services = []
|
services = []
|
||||||
++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; })
|
++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
|
||||||
++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
|
++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
|
||||||
++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30";
|
++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
|
||||||
RestartSec = "20s";
|
++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
|
||||||
PrivateDevices = "no"; # osd needs disk access
|
++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
|
||||||
})
|
|
||||||
++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { })
|
|
||||||
++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
|
|
||||||
in
|
in
|
||||||
mkMerge services;
|
mkMerge services;
|
||||||
|
|
||||||
|
@ -403,7 +402,9 @@ in
|
||||||
systemd.tmpfiles.rules = [
|
systemd.tmpfiles.rules = [
|
||||||
"d /etc/ceph - ceph ceph - -"
|
"d /etc/ceph - ceph ceph - -"
|
||||||
"d /run/ceph 0770 ceph ceph -"
|
"d /run/ceph 0770 ceph ceph -"
|
||||||
"d /var/lib/ceph - ceph ceph - -"
|
"d /var/lib/ceph - ceph ceph - -"]
|
||||||
];
|
++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"]
|
||||||
|
++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"]
|
||||||
|
++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"];
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -114,18 +114,6 @@ let
|
||||||
$osd0->waitForUnit("network.target");
|
$osd0->waitForUnit("network.target");
|
||||||
$osd1->waitForUnit("network.target");
|
$osd1->waitForUnit("network.target");
|
||||||
|
|
||||||
# Create the ceph-related directories
|
|
||||||
$monA->mustSucceed(
|
|
||||||
"mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}",
|
|
||||||
"mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}",
|
|
||||||
);
|
|
||||||
$osd0->mustSucceed(
|
|
||||||
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
|
||||||
);
|
|
||||||
$osd1->mustSucceed(
|
|
||||||
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
|
||||||
);
|
|
||||||
|
|
||||||
# Bootstrap ceph-mon daemon
|
# Bootstrap ceph-mon daemon
|
||||||
$monA->mustSucceed(
|
$monA->mustSucceed(
|
||||||
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
|
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
|
||||||
|
@ -133,6 +121,7 @@ let
|
||||||
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
|
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
|
||||||
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
|
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
|
||||||
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
|
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
|
||||||
|
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
|
||||||
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
|
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
|
||||||
"systemctl start ceph-mon-${cfg.monA.name}"
|
"systemctl start ceph-mon-${cfg.monA.name}"
|
||||||
);
|
);
|
||||||
|
@ -159,12 +148,14 @@ let
|
||||||
# Bootstrap both OSDs
|
# Bootstrap both OSDs
|
||||||
$osd0->mustSucceed(
|
$osd0->mustSucceed(
|
||||||
"mkfs.xfs /dev/vdb",
|
"mkfs.xfs /dev/vdb",
|
||||||
|
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
||||||
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
||||||
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
|
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
|
||||||
"echo '{\"cephx_secret\": \"${cfg.osd0.key}\"}' | ceph osd new ${cfg.osd0.uuid} -i -",
|
"echo '{\"cephx_secret\": \"${cfg.osd0.key}\"}' | ceph osd new ${cfg.osd0.uuid} -i -",
|
||||||
);
|
);
|
||||||
$osd1->mustSucceed(
|
$osd1->mustSucceed(
|
||||||
"mkfs.xfs /dev/vdb",
|
"mkfs.xfs /dev/vdb",
|
||||||
|
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
||||||
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
||||||
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
|
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
|
||||||
"echo '{\"cephx_secret\": \"${cfg.osd1.key}\"}' | ceph osd new ${cfg.osd1.uuid} -i -"
|
"echo '{\"cephx_secret\": \"${cfg.osd1.key}\"}' | ceph osd new ${cfg.osd1.uuid} -i -"
|
||||||
|
|
|
@ -77,14 +77,6 @@ let
|
||||||
|
|
||||||
$monA->waitForUnit("network.target");
|
$monA->waitForUnit("network.target");
|
||||||
|
|
||||||
# Create the ceph-related directories
|
|
||||||
$monA->mustSucceed(
|
|
||||||
"mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}",
|
|
||||||
"mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}",
|
|
||||||
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
|
||||||
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
|
||||||
);
|
|
||||||
|
|
||||||
# Bootstrap ceph-mon daemon
|
# Bootstrap ceph-mon daemon
|
||||||
$monA->mustSucceed(
|
$monA->mustSucceed(
|
||||||
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
|
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
|
||||||
|
@ -101,8 +93,9 @@ let
|
||||||
# Can't check ceph status until a mon is up
|
# Can't check ceph status until a mon is up
|
||||||
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
|
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
|
||||||
|
|
||||||
# Start the ceph-mgr daemon, it has no deps and hardly any setup
|
# Start the ceph-mgr daemon, after copying in the keyring
|
||||||
$monA->mustSucceed(
|
$monA->mustSucceed(
|
||||||
|
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
|
||||||
"ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
|
"ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
|
||||||
"systemctl start ceph-mgr-${cfg.monA.name}"
|
"systemctl start ceph-mgr-${cfg.monA.name}"
|
||||||
);
|
);
|
||||||
|
@ -114,7 +107,9 @@ let
|
||||||
$monA->mustSucceed(
|
$monA->mustSucceed(
|
||||||
"mkfs.xfs /dev/vdb",
|
"mkfs.xfs /dev/vdb",
|
||||||
"mkfs.xfs /dev/vdc",
|
"mkfs.xfs /dev/vdc",
|
||||||
|
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
||||||
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
||||||
|
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
||||||
"mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
"mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
||||||
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
|
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
|
||||||
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
|
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
|
||||||
|
|
Loading…
Reference in a new issue