diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix index 887a6ff925d..ef0d2d90a1a 100644 --- a/nixos/modules/services/network-filesystems/ceph.nix +++ b/nixos/modules/services/network-filesystems/ceph.nix @@ -9,12 +9,12 @@ let expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars); expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value); - makeServices = (daemonType: daemonIds: extraServiceConfig: + makeServices = (daemonType: daemonIds: mkMerge (map (daemonId: - { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; }) + { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; }) daemonIds)); - makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: { + makeService = (daemonType: daemonId: clusterName: ceph: { enable = true; description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}"; after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target"; @@ -22,6 +22,8 @@ let partOf = [ "ceph-${daemonType}.target" ]; wantedBy = [ "ceph-${daemonType}.target" ]; + path = [ pkgs.getopt ]; + serviceConfig = { LimitNOFILE = 1048576; LimitNPROC = 1048576; @@ -34,22 +36,22 @@ let Restart = "on-failure"; StartLimitBurst = "5"; StartLimitInterval = "30min"; + StateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; + User = "ceph"; + Group = if daemonType == "osd" then "disk" else "ceph"; ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \ - -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \ - --setgroup ${if daemonType == "osd" then "disk" else "ceph"}''; - } // extraServiceConfig - // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \ - --id ${daemonId} --cluster ${clusterName}''; }; - } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { - preStart = '' - daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}" - if [ ! -d $daemonPath ]; then - mkdir -m 755 -p $daemonPath - chown -R ceph:ceph $daemonPath - fi - ''; - } // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; } - ); + -f --cluster ${clusterName} --id ${daemonId}''; + } // optionalAttrs (daemonType == "osd") { + ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}''; + StartLimitBurst = "30"; + RestartSec = "20s"; + PrivateDevices = "no"; # osd needs disk access + } // optionalAttrs ( daemonType == "mon") { + RestartSec = "10"; + } // optionalAttrs (lib.elem daemonType ["mgr" "mds"]) { + StartLimitBurst = "3"; + }; + }); makeTarget = (daemonType: { @@ -377,14 +379,11 @@ in systemd.services = let services = [] - ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; }) - ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; }) - ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30"; - RestartSec = "20s"; - PrivateDevices = "no"; # osd needs disk access - }) - ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { }) - ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; }); + ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons) + ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons) + ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons) + ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons) + ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons); in mkMerge services; @@ -403,7 +402,9 @@ in systemd.tmpfiles.rules = [ "d /etc/ceph - ceph ceph - -" "d /run/ceph 0770 ceph ceph -" - "d /var/lib/ceph - ceph ceph - -" - ]; + "d /var/lib/ceph - ceph ceph - -"] + ++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"] + ++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"] + ++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"]; }; } diff --git a/nixos/tests/ceph-multi-node.nix b/nixos/tests/ceph-multi-node.nix index 13d0851366e..c34b134de53 100644 --- a/nixos/tests/ceph-multi-node.nix +++ b/nixos/tests/ceph-multi-node.nix @@ -114,18 +114,6 @@ let $osd0->waitForUnit("network.target"); $osd1->waitForUnit("network.target"); - # Create the ceph-related directories - $monA->mustSucceed( - "mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}", - "mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}", - ); - $osd0->mustSucceed( - "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}", - ); - $osd1->mustSucceed( - "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}", - ); - # Bootstrap ceph-mon daemon $monA->mustSucceed( "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", @@ -133,6 +121,7 @@ let "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap", "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring", + "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/", "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done", "systemctl start ceph-mon-${cfg.monA.name}" ); @@ -159,12 +148,14 @@ let # Bootstrap both OSDs $osd0->mustSucceed( "mkfs.xfs /dev/vdb", + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}", "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}", "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}", "echo '{\"cephx_secret\": \"${cfg.osd0.key}\"}' | ceph osd new ${cfg.osd0.uuid} -i -", ); $osd1->mustSucceed( "mkfs.xfs /dev/vdb", + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}", "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}", "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}", "echo '{\"cephx_secret\": \"${cfg.osd1.key}\"}' | ceph osd new ${cfg.osd1.uuid} -i -" diff --git a/nixos/tests/ceph-single-node.nix b/nixos/tests/ceph-single-node.nix index cb37b4b43bc..5938300ebb4 100644 --- a/nixos/tests/ceph-single-node.nix +++ b/nixos/tests/ceph-single-node.nix @@ -77,14 +77,6 @@ let $monA->waitForUnit("network.target"); - # Create the ceph-related directories - $monA->mustSucceed( - "mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}", - "mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}", - "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}", - "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}", - ); - # Bootstrap ceph-mon daemon $monA->mustSucceed( "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", @@ -101,8 +93,9 @@ let # Can't check ceph status until a mon is up $monA->succeed("ceph -s | grep 'mon: 1 daemons'"); - # Start the ceph-mgr daemon, it has no deps and hardly any setup + # Start the ceph-mgr daemon, after copying in the keyring $monA->mustSucceed( + "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/", "ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring", "systemctl start ceph-mgr-${cfg.monA.name}" ); @@ -114,7 +107,9 @@ let $monA->mustSucceed( "mkfs.xfs /dev/vdb", "mkfs.xfs /dev/vdc", + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}", "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}", + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}", "mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}", "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}", "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",