From 85baedaca3c03bb376a9a895fa8421a76f5f3301 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Tue, 6 Nov 2018 14:35:48 +0100 Subject: [PATCH] ceph: 12.2.7 -> 13.2.4 * maintain only one version * ceph-client: init * include ceph-volume python tool in output nixos/ceph: extraConfig, fix test, wait for ceph-mgr to become active * run ceph with disk group permission * add extraConfig option for the global section needed per cluster * clear up how ceph.conf is generated * fix ceph testcase --- .../services/network-filesystems/ceph.nix | 102 +++++---- nixos/tests/ceph.nix | 28 ++- .../ceph/0000-ceph-volume-allow-loop.patch | 35 ++++ .../0000-dont-check-cherrypy-version.patch | 33 +++ .../ceph/0000-fix-SPDK-build-env.patch | 11 + ...kv-RocksDBStore-API-break-additional.patch | 58 ------ .../ceph/0002-fix-absolute-include-path.patch | 19 -- pkgs/tools/filesystems/ceph/default.nix | 193 +++++++++++++++++- pkgs/tools/filesystems/ceph/generic.nix | 175 ---------------- pkgs/top-level/all-packages.nix | 8 +- 10 files changed, 360 insertions(+), 302 deletions(-) create mode 100644 pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch create mode 100644 pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch create mode 100644 pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch delete mode 100644 pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch delete mode 100644 pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch delete mode 100644 pkgs/tools/filesystems/ceph/generic.nix diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix index 54841861c08..58922897a18 100644 --- a/nixos/modules/services/network-filesystems/ceph.nix +++ b/nixos/modules/services/network-filesystems/ceph.nix @@ -3,18 +3,18 @@ with lib; let - ceph = pkgs.ceph; cfg = config.services.ceph; + # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode - translateOption = replaceStrings upperChars (map (s: " ${s}") lowerChars); - generateDaemonList = (daemonType: daemons: extraServiceConfig: - mkMerge ( - map (daemon: - { "ceph-${daemonType}-${daemon}" = generateServiceFile daemonType daemon cfg.global.clusterName ceph extraServiceConfig; } - ) daemons - ) - ); - generateServiceFile = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: { + expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars); + expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value); + + makeServices = (daemonType: daemonIds: extraServiceConfig: + mkMerge (map (daemonId: + { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; }) + daemonIds)); + + makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: { enable = true; description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}"; after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target"; @@ -34,23 +34,29 @@ let Restart = "on-failure"; StartLimitBurst = "5"; StartLimitInterval = "30min"; - ExecStart = "${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} -f --cluster ${clusterName} --id ${if daemonType == "rgw" then "client.${daemonId}" else daemonId} --setuser ceph --setgroup ceph"; + ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \ + -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \ + --setgroup ${if daemonType == "osd" then "disk" else "ceph"}''; } // extraServiceConfig - // optionalAttrs (daemonType == "osd") { ExecStartPre = "${ceph.out}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}"; }; - } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { preStart = '' + // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.out}/libexec/ceph/ceph-osd-prestart.sh \ + --id ${daemonId} --cluster ${clusterName}''; }; + } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { + preStart = '' daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}" - if [ ! -d ''$daemonPath ]; then - mkdir -m 755 -p ''$daemonPath - chown -R ceph:ceph ''$daemonPath + if [ ! -d $daemonPath ]; then + mkdir -m 755 -p $daemonPath + chown -R ceph:ceph $daemonPath fi ''; } // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; } ); - generateTargetFile = (daemonType: + + makeTarget = (daemonType: { "ceph-${daemonType}" = { description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once"; partOf = [ "ceph.target" ]; + wantedBy = [ "ceph.target" ]; before = [ "ceph.target" ]; }; } @@ -157,6 +163,27 @@ in A comma-separated list of subnets that will be used as cluster networks in the cluster. ''; }; + + rgwMimeTypesFile = mkOption { + type = with types; nullOr path; + default = "${pkgs.mime-types}/etc/mime.types"; + description = '' + Path to mime types used by radosgw. + ''; + }; + }; + + extraConfig = mkOption { + type = with types; attrsOf str; + default = {}; + example = '' + { + "ms bind ipv6" = "true"; + }; + ''; + description = '' + Extra configuration to add to the global section. Use for setting values that are common for all daemons in the cluster. + ''; }; mgr = { @@ -216,6 +243,7 @@ in to the id part in ceph i.e. [ "name1" ] would result in osd.name1 ''; }; + extraConfig = mkOption { type = with types; attrsOf str; default = { @@ -296,9 +324,6 @@ in { assertion = cfg.global.fsid != ""; message = "fsid has to be set to a valid uuid for the cluster to function"; } - { assertion = cfg.mgr.enable == true; - message = "ceph 12.x requires atleast 1 MGR daemon enabled for the cluster to function"; - } { assertion = cfg.mon.enable == true -> cfg.mon.daemons != []; message = "have to set id of atleast one MON if you're going to enable Monitor"; } @@ -317,14 +342,12 @@ in ''Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function''; environment.etc."ceph/ceph.conf".text = let - # Translate camelCaseOptions to the expected camel case option for ceph.conf - translatedGlobalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) cfg.global; # Merge the extraConfig set for mgr daemons, as mgr don't have their own section - globalAndMgrConfig = translatedGlobalConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig; + globalSection = expandCamelCaseAttrs (cfg.global // cfg.extraConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig); # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf - globalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) (filterAttrs (name: value: value != null) globalAndMgrConfig); + globalSection' = filterAttrs (name: value: value != null) globalSection; totalConfig = { - "global" = globalConfig; + "global" = globalSection'; } // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { "mon" = cfg.mon.extraConfig; } // optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { "mds" = cfg.mds.extraConfig; } // optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { "osd" = cfg.osd.extraConfig; } @@ -336,8 +359,9 @@ in name = "ceph"; uid = config.ids.uids.ceph; description = "Ceph daemon user"; + group = "ceph"; + extraGroups = [ "disk" ]; }; - users.groups = singleton { name = "ceph"; gid = config.ids.gids.ceph; @@ -345,22 +369,26 @@ in systemd.services = let services = [] - ++ optional cfg.mon.enable (generateDaemonList "mon" cfg.mon.daemons { RestartSec = "10"; }) - ++ optional cfg.mds.enable (generateDaemonList "mds" cfg.mds.daemons { StartLimitBurst = "3"; }) - ++ optional cfg.osd.enable (generateDaemonList "osd" cfg.osd.daemons { StartLimitBurst = "30"; RestartSec = "20s"; }) - ++ optional cfg.rgw.enable (generateDaemonList "rgw" cfg.rgw.daemons { }) - ++ optional cfg.mgr.enable (generateDaemonList "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; }); + ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; }) + ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; }) + ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30"; + RestartSec = "20s"; + PrivateDevices = "no"; # osd needs disk access + }) + ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { }) + ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; }); in mkMerge services; systemd.targets = let targets = [ - { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once"; }; } - ] ++ optional cfg.mon.enable (generateTargetFile "mon") - ++ optional cfg.mds.enable (generateTargetFile "mds") - ++ optional cfg.osd.enable (generateTargetFile "osd") - ++ optional cfg.rgw.enable (generateTargetFile "rgw") - ++ optional cfg.mgr.enable (generateTargetFile "mgr"); + { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once"; + wantedBy = [ "multi-user.target" ]; }; } + ] ++ optional cfg.mon.enable (makeTarget "mon") + ++ optional cfg.mds.enable (makeTarget "mds") + ++ optional cfg.osd.enable (makeTarget "osd") + ++ optional cfg.rgw.enable (makeTarget "rgw") + ++ optional cfg.mgr.enable (makeTarget "mgr"); in mkMerge targets; diff --git a/nixos/tests/ceph.nix b/nixos/tests/ceph.nix index 7408029c460..a80f63da665 100644 --- a/nixos/tests/ceph.nix +++ b/nixos/tests/ceph.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({pkgs, ...}: rec { +import ./make-test.nix ({pkgs, lib, ...}: rec { name = "All-in-one-basic-ceph-cluster"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ lejonet ]; @@ -40,6 +40,9 @@ import ./make-test.nix ({pkgs, ...}: rec { services.ceph.mon = { enable = true; daemons = [ "aio" ]; + extraConfig = { + "mgr initial modules" = "dashboard"; + }; }; services.ceph.mgr = { @@ -51,6 +54,9 @@ import ./make-test.nix ({pkgs, ...}: rec { enable = true; daemons = [ "0" "1" ]; }; + + # So that we don't have to battle systemd when bootstraping + systemd.targets.ceph.wantedBy = lib.mkForce []; }; }; @@ -69,7 +75,6 @@ import ./make-test.nix ({pkgs, ...}: rec { # Bootstrap ceph-mon daemon $aio->mustSucceed( - "mkdir -p /var/lib/ceph/bootstrap-osd && chown ceph:ceph /var/lib/ceph/bootstrap-osd", "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", @@ -90,6 +95,7 @@ import ./make-test.nix ({pkgs, ...}: rec { ); $aio->waitForUnit("ceph-mgr-aio"); $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'"); + $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'"); # Bootstrap both OSDs $aio->mustSucceed( @@ -135,5 +141,23 @@ import ./make-test.nix ({pkgs, ...}: rec { "ceph osd pool ls | grep 'aio-test'", "ceph osd pool delete aio-other-test aio-other-test --yes-i-really-really-mean-it" ); + + # As we disable the target in the config, we still want to test that it works as intended + $aio->mustSucceed( + "systemctl stop ceph-osd-0", + "systemctl stop ceph-osd-1", + "systemctl stop ceph-mgr-aio", + "systemctl stop ceph-mon-aio" + ); + $aio->succeed("systemctl start ceph.target"); + $aio->waitForUnit("ceph-mon-aio"); + $aio->waitForUnit("ceph-mgr-aio"); + $aio->waitForUnit("ceph-osd-0"); + $aio->waitForUnit("ceph-osd-1"); + $aio->succeed("ceph -s | grep 'mon: 1 daemons'"); + $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'"); + $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'"); + $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'"); + $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); ''; }) diff --git a/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch b/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch new file mode 100644 index 00000000000..d103c42b518 --- /dev/null +++ b/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch @@ -0,0 +1,35 @@ +diff --git a/src/ceph-disk/ceph_disk/main.py b/src/ceph-disk/ceph_disk/main.py +index f8ef35503e..2cc8a75d2b 100644 +--- a/src/ceph-disk/ceph_disk/main.py ++++ b/src/ceph-disk/ceph_disk/main.py +@@ -881,13 +881,14 @@ def is_partition(dev): + if not stmode_is_diskdevice(st.st_mode): + raise Error('not a block device', dev) + ++ major = os.major(st.st_rdev) ++ minor = os.minor(st.st_rdev) ++ + name = get_dev_name(dev) +- if os.path.exists(os.path.join(BLOCKDIR, name)): ++ if os.path.exists(os.path.join(BLOCKDIR, name)) or os.path.exists(os.path.join(BLOCKDIR, "loop%d" % minor)): + return False + + # make sure it is a partition of something else +- major = os.major(st.st_rdev) +- minor = os.minor(st.st_rdev) + if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)): + return True + +diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py +index c3a7915400..9a708b05f2 100644 +--- a/src/ceph-volume/ceph_volume/util/disk.py ++++ b/src/ceph-volume/ceph_volume/util/disk.py +@@ -203,7 +203,7 @@ def is_device(dev): + # use lsblk first, fall back to using stat + TYPE = lsblk(dev).get('TYPE') + if TYPE: +- return TYPE == 'disk' ++ return TYPE == 'disk' or TYPE == 'loop' + + # fallback to stat + return _stat_is_device(os.lstat(dev).st_mode) diff --git a/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch new file mode 100644 index 00000000000..154e90d9146 --- /dev/null +++ b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch @@ -0,0 +1,33 @@ +diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py +index 0b53743d74..2e17ce1168 100644 +--- a/src/pybind/mgr/dashboard/module.py ++++ b/src/pybind/mgr/dashboard/module.py +@@ -28,28 +28,6 @@ except ImportError: + # To be picked up and reported by .can_run() + cherrypy = None + +- +-# The SSL code in CherryPy 3.5.0 is buggy. It was fixed long ago, +-# but 3.5.0 is still shipping in major linux distributions +-# (Fedora 27, Ubuntu Xenial), so we must monkey patch it to get SSL working. +-if cherrypy is not None: +- v = StrictVersion(cherrypy.__version__) +- # It was fixed in 3.7.0. Exact lower bound version is probably earlier, +- # but 3.5.0 is what this monkey patch is tested on. +- if v >= StrictVersion("3.5.0") and v < StrictVersion("3.7.0"): +- from cherrypy.wsgiserver.wsgiserver2 import HTTPConnection,\ +- CP_fileobject +- +- def fixed_init(hc_self, server, sock, makefile=CP_fileobject): +- hc_self.server = server +- hc_self.socket = sock +- hc_self.rfile = makefile(sock, "rb", hc_self.rbufsize) +- hc_self.wfile = makefile(sock, "wb", hc_self.wbufsize) +- hc_self.requests_seen = 0 +- +- HTTPConnection.__init__ = fixed_init +- +- + if 'COVERAGE_ENABLED' in os.environ: + import coverage + _cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__))) diff --git a/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch b/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch new file mode 100644 index 00000000000..b04082537b5 --- /dev/null +++ b/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch @@ -0,0 +1,11 @@ +--- a/cmake/modules/BuildSPDK.cmake 2018-08-09 09:22:34.950684960 +0200 ++++ b/cmake/modules/BuildSPDK.cmake 2018-08-09 09:21:59.986964224 +0200 +@@ -16,7 +16,7 @@ + # unset $CFLAGS, otherwise it will interfere with how SPDK sets + # its include directory. + # unset $LDFLAGS, otherwise SPDK will fail to mock some functions. +- BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC" ++ BUILD_COMMAND env PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC" C_OPT="-mssse3" + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "true") + ExternalProject_Get_Property(spdk-ext source_dir) diff --git a/pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch b/pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch deleted file mode 100644 index 0b6f1d30a4c..00000000000 --- a/pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch +++ /dev/null @@ -1,58 +0,0 @@ -Seulement dans ceph: ceph.old -diff -ur ceph.old/src/kv/RocksDBStore.cc ceph/src/kv/RocksDBStore.cc ---- ceph.old/src/kv/RocksDBStore.cc 1980-01-02 00:00:00.000000000 +0100 -+++ ceph/src/kv/RocksDBStore.cc 2018-01-24 14:08:35.017553372 +0100 -@@ -505,7 +505,7 @@ - // considering performance overhead, default is disabled - if (g_conf->rocksdb_perf) { - rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex); -- rocksdb::perf_context.Reset(); -+ rocksdb::get_perf_context()->Reset(); - } - - RocksDBTransactionImpl * _t = -@@ -532,13 +532,13 @@ - utime_t write_wal_time; - utime_t write_pre_and_post_process_time; - write_wal_time.set_from_double( -- static_cast(rocksdb::perf_context.write_wal_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_wal_time)/1000000000); - write_memtable_time.set_from_double( -- static_cast(rocksdb::perf_context.write_memtable_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_memtable_time)/1000000000); - write_delay_time.set_from_double( -- static_cast(rocksdb::perf_context.write_delay_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_delay_time)/1000000000); - write_pre_and_post_process_time.set_from_double( -- static_cast(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000); - logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time); - logger->tinc(l_rocksdb_write_delay_time, write_delay_time); - logger->tinc(l_rocksdb_write_wal_time, write_wal_time); -@@ -558,7 +558,7 @@ - // considering performance overhead, default is disabled - if (g_conf->rocksdb_perf) { - rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex); -- rocksdb::perf_context.Reset(); -+ rocksdb::get_perf_context()->Reset(); - } - - RocksDBTransactionImpl * _t = -@@ -586,13 +586,13 @@ - utime_t write_wal_time; - utime_t write_pre_and_post_process_time; - write_wal_time.set_from_double( -- static_cast(rocksdb::perf_context.write_wal_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_wal_time)/1000000000); - write_memtable_time.set_from_double( -- static_cast(rocksdb::perf_context.write_memtable_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_memtable_time)/1000000000); - write_delay_time.set_from_double( -- static_cast(rocksdb::perf_context.write_delay_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_delay_time)/1000000000); - write_pre_and_post_process_time.set_from_double( -- static_cast(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000); -+ static_cast(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000); - logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time); - logger->tinc(l_rocksdb_write_delay_time, write_delay_time); - logger->tinc(l_rocksdb_write_wal_time, write_wal_time); diff --git a/pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch b/pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch deleted file mode 100644 index 01aa194dddf..00000000000 --- a/pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff -ru ceph/src/key_value_store/kv_flat_btree_async.cc ceph-copy/src/key_value_store/kv_flat_btree_async.cc ---- ceph/src/key_value_store/kv_flat_btree_async.cc 1980-01-02 00:00:00.000000000 +0100 -+++ ceph-copy/src/key_value_store/kv_flat_btree_async.cc 2018-02-13 21:49:59.232860487 +0100 -@@ -15,13 +15,13 @@ - #include "key_value_store/kv_flat_btree_async.h" - #include "key_value_store/kvs_arg_types.h" - #include "include/rados/librados.hpp" --#include "/usr/include/asm-generic/errno.h" --#include "/usr/include/asm-generic/errno-base.h" - #include "common/ceph_context.h" - #include "common/Clock.h" - #include "include/types.h" - - -+#include -+#include - #include - #include - #include diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index ef3e36ff627..b9c71c49e80 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -1,12 +1,189 @@ -{ callPackage, fetchgit, fetchpatch, ... } @ args: +{ stdenv, runCommand, fetchurl +, ensureNewerSourcesHook +, cmake, pkgconfig +, which, git +, boost, python2Packages +, libxml2, zlib, lz4 +, openldap, lttng-ust +, babeltrace, gperf +, cunit, snappy +, rocksdb, makeWrapper +, leveldb, oathToolkit -callPackage ./generic.nix (args // rec { - version = "12.2.7"; +# Optional Dependencies +, yasm ? null, fcgi ? null, expat ? null +, curl ? null, fuse ? null +, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null +, libs3 ? null - src = fetchgit { - url = "https://github.com/ceph/ceph.git"; - rev = "refs/tags/v${version}"; - sha256 = "031nfw2g2fdpxxx39g862phgmdx68hj9r54axazandghfhc1bzrl"; +# Mallocs +, jemalloc ? null, gperftools ? null + +# Crypto Dependencies +, cryptopp ? null +, nss ? null, nspr ? null + +# Linux Only Dependencies +, linuxHeaders, utillinux, libuuid, udev, keyutils +, libaio ? null, libxfs ? null, zfs ? null +, ... +}: + +# We must have one crypto library +assert cryptopp != null || (nss != null && nspr != null); + +with stdenv; with stdenv.lib; +let + shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null; + + optYasm = shouldUsePkg yasm; + optFcgi = shouldUsePkg fcgi; + optExpat = shouldUsePkg expat; + optCurl = shouldUsePkg curl; + optFuse = shouldUsePkg fuse; + optLibedit = shouldUsePkg libedit; + optLibatomic_ops = shouldUsePkg libatomic_ops; + optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client; + optLibs3 = shouldUsePkg libs3; + + optJemalloc = shouldUsePkg jemalloc; + optGperftools = shouldUsePkg gperftools; + + optCryptopp = shouldUsePkg cryptopp; + optNss = shouldUsePkg nss; + optNspr = shouldUsePkg nspr; + + optLibaio = shouldUsePkg libaio; + optLibxfs = shouldUsePkg libxfs; + optZfs = shouldUsePkg zfs; + + hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null; + + + # Malloc implementation (can be jemalloc, tcmalloc or null) + malloc = if optJemalloc != null then optJemalloc else optGperftools; + + # We prefer nss over cryptopp + cryptoStr = if optNss != null && optNspr != null then "nss" else + if optCryptopp != null then "cryptopp" else "none"; + + cryptoLibsMap = { + nss = [ optNss optNspr ]; + cryptopp = [ optCryptopp ]; + none = [ ]; }; -}) + ceph-python-env = python2Packages.python.withPackages (ps: [ + ps.sphinx + ps.flask + ps.cython + ps.setuptools + ps.virtualenv + # Libraries needed by the python tools + ps.Mako + ps.cherrypy + ps.pecan + ps.prettytable + ps.webob + ps.bcrypt + ]); + + version = "13.2.4"; +in rec { + ceph = stdenv.mkDerivation { + name="ceph-${version}"; + + src = fetchurl { + url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz"; + sha256 = "0g2mc6rp84ia44vz8kl449820m9hmfavzfmwn8fy6li14xr8a00w"; + }; + + patches = [ + ./0000-fix-SPDK-build-env.patch + # TODO: remove when https://github.com/ceph/ceph/pull/21289 is merged + ./0000-ceph-volume-allow-loop.patch + ./0000-dont-check-cherrypy-version.patch + ]; + + nativeBuildInputs = [ + cmake + pkgconfig which git python2Packages.wrapPython makeWrapper + (ensureNewerSourcesHook { year = "1980"; }) + ]; + + buildInputs = cryptoLibsMap.${cryptoStr} ++ [ + boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3 + malloc zlib openldap lttng-ust babeltrace gperf cunit + snappy rocksdb lz4 oathToolkit leveldb + optKinetic-cpp-client + ] ++ optionals stdenv.isLinux [ + linuxHeaders utillinux libuuid udev keyutils optLibaio optLibxfs optZfs + ] ++ optionals hasRadosgw [ + optFcgi optExpat optCurl optFuse optLibedit + ]; + + preConfigure ='' + substituteInPlace src/common/module.c --replace "/sbin/modinfo" "modinfo" + substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe" + # Since Boost 1.67 this seems to have changed + substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python27)" + substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python27 " + + # for pybind/rgw to find internal dep + export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH" + # install target needs to be in PYTHONPATH for "*.pth support" check to succeed + export PYTHONPATH=$lib/lib/python2.7/site-packages/:$out/lib/python2.7/site-packages/ + + patchShebangs src/spdk + ''; + + cmakeFlags = [ + "-DWITH_SYSTEM_ROCKSDB=ON" + "-DROCKSDB_INCLUDE_DIR=${rocksdb}/include/rocksdb" + "-DWITH_SYSTEM_BOOST=ON" + "-DWITH_SYSTEMD=OFF" + "-DWITH_TESTS=OFF" + # TODO breaks with sandbox, tries to download stuff with npm + "-DWITH_MGR_DASHBOARD_FRONTEND=OFF" + ]; + + postFixup = '' + wrapPythonPrograms + wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$lib/lib/ceph/mgr:$out/lib/python2.7/site-packages/" + ''; + + enableParallelBuilding = true; + + outputs = [ "out" "lib" "dev" "doc" "man" ]; + + meta = { + homepage = https://ceph.com/; + description = "Distributed storage system"; + license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ]; + maintainers = with maintainers; [ adev ak krav ]; + platforms = platforms.unix; + }; + + passthru.version = version; + }; + + ceph-client = runCommand "ceph-client-${version}" { + meta = { + homepage = https://ceph.com/; + description = "Tools needed to mount Ceph's RADOS Block Devices"; + license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ]; + maintainers = with maintainers; [ adev ak krav ]; + platforms = platforms.unix; + }; + } '' + mkdir -p $out/{bin,etc,lib/python2.7/site-packages} + cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin + cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin + cp -r ${ceph}/bin/rbd-replay* $out/bin + cp -r ${ceph}/lib/python2.7/site-packages $out/lib/python2.7/ + cp -r ${ceph}/etc/bash_completion.d $out/etc + # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths + substituteInPlace $out/bin/ceph --replace ${ceph} $out + substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out + ''; +} diff --git a/pkgs/tools/filesystems/ceph/generic.nix b/pkgs/tools/filesystems/ceph/generic.nix deleted file mode 100644 index e8a4917be99..00000000000 --- a/pkgs/tools/filesystems/ceph/generic.nix +++ /dev/null @@ -1,175 +0,0 @@ -{ stdenv, ensureNewerSourcesHook, cmake, pkgconfig -, which, git -, boost, python2Packages -, libxml2, zlib -, openldap, lttng-ust -, babeltrace, gperf -, cunit, snappy -, rocksdb, makeWrapper - -# Optional Dependencies -, yasm ? null, fcgi ? null, expat ? null -, curl ? null, fuse ? null -, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null -, libs3 ? null - -# Mallocs -, jemalloc ? null, gperftools ? null - -# Crypto Dependencies -, cryptopp ? null -, nss ? null, nspr ? null - -# Linux Only Dependencies -, linuxHeaders, libuuid, udev, keyutils, libaio ? null, libxfs ? null -, zfs ? null - -# Version specific arguments -, version, src ? [], buildInputs ? [] -, ... -}: - -# We must have one crypto library -assert cryptopp != null || (nss != null && nspr != null); - -with stdenv; -with stdenv.lib; -let - - shouldUsePkg = pkg_: let pkg = (builtins.tryEval pkg_).value; - in if lib.any (lib.meta.platformMatch stdenv.hostPlatform) pkg.meta.platforms - then pkg else null; - - optYasm = shouldUsePkg yasm; - optFcgi = shouldUsePkg fcgi; - optExpat = shouldUsePkg expat; - optCurl = shouldUsePkg curl; - optFuse = shouldUsePkg fuse; - optLibedit = shouldUsePkg libedit; - optLibatomic_ops = shouldUsePkg libatomic_ops; - optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client; - optLibs3 = if versionAtLeast version "10.0.0" then null else shouldUsePkg libs3; - - optJemalloc = shouldUsePkg jemalloc; - optGperftools = shouldUsePkg gperftools; - - optCryptopp = shouldUsePkg cryptopp; - optNss = shouldUsePkg nss; - optNspr = shouldUsePkg nspr; - - optLibaio = shouldUsePkg libaio; - optLibxfs = shouldUsePkg libxfs; - optZfs = shouldUsePkg zfs; - - hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null; - - - # TODO: Reenable when kinetic support is fixed - #hasKinetic = versionAtLeast version "9.0.0" && optKinetic-cpp-client != null; - hasKinetic = false; - - # Malloc implementation (can be jemalloc, tcmalloc or null) - malloc = if optJemalloc != null then optJemalloc else optGperftools; - - # We prefer nss over cryptopp - cryptoStr = if optNss != null && optNspr != null then "nss" else - if optCryptopp != null then "cryptopp" else "none"; - cryptoLibsMap = { - nss = [ optNss optNspr ]; - cryptopp = [ optCryptopp ]; - none = [ ]; - }; - - ceph-python-env = python2Packages.python.withPackages (ps: [ - ps.sphinx - ps.flask - ps.cython - ps.setuptools - ps.pip - # Libraries needed by the python tools - ps.Mako - ps.pecan - ps.prettytable - ps.webob - ps.cherrypy - ]); - -in -stdenv.mkDerivation { - pname = "ceph"; - inherit version; - - inherit src; - - patches = [ - # ./ceph-patch-cmake-path.patch - ./0001-kv-RocksDBStore-API-break-additional.patch - ] ++ optionals stdenv.isLinux [ - ./0002-fix-absolute-include-path.patch - ]; - - nativeBuildInputs = [ - cmake - pkgconfig which git python2Packages.wrapPython makeWrapper - (ensureNewerSourcesHook { year = "1980"; }) - ]; - - buildInputs = buildInputs ++ cryptoLibsMap.${cryptoStr} ++ [ - boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3 - malloc zlib openldap lttng-ust babeltrace gperf cunit - snappy rocksdb - ] ++ optionals stdenv.isLinux [ - linuxHeaders libuuid udev keyutils optLibaio optLibxfs optZfs - ] ++ optionals hasRadosgw [ - optFcgi optExpat optCurl optFuse optLibedit - ] ++ optionals hasKinetic [ - optKinetic-cpp-client - ]; - - - preConfigure ='' - # rip off submodule that interfer with system libs - rm -rf src/boost - rm -rf src/rocksdb - - # require LD_LIBRARY_PATH for cython to find internal dep - export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH" - - # requires setuptools due to embedded in-cmake setup.py usage - export PYTHONPATH="${python2Packages.setuptools}/lib/python2.7/site-packages/:$PYTHONPATH" - ''; - - cmakeFlags = [ - "-DENABLE_GIT_VERSION=OFF" - "-DWITH_SYSTEM_BOOST=ON" - "-DWITH_SYSTEM_ROCKSDB=ON" - "-DWITH_LEVELDB=OFF" - - # enforce shared lib - "-DBUILD_SHARED_LIBS=ON" - - # disable cephfs, cmake build broken for now - "-DWITH_CEPHFS=OFF" - "-DWITH_LIBCEPHFS=OFF" - ]; - - postFixup = '' - wrapPythonPrograms - wrapProgram $out/bin/ceph-mgr --set PYTHONPATH $out/${python2Packages.python.sitePackages} - ''; - - enableParallelBuilding = true; - - outputs = [ "dev" "lib" "out" "doc" ]; - - meta = { - homepage = https://ceph.com/; - description = "Distributed storage system"; - license = licenses.lgpl21; - maintainers = with maintainers; [ adev ak ]; - platforms = platforms.unix; - broken = true; - }; - - passthru.version = version; -} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index b5a76fc5541..85f7a786a6c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2193,9 +2193,11 @@ in nrg2iso = callPackage ../tools/cd-dvd/nrg2iso { }; libceph = ceph.lib; - ceph = callPackage ../tools/filesystems/ceph { - boost = boost166.override { enablePython = true; }; - }; + inherit (callPackages ../tools/filesystems/ceph { + boost = boost167.override { enablePython = true; }; + }) + ceph + ceph-client; ceph-dev = ceph; inherit (callPackages ../tools/security/certmgr { })