cassandra: rewrote service from scratch
Adds a replacement for the previously broken `services.database.cassandra` with tests for a multi-node setup.
This commit is contained in:
parent
291018b34e
commit
31e11bdd60
|
@ -73,6 +73,14 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull'
|
||||||
</para>
|
</para>
|
||||||
|
|
||||||
<itemizedlist>
|
<itemizedlist>
|
||||||
|
<listitem>
|
||||||
|
<para>
|
||||||
|
The <varname>services.cassandra</varname> module has been reworked and
|
||||||
|
was rewritten from scratch. The service has succeeding tests for
|
||||||
|
the versions 2.1, 2.2, 3.0 and 3.11 of <link
|
||||||
|
xlink:href="https://cassandra.apache.org/">Apache Cassandra</link>.
|
||||||
|
</para>
|
||||||
|
</listitem>
|
||||||
<listitem>
|
<listitem>
|
||||||
<para>
|
<para>
|
||||||
There is a new <varname>services.foundationdb</varname> module for deploying
|
There is a new <varname>services.foundationdb</varname> module for deploying
|
||||||
|
@ -119,6 +127,12 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull'
|
||||||
</para>
|
</para>
|
||||||
|
|
||||||
<itemizedlist>
|
<itemizedlist>
|
||||||
|
<listitem>
|
||||||
|
<para>
|
||||||
|
The deprecated <varname>services.cassandra</varname> module has
|
||||||
|
seen a complete rewrite. (See above.)
|
||||||
|
</para>
|
||||||
|
</listitem>
|
||||||
<listitem>
|
<listitem>
|
||||||
<para>
|
<para>
|
||||||
<literal>lib.strict</literal> is removed. Use
|
<literal>lib.strict</literal> is removed. Use
|
||||||
|
|
|
@ -324,6 +324,7 @@
|
||||||
hadoop = 297;
|
hadoop = 297;
|
||||||
hydron = 298;
|
hydron = 298;
|
||||||
cfssl = 299;
|
cfssl = 299;
|
||||||
|
cassandra = 300;
|
||||||
|
|
||||||
# When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
|
# When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
|
||||||
|
|
||||||
|
@ -608,6 +609,7 @@
|
||||||
hadoop = 297;
|
hadoop = 297;
|
||||||
hydron = 298;
|
hydron = 298;
|
||||||
cfssl = 299;
|
cfssl = 299;
|
||||||
|
cassandra = 300;
|
||||||
|
|
||||||
# When adding a gid, make sure it doesn't match an existing
|
# When adding a gid, make sure it doesn't match an existing
|
||||||
# uid. Users and groups with the same name should have equal
|
# uid. Users and groups with the same name should have equal
|
||||||
|
|
|
@ -201,6 +201,7 @@
|
||||||
./services/databases/4store-endpoint.nix
|
./services/databases/4store-endpoint.nix
|
||||||
./services/databases/4store.nix
|
./services/databases/4store.nix
|
||||||
./services/databases/aerospike.nix
|
./services/databases/aerospike.nix
|
||||||
|
./services/databases/cassandra.nix
|
||||||
./services/databases/clickhouse.nix
|
./services/databases/clickhouse.nix
|
||||||
./services/databases/couchdb.nix
|
./services/databases/couchdb.nix
|
||||||
./services/databases/firebird.nix
|
./services/databases/firebird.nix
|
||||||
|
|
|
@ -4,445 +4,288 @@ with lib;
|
||||||
|
|
||||||
let
|
let
|
||||||
cfg = config.services.cassandra;
|
cfg = config.services.cassandra;
|
||||||
cassandraPackage = cfg.package.override {
|
defaultUser = "cassandra";
|
||||||
jre = cfg.jre;
|
cassandraConfig = flip recursiveUpdate cfg.extraConfig
|
||||||
};
|
({ commitlog_sync = "batch";
|
||||||
cassandraUser = {
|
commitlog_sync_batch_window_in_ms = 2;
|
||||||
name = cfg.user;
|
partitioner = "org.apache.cassandra.dht.Murmur3Partitioner";
|
||||||
home = "/var/lib/cassandra";
|
endpoint_snitch = "SimpleSnitch";
|
||||||
description = "Cassandra role user";
|
seed_provider =
|
||||||
};
|
[{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
|
||||||
|
parameters = [ { seeds = "127.0.0.1"; } ];
|
||||||
cassandraRackDcProperties = ''
|
}];
|
||||||
dc=${cfg.dc}
|
data_file_directories = [ "${cfg.homeDir}/data" ];
|
||||||
rack=${cfg.rack}
|
commitlog_directory = "${cfg.homeDir}/commitlog";
|
||||||
'';
|
saved_caches_directory = "${cfg.homeDir}/saved_caches";
|
||||||
|
} // (if builtins.compareVersions cfg.package.version "3" >= 0
|
||||||
cassandraConf = ''
|
then { hints_directory = "${cfg.homeDir}/hints"; }
|
||||||
cluster_name: ${cfg.clusterName}
|
else {})
|
||||||
num_tokens: 256
|
);
|
||||||
auto_bootstrap: ${boolToString cfg.autoBootstrap}
|
cassandraConfigWithAddresses = cassandraConfig //
|
||||||
hinted_handoff_enabled: ${boolToString cfg.hintedHandOff}
|
( if isNull cfg.listenAddress
|
||||||
hinted_handoff_throttle_in_kb: ${builtins.toString cfg.hintedHandOffThrottle}
|
then { listen_interface = cfg.listenInterface; }
|
||||||
max_hints_delivery_threads: 2
|
else { listen_address = cfg.listenAddress; }
|
||||||
max_hint_window_in_ms: 10800000 # 3 hours
|
) // (
|
||||||
authenticator: ${cfg.authenticator}
|
if isNull cfg.rpcAddress
|
||||||
authorizer: ${cfg.authorizer}
|
then { rpc_interface = cfg.rpcInterface; }
|
||||||
permissions_validity_in_ms: 2000
|
else { rpc_address = cfg.rpcAddress; }
|
||||||
partitioner: org.apache.cassandra.dht.Murmur3Partitioner
|
);
|
||||||
data_file_directories:
|
cassandraEtc = pkgs.stdenv.mkDerivation
|
||||||
${builtins.concatStringsSep "\n" (map (v: " - "+v) cfg.dataDirs)}
|
{ name = "cassandra-etc";
|
||||||
commitlog_directory: ${cfg.commitLogDirectory}
|
cassandraYaml = builtins.toJSON cassandraConfigWithAddresses;
|
||||||
disk_failure_policy: stop
|
cassandraEnvPkg = "${cfg.package}/conf/cassandra-env.sh";
|
||||||
key_cache_size_in_mb:
|
buildCommand = ''
|
||||||
key_cache_save_period: 14400
|
mkdir -p "$out"
|
||||||
row_cache_size_in_mb: 0
|
|
||||||
row_cache_save_period: 0
|
|
||||||
saved_caches_directory: ${cfg.savedCachesDirectory}
|
|
||||||
commitlog_sync: ${cfg.commitLogSync}
|
|
||||||
commitlog_sync_period_in_ms: ${builtins.toString cfg.commitLogSyncPeriod}
|
|
||||||
commitlog_segment_size_in_mb: 32
|
|
||||||
seed_provider:
|
|
||||||
- class_name: org.apache.cassandra.locator.SimpleSeedProvider
|
|
||||||
parameters:
|
|
||||||
- seeds: "${builtins.concatStringsSep "," cfg.seeds}"
|
|
||||||
concurrent_reads: ${builtins.toString cfg.concurrentReads}
|
|
||||||
concurrent_writes: ${builtins.toString cfg.concurrentWrites}
|
|
||||||
memtable_flush_queue_size: 4
|
|
||||||
trickle_fsync: false
|
|
||||||
trickle_fsync_interval_in_kb: 10240
|
|
||||||
storage_port: 7000
|
|
||||||
ssl_storage_port: 7001
|
|
||||||
listen_address: ${cfg.listenAddress}
|
|
||||||
start_native_transport: true
|
|
||||||
native_transport_port: 9042
|
|
||||||
start_rpc: true
|
|
||||||
rpc_address: ${cfg.rpcAddress}
|
|
||||||
rpc_port: 9160
|
|
||||||
rpc_keepalive: true
|
|
||||||
rpc_server_type: sync
|
|
||||||
thrift_framed_transport_size_in_mb: 15
|
|
||||||
incremental_backups: ${boolToString cfg.incrementalBackups}
|
|
||||||
snapshot_before_compaction: false
|
|
||||||
auto_snapshot: true
|
|
||||||
column_index_size_in_kb: 64
|
|
||||||
in_memory_compaction_limit_in_mb: 64
|
|
||||||
multithreaded_compaction: false
|
|
||||||
compaction_throughput_mb_per_sec: 16
|
|
||||||
compaction_preheat_key_cache: true
|
|
||||||
read_request_timeout_in_ms: 10000
|
|
||||||
range_request_timeout_in_ms: 10000
|
|
||||||
write_request_timeout_in_ms: 10000
|
|
||||||
cas_contention_timeout_in_ms: 1000
|
|
||||||
truncate_request_timeout_in_ms: 60000
|
|
||||||
request_timeout_in_ms: 10000
|
|
||||||
cross_node_timeout: false
|
|
||||||
endpoint_snitch: ${cfg.snitch}
|
|
||||||
dynamic_snitch_update_interval_in_ms: 100
|
|
||||||
dynamic_snitch_reset_interval_in_ms: 600000
|
|
||||||
dynamic_snitch_badness_threshold: 0.1
|
|
||||||
request_scheduler: org.apache.cassandra.scheduler.NoScheduler
|
|
||||||
server_encryption_options:
|
|
||||||
internode_encryption: ${cfg.internodeEncryption}
|
|
||||||
keystore: ${cfg.keyStorePath}
|
|
||||||
keystore_password: ${cfg.keyStorePassword}
|
|
||||||
truststore: ${cfg.trustStorePath}
|
|
||||||
truststore_password: ${cfg.trustStorePassword}
|
|
||||||
client_encryption_options:
|
|
||||||
enabled: ${boolToString cfg.clientEncryption}
|
|
||||||
keystore: ${cfg.keyStorePath}
|
|
||||||
keystore_password: ${cfg.keyStorePassword}
|
|
||||||
internode_compression: all
|
|
||||||
inter_dc_tcp_nodelay: false
|
|
||||||
preheat_kernel_page_cache: false
|
|
||||||
streaming_socket_timeout_in_ms: ${toString cfg.streamingSocketTimoutInMS}
|
|
||||||
'';
|
|
||||||
|
|
||||||
cassandraLog = ''
|
|
||||||
log4j.rootLogger=${cfg.logLevel},stdout
|
|
||||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
|
||||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
|
||||||
log4j.appender.stdout.layout.ConversionPattern=%5p [%t] %d{HH:mm:ss,SSS} %m%n
|
|
||||||
'';
|
|
||||||
|
|
||||||
cassandraConfFile = pkgs.writeText "cassandra.yaml" cassandraConf;
|
|
||||||
cassandraLogFile = pkgs.writeText "log4j-server.properties" cassandraLog;
|
|
||||||
cassandraRackFile = pkgs.writeText "cassandra-rackdc.properties" cassandraRackDcProperties;
|
|
||||||
|
|
||||||
cassandraEnvironment = {
|
|
||||||
CASSANDRA_HOME = cassandraPackage;
|
|
||||||
JAVA_HOME = cfg.jre;
|
|
||||||
CASSANDRA_CONF = "/etc/cassandra";
|
|
||||||
};
|
|
||||||
|
|
||||||
|
echo "$cassandraYaml" > "$out/cassandra.yaml"
|
||||||
|
ln -s "$cassandraEnvPkg" "$out/cassandra-env.sh"
|
||||||
|
'';
|
||||||
|
};
|
||||||
in {
|
in {
|
||||||
|
|
||||||
###### interface
|
|
||||||
|
|
||||||
options.services.cassandra = {
|
options.services.cassandra = {
|
||||||
enable = mkOption {
|
enable = mkEnableOption ''
|
||||||
description = "Whether to enable cassandra.";
|
Apache Cassandra – Scalable and highly available database.
|
||||||
default = false;
|
'';
|
||||||
type = types.bool;
|
|
||||||
};
|
|
||||||
package = mkOption {
|
|
||||||
description = "Cassandra package to use.";
|
|
||||||
default = pkgs.cassandra;
|
|
||||||
defaultText = "pkgs.cassandra";
|
|
||||||
type = types.package;
|
|
||||||
};
|
|
||||||
jre = mkOption {
|
|
||||||
description = "JRE package to run cassandra service.";
|
|
||||||
default = pkgs.jre;
|
|
||||||
defaultText = "pkgs.jre";
|
|
||||||
type = types.package;
|
|
||||||
};
|
|
||||||
user = mkOption {
|
user = mkOption {
|
||||||
description = "User that runs cassandra service.";
|
type = types.str;
|
||||||
default = "cassandra";
|
default = defaultUser;
|
||||||
type = types.string;
|
description = "Run Apache Cassandra under this user.";
|
||||||
};
|
};
|
||||||
group = mkOption {
|
group = mkOption {
|
||||||
description = "Group that runs cassandra service.";
|
|
||||||
default = "cassandra";
|
|
||||||
type = types.string;
|
|
||||||
};
|
|
||||||
envFile = mkOption {
|
|
||||||
description = "path to cassandra-env.sh";
|
|
||||||
default = "${cassandraPackage}/conf/cassandra-env.sh";
|
|
||||||
defaultText = "\${cassandraPackage}/conf/cassandra-env.sh";
|
|
||||||
type = types.path;
|
|
||||||
};
|
|
||||||
clusterName = mkOption {
|
|
||||||
description = "set cluster name";
|
|
||||||
default = "cassandra";
|
|
||||||
example = "prod-cluster0";
|
|
||||||
type = types.string;
|
|
||||||
};
|
|
||||||
commitLogDirectory = mkOption {
|
|
||||||
description = "directory for commit logs";
|
|
||||||
default = "/var/lib/cassandra/commit_log";
|
|
||||||
type = types.string;
|
|
||||||
};
|
|
||||||
savedCachesDirectory = mkOption {
|
|
||||||
description = "directory for saved caches";
|
|
||||||
default = "/var/lib/cassandra/saved_caches";
|
|
||||||
type = types.string;
|
|
||||||
};
|
|
||||||
hintedHandOff = mkOption {
|
|
||||||
description = "enable hinted handoff";
|
|
||||||
default = true;
|
|
||||||
type = types.bool;
|
|
||||||
};
|
|
||||||
hintedHandOffThrottle = mkOption {
|
|
||||||
description = "hinted hand off throttle rate in kb";
|
|
||||||
default = 1024;
|
|
||||||
type = types.int;
|
|
||||||
};
|
|
||||||
commitLogSync = mkOption {
|
|
||||||
description = "commitlog sync method";
|
|
||||||
default = "periodic";
|
|
||||||
type = types.str;
|
type = types.str;
|
||||||
example = "batch";
|
default = defaultUser;
|
||||||
|
description = "Run Apache Cassandra under this group.";
|
||||||
};
|
};
|
||||||
commitLogSyncPeriod = mkOption {
|
homeDir = mkOption {
|
||||||
description = "commitlog sync period in ms ";
|
|
||||||
default = 10000;
|
|
||||||
type = types.int;
|
|
||||||
};
|
|
||||||
envScript = mkOption {
|
|
||||||
default = "${cassandraPackage}/conf/cassandra-env.sh";
|
|
||||||
defaultText = "\${cassandraPackage}/conf/cassandra-env.sh";
|
|
||||||
type = types.path;
|
type = types.path;
|
||||||
description = "Supply your own cassandra-env.sh rather than using the default";
|
default = "/var/lib/cassandra";
|
||||||
|
description = ''
|
||||||
|
Home directory for Apache Cassandra.
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
extraParams = mkOption {
|
package = mkOption {
|
||||||
description = "add additional lines to cassandra-env.sh";
|
type = types.package;
|
||||||
|
default = pkgs.cassandra;
|
||||||
|
defaultText = "pkgs.cassandra";
|
||||||
|
example = literalExample "pkgs.cassandra_3_11";
|
||||||
|
description = ''
|
||||||
|
The Apache Cassandra package to use.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
jvmOpts = mkOption {
|
||||||
|
type = types.listOf types.str;
|
||||||
default = [];
|
default = [];
|
||||||
example = [''JVM_OPTS="$JVM_OPTS -Dcassandra.available_processors=1"''];
|
description = ''
|
||||||
type = types.listOf types.str;
|
Populate the JVM_OPT environment variable.
|
||||||
};
|
'';
|
||||||
dataDirs = mkOption {
|
|
||||||
type = types.listOf types.path;
|
|
||||||
default = [ "/var/lib/cassandra/data" ];
|
|
||||||
description = "Data directories for cassandra";
|
|
||||||
};
|
|
||||||
logLevel = mkOption {
|
|
||||||
type = types.str;
|
|
||||||
default = "INFO";
|
|
||||||
description = "default logging level for log4j";
|
|
||||||
};
|
|
||||||
internodeEncryption = mkOption {
|
|
||||||
description = "enable internode encryption";
|
|
||||||
default = "none";
|
|
||||||
example = "all";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
clientEncryption = mkOption {
|
|
||||||
description = "enable client encryption";
|
|
||||||
default = false;
|
|
||||||
type = types.bool;
|
|
||||||
};
|
|
||||||
trustStorePath = mkOption {
|
|
||||||
description = "path to truststore";
|
|
||||||
default = ".conf/truststore";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
keyStorePath = mkOption {
|
|
||||||
description = "path to keystore";
|
|
||||||
default = ".conf/keystore";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
keyStorePassword = mkOption {
|
|
||||||
description = "password to keystore";
|
|
||||||
default = "cassandra";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
trustStorePassword = mkOption {
|
|
||||||
description = "password to truststore";
|
|
||||||
default = "cassandra";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
seeds = mkOption {
|
|
||||||
description = "password to truststore";
|
|
||||||
default = [ "127.0.0.1" ];
|
|
||||||
type = types.listOf types.str;
|
|
||||||
};
|
|
||||||
concurrentWrites = mkOption {
|
|
||||||
description = "number of concurrent writes allowed";
|
|
||||||
default = 32;
|
|
||||||
type = types.int;
|
|
||||||
};
|
|
||||||
concurrentReads = mkOption {
|
|
||||||
description = "number of concurrent reads allowed";
|
|
||||||
default = 32;
|
|
||||||
type = types.int;
|
|
||||||
};
|
};
|
||||||
listenAddress = mkOption {
|
listenAddress = mkOption {
|
||||||
description = "listen address";
|
type = types.nullOr types.str;
|
||||||
default = "localhost";
|
default = "127.0.0.1";
|
||||||
type = types.str;
|
example = literalExample "null";
|
||||||
|
description = ''
|
||||||
|
Address or interface to bind to and tell other Cassandra nodes
|
||||||
|
to connect to. You _must_ change this if you want multiple
|
||||||
|
nodes to be able to communicate!
|
||||||
|
|
||||||
|
Set listenAddress OR listenInterface, not both.
|
||||||
|
|
||||||
|
Leaving it blank leaves it up to
|
||||||
|
InetAddress.getLocalHost(). This will always do the Right
|
||||||
|
Thing _if_ the node is properly configured (hostname, name
|
||||||
|
resolution, etc), and the Right Thing is to use the address
|
||||||
|
associated with the hostname (it might not be).
|
||||||
|
|
||||||
|
Setting listen_address to 0.0.0.0 is always wrong.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
listenInterface = mkOption {
|
||||||
|
type = types.nullOr types.str;
|
||||||
|
default = null;
|
||||||
|
example = "eth1";
|
||||||
|
description = ''
|
||||||
|
Set listenAddress OR listenInterface, not both. Interfaces
|
||||||
|
must correspond to a single address, IP aliasing is not
|
||||||
|
supported.
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
rpcAddress = mkOption {
|
rpcAddress = mkOption {
|
||||||
description = "rpc listener address";
|
type = types.nullOr types.str;
|
||||||
default = "localhost";
|
default = "127.0.0.1";
|
||||||
type = types.str;
|
example = literalExample "null";
|
||||||
};
|
|
||||||
incrementalBackups = mkOption {
|
|
||||||
description = "enable incremental backups";
|
|
||||||
default = false;
|
|
||||||
type = types.bool;
|
|
||||||
};
|
|
||||||
snitch = mkOption {
|
|
||||||
description = "snitch to use for topology discovery";
|
|
||||||
default = "GossipingPropertyFileSnitch";
|
|
||||||
example = "Ec2Snitch";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
dc = mkOption {
|
|
||||||
description = "datacenter for use in topology configuration";
|
|
||||||
default = "DC1";
|
|
||||||
example = "DC1";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
rack = mkOption {
|
|
||||||
description = "rack for use in topology configuration";
|
|
||||||
default = "RAC1";
|
|
||||||
example = "RAC1";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
authorizer = mkOption {
|
|
||||||
description = "
|
|
||||||
Authorization backend, implementing IAuthorizer; used to limit access/provide permissions
|
|
||||||
";
|
|
||||||
default = "AllowAllAuthorizer";
|
|
||||||
example = "CassandraAuthorizer";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
authenticator = mkOption {
|
|
||||||
description = "
|
|
||||||
Authentication backend, implementing IAuthenticator; used to identify users
|
|
||||||
";
|
|
||||||
default = "AllowAllAuthenticator";
|
|
||||||
example = "PasswordAuthenticator";
|
|
||||||
type = types.str;
|
|
||||||
};
|
|
||||||
autoBootstrap = mkOption {
|
|
||||||
description = "It makes new (non-seed) nodes automatically migrate the right data to themselves.";
|
|
||||||
default = true;
|
|
||||||
type = types.bool;
|
|
||||||
};
|
|
||||||
streamingSocketTimoutInMS = mkOption {
|
|
||||||
description = "Enable or disable socket timeout for streaming operations";
|
|
||||||
default = 3600000; #CASSANDRA-8611
|
|
||||||
example = 120;
|
|
||||||
type = types.int;
|
|
||||||
};
|
|
||||||
repairStartAt = mkOption {
|
|
||||||
default = "Sun";
|
|
||||||
type = types.string;
|
|
||||||
description = ''
|
description = ''
|
||||||
Defines realtime (i.e. wallclock) timers with calendar event
|
The address or interface to bind the native transport server to.
|
||||||
expressions. For more details re: systemd OnCalendar at
|
|
||||||
https://www.freedesktop.org/software/systemd/man/systemd.time.html#Displaying%20Time%20Spans
|
Set rpcAddress OR rpcInterface, not both.
|
||||||
'';
|
|
||||||
example = ["weekly" "daily" "08:05:40" "mon,fri *-1/2-1,3 *:30:45"];
|
Leaving rpcAddress blank has the same effect as on
|
||||||
};
|
listenAddress (i.e. it will be based on the configured hostname
|
||||||
repairRandomizedDelayInSec = mkOption {
|
of the node).
|
||||||
default = 0;
|
|
||||||
type = types.int;
|
Note that unlike listenAddress, you can specify 0.0.0.0, but you
|
||||||
description = ''Delay the timer by a randomly selected, evenly distributed
|
must also set extraConfig.broadcast_rpc_address to a value other
|
||||||
amount of time between 0 and the specified time value. re: systemd timer
|
than 0.0.0.0.
|
||||||
RandomizedDelaySec for more details
|
|
||||||
|
For security reasons, you should not expose this port to the
|
||||||
|
internet. Firewall it if needed.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
repairPostStop = mkOption {
|
rpcInterface = mkOption {
|
||||||
|
type = types.nullOr types.str;
|
||||||
default = null;
|
default = null;
|
||||||
type = types.nullOr types.string;
|
example = "eth1";
|
||||||
description = ''
|
description = ''
|
||||||
Run a script when repair is over. One can use it to send statsd events, email, etc.
|
Set rpcAddress OR rpcInterface, not both. Interfaces must
|
||||||
|
correspond to a single address, IP aliasing is not supported.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
repairPostStart = mkOption {
|
|
||||||
default = null;
|
extraConfig = mkOption {
|
||||||
type = types.nullOr types.string;
|
type = types.attrs;
|
||||||
|
default = {};
|
||||||
|
example =
|
||||||
|
{ commitlog_sync_batch_window_in_ms = 3;
|
||||||
|
};
|
||||||
description = ''
|
description = ''
|
||||||
Run a script when repair starts. One can use it to send statsd events, email, etc.
|
Extra options to be merged into cassandra.yaml as nix attribute set.
|
||||||
It has same semantics as systemd ExecStopPost; So, if it fails, unit is consisdered
|
|
||||||
failed.
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
fullRepairInterval = mkOption {
|
||||||
|
type = types.nullOr types.str;
|
||||||
|
default = "3w";
|
||||||
|
example = literalExample "null";
|
||||||
|
description = ''
|
||||||
|
Set the interval how often full repairs are run, i.e.
|
||||||
|
`nodetool repair --full` is executed. See
|
||||||
|
https://cassandra.apache.org/doc/latest/operating/repair.html
|
||||||
|
for more information.
|
||||||
|
|
||||||
|
Set to `null` to disable full repairs.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
fullRepairOptions = mkOption {
|
||||||
|
type = types.listOf types.str;
|
||||||
|
default = [];
|
||||||
|
example = [ "--partitioner-range" ];
|
||||||
|
description = ''
|
||||||
|
Options passed through to the full repair command.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
incrementalRepairInterval = mkOption {
|
||||||
|
type = types.nullOr types.str;
|
||||||
|
default = "3d";
|
||||||
|
example = literalExample "null";
|
||||||
|
description = ''
|
||||||
|
Set the interval how often incremental repairs are run, i.e.
|
||||||
|
`nodetool repair` is executed. See
|
||||||
|
https://cassandra.apache.org/doc/latest/operating/repair.html
|
||||||
|
for more information.
|
||||||
|
|
||||||
|
Set to `null` to disable incremental repairs.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
incrementalRepairOptions = mkOption {
|
||||||
|
type = types.listOf types.string;
|
||||||
|
default = [];
|
||||||
|
example = [ "--partitioner-range" ];
|
||||||
|
description = ''
|
||||||
|
Options passed through to the incremental repair command.
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
###### implementation
|
|
||||||
|
|
||||||
config = mkIf cfg.enable {
|
config = mkIf cfg.enable {
|
||||||
|
assertions =
|
||||||
environment.etc."cassandra/cassandra-rackdc.properties" = {
|
[ { assertion =
|
||||||
source = cassandraRackFile;
|
((isNull cfg.listenAddress)
|
||||||
};
|
|| (isNull cfg.listenInterface)
|
||||||
environment.etc."cassandra/cassandra.yaml" = {
|
) && !((isNull cfg.listenAddress)
|
||||||
source = cassandraConfFile;
|
&& (isNull cfg.listenInterface)
|
||||||
};
|
);
|
||||||
environment.etc."cassandra/log4j-server.properties" = {
|
message = "You have to set either listenAddress or listenInterface";
|
||||||
source = cassandraLogFile;
|
}
|
||||||
};
|
{ assertion =
|
||||||
environment.etc."cassandra/cassandra-env.sh" = {
|
((isNull cfg.rpcAddress)
|
||||||
text = ''
|
|| (isNull cfg.rpcInterface)
|
||||||
${builtins.readFile cfg.envFile}
|
) && !((isNull cfg.rpcAddress)
|
||||||
${concatStringsSep "\n" cfg.extraParams}
|
&& (isNull cfg.rpcInterface)
|
||||||
'';
|
);
|
||||||
};
|
message = "You have to set either rpcAddress or rpcInterface";
|
||||||
systemd.services.cassandra = {
|
}
|
||||||
description = "Cassandra Daemon";
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
after = [ "network.target" ];
|
|
||||||
environment = cassandraEnvironment;
|
|
||||||
restartTriggers = [ cassandraConfFile cassandraLogFile cassandraRackFile ];
|
|
||||||
serviceConfig = {
|
|
||||||
|
|
||||||
User = cfg.user;
|
|
||||||
PermissionsStartOnly = true;
|
|
||||||
LimitAS = "infinity";
|
|
||||||
LimitNOFILE = "100000";
|
|
||||||
LimitNPROC = "32768";
|
|
||||||
LimitMEMLOCK = "infinity";
|
|
||||||
|
|
||||||
};
|
|
||||||
script = ''
|
|
||||||
${cassandraPackage}/bin/cassandra -f
|
|
||||||
'';
|
|
||||||
path = [
|
|
||||||
cfg.jre
|
|
||||||
cassandraPackage
|
|
||||||
pkgs.coreutils
|
|
||||||
];
|
];
|
||||||
preStart = ''
|
users = mkIf (cfg.user == defaultUser) {
|
||||||
mkdir -m 0700 -p /etc/cassandra/triggers
|
extraUsers."${defaultUser}" =
|
||||||
mkdir -m 0700 -p /var/lib/cassandra /var/log/cassandra
|
{ group = cfg.group;
|
||||||
chown ${cfg.user} /var/lib/cassandra /var/log/cassandra /etc/cassandra/triggers
|
home = cfg.homeDir;
|
||||||
'';
|
createHome = true;
|
||||||
postStart = ''
|
uid = config.ids.uids.cassandra;
|
||||||
sleep 2
|
description = "Cassandra service user";
|
||||||
while ! nodetool status >/dev/null 2>&1; do
|
};
|
||||||
sleep 2
|
extraGroups."${defaultUser}".gid = config.ids.gids.cassandra;
|
||||||
done
|
|
||||||
nodetool status
|
|
||||||
'';
|
|
||||||
};
|
};
|
||||||
|
|
||||||
environment.systemPackages = [ cassandraPackage ];
|
systemd.services.cassandra =
|
||||||
|
{ description = "Apache Cassandra service";
|
||||||
networking.firewall.allowedTCPPorts = [
|
after = [ "network.target" ];
|
||||||
7000
|
environment =
|
||||||
7001
|
{ CASSANDRA_CONF = "${cassandraEtc}";
|
||||||
9042
|
JVM_OPTS = builtins.concatStringsSep " " cfg.jvmOpts;
|
||||||
9160
|
};
|
||||||
];
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
serviceConfig =
|
||||||
users.users.cassandra =
|
{ User = cfg.user;
|
||||||
if config.ids.uids ? "cassandra"
|
Group = cfg.group;
|
||||||
then { uid = config.ids.uids.cassandra; } // cassandraUser
|
ExecStart = "${cfg.package}/bin/cassandra -f";
|
||||||
else cassandraUser ;
|
SuccessExitStatus = 143;
|
||||||
|
};
|
||||||
boot.kernel.sysctl."vm.swappiness" = pkgs.lib.mkOptionDefault 0;
|
|
||||||
|
|
||||||
systemd.timers."cassandra-repair" = {
|
|
||||||
timerConfig = {
|
|
||||||
OnCalendar = "${toString cfg.repairStartAt}";
|
|
||||||
RandomizedDelaySec = cfg.repairRandomizedDelayInSec;
|
|
||||||
};
|
};
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services."cassandra-repair" = {
|
systemd.services.cassandra-full-repair =
|
||||||
description = "Cassandra repair daemon";
|
{ description = "Perform a full repair on this Cassandra node";
|
||||||
environment = cassandraEnvironment;
|
after = [ "cassandra.service" ];
|
||||||
script = "${cassandraPackage}/bin/nodetool repair -pr";
|
requires = [ "cassandra.service" ];
|
||||||
postStop = mkIf (cfg.repairPostStop != null) cfg.repairPostStop;
|
serviceConfig =
|
||||||
postStart = mkIf (cfg.repairPostStart != null) cfg.repairPostStart;
|
{ User = cfg.user;
|
||||||
serviceConfig = {
|
Group = cfg.group;
|
||||||
User = cfg.user;
|
ExecStart =
|
||||||
|
lib.concatStringsSep " "
|
||||||
|
([ "${cfg.package}/bin/nodetool" "repair" "--full"
|
||||||
|
] ++ cfg.fullRepairOptions);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
systemd.timers.cassandra-full-repair =
|
||||||
|
mkIf (!isNull cfg.fullRepairInterval) {
|
||||||
|
description = "Schedule full repairs on Cassandra";
|
||||||
|
wantedBy = [ "timers.target" ];
|
||||||
|
timerConfig =
|
||||||
|
{ OnBootSec = cfg.fullRepairInterval;
|
||||||
|
OnUnitActiveSec = cfg.fullRepairInterval;
|
||||||
|
Persistent = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.services.cassandra-incremental-repair =
|
||||||
|
{ description = "Perform an incremental repair on this cassandra node.";
|
||||||
|
after = [ "cassandra.service" ];
|
||||||
|
requires = [ "cassandra.service" ];
|
||||||
|
serviceConfig =
|
||||||
|
{ User = cfg.user;
|
||||||
|
Group = cfg.group;
|
||||||
|
ExecStart =
|
||||||
|
lib.concatStringsSep " "
|
||||||
|
([ "${cfg.package}/bin/nodetool" "repair"
|
||||||
|
] ++ cfg.incrementalRepairOptions);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
systemd.timers.cassandra-incremental-repair =
|
||||||
|
mkIf (!isNull cfg.incrementalRepairInterval) {
|
||||||
|
description = "Schedule incremental repairs on Cassandra";
|
||||||
|
wantedBy = [ "timers.target" ];
|
||||||
|
timerConfig =
|
||||||
|
{ OnBootSec = cfg.incrementalRepairInterval;
|
||||||
|
OnUnitActiveSec = cfg.incrementalRepairInterval;
|
||||||
|
Persistent = true;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
};
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,68 +1,71 @@
|
||||||
import ./make-test.nix ({ pkgs, ...}:
|
import ./make-test.nix ({ pkgs, ...}:
|
||||||
let
|
let
|
||||||
user = "cassandra";
|
# Change this to test a different version of Cassandra:
|
||||||
nodeCfg = nodes: selfIP: cassandraOpts:
|
testPackage = pkgs.cassandra;
|
||||||
{
|
cassandraCfg =
|
||||||
services.cassandra = {
|
{ enable = true;
|
||||||
enable = true;
|
listenAddress = null;
|
||||||
listenAddress = selfIP;
|
listenInterface = "eth1";
|
||||||
rpcAddress = "0.0.0.0";
|
rpcAddress = null;
|
||||||
seeds = [ "192.168.1.1" ];
|
rpcInterface = "eth1";
|
||||||
package = pkgs.cassandra_2_0;
|
extraConfig =
|
||||||
jre = pkgs.openjdk;
|
{ start_native_transport = true;
|
||||||
clusterName = "ci ahoy";
|
seed_provider =
|
||||||
authenticator = "PasswordAuthenticator";
|
[{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
|
||||||
authorizer = "CassandraAuthorizer";
|
parameters = [ { seeds = "cass0"; } ];
|
||||||
user = user;
|
}];
|
||||||
} // cassandraOpts;
|
};
|
||||||
nixpkgs.config.allowUnfree = true;
|
package = testPackage;
|
||||||
|
};
|
||||||
|
nodeCfg = extra: {pkgs, config, ...}:
|
||||||
|
{ environment.systemPackages = [ testPackage ];
|
||||||
|
networking.firewall.enable = false;
|
||||||
|
services.cassandra = cassandraCfg // extra;
|
||||||
virtualisation.memorySize = 1024;
|
virtualisation.memorySize = 1024;
|
||||||
};
|
};
|
||||||
|
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
name = "cassandra-ci";
|
name = "cassandra-ci";
|
||||||
|
|
||||||
nodes = {
|
nodes = {
|
||||||
cass0 = { nodes, ... }: nodeCfg nodes "192.168.1.1" {};
|
cass0 = nodeCfg {};
|
||||||
cass1 = { nodes, ... }: nodeCfg nodes "192.168.1.2" {};
|
cass1 = nodeCfg {};
|
||||||
cass2 = { nodes, ... }: nodeCfg nodes "192.168.1.3" {
|
cass2 = nodeCfg { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
|
||||||
extraParams = [
|
|
||||||
''JVM_OPTS="$JVM_OPTS -Dcassandra.replace_address=192.168.1.2"''
|
|
||||||
];
|
|
||||||
listenAddress = "192.168.1.3";
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
testScript = ''
|
testScript = ''
|
||||||
subtest "start seed", sub {
|
subtest "timers exist", sub {
|
||||||
|
$cass0->succeed("systemctl list-timers | grep cassandra-full-repair.timer");
|
||||||
|
$cass0->succeed("systemctl list-timers | grep cassandra-incremental-repair.timer");
|
||||||
|
};
|
||||||
|
subtest "can connect via cqlsh", sub {
|
||||||
$cass0->waitForUnit("cassandra.service");
|
$cass0->waitForUnit("cassandra.service");
|
||||||
$cass0->waitForOpenPort(9160);
|
$cass0->waitUntilSucceeds("nc -z cass0 9042");
|
||||||
$cass0->execute("echo show version | cqlsh localhost -u cassandra -p cassandra");
|
$cass0->succeed("echo 'show version;' | cqlsh cass0");
|
||||||
sleep 2;
|
|
||||||
$cass0->succeed("echo show version | cqlsh localhost -u cassandra -p cassandra");
|
|
||||||
$cass1->start;
|
|
||||||
};
|
};
|
||||||
subtest "cassandra user/group", sub {
|
subtest "nodetool is operational", sub {
|
||||||
$cass0->succeed("id \"${user}\" >/dev/null");
|
$cass0->waitForUnit("cassandra.service");
|
||||||
$cass1->succeed("id \"${user}\" >/dev/null");
|
$cass0->waitUntilSucceeds("nc -z localhost 7199");
|
||||||
|
$cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass0'");
|
||||||
};
|
};
|
||||||
subtest "bring up cassandra cluster", sub {
|
subtest "bring up cluster", sub {
|
||||||
$cass1->waitForUnit("cassandra.service");
|
$cass1->waitForUnit("cassandra.service");
|
||||||
$cass0->waitUntilSucceeds("nodetool status | grep -c UN | grep 2");
|
$cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2");
|
||||||
|
$cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");
|
||||||
};
|
};
|
||||||
subtest "break and fix node", sub {
|
subtest "break and fix node", sub {
|
||||||
$cass0->block;
|
$cass1->block;
|
||||||
$cass0->waitUntilSucceeds("nodetool status | grep -c DN | grep 1");
|
$cass0->waitUntilSucceeds("nodetool status --resolve-ip | egrep -c '^DN[[:space:]]+cass1'");
|
||||||
$cass0->unblock;
|
$cass0->succeed("nodetool status | egrep -c '^UN' | grep 1");
|
||||||
$cass0->waitUntilSucceeds("nodetool status | grep -c UN | grep 2");
|
$cass1->unblock;
|
||||||
|
$cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2");
|
||||||
|
$cass0->succeed("nodetool status | egrep -c '^UN' | grep 2");
|
||||||
};
|
};
|
||||||
subtest "replace crashed node", sub {
|
subtest "replace crashed node", sub {
|
||||||
$cass1->crash;
|
$cass1->crash;
|
||||||
$cass2->start;
|
|
||||||
$cass2->waitForUnit("cassandra.service");
|
$cass2->waitForUnit("cassandra.service");
|
||||||
$cass0->waitUntilFails("nodetool status | grep UN | grep 192.168.1.2");
|
$cass0->waitUntilFails("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");
|
||||||
$cass0->waitUntilSucceeds("nodetool status | grep UN | grep 192.168.1.3");
|
$cass0->waitUntilSucceeds("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass2'");
|
||||||
};
|
};
|
||||||
'';
|
'';
|
||||||
})
|
})
|
||||||
|
|
|
@ -16,6 +16,7 @@ in
|
||||||
|
|
||||||
stdenv.mkDerivation rec {
|
stdenv.mkDerivation rec {
|
||||||
name = "cassandra-${version}";
|
name = "cassandra-${version}";
|
||||||
|
inherit version;
|
||||||
|
|
||||||
src = fetchurl {
|
src = fetchurl {
|
||||||
inherit sha256;
|
inherit sha256;
|
||||||
|
|
Loading…
Reference in a new issue