Merge pull request #44340 from shmish111/es-curator
nixos/curator: init elasticsearch curator
This commit is contained in:
commit
a144c798e5
|
@ -111,6 +111,12 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull'
|
|||
<link xlink:href="https://github.com/strongswan/strongswan/blob/master/README_LEGACY.md">stroke configuration interface</link>.
|
||||
</para>
|
||||
</listitem>
|
||||
<listitem>
|
||||
<para>
|
||||
The new <varname>services.elasticsearch-curator</varname> service
|
||||
periodically curates or manages, your Elasticsearch indices and snapshots.
|
||||
</para>
|
||||
</listitem>
|
||||
</itemizedlist>
|
||||
</section>
|
||||
|
||||
|
|
|
@ -623,6 +623,7 @@
|
|||
./services/scheduling/fcron.nix
|
||||
./services/scheduling/marathon.nix
|
||||
./services/search/elasticsearch.nix
|
||||
./services/search/elasticsearch-curator.nix
|
||||
./services/search/hound.nix
|
||||
./services/search/kibana.nix
|
||||
./services/search/solr.nix
|
||||
|
|
93
nixos/modules/services/search/elasticsearch-curator.nix
Normal file
93
nixos/modules/services/search/elasticsearch-curator.nix
Normal file
|
@ -0,0 +1,93 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.services.elasticsearch-curator;
|
||||
curatorConfig = pkgs.writeTextFile {
|
||||
name = "config.yaml";
|
||||
text = ''
|
||||
---
|
||||
# Remember, leave a key empty if there is no value. None will be a string,
|
||||
# not a Python "NoneType"
|
||||
client:
|
||||
hosts: ${builtins.toJSON cfg.hosts}
|
||||
port: ${toString cfg.port}
|
||||
url_prefix:
|
||||
use_ssl: False
|
||||
certificate:
|
||||
client_cert:
|
||||
client_key:
|
||||
ssl_no_validate: False
|
||||
http_auth:
|
||||
timeout: 30
|
||||
master_only: False
|
||||
logging:
|
||||
loglevel: INFO
|
||||
logfile:
|
||||
logformat: default
|
||||
blacklist: ['elasticsearch', 'urllib3']
|
||||
'';
|
||||
};
|
||||
curatorAction = pkgs.writeTextFile {
|
||||
name = "action.yaml";
|
||||
text = cfg.actionYAML;
|
||||
};
|
||||
in {
|
||||
|
||||
options.services.elasticsearch-curator = {
|
||||
|
||||
enable = mkEnableOption "elasticsearch curator";
|
||||
interval = mkOption {
|
||||
description = "The frequency to run curator, a systemd.time such as 'hourly'";
|
||||
default = "hourly";
|
||||
type = types.str;
|
||||
};
|
||||
hosts = mkOption {
|
||||
description = "a list of elasticsearch hosts to connect to";
|
||||
type = types.listOf types.str;
|
||||
default = ["localhost"];
|
||||
};
|
||||
port = mkOption {
|
||||
description = "the port that elasticsearch is listening on";
|
||||
type = types.int;
|
||||
default = 9200;
|
||||
};
|
||||
actionYAML = mkOption {
|
||||
description = "curator action.yaml file contents, alternatively use curator-cli which takes a simple action command";
|
||||
example = ''
|
||||
---
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete indices older than 45 days (based on index name), for logstash-
|
||||
prefixed indices. Ignore the error if the filter does not result in an
|
||||
actionable list of indices (ignore_empty_list) and exit cleanly.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: prefix
|
||||
value: logstash-
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: 45
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
|
||||
systemd.services.elasticsearch-curator = {
|
||||
startAt = cfg.interval;
|
||||
serviceConfig = {
|
||||
ExecStart = ''${pkgs.python36Packages.elasticsearch-curator}/bin/curator --config ${curatorConfig} ${curatorAction}'';
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
|
@ -63,6 +63,33 @@ let
|
|||
package = elk.kibana;
|
||||
elasticsearch.url = esUrl;
|
||||
};
|
||||
|
||||
elasticsearch-curator = {
|
||||
enable = true;
|
||||
actionYAML = ''
|
||||
---
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete indices older than 1 second (based on index name), for logstash-
|
||||
prefixed indices. Ignore the error if the filter does not result in an
|
||||
actionable list of indices (ignore_empty_list) and exit cleanly.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: prefix
|
||||
value: logstash-
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: seconds
|
||||
unit_count: 1
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
@ -91,6 +118,11 @@ let
|
|||
# See if logstash messages arive in elasticsearch.
|
||||
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
|
||||
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
|
||||
|
||||
# Test elasticsearch-curator.
|
||||
$one->systemctl("stop logstash");
|
||||
$one->systemctl("start elasticsearch-curator");
|
||||
$one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
|
||||
'';
|
||||
};
|
||||
in mapAttrs mkElkTest {
|
||||
|
|
|
@ -1,8 +1,10 @@
|
|||
{ stdenv
|
||||
, buildPythonPackage
|
||||
, fetchPypi
|
||||
, boto3
|
||||
, click
|
||||
, certifi
|
||||
, requests-aws4auth
|
||||
, voluptuous
|
||||
, pyyaml
|
||||
, elasticsearch
|
||||
|
@ -22,17 +24,17 @@ buildPythonPackage rec {
|
|||
sha256 = "e75abeb7f7be939b1c64c071898760dc10ab5f08307c253fc074abf8a41a76f0";
|
||||
};
|
||||
|
||||
# The integration tests require a running elasticsearch cluster.
|
||||
postUnpackPhase = ''
|
||||
rm -r test/integration
|
||||
'';
|
||||
# The test hangs so we disable it.
|
||||
doCheck = false;
|
||||
|
||||
propagatedBuildInputs = [
|
||||
click
|
||||
certifi
|
||||
requests-aws4auth
|
||||
voluptuous
|
||||
pyyaml
|
||||
elasticsearch
|
||||
boto3
|
||||
];
|
||||
|
||||
checkInputs = [
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
{ lib, buildPythonPackage, fetchPypi, fetchzip, isPy3k, requests }:
|
||||
with lib;
|
||||
buildPythonPackage rec {
|
||||
pname = "requests-aws4auth";
|
||||
version = "0.9";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "0g52a1pm53aqkc9qb5q1m918c1qy6q47c1qz63p5ilynfbs3m5y9";
|
||||
};
|
||||
|
||||
postPatch = optionalString isPy3k ''
|
||||
sed "s/path_encoding_style/'path_encoding_style'/" \
|
||||
-i requests_aws4auth/service_parameters.py
|
||||
'';
|
||||
|
||||
propagatedBuildInputs = [ requests ];
|
||||
|
||||
# The test fail on Python >= 3 because of module import errors.
|
||||
doCheck = !isPy3k;
|
||||
|
||||
meta = {
|
||||
description = "Amazon Web Services version 4 authentication for the Python Requests library.";
|
||||
homepage = https://github.com/sam-washington/requests-aws4auth;
|
||||
license = licenses.mit;
|
||||
maintainers = [ maintainers.basvandijk ];
|
||||
};
|
||||
}
|
|
@ -2009,6 +2009,8 @@ in {
|
|||
|
||||
requests-unixsocket = callPackage ../development/python-modules/requests-unixsocket {};
|
||||
|
||||
requests-aws4auth = callPackage ../development/python-modules/requests-aws4auth { };
|
||||
|
||||
howdoi = callPackage ../development/python-modules/howdoi {};
|
||||
|
||||
neurotools = callPackage ../development/python-modules/neurotools {};
|
||||
|
|
Loading…
Reference in a new issue