Merge pull request #44340 from shmish111/es-curator

nixos/curator: init elasticsearch curator

nixos/doc/manual/release-notes/rl-1809.xml

@@ -111,6 +111,12 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull'
       <link xlink:href="https://github.com/strongswan/strongswan/blob/master/README_LEGACY.md">stroke configuration interface</link>.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The new <varname>services.elasticsearch-curator</varname> service
+      periodically curates or manages, your Elasticsearch indices and snapshots.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/module-list.nix

@@ -623,6 +623,7 @@
   ./services/scheduling/fcron.nix
   ./services/scheduling/marathon.nix
   ./services/search/elasticsearch.nix
+  ./services/search/elasticsearch-curator.nix
   ./services/search/hound.nix
   ./services/search/kibana.nix
   ./services/search/solr.nix

nixos/modules/services/search/elasticsearch-curator.nix

@@ -0,0 +1,93 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+    cfg = config.services.elasticsearch-curator;
+    curatorConfig = pkgs.writeTextFile {
+      name = "config.yaml";
+      text = ''
+        ---
+        # Remember, leave a key empty if there is no value.  None will be a string,
+        # not a Python "NoneType"
+        client:
+          hosts: ${builtins.toJSON cfg.hosts}
+          port: ${toString cfg.port}
+          url_prefix:
+          use_ssl: False
+          certificate:
+          client_cert:
+          client_key:
+          ssl_no_validate: False
+          http_auth:
+          timeout: 30
+          master_only: False
+        logging:
+          loglevel: INFO
+          logfile:
+          logformat: default
+          blacklist: ['elasticsearch', 'urllib3']
+        '';
+    };
+    curatorAction = pkgs.writeTextFile {
+      name = "action.yaml";
+      text = cfg.actionYAML;
+    };
+in {
+
+  options.services.elasticsearch-curator = {
+
+    enable = mkEnableOption "elasticsearch curator";
+    interval = mkOption {
+      description = "The frequency to run curator, a systemd.time such as 'hourly'";
+      default = "hourly";
+      type = types.str;
+    };
+    hosts = mkOption {
+      description = "a list of elasticsearch hosts to connect to";
+      type = types.listOf types.str;
+      default = ["localhost"];
+    };
+    port = mkOption {
+      description = "the port that elasticsearch is listening on";
+      type = types.int;
+      default = 9200;
+    };
+    actionYAML = mkOption {
+      description = "curator action.yaml file contents, alternatively use curator-cli which takes a simple action command";
+      example = ''
+        ---
+        actions:
+          1:
+            action: delete_indices
+            description: >-
+              Delete indices older than 45 days (based on index name), for logstash-
+              prefixed indices. Ignore the error if the filter does not result in an
+              actionable list of indices (ignore_empty_list) and exit cleanly.
+            options:
+              ignore_empty_list: True
+              disable_action: False
+            filters:
+            - filtertype: pattern
+              kind: prefix
+              value: logstash-
+            - filtertype: age
+              source: name
+              direction: older
+              timestring: '%Y.%m.%d'
+              unit: days
+              unit_count: 45
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    systemd.services.elasticsearch-curator = {
+      startAt = cfg.interval;
+      serviceConfig = {
+        ExecStart = ''${pkgs.python36Packages.elasticsearch-curator}/bin/curator --config ${curatorConfig} ${curatorAction}'';
+      };
+    };
+  };
+}

nixos/tests/elk.nix

@@ -63,6 +63,33 @@ let
                 package = elk.kibana;
                 elasticsearch.url = esUrl;
               };
+
+              elasticsearch-curator = {
+                enable = true;
+                actionYAML = ''
+                ---
+                actions:
+                  1:
+                    action: delete_indices
+                    description: >-
+                      Delete indices older than 1 second (based on index name), for logstash-
+                      prefixed indices. Ignore the error if the filter does not result in an
+                      actionable list of indices (ignore_empty_list) and exit cleanly.
+                    options:
+                      ignore_empty_list: True
+                      disable_action: False
+                    filters:
+                    - filtertype: pattern
+                      kind: prefix
+                      value: logstash-
+                    - filtertype: age
+                      source: name
+                      direction: older
+                      timestring: '%Y.%m.%d'
+                      unit: seconds
+                      unit_count: 1
+                '';
+              };
             };
           };
       };
@@ -91,6 +118,11 @@ let
       # See if logstash messages arive in elasticsearch.
       $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
       $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
+
+      # Test elasticsearch-curator.
+      $one->systemctl("stop logstash");
+      $one->systemctl("start elasticsearch-curator");
+      $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
     '';
   };
 in mapAttrs mkElkTest {

pkgs/development/python-modules/elasticsearch-curator/default.nix

@@ -1,8 +1,10 @@
 { stdenv
 , buildPythonPackage
 , fetchPypi
+, boto3
 , click
 , certifi
+, requests-aws4auth
 , voluptuous
 , pyyaml
 , elasticsearch
@@ -22,17 +24,17 @@ buildPythonPackage rec {
     sha256 = "e75abeb7f7be939b1c64c071898760dc10ab5f08307c253fc074abf8a41a76f0";
   };
 
-  # The integration tests require a running elasticsearch cluster.
-  postUnpackPhase = ''
-    rm -r test/integration
-  '';
+  # The test hangs so we disable it.
+  doCheck = false;
 
   propagatedBuildInputs = [
     click
     certifi
+    requests-aws4auth
     voluptuous
     pyyaml
     elasticsearch
+    boto3
   ];
 
   checkInputs = [

pkgs/development/python-modules/requests-aws4auth/default.nix

@@ -0,0 +1,28 @@
+{ lib, buildPythonPackage, fetchPypi, fetchzip, isPy3k, requests }:
+with lib;
+buildPythonPackage rec {
+  pname = "requests-aws4auth";
+  version = "0.9";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0g52a1pm53aqkc9qb5q1m918c1qy6q47c1qz63p5ilynfbs3m5y9";
+  };
+
+  postPatch = optionalString isPy3k ''
+    sed "s/path_encoding_style/'path_encoding_style'/" \
+      -i requests_aws4auth/service_parameters.py
+  '';
+
+  propagatedBuildInputs = [ requests ];
+
+  # The test fail on Python >= 3 because of module import errors.
+  doCheck = !isPy3k;
+
+  meta = {
+    description = "Amazon Web Services version 4 authentication for the Python Requests library.";
+    homepage = https://github.com/sam-washington/requests-aws4auth;
+    license = licenses.mit;
+    maintainers = [ maintainers.basvandijk ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -2009,6 +2009,8 @@ in {
 
   requests-unixsocket = callPackage ../development/python-modules/requests-unixsocket {};
 
+  requests-aws4auth = callPackage ../development/python-modules/requests-aws4auth { };
+
   howdoi = callPackage ../development/python-modules/howdoi {};
 
   neurotools = callPackage ../development/python-modules/neurotools {};