elk7: init at 7.0.0

This adds the following new packages:

+ elasticsearch7
+ elasticsearch7-oss
+ logstash7
+ logstash7-oss
+ kibana7
+ kibana7-oss
+ filebeat7
+ heartbeat7
+ metricbeat7
+ packetbeat7
+ journalbeat7

The default major version of the ELK stack stays at 6. We should
probably set it to 7 in a next commit.

nixos/modules/services/search/kibana.nix

@@ -5,6 +5,9 @@ with lib;
 let
   cfg = config.services.kibana;
 
+  ge7 = builtins.compareVersions cfg.package.version "7" >= 0;
+  lt6_6 = builtins.compareVersions cfg.package.version "6.6" < 0;
+
   cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON (
     (filterAttrsRecursive (n: v: v != null) ({
       server.host = cfg.listenAddress;
@@ -16,6 +19,7 @@ let
       kibana.defaultAppId = cfg.defaultAppId;
 
       elasticsearch.url = cfg.elasticsearch.url;
+      elasticsearch.hosts = cfg.elasticsearch.hosts;
       elasticsearch.username = cfg.elasticsearch.username;
       elasticsearch.password = cfg.elasticsearch.password;
 
@@ -67,9 +71,30 @@ in {
 
     elasticsearch = {
       url = mkOption {
-        description = "Elasticsearch url";
-        default = "http://localhost:9200";
-        type = types.str;
+        description = ''
+          Elasticsearch url.
+
+          Defaults to <literal>"http://localhost:9200"</literal>.
+
+          Don't set this when using Kibana >= 7.0.0 because it will result in a
+          configuration error. Use <option>services.kibana.elasticsearch.hosts</option>
+          instead.
+        '';
+        default = null;
+        type = types.nullOr types.str;
+      };
+
+      hosts = mkOption {
+        description = ''
+          The URLs of the Elasticsearch instances to use for all your queries.
+          All nodes listed here must be on the same cluster.
+
+          Defaults to <literal>[ "http://localhost:9200" ]</literal>.
+
+          This option is only valid when using kibana >= 6.6.
+        '';
+        default = null;
+        type = types.nullOr (types.listOf types.str);
       };
 
       username = mkOption {
@@ -143,6 +168,19 @@ in {
   };
 
   config = mkIf (cfg.enable) {
+    assertions = [
+      {
+        assertion = ge7 -> cfg.elasticsearch.url == null;
+        message =
+          "The option services.kibana.elasticsearch.url has been removed when using kibana >= 7.0.0. " +
+          "Please use option services.kibana.elasticsearch.hosts instead.";
+      }
+      {
+        assertion = lt6_6 -> cfg.elasticsearch.hosts == null;
+        message =
+          "The option services.kibana.elasticsearch.hosts is only valid for kibana >= 6.6.";
+      }
+    ];
     systemd.services.kibana = {
       description = "Kibana Service";
       wantedBy = [ "multi-user.target" ];

nixos/tests/elk.nix

@@ -12,7 +12,9 @@ with pkgs.lib;
 let
   esUrl = "http://localhost:9200";
 
-  mkElkTest = name : elk : makeTest {
+  mkElkTest = name : elk :
+   let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
+   in makeTest {
     inherit name;
     meta = with pkgs.stdenv.lib.maintainers; {
       maintainers = [ eelco offline basvandijk ];
@@ -69,11 +71,11 @@ let
               kibana = {
                 enable = true;
                 package = elk.kibana;
-                elasticsearch.url = esUrl;
               };
 
               elasticsearch-curator = {
-                enable = true;
+                # The current version of curator (5.6) doesn't support elasticsearch >= 7.0.0.
+                enable = !elasticsearchGe7;
                 actionYAML = ''
                 ---
                 actions:
@@ -126,7 +128,7 @@ let
       # See if logstash messages arive in elasticsearch.
       $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
       $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
-
+    '' + optionalString (!elasticsearchGe7) ''
       # Test elasticsearch-curator.
       $one->systemctl("stop logstash");
       $one->systemctl("start elasticsearch-curator");
@@ -151,4 +153,16 @@ in mapAttrs mkElkTest {
       logstash      = pkgs.logstash6-oss;
       kibana        = pkgs.kibana6-oss;
     };
+  "ELK-7" =
+    if enableUnfree
+    then {
+      elasticsearch = pkgs.elasticsearch7;
+      logstash      = pkgs.logstash7;
+      kibana        = pkgs.kibana7;
+    }
+    else {
+      elasticsearch = pkgs.elasticsearch7-oss;
+      logstash      = pkgs.logstash7-oss;
+      kibana        = pkgs.kibana7-oss;
+    };
 }

pkgs/development/tools/misc/kibana/7.x.nix

@@ -0,0 +1,64 @@
+{ elk7Version
+, enableUnfree ? true
+, stdenv
+, makeWrapper
+, fetchzip
+, fetchurl
+, nodejs-10_x
+, coreutils
+, which
+}:
+
+with stdenv.lib;
+let
+  nodejs = nodejs-10_x;
+  inherit (builtins) elemAt;
+  info = splitString "-" stdenv.hostPlatform.system;
+  arch = elemAt info 0;
+  plat = elemAt info 1;
+  shas =
+    if enableUnfree
+    then {
+      "x86_64-linux"  = "0awkbnbrffv2m6n9zqv8k4m23g4mril69rmj84ndqpdnv7shrk4l";
+      "x86_64-darwin" = "0mbaya0xq99vfl8bn1wgi4m43c8qjp910bxf26z436km64311pip";
+    }
+    else {
+      "x86_64-linux"  = "0fy8vh0x2p8n2r3n5f67h1r5z75qfx5q4fghgqwnqymray5798v4";
+      "x86_64-darwin" = "0zyhvbwvdj95v2nswx5b83n4cw3zy5a6bkrvypw6k1i833c6v5g7";
+    };
+
+in stdenv.mkDerivation rec {
+  name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}";
+  version = elk7Version;
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz";
+    sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
+  };
+
+  patches = [
+    # Kibana specifies it specifically needs nodejs 10.15.2 but nodejs in nixpkgs is at 10.15.3.
+    # The <nixpkgs/nixos/tests/elk.nix> test succeeds with this newer version so lets just
+    # disable the version check.
+    ./disable-nodejs-version-check.patch
+  ];
+
+  buildInputs = [ makeWrapper ];
+
+  installPhase = ''
+    mkdir -p $out/libexec/kibana $out/bin
+    mv * $out/libexec/kibana/
+    rm -r $out/libexec/kibana/node
+    makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
+      --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
+    sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
+  '';
+
+  meta = {
+    description = "Visualize logs and time-stamped data";
+    homepage = http://www.elasticsearch.org/overview/kibana;
+    license = if enableUnfree then licenses.elastic else licenses.asl20;
+    maintainers = with maintainers; [ offline rickynils basvandijk ];
+    platforms = with platforms; unix;
+  };
+}

pkgs/misc/logging/beats/7.x.nix

@@ -0,0 +1,49 @@
+{ stdenv, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, systemd }:
+
+let beat = package : extraArgs : buildGoPackage (rec {
+      name = "${package}-${version}";
+      version = elk7Version;
+
+      src = fetchFromGitHub {
+        owner = "elastic";
+        repo = "beats";
+        rev = "v${version}";
+        sha256 = "0n1sjngc82b7wysw5aaiqvllq4c8rx2jj7khw4vrypc40f8ahjs5";
+      };
+
+      goPackagePath = "github.com/elastic/beats";
+
+      subPackages = [ package ];
+
+      meta = with stdenv.lib; {
+        homepage = https://www.elastic.co/products/beats;
+        license = licenses.asl20;
+        maintainers = with maintainers; [ fadenb basvandijk ];
+        platforms = platforms.linux;
+      };
+    } // extraArgs);
+in {
+  filebeat7   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
+  heartbeat7  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
+  metricbeat7 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+  packetbeat7 = beat "packetbeat" {
+    buildInputs = [ libpcap ];
+    meta.description = "Network packet analyzer that ships data to Elasticsearch";
+    meta.longDescription = ''
+      Packetbeat is an open source network packet analyzer that ships the
+      data to Elasticsearch.
+
+      Think of it like a distributed real-time Wireshark with a lot more
+      analytics features. The Packetbeat shippers sniff the traffic between
+      your application processes, parse on the fly protocols like HTTP, MySQL,
+      PostgreSQL, Redis or Thrift and correlate the messages into transactions.
+    '';
+  };
+  journalbeat7  = beat "journalbeat" {
+    meta.description = ''
+      Journalbeat is an open source data collector to read and forward
+      journal entries from Linuxes with systemd.
+    '';
+    buildInputs = [ systemd.dev ];
+  };
+}

pkgs/servers/search/elasticsearch/7.x.nix

@@ -0,0 +1,83 @@
+{ elk7Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre_headless
+, utillinux
+, autoPatchelfHook
+, zlib
+}:
+
+with stdenv.lib;
+let
+  info = splitString "-" stdenv.hostPlatform.system;
+  arch = elemAt info 0;
+  plat = elemAt info 1;
+  shas =
+    if enableUnfree
+    then {
+      "x86_64-linux"  = "1fi57xqwgxx0ivjyfvaybzz2k457qw59fn9qr26d86lnkigfxpk8";
+      "x86_64-darwin" = "06hj96d4vl9q24dfx8ffydfs7qd440ys29654jgqp8sp7js7hjxp";
+    }
+    else {
+      "x86_64-linux"  = "1jrcdxm1swf8ahkv3h7kyzzhdq9nwwfhimpflzdq2d831fx525y8";
+      "x86_64-darwin" = "119ym2d5fqwba3aq2izh5qj8vxawb7hf183cgg00s1sm1mm8lviv";
+    };
+in
+stdenv.mkDerivation (rec {
+  version = elk7Version;
+  name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}";
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}-${plat}-${arch}.tar.gz";
+    sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
+  };
+
+  patches = [ ./es-home-6.x.patch ];
+
+  postPatch = ''
+    substituteInPlace bin/elasticsearch-env --replace \
+      "ES_CLASSPATH=\"\$ES_HOME/lib/*\"" \
+      "ES_CLASSPATH=\"$out/lib/*\""
+
+    substituteInPlace bin/elasticsearch-cli --replace \
+      "ES_CLASSPATH=\"\$ES_CLASSPATH:\$ES_HOME/\$additional_classpath_directory/*\"" \
+      "ES_CLASSPATH=\"\$ES_CLASSPATH:$out/\$additional_classpath_directory/*\""
+  '';
+
+  buildInputs = [ makeWrapper jre_headless utillinux ]
+             ++ optional enableUnfree zlib;
+
+  installPhase = ''
+    mkdir -p $out
+    cp -R bin config lib modules plugins $out
+
+    chmod +x $out/bin/*
+
+    wrapProgram $out/bin/elasticsearch \
+      --prefix PATH : "${utillinux}/bin/" \
+      --set JAVA_HOME "${jre_headless}"
+
+    wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
+  '';
+
+  passthru = { inherit enableUnfree; };
+
+  meta = {
+    description = "Open Source, Distributed, RESTful Search Engine";
+    license = if enableUnfree then licenses.elastic else licenses.asl20;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ apeschar basvandijk ];
+  };
+} // optionalAttrs enableUnfree {
+  dontPatchELF = true;
+  nativeBuildInputs = [ autoPatchelfHook ];
+  runtimeDependencies = [ zlib ];
+  postFixup = ''
+    for exe in $(find $out/modules/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
+      echo "patching $exe..."
+      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
+    done
+  '';
+})

pkgs/tools/misc/logstash/7.x.nix

@@ -0,0 +1,53 @@
+{ elk7Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre
+}:
+
+with stdenv.lib;
+
+stdenv.mkDerivation rec {
+  version = elk7Version;
+  name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}";
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
+    sha256 =
+      if enableUnfree
+      then "1dzk9amg23vbkj92wx529i1m2rrfjc0gzkbg6w3f6xaix5zch0vp"
+      else "07q2gbd3xidcklid5ny22zpb3q5v0z3k2kva6g214k7rnkr9bhyy";
+  };
+
+  dontBuild         = true;
+  dontPatchELF      = true;
+  dontStrip         = true;
+  dontPatchShebangs = true;
+
+  buildInputs = [
+    makeWrapper jre
+  ];
+
+  installPhase = ''
+    mkdir -p $out
+    cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
+
+    patchShebangs $out/bin/logstash
+    patchShebangs $out/bin/logstash-plugin
+
+    wrapProgram $out/bin/logstash \
+       --set JAVA_HOME "${jre}"
+
+    wrapProgram $out/bin/logstash-plugin \
+       --set JAVA_HOME "${jre}"
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
+    homepage    = https://www.elastic.co/products/logstash;
+    license     = if enableUnfree then licenses.elastic else licenses.asl20;
+    platforms   = platforms.unix;
+    maintainers = with maintainers; [ wjlroe offline basvandijk ];
+  };
+}

pkgs/top-level/all-packages.nix

@@ -948,6 +948,16 @@ in
 
   bchunk = callPackage ../tools/cd-dvd/bchunk { };
 
+  inherit (callPackages ../misc/logging/beats/5.x.nix {
+    # XXX: this is failing with Go 1.12. Error is related to cgo, an
+    # update to this package might fix it.
+    buildGoPackage = buildGo111Package;
+  })
+    filebeat5
+    heartbeat5
+    metricbeat5
+    packetbeat5;
+
   inherit (callPackages ../misc/logging/beats/6.x.nix {
     # XXX: this is failing with Go 1.12. Error is related to cgo, an
     # update to this package might fix it.
@@ -959,20 +969,21 @@ in
     packetbeat6
     journalbeat6;
 
-  filebeat = filebeat6;
-  heartbeat = heartbeat6;
-  metricbeat = metricbeat6;
-  packetbeat = packetbeat6;
-
-  inherit (callPackages ../misc/logging/beats/5.x.nix {
+  inherit (callPackages ../misc/logging/beats/7.x.nix {
     # XXX: this is failing with Go 1.12. Error is related to cgo, an
     # update to this package might fix it.
     buildGoPackage = buildGo111Package;
   })
-    filebeat5
-    heartbeat5
-    metricbeat5
-    packetbeat5;
+    filebeat7
+    heartbeat7
+    metricbeat7
+    packetbeat7
+    journalbeat7;
+
+  filebeat = filebeat6;
+  heartbeat = heartbeat6;
+  metricbeat = metricbeat6;
+  packetbeat = packetbeat6;
 
   bfr = callPackage ../tools/misc/bfr { };
 
@@ -2572,10 +2583,15 @@ in
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   elk5Version = "5.6.9";
   elk6Version = "6.7.1";
+  elk7Version = "7.0.0";
 
   elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
-  elasticsearch6 = callPackage ../servers/search/elasticsearch { };
-  elasticsearch6-oss = callPackage ../servers/search/elasticsearch {
+  elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
+  elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix {
+    enableUnfree = false;
+  };
+  elasticsearch7 = callPackage ../servers/search/elasticsearch/7.x.nix { };
+  elasticsearch7-oss = callPackage ../servers/search/elasticsearch/7.x.nix {
     enableUnfree = false;
   };
   elasticsearch = elasticsearch6;
@@ -3778,8 +3794,12 @@ in
   keyfuzz = callPackage ../tools/inputmethods/keyfuzz { };
 
   kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
-  kibana6 = callPackage ../development/tools/misc/kibana/default.nix { };
-  kibana6-oss = callPackage ../development/tools/misc/kibana/default.nix {
+  kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
+  kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix {
+    enableUnfree = false;
+  };
+  kibana7 = callPackage ../development/tools/misc/kibana/7.x.nix { };
+  kibana7-oss = callPackage ../development/tools/misc/kibana/7.x.nix {
     enableUnfree = false;
   };
   kibana = kibana6;
@@ -3868,8 +3888,12 @@ in
   lockfileProgs = callPackage ../tools/misc/lockfile-progs { };
 
   logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
-  logstash6 = callPackage ../tools/misc/logstash { };
-  logstash6-oss = callPackage ../tools/misc/logstash {
+  logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
+  logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix {
+    enableUnfree = false;
+  };
+  logstash7 = callPackage ../tools/misc/logstash/7.x.nix { };
+  logstash7-oss = callPackage ../tools/misc/logstash/7.x.nix {
     enableUnfree = false;
   };
   logstash = logstash6;