diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix
index 3539b3ddb4f..ba58630a467 100644
--- a/nixos/modules/services/search/kibana.nix
+++ b/nixos/modules/services/search/kibana.nix
@@ -5,6 +5,9 @@ with lib;
let
cfg = config.services.kibana;
+ ge7 = builtins.compareVersions cfg.package.version "7" >= 0;
+ lt6_6 = builtins.compareVersions cfg.package.version "6.6" < 0;
+
cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON (
(filterAttrsRecursive (n: v: v != null) ({
server.host = cfg.listenAddress;
@@ -16,6 +19,7 @@ let
kibana.defaultAppId = cfg.defaultAppId;
elasticsearch.url = cfg.elasticsearch.url;
+ elasticsearch.hosts = cfg.elasticsearch.hosts;
elasticsearch.username = cfg.elasticsearch.username;
elasticsearch.password = cfg.elasticsearch.password;
@@ -67,9 +71,30 @@ in {
elasticsearch = {
url = mkOption {
- description = "Elasticsearch url";
- default = "http://localhost:9200";
- type = types.str;
+ description = ''
+ Elasticsearch url.
+
+ Defaults to "http://localhost:9200".
+
+ Don't set this when using Kibana >= 7.0.0 because it will result in a
+ configuration error. Use
+ instead.
+ '';
+ default = null;
+ type = types.nullOr types.str;
+ };
+
+ hosts = mkOption {
+ description = ''
+ The URLs of the Elasticsearch instances to use for all your queries.
+ All nodes listed here must be on the same cluster.
+
+ Defaults to [ "http://localhost:9200" ].
+
+ This option is only valid when using kibana >= 6.6.
+ '';
+ default = null;
+ type = types.nullOr (types.listOf types.str);
};
username = mkOption {
@@ -143,6 +168,19 @@ in {
};
config = mkIf (cfg.enable) {
+ assertions = [
+ {
+ assertion = ge7 -> cfg.elasticsearch.url == null;
+ message =
+ "The option services.kibana.elasticsearch.url has been removed when using kibana >= 7.0.0. " +
+ "Please use option services.kibana.elasticsearch.hosts instead.";
+ }
+ {
+ assertion = lt6_6 -> cfg.elasticsearch.hosts == null;
+ message =
+ "The option services.kibana.elasticsearch.hosts is only valid for kibana >= 6.6.";
+ }
+ ];
systemd.services.kibana = {
description = "Kibana Service";
wantedBy = [ "multi-user.target" ];
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index a82e75799ae..3b3fbd73dd5 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -12,7 +12,9 @@ with pkgs.lib;
let
esUrl = "http://localhost:9200";
- mkElkTest = name : elk : makeTest {
+ mkElkTest = name : elk :
+ let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
+ in makeTest {
inherit name;
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ eelco offline basvandijk ];
@@ -69,11 +71,11 @@ let
kibana = {
enable = true;
package = elk.kibana;
- elasticsearch.url = esUrl;
};
elasticsearch-curator = {
- enable = true;
+ # The current version of curator (5.6) doesn't support elasticsearch >= 7.0.0.
+ enable = !elasticsearchGe7;
actionYAML = ''
---
actions:
@@ -126,7 +128,7 @@ let
# See if logstash messages arive in elasticsearch.
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
-
+ '' + optionalString (!elasticsearchGe7) ''
# Test elasticsearch-curator.
$one->systemctl("stop logstash");
$one->systemctl("start elasticsearch-curator");
@@ -151,4 +153,16 @@ in mapAttrs mkElkTest {
logstash = pkgs.logstash6-oss;
kibana = pkgs.kibana6-oss;
};
+ "ELK-7" =
+ if enableUnfree
+ then {
+ elasticsearch = pkgs.elasticsearch7;
+ logstash = pkgs.logstash7;
+ kibana = pkgs.kibana7;
+ }
+ else {
+ elasticsearch = pkgs.elasticsearch7-oss;
+ logstash = pkgs.logstash7-oss;
+ kibana = pkgs.kibana7-oss;
+ };
}
diff --git a/pkgs/development/tools/misc/kibana/default.nix b/pkgs/development/tools/misc/kibana/6.x.nix
similarity index 100%
rename from pkgs/development/tools/misc/kibana/default.nix
rename to pkgs/development/tools/misc/kibana/6.x.nix
diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix
new file mode 100644
index 00000000000..19590239caf
--- /dev/null
+++ b/pkgs/development/tools/misc/kibana/7.x.nix
@@ -0,0 +1,64 @@
+{ elk7Version
+, enableUnfree ? true
+, stdenv
+, makeWrapper
+, fetchzip
+, fetchurl
+, nodejs-10_x
+, coreutils
+, which
+}:
+
+with stdenv.lib;
+let
+ nodejs = nodejs-10_x;
+ inherit (builtins) elemAt;
+ info = splitString "-" stdenv.hostPlatform.system;
+ arch = elemAt info 0;
+ plat = elemAt info 1;
+ shas =
+ if enableUnfree
+ then {
+ "x86_64-linux" = "0awkbnbrffv2m6n9zqv8k4m23g4mril69rmj84ndqpdnv7shrk4l";
+ "x86_64-darwin" = "0mbaya0xq99vfl8bn1wgi4m43c8qjp910bxf26z436km64311pip";
+ }
+ else {
+ "x86_64-linux" = "0fy8vh0x2p8n2r3n5f67h1r5z75qfx5q4fghgqwnqymray5798v4";
+ "x86_64-darwin" = "0zyhvbwvdj95v2nswx5b83n4cw3zy5a6bkrvypw6k1i833c6v5g7";
+ };
+
+in stdenv.mkDerivation rec {
+ name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}";
+ version = elk7Version;
+
+ src = fetchurl {
+ url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz";
+ sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
+ };
+
+ patches = [
+ # Kibana specifies it specifically needs nodejs 10.15.2 but nodejs in nixpkgs is at 10.15.3.
+ # The test succeeds with this newer version so lets just
+ # disable the version check.
+ ./disable-nodejs-version-check.patch
+ ];
+
+ buildInputs = [ makeWrapper ];
+
+ installPhase = ''
+ mkdir -p $out/libexec/kibana $out/bin
+ mv * $out/libexec/kibana/
+ rm -r $out/libexec/kibana/node
+ makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
+ --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
+ sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
+ '';
+
+ meta = {
+ description = "Visualize logs and time-stamped data";
+ homepage = http://www.elasticsearch.org/overview/kibana;
+ license = if enableUnfree then licenses.elastic else licenses.asl20;
+ maintainers = with maintainers; [ offline rickynils basvandijk ];
+ platforms = with platforms; unix;
+ };
+}
diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix
new file mode 100644
index 00000000000..6d00b02d80e
--- /dev/null
+++ b/pkgs/misc/logging/beats/7.x.nix
@@ -0,0 +1,49 @@
+{ stdenv, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, systemd }:
+
+let beat = package : extraArgs : buildGoPackage (rec {
+ name = "${package}-${version}";
+ version = elk7Version;
+
+ src = fetchFromGitHub {
+ owner = "elastic";
+ repo = "beats";
+ rev = "v${version}";
+ sha256 = "0n1sjngc82b7wysw5aaiqvllq4c8rx2jj7khw4vrypc40f8ahjs5";
+ };
+
+ goPackagePath = "github.com/elastic/beats";
+
+ subPackages = [ package ];
+
+ meta = with stdenv.lib; {
+ homepage = https://www.elastic.co/products/beats;
+ license = licenses.asl20;
+ maintainers = with maintainers; [ fadenb basvandijk ];
+ platforms = platforms.linux;
+ };
+ } // extraArgs);
+in {
+ filebeat7 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";};
+ heartbeat7 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";};
+ metricbeat7 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+ packetbeat7 = beat "packetbeat" {
+ buildInputs = [ libpcap ];
+ meta.description = "Network packet analyzer that ships data to Elasticsearch";
+ meta.longDescription = ''
+ Packetbeat is an open source network packet analyzer that ships the
+ data to Elasticsearch.
+
+ Think of it like a distributed real-time Wireshark with a lot more
+ analytics features. The Packetbeat shippers sniff the traffic between
+ your application processes, parse on the fly protocols like HTTP, MySQL,
+ PostgreSQL, Redis or Thrift and correlate the messages into transactions.
+ '';
+ };
+ journalbeat7 = beat "journalbeat" {
+ meta.description = ''
+ Journalbeat is an open source data collector to read and forward
+ journal entries from Linuxes with systemd.
+ '';
+ buildInputs = [ systemd.dev ];
+ };
+}
diff --git a/pkgs/servers/search/elasticsearch/default.nix b/pkgs/servers/search/elasticsearch/6.x.nix
similarity index 100%
rename from pkgs/servers/search/elasticsearch/default.nix
rename to pkgs/servers/search/elasticsearch/6.x.nix
diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix
new file mode 100644
index 00000000000..7ea33935faa
--- /dev/null
+++ b/pkgs/servers/search/elasticsearch/7.x.nix
@@ -0,0 +1,83 @@
+{ elk7Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre_headless
+, utillinux
+, autoPatchelfHook
+, zlib
+}:
+
+with stdenv.lib;
+let
+ info = splitString "-" stdenv.hostPlatform.system;
+ arch = elemAt info 0;
+ plat = elemAt info 1;
+ shas =
+ if enableUnfree
+ then {
+ "x86_64-linux" = "1fi57xqwgxx0ivjyfvaybzz2k457qw59fn9qr26d86lnkigfxpk8";
+ "x86_64-darwin" = "06hj96d4vl9q24dfx8ffydfs7qd440ys29654jgqp8sp7js7hjxp";
+ }
+ else {
+ "x86_64-linux" = "1jrcdxm1swf8ahkv3h7kyzzhdq9nwwfhimpflzdq2d831fx525y8";
+ "x86_64-darwin" = "119ym2d5fqwba3aq2izh5qj8vxawb7hf183cgg00s1sm1mm8lviv";
+ };
+in
+stdenv.mkDerivation (rec {
+ version = elk7Version;
+ name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}";
+
+ src = fetchurl {
+ url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}-${plat}-${arch}.tar.gz";
+ sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
+ };
+
+ patches = [ ./es-home-6.x.patch ];
+
+ postPatch = ''
+ substituteInPlace bin/elasticsearch-env --replace \
+ "ES_CLASSPATH=\"\$ES_HOME/lib/*\"" \
+ "ES_CLASSPATH=\"$out/lib/*\""
+
+ substituteInPlace bin/elasticsearch-cli --replace \
+ "ES_CLASSPATH=\"\$ES_CLASSPATH:\$ES_HOME/\$additional_classpath_directory/*\"" \
+ "ES_CLASSPATH=\"\$ES_CLASSPATH:$out/\$additional_classpath_directory/*\""
+ '';
+
+ buildInputs = [ makeWrapper jre_headless utillinux ]
+ ++ optional enableUnfree zlib;
+
+ installPhase = ''
+ mkdir -p $out
+ cp -R bin config lib modules plugins $out
+
+ chmod +x $out/bin/*
+
+ wrapProgram $out/bin/elasticsearch \
+ --prefix PATH : "${utillinux}/bin/" \
+ --set JAVA_HOME "${jre_headless}"
+
+ wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
+ '';
+
+ passthru = { inherit enableUnfree; };
+
+ meta = {
+ description = "Open Source, Distributed, RESTful Search Engine";
+ license = if enableUnfree then licenses.elastic else licenses.asl20;
+ platforms = platforms.unix;
+ maintainers = with maintainers; [ apeschar basvandijk ];
+ };
+} // optionalAttrs enableUnfree {
+ dontPatchELF = true;
+ nativeBuildInputs = [ autoPatchelfHook ];
+ runtimeDependencies = [ zlib ];
+ postFixup = ''
+ for exe in $(find $out/modules/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
+ echo "patching $exe..."
+ patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
+ done
+ '';
+})
diff --git a/pkgs/tools/misc/logstash/default.nix b/pkgs/tools/misc/logstash/6.x.nix
similarity index 100%
rename from pkgs/tools/misc/logstash/default.nix
rename to pkgs/tools/misc/logstash/6.x.nix
diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix
new file mode 100644
index 00000000000..bb4a6b1436b
--- /dev/null
+++ b/pkgs/tools/misc/logstash/7.x.nix
@@ -0,0 +1,53 @@
+{ elk7Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre
+}:
+
+with stdenv.lib;
+
+stdenv.mkDerivation rec {
+ version = elk7Version;
+ name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}";
+
+ src = fetchurl {
+ url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
+ sha256 =
+ if enableUnfree
+ then "1dzk9amg23vbkj92wx529i1m2rrfjc0gzkbg6w3f6xaix5zch0vp"
+ else "07q2gbd3xidcklid5ny22zpb3q5v0z3k2kva6g214k7rnkr9bhyy";
+ };
+
+ dontBuild = true;
+ dontPatchELF = true;
+ dontStrip = true;
+ dontPatchShebangs = true;
+
+ buildInputs = [
+ makeWrapper jre
+ ];
+
+ installPhase = ''
+ mkdir -p $out
+ cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
+
+ patchShebangs $out/bin/logstash
+ patchShebangs $out/bin/logstash-plugin
+
+ wrapProgram $out/bin/logstash \
+ --set JAVA_HOME "${jre}"
+
+ wrapProgram $out/bin/logstash-plugin \
+ --set JAVA_HOME "${jre}"
+ '';
+
+ meta = with stdenv.lib; {
+ description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
+ homepage = https://www.elastic.co/products/logstash;
+ license = if enableUnfree then licenses.elastic else licenses.asl20;
+ platforms = platforms.unix;
+ maintainers = with maintainers; [ wjlroe offline basvandijk ];
+ };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 34db5cf8982..c99e06da534 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -948,6 +948,16 @@ in
bchunk = callPackage ../tools/cd-dvd/bchunk { };
+ inherit (callPackages ../misc/logging/beats/5.x.nix {
+ # XXX: this is failing with Go 1.12. Error is related to cgo, an
+ # update to this package might fix it.
+ buildGoPackage = buildGo111Package;
+ })
+ filebeat5
+ heartbeat5
+ metricbeat5
+ packetbeat5;
+
inherit (callPackages ../misc/logging/beats/6.x.nix {
# XXX: this is failing with Go 1.12. Error is related to cgo, an
# update to this package might fix it.
@@ -959,20 +969,21 @@ in
packetbeat6
journalbeat6;
- filebeat = filebeat6;
- heartbeat = heartbeat6;
- metricbeat = metricbeat6;
- packetbeat = packetbeat6;
-
- inherit (callPackages ../misc/logging/beats/5.x.nix {
+ inherit (callPackages ../misc/logging/beats/7.x.nix {
# XXX: this is failing with Go 1.12. Error is related to cgo, an
# update to this package might fix it.
buildGoPackage = buildGo111Package;
})
- filebeat5
- heartbeat5
- metricbeat5
- packetbeat5;
+ filebeat7
+ heartbeat7
+ metricbeat7
+ packetbeat7
+ journalbeat7;
+
+ filebeat = filebeat6;
+ heartbeat = heartbeat6;
+ metricbeat = metricbeat6;
+ packetbeat = packetbeat6;
bfr = callPackage ../tools/misc/bfr { };
@@ -2572,10 +2583,15 @@ in
# The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
elk5Version = "5.6.9";
elk6Version = "6.7.1";
+ elk7Version = "7.0.0";
elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
- elasticsearch6 = callPackage ../servers/search/elasticsearch { };
- elasticsearch6-oss = callPackage ../servers/search/elasticsearch {
+ elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
+ elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix {
+ enableUnfree = false;
+ };
+ elasticsearch7 = callPackage ../servers/search/elasticsearch/7.x.nix { };
+ elasticsearch7-oss = callPackage ../servers/search/elasticsearch/7.x.nix {
enableUnfree = false;
};
elasticsearch = elasticsearch6;
@@ -3778,8 +3794,12 @@ in
keyfuzz = callPackage ../tools/inputmethods/keyfuzz { };
kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
- kibana6 = callPackage ../development/tools/misc/kibana/default.nix { };
- kibana6-oss = callPackage ../development/tools/misc/kibana/default.nix {
+ kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
+ kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix {
+ enableUnfree = false;
+ };
+ kibana7 = callPackage ../development/tools/misc/kibana/7.x.nix { };
+ kibana7-oss = callPackage ../development/tools/misc/kibana/7.x.nix {
enableUnfree = false;
};
kibana = kibana6;
@@ -3868,8 +3888,12 @@ in
lockfileProgs = callPackage ../tools/misc/lockfile-progs { };
logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
- logstash6 = callPackage ../tools/misc/logstash { };
- logstash6-oss = callPackage ../tools/misc/logstash {
+ logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
+ logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix {
+ enableUnfree = false;
+ };
+ logstash7 = callPackage ../tools/misc/logstash/7.x.nix { };
+ logstash7-oss = callPackage ../tools/misc/logstash/7.x.nix {
enableUnfree = false;
};
logstash = logstash6;