elk7: init at 7.0.0

This adds the following new packages:

+ elasticsearch7
+ elasticsearch7-oss
+ logstash7
+ logstash7-oss
+ kibana7
+ kibana7-oss
+ filebeat7
+ heartbeat7
+ metricbeat7
+ packetbeat7
+ journalbeat7

The default major version of the ELK stack stays at 6. We should
probably set it to 7 in a next commit.
This commit is contained in:
Bas van Dijk 2019-04-14 21:39:46 +02:00
parent 917a7fa1cb
commit 13352f28d2
10 changed files with 348 additions and 23 deletions

View file

@ -5,6 +5,9 @@ with lib;
let let
cfg = config.services.kibana; cfg = config.services.kibana;
ge7 = builtins.compareVersions cfg.package.version "7" >= 0;
lt6_6 = builtins.compareVersions cfg.package.version "6.6" < 0;
cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON ( cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON (
(filterAttrsRecursive (n: v: v != null) ({ (filterAttrsRecursive (n: v: v != null) ({
server.host = cfg.listenAddress; server.host = cfg.listenAddress;
@ -16,6 +19,7 @@ let
kibana.defaultAppId = cfg.defaultAppId; kibana.defaultAppId = cfg.defaultAppId;
elasticsearch.url = cfg.elasticsearch.url; elasticsearch.url = cfg.elasticsearch.url;
elasticsearch.hosts = cfg.elasticsearch.hosts;
elasticsearch.username = cfg.elasticsearch.username; elasticsearch.username = cfg.elasticsearch.username;
elasticsearch.password = cfg.elasticsearch.password; elasticsearch.password = cfg.elasticsearch.password;
@ -67,9 +71,30 @@ in {
elasticsearch = { elasticsearch = {
url = mkOption { url = mkOption {
description = "Elasticsearch url"; description = ''
default = "http://localhost:9200"; Elasticsearch url.
type = types.str;
Defaults to <literal>"http://localhost:9200"</literal>.
Don't set this when using Kibana >= 7.0.0 because it will result in a
configuration error. Use <option>services.kibana.elasticsearch.hosts</option>
instead.
'';
default = null;
type = types.nullOr types.str;
};
hosts = mkOption {
description = ''
The URLs of the Elasticsearch instances to use for all your queries.
All nodes listed here must be on the same cluster.
Defaults to <literal>[ "http://localhost:9200" ]</literal>.
This option is only valid when using kibana >= 6.6.
'';
default = null;
type = types.nullOr (types.listOf types.str);
}; };
username = mkOption { username = mkOption {
@ -143,6 +168,19 @@ in {
}; };
config = mkIf (cfg.enable) { config = mkIf (cfg.enable) {
assertions = [
{
assertion = ge7 -> cfg.elasticsearch.url == null;
message =
"The option services.kibana.elasticsearch.url has been removed when using kibana >= 7.0.0. " +
"Please use option services.kibana.elasticsearch.hosts instead.";
}
{
assertion = lt6_6 -> cfg.elasticsearch.hosts == null;
message =
"The option services.kibana.elasticsearch.hosts is only valid for kibana >= 6.6.";
}
];
systemd.services.kibana = { systemd.services.kibana = {
description = "Kibana Service"; description = "Kibana Service";
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];

View file

@ -12,7 +12,9 @@ with pkgs.lib;
let let
esUrl = "http://localhost:9200"; esUrl = "http://localhost:9200";
mkElkTest = name : elk : makeTest { mkElkTest = name : elk :
let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
in makeTest {
inherit name; inherit name;
meta = with pkgs.stdenv.lib.maintainers; { meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ eelco offline basvandijk ]; maintainers = [ eelco offline basvandijk ];
@ -69,11 +71,11 @@ let
kibana = { kibana = {
enable = true; enable = true;
package = elk.kibana; package = elk.kibana;
elasticsearch.url = esUrl;
}; };
elasticsearch-curator = { elasticsearch-curator = {
enable = true; # The current version of curator (5.6) doesn't support elasticsearch >= 7.0.0.
enable = !elasticsearchGe7;
actionYAML = '' actionYAML = ''
--- ---
actions: actions:
@ -126,7 +128,7 @@ let
# See if logstash messages arive in elasticsearch. # See if logstash messages arive in elasticsearch.
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
'' + optionalString (!elasticsearchGe7) ''
# Test elasticsearch-curator. # Test elasticsearch-curator.
$one->systemctl("stop logstash"); $one->systemctl("stop logstash");
$one->systemctl("start elasticsearch-curator"); $one->systemctl("start elasticsearch-curator");
@ -151,4 +153,16 @@ in mapAttrs mkElkTest {
logstash = pkgs.logstash6-oss; logstash = pkgs.logstash6-oss;
kibana = pkgs.kibana6-oss; kibana = pkgs.kibana6-oss;
}; };
"ELK-7" =
if enableUnfree
then {
elasticsearch = pkgs.elasticsearch7;
logstash = pkgs.logstash7;
kibana = pkgs.kibana7;
}
else {
elasticsearch = pkgs.elasticsearch7-oss;
logstash = pkgs.logstash7-oss;
kibana = pkgs.kibana7-oss;
};
} }

View file

@ -0,0 +1,64 @@
{ elk7Version
, enableUnfree ? true
, stdenv
, makeWrapper
, fetchzip
, fetchurl
, nodejs-10_x
, coreutils
, which
}:
with stdenv.lib;
let
nodejs = nodejs-10_x;
inherit (builtins) elemAt;
info = splitString "-" stdenv.hostPlatform.system;
arch = elemAt info 0;
plat = elemAt info 1;
shas =
if enableUnfree
then {
"x86_64-linux" = "0awkbnbrffv2m6n9zqv8k4m23g4mril69rmj84ndqpdnv7shrk4l";
"x86_64-darwin" = "0mbaya0xq99vfl8bn1wgi4m43c8qjp910bxf26z436km64311pip";
}
else {
"x86_64-linux" = "0fy8vh0x2p8n2r3n5f67h1r5z75qfx5q4fghgqwnqymray5798v4";
"x86_64-darwin" = "0zyhvbwvdj95v2nswx5b83n4cw3zy5a6bkrvypw6k1i833c6v5g7";
};
in stdenv.mkDerivation rec {
name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}";
version = elk7Version;
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz";
sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
};
patches = [
# Kibana specifies it specifically needs nodejs 10.15.2 but nodejs in nixpkgs is at 10.15.3.
# The <nixpkgs/nixos/tests/elk.nix> test succeeds with this newer version so lets just
# disable the version check.
./disable-nodejs-version-check.patch
];
buildInputs = [ makeWrapper ];
installPhase = ''
mkdir -p $out/libexec/kibana $out/bin
mv * $out/libexec/kibana/
rm -r $out/libexec/kibana/node
makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
--prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
'';
meta = {
description = "Visualize logs and time-stamped data";
homepage = http://www.elasticsearch.org/overview/kibana;
license = if enableUnfree then licenses.elastic else licenses.asl20;
maintainers = with maintainers; [ offline rickynils basvandijk ];
platforms = with platforms; unix;
};
}

View file

@ -0,0 +1,49 @@
{ stdenv, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, systemd }:
let beat = package : extraArgs : buildGoPackage (rec {
name = "${package}-${version}";
version = elk7Version;
src = fetchFromGitHub {
owner = "elastic";
repo = "beats";
rev = "v${version}";
sha256 = "0n1sjngc82b7wysw5aaiqvllq4c8rx2jj7khw4vrypc40f8ahjs5";
};
goPackagePath = "github.com/elastic/beats";
subPackages = [ package ];
meta = with stdenv.lib; {
homepage = https://www.elastic.co/products/beats;
license = licenses.asl20;
maintainers = with maintainers; [ fadenb basvandijk ];
platforms = platforms.linux;
};
} // extraArgs);
in {
filebeat7 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";};
heartbeat7 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";};
metricbeat7 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
packetbeat7 = beat "packetbeat" {
buildInputs = [ libpcap ];
meta.description = "Network packet analyzer that ships data to Elasticsearch";
meta.longDescription = ''
Packetbeat is an open source network packet analyzer that ships the
data to Elasticsearch.
Think of it like a distributed real-time Wireshark with a lot more
analytics features. The Packetbeat shippers sniff the traffic between
your application processes, parse on the fly protocols like HTTP, MySQL,
PostgreSQL, Redis or Thrift and correlate the messages into transactions.
'';
};
journalbeat7 = beat "journalbeat" {
meta.description = ''
Journalbeat is an open source data collector to read and forward
journal entries from Linuxes with systemd.
'';
buildInputs = [ systemd.dev ];
};
}

View file

@ -0,0 +1,83 @@
{ elk7Version
, enableUnfree ? true
, stdenv
, fetchurl
, makeWrapper
, jre_headless
, utillinux
, autoPatchelfHook
, zlib
}:
with stdenv.lib;
let
info = splitString "-" stdenv.hostPlatform.system;
arch = elemAt info 0;
plat = elemAt info 1;
shas =
if enableUnfree
then {
"x86_64-linux" = "1fi57xqwgxx0ivjyfvaybzz2k457qw59fn9qr26d86lnkigfxpk8";
"x86_64-darwin" = "06hj96d4vl9q24dfx8ffydfs7qd440ys29654jgqp8sp7js7hjxp";
}
else {
"x86_64-linux" = "1jrcdxm1swf8ahkv3h7kyzzhdq9nwwfhimpflzdq2d831fx525y8";
"x86_64-darwin" = "119ym2d5fqwba3aq2izh5qj8vxawb7hf183cgg00s1sm1mm8lviv";
};
in
stdenv.mkDerivation (rec {
version = elk7Version;
name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}";
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}-${plat}-${arch}.tar.gz";
sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
};
patches = [ ./es-home-6.x.patch ];
postPatch = ''
substituteInPlace bin/elasticsearch-env --replace \
"ES_CLASSPATH=\"\$ES_HOME/lib/*\"" \
"ES_CLASSPATH=\"$out/lib/*\""
substituteInPlace bin/elasticsearch-cli --replace \
"ES_CLASSPATH=\"\$ES_CLASSPATH:\$ES_HOME/\$additional_classpath_directory/*\"" \
"ES_CLASSPATH=\"\$ES_CLASSPATH:$out/\$additional_classpath_directory/*\""
'';
buildInputs = [ makeWrapper jre_headless utillinux ]
++ optional enableUnfree zlib;
installPhase = ''
mkdir -p $out
cp -R bin config lib modules plugins $out
chmod +x $out/bin/*
wrapProgram $out/bin/elasticsearch \
--prefix PATH : "${utillinux}/bin/" \
--set JAVA_HOME "${jre_headless}"
wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
'';
passthru = { inherit enableUnfree; };
meta = {
description = "Open Source, Distributed, RESTful Search Engine";
license = if enableUnfree then licenses.elastic else licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [ apeschar basvandijk ];
};
} // optionalAttrs enableUnfree {
dontPatchELF = true;
nativeBuildInputs = [ autoPatchelfHook ];
runtimeDependencies = [ zlib ];
postFixup = ''
for exe in $(find $out/modules/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
echo "patching $exe..."
patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
done
'';
})

View file

@ -0,0 +1,53 @@
{ elk7Version
, enableUnfree ? true
, stdenv
, fetchurl
, makeWrapper
, jre
}:
with stdenv.lib;
stdenv.mkDerivation rec {
version = elk7Version;
name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}";
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
sha256 =
if enableUnfree
then "1dzk9amg23vbkj92wx529i1m2rrfjc0gzkbg6w3f6xaix5zch0vp"
else "07q2gbd3xidcklid5ny22zpb3q5v0z3k2kva6g214k7rnkr9bhyy";
};
dontBuild = true;
dontPatchELF = true;
dontStrip = true;
dontPatchShebangs = true;
buildInputs = [
makeWrapper jre
];
installPhase = ''
mkdir -p $out
cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
patchShebangs $out/bin/logstash
patchShebangs $out/bin/logstash-plugin
wrapProgram $out/bin/logstash \
--set JAVA_HOME "${jre}"
wrapProgram $out/bin/logstash-plugin \
--set JAVA_HOME "${jre}"
'';
meta = with stdenv.lib; {
description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
homepage = https://www.elastic.co/products/logstash;
license = if enableUnfree then licenses.elastic else licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [ wjlroe offline basvandijk ];
};
}

View file

@ -948,6 +948,16 @@ in
bchunk = callPackage ../tools/cd-dvd/bchunk { }; bchunk = callPackage ../tools/cd-dvd/bchunk { };
inherit (callPackages ../misc/logging/beats/5.x.nix {
# XXX: this is failing with Go 1.12. Error is related to cgo, an
# update to this package might fix it.
buildGoPackage = buildGo111Package;
})
filebeat5
heartbeat5
metricbeat5
packetbeat5;
inherit (callPackages ../misc/logging/beats/6.x.nix { inherit (callPackages ../misc/logging/beats/6.x.nix {
# XXX: this is failing with Go 1.12. Error is related to cgo, an # XXX: this is failing with Go 1.12. Error is related to cgo, an
# update to this package might fix it. # update to this package might fix it.
@ -959,20 +969,21 @@ in
packetbeat6 packetbeat6
journalbeat6; journalbeat6;
filebeat = filebeat6; inherit (callPackages ../misc/logging/beats/7.x.nix {
heartbeat = heartbeat6;
metricbeat = metricbeat6;
packetbeat = packetbeat6;
inherit (callPackages ../misc/logging/beats/5.x.nix {
# XXX: this is failing with Go 1.12. Error is related to cgo, an # XXX: this is failing with Go 1.12. Error is related to cgo, an
# update to this package might fix it. # update to this package might fix it.
buildGoPackage = buildGo111Package; buildGoPackage = buildGo111Package;
}) })
filebeat5 filebeat7
heartbeat5 heartbeat7
metricbeat5 metricbeat7
packetbeat5; packetbeat7
journalbeat7;
filebeat = filebeat6;
heartbeat = heartbeat6;
metricbeat = metricbeat6;
packetbeat = packetbeat6;
bfr = callPackage ../tools/misc/bfr { }; bfr = callPackage ../tools/misc/bfr { };
@ -2572,10 +2583,15 @@ in
# The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
elk5Version = "5.6.9"; elk5Version = "5.6.9";
elk6Version = "6.7.1"; elk6Version = "6.7.1";
elk7Version = "7.0.0";
elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { }; elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
elasticsearch6 = callPackage ../servers/search/elasticsearch { }; elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
elasticsearch6-oss = callPackage ../servers/search/elasticsearch { elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix {
enableUnfree = false;
};
elasticsearch7 = callPackage ../servers/search/elasticsearch/7.x.nix { };
elasticsearch7-oss = callPackage ../servers/search/elasticsearch/7.x.nix {
enableUnfree = false; enableUnfree = false;
}; };
elasticsearch = elasticsearch6; elasticsearch = elasticsearch6;
@ -3778,8 +3794,12 @@ in
keyfuzz = callPackage ../tools/inputmethods/keyfuzz { }; keyfuzz = callPackage ../tools/inputmethods/keyfuzz { };
kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { }; kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
kibana6 = callPackage ../development/tools/misc/kibana/default.nix { }; kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
kibana6-oss = callPackage ../development/tools/misc/kibana/default.nix { kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix {
enableUnfree = false;
};
kibana7 = callPackage ../development/tools/misc/kibana/7.x.nix { };
kibana7-oss = callPackage ../development/tools/misc/kibana/7.x.nix {
enableUnfree = false; enableUnfree = false;
}; };
kibana = kibana6; kibana = kibana6;
@ -3868,8 +3888,12 @@ in
lockfileProgs = callPackage ../tools/misc/lockfile-progs { }; lockfileProgs = callPackage ../tools/misc/lockfile-progs { };
logstash5 = callPackage ../tools/misc/logstash/5.x.nix { }; logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
logstash6 = callPackage ../tools/misc/logstash { }; logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
logstash6-oss = callPackage ../tools/misc/logstash { logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix {
enableUnfree = false;
};
logstash7 = callPackage ../tools/misc/logstash/7.x.nix { };
logstash7-oss = callPackage ../tools/misc/logstash/7.x.nix {
enableUnfree = false; enableUnfree = false;
}; };
logstash = logstash6; logstash = logstash6;