From 13352f28d2b587a01c262cbee29678209c919e95 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Sun, 14 Apr 2019 21:39:46 +0200 Subject: [PATCH] elk7: init at 7.0.0 This adds the following new packages: + elasticsearch7 + elasticsearch7-oss + logstash7 + logstash7-oss + kibana7 + kibana7-oss + filebeat7 + heartbeat7 + metricbeat7 + packetbeat7 + journalbeat7 The default major version of the ELK stack stays at 6. We should probably set it to 7 in a next commit. --- nixos/modules/services/search/kibana.nix | 44 +++++++++- nixos/tests/elk.nix | 22 ++++- .../misc/kibana/{default.nix => 6.x.nix} | 0 pkgs/development/tools/misc/kibana/7.x.nix | 64 ++++++++++++++ pkgs/misc/logging/beats/7.x.nix | 49 +++++++++++ .../elasticsearch/{default.nix => 6.x.nix} | 0 pkgs/servers/search/elasticsearch/7.x.nix | 83 +++++++++++++++++++ .../misc/logstash/{default.nix => 6.x.nix} | 0 pkgs/tools/misc/logstash/7.x.nix | 53 ++++++++++++ pkgs/top-level/all-packages.nix | 56 +++++++++---- 10 files changed, 348 insertions(+), 23 deletions(-) rename pkgs/development/tools/misc/kibana/{default.nix => 6.x.nix} (100%) create mode 100644 pkgs/development/tools/misc/kibana/7.x.nix create mode 100644 pkgs/misc/logging/beats/7.x.nix rename pkgs/servers/search/elasticsearch/{default.nix => 6.x.nix} (100%) create mode 100644 pkgs/servers/search/elasticsearch/7.x.nix rename pkgs/tools/misc/logstash/{default.nix => 6.x.nix} (100%) create mode 100644 pkgs/tools/misc/logstash/7.x.nix diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix index 3539b3ddb4f..ba58630a467 100644 --- a/nixos/modules/services/search/kibana.nix +++ b/nixos/modules/services/search/kibana.nix @@ -5,6 +5,9 @@ with lib; let cfg = config.services.kibana; + ge7 = builtins.compareVersions cfg.package.version "7" >= 0; + lt6_6 = builtins.compareVersions cfg.package.version "6.6" < 0; + cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON ( (filterAttrsRecursive (n: v: v != null) ({ server.host = cfg.listenAddress; @@ -16,6 +19,7 @@ let kibana.defaultAppId = cfg.defaultAppId; elasticsearch.url = cfg.elasticsearch.url; + elasticsearch.hosts = cfg.elasticsearch.hosts; elasticsearch.username = cfg.elasticsearch.username; elasticsearch.password = cfg.elasticsearch.password; @@ -67,9 +71,30 @@ in { elasticsearch = { url = mkOption { - description = "Elasticsearch url"; - default = "http://localhost:9200"; - type = types.str; + description = '' + Elasticsearch url. + + Defaults to "http://localhost:9200". + + Don't set this when using Kibana >= 7.0.0 because it will result in a + configuration error. Use + instead. + ''; + default = null; + type = types.nullOr types.str; + }; + + hosts = mkOption { + description = '' + The URLs of the Elasticsearch instances to use for all your queries. + All nodes listed here must be on the same cluster. + + Defaults to [ "http://localhost:9200" ]. + + This option is only valid when using kibana >= 6.6. + ''; + default = null; + type = types.nullOr (types.listOf types.str); }; username = mkOption { @@ -143,6 +168,19 @@ in { }; config = mkIf (cfg.enable) { + assertions = [ + { + assertion = ge7 -> cfg.elasticsearch.url == null; + message = + "The option services.kibana.elasticsearch.url has been removed when using kibana >= 7.0.0. " + + "Please use option services.kibana.elasticsearch.hosts instead."; + } + { + assertion = lt6_6 -> cfg.elasticsearch.hosts == null; + message = + "The option services.kibana.elasticsearch.hosts is only valid for kibana >= 6.6."; + } + ]; systemd.services.kibana = { description = "Kibana Service"; wantedBy = [ "multi-user.target" ]; diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index a82e75799ae..3b3fbd73dd5 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -12,7 +12,9 @@ with pkgs.lib; let esUrl = "http://localhost:9200"; - mkElkTest = name : elk : makeTest { + mkElkTest = name : elk : + let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0; + in makeTest { inherit name; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ eelco offline basvandijk ]; @@ -69,11 +71,11 @@ let kibana = { enable = true; package = elk.kibana; - elasticsearch.url = esUrl; }; elasticsearch-curator = { - enable = true; + # The current version of curator (5.6) doesn't support elasticsearch >= 7.0.0. + enable = !elasticsearchGe7; actionYAML = '' --- actions: @@ -126,7 +128,7 @@ let # See if logstash messages arive in elasticsearch. $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); - + '' + optionalString (!elasticsearchGe7) '' # Test elasticsearch-curator. $one->systemctl("stop logstash"); $one->systemctl("start elasticsearch-curator"); @@ -151,4 +153,16 @@ in mapAttrs mkElkTest { logstash = pkgs.logstash6-oss; kibana = pkgs.kibana6-oss; }; + "ELK-7" = + if enableUnfree + then { + elasticsearch = pkgs.elasticsearch7; + logstash = pkgs.logstash7; + kibana = pkgs.kibana7; + } + else { + elasticsearch = pkgs.elasticsearch7-oss; + logstash = pkgs.logstash7-oss; + kibana = pkgs.kibana7-oss; + }; } diff --git a/pkgs/development/tools/misc/kibana/default.nix b/pkgs/development/tools/misc/kibana/6.x.nix similarity index 100% rename from pkgs/development/tools/misc/kibana/default.nix rename to pkgs/development/tools/misc/kibana/6.x.nix diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix new file mode 100644 index 00000000000..19590239caf --- /dev/null +++ b/pkgs/development/tools/misc/kibana/7.x.nix @@ -0,0 +1,64 @@ +{ elk7Version +, enableUnfree ? true +, stdenv +, makeWrapper +, fetchzip +, fetchurl +, nodejs-10_x +, coreutils +, which +}: + +with stdenv.lib; +let + nodejs = nodejs-10_x; + inherit (builtins) elemAt; + info = splitString "-" stdenv.hostPlatform.system; + arch = elemAt info 0; + plat = elemAt info 1; + shas = + if enableUnfree + then { + "x86_64-linux" = "0awkbnbrffv2m6n9zqv8k4m23g4mril69rmj84ndqpdnv7shrk4l"; + "x86_64-darwin" = "0mbaya0xq99vfl8bn1wgi4m43c8qjp910bxf26z436km64311pip"; + } + else { + "x86_64-linux" = "0fy8vh0x2p8n2r3n5f67h1r5z75qfx5q4fghgqwnqymray5798v4"; + "x86_64-darwin" = "0zyhvbwvdj95v2nswx5b83n4cw3zy5a6bkrvypw6k1i833c6v5g7"; + }; + +in stdenv.mkDerivation rec { + name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}"; + version = elk7Version; + + src = fetchurl { + url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz"; + sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture"); + }; + + patches = [ + # Kibana specifies it specifically needs nodejs 10.15.2 but nodejs in nixpkgs is at 10.15.3. + # The test succeeds with this newer version so lets just + # disable the version check. + ./disable-nodejs-version-check.patch + ]; + + buildInputs = [ makeWrapper ]; + + installPhase = '' + mkdir -p $out/libexec/kibana $out/bin + mv * $out/libexec/kibana/ + rm -r $out/libexec/kibana/node + makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ + --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" + sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana + ''; + + meta = { + description = "Visualize logs and time-stamped data"; + homepage = http://www.elasticsearch.org/overview/kibana; + license = if enableUnfree then licenses.elastic else licenses.asl20; + maintainers = with maintainers; [ offline rickynils basvandijk ]; + platforms = with platforms; unix; + }; +} diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix new file mode 100644 index 00000000000..6d00b02d80e --- /dev/null +++ b/pkgs/misc/logging/beats/7.x.nix @@ -0,0 +1,49 @@ +{ stdenv, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, systemd }: + +let beat = package : extraArgs : buildGoPackage (rec { + name = "${package}-${version}"; + version = elk7Version; + + src = fetchFromGitHub { + owner = "elastic"; + repo = "beats"; + rev = "v${version}"; + sha256 = "0n1sjngc82b7wysw5aaiqvllq4c8rx2jj7khw4vrypc40f8ahjs5"; + }; + + goPackagePath = "github.com/elastic/beats"; + + subPackages = [ package ]; + + meta = with stdenv.lib; { + homepage = https://www.elastic.co/products/beats; + license = licenses.asl20; + maintainers = with maintainers; [ fadenb basvandijk ]; + platforms = platforms.linux; + }; + } // extraArgs); +in { + filebeat7 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; + heartbeat7 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; + metricbeat7 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; + packetbeat7 = beat "packetbeat" { + buildInputs = [ libpcap ]; + meta.description = "Network packet analyzer that ships data to Elasticsearch"; + meta.longDescription = '' + Packetbeat is an open source network packet analyzer that ships the + data to Elasticsearch. + + Think of it like a distributed real-time Wireshark with a lot more + analytics features. The Packetbeat shippers sniff the traffic between + your application processes, parse on the fly protocols like HTTP, MySQL, + PostgreSQL, Redis or Thrift and correlate the messages into transactions. + ''; + }; + journalbeat7 = beat "journalbeat" { + meta.description = '' + Journalbeat is an open source data collector to read and forward + journal entries from Linuxes with systemd. + ''; + buildInputs = [ systemd.dev ]; + }; +} diff --git a/pkgs/servers/search/elasticsearch/default.nix b/pkgs/servers/search/elasticsearch/6.x.nix similarity index 100% rename from pkgs/servers/search/elasticsearch/default.nix rename to pkgs/servers/search/elasticsearch/6.x.nix diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix new file mode 100644 index 00000000000..7ea33935faa --- /dev/null +++ b/pkgs/servers/search/elasticsearch/7.x.nix @@ -0,0 +1,83 @@ +{ elk7Version +, enableUnfree ? true +, stdenv +, fetchurl +, makeWrapper +, jre_headless +, utillinux +, autoPatchelfHook +, zlib +}: + +with stdenv.lib; +let + info = splitString "-" stdenv.hostPlatform.system; + arch = elemAt info 0; + plat = elemAt info 1; + shas = + if enableUnfree + then { + "x86_64-linux" = "1fi57xqwgxx0ivjyfvaybzz2k457qw59fn9qr26d86lnkigfxpk8"; + "x86_64-darwin" = "06hj96d4vl9q24dfx8ffydfs7qd440ys29654jgqp8sp7js7hjxp"; + } + else { + "x86_64-linux" = "1jrcdxm1swf8ahkv3h7kyzzhdq9nwwfhimpflzdq2d831fx525y8"; + "x86_64-darwin" = "119ym2d5fqwba3aq2izh5qj8vxawb7hf183cgg00s1sm1mm8lviv"; + }; +in +stdenv.mkDerivation (rec { + version = elk7Version; + name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}"; + + src = fetchurl { + url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}-${plat}-${arch}.tar.gz"; + sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture"); + }; + + patches = [ ./es-home-6.x.patch ]; + + postPatch = '' + substituteInPlace bin/elasticsearch-env --replace \ + "ES_CLASSPATH=\"\$ES_HOME/lib/*\"" \ + "ES_CLASSPATH=\"$out/lib/*\"" + + substituteInPlace bin/elasticsearch-cli --replace \ + "ES_CLASSPATH=\"\$ES_CLASSPATH:\$ES_HOME/\$additional_classpath_directory/*\"" \ + "ES_CLASSPATH=\"\$ES_CLASSPATH:$out/\$additional_classpath_directory/*\"" + ''; + + buildInputs = [ makeWrapper jre_headless utillinux ] + ++ optional enableUnfree zlib; + + installPhase = '' + mkdir -p $out + cp -R bin config lib modules plugins $out + + chmod +x $out/bin/* + + wrapProgram $out/bin/elasticsearch \ + --prefix PATH : "${utillinux}/bin/" \ + --set JAVA_HOME "${jre_headless}" + + wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}" + ''; + + passthru = { inherit enableUnfree; }; + + meta = { + description = "Open Source, Distributed, RESTful Search Engine"; + license = if enableUnfree then licenses.elastic else licenses.asl20; + platforms = platforms.unix; + maintainers = with maintainers; [ apeschar basvandijk ]; + }; +} // optionalAttrs enableUnfree { + dontPatchELF = true; + nativeBuildInputs = [ autoPatchelfHook ]; + runtimeDependencies = [ zlib ]; + postFixup = '' + for exe in $(find $out/modules/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do + echo "patching $exe..." + patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe" + done + ''; +}) diff --git a/pkgs/tools/misc/logstash/default.nix b/pkgs/tools/misc/logstash/6.x.nix similarity index 100% rename from pkgs/tools/misc/logstash/default.nix rename to pkgs/tools/misc/logstash/6.x.nix diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix new file mode 100644 index 00000000000..bb4a6b1436b --- /dev/null +++ b/pkgs/tools/misc/logstash/7.x.nix @@ -0,0 +1,53 @@ +{ elk7Version +, enableUnfree ? true +, stdenv +, fetchurl +, makeWrapper +, jre +}: + +with stdenv.lib; + +stdenv.mkDerivation rec { + version = elk7Version; + name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}"; + + src = fetchurl { + url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; + sha256 = + if enableUnfree + then "1dzk9amg23vbkj92wx529i1m2rrfjc0gzkbg6w3f6xaix5zch0vp" + else "07q2gbd3xidcklid5ny22zpb3q5v0z3k2kva6g214k7rnkr9bhyy"; + }; + + dontBuild = true; + dontPatchELF = true; + dontStrip = true; + dontPatchShebangs = true; + + buildInputs = [ + makeWrapper jre + ]; + + installPhase = '' + mkdir -p $out + cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out + + patchShebangs $out/bin/logstash + patchShebangs $out/bin/logstash-plugin + + wrapProgram $out/bin/logstash \ + --set JAVA_HOME "${jre}" + + wrapProgram $out/bin/logstash-plugin \ + --set JAVA_HOME "${jre}" + ''; + + meta = with stdenv.lib; { + description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; + homepage = https://www.elastic.co/products/logstash; + license = if enableUnfree then licenses.elastic else licenses.asl20; + platforms = platforms.unix; + maintainers = with maintainers; [ wjlroe offline basvandijk ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 34db5cf8982..c99e06da534 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -948,6 +948,16 @@ in bchunk = callPackage ../tools/cd-dvd/bchunk { }; + inherit (callPackages ../misc/logging/beats/5.x.nix { + # XXX: this is failing with Go 1.12. Error is related to cgo, an + # update to this package might fix it. + buildGoPackage = buildGo111Package; + }) + filebeat5 + heartbeat5 + metricbeat5 + packetbeat5; + inherit (callPackages ../misc/logging/beats/6.x.nix { # XXX: this is failing with Go 1.12. Error is related to cgo, an # update to this package might fix it. @@ -959,20 +969,21 @@ in packetbeat6 journalbeat6; - filebeat = filebeat6; - heartbeat = heartbeat6; - metricbeat = metricbeat6; - packetbeat = packetbeat6; - - inherit (callPackages ../misc/logging/beats/5.x.nix { + inherit (callPackages ../misc/logging/beats/7.x.nix { # XXX: this is failing with Go 1.12. Error is related to cgo, an # update to this package might fix it. buildGoPackage = buildGo111Package; }) - filebeat5 - heartbeat5 - metricbeat5 - packetbeat5; + filebeat7 + heartbeat7 + metricbeat7 + packetbeat7 + journalbeat7; + + filebeat = filebeat6; + heartbeat = heartbeat6; + metricbeat = metricbeat6; + packetbeat = packetbeat6; bfr = callPackage ../tools/misc/bfr { }; @@ -2572,10 +2583,15 @@ in # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. elk5Version = "5.6.9"; elk6Version = "6.7.1"; + elk7Version = "7.0.0"; elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { }; - elasticsearch6 = callPackage ../servers/search/elasticsearch { }; - elasticsearch6-oss = callPackage ../servers/search/elasticsearch { + elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { }; + elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix { + enableUnfree = false; + }; + elasticsearch7 = callPackage ../servers/search/elasticsearch/7.x.nix { }; + elasticsearch7-oss = callPackage ../servers/search/elasticsearch/7.x.nix { enableUnfree = false; }; elasticsearch = elasticsearch6; @@ -3778,8 +3794,12 @@ in keyfuzz = callPackage ../tools/inputmethods/keyfuzz { }; kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { }; - kibana6 = callPackage ../development/tools/misc/kibana/default.nix { }; - kibana6-oss = callPackage ../development/tools/misc/kibana/default.nix { + kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { }; + kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix { + enableUnfree = false; + }; + kibana7 = callPackage ../development/tools/misc/kibana/7.x.nix { }; + kibana7-oss = callPackage ../development/tools/misc/kibana/7.x.nix { enableUnfree = false; }; kibana = kibana6; @@ -3868,8 +3888,12 @@ in lockfileProgs = callPackage ../tools/misc/lockfile-progs { }; logstash5 = callPackage ../tools/misc/logstash/5.x.nix { }; - logstash6 = callPackage ../tools/misc/logstash { }; - logstash6-oss = callPackage ../tools/misc/logstash { + logstash6 = callPackage ../tools/misc/logstash/6.x.nix { }; + logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix { + enableUnfree = false; + }; + logstash7 = callPackage ../tools/misc/logstash/7.x.nix { }; + logstash7-oss = callPackage ../tools/misc/logstash/7.x.nix { enableUnfree = false; }; logstash = logstash6;