b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

nixos/journalbeat: support journalbeat >= 6 & add test

Browse source
This commit is contained in:
Bas van Dijk 2019-05-10 15:41:41 +02:00
parent a662f99139
commit 477c552c7d
2 changed files with 91 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
nixos/modules/services/logging/journalbeat.nix
View file

@ -5,11 +5,13 @@ with lib;
let
cfg = config.services.journalbeat;
lt6 = builtins.compareVersions cfg.package.version "6" < 0;
journalbeatYml = pkgs.writeText "journalbeat.yml" ''
name: ${cfg.name}
tags: ${builtins.toJSON cfg.tags}
journalbeat.cursor_state_file: ${cfg.stateDir}/cursor-state
${optionalString lt6 "journalbeat.cursor_state_file: /var/lib/${cfg.stateDir}/cursor-state"}
${cfg.extraConfig}
'';
@ -22,6 +24,16 @@ in
enable = mkEnableOption "journalbeat";
package = mkOption {
type = types.package;
default = pkgs.journalbeat;
defaultText = "pkgs.journalbeat";
example = literalExample "pkgs.journalbeat7";
description = ''
The journalbeat package to use
'';
};
name = mkOption {
type = types.str;
default = "journalbeat";
@ -36,13 +48,17 @@ in
stateDir = mkOption {
type = types.str;
default = "/var/lib/journalbeat";
description = "The state directory. Journalbeat's own logs and other data are stored here.";
default = "journalbeat";
description = ''
Directory below <literal>/var/lib/</literal> to store journalbeat's
own logs and other data. This directory will be created automatically
using systemd's StateDirectory mechanism.
'';
};
extraConfig = mkOption {
type = types.lines;
default = ''
default = optionalString lt6 ''
journalbeat:
seek_position: cursor
cursor_seek_fallback: tail
@ -61,7 +77,16 @@ in
config = mkIf cfg.enable {
systemd.services.journalbeat = with pkgs; {
assertions = [
{
assertion = !hasPrefix "/" cfg.stateDir;
message =
"The option services.journalbeat.stateDir shouldn't be an absolute directory." +
" It should be a directory relative to /var/lib/.";
}
];
systemd.services.journalbeat = {
description = "Journalbeat log shipper";
wantedBy = [ "multi-user.target" ];
preStart = ''
@ -69,7 +94,13 @@ in
mkdir -p ${cfg.stateDir}/logs
'';
serviceConfig = {
ExecStart = "${pkgs.journalbeat}/bin/journalbeat -c ${journalbeatYml} -path.data ${cfg.stateDir}/data -path.logs ${cfg.stateDir}/logs";
StateDirectory = cfg.stateDir;
ExecStart = ''
${cfg.package}/bin/journalbeat \
-c ${journalbeatYml} \
-path.data /var/lib/${cfg.stateDir}/data \
-path.logs /var/lib/${cfg.stateDir}/logs'';
Restart = "always";
};
};
};

61
nixos/tests/elk.nix
View file

@ -12,6 +12,11 @@ with pkgs.lib;
let
esUrl = "http://localhost:9200";
totalHits = message :
"curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
"| jq .hits.total";
mkElkTest = name : elk :
let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
in makeTest {
@ -21,7 +26,7 @@ let
};
nodes = {
one =
{ pkgs, ... }: {
{ pkgs, lib, ... }: {
# Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
#
# OpenJDK 64-Bit Server VM warning:
@ -40,6 +45,26 @@ let
environment.systemPackages = [ pkgs.jq ];
services = {
journalbeat = let lt6 = builtins.compareVersions
elk.journalbeat.version "6" < 0; in {
enable = true;
package = elk.journalbeat;
extraConfig = mkOptionDefault (''
logging:
to_syslog: true
level: warning
metrics.enabled: false
output.elasticsearch:
hosts: [ "127.0.0.1:9200" ]
${optionalString lt6 "template.enabled: false"}
'' + optionalString (!lt6) ''
journalbeat.inputs:
- paths: []
seek: cursor
'');
};
logstash = {
enable = true;
package = elk.logstash;
@ -107,14 +132,19 @@ let
testScript = ''
startAll;
# Wait until elasticsearch is listening for connections.
$one->waitForUnit("elasticsearch.service");
$one->waitForOpenPort(9200);
# Continue as long as the status is not "red". The status is probably
# "yellow" instead of "green" because we are using a single elasticsearch
# node which elasticsearch considers risky.
#
# TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
# TODO: extend this test with multiple elasticsearch nodes
# and see if the status turns "green".
$one->waitUntilSucceeds(
"curl --silent --show-error '${esUrl}/_cluster/health' " .
"| jq .status | grep -v red");
# Perform some simple logstash tests.
$one->waitForUnit("logstash.service");
@ -123,16 +153,28 @@ let
# See if kibana is healthy.
$one->waitForUnit("kibana.service");
$one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
$one->waitUntilSucceeds(
"curl --silent --show-error 'http://localhost:5601/api/status' " .
"| jq .status.overall.state | grep green");
# See if logstash messages arive in elasticsearch.
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
$one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
$one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
# Test if a message logged to the journal
# is ingested by elasticsearch via journalbeat.
$one->waitForUnit("journalbeat.service");
$one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
$one->waitUntilSucceeds(
"${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
'' + optionalString (!elasticsearchGe7) ''
# Test elasticsearch-curator.
$one->systemctl("stop logstash");
$one->systemctl("start elasticsearch-curator");
$one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
$one->waitUntilSucceeds(
"! curl --silent --show-error '${esUrl}/_cat/indices' " .
"| grep logstash | grep -q ^$1");
'';
};
in mapAttrs mkElkTest {
@ -140,6 +182,7 @@ in mapAttrs mkElkTest {
elasticsearch = pkgs.elasticsearch5;
logstash = pkgs.logstash5;
kibana = pkgs.kibana5;
journalbeat = pkgs.journalbeat5;
};
"ELK-6" =
if enableUnfree
@ -147,11 +190,13 @@ in mapAttrs mkElkTest {
elasticsearch = pkgs.elasticsearch6;
logstash = pkgs.logstash6;
kibana = pkgs.kibana6;
journalbeat = pkgs.journalbeat6;
}
else {
elasticsearch = pkgs.elasticsearch6-oss;
logstash = pkgs.logstash6-oss;
kibana = pkgs.kibana6-oss;
journalbeat = pkgs.journalbeat6;
};
"ELK-7" =
if enableUnfree
@ -159,10 +204,12 @@ in mapAttrs mkElkTest {
elasticsearch = pkgs.elasticsearch7;
logstash = pkgs.logstash7;
kibana = pkgs.kibana7;
journalbeat = pkgs.journalbeat7;
}
else {
elasticsearch = pkgs.elasticsearch7-oss;
logstash = pkgs.logstash7-oss;
kibana = pkgs.kibana7-oss;
journalbeat = pkgs.journalbeat7;
};
}