Merge pull request #77303 from tfc/port-elk-initdb-kafka
Port elk initdb kafka
This commit is contained in:
Florian Klink
GitHub
commit
47e0ce7f1a
|
|
@ -6,20 +6,12 @@
|
|||
# NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true
|
||||
}:
|
||||
|
||||
with import ../lib/testing.nix { inherit system pkgs; };
|
||||
with pkgs.lib;
|
||||
|
||||
let
|
||||
esUrl = "http://localhost:9200";
|
||||
|
||||
totalHits = message :
|
||||
"curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
|
||||
''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
|
||||
"| jq .hits.total";
|
||||
|
||||
mkElkTest = name : elk :
|
||||
let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
|
||||
in makeTest {
|
||||
in import ./make-test-python.nix ({
|
||||
inherit name;
|
||||
meta = with pkgs.stdenv.lib.maintainers; {
|
||||
maintainers = [ eelco offline basvandijk ];
|
||||
|
|
@ -50,15 +42,15 @@ let
|
|||
elk.journalbeat.version "6" < 0; in {
|
||||
enable = true;
|
||||
package = elk.journalbeat;
|
||||
extraConfig = mkOptionDefault (''
|
||||
extraConfig = pkgs.lib.mkOptionDefault (''
|
||||
logging:
|
||||
to_syslog: true
|
||||
level: warning
|
||||
metrics.enabled: false
|
||||
output.elasticsearch:
|
||||
hosts: [ "127.0.0.1:9200" ]
|
||||
${optionalString lt6 "template.enabled: false"}
|
||||
'' + optionalString (!lt6) ''
|
||||
${pkgs.lib.optionalString lt6 "template.enabled: false"}
|
||||
'' + pkgs.lib.optionalString (!lt6) ''
|
||||
journalbeat.inputs:
|
||||
- paths: []
|
||||
seek: cursor
|
||||
|
|
@ -130,11 +122,23 @@ let
|
|||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
import json
|
||||
|
||||
# Wait until elasticsearch is listening for connections.
|
||||
$one->waitForUnit("elasticsearch.service");
|
||||
$one->waitForOpenPort(9200);
|
||||
|
||||
def total_hits(message):
|
||||
dictionary = {"query": {"match": {"message": message}}}
|
||||
return (
|
||||
"curl --silent --show-error '${esUrl}/_search' "
|
||||
+ "-H 'Content-Type: application/json' "
|
||||
+ "-d '{}' ".format(json.dumps(dictionary))
|
||||
+ "| jq .hits.total"
|
||||
)
|
||||
|
||||
|
||||
start_all()
|
||||
|
||||
one.wait_for_unit("elasticsearch.service")
|
||||
one.wait_for_open_port(9200)
|
||||
|
||||
# Continue as long as the status is not "red". The status is probably
|
||||
# "yellow" instead of "green" because we are using a single elasticsearch
|
||||
|
|
@ -142,42 +146,43 @@ let
|
|||
#
|
||||
# TODO: extend this test with multiple elasticsearch nodes
|
||||
# and see if the status turns "green".
|
||||
$one->waitUntilSucceeds(
|
||||
"curl --silent --show-error '${esUrl}/_cluster/health' " .
|
||||
"| jq .status | grep -v red");
|
||||
one.wait_until_succeeds(
|
||||
"curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
|
||||
)
|
||||
|
||||
# Perform some simple logstash tests.
|
||||
$one->waitForUnit("logstash.service");
|
||||
$one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers");
|
||||
$one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons");
|
||||
with subtest("Perform some simple logstash tests"):
|
||||
one.wait_for_unit("logstash.service")
|
||||
one.wait_until_succeeds("cat /tmp/logstash.out | grep flowers")
|
||||
one.wait_until_succeeds("cat /tmp/logstash.out | grep -v dragons")
|
||||
|
||||
# See if kibana is healthy.
|
||||
$one->waitForUnit("kibana.service");
|
||||
$one->waitUntilSucceeds(
|
||||
"curl --silent --show-error 'http://localhost:5601/api/status' " .
|
||||
"| jq .status.overall.state | grep green");
|
||||
with subtest("Kibana is healthy"):
|
||||
one.wait_for_unit("kibana.service")
|
||||
one.wait_until_succeeds(
|
||||
"curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
|
||||
)
|
||||
|
||||
# See if logstash messages arive in elasticsearch.
|
||||
$one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
|
||||
$one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
|
||||
with subtest("Logstash messages arive in elasticsearch"):
|
||||
one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
|
||||
one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
|
||||
|
||||
# Test if a message logged to the journal
|
||||
# is ingested by elasticsearch via journalbeat.
|
||||
$one->waitForUnit("journalbeat.service");
|
||||
$one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
|
||||
$one->waitUntilSucceeds(
|
||||
"${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
|
||||
|
||||
'' + optionalString (!elasticsearchGe7) ''
|
||||
# Test elasticsearch-curator.
|
||||
$one->systemctl("stop logstash");
|
||||
$one->systemctl("start elasticsearch-curator");
|
||||
$one->waitUntilSucceeds(
|
||||
"! curl --silent --show-error '${esUrl}/_cat/indices' " .
|
||||
"| grep logstash | grep -q ^$1");
|
||||
with subtest(
|
||||
"A message logged to the journal is ingested by elasticsearch via journalbeat"
|
||||
):
|
||||
one.wait_for_unit("journalbeat.service")
|
||||
one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
|
||||
one.wait_until_succeeds(
|
||||
total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
|
||||
)
|
||||
'' + pkgs.lib.optionalString (!elasticsearchGe7) ''
|
||||
with subtest("Elasticsearch-curator works"):
|
||||
one.systemctl("stop logstash")
|
||||
one.systemctl("start elasticsearch-curator")
|
||||
one.wait_until_succeeds(
|
||||
'! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep -q ^'
|
||||
)
|
||||
'';
|
||||
};
|
||||
in mapAttrs mkElkTest {
|
||||
}) {};
|
||||
in pkgs.lib.mapAttrs mkElkTest {
|
||||
ELK-6 =
|
||||
if enableUnfree
|
||||
then {
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
let
|
||||
pkgs = import <nixpkgs> { };
|
||||
in
|
||||
with import <nixpkgs/nixos/lib/testing.nix> { inherit pkgs; system = builtins.currentSystem; };
|
||||
with pkgs.lib;
|
||||
|
||||
makeTest {
|
||||
name = "pg-initdb";
|
||||
|
||||
machine = {...}:
|
||||
{
|
||||
documentation.enable = false;
|
||||
services.postgresql.enable = true;
|
||||
services.postgresql.package = pkgs.postgresql_9_6;
|
||||
environment.pathsToLink = [
|
||||
"/share/postgresql"
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
$machine->start;
|
||||
$machine->succeed("sudo -u postgres initdb -D /tmp/testpostgres2");
|
||||
$machine->shutdown;
|
||||
'';
|
||||
|
||||
}
|
||||
|
|
@ -3,11 +3,10 @@
|
|||
pkgs ? import ../.. { inherit system config; }
|
||||
}:
|
||||
|
||||
with import ../lib/testing.nix { inherit system pkgs; };
|
||||
with pkgs.lib;
|
||||
|
||||
let
|
||||
makeKafkaTest = name: kafkaPackage: (makeTest {
|
||||
makeKafkaTest = name: kafkaPackage: (import ./make-test-python.nix ({
|
||||
inherit name;
|
||||
meta = with pkgs.stdenv.lib.maintainers; {
|
||||
maintainers = [ nequissimus ];
|
||||
|
|
@ -45,24 +44,40 @@ let
|
|||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
start_all()
|
||||
|
||||
$zookeeper1->waitForUnit("default.target");
|
||||
$zookeeper1->waitForUnit("zookeeper.service");
|
||||
$zookeeper1->waitForOpenPort(2181);
|
||||
zookeeper1.wait_for_unit("default.target")
|
||||
zookeeper1.wait_for_unit("zookeeper.service")
|
||||
zookeeper1.wait_for_open_port(2181)
|
||||
|
||||
$kafka->waitForUnit("default.target");
|
||||
$kafka->waitForUnit("apache-kafka.service");
|
||||
$kafka->waitForOpenPort(9092);
|
||||
kafka.wait_for_unit("default.target")
|
||||
kafka.wait_for_unit("apache-kafka.service")
|
||||
kafka.wait_for_open_port(9092)
|
||||
|
||||
$kafka->waitUntilSucceeds("${kafkaPackage}/bin/kafka-topics.sh --create --zookeeper zookeeper1:2181 --partitions 1 --replication-factor 1 --topic testtopic");
|
||||
$kafka->mustSucceed("echo 'test 1' | ${kafkaPackage}/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic testtopic");
|
||||
kafka.wait_until_succeeds(
|
||||
"${kafkaPackage}/bin/kafka-topics.sh --create "
|
||||
+ "--zookeeper zookeeper1:2181 --partitions 1 "
|
||||
+ "--replication-factor 1 --topic testtopic"
|
||||
)
|
||||
kafka.succeed(
|
||||
"echo 'test 1' | "
|
||||
+ "${kafkaPackage}/bin/kafka-console-producer.sh "
|
||||
+ "--broker-list localhost:9092 --topic testtopic"
|
||||
)
|
||||
'' + (if name == "kafka_0_9" then ''
|
||||
$kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --zookeeper zookeeper1:2181 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'");
|
||||
assert "test 1" in kafka.succeed(
|
||||
"${kafkaPackage}/bin/kafka-console-consumer.sh "
|
||||
+ "--zookeeper zookeeper1:2181 --topic testtopic "
|
||||
+ "--from-beginning --max-messages 1"
|
||||
)
|
||||
'' else ''
|
||||
$kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'");
|
||||
assert "test 1" in kafka.succeed(
|
||||
"${kafkaPackage}/bin/kafka-console-consumer.sh "
|
||||
+ "--bootstrap-server localhost:9092 --topic testtopic "
|
||||
+ "--from-beginning --max-messages 1"
|
||||
)
|
||||
'');
|
||||
});
|
||||
}) {});
|
||||
|
||||
in with pkgs; {
|
||||
kafka_0_9 = makeKafkaTest "kafka_0_9" apacheKafka_0_9;
|
||||
|
|
|
|||
|
|
@ -29,11 +29,15 @@ let
|
|||
|
||||
machine = {...}:
|
||||
{
|
||||
services.postgresql.enable = true;
|
||||
services.postgresql.package = postgresql-package;
|
||||
services.postgresql = {
|
||||
enable = true;
|
||||
package = postgresql-package;
|
||||
};
|
||||
|
||||
services.postgresqlBackup.enable = true;
|
||||
services.postgresqlBackup.databases = optional (!backup-all) "postgres";
|
||||
services.postgresqlBackup = {
|
||||
enable = true;
|
||||
databases = optional (!backup-all) "postgres";
|
||||
};
|
||||
};
|
||||
|
||||
testScript = let
|
||||
|
|
@ -49,23 +53,32 @@ let
|
|||
machine.start()
|
||||
machine.wait_for_unit("postgresql")
|
||||
|
||||
# postgresql should be available just after unit start
|
||||
machine.succeed(
|
||||
"cat ${test-sql} | sudo -u postgres psql"
|
||||
)
|
||||
machine.shutdown() # make sure that postgresql survive restart (bug #1735)
|
||||
time.sleep(2)
|
||||
machine.start()
|
||||
machine.wait_for_unit("postgresql")
|
||||
with subtest("Postgresql is available just after unit start"):
|
||||
machine.succeed(
|
||||
"cat ${test-sql} | sudo -u postgres psql"
|
||||
)
|
||||
|
||||
with subtest("Postgresql survives restart (bug #1735)"):
|
||||
machine.shutdown()
|
||||
time.sleep(2)
|
||||
machine.start()
|
||||
machine.wait_for_unit("postgresql")
|
||||
|
||||
machine.fail(check_count("SELECT * FROM sth;", 3))
|
||||
machine.succeed(check_count("SELECT * FROM sth;", 5))
|
||||
machine.fail(check_count("SELECT * FROM sth;", 4))
|
||||
machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1))
|
||||
|
||||
# Check backup service
|
||||
machine.succeed("systemctl start ${backupService}.service")
|
||||
machine.succeed("zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'")
|
||||
machine.succeed("stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600")
|
||||
with subtest("Backup service works"):
|
||||
machine.succeed(
|
||||
"systemctl start ${backupService}.service",
|
||||
"zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'",
|
||||
"stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600",
|
||||
)
|
||||
|
||||
with subtest("Initdb works"):
|
||||
machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2")
|
||||
|
||||
machine.shutdown()
|
||||
'';
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue