From 6ef8cad2a7683e933411a3ff1c1cb70c4e45032b Mon Sep 17 00:00:00 2001 From: Matej Cotman Date: Sat, 9 Sep 2017 02:00:35 +0200 Subject: [PATCH] kubernetes: fix tests --- nixos/tests/kubernetes/base.nix | 113 ++++++ nixos/tests/kubernetes/certs.nix | 400 +++++++++---------- nixos/tests/kubernetes/default.nix | 8 +- nixos/tests/kubernetes/dns.nix | 150 ++++--- nixos/tests/kubernetes/e2e.nix | 40 ++ nixos/tests/kubernetes/kubernetes-common.nix | 111 +++-- nixos/tests/kubernetes/kubernetes-master.nix | 63 --- nixos/tests/kubernetes/multinode-kubectl.nix | 123 ------ nixos/tests/kubernetes/rbac.nix | 145 ++++--- nixos/tests/kubernetes/singlenode.nix | 75 ---- 10 files changed, 538 insertions(+), 690 deletions(-) create mode 100644 nixos/tests/kubernetes/base.nix create mode 100644 nixos/tests/kubernetes/e2e.nix delete mode 100644 nixos/tests/kubernetes/kubernetes-master.nix delete mode 100644 nixos/tests/kubernetes/multinode-kubectl.nix delete mode 100644 nixos/tests/kubernetes/singlenode.nix diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix new file mode 100644 index 00000000000..acf2e025081 --- /dev/null +++ b/nixos/tests/kubernetes/base.nix @@ -0,0 +1,113 @@ +{ system ? builtins.currentSystem }: + +with import ../../lib/testing.nix { inherit system; }; +with import ../../lib/qemu-flags.nix; +with pkgs.lib; + +let + mkKubernetesBaseTest = + { name, domain ? "my.zyx", test, machines + , pkgs ? import { inherit system; } + , certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; } + , extraConfiguration ? null }: + let + masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines)); + master = machines.${masterName}; + extraHosts = '' + ${master.ip} etcd.${domain} + ${master.ip} api.${domain} + ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)} + ''; + in makeTest { + inherit name; + + nodes = mapAttrs (machineName: machine: + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = mkDefault 768; + virtualisation.diskSize = mkDefault 4096; + networking = { + inherit domain extraHosts; + primaryIPAddress = mkForce machine.ip; + + firewall = { + allowedTCPPorts = [ + 10250 # kubelet + ]; + trustedInterfaces = ["docker0"]; + + extraCommands = concatMapStrings (node: '' + iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT + '') (attrValues nodes); + }; + }; + programs.bash.enableCompletion = true; + environment.variables = { + ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem"; + ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem"; + ETCDCTL_CA_FILE = "${certs.worker}/ca.pem"; + ETCDCTL_PEERS = "https://etcd.${domain}:2379"; + }; + services.flannel.iface = "eth1"; + services.kubernetes.apiserver.advertiseAddress = master.ip; + } + (optionalAttrs (any (role: role == "master") machine.roles) { + networking.firewall.allowedTCPPorts = [ + 2379 2380 # etcd + 443 # kubernetes apiserver + ]; + services.etcd = { + enable = true; + certFile = "${certs.master}/etcd.pem"; + keyFile = "${certs.master}/etcd-key.pem"; + trustedCaFile = "${certs.master}/ca.pem"; + peerClientCertAuth = true; + listenClientUrls = ["https://0.0.0.0:2379"]; + listenPeerUrls = ["https://0.0.0.0:2380"]; + advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"]; + initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"]; + initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"]; + }; + }) + (import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; }) + (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; })) + (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; })) + ] + ) machines; + + testScript = '' + startAll; + + ${test} + ''; + }; + + mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({ + machines = { + machine1 = { + roles = ["master"]; + ip = "192.168.1.1"; + }; + machine2 = { + roles = ["node"]; + ip = "192.168.1.2"; + }; + }; + } // attrs // { + name = "kubernetes-${attrs.name}-multinode"; + }); + + mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({ + machines = { + machine1 = { + roles = ["master" "node"]; + ip = "192.168.1.1"; + }; + }; + } // attrs // { + name = "kubernetes-${attrs.name}-singlenode"; + }); +in { + inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest; +} diff --git a/nixos/tests/kubernetes/certs.nix b/nixos/tests/kubernetes/certs.nix index f167224e549..f108e35b98c 100644 --- a/nixos/tests/kubernetes/certs.nix +++ b/nixos/tests/kubernetes/certs.nix @@ -1,229 +1,185 @@ { pkgs ? import {}, - servers ? {test = "1.2.3.4";}, - internalDomain ? "cluster.local", - externalDomain ? "nixos.xyz" + internalDomain ? "cloud.yourdomain.net", + externalDomain ? "myawesomecluster.cluster.yourdomain.net", + serviceClusterIp ? "10.0.0.1" }: let - mkAltNames = ipFrom: dnsFrom: - pkgs.lib.concatImapStringsSep "\n" (i: v: "IP.${toString (i+ipFrom)} = ${v.ip}\nDNS.${toString (i+dnsFrom)} = ${v.name}.${externalDomain}") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers); + runWithCFSSL = name: cmd: + builtins.fromJSON (builtins.readFile ( + pkgs.runCommand "${name}-cfss.json" { + buildInputs = [ pkgs.cfssl ]; + } "cfssl ${cmd} > $out" + )); - runWithOpenSSL = file: cmd: pkgs.runCommand file { - buildInputs = [ pkgs.openssl ]; - passthru = { inherit file; }; - } cmd; - - ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048"; - ca_pem = runWithOpenSSL "ca.pem" '' - openssl req \ - -x509 -new -nodes -key ${ca_key} \ - -days 10000 -out $out -subj "/CN=etcd-ca" - ''; - - etcd_cnf = pkgs.writeText "openssl.cnf" '' - [req] - req_extensions = v3_req - distinguished_name = req_distinguished_name - [req_distinguished_name] - [ v3_req ] - basicConstraints = CA:FALSE - keyUsage = digitalSignature, keyEncipherment - extendedKeyUsage = serverAuth - subjectAltName = @alt_names - [alt_names] - DNS.1 = etcd.kubernetes.${externalDomain} - IP.1 = 127.0.0.1 - ''; - etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048"; - etcd_csr = runWithOpenSSL "etcd.csr" '' - openssl req \ - -new -key ${etcd_key} \ - -out $out -subj "/CN=etcd" \ - -config ${etcd_cnf} - ''; - etcd_cert = runWithOpenSSL "etcd.pem" '' - openssl x509 \ - -req -in ${etcd_csr} \ - -CA ${ca_pem} -CAkey ${ca_key} \ - -CAcreateserial -out $out \ - -days 3650 -extensions v3_req \ - -extfile ${etcd_cnf} - ''; - - etcd_client_key = runWithOpenSSL "etcd-client-key.pem" - "openssl genrsa -out $out 2048"; - - etcd_client_csr = runWithOpenSSL "etcd-client.csr" '' - openssl req \ - -new -key ${etcd_client_key} \ - -out $out -subj "/CN=etcd-client" \ - -config ${client_openssl_cnf} - ''; - - etcd_client_cert = runWithOpenSSL "etcd-client.crt" '' - openssl x509 \ - -req -in ${etcd_client_csr} \ - -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ - -out $out -days 3650 -extensions v3_req \ - -extfile ${client_openssl_cnf} - ''; - - admin_key = runWithOpenSSL "admin-key.pem" - "openssl genrsa -out $out 2048"; - - admin_csr = runWithOpenSSL "admin.csr" '' - openssl req \ - -new -key ${admin_key} \ - -out $out -subj "/CN=admin/O=system:masters" \ - -config ${client_openssl_cnf} - ''; - - admin_cert = runWithOpenSSL "admin.crt" '' - openssl x509 \ - -req -in ${admin_csr} \ - -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ - -out $out -days 3650 -extensions v3_req \ - -extfile ${client_openssl_cnf} - ''; - - apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048"; - - apiserver_csr = runWithOpenSSL "apiserver.csr" '' - openssl req \ - -new -key ${apiserver_key} \ - -out $out -subj "/CN=kube-apiserver" \ - -config ${apiserver_cnf} - ''; - - apiserver_cert = runWithOpenSSL "apiserver.pem" '' - openssl x509 \ - -req -in ${apiserver_csr} \ - -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ - -out $out -days 3650 -extensions v3_req \ - -extfile ${apiserver_cnf} - ''; - - worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048"; - - worker_csr = runWithOpenSSL "worker.csr" '' - openssl req \ - -new -key ${worker_key} \ - -out $out -subj "/CN=kube-worker/O=system:authenticated" \ - -config ${worker_cnf} - ''; - - worker_cert = runWithOpenSSL "worker.pem" '' - openssl x509 \ - -req -in ${worker_csr} \ - -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ - -out $out -days 3650 -extensions v3_req \ - -extfile ${worker_cnf} - ''; - - openssl_cnf = pkgs.writeText "openssl.cnf" '' - [req] - req_extensions = v3_req - distinguished_name = req_distinguished_name - [req_distinguished_name] - [ v3_req ] - basicConstraints = CA:FALSE - keyUsage = digitalSignature, keyEncipherment - extendedKeyUsage = serverAuth - subjectAltName = @alt_names - [alt_names] - DNS.1 = *.cluster.${externalDomain} - IP.1 = 127.0.0.1 - ${mkAltNames 1 1} - ''; - - client_openssl_cnf = pkgs.writeText "client-openssl.cnf" '' - [req] - req_extensions = v3_req - distinguished_name = req_distinguished_name - [req_distinguished_name] - [ v3_req ] - basicConstraints = CA:FALSE - keyUsage = digitalSignature, keyEncipherment - extendedKeyUsage = clientAuth - subjectAltName = @alt_names - [alt_names] - DNS.1 = kubernetes - DNS.2 = kubernetes.default - DNS.3 = kubernetes.default.svc - DNS.4 = kubernetes.default.svc.${internalDomain} - DNS.5 = kubernetes.${externalDomain} - DNS.6 = *.cluster.${externalDomain} - IP.1 = 10.1.10.1 - ${mkAltNames 1 6} - ''; - - apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" '' - [req] - req_extensions = v3_req - distinguished_name = req_distinguished_name - [req_distinguished_name] - [ v3_req ] - basicConstraints = CA:FALSE - keyUsage = nonRepudiation, digitalSignature, keyEncipherment - extendedKeyUsage = serverAuth - subjectAltName = @alt_names - [alt_names] - DNS.1 = kubernetes - DNS.2 = kubernetes.default - DNS.3 = kubernetes.default.svc - DNS.4 = kubernetes.default.svc.${internalDomain} - DNS.5 = kubernetes.${externalDomain} - DNS.6 = *.cluster.${externalDomain} - IP.1 = 10.1.10.1 - ${mkAltNames 1 6} - ''; - - worker_cnf = pkgs.writeText "worker-openssl.cnf" '' - [req] - req_extensions = v3_req - distinguished_name = req_distinguished_name - [req_distinguished_name] - [ v3_req ] - basicConstraints = CA:FALSE - keyUsage = nonRepudiation, digitalSignature, keyEncipherment - subjectAltName = @alt_names - [alt_names] - DNS.1 = *.cluster.${externalDomain} - IP.1 = 10.1.10.1 - ${mkAltNames 1 1} - ''; - - ln = cert: target: '' - cp -v ${cert} ${target}/${cert.file} - ''; -in - pkgs.stdenv.mkDerivation rec { - name = "kubernetes-certs"; - unpackPhase = "true"; - installPhase = '' - set -xe + writeCFSSL = content: + pkgs.runCommand content.name { + buildInputs = [ pkgs.cfssl ]; + } '' mkdir -p $out - ${ln ca_key "$out"} - ${ln ca_pem "$out"} - - ${ln etcd_key "$out"} - ${ln etcd_csr "$out"} - ${ln etcd_cert "$out"} - - ${ln etcd_client_key "$out"} - ${ln etcd_client_csr "$out"} - ${ln etcd_client_cert "$out"} - - ${ln apiserver_key "$out"} - ${ln apiserver_csr "$out"} - ${ln apiserver_cert "$out"} - - ${ln worker_key "$out"} - ${ln worker_csr "$out"} - ${ln worker_cert "$out"} - - ${ln admin_key "$out"} - ${ln admin_csr "$out"} - ${ln admin_cert "$out"} + cd $out + cat ${writeFile content} | cfssljson -bare ${content.name} ''; - } + + noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content; + noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content; + + writeFile = content: pkgs.writeText "content" ( + if pkgs.lib.isAttrs content then builtins.toJSON content + else toString content + ); + + createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }: + noCSR ( + (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile { + CN = cn; + hosts = hosts; + key = { algo = "rsa"; inherit size; }; + }}") // { inherit name; } + ); + + createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }: + noCSR ( + (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile { + CN = cn; + names = map (group: {O = group;}) groups; + hosts = [""]; + key = { algo = "rsa"; inherit size; }; + }}") // { inherit name; } + ); + + createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }: + (noCSR (runWithCFSSL CN "genkey -initca ${writeFile { + key = { algo = "rsa"; inherit size; }; + names = [{ inherit C ST L O OU CN emailAddress; }]; + }}")) // { + inherit name; + config.signing = { + default.expiry = expiry; + profiles = { + server = { + inherit expiry; + usages = [ + "signing" + "key encipherment" + "server auth" + ]; + }; + client = { + inherit expiry; + usages = [ + "signing" + "key encipherment" + "client auth" + ]; + }; + peer = { + inherit expiry; + usages = [ + "signing" + "key encipherment" + "server auth" + "client auth" + ]; + }; + }; + }; + }; + + ca = createSigningCertKey {}; + + kube-apiserver = createServingCertKey { + inherit ca; + cn = "kube-apiserver"; + hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp]; + }; + + kubelet = createServingCertKey { + inherit ca; + cn = "kubelet"; + hosts = ["*.${externalDomain}"]; + }; + + service-accounts = createServingCertKey { + inherit ca; + cn = "kube-service-accounts"; + }; + + etcd = createServingCertKey { + inherit ca; + cn = "etcd"; + hosts = ["etcd.${externalDomain}"]; + }; + + etcd-client = createClientCertKey { + inherit ca; + cn = "etcd-client"; + }; + + kubelet-client = createClientCertKey { + inherit ca; + cn = "kubelet-client"; + groups = ["system:masters"]; + }; + + apiserver-client = { + kubelet = createClientCertKey { + inherit ca; + cn = "apiserver-client-kubelet"; + groups = ["system:nodes"]; + }; + + kube-proxy = createClientCertKey { + inherit ca; + name = "apiserver-client-kube-proxy"; + cn = "system:kube-proxy"; + groups = ["system:kube-proxy" "system:nodes"]; + }; + + kube-controller-manager = createClientCertKey { + inherit ca; + name = "apiserver-client-kube-controller-manager"; + cn = "system:kube-controller-manager"; + groups = ["system:masters"]; + }; + + kube-scheduler = createClientCertKey { + inherit ca; + name = "apiserver-client-kube-scheduler"; + cn = "system:kube-scheduler"; + groups = ["system:kube-scheduler"]; + }; + + admin = createClientCertKey { + inherit ca; + cn = "admin"; + groups = ["system:masters"]; + }; + }; +in { + master = pkgs.buildEnv { + name = "master-keys"; + paths = [ + (writeCFSSL (noKey ca)) + (writeCFSSL kube-apiserver) + (writeCFSSL kubelet-client) + (writeCFSSL apiserver-client.kube-controller-manager) + (writeCFSSL apiserver-client.kube-scheduler) + (writeCFSSL service-accounts) + (writeCFSSL etcd) + ]; + }; + + worker = pkgs.buildEnv { + name = "worker-keys"; + paths = [ + (writeCFSSL (noKey ca)) + (writeCFSSL kubelet) + (writeCFSSL apiserver-client.kubelet) + (writeCFSSL apiserver-client.kube-proxy) + (writeCFSSL etcd-client) + ]; + }; + + admin = writeCFSSL apiserver-client.admin; +} diff --git a/nixos/tests/kubernetes/default.nix b/nixos/tests/kubernetes/default.nix index 2b61980349e..a801759bf58 100644 --- a/nixos/tests/kubernetes/default.nix +++ b/nixos/tests/kubernetes/default.nix @@ -1,7 +1,7 @@ { system ? builtins.currentSystem }: { - kubernetes-singlenode = import ./singlenode.nix { inherit system; }; - kubernetes-multinode-kubectl = import ./multinode-kubectl.nix { inherit system; }; - kubernetes-rbac = import ./rbac.nix { inherit system; }; - kubernetes-dns = import ./dns.nix { inherit system; }; + dns = import ./dns.nix { inherit system; }; + # e2e = import ./e2e.nix { inherit system; }; # TODO: make it pass + # the following test(s) can be removed when e2e is working: + rbac = import ./rbac.nix { inherit system; }; } diff --git a/nixos/tests/kubernetes/dns.nix b/nixos/tests/kubernetes/dns.nix index 4659c520dee..74d98dabec8 100644 --- a/nixos/tests/kubernetes/dns.nix +++ b/nixos/tests/kubernetes/dns.nix @@ -1,16 +1,11 @@ -{ system ? builtins.currentSystem }: - -with import ../../lib/testing.nix { inherit system; }; -with import ../../lib/qemu-flags.nix; -with pkgs.lib; - +{ system ? builtins.currentSystem, pkgs ? import { inherit system; } }: +with import ./base.nix { inherit system; }; let - servers.master = "192.168.1.1"; - servers.one = "192.168.1.10"; + domain = "my.zyx"; - certs = import ./certs.nix { inherit servers; }; + certs = import ./certs.nix { externalDomain = domain; }; - redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON { + redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON { kind = "Pod"; apiVersion = "v1"; metadata.name = "redis"; @@ -40,64 +35,93 @@ let redisImage = pkgs.dockerTools.buildImage { name = "redis"; tag = "latest"; - contents = [ pkgs.redis pkgs.bind.dnsutils pkgs.coreutils pkgs.inetutils pkgs.nmap ]; + contents = [ pkgs.redis pkgs.bind.host ]; config.Entrypoint = "/bin/redis-server"; }; - test = '' - $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); - $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); + probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "probe"; + metadata.labels.name = "probe"; + spec.containers = [{ + name = "probe"; + image = "probe"; + args = [ "-f" ]; + tty = true; + imagePullPolicy = "Never"; + }]; + }); - $one->execute("docker load < ${redisImage}"); - - $master->waitUntilSucceeds("kubectl create -f ${redisPod} || kubectl apply -f ${redisPod}"); - $master->waitUntilSucceeds("kubectl create -f ${redisService} || kubectl apply -f ${redisService}"); - - $master->waitUntilSucceeds("kubectl get pod redis | grep Running"); - - $master->succeed("dig \@192.168.1.1 redis.default.svc.cluster.local"); - $one->succeed("dig \@192.168.1.10 redis.default.svc.cluster.local"); - - - $master->succeed("kubectl exec -ti redis -- cat /etc/resolv.conf | grep 'nameserver 192.168.1.10'"); - - $master->succeed("kubectl exec -ti redis -- dig \@192.168.1.10 redis.default.svc.cluster.local"); - ''; - -in makeTest { - name = "kubernetes-dns"; - - nodes = { - master = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.master; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - (import ./kubernetes-master.nix { inherit pkgs config certs; }) - ]; - - one = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.one; - services.kubernetes.roles = ["node"]; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - ]; + probeImage = pkgs.dockerTools.buildImage { + name = "probe"; + tag = "latest"; + contents = [ pkgs.bind.host pkgs.busybox ]; + config.Entrypoint = "/bin/tail"; }; - testScript = '' - startAll; + extraConfiguration = { config, pkgs, lib, nodes, ... }: { + environment.systemPackages = [ pkgs.bind.host ]; + # virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}"; + services.dnsmasq.enable = true; + services.dnsmasq.servers = [ + "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53" + ]; + }; - ${test} - ''; + base = { + name = "dns"; + inherit domain certs extraConfiguration; + }; + + singleNodeTest = { + test = '' + # prepare machine1 for test + $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready"); + $machine1->execute("docker load < ${redisImage}"); + $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}"); + $machine1->waitUntilSucceeds("kubectl create -f ${redisService}"); + $machine1->execute("docker load < ${probeImage}"); + $machine1->waitUntilSucceeds("kubectl create -f ${probePod}"); + + # check if pods are running + $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running"); + $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running"); + $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'"); + + # check dns on host (dnsmasq) + $machine1->succeed("host redis.default.svc.cluster.local"); + + # check dns inside the container + $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local"); + ''; + }; + + multiNodeTest = { + test = '' + # prepare machines for test + $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready"); + $machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready"); + $machine2->execute("docker load < ${redisImage}"); + $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}"); + $machine1->waitUntilSucceeds("kubectl create -f ${redisService}"); + $machine2->execute("docker load < ${probeImage}"); + $machine1->waitUntilSucceeds("kubectl create -f ${probePod}"); + + # check if pods are running + $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running"); + $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running"); + $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'"); + + # check dns on hosts (dnsmasq) + $machine1->succeed("host redis.default.svc.cluster.local"); + $machine2->succeed("host redis.default.svc.cluster.local"); + + # check dns inside the container + $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local"); + ''; + }; +in { + singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest); + multinode = mkKubernetesMultiNodeTest (base // multiNodeTest); } diff --git a/nixos/tests/kubernetes/e2e.nix b/nixos/tests/kubernetes/e2e.nix new file mode 100644 index 00000000000..d9d7ba9bb2c --- /dev/null +++ b/nixos/tests/kubernetes/e2e.nix @@ -0,0 +1,40 @@ +{ system ? builtins.currentSystem, pkgs ? import { inherit system; } }: +with import ./base.nix { inherit system; }; +let + domain = "my.zyx"; + certs = import ./certs.nix { externalDomain = domain; }; + kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON { + apiVersion = "v1"; + kind = "Config"; + clusters = [{ + name = "local"; + cluster.certificate-authority = "${certs.master}/ca.pem"; + cluster.server = "https://api.${domain}"; + }]; + users = [{ + name = "kubelet"; + user = { + client-certificate = "${certs.admin}/admin.pem"; + client-key = "${certs.admin}/admin-key.pem"; + }; + }]; + contexts = [{ + context = { + cluster = "local"; + user = "kubelet"; + }; + current-context = "kubelet-context"; + }]; + }); + + base = { + name = "e2e"; + inherit domain certs; + test = '' + $machine1->succeed("e2e.test -kubeconfig ${kubeconfig} -provider local -ginkgo.focus '\\[Conformance\\]' -ginkgo.skip '\\[Flaky\\]|\\[Serial\\]'"); + ''; + }; +in { + singlenode = mkKubernetesSingleNodeTest base; + multinode = mkKubernetesMultiNodeTest base; +} diff --git a/nixos/tests/kubernetes/kubernetes-common.nix b/nixos/tests/kubernetes/kubernetes-common.nix index 9f9e730fa65..00a5c9aba4e 100644 --- a/nixos/tests/kubernetes/kubernetes-common.nix +++ b/nixos/tests/kubernetes/kubernetes-common.nix @@ -1,72 +1,59 @@ -{ config, pkgs, certs, servers }: - +{ roles, config, pkgs, certs }: +with pkgs.lib; let - etcd_key = "${certs}/etcd-key.pem"; - etcd_cert = "${certs}/etcd.pem"; - ca_pem = "${certs}/ca.pem"; - etcd_client_cert = "${certs}/etcd-client.crt"; - etcd_client_key = "${certs}/etcd-client-key.pem"; + base = { + inherit roles; + featureGates = ["AllAlpha"]; + flannel.enable = true; + addons.dashboard.enable = true; + verbose = true; - worker_key = "${certs}/worker-key.pem"; - worker_cert = "${certs}/worker.pem"; - - rootCaFile = pkgs.writeScript "rootCaFile.pem" '' - ${pkgs.lib.readFile "${certs}/ca.pem"} - - ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")} - ''; - - mkHosts = - pkgs.lib.concatMapStringsSep "\n" (v: "${v.ip} ${v.name}.nixos.xyz") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers); - -in -{ - programs.bash.enableCompletion = true; - environment.systemPackages = with pkgs; [ netcat bind etcd.bin ]; - - networking = { - firewall.allowedTCPPorts = [ - 10250 # kubelet - ]; - extraHosts = '' - # register "external" domains - ${servers.master} etcd.kubernetes.nixos.xyz - ${servers.master} kubernetes.nixos.xyz - ${mkHosts} - ''; - }; - services.flannel.iface = "eth1"; - environment.variables = { - ETCDCTL_CERT_FILE = "${etcd_client_cert}"; - ETCDCTL_KEY_FILE = "${etcd_client_key}"; - ETCDCTL_CA_FILE = "${rootCaFile}"; - ETCDCTL_PEERS = "https://etcd.kubernetes.nixos.xyz:2379"; - }; - - services.kubernetes = { - kubelet = { - tlsKeyFile = worker_key; - tlsCertFile = worker_cert; - hostname = "${config.networking.hostName}.nixos.xyz"; - nodeIp = config.networking.primaryIPAddress; + caFile = "${certs.master}/ca.pem"; + apiserver = { + tlsCertFile = "${certs.master}/kube-apiserver.pem"; + tlsKeyFile = "${certs.master}/kube-apiserver-key.pem"; + kubeletClientCertFile = "${certs.master}/kubelet-client.pem"; + kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem"; + serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem"; }; etcd = { - servers = ["https://etcd.kubernetes.nixos.xyz:2379"]; - keyFile = etcd_client_key; - certFile = etcd_client_cert; - caFile = ca_pem; + servers = ["https://etcd.${config.networking.domain}:2379"]; + certFile = "${certs.worker}/etcd-client.pem"; + keyFile = "${certs.worker}/etcd-client-key.pem"; }; kubeconfig = { - server = "https://kubernetes.nixos.xyz"; - caFile = rootCaFile; - certFile = worker_cert; - keyFile = worker_key; + server = "https://api.${config.networking.domain}"; + }; + kubelet = { + tlsCertFile = "${certs.worker}/kubelet.pem"; + tlsKeyFile = "${certs.worker}/kubelet-key.pem"; + hostname = "${config.networking.hostName}.${config.networking.domain}"; + kubeconfig = { + certFile = "${certs.worker}/apiserver-client-kubelet.pem"; + keyFile = "${certs.worker}/apiserver-client-kubelet-key.pem"; + }; + }; + controllerManager = { + serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem"; + kubeconfig = { + certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem"; + keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem"; + }; + }; + scheduler = { + kubeconfig = { + certFile = "${certs.master}/apiserver-client-kube-scheduler.pem"; + keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem"; + }; + }; + proxy = { + kubeconfig = { + certFile = "${certs.worker}/apiserver-client-kube-proxy.pem"; + keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem"; + }; }; - flannel.enable = true; - - dns.port = 4453; }; - services.dnsmasq.enable = true; - services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; +in { + services.kubernetes = base; } diff --git a/nixos/tests/kubernetes/kubernetes-master.nix b/nixos/tests/kubernetes/kubernetes-master.nix deleted file mode 100644 index 28cce6ea653..00000000000 --- a/nixos/tests/kubernetes/kubernetes-master.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ config, pkgs, certs }: -let - etcd_key = "${certs}/etcd-key.pem"; - etcd_cert = "${certs}/etcd.pem"; - ca_pem = "${certs}/ca.pem"; - etcd_client_cert = "${certs}/etcd-client.crt"; - etcd_client_key = "${certs}/etcd-client-key.pem"; - - apiserver_key = "${certs}/apiserver-key.pem"; - apiserver_cert = "${certs}/apiserver.pem"; - worker_key = "${certs}/worker-key.pem"; - worker_cert = "${certs}/worker.pem"; - - - rootCaFile = pkgs.writeScript "rootCaFile.pem" '' - ${pkgs.lib.readFile "${certs}/ca.pem"} - - ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")} - ''; -in -{ - networking = { - firewall = { - enable = true; - allowPing = true; - allowedTCPPorts = [ - 2379 2380 # etcd - 443 # kubernetes apiserver - ]; - }; - }; - - services.etcd = { - enable = pkgs.lib.mkForce true; - keyFile = etcd_key; - certFile = etcd_cert; - trustedCaFile = rootCaFile; - peerClientCertAuth = true; - listenClientUrls = ["https://0.0.0.0:2379"]; - listenPeerUrls = ["https://0.0.0.0:2380"]; - - advertiseClientUrls = ["https://etcd.kubernetes.nixos.xyz:2379"]; - initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"]; - initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"]; - }; - - services.kubernetes = { - roles = ["master"]; - scheduler.leaderElect = true; - controllerManager.rootCaFile = rootCaFile; - controllerManager.serviceAccountKeyFile = apiserver_key; - apiserver = { - publicAddress = "192.168.1.1"; - advertiseAddress = "192.168.1.1"; - tlsKeyFile = apiserver_key; - tlsCertFile = apiserver_cert; - clientCaFile = rootCaFile; - kubeletClientCaFile = rootCaFile; - kubeletClientKeyFile = worker_key; - kubeletClientCertFile = worker_cert; - }; - }; -} diff --git a/nixos/tests/kubernetes/multinode-kubectl.nix b/nixos/tests/kubernetes/multinode-kubectl.nix deleted file mode 100644 index c5dd999a01e..00000000000 --- a/nixos/tests/kubernetes/multinode-kubectl.nix +++ /dev/null @@ -1,123 +0,0 @@ -{ system ? builtins.currentSystem }: - -with import ../../lib/testing.nix { inherit system; }; -with import ../../lib/qemu-flags.nix; -with pkgs.lib; - -let - servers.master = "192.168.1.1"; - servers.one = "192.168.1.10"; - servers.two = "192.168.1.20"; - - certs = import ./certs.nix { inherit servers; }; - - kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON { - kind = "Pod"; - apiVersion = "v1"; - metadata.name = "kubectl"; - metadata.labels.name = "kubectl"; - spec.containers = [{ - name = "kubectl"; - image = "kubectl:latest"; - command = ["${pkgs.busybox}/bin/tail" "-f"]; - imagePullPolicy = "Never"; - tty = true; - }]; - }); - - kubectlImage = pkgs.dockerTools.buildImage { - name = "kubectl"; - tag = "latest"; - contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ]; - config.Entrypoint = "${pkgs.busybox}/bin/sh"; - }; - - kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON { - apiVersion = "v1"; - kind = "Config"; - clusters = [{ - name = "local"; - cluster.certificate-authority = "/ca.pem"; - cluster.server = "https://${servers.master}"; - }]; - users = [{ - name = "kubelet"; - user = { - client-certificate = "/admin.crt"; - client-key = "/admin-key.pem"; - }; - }]; - contexts = [{ - context = { - cluster = "local"; - user = "kubelet"; - }; - current-context = "kubelet-context"; - }]; - }); - - test = '' - $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); - $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); - $master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready"); - - $one->execute("docker load < ${kubectlImage}"); - $two->execute("docker load < ${kubectlImage}"); - - $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}"); - - $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running"); - - $master->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version"); - ''; - -in makeTest { - name = "kubernetes-multinode-kubectl"; - - nodes = { - master = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.master; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - (import ./kubernetes-master.nix { inherit pkgs config certs; }) - ]; - - one = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.one; - services.kubernetes.roles = ["node"]; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - ]; - - two = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.two; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.two; - services.kubernetes.roles = ["node"]; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - ]; - }; - - testScript = '' - startAll; - - ${test} - ''; -} diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix index dfb55e7e058..1966fed3a5f 100644 --- a/nixos/tests/kubernetes/rbac.nix +++ b/nixos/tests/kubernetes/rbac.nix @@ -1,14 +1,6 @@ -{ system ? builtins.currentSystem }: - -with import ../../lib/testing.nix { inherit system; }; -with import ../../lib/qemu-flags.nix; -with pkgs.lib; - +{ system ? builtins.currentSystem, pkgs ? import { inherit system; } }: +with import ./base.nix { inherit system; }; let - servers.master = "192.168.1.1"; - servers.one = "192.168.1.10"; - - certs = import ./certs.nix { inherit servers; }; roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON { kind = "ServiceAccount"; @@ -20,21 +12,21 @@ let }); roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON { - "apiVersion" = "rbac.authorization.k8s.io/v1beta1"; - "kind" = "RoleBinding"; - "metadata" = { - "name" = "read-pods"; - "namespace" = "default"; + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "RoleBinding"; + metadata = { + name = "read-pods"; + namespace = "default"; }; - "roleRef" = { - "apiGroup" = "rbac.authorization.k8s.io"; - "kind" = "Role"; - "name" = "pod-reader"; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "Role"; + name = "pod-reader"; }; - "subjects" = [{ - "kind" = "ServiceAccount"; - "name" = "read-only"; - "namespace" = "default"; + subjects = [{ + kind = "ServiceAccount"; + name = "read-only"; + namespace = "default"; }]; }); @@ -62,7 +54,7 @@ let spec.containers = [{ name = "kubectl"; image = "kubectl:latest"; - command = ["${pkgs.busybox}/bin/tail" "-f"]; + command = ["/bin/tail" "-f"]; imagePullPolicy = "Never"; tty = true; }]; @@ -78,71 +70,68 @@ let spec.containers = [{ name = "kubectl-2"; image = "kubectl:latest"; - command = ["${pkgs.busybox}/bin/tail" "-f"]; + command = ["/bin/tail" "-f"]; imagePullPolicy = "Never"; tty = true; }]; }); + kubectl = pkgs.runCommand "copy-kubectl" { buildInputs = [ pkgs.kubernetes ]; } '' + mkdir -p $out/bin + cp ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl + ''; + kubectlImage = pkgs.dockerTools.buildImage { name = "kubectl"; tag = "latest"; - contents = [ pkgs.kubernetes pkgs.busybox kubectlPod2 ]; # certs kubeconfig - config.Entrypoint = "${pkgs.busybox}/bin/sh"; + contents = [ kubectl pkgs.busybox kubectlPod2 ]; + config.Entrypoint = "/bin/sh"; }; - test = '' - $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); - $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); - - $one->execute("docker load < ${kubectlImage}"); - - $master->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}"); - $master->waitUntilSucceeds("kubectl apply -f ${roRole}"); - $master->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}"); - $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}"); - - $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running"); - - $master->succeed("kubectl exec -ti kubectl -- kubectl get pods"); - $master->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json"); - $master->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl"); - ''; - -in makeTest { - name = "kubernetes-rbac"; - - nodes = { - master = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.master; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - (import ./kubernetes-master.nix { inherit pkgs config certs; }) - ]; - - one = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.one; - services.kubernetes.roles = ["node"]; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - ]; + base = { + name = "rbac"; }; - testScript = '' - startAll; + singlenode = base // { + test = '' + $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready"); - ${test} - ''; + $machine1->execute("docker load < ${kubectlImage}"); + + $machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}"); + $machine1->waitUntilSucceeds("kubectl apply -f ${roRole}"); + $machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}"); + $machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}"); + + $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running"); + + $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods"); + $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json"); + $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl"); + ''; + }; + + multinode = base // { + test = '' + $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready"); + $machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready"); + + $machine2->execute("docker load < ${kubectlImage}"); + + $machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}"); + $machine1->waitUntilSucceeds("kubectl apply -f ${roRole}"); + $machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}"); + $machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}"); + + $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running"); + + $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods"); + $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json"); + $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl"); + ''; + }; + +in { + singlenode = mkKubernetesSingleNodeTest singlenode; + multinode = mkKubernetesMultiNodeTest multinode; } diff --git a/nixos/tests/kubernetes/singlenode.nix b/nixos/tests/kubernetes/singlenode.nix deleted file mode 100644 index d924da155bd..00000000000 --- a/nixos/tests/kubernetes/singlenode.nix +++ /dev/null @@ -1,75 +0,0 @@ -{ system ? builtins.currentSystem }: - -with import ../../lib/testing.nix { inherit system; }; -with import ../../lib/qemu-flags.nix; -with pkgs.lib; - -let - redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON { - kind = "Pod"; - apiVersion = "v1"; - metadata.name = "redis"; - metadata.labels.name = "redis"; - spec.containers = [{ - name = "redis"; - image = "redis"; - args = ["--bind" "0.0.0.0"]; - imagePullPolicy = "Never"; - ports = [{ - name = "redis-server"; - containerPort = 6379; - }]; - }]; - }); - - redisService = pkgs.writeText "redis-service.json" (builtins.toJSON { - kind = "Service"; - apiVersion = "v1"; - metadata.name = "redis"; - spec = { - ports = [{port = 6379; targetPort = 6379;}]; - selector = {name = "redis";}; - }; - }); - - redisImage = pkgs.dockerTools.buildImage { - name = "redis"; - tag = "latest"; - contents = pkgs.redis; - config.Entrypoint = "/bin/redis-server"; - }; - - testSimplePod = '' - $kubernetes->execute("docker load < ${redisImage}"); - $kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}"); - $kubernetes->succeed("kubectl create -f ${redisService}"); - $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running"); - $kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379"); - ''; -in makeTest { - name = "kubernetes-singlenode"; - - nodes = { - kubernetes = - { config, pkgs, lib, nodes, ... }: - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 2048; - - programs.bash.enableCompletion = true; - environment.systemPackages = with pkgs; [ netcat bind ]; - - services.kubernetes.roles = ["master" "node"]; - services.kubernetes.dns.port = 4453; - - services.dnsmasq.enable = true; - services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; - }; - }; - - testScript = '' - startAll; - - ${testSimplePod} - ''; -}