b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

kubernetes: fix tests

Browse source
This commit is contained in:
Matej Cotman 2017-09-09 02:00:35 +02:00 committed by Robin Gloster
parent 1febe07de7
commit 6ef8cad2a7
10 changed files with 538 additions and 690 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
nixos/tests/kubernetes/base.nix Normal file
View file

@ -0,0 +1,113 @@
{ system ? builtins.currentSystem }:
with import ../../lib/testing.nix { inherit system; };
with import ../../lib/qemu-flags.nix;
with pkgs.lib;
let
mkKubernetesBaseTest =
{ name, domain ? "my.zyx", test, machines
, pkgs ? import <nixpkgs> { inherit system; }
, certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; }
, extraConfiguration ? null }:
let
masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
master = machines.${masterName};
extraHosts = ''
${master.ip} etcd.${domain}
${master.ip} api.${domain}
${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
'';
in makeTest {
inherit name;
nodes = mapAttrs (machineName: machine:
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = mkDefault 768;
virtualisation.diskSize = mkDefault 4096;
networking = {
inherit domain extraHosts;
primaryIPAddress = mkForce machine.ip;
firewall = {
allowedTCPPorts = [
10250 # kubelet
];
trustedInterfaces = ["docker0"];
extraCommands = concatMapStrings (node: ''
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
'') (attrValues nodes);
};
};
programs.bash.enableCompletion = true;
environment.variables = {
ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";
ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";
ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";
ETCDCTL_PEERS = "https://etcd.${domain}:2379";
};
services.flannel.iface = "eth1";
services.kubernetes.apiserver.advertiseAddress = master.ip;
}
(optionalAttrs (any (role: role == "master") machine.roles) {
networking.firewall.allowedTCPPorts = [
2379 2380 # etcd
443 # kubernetes apiserver
];
services.etcd = {
enable = true;
certFile = "${certs.master}/etcd.pem";
keyFile = "${certs.master}/etcd-key.pem";
trustedCaFile = "${certs.master}/ca.pem";
peerClientCertAuth = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];
initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
};
})
(import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })
(optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
(optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
]
) machines;
testScript = ''
startAll;
${test}
'';
};
mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
machines = {
machine1 = {
roles = ["master"];
ip = "192.168.1.1";
};
machine2 = {
roles = ["node"];
ip = "192.168.1.2";
};
};
} // attrs // {
name = "kubernetes-${attrs.name}-multinode";
});
mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
machines = {
machine1 = {
roles = ["master" "node"];
ip = "192.168.1.1";
};
};
} // attrs // {
name = "kubernetes-${attrs.name}-singlenode";
});
in {
inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
}

400
nixos/tests/kubernetes/certs.nix
View file

@ -1,229 +1,185 @@
{
pkgs ? import <nixpkgs> {},
servers ? {test = "1.2.3.4";},
internalDomain ? "cluster.local",
externalDomain ? "nixos.xyz"
internalDomain ? "cloud.yourdomain.net",
externalDomain ? "myawesomecluster.cluster.yourdomain.net",
serviceClusterIp ? "10.0.0.1"
}:
let
mkAltNames = ipFrom: dnsFrom:
pkgs.lib.concatImapStringsSep "\n" (i: v: "IP.${toString (i+ipFrom)} = ${v.ip}\nDNS.${toString (i+dnsFrom)} = ${v.name}.${externalDomain}") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);
runWithCFSSL = name: cmd:
builtins.fromJSON (builtins.readFile (
pkgs.runCommand "${name}-cfss.json" {
buildInputs = [ pkgs.cfssl ];
} "cfssl ${cmd} > $out"
));
runWithOpenSSL = file: cmd: pkgs.runCommand file {
buildInputs = [ pkgs.openssl ];
passthru = { inherit file; };
} cmd;
ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
ca_pem = runWithOpenSSL "ca.pem" ''
openssl req \
-x509 -new -nodes -key ${ca_key} \
-days 10000 -out $out -subj "/CN=etcd-ca"
'';
etcd_cnf = pkgs.writeText "openssl.cnf" ''
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = etcd.kubernetes.${externalDomain}
IP.1 = 127.0.0.1
'';
etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
etcd_csr = runWithOpenSSL "etcd.csr" ''
openssl req \
-new -key ${etcd_key} \
-out $out -subj "/CN=etcd" \
-config ${etcd_cnf}
'';
etcd_cert = runWithOpenSSL "etcd.pem" ''
openssl x509 \
-req -in ${etcd_csr} \
-CA ${ca_pem} -CAkey ${ca_key} \
-CAcreateserial -out $out \
-days 3650 -extensions v3_req \
-extfile ${etcd_cnf}
'';
etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
"openssl genrsa -out $out 2048";
etcd_client_csr = runWithOpenSSL "etcd-client.csr" ''
openssl req \
-new -key ${etcd_client_key} \
-out $out -subj "/CN=etcd-client" \
-config ${client_openssl_cnf}
'';
etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
openssl x509 \
-req -in ${etcd_client_csr} \
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-out $out -days 3650 -extensions v3_req \
-extfile ${client_openssl_cnf}
'';
admin_key = runWithOpenSSL "admin-key.pem"
"openssl genrsa -out $out 2048";
admin_csr = runWithOpenSSL "admin.csr" ''
openssl req \
-new -key ${admin_key} \
-out $out -subj "/CN=admin/O=system:masters" \
-config ${client_openssl_cnf}
'';
admin_cert = runWithOpenSSL "admin.crt" ''
openssl x509 \
-req -in ${admin_csr} \
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-out $out -days 3650 -extensions v3_req \
-extfile ${client_openssl_cnf}
'';
apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048";
apiserver_csr = runWithOpenSSL "apiserver.csr" ''
openssl req \
-new -key ${apiserver_key} \
-out $out -subj "/CN=kube-apiserver" \
-config ${apiserver_cnf}
'';
apiserver_cert = runWithOpenSSL "apiserver.pem" ''
openssl x509 \
-req -in ${apiserver_csr} \
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-out $out -days 3650 -extensions v3_req \
-extfile ${apiserver_cnf}
'';
worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";
worker_csr = runWithOpenSSL "worker.csr" ''
openssl req \
-new -key ${worker_key} \
-out $out -subj "/CN=kube-worker/O=system:authenticated" \
-config ${worker_cnf}
'';
worker_cert = runWithOpenSSL "worker.pem" ''
openssl x509 \
-req -in ${worker_csr} \
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-out $out -days 3650 -extensions v3_req \
-extfile ${worker_cnf}
'';
openssl_cnf = pkgs.writeText "openssl.cnf" ''
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = *.cluster.${externalDomain}
IP.1 = 127.0.0.1
${mkAltNames 1 1}
'';
client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.${internalDomain}
DNS.5 = kubernetes.${externalDomain}
DNS.6 = *.cluster.${externalDomain}
IP.1 = 10.1.10.1
${mkAltNames 1 6}
'';
apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.${internalDomain}
DNS.5 = kubernetes.${externalDomain}
DNS.6 = *.cluster.${externalDomain}
IP.1 = 10.1.10.1
${mkAltNames 1 6}
'';
worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = *.cluster.${externalDomain}
IP.1 = 10.1.10.1
${mkAltNames 1 1}
'';
ln = cert: target: ''
cp -v ${cert} ${target}/${cert.file}
'';
in
pkgs.stdenv.mkDerivation rec {
name = "kubernetes-certs";
unpackPhase = "true";
installPhase = ''
set -xe
writeCFSSL = content:
pkgs.runCommand content.name {
buildInputs = [ pkgs.cfssl ];
} ''
mkdir -p $out
${ln ca_key "$out"}
${ln ca_pem "$out"}
${ln etcd_key "$out"}
${ln etcd_csr "$out"}
${ln etcd_cert "$out"}
${ln etcd_client_key "$out"}
${ln etcd_client_csr "$out"}
${ln etcd_client_cert "$out"}
${ln apiserver_key "$out"}
${ln apiserver_csr "$out"}
${ln apiserver_cert "$out"}
${ln worker_key "$out"}
${ln worker_csr "$out"}
${ln worker_cert "$out"}
${ln admin_key "$out"}
${ln admin_csr "$out"}
${ln admin_cert "$out"}
cd $out
cat ${writeFile content} | cfssljson -bare ${content.name}
'';
}
noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
writeFile = content: pkgs.writeText "content" (
if pkgs.lib.isAttrs content then builtins.toJSON content
else toString content
);
createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
noCSR (
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile {
CN = cn;
hosts = hosts;
key = { algo = "rsa"; inherit size; };
}}") // { inherit name; }
);
createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }:
noCSR (
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile {
CN = cn;
names = map (group: {O = group;}) groups;
hosts = [""];
key = { algo = "rsa"; inherit size; };
}}") // { inherit name; }
);
createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }:
(noCSR (runWithCFSSL CN "genkey -initca ${writeFile {
key = { algo = "rsa"; inherit size; };
names = [{ inherit C ST L O OU CN emailAddress; }];
}}")) // {
inherit name;
config.signing = {
default.expiry = expiry;
profiles = {
server = {
inherit expiry;
usages = [
"signing"
"key encipherment"
"server auth"
];
};
client = {
inherit expiry;
usages = [
"signing"
"key encipherment"
"client auth"
];
};
peer = {
inherit expiry;
usages = [
"signing"
"key encipherment"
"server auth"
"client auth"
];
};
};
};
};
ca = createSigningCertKey {};
kube-apiserver = createServingCertKey {
inherit ca;
cn = "kube-apiserver";
hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp];
};
kubelet = createServingCertKey {
inherit ca;
cn = "kubelet";
hosts = ["*.${externalDomain}"];
};
service-accounts = createServingCertKey {
inherit ca;
cn = "kube-service-accounts";
};
etcd = createServingCertKey {
inherit ca;
cn = "etcd";
hosts = ["etcd.${externalDomain}"];
};
etcd-client = createClientCertKey {
inherit ca;
cn = "etcd-client";
};
kubelet-client = createClientCertKey {
inherit ca;
cn = "kubelet-client";
groups = ["system:masters"];
};
apiserver-client = {
kubelet = createClientCertKey {
inherit ca;
cn = "apiserver-client-kubelet";
groups = ["system:nodes"];
};
kube-proxy = createClientCertKey {
inherit ca;
name = "apiserver-client-kube-proxy";
cn = "system:kube-proxy";
groups = ["system:kube-proxy" "system:nodes"];
};
kube-controller-manager = createClientCertKey {
inherit ca;
name = "apiserver-client-kube-controller-manager";
cn = "system:kube-controller-manager";
groups = ["system:masters"];
};
kube-scheduler = createClientCertKey {
inherit ca;
name = "apiserver-client-kube-scheduler";
cn = "system:kube-scheduler";
groups = ["system:kube-scheduler"];
};
admin = createClientCertKey {
inherit ca;
cn = "admin";
groups = ["system:masters"];
};
};
in {
master = pkgs.buildEnv {
name = "master-keys";
paths = [
(writeCFSSL (noKey ca))
(writeCFSSL kube-apiserver)
(writeCFSSL kubelet-client)
(writeCFSSL apiserver-client.kube-controller-manager)
(writeCFSSL apiserver-client.kube-scheduler)
(writeCFSSL service-accounts)
(writeCFSSL etcd)
];
};
worker = pkgs.buildEnv {
name = "worker-keys";
paths = [
(writeCFSSL (noKey ca))
(writeCFSSL kubelet)
(writeCFSSL apiserver-client.kubelet)
(writeCFSSL apiserver-client.kube-proxy)
(writeCFSSL etcd-client)
];
};
admin = writeCFSSL apiserver-client.admin;
}

8
nixos/tests/kubernetes/default.nix
View file

@ -1,7 +1,7 @@
{ system ? builtins.currentSystem }:
{
kubernetes-singlenode = import ./singlenode.nix { inherit system; };
kubernetes-multinode-kubectl = import ./multinode-kubectl.nix { inherit system; };
kubernetes-rbac = import ./rbac.nix { inherit system; };
kubernetes-dns = import ./dns.nix { inherit system; };
dns = import ./dns.nix { inherit system; };
# e2e = import ./e2e.nix { inherit system; }; # TODO: make it pass
# the following test(s) can be removed when e2e is working:
rbac = import ./rbac.nix { inherit system; };
}

150
nixos/tests/kubernetes/dns.nix
View file

@ -1,16 +1,11 @@
{ system ? builtins.currentSystem }:
with import ../../lib/testing.nix { inherit system; };
with import ../../lib/qemu-flags.nix;
with pkgs.lib;
{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
with import ./base.nix { inherit system; };
let
servers.master = "192.168.1.1";
servers.one = "192.168.1.10";
domain = "my.zyx";
certs = import ./certs.nix { inherit servers; };
certs = import ./certs.nix { externalDomain = domain; };
redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {
kind = "Pod";
apiVersion = "v1";
metadata.name = "redis";
@ -40,64 +35,93 @@ let
redisImage = pkgs.dockerTools.buildImage {
name = "redis";
tag = "latest";
contents = [ pkgs.redis pkgs.bind.dnsutils pkgs.coreutils pkgs.inetutils pkgs.nmap ];
contents = [ pkgs.redis pkgs.bind.host ];
config.Entrypoint = "/bin/redis-server";
};
test = ''
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
kind = "Pod";
apiVersion = "v1";
metadata.name = "probe";
metadata.labels.name = "probe";
spec.containers = [{
name = "probe";
image = "probe";
args = [ "-f" ];
tty = true;
imagePullPolicy = "Never";
}];
});
$one->execute("docker load < ${redisImage}");
$master->waitUntilSucceeds("kubectl create -f ${redisPod} || kubectl apply -f ${redisPod}");
$master->waitUntilSucceeds("kubectl create -f ${redisService} || kubectl apply -f ${redisService}");
$master->waitUntilSucceeds("kubectl get pod redis | grep Running");
$master->succeed("dig \@192.168.1.1 redis.default.svc.cluster.local");
$one->succeed("dig \@192.168.1.10 redis.default.svc.cluster.local");
$master->succeed("kubectl exec -ti redis -- cat /etc/resolv.conf | grep 'nameserver 192.168.1.10'");
$master->succeed("kubectl exec -ti redis -- dig \@192.168.1.10 redis.default.svc.cluster.local");
'';
in makeTest {
name = "kubernetes-dns";
nodes = {
master =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.master;
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
(import ./kubernetes-master.nix { inherit pkgs config certs; })
];
one =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.one;
services.kubernetes.roles = ["node"];
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
];
probeImage = pkgs.dockerTools.buildImage {
name = "probe";
tag = "latest";
contents = [ pkgs.bind.host pkgs.busybox ];
config.Entrypoint = "/bin/tail";
};
testScript = ''
startAll;
extraConfiguration = { config, pkgs, lib, nodes, ... }: {
environment.systemPackages = [ pkgs.bind.host ];
# virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}";
services.dnsmasq.enable = true;
services.dnsmasq.servers = [
"/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
];
};
${test}
'';
base = {
name = "dns";
inherit domain certs extraConfiguration;
};
singleNodeTest = {
test = ''
# prepare machine1 for test
$machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
$machine1->execute("docker load < ${redisImage}");
$machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
$machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
$machine1->execute("docker load < ${probeImage}");
$machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
# check if pods are running
$machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
$machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
$machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
# check dns on host (dnsmasq)
$machine1->succeed("host redis.default.svc.cluster.local");
# check dns inside the container
$machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
'';
};
multiNodeTest = {
test = ''
# prepare machines for test
$machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
$machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
$machine2->execute("docker load < ${redisImage}");
$machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
$machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
$machine2->execute("docker load < ${probeImage}");
$machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
# check if pods are running
$machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
$machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
$machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
# check dns on hosts (dnsmasq)
$machine1->succeed("host redis.default.svc.cluster.local");
$machine2->succeed("host redis.default.svc.cluster.local");
# check dns inside the container
$machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
'';
};
in {
singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
}

40
nixos/tests/kubernetes/e2e.nix Normal file
View file

@ -0,0 +1,40 @@
{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
with import ./base.nix { inherit system; };
let
domain = "my.zyx";
certs = import ./certs.nix { externalDomain = domain; };
kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON {
apiVersion = "v1";
kind = "Config";
clusters = [{
name = "local";
cluster.certificate-authority = "${certs.master}/ca.pem";
cluster.server = "https://api.${domain}";
}];
users = [{
name = "kubelet";
user = {
client-certificate = "${certs.admin}/admin.pem";
client-key = "${certs.admin}/admin-key.pem";
};
}];
contexts = [{
context = {
cluster = "local";
user = "kubelet";
};
current-context = "kubelet-context";
}];
});
base = {
name = "e2e";
inherit domain certs;
test = ''
$machine1->succeed("e2e.test -kubeconfig ${kubeconfig} -provider local -ginkgo.focus '\\[Conformance\\]' -ginkgo.skip '\\[Flaky\\]|\\[Serial\\]'");
'';
};
in {
singlenode = mkKubernetesSingleNodeTest base;
multinode = mkKubernetesMultiNodeTest base;
}

111
nixos/tests/kubernetes/kubernetes-common.nix
View file

@ -1,72 +1,59 @@
{ config, pkgs, certs, servers }:
{ roles, config, pkgs, certs }:
with pkgs.lib;
let
etcd_key = "${certs}/etcd-key.pem";
etcd_cert = "${certs}/etcd.pem";
ca_pem = "${certs}/ca.pem";
etcd_client_cert = "${certs}/etcd-client.crt";
etcd_client_key = "${certs}/etcd-client-key.pem";
base = {
inherit roles;
featureGates = ["AllAlpha"];
flannel.enable = true;
addons.dashboard.enable = true;
verbose = true;
worker_key = "${certs}/worker-key.pem";
worker_cert = "${certs}/worker.pem";
rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
${pkgs.lib.readFile "${certs}/ca.pem"}
${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}
'';
mkHosts =
pkgs.lib.concatMapStringsSep "\n" (v: "${v.ip} ${v.name}.nixos.xyz") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);
in
{
programs.bash.enableCompletion = true;
environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];
networking = {
firewall.allowedTCPPorts = [
10250 # kubelet
];
extraHosts = ''
# register "external" domains
${servers.master} etcd.kubernetes.nixos.xyz
${servers.master} kubernetes.nixos.xyz
${mkHosts}
'';
};
services.flannel.iface = "eth1";
environment.variables = {
ETCDCTL_CERT_FILE = "${etcd_client_cert}";
ETCDCTL_KEY_FILE = "${etcd_client_key}";
ETCDCTL_CA_FILE = "${rootCaFile}";
ETCDCTL_PEERS = "https://etcd.kubernetes.nixos.xyz:2379";
};
services.kubernetes = {
kubelet = {
tlsKeyFile = worker_key;
tlsCertFile = worker_cert;
hostname = "${config.networking.hostName}.nixos.xyz";
nodeIp = config.networking.primaryIPAddress;
caFile = "${certs.master}/ca.pem";
apiserver = {
tlsCertFile = "${certs.master}/kube-apiserver.pem";
tlsKeyFile = "${certs.master}/kube-apiserver-key.pem";
kubeletClientCertFile = "${certs.master}/kubelet-client.pem";
kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem";
serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem";
};
etcd = {
servers = ["https://etcd.kubernetes.nixos.xyz:2379"];
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
servers = ["https://etcd.${config.networking.domain}:2379"];
certFile = "${certs.worker}/etcd-client.pem";
keyFile = "${certs.worker}/etcd-client-key.pem";
};
kubeconfig = {
server = "https://kubernetes.nixos.xyz";
caFile = rootCaFile;
certFile = worker_cert;
keyFile = worker_key;
server = "https://api.${config.networking.domain}";
};
kubelet = {
tlsCertFile = "${certs.worker}/kubelet.pem";
tlsKeyFile = "${certs.worker}/kubelet-key.pem";
hostname = "${config.networking.hostName}.${config.networking.domain}";
kubeconfig = {
certFile = "${certs.worker}/apiserver-client-kubelet.pem";
keyFile = "${certs.worker}/apiserver-client-kubelet-key.pem";
};
};
controllerManager = {
serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem";
kubeconfig = {
certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem";
keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem";
};
};
scheduler = {
kubeconfig = {
certFile = "${certs.master}/apiserver-client-kube-scheduler.pem";
keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem";
};
};
proxy = {
kubeconfig = {
certFile = "${certs.worker}/apiserver-client-kube-proxy.pem";
keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem";
};
};
flannel.enable = true;
dns.port = 4453;
};
services.dnsmasq.enable = true;
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
in {
services.kubernetes = base;
}

63
nixos/tests/kubernetes/kubernetes-master.nix
View file

@ -1,63 +0,0 @@
{ config, pkgs, certs }:
let
etcd_key = "${certs}/etcd-key.pem";
etcd_cert = "${certs}/etcd.pem";
ca_pem = "${certs}/ca.pem";
etcd_client_cert = "${certs}/etcd-client.crt";
etcd_client_key = "${certs}/etcd-client-key.pem";
apiserver_key = "${certs}/apiserver-key.pem";
apiserver_cert = "${certs}/apiserver.pem";
worker_key = "${certs}/worker-key.pem";
worker_cert = "${certs}/worker.pem";
rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
${pkgs.lib.readFile "${certs}/ca.pem"}
${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}
'';
in
{
networking = {
firewall = {
enable = true;
allowPing = true;
allowedTCPPorts = [
2379 2380 # etcd
443 # kubernetes apiserver
];
};
};
services.etcd = {
enable = pkgs.lib.mkForce true;
keyFile = etcd_key;
certFile = etcd_cert;
trustedCaFile = rootCaFile;
peerClientCertAuth = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
advertiseClientUrls = ["https://etcd.kubernetes.nixos.xyz:2379"];
initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"];
initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"];
};
services.kubernetes = {
roles = ["master"];
scheduler.leaderElect = true;
controllerManager.rootCaFile = rootCaFile;
controllerManager.serviceAccountKeyFile = apiserver_key;
apiserver = {
publicAddress = "192.168.1.1";
advertiseAddress = "192.168.1.1";
tlsKeyFile = apiserver_key;
tlsCertFile = apiserver_cert;
clientCaFile = rootCaFile;
kubeletClientCaFile = rootCaFile;
kubeletClientKeyFile = worker_key;
kubeletClientCertFile = worker_cert;
};
};
}

123
nixos/tests/kubernetes/multinode-kubectl.nix
View file

@ -1,123 +0,0 @@
{ system ? builtins.currentSystem }:
with import ../../lib/testing.nix { inherit system; };
with import ../../lib/qemu-flags.nix;
with pkgs.lib;
let
servers.master = "192.168.1.1";
servers.one = "192.168.1.10";
servers.two = "192.168.1.20";
certs = import ./certs.nix { inherit servers; };
kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON {
kind = "Pod";
apiVersion = "v1";
metadata.name = "kubectl";
metadata.labels.name = "kubectl";
spec.containers = [{
name = "kubectl";
image = "kubectl:latest";
command = ["${pkgs.busybox}/bin/tail" "-f"];
imagePullPolicy = "Never";
tty = true;
}];
});
kubectlImage = pkgs.dockerTools.buildImage {
name = "kubectl";
tag = "latest";
contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ];
config.Entrypoint = "${pkgs.busybox}/bin/sh";
};
kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON {
apiVersion = "v1";
kind = "Config";
clusters = [{
name = "local";
cluster.certificate-authority = "/ca.pem";
cluster.server = "https://${servers.master}";
}];
users = [{
name = "kubelet";
user = {
client-certificate = "/admin.crt";
client-key = "/admin-key.pem";
};
}];
contexts = [{
context = {
cluster = "local";
user = "kubelet";
};
current-context = "kubelet-context";
}];
});
test = ''
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
$master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");
$one->execute("docker load < ${kubectlImage}");
$two->execute("docker load < ${kubectlImage}");
$master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
$master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
$master->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version");
'';
in makeTest {
name = "kubernetes-multinode-kubectl";
nodes = {
master =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.master;
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
(import ./kubernetes-master.nix { inherit pkgs config certs; })
];
one =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.one;
services.kubernetes.roles = ["node"];
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
];
two =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.two; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.two;
services.kubernetes.roles = ["node"];
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
];
};
testScript = ''
startAll;
${test}
'';
}

145
nixos/tests/kubernetes/rbac.nix
View file

@ -1,14 +1,6 @@
{ system ? builtins.currentSystem }:
with import ../../lib/testing.nix { inherit system; };
with import ../../lib/qemu-flags.nix;
with pkgs.lib;
{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
with import ./base.nix { inherit system; };
let
servers.master = "192.168.1.1";
servers.one = "192.168.1.10";
certs = import ./certs.nix { inherit servers; };
roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON {
kind = "ServiceAccount";
@ -20,21 +12,21 @@ let
});
roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {
"apiVersion" = "rbac.authorization.k8s.io/v1beta1";
"kind" = "RoleBinding";
"metadata" = {
"name" = "read-pods";
"namespace" = "default";
apiVersion = "rbac.authorization.k8s.io/v1beta1";
kind = "RoleBinding";
metadata = {
name = "read-pods";
namespace = "default";
};
"roleRef" = {
"apiGroup" = "rbac.authorization.k8s.io";
"kind" = "Role";
"name" = "pod-reader";
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "Role";
name = "pod-reader";
};
"subjects" = [{
"kind" = "ServiceAccount";
"name" = "read-only";
"namespace" = "default";
subjects = [{
kind = "ServiceAccount";
name = "read-only";
namespace = "default";
}];
});
@ -62,7 +54,7 @@ let
spec.containers = [{
name = "kubectl";
image = "kubectl:latest";
command = ["${pkgs.busybox}/bin/tail" "-f"];
command = ["/bin/tail" "-f"];
imagePullPolicy = "Never";
tty = true;
}];
@ -78,71 +70,68 @@ let
spec.containers = [{
name = "kubectl-2";
image = "kubectl:latest";
command = ["${pkgs.busybox}/bin/tail" "-f"];
command = ["/bin/tail" "-f"];
imagePullPolicy = "Never";
tty = true;
}];
});
kubectl = pkgs.runCommand "copy-kubectl" { buildInputs = [ pkgs.kubernetes ]; } ''
mkdir -p $out/bin
cp ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl
'';
kubectlImage = pkgs.dockerTools.buildImage {
name = "kubectl";
tag = "latest";
contents = [ pkgs.kubernetes pkgs.busybox kubectlPod2 ]; # certs kubeconfig
config.Entrypoint = "${pkgs.busybox}/bin/sh";
contents = [ kubectl pkgs.busybox kubectlPod2 ];
config.Entrypoint = "/bin/sh";
};
test = ''
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
$one->execute("docker load < ${kubectlImage}");
$master->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
$master->waitUntilSucceeds("kubectl apply -f ${roRole}");
$master->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
$master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
$master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
$master->succeed("kubectl exec -ti kubectl -- kubectl get pods");
$master->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
$master->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
'';
in makeTest {
name = "kubernetes-rbac";
nodes = {
master =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.master;
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
(import ./kubernetes-master.nix { inherit pkgs config certs; })
];
one =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.one;
services.kubernetes.roles = ["node"];
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
];
base = {
name = "rbac";
};
testScript = ''
startAll;
singlenode = base // {
test = ''
$machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
${test}
'';
$machine1->execute("docker load < ${kubectlImage}");
$machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
$machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");
$machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
$machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");
$machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
$machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
$machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
$machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
'';
};
multinode = base // {
test = ''
$machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
$machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");
$machine2->execute("docker load < ${kubectlImage}");
$machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
$machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");
$machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
$machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");
$machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
$machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
$machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
$machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
'';
};
in {
singlenode = mkKubernetesSingleNodeTest singlenode;
multinode = mkKubernetesMultiNodeTest multinode;
}

75
nixos/tests/kubernetes/singlenode.nix
View file

@ -1,75 +0,0 @@
{ system ? builtins.currentSystem }:
with import ../../lib/testing.nix { inherit system; };
with import ../../lib/qemu-flags.nix;
with pkgs.lib;
let
redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
kind = "Pod";
apiVersion = "v1";
metadata.name = "redis";
metadata.labels.name = "redis";
spec.containers = [{
name = "redis";
image = "redis";
args = ["--bind" "0.0.0.0"];
imagePullPolicy = "Never";
ports = [{
name = "redis-server";
containerPort = 6379;
}];
}];
});
redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
kind = "Service";
apiVersion = "v1";
metadata.name = "redis";
spec = {
ports = [{port = 6379; targetPort = 6379;}];
selector = {name = "redis";};
};
});
redisImage = pkgs.dockerTools.buildImage {
name = "redis";
tag = "latest";
contents = pkgs.redis;
config.Entrypoint = "/bin/redis-server";
};
testSimplePod = ''
$kubernetes->execute("docker load < ${redisImage}");
$kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}");
$kubernetes->succeed("kubectl create -f ${redisService}");
$kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
$kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379");
'';
in makeTest {
name = "kubernetes-singlenode";
nodes = {
kubernetes =
{ config, pkgs, lib, nodes, ... }:
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 2048;
programs.bash.enableCompletion = true;
environment.systemPackages = with pkgs; [ netcat bind ];
services.kubernetes.roles = ["master" "node"];
services.kubernetes.dns.port = 4453;
services.dnsmasq.enable = true;
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
};
};
testScript = ''
startAll;
${testSimplePod}
'';
}