kubernetes module: refactor module system, kube-dns as module

This commit is contained in:
Jaka Hudoklin 2017-09-01 12:31:56 +02:00 committed by Robin Gloster
parent 2beadcf181
commit ddf5de5de0
7 changed files with 478 additions and 239 deletions

View file

@ -158,6 +158,8 @@
./services/backup/znapzend.nix
./services/cluster/fleet.nix
./services/cluster/kubernetes/default.nix
./services/cluster/kubernetes/dns.nix
./services/cluster/kubernetes/dashboard.nix
./services/cluster/panamax.nix
./services/computing/boinc/client.nix
./services/computing/torque/server.nix

View file

@ -1,85 +1,160 @@
{ cfg }: {
"dashboard-controller" = {
"apiVersion" = "extensions/v1beta1";
"kind" = "Deployment";
"metadata" = {
"labels" = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kubernetes-dashboard";
"kubernetes.io/cluster-service" = "true";
};
"name" = "kubernetes-dashboard";
"namespace" = "kube-system";
};
"spec" = {
"selector" = {
"matchLabels" = {
"k8s-app" = "kubernetes-dashboard";
};
};
"template" = {
"metadata" = {
"annotations" = {
"scheduler.alpha.kubernetes.io/critical-pod" = "";
};
"labels" = {
"k8s-app" = "kubernetes-dashboard";
};
};
"spec" = {
"containers" = [{
"image" = "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.0";
"livenessProbe" = {
"httpGet" = {
"path" = "/";
"port" = 9090;
};
"initialDelaySeconds" = 30;
"timeoutSeconds" = 30;
};
"name" = "kubernetes-dashboard";
"ports" = [{
"containerPort" = 9090;
}];
"resources" = {
"limits" = {
"cpu" = "100m";
"memory" = "50Mi";
};
"requests" = {
"cpu" = "100m";
"memory" = "50Mi";
};
};
}];
"tolerations" = [{
"key" = "CriticalAddonsOnly";
"operator" = "Exists";
}];
};
};
};
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.kubernetes.addons.dashboard;
name = "gcr.io/google_containers/kubernetes-dashboard-amd64";
version = "v1.6.3";
image = pkgs.dockerTools.pullImage {
imageName = name;
imageTag = version;
sha256 = "1sf54d96nkgic9hir9c6p14gw24ns1k5d5a0r1sg414kjrvic0b4";
};
in {
options.services.kubernetes.addons.dashboard = {
enable = mkEnableOption "kubernetes dashboard addon";
enableRBAC = mkOption {
description = "Whether to enable role based access control is enabled for kubernetes dashboard";
type = types.bool;
default = elem "RBAC" config.services.kubernetes.apiserver.authorizationMode;
};
"dashboard-service" = {
"apiVersion" = "v1";
"kind" = "Service";
"metadata" = {
"labels" = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kubernetes-dashboard";
};
config = mkIf cfg.enable {
services.kubernetes.kubelet.seedDockerImages = [image];
services.kubernetes.addonManager.addons = {
kubernetes-dashboard-deployment = {
kind = "Deployment";
apiVersion = "apps/v1beta1";
metadata = {
labels = {
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
k8s-app = "kubernetes-dashboard";
version = version;
"kubernetes.io/cluster-service" = "true";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
name = "kubernetes-dashboard";
namespace = "kube-system";
};
spec = {
replicas = 1;
revisionHistoryLimit = 10;
selector.matchLabels."k8s-app" = "kubernetes-dashboard";
template = {
metadata = {
labels = {
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
k8s-app = "kubernetes-dashboard";
version = version;
"kubernetes.io/cluster-service" = "true";
};
annotations = {
"scheduler.alpha.kubernetes.io/critical-pod" = "";
#"scheduler.alpha.kubernetes.io/tolerations" = ''[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'';
};
};
"name" = "kubernetes-dashboard";
"namespace" = "kube-system";
};
"spec" = {
"ports" = [{
"port" = 80;
"targetPort" = 9090;
}];
"selector" = {
"k8s-app" = "kubernetes-dashboard";
spec = {
containers = [{
name = "kubernetes-dashboard";
image = "${name}:${version}";
ports = [{
containerPort = 9090;
protocol = "TCP";
}];
resources = {
limits = {
cpu = "100m";
memory = "50Mi";
};
requests = {
cpu = "100m";
memory = "50Mi";
};
};
livenessProbe = {
httpGet = {
path = "/";
port = 9090;
};
initialDelaySeconds = 30;
timeoutSeconds = 30;
};
}];
serviceAccountName = "kubernetes-dashboard";
tolerations = [{
key = "node-role.kubernetes.io/master";
effect = "NoSchedule";
}];
};
};
};
};
};
kubernetes-dashboard-svc = {
apiVersion = "v1";
kind = "Service";
metadata = {
labels = {
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
k8s-app = "kubernetes-dashboard";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/name" = "KubeDashboard";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
name = "kubernetes-dashboard";
namespace = "kube-system";
};
spec = {
ports = [{
port = 80;
targetPort = 9090;
}];
selector.k8s-app = "kubernetes-dashboard";
};
};
kubernetes-dashboard-sa = {
apiVersion = "v1";
kind = "ServiceAccount";
metadata = {
labels = {
k8s-app = "kubernetes-dashboard";
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
name = "kubernetes-dashboard";
namespace = "kube-system";
};
};
} // (optionalAttrs cfg.enableRBAC {
kubernetes-dashboard-crb = {
apiVersion = "rbac.authorization.k8s.io/v1beta1";
kind = "ClusterRoleBinding";
metadata = {
name = "kubernetes-dashboard";
labels = {
k8s-app = "kubernetes-dashboard";
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
};
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "ClusterRole";
name = "cluster-admin";
};
subjects = [{
kind = "ServiceAccount";
name = "kubernetes-dashboard";
namespace = "kube-system";
}];
};
});
};
}

View file

@ -587,7 +587,7 @@ in {
clusterDomain = mkOption {
description = "Use alternative domain.";
default = "cluster.local";
default = config.services.kubernetes.addons.dns.clusterDomain;
type = types.str;
};
@ -705,16 +705,10 @@ in {
type = types.bool;
};
versionTag = mkOption {
description = "Version tag of Kubernetes addon manager image.";
default = "v6.4-beta.1";
type = types.str;
};
addons = mkOption {
description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
default = { };
type = types.attrsOf types.attrs;
type = types.attrsOf (types.either types.attrs (types.listOf types.attrs));
example = literalExample ''
{
"my-service" = {
@ -732,30 +726,6 @@ in {
};
};
dns = {
enable = mkEnableOption "Kubernetes DNS service.";
port = mkOption {
description = "Kubernetes DNS listening port.";
default = 53;
type = types.int;
};
domain = mkOption {
description = "Kubernetes DNS domain under which to create names.";
default = cfg.kubelet.clusterDomain;
type = types.str;
};
kubeconfig = mkKubeConfigOptions "Kubernetes dns";
extraOpts = mkOption {
description = "Kubernetes DNS extra command line options.";
default = "";
type = types.str;
};
};
path = mkOption {
description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
type = types.listOf types.package;
@ -1052,8 +1022,8 @@ in {
services.kubernetes.apiserver.enable = mkDefault true;
services.kubernetes.scheduler.enable = mkDefault true;
services.kubernetes.controllerManager.enable = mkDefault true;
services.kubernetes.dns.enable = mkDefault true;
services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
services.kubernetes.addonManager.enable = mkDefault true;
})
# if this node is only a master make it unschedulable by default
@ -1074,47 +1044,24 @@ in {
services.kubernetes.kubelet.enable = mkDefault true;
services.kubernetes.proxy.enable = mkDefault true;
services.kubernetes.dns.enable = mkDefault true;
})
(mkIf cfg.addonManager.enable {
services.kubernetes.kubelet.manifests = import ./kube-addon-manager.nix { inherit cfg addons; };
environment.etc."kubernetes/addons".source = "${addons}/";
})
(mkIf cfg.dns.enable {
systemd.services.kube-dns = {
description = "Kubernetes DNS Service";
systemd.services.kube-addon-manager = {
description = "Kubernetes addon manager";
wantedBy = [ "kubernetes.target" ];
after = [ "kube-apiserver.service" ];
environment.ADDON_PATH = "/etc/kubernetes/addons/";
serviceConfig = {
Slice = "kubernetes.slice";
ExecStart = ''${pkgs.kube-dns}/bin/kube-dns \
--kubecfg-file=${mkKubeConfig "kube-dns" cfg.dns.kubeconfig} \
--dns-port=${toString cfg.dns.port} \
--domain=${cfg.dns.domain} \
${optionalString cfg.verbose "--v=6"} \
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
${cfg.dns.extraOpts}
'';
ExecStart = "${cfg.package}/bin/kube-addons";
WorkingDirectory = cfg.dataDir;
User = "kubernetes";
Group = "kubernetes";
AmbientCapabilities = "cap_net_bind_service";
SendSIGHUP = true;
RestartSec = "30s";
Restart = "always";
StartLimitInterval = "1m";
};
};
networking.firewall.extraCommands = ''
# allow container to host communication for DNS traffic
${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
'';
services.kubernetes.dns.kubeconfig = kubeConfigDefaults;
})
(mkIf (
@ -1122,8 +1069,7 @@ in {
cfg.scheduler.enable ||
cfg.controllerManager.enable ||
cfg.kubelet.enable ||
cfg.proxy.enable ||
cfg.dns.enable
cfg.proxy.enable
) {
systemd.targets.kubernetes = {
description = "Kubernetes";
@ -1147,6 +1093,9 @@ in {
createHome = true;
};
users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
# dns addon is enabled by default
services.kubernetes.addons.dns.enable = mkDefault true;
})
(mkIf cfg.flannel.enable {

View file

@ -0,0 +1,311 @@
{ config, pkgs, lib, ... }:
with lib;
let
version = "1.14.4";
k8s-dns-kube-dns = pkgs.dockerTools.pullImage {
imageName = "gcr.io/google_containers/k8s-dns-kube-dns-amd64";
imageTag = version;
sha256 = "1zq8xl0rac99r925xmalmx1xc3pgzprh6csrzck9k8fnqp50smdk";
};
k8s-dns-dnsmasq-nanny = pkgs.dockerTools.pullImage {
imageName = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64";
imageTag = version;
sha256 = "1dzw1qb97w2km6c5p564g1r5lnvyblwg5h2cssn09v7x4w3lf732";
};
k8s-dns-sidecar = pkgs.dockerTools.pullImage {
imageName = "gcr.io/google_containers/k8s-dns-sidecar-amd64";
imageTag = version;
sha256 = "1k9k6xklc1b74slpldnjsciikhkfhi0nl2n1vn1xr1y40dggssy4";
};
cfg = config.services.kubernetes.addons.dns;
in {
options.services.kubernetes.addons.dns = {
enable = mkEnableOption "kubernetes dns addon";
clusterIp = mkOption {
description = "Dns addon clusterIP";
# this default is also what kubernetes users
default = (
concatStringsSep "." (
take 3 (splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
))
) + ".254";
type = types.str;
};
clusterDomain = mkOption {
description = "Dns cluster domain";
default = "cluster.local";
type = types.str;
};
};
config = mkIf cfg.enable {
services.kubernetes.kubelet.seedDockerImages = [
k8s-dns-kube-dns
k8s-dns-dnsmasq-nanny
k8s-dns-sidecar
];
services.kubernetes.addonManager.addons = {
kubedns-deployment = {
apiVersion = "apps/v1beta1";
kind = "Deployment";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
};
name = "kube-dns";
namespace = "kube-system";
};
spec = {
selector.matchLabels."k8s-app" = "kube-dns";
strategy = {
rollingUpdate = {
maxSurge = "10%";
maxUnavailable = 0;
};
};
template = {
metadata = {
annotations."scheduler.alpha.kubernetes.io/critical-pod" = "";
labels.k8s-app = "kube-dns";
};
spec = {
containers = [
{
name = "kubedns";
args = [
"--domain=${cfg.clusterDomain}"
"--dns-port=10053"
"--config-dir=/kube-dns-config"
"--v=2"
];
env = [
{
name = "PROMETHEUS_PORT";
value = "10055";
}
];
image = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:${version}";
livenessProbe = {
failureThreshold = 5;
httpGet = {
path = "/healthcheck/kubedns";
port = 10054;
scheme = "HTTP";
};
initialDelaySeconds = 60;
successThreshold = 1;
timeoutSeconds = 5;
};
ports = [
{
containerPort = 10053;
name = "dns-local";
protocol = "UDP";
}
{
containerPort = 10053;
name = "dns-tcp-local";
protocol = "TCP";
}
{
containerPort = 10055;
name = "metrics";
protocol = "TCP";
}
];
readinessProbe = {
httpGet = {
path = "/readiness";
port = 8081;
scheme = "HTTP";
};
initialDelaySeconds = 3;
timeoutSeconds = 5;
};
resources = {
limits.memory = "170Mi";
requests = {
cpu = "100m";
memory = "70Mi";
};
};
volumeMounts = [
{
mountPath = "/kube-dns-config";
name = "kube-dns-config";
}
];
}
{
args = [
"-v=2"
"-logtostderr"
"-configDir=/etc/k8s/dns/dnsmasq-nanny"
"-restartDnsmasq=true"
"--"
"-k"
"--cache-size=1000"
"--log-facility=-"
"--server=/${cfg.clusterDomain}/127.0.0.1#10053"
"--server=/in-addr.arpa/127.0.0.1#10053"
"--server=/ip6.arpa/127.0.0.1#10053"
];
image = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:${version}";
livenessProbe = {
failureThreshold = 5;
httpGet = {
path = "/healthcheck/dnsmasq";
port = 10054;
scheme = "HTTP";
};
initialDelaySeconds = 60;
successThreshold = 1;
timeoutSeconds = 5;
};
name = "dnsmasq";
ports = [
{
containerPort = 53;
name = "dns";
protocol = "UDP";
}
{
containerPort = 53;
name = "dns-tcp";
protocol = "TCP";
}
];
resources = {
requests = {
cpu = "150m";
memory = "20Mi";
};
};
volumeMounts = [
{
mountPath = "/etc/k8s/dns/dnsmasq-nanny";
name = "kube-dns-config";
}
];
}
{
name = "sidecar";
image = "gcr.io/google_containers/k8s-dns-sidecar-amd64:${version}";
args = [
"--v=2"
"--logtostderr"
"--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cfg.clusterDomain},5,A"
"--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cfg.clusterDomain},5,A"
];
livenessProbe = {
failureThreshold = 5;
httpGet = {
path = "/metrics";
port = 10054;
scheme = "HTTP";
};
initialDelaySeconds = 60;
successThreshold = 1;
timeoutSeconds = 5;
};
ports = [
{
containerPort = 10054;
name = "metrics";
protocol = "TCP";
}
];
resources = {
requests = {
cpu = "10m";
memory = "20Mi";
};
};
}
];
dnsPolicy = "Default";
serviceAccountName = "kube-dns";
tolerations = [
{
key = "CriticalAddonsOnly";
operator = "Exists";
}
];
volumes = [
{
configMap = {
name = "kube-dns";
optional = true;
};
name = "kube-dns-config";
}
];
};
};
};
};
kubedns-svc = {
apiVersion = "v1";
kind = "Service";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/name" = "KubeDNS";
};
name = "kube-dns";
namespace = "kube-system";
};
spec = {
clusterIP = cfg.clusterIp;
ports = [
{name = "dns"; port = 53; protocol = "UDP";}
{name = "dns-tcp"; port = 53; protocol = "TCP";}
];
selector.k8s-app = "kube-dns";
};
};
kubedns-sa = {
apiVersion = "v1";
kind = "ServiceAccount";
metadata = {
name = "kube-dns";
namespace = "kube-system";
labels = {
"kubernetes.io/cluster-service" = "true";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
};
};
kubedns-cm = {
apiVersion = "v1";
kind = "ConfigMap";
metadata = {
name = "kube-dns";
namespace = "kube-system";
labels = {
"addonmanager.kubernetes.io/mode" = "EnsureExists";
};
};
};
};
services.kubernetes.kubelet.clusterDns = mkDefault cfg.clusterIp;
};
}

View file

@ -1,54 +0,0 @@
{ cfg, addons }: {
"kube-addon-manager" = {
"apiVersion" = "v1";
"kind" = "Pod";
"metadata" = {
"labels" = {
"component" = "kube-addon-manager";
};
"name" = "kube-addon-manager";
"namespace" = "kube-system";
};
"spec" = {
"containers" = [{
"command" = ["/bin/bash"
"-c"
"/opt/kube-addons.sh | tee /var/log/kube-addon-manager.log"
];
"env" = [{
"name" = "KUBECTL_OPTS";
"value" = "--server=${cfg.kubeconfig.server}";
}];
"image" = "gcr.io/google-containers/kube-addon-manager:${cfg.addonManager.versionTag}";
"name" = "kube-addon-manager";
"resources" = {
"requests" = {
"cpu" = "5m";
"memory" = "50Mi";
};
};
"volumeMounts" = [{
"mountPath" = "/etc/kubernetes/addons/";
"name" = "addons";
"readOnly" = true;
} {
"mountPath" = "/var/log";
"name" = "varlog";
"readOnly" = false;
}];
}];
"hostNetwork" = true;
"volumes" = [{
"hostPath" = {
"path" = "${addons}/";
};
"name" = "addons";
} {
"hostPath" = {
"path" = "/var/log";
};
"name" = "varlog";
}];
};
};
}

View file

@ -1,42 +0,0 @@
{ stdenv, lib, buildGoPackage, fetchFromGitHub, go }:
with lib;
stdenv.mkDerivation rec {
name = "kube-dns-${version}";
version = "1.14.1";
src = fetchFromGitHub {
owner = "kubernetes";
repo = "dns";
rev = "${version}";
sha256 = "13l42wm2rcz1ina8hhhagbnck6f1gdbwj33dmnrr52pwi1xh52f7";
};
buildInputs = [ go ];
buildPhase = ''
export GOPATH="$(pwd)/.gopath"
mkdir $GOPATH
ln -s $(pwd)/vendor $GOPATH/src
mkdir $GOPATH/src/k8s.io/dns
ln -s $(pwd)/cmd $GOPATH/src/k8s.io/dns/cmd
ln -s $(pwd)/pkg $GOPATH/src/k8s.io/dns/pkg
# build only kube-dns, we do not need anything else
go build -o kube-dns ./cmd/kube-dns
'';
installPhase = ''
mkdir -p $out/bin
cp kube-dns $out/bin
'';
meta = {
description = "Kubernetes DNS service";
license = licenses.asl20;
homepage = https://github.com/kubernetes/dns;
maintainers = with maintainers; [ matejc ];
platforms = platforms.unix;
};
}

View file

@ -15117,8 +15117,6 @@ with pkgs;
kubernetes = callPackage ../applications/networking/cluster/kubernetes { };
kube-dns = callPackage ../applications/networking/cluster/kubernetes/dns.nix { };
kupfer = callPackage ../applications/misc/kupfer { };
lame = callPackage ../development/libraries/lame { };