diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 7edea2c9538..cb71e93e578 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -157,7 +157,7 @@
./services/backup/tarsnap.nix
./services/backup/znapzend.nix
./services/cluster/fleet.nix
- ./services/cluster/kubernetes.nix
+ ./services/cluster/kubernetes/default.nix
./services/cluster/panamax.nix
./services/computing/boinc/client.nix
./services/computing/torque/server.nix
diff --git a/nixos/modules/services/cluster/kubernetes/dashboard.nix b/nixos/modules/services/cluster/kubernetes/dashboard.nix
new file mode 100644
index 00000000000..337c2634a37
--- /dev/null
+++ b/nixos/modules/services/cluster/kubernetes/dashboard.nix
@@ -0,0 +1,85 @@
+{ cfg }: {
+ "dashboard-controller" = {
+ "apiVersion" = "extensions/v1beta1";
+ "kind" = "Deployment";
+ "metadata" = {
+ "labels" = {
+ "addonmanager.kubernetes.io/mode" = "Reconcile";
+ "k8s-app" = "kubernetes-dashboard";
+ "kubernetes.io/cluster-service" = "true";
+ };
+ "name" = "kubernetes-dashboard";
+ "namespace" = "kube-system";
+ };
+ "spec" = {
+ "selector" = {
+ "matchLabels" = {
+ "k8s-app" = "kubernetes-dashboard";
+ };
+ };
+ "template" = {
+ "metadata" = {
+ "annotations" = {
+ "scheduler.alpha.kubernetes.io/critical-pod" = "";
+ };
+ "labels" = {
+ "k8s-app" = "kubernetes-dashboard";
+ };
+ };
+ "spec" = {
+ "containers" = [{
+ "image" = "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.0";
+ "livenessProbe" = {
+ "httpGet" = {
+ "path" = "/";
+ "port" = 9090;
+ };
+ "initialDelaySeconds" = 30;
+ "timeoutSeconds" = 30;
+ };
+ "name" = "kubernetes-dashboard";
+ "ports" = [{
+ "containerPort" = 9090;
+ }];
+ "resources" = {
+ "limits" = {
+ "cpu" = "100m";
+ "memory" = "50Mi";
+ };
+ "requests" = {
+ "cpu" = "100m";
+ "memory" = "50Mi";
+ };
+ };
+ }];
+ "tolerations" = [{
+ "key" = "CriticalAddonsOnly";
+ "operator" = "Exists";
+ }];
+ };
+ };
+ };
+ };
+ "dashboard-service" = {
+ "apiVersion" = "v1";
+ "kind" = "Service";
+ "metadata" = {
+ "labels" = {
+ "addonmanager.kubernetes.io/mode" = "Reconcile";
+ "k8s-app" = "kubernetes-dashboard";
+ "kubernetes.io/cluster-service" = "true";
+ };
+ "name" = "kubernetes-dashboard";
+ "namespace" = "kube-system";
+ };
+ "spec" = {
+ "ports" = [{
+ "port" = 80;
+ "targetPort" = 9090;
+ }];
+ "selector" = {
+ "k8s-app" = "kubernetes-dashboard";
+ };
+ };
+ };
+}
diff --git a/nixos/modules/services/cluster/kubernetes.nix b/nixos/modules/services/cluster/kubernetes/default.nix
similarity index 78%
rename from nixos/modules/services/cluster/kubernetes.nix
rename to nixos/modules/services/cluster/kubernetes/default.nix
index 8177233b3ed..7160bcca153 100644
--- a/nixos/modules/services/cluster/kubernetes.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -53,6 +53,35 @@ let
) cfg.kubelet.manifests;
};
+ addons = pkgs.runCommand "kubernetes-addons" { } ''
+ mkdir -p $out
+ # since we are mounting the addons to the addon manager, they need to be copied
+ ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon:
+ pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)
+ ) (cfg.addonManager.addons))}
+ '';
+
+ taintOptions = { name, ... }: {
+ options = {
+ key = mkOption {
+ description = "Key of taint.";
+ default = name;
+ type = types.str;
+ };
+ value = mkOption {
+ description = "Value of taint.";
+ type = types.str;
+ };
+ effect = mkOption {
+ description = "Effect of taint.";
+ example = "NoSchedule";
+ type = types.str;
+ };
+ };
+ };
+
+ taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints);
+
in {
###### interface
@@ -77,7 +106,7 @@ in {
};
verbose = mkOption {
- description = "Kubernetes enable verbose mode for debugging";
+ description = "Kubernetes enable verbose mode for debugging.";
default = false;
type = types.bool;
};
@@ -90,19 +119,19 @@ in {
};
keyFile = mkOption {
- description = "Etcd key file";
+ description = "Etcd key file.";
default = null;
type = types.nullOr types.path;
};
certFile = mkOption {
- description = "Etcd cert file";
+ description = "Etcd cert file.";
default = null;
type = types.nullOr types.path;
};
caFile = mkOption {
- description = "Etcd ca file";
+ description = "Etcd ca file.";
default = null;
type = types.nullOr types.path;
};
@@ -110,25 +139,25 @@ in {
kubeconfig = {
server = mkOption {
- description = "Kubernetes apiserver server address";
+ description = "Kubernetes apiserver server address.";
default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
type = types.str;
};
caFile = mkOption {
- description = "Certificate authrority file to use to connect to kuberentes apiserver";
+ description = "Certificate authrority file to use to connect to Kubernetes apiserver.";
type = types.nullOr types.path;
default = null;
};
certFile = mkOption {
- description = "Client certificate file to use to connect to kubernetes";
+ description = "Client certificate file to use to connect to Kubernetes.";
type = types.nullOr types.path;
default = null;
};
keyFile = mkOption {
- description = "Client key file to use to connect to kubernetes";
+ description = "Client key file to use to connect to Kubernetes.";
type = types.nullOr types.path;
default = null;
};
@@ -142,7 +171,7 @@ in {
apiserver = {
enable = mkOption {
- description = "Whether to enable kubernetes apiserver.";
+ description = "Whether to enable Kubernetes apiserver.";
default = false;
type = types.bool;
};
@@ -172,6 +201,14 @@ in {
type = types.nullOr types.str;
};
+ storageBackend = mkOption {
+ description = ''
+ Kubernetes apiserver storage backend.
+ '';
+ default = "etcd2";
+ type = types.enum ["etcd2" "etcd3"];
+ };
+
port = mkOption {
description = "Kubernetes apiserver listening port.";
default = 8080;
@@ -227,7 +264,7 @@ in {
Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See
'';
- default = ["ABAC" "RBAC"];
+ default = ["ABAC"];
type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC"]);
};
@@ -274,7 +311,7 @@ in {
apiVersion = "abac.authorization.kubernetes.io/v1beta1";
kind = "Policy";
spec = {
- user = "kube";
+ user = "kube-worker";
namespace = "*";
resource = "*";
apiGroup = "*";
@@ -285,7 +322,29 @@ in {
apiVersion = "abac.authorization.kubernetes.io/v1beta1";
kind = "Policy";
spec = {
- user = "system:serviceaccount:kube-system:default";
+ user = "kube_proxy";
+ namespace = "*";
+ resource = "*";
+ apiGroup = "*";
+ nonResourcePath = "*";
+ };
+ }
+ {
+ apiVersion = "abac.authorization.kubernetes.io/v1beta1";
+ kind = "Policy";
+ spec = {
+ user = "client";
+ namespace = "*";
+ resource = "*";
+ apiGroup = "*";
+ nonResourcePath = "*";
+ };
+ }
+ {
+ apiVersion = "abac.authorization.kubernetes.io/v1beta1";
+ kind = "Policy";
+ spec = {
+ group = "system:serviceaccounts";
namespace = "*";
resource = "*";
apiGroup = "*";
@@ -297,19 +356,19 @@ in {
};
autorizationRBACSuperAdmin = mkOption {
- description = "Role based authorization super admin";
+ description = "Role based authorization super admin.";
default = "admin";
type = types.str;
};
allowPrivileged = mkOption {
- description = "Whether to allow privileged containers on kubernetes.";
+ description = "Whether to allow privileged containers on Kubernetes.";
default = true;
type = types.bool;
};
portalNet = mkOption {
- description = "Kubernetes CIDR notation IP range from which to assign portal IPs";
+ description = "Kubernetes CIDR notation IP range from which to assign portal IPs.";
default = "10.10.10.10/24";
type = types.str;
};
@@ -319,7 +378,7 @@ in {
Api runtime configuration. See
'';
- default = "rbac.authorization.k8s.io/v1alpha1";
+ default = "";
example = "api/all=false,api/v1=true";
type = types.str;
};
@@ -348,25 +407,25 @@ in {
};
kubeletClientCaFile = mkOption {
- description = "Path to a cert file for connecting to kubelet";
+ description = "Path to a cert file for connecting to kubelet.";
default = null;
type = types.nullOr types.path;
};
kubeletClientCertFile = mkOption {
- description = "Client certificate to use for connections to kubelet";
+ description = "Client certificate to use for connections to kubelet.";
default = null;
type = types.nullOr types.path;
};
kubeletClientKeyFile = mkOption {
- description = "Key to use for connections to kubelet";
+ description = "Key to use for connections to kubelet.";
default = null;
type = types.nullOr types.path;
};
kubeletHttps = mkOption {
- description = "Whether to use https for connections to kubelet";
+ description = "Whether to use https for connections to kubelet.";
default = true;
type = types.bool;
};
@@ -380,7 +439,7 @@ in {
scheduler = {
enable = mkOption {
- description = "Whether to enable kubernetes scheduler.";
+ description = "Whether to enable Kubernetes scheduler.";
default = false;
type = types.bool;
};
@@ -398,7 +457,7 @@ in {
};
leaderElect = mkOption {
- description = "Whether to start leader election before executing main loop";
+ description = "Whether to start leader election before executing main loop.";
type = types.bool;
default = false;
};
@@ -412,7 +471,7 @@ in {
controllerManager = {
enable = mkOption {
- description = "Whether to enable kubernetes controller manager.";
+ description = "Whether to enable Kubernetes controller manager.";
default = false;
type = types.bool;
};
@@ -430,7 +489,7 @@ in {
};
leaderElect = mkOption {
- description = "Whether to start leader election before executing main loop";
+ description = "Whether to start leader election before executing main loop.";
type = types.bool;
default = false;
};
@@ -453,12 +512,6 @@ in {
type = types.nullOr types.path;
};
- clusterCidr = mkOption {
- description = "Kubernetes controller manager CIDR Range for Pods in cluster";
- default = "10.10.0.0/16";
- type = types.str;
- };
-
extraOpts = mkOption {
description = "Kubernetes controller manager extra command line options.";
default = "";
@@ -468,7 +521,7 @@ in {
kubelet = {
enable = mkOption {
- description = "Whether to enable kubernetes kubelet.";
+ description = "Whether to enable Kubernetes kubelet.";
default = false;
type = types.bool;
};
@@ -518,13 +571,13 @@ in {
};
hostname = mkOption {
- description = "Kubernetes kubelet hostname override";
+ description = "Kubernetes kubelet hostname override.";
default = config.networking.hostName;
type = types.str;
};
allowPrivileged = mkOption {
- description = "Whether to allow kubernetes containers to request privileged mode.";
+ description = "Whether to allow Kubernetes containers to request privileged mode.";
default = true;
type = types.bool;
};
@@ -536,7 +589,7 @@ in {
};
clusterDns = mkOption {
- description = "Use alternative dns.";
+ description = "Use alternative DNS.";
default = "10.10.0.1";
type = types.str;
};
@@ -548,20 +601,20 @@ in {
};
networkPlugin = mkOption {
- description = "Network plugin to use by kubernetes";
+ description = "Network plugin to use by Kubernetes.";
type = types.nullOr (types.enum ["cni" "kubenet"]);
default = "kubenet";
};
cni = {
packages = mkOption {
- description = "List of network plugin packages to install";
+ description = "List of network plugin packages to install.";
type = types.listOf types.package;
default = [];
};
config = mkOption {
- description = "Kubernetes CNI configuration";
+ description = "Kubernetes CNI configuration.";
type = types.listOf types.attrs;
default = [];
example = literalExample ''
@@ -588,11 +641,29 @@ in {
};
manifests = mkOption {
- description = "List of manifests to bootstrap with kubelet";
+ description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
type = types.attrsOf types.attrs;
default = {};
};
+ applyManifests = mkOption {
+ description = "Whether to apply manifests.";
+ default = false;
+ type = types.bool;
+ };
+
+ unschedulable = mkOption {
+ description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
+ default = false;
+ type = types.bool;
+ };
+
+ taints = mkOption {
+ description = ".";
+ default = {};
+ type = types.attrsOf (types.submodule [ taintOptions ]);
+ };
+
extraOpts = mkOption {
description = "Kubernetes kubelet extra command line options.";
default = "";
@@ -602,7 +673,7 @@ in {
proxy = {
enable = mkOption {
- description = "Whether to enable kubernetes proxy.";
+ description = "Whether to enable Kubernetes proxy.";
default = false;
type = types.bool;
};
@@ -620,12 +691,74 @@ in {
};
};
+ addonManager = {
+ enable = mkOption {
+ description = "Whether to enable Kubernetes addon manager.";
+ default = false;
+ type = types.bool;
+ };
+
+ versionTag = mkOption {
+ description = "Version tag of Kubernetes addon manager image.";
+ default = "v6.4-beta.1";
+ type = types.str;
+ };
+
+ addons = mkOption {
+ description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
+ default = { };
+ type = types.attrsOf types.attrs;
+ example = literalExample ''
+ {
+ "my-service" = {
+ "apiVersion" = "v1";
+ "kind" = "Service";
+ "metadata" = {
+ "name" = "my-service";
+ "namespace" = "default";
+ };
+ "spec" = { ... };
+ };
+ }
+ // import { cfg = config.services.kubernetes; };
+ '';
+ };
+ };
+
+ dns = {
+ enable = mkEnableOption "Kubernetes DNS service.";
+
+ port = mkOption {
+ description = "Kubernetes DNS listening port.";
+ default = 53;
+ type = types.int;
+ };
+
+ domain = mkOption {
+ description = "Kubernetes DNS domain under which to create names.";
+ default = cfg.kubelet.clusterDomain;
+ type = types.str;
+ };
+
+ extraOpts = mkOption {
+ description = "Kubernetes DNS extra command line options.";
+ default = "";
+ type = types.str;
+ };
+ };
+
path = mkOption {
- description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added";
+ description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
type = types.listOf types.package;
default = [];
};
+ clusterCidr = mkOption {
+ description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
+ default = "10.10.0.0/16";
+ type = types.str;
+ };
+
};
###### implementation
@@ -645,7 +778,10 @@ in {
serviceConfig = {
Slice = "kubernetes.slice";
ExecStart = ''${cfg.package}/bin/kubelet \
- --pod-manifest-path=${manifests} \
+ ${optionalString cfg.kubelet.applyManifests
+ "--pod-manifest-path=${manifests}"} \
+ ${optionalString (taints != "")
+ "--register-with-taints=${taints}"} \
--kubeconfig=${kubeconfig} \
--require-kubeconfig \
--address=${cfg.kubelet.address} \
@@ -677,15 +813,24 @@ in {
};
};
+ # Allways include cni plugins
+ services.kubernetes.kubelet.cni.packages = [pkgs.cni];
+ })
+
+ (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
environment.etc = mapAttrs' (name: manifest:
nameValuePair "kubernetes/manifests/${name}.json" {
text = builtins.toJSON manifest;
mode = "0755";
}
) cfg.kubelet.manifests;
+ })
- # Allways include cni plugins
- services.kubernetes.kubelet.cni.packages = [pkgs.cni];
+ (mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) {
+ services.kubernetes.kubelet.taints.unschedulable = {
+ value = "true";
+ effect = "NoSchedule";
+ };
})
(mkIf cfg.apiserver.enable {
@@ -728,7 +873,7 @@ in {
--authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \
${optionalString (elem "ABAC" cfg.apiserver.authorizationMode)
"--authorization-policy-file=${
- pkgs.writeText "kube-auth-policy"
+ pkgs.writeText "kube-auth-policy.jsonl"
(concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
}"
} \
@@ -743,7 +888,7 @@ in {
"--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
${optionalString cfg.verbose "--v=6"} \
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
- --storage-backend=etcd2 \
+ --storage-backend=${cfg.apiserver.storageBackend} \
${cfg.apiserver.extraOpts}
'';
WorkingDirectory = cfg.dataDir;
@@ -799,8 +944,8 @@ in {
${if (cfg.controllerManager.rootCaFile!=null)
then "--root-ca-file=${cfg.controllerManager.rootCaFile}"
else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \
- ${optionalString (cfg.controllerManager.clusterCidr!=null)
- "--cluster-cidr=${cfg.controllerManager.clusterCidr}"} \
+ ${optionalString (cfg.clusterCidr!=null)
+ "--cluster-cidr=${cfg.clusterCidr}"} \
--allocate-node-cidrs=true \
${optionalString cfg.verbose "--v=6"} \
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
@@ -819,7 +964,7 @@ in {
description = "Kubernetes Proxy Service";
wantedBy = [ "kubernetes.target" ];
after = [ "kube-apiserver.service" ];
- path = [pkgs.iptables];
+ path = [pkgs.iptables pkgs.conntrack_tools];
serviceConfig = {
Slice = "kubernetes.slice";
ExecStart = ''${cfg.package}/bin/kube-proxy \
@@ -827,6 +972,8 @@ in {
--bind-address=${cfg.proxy.address} \
${optionalString cfg.verbose "--v=6"} \
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
+ ${optionalString (cfg.clusterCidr!=null)
+ "--cluster-cidr=${cfg.clusterCidr}"} \
${cfg.proxy.extraOpts}
'';
WorkingDirectory = cfg.dataDir;
@@ -842,12 +989,18 @@ in {
virtualisation.docker.enable = mkDefault true;
services.kubernetes.kubelet.enable = mkDefault true;
services.kubernetes.kubelet.allowPrivileged = mkDefault true;
+ services.kubernetes.kubelet.applyManifests = mkDefault true;
services.kubernetes.apiserver.enable = mkDefault true;
services.kubernetes.scheduler.enable = mkDefault true;
services.kubernetes.controllerManager.enable = mkDefault true;
+ services.kubernetes.dns.enable = mkDefault true;
services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
})
+ (mkIf (all (el: el == "master") cfg.roles) {
+ services.kubernetes.kubelet.unschedulable = mkDefault true;
+ })
+
(mkIf (any (el: el == "node") cfg.roles) {
virtualisation.docker.enable = mkDefault true;
virtualisation.docker.logDriver = mkDefault "json-file";
@@ -855,12 +1008,42 @@ in {
services.kubernetes.proxy.enable = mkDefault true;
})
+ (mkIf cfg.addonManager.enable {
+ services.kubernetes.kubelet.manifests = import ./kube-addon-manager.nix { inherit cfg addons; };
+ environment.etc."kubernetes/addons".source = "${addons}/";
+ })
+
+ (mkIf cfg.dns.enable {
+ systemd.services.kube-dns = {
+ description = "Kubernetes DNS Service";
+ wantedBy = [ "kubernetes.target" ];
+ after = [ "kube-apiserver.service" ];
+ serviceConfig = {
+ Slice = "kubernetes.slice";
+ ExecStart = ''${pkgs.kube-dns}/bin/kube-dns \
+ --kubecfg-file=${kubeconfig} \
+ --dns-port=${toString cfg.dns.port} \
+ --domain=${cfg.dns.domain} \
+ ${optionalString cfg.verbose "--v=6"} \
+ ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
+ ${cfg.dns.extraOpts}
+ '';
+ WorkingDirectory = cfg.dataDir;
+ User = "kubernetes";
+ Group = "kubernetes";
+ AmbientCapabilities = "cap_net_bind_service";
+ SendSIGHUP = true;
+ };
+ };
+ })
+
(mkIf (
cfg.apiserver.enable ||
cfg.scheduler.enable ||
cfg.controllerManager.enable ||
cfg.kubelet.enable ||
- cfg.proxy.enable
+ cfg.proxy.enable ||
+ cfg.dns.enable
) {
systemd.targets.kubernetes = {
description = "Kubernetes";
diff --git a/nixos/modules/services/cluster/kubernetes/dns.nix b/nixos/modules/services/cluster/kubernetes/dns.nix
new file mode 100644
index 00000000000..ac59eaf8725
--- /dev/null
+++ b/nixos/modules/services/cluster/kubernetes/dns.nix
@@ -0,0 +1,240 @@
+{ cfg }: {
+ "kubedns-cm" = {
+ "apiVersion" = "v1";
+ "kind" = "ConfigMap";
+ "metadata" = {
+ "labels" = {
+ "addonmanager.kubernetes.io/mode" = "EnsureExists";
+ };
+ "name" = "kube-dns";
+ "namespace" = "kube-system";
+ };
+ };
+ "kubedns-controller" = {
+ "apiVersion" = "extensions/v1beta1";
+ "kind" = "Deployment";
+ "metadata" = {
+ "labels" = {
+ "addonmanager.kubernetes.io/mode" = "Reconcile";
+ "k8s-app" = "kube-dns";
+ "kubernetes.io/cluster-service" = "true";
+ };
+ "name" = "kube-dns";
+ "namespace" = "kube-system";
+ };
+ "spec" = {
+ "selector" = {
+ "matchLabels" = {
+ "k8s-app" = "kube-dns";
+ };
+ };
+ "strategy" = {
+ "rollingUpdate" = {
+ "maxSurge" = "10%";
+ "maxUnavailable" = 0;
+ };
+ };
+ "template" = {
+ "metadata" = {
+ "annotations" = {
+ "scheduler.alpha.kubernetes.io/critical-pod" = "";
+ };
+ "labels" = {
+ "k8s-app" = "kube-dns";
+ };
+ };
+ "spec" = {
+ "containers" = [{
+ "args" = ["--domain=${cfg.dns.domain}."
+ "--dns-port=10053"
+ "--config-dir=/kube-dns-config"
+ "--kube-master-url=${cfg.kubeconfig.server}"
+ "--v=2"
+ ];
+ "env" = [{
+ "name" = "PROMETHEUS_PORT";
+ "value" = "10055";
+ }];
+ "image" = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1";
+ "livenessProbe" = {
+ "failureThreshold" = 5;
+ "httpGet" = {
+ "path" = "/healthcheck/kubedns";
+ "port" = 10054;
+ "scheme" = "HTTP";
+ };
+ "initialDelaySeconds" = 60;
+ "successThreshold" = 1;
+ "timeoutSeconds" = 5;
+ };
+ "name" = "kubedns";
+ "ports" = [{
+ "containerPort" = 10053;
+ "name" = "dns-local";
+ "protocol" = "UDP";
+ } {
+ "containerPort" = 10053;
+ "name" = "dns-tcp-local";
+ "protocol" = "TCP";
+ } {
+ "containerPort" = 10055;
+ "name" = "metrics";
+ "protocol" = "TCP";
+ }];
+ "readinessProbe" = {
+ "httpGet" = {
+ "path" = "/readiness";
+ "port" = 8081;
+ "scheme" = "HTTP";
+ };
+ "initialDelaySeconds" = 3;
+ "timeoutSeconds" = 5;
+ };
+ "resources" = {
+ "limits" = {
+ "memory" = "170Mi";
+ };
+ "requests" = {
+ "cpu" = "100m";
+ "memory" = "70Mi";
+ };
+ };
+ "volumeMounts" = [{
+ "mountPath" = "/kube-dns-config";
+ "name" = "kube-dns-config";
+ }];
+ } {
+ "args" = ["-v=2"
+ "-logtostderr"
+ "-configDir=/etc/k8s/dns/dnsmasq-nanny"
+ "-restartDnsmasq=true"
+ "--"
+ "-k"
+ "--cache-size=1000"
+ "--log-facility=-"
+ "--server=/${cfg.dns.domain}/127.0.0.1#10053"
+ "--server=/in-addr.arpa/127.0.0.1#10053"
+ "--server=/ip6.arpa/127.0.0.1#10053"
+ ];
+ "image" = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1";
+ "livenessProbe" = {
+ "failureThreshold" = 5;
+ "httpGet" = {
+ "path" = "/healthcheck/dnsmasq";
+ "port" = 10054;
+ "scheme" = "HTTP";
+ };
+ "initialDelaySeconds" = 60;
+ "successThreshold" = 1;
+ "timeoutSeconds" = 5;
+ };
+ "name" = "dnsmasq";
+ "ports" = [{
+ "containerPort" = 53;
+ "name" = "dns";
+ "protocol" = "UDP";
+ } {
+ "containerPort" = 53;
+ "name" = "dns-tcp";
+ "protocol" = "TCP";
+ }];
+ "resources" = {
+ "requests" = {
+ "cpu" = "150m";
+ "memory" = "20Mi";
+ };
+ };
+ "volumeMounts" = [{
+ "mountPath" = "/etc/k8s/dns/dnsmasq-nanny";
+ "name" = "kube-dns-config";
+ }];
+ } {
+ "args" = ["--v=2"
+ "--logtostderr"
+ "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cfg.dns.domain},5,A"
+ "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cfg.dns.domain},5,A"
+ ];
+ "image" = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1";
+ "livenessProbe" = {
+ "failureThreshold" = 5;
+ "httpGet" = {
+ "path" = "/metrics";
+ "port" = 10054;
+ "scheme" = "HTTP";
+ };
+ "initialDelaySeconds" = 60;
+ "successThreshold" = 1;
+ "timeoutSeconds" = 5;
+ };
+ "name" = "sidecar";
+ "ports" = [{
+ "containerPort" = 10054;
+ "name" = "metrics";
+ "protocol" = "TCP";
+ }];
+ "resources" = {
+ "requests" = {
+ "cpu" = "10m";
+ "memory" = "20Mi";
+ };
+ };
+ }];
+ "dnsPolicy" = "Default";
+ "serviceAccountName" = "kube-dns";
+ "tolerations" = [{
+ "key" = "CriticalAddonsOnly";
+ "operator" = "Exists";
+ }];
+ "volumes" = [{
+ "configMap" = {
+ "name" = "kube-dns";
+ "optional" = true;
+ };
+ "name" = "kube-dns-config";
+ }];
+ };
+ };
+ };
+ };
+ "kubedns-sa" = {
+ "apiVersion" = "v1";
+ "kind" = "ServiceAccount";
+ "metadata" = {
+ "labels" = {
+ "addonmanager.kubernetes.io/mode" = "Reconcile";
+ "kubernetes.io/cluster-service" = "true";
+ };
+ "name" = "kube-dns";
+ "namespace" = "kube-system";
+ };
+ };
+ "kubedns-svc" = {
+ "apiVersion" = "v1";
+ "kind" = "Service";
+ "metadata" = {
+ "labels" = {
+ "addonmanager.kubernetes.io/mode" = "Reconcile";
+ "k8s-app" = "kube-dns";
+ "kubernetes.io/cluster-service" = "true";
+ "kubernetes.io/name" = "KubeDNS";
+ };
+ "name" = "kube-dns";
+ "namespace" = "kube-system";
+ };
+ "spec" = {
+ "clusterIP" = "${cfg.dns.serverIp}";
+ "ports" = [{
+ "name" = "dns";
+ "port" = 53;
+ "protocol" = "UDP";
+ } {
+ "name" = "dns-tcp";
+ "port" = 53;
+ "protocol" = "TCP";
+ }];
+ "selector" = {
+ "k8s-app" = "kube-dns";
+ };
+ };
+ };
+}
diff --git a/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix b/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix
new file mode 100644
index 00000000000..f1a367dfff1
--- /dev/null
+++ b/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix
@@ -0,0 +1,54 @@
+{ cfg, addons }: {
+ "kube-addon-manager" = {
+ "apiVersion" = "v1";
+ "kind" = "Pod";
+ "metadata" = {
+ "labels" = {
+ "component" = "kube-addon-manager";
+ };
+ "name" = "kube-addon-manager";
+ "namespace" = "kube-system";
+ };
+ "spec" = {
+ "containers" = [{
+ "command" = ["/bin/bash"
+ "-c"
+ "/opt/kube-addons.sh | tee /var/log/kube-addon-manager.log"
+ ];
+ "env" = [{
+ "name" = "KUBECTL_OPTS";
+ "value" = "--server=${cfg.kubeconfig.server}";
+ }];
+ "image" = "gcr.io/google-containers/kube-addon-manager:${cfg.addonManager.versionTag}";
+ "name" = "kube-addon-manager";
+ "resources" = {
+ "requests" = {
+ "cpu" = "5m";
+ "memory" = "50Mi";
+ };
+ };
+ "volumeMounts" = [{
+ "mountPath" = "/etc/kubernetes/addons/";
+ "name" = "addons";
+ "readOnly" = true;
+ } {
+ "mountPath" = "/var/log";
+ "name" = "varlog";
+ "readOnly" = false;
+ }];
+ }];
+ "hostNetwork" = true;
+ "volumes" = [{
+ "hostPath" = {
+ "path" = "${addons}/";
+ };
+ "name" = "addons";
+ } {
+ "hostPath" = {
+ "path" = "/var/log";
+ };
+ "name" = "varlog";
+ }];
+ };
+ };
+}