{ config, lib, pkgs, ... }: with lib; let top = config.services.kubernetes; cfg = top.apiserver; isRBACEnabled = elem "RBAC" cfg.authorizationMode; apiserverServiceIP = (concatStringsSep "." ( take 3 (splitString "." cfg.serviceClusterIpRange )) + ".1"); in { ###### interface options.services.kubernetes.apiserver = with lib.types; { advertiseAddress = mkOption { description = '' Kubernetes apiserver IP address on which to advertise the apiserver to members of the cluster. This address must be reachable by the rest of the cluster. ''; default = null; type = nullOr str; }; allowPrivileged = mkOption { description = "Whether to allow privileged containers on Kubernetes."; default = false; type = bool; }; authorizationMode = mkOption { description = '' Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See ''; default = ["RBAC" "Node"]; # Enabling RBAC by default, although kubernetes default is AllowAllow type = listOf (enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]); }; authorizationPolicy = mkOption { description = '' Kubernetes apiserver authorization policy file. See ''; default = []; type = listOf attrs; }; basicAuthFile = mkOption { description = '' Kubernetes apiserver basic authentication file. See ''; default = null; type = nullOr path; }; bindAddress = mkOption { description = '' The IP address on which to listen for the --secure-port port. The associated interface(s) must be reachable by the rest of the cluster, and by CLI/web clients. ''; default = "0.0.0.0"; type = str; }; clientCaFile = mkOption { description = "Kubernetes apiserver CA file for client auth."; default = top.caFile; type = nullOr path; }; disableAdmissionPlugins = mkOption { description = '' Kubernetes admission control plugins to disable. See ''; default = []; type = listOf str; }; enable = mkEnableOption "Kubernetes apiserver"; enableAdmissionPlugins = mkOption { description = '' Kubernetes admission control plugins to enable. See ''; default = [ "NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction" ]; example = [ "NamespaceLifecycle" "NamespaceExists" "LimitRanger" "SecurityContextDeny" "ServiceAccount" "ResourceQuota" "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass" ]; type = listOf str; }; etcd = { servers = mkOption { description = "List of etcd servers."; default = ["http://127.0.0.1:2379"]; type = types.listOf types.str; }; keyFile = mkOption { description = "Etcd key file."; default = null; type = types.nullOr types.path; }; certFile = mkOption { description = "Etcd cert file."; default = null; type = types.nullOr types.path; }; caFile = mkOption { description = "Etcd ca file."; default = top.caFile; type = types.nullOr types.path; }; }; extraOpts = mkOption { description = "Kubernetes apiserver extra command line options."; default = ""; type = str; }; extraSANs = mkOption { description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert."; default = []; type = listOf str; }; featureGates = mkOption { description = "List set of feature gates"; default = top.featureGates; type = listOf str; }; insecureBindAddress = mkOption { description = "The IP address on which to serve the --insecure-port."; default = "127.0.0.1"; type = str; }; insecurePort = mkOption { description = "Kubernetes apiserver insecure listening port. (0 = disabled)"; default = 0; type = int; }; kubeletClientCaFile = mkOption { description = "Path to a cert file for connecting to kubelet."; default = top.caFile; type = nullOr path; }; kubeletClientCertFile = mkOption { description = "Client certificate to use for connections to kubelet."; default = null; type = nullOr path; }; kubeletClientKeyFile = mkOption { description = "Key to use for connections to kubelet."; default = null; type = nullOr path; }; kubeletHttps = mkOption { description = "Whether to use https for connections to kubelet."; default = true; type = bool; }; preferredAddressTypes = mkOption { description = "List of the preferred NodeAddressTypes to use for kubelet connections."; type = nullOr str; default = null; }; proxyClientCertFile = mkOption { description = "Client certificate to use for connections to proxy."; default = null; type = nullOr path; }; proxyClientKeyFile = mkOption { description = "Key to use for connections to proxy."; default = null; type = nullOr path; }; runtimeConfig = mkOption { description = '' Api runtime configuration. See ''; default = "authentication.k8s.io/v1beta1=true"; example = "api/all=false,api/v1=true"; type = str; }; storageBackend = mkOption { description = '' Kubernetes apiserver storage backend. ''; default = "etcd3"; type = enum ["etcd2" "etcd3"]; }; securePort = mkOption { description = "Kubernetes apiserver secure port."; default = 6443; type = int; }; serviceAccountKeyFile = mkOption { description = '' Kubernetes apiserver PEM-encoded x509 RSA private or public key file, used to verify ServiceAccount tokens. By default tls private key file is used. ''; default = null; type = nullOr path; }; serviceClusterIpRange = mkOption { description = '' A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes for pods. ''; default = "10.0.0.0/24"; type = str; }; tlsCertFile = mkOption { description = "Kubernetes apiserver certificate file."; default = null; type = nullOr path; }; tlsKeyFile = mkOption { description = "Kubernetes apiserver private key file."; default = null; type = nullOr path; }; tokenAuthFile = mkOption { description = '' Kubernetes apiserver token authentication file. See ''; default = null; type = nullOr path; }; verbosity = mkOption { description = '' Optional glog verbosity level for logging statements. See ''; default = null; type = nullOr int; }; webhookConfig = mkOption { description = '' Kubernetes apiserver Webhook config file. It uses the kubeconfig file format. See ''; default = null; type = nullOr path; }; }; ###### implementation config = mkMerge [ (mkIf cfg.enable { systemd.services.kube-apiserver = { description = "Kubernetes APIServer Service"; wantedBy = [ "kubernetes.target" ]; after = [ "network.target" ]; serviceConfig = { Slice = "kubernetes.slice"; ExecStart = ''${top.package}/bin/kube-apiserver \ --allow-privileged=${boolToString cfg.allowPrivileged} \ --authorization-mode=${concatStringsSep "," cfg.authorizationMode} \ ${optionalString (elem "ABAC" cfg.authorizationMode) "--authorization-policy-file=${ pkgs.writeText "kube-auth-policy.jsonl" (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy) }" } \ ${optionalString (elem "Webhook" cfg.authorizationMode) "--authorization-webhook-config-file=${cfg.webhookConfig}" } \ --bind-address=${cfg.bindAddress} \ ${optionalString (cfg.advertiseAddress != null) "--advertise-address=${cfg.advertiseAddress}"} \ ${optionalString (cfg.clientCaFile != null) "--client-ca-file=${cfg.clientCaFile}"} \ --disable-admission-plugins=${concatStringsSep "," cfg.disableAdmissionPlugins} \ --enable-admission-plugins=${concatStringsSep "," cfg.enableAdmissionPlugins} \ --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \ ${optionalString (cfg.etcd.caFile != null) "--etcd-cafile=${cfg.etcd.caFile}"} \ ${optionalString (cfg.etcd.certFile != null) "--etcd-certfile=${cfg.etcd.certFile}"} \ ${optionalString (cfg.etcd.keyFile != null) "--etcd-keyfile=${cfg.etcd.keyFile}"} \ ${optionalString (cfg.featureGates != []) "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ ${optionalString (cfg.basicAuthFile != null) "--basic-auth-file=${cfg.basicAuthFile}"} \ --kubelet-https=${boolToString cfg.kubeletHttps} \ ${optionalString (cfg.kubeletClientCaFile != null) "--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"} \ ${optionalString (cfg.kubeletClientCertFile != null) "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"} \ ${optionalString (cfg.kubeletClientKeyFile != null) "--kubelet-client-key=${cfg.kubeletClientKeyFile}"} \ ${optionalString (cfg.preferredAddressTypes != null) "--kubelet-preferred-address-types=${cfg.preferredAddressTypes}"} \ ${optionalString (cfg.proxyClientCertFile != null) "--proxy-client-cert-file=${cfg.proxyClientCertFile}"} \ ${optionalString (cfg.proxyClientKeyFile != null) "--proxy-client-key-file=${cfg.proxyClientKeyFile}"} \ --insecure-bind-address=${cfg.insecureBindAddress} \ --insecure-port=${toString cfg.insecurePort} \ ${optionalString (cfg.runtimeConfig != "") "--runtime-config=${cfg.runtimeConfig}"} \ --secure-port=${toString cfg.securePort} \ ${optionalString (cfg.serviceAccountKeyFile!=null) "--service-account-key-file=${cfg.serviceAccountKeyFile}"} \ --service-cluster-ip-range=${cfg.serviceClusterIpRange} \ --storage-backend=${cfg.storageBackend} \ ${optionalString (cfg.tlsCertFile != null) "--tls-cert-file=${cfg.tlsCertFile}"} \ ${optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"} \ ${optionalString (cfg.tokenAuthFile != null) "--token-auth-file=${cfg.tokenAuthFile}"} \ ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ ${cfg.extraOpts} ''; WorkingDirectory = top.dataDir; User = "kubernetes"; Group = "kubernetes"; AmbientCapabilities = "cap_net_bind_service"; Restart = "on-failure"; RestartSec = 5; }; }; services.etcd = { clientCertAuth = mkDefault true; peerClientCertAuth = mkDefault true; listenClientUrls = mkDefault ["https://0.0.0.0:2379"]; listenPeerUrls = mkDefault ["https://0.0.0.0:2380"]; advertiseClientUrls = mkDefault ["https://${top.masterAddress}:2379"]; initialCluster = mkDefault ["${top.masterAddress}=https://${top.masterAddress}:2380"]; name = mkDefault top.masterAddress; initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"]; }; services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled { apiserver-kubelet-api-admin-crb = { apiVersion = "rbac.authorization.k8s.io/v1"; kind = "ClusterRoleBinding"; metadata = { name = "system:kube-apiserver:kubelet-api-admin"; }; roleRef = { apiGroup = "rbac.authorization.k8s.io"; kind = "ClusterRole"; name = "system:kubelet-api-admin"; }; subjects = [{ kind = "User"; name = "system:kube-apiserver"; }]; }; }; services.kubernetes.pki.certs = with top.lib; { apiServer = mkCert { name = "kube-apiserver"; CN = "kubernetes"; hosts = [ "kubernetes.default.svc" "kubernetes.default.svc.${top.addons.dns.clusterDomain}" cfg.advertiseAddress top.masterAddress apiserverServiceIP "127.0.0.1" ] ++ cfg.extraSANs; action = "systemctl restart kube-apiserver.service"; }; apiserverProxyClient = mkCert { name = "kube-apiserver-proxy-client"; CN = "front-proxy-client"; action = "systemctl restart kube-apiserver.service"; }; apiserverKubeletClient = mkCert { name = "kube-apiserver-kubelet-client"; CN = "system:kube-apiserver"; action = "systemctl restart kube-apiserver.service"; }; apiserverEtcdClient = mkCert { name = "kube-apiserver-etcd-client"; CN = "etcd-client"; action = "systemctl restart kube-apiserver.service"; }; clusterAdmin = mkCert { name = "cluster-admin"; CN = "cluster-admin"; fields = { O = "system:masters"; }; privateKeyOwner = "root"; }; etcd = mkCert { name = "etcd"; CN = top.masterAddress; hosts = [ "etcd.local" "etcd.${top.addons.dns.clusterDomain}" top.masterAddress cfg.advertiseAddress ]; privateKeyOwner = "etcd"; action = "systemctl restart etcd.service"; }; }; }) ]; }