Revert "Merge pull request #56789 from mayflower/upstream-k8s-refactor"

This reverts commit 7dc6e77bc2a03e660cab2c4cbf52f235bc52683e, reversing
changes made to bce47ea9d5fa962736ddd4a254a27a5fd2cdee9a.

Motivation for the revert in #67563

1 files changed, 75 insertions, 91 deletions

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 47384ae50a07..733479e24c97 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -27,11 +27,12 @@ let
   certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
   cfsslAPITokenLength = 32;
 
-  clusterAdminKubeconfig = with cfg.certs.clusterAdmin; {
-    server = top.apiserverAddress;
-    certFile = cert;
-    keyFile = key;
-  };
+  clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
+    top.lib.mkKubeConfig "cluster-admin" {
+        server = top.apiserverAddress;
+        certFile = cert;
+        keyFile = key;
+    };
 
   remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
 in
@@ -118,11 +119,6 @@ in
     cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
     cfsslCert = "${cfsslCertPathPrefix}.pem";
     cfsslKey = "${cfsslCertPathPrefix}-key.pem";
-
-    certmgrPaths = [
-      top.caFile
-      certmgrAPITokenPath
-    ];
   in
   {
 
@@ -172,40 +168,13 @@ in
         chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
       '')]);
 
-    systemd.targets.cfssl-online = {
-      wantedBy = [ "network-online.target" ];
-      after = [ "cfssl.service" "network-online.target" "cfssl-online.service" ];
-    };
-
-    systemd.services.cfssl-online = {
-      description = "Wait for ${remote} to be reachable.";
-      wantedBy = [ "cfssl-online.target" ];
-      before = [ "cfssl-online.target" ];
-      path = [ pkgs.curl ];
-      preStart = ''
-        until curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o /dev/null; do
-          echo curl ${remote}/api/v1/cfssl/info: exit status $?
-          sleep 2
-        done
-      '';
-      script = "echo Ok";
-      serviceConfig = {
-        TimeoutSec = "300";
-      };
-    };
-
     systemd.services.kube-certmgr-bootstrap = {
       description = "Kubernetes certmgr bootstrapper";
-      wantedBy = [ "cfssl-online.target" ];
-      after = [ "cfssl-online.target" ];
-      before = [ "certmgr.service" ];
-      path = with pkgs; [ curl cfssl ];
+      wantedBy = [ "certmgr.service" ];
+      after = [ "cfssl.target" ];
       script = concatStringsSep "\n" [''
         set -e
 
-        mkdir -p $(dirname ${certmgrAPITokenPath})
-        mkdir -p $(dirname ${top.caFile})
-
         # If there's a cfssl (cert issuer) running locally, then don't rely on user to
         # manually paste it in place. Just symlink.
         # otherwise, create the target file, ready for users to insert the token
@@ -217,18 +186,15 @@ in
         fi
       ''
       (optionalString (cfg.pkiTrustOnBootstrap) ''
-        if [ ! -s "${top.caFile}" ]; then
-          until test -s ${top.caFile}.json; do
-            sleep 2
-            curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o ${top.caFile}.json
-          done
-          cfssljson -f ${top.caFile}.json -stdout >${top.caFile}
-          rm ${top.caFile}.json
+        if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
+          ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
+            ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
         fi
       '')
       ];
       serviceConfig = {
-        TimeoutSec = "500";
+        RestartSec = "10s";
+        Restart = "on-failure";
       };
     };
 
@@ -264,28 +230,35 @@ in
           mapAttrs mkSpec cfg.certs;
       };
 
-      systemd.services.certmgr = {
-        wantedBy = [ "cfssl-online.target" ];
-        after = [ "cfssl-online.target" "kube-certmgr-bootstrap.service" ];
-        preStart = ''
-          while ! test -s ${certmgrAPITokenPath} ; do
-            sleep 1
-            echo Waiting for ${certmgrAPITokenPath}
-          done
-        '';
-        unitConfig.ConditionPathExists = certmgrPaths;
-      };
-
-      systemd.paths.certmgr = {
-        wantedBy = [ "certmgr.service" ];
-        pathConfig = {
-          PathExists = certmgrPaths;
-          PathChanged = certmgrPaths;
-        };
-      };
-
-      environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (cfg.etcClusterAdminKubeconfig != null)
-        (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig);
+      #TODO: Get rid of kube-addon-manager in the future for the following reasons
+      # - it is basically just a shell script wrapped around kubectl
+      # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+      # - it is designed to be used with k8s system components only
+      # - it would be better with a more Nix-oriented way of managing addons
+      systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
+        environment.KUBECONFIG = with cfg.certs.addonManager;
+          top.lib.mkKubeConfig "addon-manager" {
+            server = top.apiserverAddress;
+            certFile = cert;
+            keyFile = key;
+          };
+        }
+
+        (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
+          serviceConfig.PermissionsStartOnly = true;
+          preStart = with pkgs;
+          let
+            files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
+              top.addonManager.bootstrapAddons;
+          in
+          ''
+            export KUBECONFIG=${clusterAdminKubeconfig}
+            ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
+          '';
+        })]);
+
+      environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
+        clusterAdminKubeconfig;
 
       environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
       (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
@@ -311,22 +284,38 @@ in
           exit 1
         fi
 
-        do_restart=$(test -s ${certmgrAPITokenPath} && echo -n y || echo -n n)
-
         echo $token > ${certmgrAPITokenPath}
         chmod 600 ${certmgrAPITokenPath}
 
-        if [ y = $do_restart ]; then
-          echo "Restarting certmgr..." >&1
-          systemctl restart certmgr
-        fi
+        echo "Restarting certmgr..." >&1
+        systemctl restart certmgr
+
+        echo "Waiting for certs to appear..." >&1
+
+        ${optionalString top.kubelet.enable ''
+          while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
+          echo "Restarting kubelet..." >&1
+          systemctl restart kubelet
+        ''}
+
+        ${optionalString top.proxy.enable ''
+          while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
+          echo "Restarting kube-proxy..." >&1
+          systemctl restart kube-proxy
+        ''}
 
-        echo "Node joined succesfully" >&1
+        ${optionalString top.flannel.enable ''
+          while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
+          echo "Restarting flannel..." >&1
+          systemctl restart flannel
+        ''}
+
+        echo "Node joined succesfully"
       '')];
 
       # isolate etcd on loopback at the master node
       # easyCerts doesn't support multimaster clusters anyway atm.
-      services.etcd = mkIf top.apiserver.enable (with cfg.certs.etcd; {
+      services.etcd = with cfg.certs.etcd; {
         listenClientUrls = ["https://127.0.0.1:2379"];
         listenPeerUrls = ["https://127.0.0.1:2380"];
         advertiseClientUrls = ["https://etcd.local:2379"];
@@ -335,11 +324,19 @@ in
         certFile = mkDefault cert;
         keyFile = mkDefault key;
         trustedCaFile = mkDefault caCert;
-      });
+      };
       networking.extraHosts = mkIf (config.services.etcd.enable) ''
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
+      services.flannel = with cfg.certs.flannelClient; {
+        kubeconfig = top.lib.mkKubeConfig "flannel" {
+          server = top.apiserverAddress;
+          certFile = cert;
+          keyFile = key;
+        };
+      };
+
       services.kubernetes = {
 
         apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
@@ -359,13 +356,6 @@ in
           proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
           proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
         });
-        addonManager = mkIf top.addonManager.enable {
-          kubeconfig = with cfg.certs.addonManager; {
-            certFile = mkDefault cert;
-            keyFile = mkDefault key;
-          };
-          bootstrapAddonsKubeconfig = clusterAdminKubeconfig;
-        };
         controllerManager = mkIf top.controllerManager.enable {
           serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
           rootCaFile = cfg.certs.controllerManagerClient.caCert;
@@ -374,12 +364,6 @@ in
             keyFile = mkDefault key;
           };
         };
-        flannel = mkIf top.flannel.enable {
-          kubeconfig = with cfg.certs.flannelClient; {
-            certFile = cert;
-            keyFile = key;
-          };
-        };
         scheduler = mkIf top.scheduler.enable {
           kubeconfig = with cfg.certs.schedulerClient; {
             certFile = mkDefault cert;