update kubeadm job

Author: kvaps

deploy/helm/kubernetes/scripts/configure-cluster.sh

@@ -1,47 +1,15 @@
 #!/bin/sh
 set -e
 set -x
-
-# ------------------------------------------------------------------------------
-# Setup environment
-# ------------------------------------------------------------------------------
-
-mkdir -p /etc/kubernetes/pki
-ln -sf /pki/apiserver-etcd-client/tls.crt /etc/kubernetes/pki/apiserver-etcd-client.crt
-ln -sf /pki/apiserver-etcd-client/tls.key /etc/kubernetes/pki/apiserver-etcd-client.key
-ln -sf /pki/apiserver-kubelet-client/tls.crt /etc/kubernetes/pki/apiserver-kubelet-client.crt
-ln -sf /pki/apiserver-kubelet-client/tls.key /etc/kubernetes/pki/apiserver-kubelet-client.key
-ln -sf /pki/apiserver/tls.crt /etc/kubernetes/pki/apiserver.crt
-ln -sf /pki/apiserver/tls.key /etc/kubernetes/pki/apiserver.key
-ln -sf /pki/ca/tls.crt /etc/kubernetes/pki/ca.crt
-ln -sf /pki/ca/tls.key /etc/kubernetes/pki/ca.key
-ln -sf /pki/front-proxy-ca/tls.key /etc/kubernetes/pki/front-proxy-ca.crt
-ln -sf /pki/front-proxy-ca/tls.crt /etc/kubernetes/pki/front-proxy-ca.key
-ln -sf /pki/front-proxy-client/tls.key /etc/kubernetes/pki/front-proxy-client.crt
-ln -sf /pki/front-proxy-client/tls.crt /etc/kubernetes/pki/front-proxy-client.key
+ENDPOINT=$(awk -F'[ "]+' '$1 == "controlPlaneEndpoint:" {print $2}' /config/kubeadmcfg.yaml)
 
 # ------------------------------------------------------------------------------
 # Update secrets and component configs
 # ------------------------------------------------------------------------------
 
-cat >kubeadmcfg.yaml <<EOT
-apiVersion: "kubeadm.k8s.io/v1beta2" 
-kind: ClusterConfiguration 
-imageRepository: k8s.gcr.io 
-controlPlaneEndpoint: "${FULL_NAME}-apiserver:6443"
-EOT
-
-{{- if .Values.apiServer.enabled }}{{"\n"}}
-# generate sa key
-if ! kubectl get secret "${FULL_NAME}-pki-sa" >/dev/null; then
-  kubeadm init phase certs sa
-  kubectl create secret generic "${FULL_NAME}-pki-sa" --from-file=/etc/kubernetes/pki/sa.pub --from-file=/etc/kubernetes/pki/sa.key
-fi
-{{- end }}
-
 # wait for cluster
-echo "Waiting for api-server endpoint ${FULL_NAME}-apiserver:6443..."
-until kubectl --kubeconfig /etc/kubernetes/admin.conf cluster-info >/dev/null 2>/dev/null; do
+echo "Waiting for api-server endpoint ${ENDPOINT}..."
+until kubectl cluster-info >/dev/null 2>/dev/null; do
   sleep 1
 done
 
@@ -52,7 +20,7 @@ export KUBECONFIG=/etc/kubernetes/admin.conf
 
 # upload configuration
 kubeadm init phase upload-config kubeadm --config /config/kubeadmcfg.yaml
-kubectl --kubeconfig /etc/kubernetes/admin.conf patch configmap -n kube-system kubeadm-config \
+kubectl patch configmap -n kube-system kubeadm-config \
   -p '{"data":{"ClusterStatus":"apiEndpoints: {}\napiVersion: kubeadm.k8s.io/v1beta2\nkind: ClusterStatus"}}'
 
 # upload configuration
@@ -63,41 +31,55 @@ kubeadm init phase upload-config kubelet --config /config/kubeadmcfg.yaml -v1 2>
 kubeadm init phase bootstrap-token --config /config/kubeadmcfg.yaml --skip-token-print
 
 # correct apiserver address for the external clients
-tmp="$(mktemp -d)"
-kubectl --kubeconfig "$tmp/kubeconfig" config set clusters..server "https://${CONTROL_PLANE_ENDPOINT:-${FULL_NAME}-apiserver:6443}"
-kubectl --kubeconfig "$tmp/kubeconfig" config set clusters..certificate-authority-data "$(base64 /etc/kubernetes/pki/ca.crt | tr -d '\n')"
-kubectl create configmap cluster-info --from-file="$tmp/kubeconfig" --dry-run=client -o yaml | kubectl --kubeconfig /etc/kubernetes/admin.conf apply -n kube-public -f -
-rm -rf "$tmp"
+kubectl apply -n kube-public -f - <<EOT
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-info
+data:
+  kubeconfig: |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority-data: $(base64 /pki/admin-client/ca.crt | tr -d '\n')
+        server: https://${ENDPOINT}
+      name: ""
+    contexts: null
+    current-context: ""
+    kind: Config
+    preferences: {}
+    users: null
+EOT
 
 {{- if .Values.konnectivityServer.enabled }}{{"\n"}}
 # install konnectivity server
-kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /manifests/konnectivity-server-rbac.yaml
+kubectl apply -f /manifests/konnectivity-server-rbac.yaml
 {{- else }}{{"\n"}}
-kubectl --kubeconfig /etc/kubernetes/admin.conf delete clusterrolebinding/system:konnectivity-server 2>/dev/null || true
+kubectl delete clusterrolebinding/system:konnectivity-server 2>/dev/null || true
 {{- end }}
 
 {{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
 # install konnectivity agent
-kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml
+kubectl apply -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml
 {{- else }}{{"\n"}}
 # uninstall konnectivity agent
-kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-system delete deployment/konnectivity-agent serviceaccount/konnectivity-agent 2>/dev/null || true
+kubectl -n kube-system delete deployment/konnectivity-agent serviceaccount/konnectivity-agent 2>/dev/null || true
 {{- end }}
 
 {{- if .Values.coredns.enabled }}{{"\n"}}
 # install coredns addon
 kubeadm init phase addon coredns --config /config/kubeadmcfg.yaml
 {{- else }}{{"\n"}}
 # uninstall coredns addon
-kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-system delete configmap/coredns deployment/coredns 2>/dev/null || true
+kubectl -n kube-system delete configmap/coredns deployment/coredns 2>/dev/null || true
 {{- end }}
 
 {{- if .Values.kubeProxy.enabled }}{{"\n"}}
 # install kube-proxy addon
 kubeadm init phase addon kube-proxy --config /config/kubeadmcfg.yaml
 {{- else }}{{"\n"}}
 # uninstall kube-proxy addon
-kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-system delete configmap/kube-proxy daemonset/kube-proxy 2>/dev/null || true
+kubectl -n kube-system delete configmap/kube-proxy daemonset/kube-proxy 2>/dev/null || true
 {{- end }}
 
 {{- with .Values.extraManifests }}{{"\n"}}

deploy/helm/kubernetes/templates/admin-deployment.yaml

@@ -71,8 +71,6 @@ spec:
         env:
         - name: KUBECONFIG
           value: "/etc/kubernetes/admin.conf"
-        - name: FULL_NAME
-          value: "{{ $fullName }}"
         {{- with .Values.admin.extraEnv }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -82,6 +80,14 @@ spec:
           readOnly: true
         - mountPath: /pki/admin-client
           name: pki-admin-client
+        - mountPath: /scripts
+          name: scripts
+        {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }}
+        - mountPath: /manifests
+          name: manifests
+        {{- end }}
+        - mountPath: /config
+          name: config
         {{- with .Values.admin.extraVolumeMounts }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -95,6 +101,26 @@ spec:
       - secret:
           secretName: "{{ $fullName }}-pki-admin-client"
         name: pki-admin-client
+      - name: scripts
+        configMap:
+          name: "{{ $fullName }}-kubeadm-scripts"
+          defaultMode: 0777
+      {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }}
+      - name: manifests
+        projected:
+          sources:
+          {{- if or .Values.extraManifests }}
+          - secret:
+              name: "{{ $fullName }}-extra-manifests"
+          {{- end }}
+          {{- if or .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }}
+          - configMap:
+              name: "{{ $fullName }}-konnectivity-manifests"
+          {{- end }}
+      {{- end }}
+      - name: config
+        configMap:
+          name: "{{ $fullName }}-kubeadm-config"
       {{- with .Values.admin.extraVolumes }}
       {{- toYaml . | nindent 6 }}
       {{- end }}

deploy/helm/kubernetes/templates/apiserver-deployment.yaml

@@ -75,14 +75,14 @@ spec:
         - --requestheader-group-headers=X-Remote-Group
         - --requestheader-username-headers=X-Remote-User
         - --secure-port={{ .Values.apiServer.port }}
-        - --service-account-key-file=/pki/sa/sa.pub
+        - --service-account-key-file=/pki/sa/tls.crt
         - --service-cluster-ip-range={{ .Values.apiServer.serviceClusterIPRange }}
         - --tls-cert-file=/pki/apiserver-server/tls.crt
         - --tls-private-key-file=/pki/apiserver-server/tls.key
         - --egress-selector-config-file=/etc/kubernetes/egress-selector-configuration.yaml
         {{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
         - --service-account-issuer=api
-        - --service-account-signing-key-file=/pki/sa/sa.key
+        - --service-account-signing-key-file=/pki/sa/tls.key
         - --api-audiences=system:konnectivity-server
         {{- end }}
         {{- if not (hasKey .Values.apiServer.extraArgs "advertise-address") }}
@@ -154,9 +154,6 @@ spec:
       - secret:
           secretName: "{{ $fullName }}-pki-apiserver-kubelet-client"
         name: pki-apiserver-kubelet-client
-      - secret:
-          secretName: "{{ $fullName }}-pki-ca"
-        name: pki-ca
       - secret:
           secretName: "{{ $fullName }}-pki-sa"
         name: pki-sa

deploy/helm/kubernetes/templates/controller-manager-deployment.yaml

@@ -64,7 +64,7 @@ spec:
         - --requestheader-client-ca-file=/pki/front-proxy-client/tls.crt
         - --root-ca-file=/pki/ca/tls.crt
         - --secure-port={{ .Values.controllerManager.port }}
-        - --service-account-private-key-file=/pki/kubernetes-sa/sa.key
+        - --service-account-private-key-file=/pki/sa/tls.key
         - --use-service-account-credentials=true
         - --tls-cert-file=/pki/controller-manager-server/tls.crt
         - --tls-private-key-file=/pki/controller-manager-server/tls.key
@@ -102,7 +102,7 @@ spec:
           name: pki-ca
         - mountPath: /pki/front-proxy-client
           name: pki-front-proxy-client
-        - mountPath: /pki/kubernetes-sa
+        - mountPath: /pki/sa
           name: pki-sa
         {{- with .Values.controllerManager.extraVolumeMounts }}
         {{- toYaml . | nindent 8 }}

deploy/helm/kubernetes/templates/kubeadm-config.yaml

@@ -9,7 +9,9 @@ data:
   kubeadmcfg.yaml: |+
     apiVersion: kubeadm.k8s.io/v1beta2
     kind: ClusterConfiguration
-    {{- with .Values.controlPlaneEndpoint }}
-    controlPlaneEndpoint: {{ . }}
+    {{- if .Values.controlPlaneEndpoint }}
+    controlPlaneEndpoint: {{ .Values.controlPlaneEndpoint }}
+    {{- else }}
+    controlPlaneEndpoint: {{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }}
     {{- end }}
 {{- end }}

deploy/helm/kubernetes/templates/kubeadm-job.yaml

@@ -57,28 +57,17 @@ spec:
         imagePullPolicy: {{ .Values.admin.image.PullPolicy }}
         command: [ '/scripts/configure-cluster.sh' ]
         env:
-        - name: FULL_NAME
-          value: "{{ $fullName }}"
-        {{- with .Values.controlPlaneEndpoint }}
-        - name: CONTROL_PLANE_ENDPOINT
-          value: "{{ . }}"
-        {{- end }}
+        - name: KUBECONFIG
+          value: "/etc/kubernetes/admin.conf"
         {{- with .Values.admin.job.extraEnv }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
         volumeMounts:
-        - mountPath: /pki/front-proxy-client
-          name: pki-front-proxy-client
-        - mountPath: /pki/apiserver
-          name: pki-apiserver
-        - mountPath: /pki/apiserver-etcd-client
-          name: pki-apiserver-etcd-client
-        - mountPath: /pki/apiserver-kubelet-client
-          name: pki-apiserver-kubelet-client
-        - mountPath: /pki/ca
-          name: pki-ca
-        - mountPath: /pki/front-proxy-ca
-          name: pki-front-proxy-ca
+        - mountPath: /etc/kubernetes/
+          name: kubeconfig
+          readOnly: true
+        - mountPath: /pki/admin-client
+          name: pki-admin-client
         - mountPath: /scripts
           name: scripts
         {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }}
@@ -87,25 +76,19 @@ spec:
         {{- end }}
         - mountPath: /config
           name: config
+        {{- with .Values.admin.extraVolumeMounts }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- with .Values.admin.sidecars }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
       volumes:
+      - configMap:
+          name: "{{ $fullName }}-admin-conf"
+        name: kubeconfig
       - secret:
-          secretName: "{{ $fullName }}-pki-front-proxy-client"
-        name: pki-front-proxy-client
-      - secret:
-          secretName: "{{ $fullName }}-pki-apiserver-server"
-        name: pki-apiserver
-      - secret:
-          secretName: "{{ $fullName }}-pki-apiserver-etcd-client"
-        name: pki-apiserver-etcd-client
-      - secret:
-          secretName: "{{ $fullName }}-pki-apiserver-kubelet-client"
-        name: pki-apiserver-kubelet-client
-      - secret:
-          secretName: "{{ $fullName }}-pki-ca"
-        name: pki-ca
-      - secret:
-          secretName: "{{ $fullName }}-pki-front-proxy-ca"
-        name: pki-front-proxy-ca
+          secretName: "{{ $fullName }}-pki-admin-client"
+        name: pki-admin-client
       - name: scripts
         configMap:
           name: "{{ $fullName }}-kubeadm-scripts"
@@ -126,4 +109,7 @@ spec:
       - name: config
         configMap:
           name: "{{ $fullName }}-kubeadm-config"
+      {{- with .Values.admin.extraVolumes }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
 {{- end }}

deploy/helm/kubernetes/templates/kubernetes-certs.yaml

@@ -30,6 +30,27 @@ spec:
   ca:
     secretName: "{{ $fullName }}-pki-ca"
 ---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: "{{ $fullName }}-pki-sa"
+spec:
+  commonName: "{{ $certName }}-sa"
+  secretName: "{{ $fullName }}-pki-sa"
+  duration: 87600h # 3650d
+  renewBefore: 8760h # 365d
+  subject:
+    organizations:
+    - "{{ $fullName }}"
+  usages:
+  - "signing"
+  - "key encipherment"
+  - "cert sign"
+  isCA: true
+  issuerRef:
+    name: "{{ $fullName }}-selfsigning-issuer"
+    kind: Issuer
+---
 {{- $svcName1 := printf "%s-controller-manager" $fullName }}
 {{- $svcName2 := printf "%s-controller-manager.%s" $fullName .Release.Namespace }}
 {{- $svcName3 := printf "%s-controller-manager.%s.svc" $fullName .Release.Namespace }}