Skip to content

Commit 6a52355

Browse files
kvapskvaps
committed
Merge pull request #9 from kvaps/konnectivity-grpc
GRPC mode for konnectivity-server
2 parents fed78ea + c020409 commit 6a52355

File tree

9 files changed

+161
-92
lines changed

9 files changed

+161
-92
lines changed

‎deploy/helm/kubernetes/manifests/konnectivity-agent-deployment.yaml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@ metadata:
1515
namespace: kube-system
1616
name: konnectivity-agent
1717
spec:
18+
{{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
1819
replicas: {{ .Values.konnectivityAgent.replicaCount }}
20+
{{- else }}
21+
replicas: {{ .Values.apiServer.replicaCount }}
22+
{{- end }}
1923
selector:
2024
matchLabels:
2125
k8s-app: konnectivity-agent
@@ -83,8 +87,10 @@ spec:
8387
- --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
8488

8589
{{- if not (hasKey .Values.konnectivityAgent.extraArgs "proxy-server-host") }}
86-
{{- if .Values.konnectivityServer.service.loadBalancerIP }}
90+
{{- if and (eq .Values.konnectivityServer.mode "HTTPConnect") .Values.konnectivityServer.service.loadBalancerIP }}
8791
- --proxy-server-host={{ .Values.konnectivityServer.service.loadBalancerIP }}
92+
{{- else if and (eq .Values.konnectivityServer.mode "GRPC") .Values.apiServer.service.loadBalancerIP }}
93+
- --proxy-server-host={{ .Values.apiServer.service.loadBalancerIP }}
8894
{{- else }}
8995
{{- fail ".konnectivityAgent.extraArgs.proxy-server-host must be specified!" }}
9096
{{- end }}

‎deploy/helm/kubernetes/templates/_helpers.tpl‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,115 @@ Take the first IP address from the serviceClusterIPRange for the kube-dns servic
7474
{{- $octetsList := splitList "." .Values.apiServer.serviceClusterIPRange -}}
7575
{{- printf "%d.%d.%d.%d" (index $octetsList 0 | int) (index $octetsList 1 | int) (index $octetsList 2 | int) 1 -}}
7676
{{- end -}}
77+
78+
{{/*
79+
Template for konnectivityServer containers
80+
*/}}
81+
{{- define "kubernetes.konnectivityServer.containers" -}}
82+
- command:
83+
- /proxy-server
84+
- --logtostderr=true
85+
- --server-count={{ .Values.konnectivityServer.replicaCount }}
86+
- --server-id=$(POD_NAME)
87+
- --cluster-cert=/pki/apiserver/tls.crt
88+
- --cluster-key=/pki/apiserver/tls.key
89+
{{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
90+
- --mode=http-connect
91+
- --server-port={{ .Values.konnectivityServer.ports.server }}
92+
- --server-ca-cert=/pki/konnectivity-server/ca.crt
93+
- --server-cert=/pki/konnectivity-server/tls.crt
94+
- --server-key=/pki/konnectivity-server/tls.key
95+
{{- else }}
96+
- --mode=grpc
97+
- --uds-name=/run/konnectivity-server/konnectivity-server.socket
98+
- --server-port=0
99+
{{- end }}
100+
- --agent-port={{ .Values.konnectivityServer.ports.agent }}
101+
- --admin-port={{ .Values.konnectivityServer.ports.admin }}
102+
- --health-port={{ .Values.konnectivityServer.ports.health }}
103+
- --agent-namespace=kube-system
104+
- --agent-service-account=konnectivity-agent
105+
- --kubeconfig=/etc/kubernetes/konnectivity-server.conf
106+
- --authentication-audience=system:konnectivity-server
107+
{{- range $key, $value := .Values.konnectivityServer.extraArgs }}
108+
- --{{ $key }}={{ $value }}
109+
{{- end }}
110+
ports:
111+
{{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
112+
- containerPort: {{ .Values.konnectivityServer.ports.server }}
113+
name: server
114+
{{- end }}
115+
- containerPort: {{ .Values.konnectivityServer.ports.agent }}
116+
name: agent
117+
- containerPort: {{ .Values.konnectivityServer.ports.admin }}
118+
name: admin
119+
- containerPort: {{ .Values.konnectivityServer.ports.health }}
120+
name: health
121+
{{- with .Values.konnectivityServer.image }}
122+
image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}"
123+
imagePullPolicy: {{ .pullPolicy }}
124+
{{- end }}
125+
livenessProbe:
126+
failureThreshold: 8
127+
httpGet:
128+
path: /healthz
129+
port: {{ .Values.konnectivityServer.ports.health }}
130+
scheme: HTTP
131+
initialDelaySeconds: 30
132+
timeoutSeconds: 60
133+
name: konnectivity-server
134+
resources:
135+
{{- toYaml .Values.konnectivityServer.resources | nindent 10 }}
136+
env:
137+
- name: POD_NAME
138+
valueFrom:
139+
fieldRef:
140+
fieldPath: metadata.name
141+
{{- with .Values.konnectivityServer.extraEnv }}
142+
{{- toYaml . | nindent 8 }}
143+
{{- end }}
144+
volumeMounts:
145+
- mountPath: /pki/apiserver
146+
name: pki-apiserver
147+
{{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
148+
- mountPath: /pki/konnectivity-server
149+
name: pki-konnectivity-server
150+
{{- else }}
151+
- mountPath: /run/konnectivity-server
152+
name: konnectivity-uds
153+
{{- end }}
154+
- mountPath: /pki/konnectivity-server-client
155+
name: pki-konnectivity-server-client
156+
- mountPath: /etc/kubernetes/
157+
name: kubeconfig
158+
readOnly: true
159+
{{- with .Values.konnectivityServer.extraVolumeMounts }}
160+
{{- toYaml . | nindent 8 }}
161+
{{- end }}
162+
{{- end -}}
163+
164+
{{/*
165+
Template for konnectivityServer volumes
166+
*/}}
167+
{{- define "kubernetes.konnectivityServer.volumes" -}}
168+
- secret:
169+
secretName: "{{ template "kubernetes.fullname" . }}-pki-apiserver-server"
170+
name: pki-apiserver
171+
{{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
172+
- secret:
173+
secretName: "{{ template "kubernetes.fullname" . }}-pki-konnectivity-server"
174+
name: pki-konnectivity-server
175+
{{- else }}
176+
- secret:
177+
secretName: "{{ template "kubernetes.fullname" . }}-pki-konnectivity-server-client"
178+
name: pki-konnectivity-server-client
179+
- emptyDir: {}
180+
name: konnectivity-uds
181+
{{- end }}
182+
- configMap:
183+
name: "{{ template "kubernetes.fullname" . }}-konnectivity-server-conf"
184+
name: kubeconfig
185+
{{- with .Values.konnectivityServer.extraVolumes }}
186+
{{- toYaml . | nindent 6 }}
187+
{{- end }}
188+
{{- end -}}

‎deploy/helm/kubernetes/templates/apiserver-config.yaml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,23 @@ data:
1212
- name: cluster
1313
connection:
1414
{{- if and .Values.konnectivityServer.enabled }}
15-
proxyProtocol: HTTPConnect
15+
{{- if has .Values.konnectivityServer.mode (list "HTTPConnect" "GRPC") }}
16+
proxyProtocol: {{ .Values.konnectivityServer.mode }}
17+
{{- else }}
18+
{{- fail ".Values.konnectivityServer.mode supports only \"HTTPConnect\" and \"GRPC\" values" }}
19+
{{- end }}
1620
transport:
21+
{{- if eq .Values.konnectivityServer.mode "GRPC" }}
22+
uds:
23+
udsName: /run/konnectivity-server/konnectivity-server.socket
24+
{{- else }}
1725
tcp:
1826
url: "https://{{ $fullName }}-konnectivity-server:8131"
1927
TLSConfig:
2028
caBundle: /pki/konnectivity-client/ca.crt
2129
clientKey: /pki/konnectivity-client/tls.key
2230
clientCert: /pki/konnectivity-client/tls.crt
31+
{{- end }}
2332
{{- else }}
2433
proxyProtocol: Direct
2534
{{- end }}

‎deploy/helm/kubernetes/templates/apiserver-deployment.yaml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ spec:
101101
- --egress-selector-config-file=/etc/kubernetes/egress-selector-configuration.yaml
102102
- --service-account-issuer=https://kubernetes.default.svc.cluster.local
103103
- --service-account-signing-key-file=/pki/sa/tls.key
104-
{{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
104+
{{- if .Values.konnectivityAgent.enabled }}
105105
- --api-audiences=system:konnectivity-server
106106
{{- end }}
107107
{{- if not (hasKey .Values.apiServer.extraArgs "advertise-address") }}
@@ -147,13 +147,20 @@ spec:
147147
name: pki-apiserver-kubelet-client
148148
- mountPath: /pki/sa
149149
name: pki-sa
150-
{{- if .Values.konnectivityServer.enabled }}{{"\n"}}
150+
{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
151151
- mountPath: /pki/konnectivity-client
152152
name: pki-konnectivity-client
153153
{{- end }}
154+
{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
155+
- mountPath: /run/konnectivity-server
156+
name: konnectivity-uds
157+
{{- end }}
154158
{{- with .Values.apiServer.extraVolumeMounts }}
155159
{{- toYaml . | nindent 8 }}
156160
{{- end }}
161+
{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
162+
{{ template "kubernetes.konnectivityServer.containers" . }}
163+
{{- end }}
157164
{{- with .Values.apiServer.sidecars }}
158165
{{- toYaml . | nindent 6 }}
159166
{{- end }}
@@ -176,11 +183,14 @@ spec:
176183
- secret:
177184
secretName: "{{ $fullName }}-pki-sa"
178185
name: pki-sa
179-
{{- if .Values.konnectivityServer.enabled }}{{"\n"}}
186+
{{- if .Values.konnectivityServer.enabled }}
180187
- secret:
181188
secretName: "{{ $fullName }}-pki-konnectivity-client"
182189
name: pki-konnectivity-client
183190
{{- end }}
191+
{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
192+
{{ template "kubernetes.konnectivityServer.volumes" . }}
193+
{{- end }}
184194
{{- with .Values.apiServer.extraVolumes }}
185195
{{- toYaml . | nindent 6 }}
186196
{{- end }}

‎deploy/helm/kubernetes/templates/apiserver-service.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ metadata:
1111
{{- toYaml . | nindent 4 }}
1212
{{- end }}
1313
annotations:
14-
{{- if not (index .Values.apiServer.service.annotations "metallb.universe.tf/allow-shared-ip") }}
14+
{{- if and (not (index .Values.apiServer.service.annotations "metallb.universe.tf/allow-shared-ip")) (eq .Values.konnectivityServer.mode "HTTPConnect") }}
1515
metallb.universe.tf/allow-shared-ip: {{ $fullName }}
1616
{{- end }}
1717
{{- with .Values.apiServer.service.annotations }}
@@ -28,6 +28,13 @@ spec:
2828
{{- with .Values.apiServer.service.nodePort }}
2929
nodePort: {{ . }}
3030
{{- end }}
31+
{{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
32+
- port: {{ .Values.konnectivityServer.ports.agent }}
33+
name: agent
34+
{{- with .Values.konnectivityServer.service.nodePorts.client }}
35+
nodePort: {{ . }}
36+
{{- end }}
37+
{{- end }}
3138
selector:
3239
app: {{ $fullName }}-apiserver
3340
{{- end }}

‎deploy/helm/kubernetes/templates/konnectivity-certs.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if and .Values.konnectivityServer.enabled }}
1+
{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
22
{{- $fullName := include "kubernetes.fullname" . -}}
33
{{- $certName := include "kubernetes.certname" . -}}
44
---
@@ -30,6 +30,7 @@ metadata:
3030
spec:
3131
ca:
3232
secretName: "{{ $fullName }}-pki-konnectivity-ca"
33+
3334
---
3435
{{- $svcName1 := printf "%s-konnectivity-server" $fullName }}
3536
{{- $svcName2 := printf "%s-konnectivity-server.%s" $fullName .Release.Namespace }}

‎deploy/helm/kubernetes/templates/konnectivity-server-deployment.yaml‎

Lines changed: 3 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if .Values.konnectivityServer.enabled }}
1+
{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
22
{{- $fullName := include "kubernetes.fullname" . -}}
33
---
44
apiVersion: apps/v1
@@ -68,90 +68,9 @@ spec:
6868
{{- end }}
6969
automountServiceAccountToken: false
7070
containers:
71-
- command:
72-
- /proxy-server
73-
- --logtostderr=true
74-
- --server-count={{ .Values.konnectivityServer.replicaCount }}
75-
- --server-id=$(POD_NAME)
76-
- --cluster-cert=/pki/apiserver/tls.crt
77-
- --cluster-key=/pki/apiserver/tls.key
78-
- --server-ca-cert=/pki/konnectivity-server/ca.crt
79-
- --server-cert=/pki/konnectivity-server/tls.crt
80-
- --server-key=/pki/konnectivity-server/tls.key
81-
- --mode=http-connect
82-
- --server-port={{ .Values.konnectivityServer.ports.server }}
83-
- --agent-port={{ .Values.konnectivityServer.ports.agent }}
84-
- --admin-port={{ .Values.konnectivityServer.ports.admin }}
85-
- --health-port={{ .Values.konnectivityServer.ports.health }}
86-
- --agent-namespace=kube-system
87-
- --agent-service-account=konnectivity-agent
88-
- --kubeconfig=/etc/kubernetes/konnectivity-server.conf
89-
- --authentication-audience=system:konnectivity-server
90-
{{- range $key, $value := .Values.konnectivityServer.extraArgs }}
91-
- --{{ $key }}={{ $value }}
92-
{{- end }}
93-
ports:
94-
- containerPort: {{ .Values.konnectivityServer.ports.server }}
95-
name: server
96-
- containerPort: {{ .Values.konnectivityServer.ports.agent }}
97-
name: agent
98-
- containerPort: {{ .Values.konnectivityServer.ports.admin }}
99-
name: admin
100-
- containerPort: {{ .Values.konnectivityServer.ports.health }}
101-
name: health
102-
{{- with .Values.konnectivityServer.image }}
103-
image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}"
104-
imagePullPolicy: {{ .pullPolicy }}
105-
{{- end }}
106-
livenessProbe:
107-
failureThreshold: 8
108-
httpGet:
109-
path: /healthz
110-
port: {{ .Values.konnectivityServer.ports.health }}
111-
scheme: HTTP
112-
initialDelaySeconds: 30
113-
timeoutSeconds: 60
114-
name: konnectivity-server
115-
resources:
116-
{{- toYaml .Values.konnectivityServer.resources | nindent 10 }}
117-
env:
118-
- name: POD_NAME
119-
valueFrom:
120-
fieldRef:
121-
fieldPath: metadata.name
122-
{{- with .Values.konnectivityServer.extraEnv }}
123-
{{- toYaml . | nindent 8 }}
124-
{{- end }}
125-
volumeMounts:
126-
- mountPath: /pki/apiserver
127-
name: pki-apiserver
128-
- mountPath: /pki/konnectivity-server
129-
name: pki-konnectivity-server
130-
- mountPath: /pki/konnectivity-server-client
131-
name: pki-konnectivity-server-client
132-
- mountPath: /etc/kubernetes/
133-
name: kubeconfig
134-
readOnly: true
135-
{{- with .Values.konnectivityServer.extraVolumeMounts }}
136-
{{- toYaml . | nindent 8 }}
137-
{{- end }}
71+
{{ template "kubernetes.konnectivityServer.containers" . }}
13872
{{- with .Values.konnectivityServer.sidecars }}
13973
{{- toYaml . | nindent 6 }}
14074
{{- end }}
141-
volumes:
142-
- secret:
143-
secretName: "{{ $fullName }}-pki-apiserver-server"
144-
name: pki-apiserver
145-
- secret:
146-
secretName: "{{ $fullName }}-pki-konnectivity-server"
147-
name: pki-konnectivity-server
148-
- secret:
149-
secretName: "{{ $fullName }}-pki-konnectivity-server-client"
150-
name: pki-konnectivity-server-client
151-
- configMap:
152-
name: "{{ $fullName }}-konnectivity-server-conf"
153-
name: kubeconfig
154-
{{- with .Values.konnectivityServer.extraVolumes }}
155-
{{- toYaml . | nindent 6 }}
156-
{{- end }}
75+
{{ template "kubernetes.konnectivityServer.volumes" . }}
15776
{{- end }}

‎deploy/helm/kubernetes/templates/konnectivity-server-service.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled }}
1+
{{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
22
{{- $fullName := include "kubernetes.fullname" . -}}
33
---
44
apiVersion: v1

‎deploy/helm/kubernetes/values.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,11 @@ coredns:
287287

288288
konnectivityServer:
289289
enabled: false
290+
# This controls the protocol between the API Server and the Konnectivity server.
291+
# Supported values are "GRPC" and "HTTPConnect".
292+
# "GRPC" will deploy konnectivity-server as a sidecar for apiserver
293+
# "HTTPConnect" will deploy konnectivity-server as separate deployment
294+
mode: GRPC
290295
image:
291296
repository: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
292297
tag: v0.0.20

0 commit comments

Comments
 (0)