Skip to content

Commit 38aaa40

Browse files
casey-quinncasey-quinn
committed
test(k8s): add no-portfwd coverage
Signed-off-by: Casey Quinn <casey.quinn@agyn.io>
1 parent bbe4a1a commit 38aaa40

File tree

3 files changed

+256
-0
lines changed

3 files changed

+256
-0
lines changed

‎.github/workflows/test.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,14 @@ jobs:
383383
run: ./hack/bats/lib/bats-core/bin/bats --timing ./hack/bats/extras/k8s.bats
384384
env:
385385
LIMA_BATS_ALL_TESTS_RETRIES: 3
386+
- name: Cache image used by templates/k8s-no-portfwd.yaml
387+
uses: ./.github/actions/setup_cache_for_template
388+
with:
389+
template: templates/k8s-no-portfwd.yaml
390+
- name: "Run BATS k8s-no-portfwd tests"
391+
run: ./hack/bats/lib/bats-core/bin/bats --timing ./hack/bats/extras/k8s-no-portfwd.bats
392+
env:
393+
LIMA_BATS_ALL_TESTS_RETRIES: 3
386394

387395
colima:
388396
name: "Colima tests (QEMU, Linux host)"

‎hack/bats/extras/k8s-no-portfwd.bats‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# SPDX-FileCopyrightText: Copyright The Lima Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# This test verifies that a Kubernetes cluster created from the k8s-no-portfwd
5+
# template does not expose NodePort services on the host even though the
6+
# services are available within the cluster.
7+
8+
load "../helpers/load"
9+
10+
: "${TEMPLATE:=k8s-no-portfwd}"
11+
12+
NAME="k8snpf"
13+
14+
local_setup() {
15+
limactl delete --force "${NAME}-0" || :
16+
limactl start --tty=false --name "${NAME}-0" "template:${TEMPLATE}" 3>&- 4>&-
17+
local nodes=""
18+
for _ in $(seq 1 60); do
19+
nodes=$(k get node -o name 2>/dev/null || true)
20+
if [[ -n "${nodes}" ]]; then
21+
break
22+
fi
23+
sleep 5
24+
done
25+
if [[ -z "${nodes}" ]]; then
26+
echo "failed to list kubernetes nodes" >&2
27+
return 1
28+
fi
29+
for node in ${nodes}; do
30+
k wait --timeout=5m --for=condition=ready "${node}"
31+
done
32+
}
33+
34+
local_teardown() {
35+
limactl delete --force "${NAME}-0" || :
36+
}
37+
38+
k() {
39+
# The host home directory is not mounted in the case of k8s.
40+
limactl shell --workdir=/ "${NAME}-0" -- kubectl "$@"
41+
}
42+
43+
# bats test_tags=nodes:1
44+
@test 'NodePorts are not forwarded to the host' {
45+
services=(nginx coredns)
46+
for svc in "${services[@]}"; do
47+
k create deployment "$svc" --image="${TEST_CONTAINER_IMAGES["$svc"]}"
48+
done
49+
for svc in "${services[@]}"; do
50+
k rollout status deployment "$svc" --timeout 60s
51+
done
52+
53+
# TCP NodePort should be unreachable from the host.
54+
k create service nodeport nginx --node-port=31080 --tcp=80:80
55+
run curl --fail --silent --show-error --connect-timeout 5 --max-time 10 http://localhost:31080
56+
assert_failure
57+
58+
# UDP NodePort should be unreachable from the host.
59+
k expose deployment coredns --port=53 --type=NodePort \
60+
--overrides='{"spec":{"ports":[{"port":53,"protocol":"UDP","targetPort":53,"nodePort":32053}]}}'
61+
run dig @127.0.0.1 -p 32053 lima-vm.io +tries=1 +time=3
62+
assert_failure
63+
64+
# Ensure hostagent never reported setting up host forwarding.
65+
instance_dir=$(limactl list "${NAME}-0" --format '{{.Dir}}')
66+
run grep -n "Forwarding " "${instance_dir}/ha.stderr"
67+
assert_failure
68+
69+
for svc in "${services[@]}"; do
70+
k delete service "$svc"
71+
k delete deployment "$svc"
72+
done
73+
}

‎templates/k8s-no-portfwd.yaml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# Deploy kubernetes via kubeadm with host port forwarding disabled.
2+
# $ limactl start ./k8s-no-portfwd.yaml
3+
# $ limactl shell k8snpf kubectl
4+
5+
minimumLimaVersion: 2.0.0
6+
7+
base: template:_images/ubuntu-lts
8+
9+
# Mounts are disabled in this template, but can be enabled optionally.
10+
mounts: []
11+
12+
portForwards:
13+
- guestIP: "0.0.0.0"
14+
guestPortRange: [1, 65535]
15+
proto: "any"
16+
ignore: true
17+
18+
containerd:
19+
system: true
20+
user: false
21+
provision:
22+
# See <https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/>
23+
- mode: system
24+
script: |
25+
#!/bin/bash
26+
set -eux -o pipefail
27+
command -v kubeadm >/dev/null 2>&1 && exit 0
28+
# Install and configure prerequisites
29+
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
30+
overlay
31+
br_netfilter
32+
EOF
33+
modprobe overlay
34+
modprobe br_netfilter
35+
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
36+
net.bridge.bridge-nf-call-iptables = 1
37+
net.ipv4.ip_forward = 1
38+
net.bridge.bridge-nf-call-ip6tables = 1
39+
EOF
40+
sysctl --system
41+
# Installing kubeadm, kubelet and kubectl
42+
export DEBIAN_FRONTEND=noninteractive
43+
apt-get update
44+
apt-get install -y apt-transport-https ca-certificates curl
45+
VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt | sed -e 's/v//' | cut -d'.' -f1-2)
46+
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${VERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
47+
curl -fsSL https://pkgs.k8s.io/core:/stable:/v${VERSION}/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
48+
apt-get update
49+
apt-get install -y kubelet kubeadm kubectl && apt-mark hold kubelet kubeadm kubectl
50+
systemctl enable --now kubelet
51+
# See <https://kubernetes.io/docs/setup/production-environment/container-runtimes/>
52+
- mode: system
53+
script: |
54+
#!/bin/bash
55+
set -eux -o pipefail
56+
[ -e /etc/containerd/conf.d/k8s.toml ] && exit 0
57+
mkdir -p /etc/containerd/conf.d
58+
# Configuring the systemd cgroup driver
59+
# Overriding the sandbox (pause) image
60+
cat <<EOF >/etc/containerd/conf.d/k8s.toml
61+
version = 2
62+
[plugins]
63+
[plugins."io.containerd.grpc.v1.cri"]
64+
sandbox_image = "$(kubeadm config images list | grep pause | sort -r | head -n1)"
65+
[plugins."io.containerd.grpc.v1.cri".containerd]
66+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
67+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
68+
runtime_type = "io.containerd.runc.v2"
69+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
70+
SystemdCgroup = true
71+
[plugins."io.containerd.cri.v1.runtime".cni]
72+
bin_dirs = ["/usr/local/libexec/cni","/opt/cni/bin"]
73+
EOF
74+
systemctl restart containerd
75+
# See <https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/>
76+
- mode: system
77+
script: |
78+
#!/bin/bash
79+
set -eux -o pipefail
80+
test -e /etc/kubernetes/admin.conf && exit 0
81+
export KUBECONFIG=/etc/kubernetes/admin.conf
82+
systemctl stop kubelet
83+
kubeadm config images list
84+
kubeadm config images pull --cri-socket=unix:///run/containerd/containerd.sock
85+
systemctl start kubelet
86+
# Initializing your control-plane node
87+
cat <<EOF >kubeadm-config.yaml
88+
kind: InitConfiguration
89+
apiVersion: kubeadm.k8s.io/v1beta4
90+
nodeRegistration:
91+
criSocket: unix:///run/containerd/containerd.sock
92+
---
93+
kind: ClusterConfiguration
94+
apiVersion: kubeadm.k8s.io/v1beta4
95+
apiServer:
96+
certSANs: # --apiserver-cert-extra-sans
97+
- "127.0.0.1"
98+
networking:
99+
podSubnet: "10.244.0.0/16" # --pod-network-cidr
100+
---
101+
kind: KubeletConfiguration
102+
apiVersion: kubelet.config.k8s.io/v1beta1
103+
cgroupDriver: systemd
104+
EOF
105+
kubeadm init --config kubeadm-config.yaml
106+
# Installing a Pod network add-on
107+
kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.27.4/kube-flannel.yml
108+
# Control plane node isolation
109+
kubectl taint nodes --all node-role.kubernetes.io/control-plane-
110+
# Symlink the kubeconfig file to the default location for kubectl
111+
mkdir -p /root/.kube && ln -sf $KUBECONFIG /root/.kube/config
112+
# Replace the server address with localhost, so that it works from the host.
113+
# The original kubeconfig is kept unmodified, so that `kubeadm token create --print-join-command`
114+
# can still print the reachable address.
115+
sed -e "/server:/ s|https://.*:\([0-9]*\)$|https://127.0.0.1:\1|" $KUBECONFIG >/root/.kube/config.localhost
116+
- mode: system
117+
script: |
118+
#!/bin/bash
119+
set -eux -o pipefail
120+
export KUBECONFIG=/etc/kubernetes/admin.conf
121+
mkdir -p {{.Home}}/.kube
122+
cp -f $KUBECONFIG {{.Home}}/.kube/config
123+
chown -R {{.User}} {{.Home}}/.kube
124+
probes:
125+
- description: "kubeadm to be installed"
126+
script: |
127+
#!/bin/bash
128+
set -eux -o pipefail
129+
if ! timeout 30s bash -c "until command -v kubeadm >/dev/null 2>&1; do sleep 3; done"; then
130+
echo >&2 "kubeadm is not installed yet"
131+
exit 1
132+
fi
133+
hint: |
134+
See "/var/log/cloud-init-output.log" in the guest
135+
- description: "kubernetes images to be pulled"
136+
script: |
137+
#!/bin/bash
138+
set -eux -o pipefail
139+
if ! timeout 30s bash -c "images=\"$(kubeadm config images list)\"; until for image in $images; do sudo ctr -n k8s.io image inspect $image >/dev/null; done; do sleep 3; done"; then
140+
echo >&2 "k8s images are not pulled yet"
141+
exit 1
142+
fi
143+
- description: "kubeadm to be completed"
144+
script: |
145+
#!/bin/bash
146+
set -eux -o pipefail
147+
if ! timeout 300s bash -c "until test -f /etc/kubernetes/admin.conf; do sleep 3; done"; then
148+
echo >&2 "k8s is not running yet"
149+
exit 1
150+
fi
151+
hint: |
152+
The k8s kubeconfig file has not yet been created.
153+
- description: "kubernetes cluster to be running"
154+
script: |
155+
#!/bin/bash
156+
set -eux -o pipefail
157+
if ! timeout 300s bash -c "until kubectl version >/dev/null 2>&1; do sleep 3; done"; then
158+
echo >&2 "kubernetes cluster is not up and running yet"
159+
exit 1
160+
fi
161+
- description: "coredns deployment to be running"
162+
script: |
163+
#!/bin/bash
164+
set -eux -o pipefail
165+
kubectl wait -n kube-system --timeout=180s --for=condition=available deploy coredns
166+
copyToHost:
167+
- guest: "/root/.kube/config.localhost"
168+
host: "{{.Dir}}/copied-from-guest/kubeconfig.yaml"
169+
deleteOnStop: true
170+
message: |
171+
To run `kubectl` on the host (assumes kubectl is installed), run the following commands:
172+
------
173+
export KUBECONFIG="{{.Dir}}/copied-from-guest/kubeconfig.yaml"
174+
kubectl ...
175+
------

0 commit comments

Comments
 (0)