Skip to content

Fix/end to end tests #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Damien-DAGORN merged 8 commits into from
Dec 30, 2025
Merged

Fix/end to end tests #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a651a02
feat: add cache sync timeout configuration
fahedouch Dec 29, 2025
5e7d2ad
feat: add environment variable support for cache sync timeout
fahedouch Dec 29, 2025
d714c25
fix(ci): increase provider installation timeout
Damien-DAGORN Dec 29, 2025
18774f0
refactor(main): remove hardcoded QPS and Burst values from config
fahedouch Dec 29, 2025
ed3513a
fix(integration-tests): increase timeout for provider installation to…
fahedouch Dec 29, 2025
7d978cf
fix(integration-tests): improve error handling for provider installat…
fahedouch Dec 29, 2025
3d841e3
debug(ci): use tmate for debuging
Damien-DAGORN Dec 29, 2025
458a8d0
fix(ci): use OCI by default and add the possibility to skip mariadb test
Damien-DAGORN Dec 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 200 additions & 48 deletions cluster/local/integration_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,24 @@ eval $(make --no-print-directory -C ${projectdir} build.vars)

# ------------------------------

# Provide safe defaults for variables that may not be set by build.vars
: "${KIND_VERSION:=unknown}"
: "${KIND_NODE_IMAGE_TAG:=v1.23.4}"

SAFEHOSTARCH="${SAFEHOSTARCH:-amd64}"
CONTROLLER_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-${SAFEHOSTARCH}"

version_tag="$(cat ${projectdir}/_output/version)"
# tag as latest version to load into kind cluster
K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-inttests}"

# Optional: set USE_OCI=true to use an in-cluster OCI registry for provider package delivery
# Modes:
# - USE_OCI=true => Push .xpkg into an in-cluster registry and install Provider from OCI (no host cache, no PVC)
# - USE_OCI=false => Extract .xpkg to .gz on host and mount as cache for Crossplane (offline local cache)
USE_OCI=${USE_OCI:-true}


PACKAGE_NAME="provider-sql"
MARIADB_ROOT_PW=$(openssl rand -base64 32)
MARIADB_TEST_PW=$(openssl rand -base64 32)
Expand All @@ -56,6 +67,12 @@ if [ "$skipcleanup" != true ]; then
function cleanup {
echo_step "Cleaning up..."
export KUBECONFIG=
# stop port-forward if running
if [ -f "${projectdir}/.work/registry-pf.pid" ]; then
pfpid=$(cat "${projectdir}/.work/registry-pf.pid" || true)
if [ -n "$pfpid" ]; then kill "$pfpid" 2>/dev/null || true; fi
rm -f "${projectdir}/.work/registry-pf.pid"
fi
cleanup_cluster
}

Expand All @@ -77,18 +94,33 @@ integration_tests_end() {
}

setup_cluster() {
echo_step "setting up local package cache"

if [ "${USE_OCI}" = true ]; then
echo_sub_step "Mode: OCI (no host cache, no PVC)"
else
echo_sub_step "Mode: Local cache (.gz mounted via PV/PVC)"
fi
local cache_path="${projectdir}/.work/inttest-package-cache"
mkdir -p "${cache_path}"
echo "created cache dir at ${cache_path}"
"${UP}" alpha xpkg xp-extract --from-xpkg "${OUTPUT_DIR}"/xpkg/linux_"${SAFEHOSTARCH}"/"${PACKAGE_NAME}"-"${VERSION}".xpkg -o "${cache_path}/${PACKAGE_NAME}.gz"
chmod 644 "${cache_path}/${PACKAGE_NAME}.gz"

local node_image="kindest/node:${KIND_NODE_IMAGE_TAG}"
echo_step "creating k8s cluster using kind ${KIND_VERSION} and node image ${node_image}"

local config="$( cat <<EOF
if [ "${USE_OCI}" = true ]; then
echo_step "creating k8s cluster (no cache mount) using kind ${KIND_VERSION} and ${node_image}"
local config="$( cat <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
EOF
)"
echo "${config}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --image="${node_image}" --config=-
else
echo_step "setting up local package cache (.xpkg -> .gz)"
mkdir -p "${cache_path}"
echo "created cache dir at ${cache_path}"
"${UP}" alpha xpkg xp-extract --from-xpkg "${OUTPUT_DIR}"/xpkg/linux_"${SAFEHOSTARCH}"/"${PACKAGE_NAME}"-"${VERSION}".xpkg -o "${cache_path}/${PACKAGE_NAME}-${VERSION}.gz"
chmod 644 "${cache_path}/${PACKAGE_NAME}-${VERSION}.gz"

echo_step "creating k8s cluster (with cache mount) using kind ${KIND_VERSION} and ${node_image}"
local config="$( cat <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
Expand All @@ -97,21 +129,21 @@ nodes:
- hostPath: "${cache_path}/"
containerPath: /cache
EOF
)"
echo "${config}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --image="${node_image}" --config=-
)"
echo "${config}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --image="${node_image}" --config=-
fi

echo_step "tag controller image and load it into kind cluster"

docker tag "${CONTROLLER_IMAGE}" "xpkg.crossplane.io/${PACKAGE_NAME}:latest"
"${KIND}" load docker-image "xpkg.crossplane.io/${PACKAGE_NAME}:latest" --name="${K8S_CLUSTER}"
echo_step "load controller runtime image into kind cluster"
"${KIND}" load docker-image "${CONTROLLER_IMAGE}" --name="${K8S_CLUSTER}"

echo_step "create crossplane-system namespace"

"${KUBECTL}" create ns crossplane-system

echo_step "create persistent volume for mounting package-cache"
if [ "${USE_OCI}" != true ]; then
echo_step "create persistent volume for mounting package-cache"

local pv_yaml="$( cat <<EOF
local pv_yaml="$( cat <<EOF
apiVersion: v1
kind: PersistentVolume
metadata:
Expand All @@ -127,13 +159,13 @@ spec:
hostPath:
path: "/cache"
EOF
)"
)"

echo "${pv_yaml}" | "${KUBECTL}" create -f -
echo "${pv_yaml}" | "${KUBECTL}" create -f -

echo_step "create persistent volume claim for mounting package-cache"
echo_step "create persistent volume claim for mounting package-cache"

local pvc_yaml="$( cat <<EOF
local pvc_yaml="$( cat <<EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
Expand All @@ -148,9 +180,10 @@ spec:
requests:
storage: 1Mi
EOF
)"
)"

echo "${pvc_yaml}" | "${KUBECTL}" create -f -
echo "${pvc_yaml}" | "${KUBECTL}" create -f -
fi
}

cleanup_cluster() {
Expand All @@ -164,15 +197,84 @@ setup_crossplane() {
local chart_version="$("${HELM}" search repo crossplane-stable/crossplane | awk 'FNR == 2 {print $2}')"
echo_info "using crossplane version ${chart_version}"
echo
# we replace empty dir with our PVC so that the /cache dir in the kind node
# container is exposed to the crossplane pod
"${HELM}" install crossplane --namespace crossplane-system crossplane-stable/crossplane --version ${chart_version} --wait --set packageCache.pvc=package-cache
if [ "${USE_OCI}" = true ]; then
echo_sub_step "Crossplane cache: emptyDir (OCI mode)"
"${HELM}" install crossplane --namespace crossplane-system crossplane-stable/crossplane --version ${chart_version} --wait
else
echo_sub_step "Crossplane cache: PVC 'package-cache' (local .gz mode)"
"${HELM}" install crossplane --namespace crossplane-system crossplane-stable/crossplane --version ${chart_version} --wait --set packageCache.pvc=package-cache
fi
}

setup_local_registry() {
[ "${USE_OCI}" = true ] || return 0
echo_step "deploy in-cluster OCI registry"
local reg_yaml="$( cat <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: registry
namespace: crossplane-system
spec:
replicas: 1
selector:
matchLabels:
app: registry
template:
metadata:
labels:
app: registry
spec:
containers:
- name: registry
image: registry:3.0.0
ports:
- containerPort: 5000
env:
- name: REGISTRY_STORAGE_DELETE_ENABLED
value: "true"
---
apiVersion: v1
kind: Service
metadata:
name: registry
namespace: crossplane-system
spec:
selector:
app: registry
ports:
- name: http
protocol: TCP
port: 5000
targetPort: 5000
EOF
)"
echo "${reg_yaml}" | "${KUBECTL}" apply -f -
"${KUBECTL}" -n crossplane-system rollout status deploy/registry --timeout=120s

echo_step "port-forward registry for pushing xpkg"
mkdir -p "${projectdir}/.work"
( kubectl -n crossplane-system port-forward svc/registry 5000:5000 >/dev/null 2>&1 & echo $! >"${projectdir}/.work/registry-pf.pid" )
for i in {1..20}; do nc -z localhost 5000 && break || sleep 0.5; done
}
Copy link

Copilot AI Dec 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The registry port-forward process is launched in a background subshell with all output redirected to /dev/null, making it difficult to diagnose failures. The PID is captured, but if the port-forward fails immediately after starting, the subsequent nc check loop will timeout without useful diagnostic information about why the port-forward failed.

Suggested change
}
( kubectl -n crossplane-system port-forward svc/registry 5000:5000 >"${projectdir}/.work/registry-pf.log" 2>&1 & echo $! >"${projectdir}/.work/registry-pf.pid" )

Copilot uses AI. Check for mistakes.

Copy link

Copilot AI Dec 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The nc readiness check loop has no error handling or diagnostic output when it fails. If the port never becomes available after 20 iterations (10 seconds), the script continues silently without any indication that the registry may not be ready. This could lead to subsequent push failures that are harder to diagnose.

Suggested change
local registry_ready=false
for i in {1..20}; do
if nc -z localhost 5000; then
registry_ready=true
break
fi
sleep 0.5
done
if [ "${registry_ready}" != true ]; then
printf "\n${RED}Failed to connect to local registry on port 5000 after waiting 10 seconds.${NOC}\n" >&2
return 1
fi

Copilot uses AI. Check for mistakes.
push_xpkg_to_registry() {
[ "${USE_OCI}" = true ] || return 0
echo_step "push xpkg to in-cluster registry"
local xpkg_path="${OUTPUT_DIR}/xpkg/linux_${SAFEHOSTARCH}/${PACKAGE_NAME}-${VERSION}.xpkg"
local ref_ver="localhost:5000/${PACKAGE_NAME}:${version_tag}"
local ref_latest="localhost:5000/${PACKAGE_NAME}:latest"
"${UP}" xpkg push ${ref_ver} -f "${xpkg_path}"
"${UP}" xpkg push ${ref_latest} -f "${xpkg_path}"
echo_info "pushed tags: ${ref_ver}, ${ref_latest}"
}

setup_provider() {
echo_step "installing provider"

local yaml="$( cat <<EOF
if [ "${USE_OCI}" = true ]; then
echo_sub_step "Provider package from OCI: registry.crossplane-system.svc.cluster.local:5000/${PACKAGE_NAME}:latest"
local yaml="$( cat <<EOF
apiVersion: pkg.crossplane.io/v1beta1
kind: DeploymentRuntimeConfig
metadata:
Expand All @@ -185,6 +287,7 @@ spec:
spec:
containers:
- name: package-runtime
image: "${CONTROLLER_IMAGE}"
args:
- --debug
---
Expand All @@ -195,19 +298,66 @@ metadata:
spec:
runtimeConfigRef:
name: debug-config
package: "${PACKAGE_NAME}"
package: "registry.crossplane-system.svc.cluster.local:5000/${PACKAGE_NAME}:latest"
packagePullPolicy: IfNotPresent
EOF
)"
echo "${yaml}" | "${KUBECTL}" apply -f -
else
echo_sub_step "Provider package from local cache: ${PACKAGE_NAME}-${VERSION}.gz"
local yaml="$( cat <<EOF
apiVersion: pkg.crossplane.io/v1beta1
kind: DeploymentRuntimeConfig
metadata:
name: debug-config
spec:
deploymentTemplate:
spec:
selector: {}
template:
spec:
containers:
- name: package-runtime
image: "${CONTROLLER_IMAGE}"
args:
- --debug
---
apiVersion: pkg.crossplane.io/v1
kind: Provider
metadata:
name: "${PACKAGE_NAME}"
spec:
runtimeConfigRef:
name: debug-config
package: "${PACKAGE_NAME}-${VERSION}.gz"
packagePullPolicy: Never
EOF
)"
)"
echo "${yaml}" | "${KUBECTL}" apply -f -
fi

echo "${yaml}" | "${KUBECTL}" apply -f -

# printing the cache dir contents can be useful for troubleshooting failures
echo_step "check kind node cache dir contents"
docker exec "${K8S_CLUSTER}-control-plane" ls -la /cache
if [ "${USE_OCI}" != true ]; then
# printing the cache dir contents can be useful for troubleshooting local cache failures
echo_step "check kind node cache dir contents"
docker exec "${K8S_CLUSTER}-control-plane" ls -la /cache
fi

echo_step "waiting for provider to be installed"
"${KUBECTL}" wait "provider.pkg.crossplane.io/${PACKAGE_NAME}" --for=condition=healthy --timeout=60s
if ! "${KUBECTL}" wait "provider.pkg.crossplane.io/${PACKAGE_NAME}" --for=condition=healthy --timeout=240s; then
echo_error "Provider did not become healthy in time. Dumping logs..."
"${KUBECTL}" get all -n crossplane-system
"${KUBECTL}" get providerrevision -o wide
"${KUBECTL}" get provider -o wide
# Get the logs of the provider pod
POD_NAME=$("${KUBECTL}" get pods -n crossplane-system -l "pkg.crossplane.io/provider=${PACKAGE_NAME}" -o name)
if [ -n "$POD_NAME" ]; then
echo "Logs for $POD_NAME:"
"${KUBECTL}" logs -n crossplane-system "$POD_NAME" -c package-runtime
else
echo "No pod found for provider ${PACKAGE_NAME}"
fi
exit 1
Comment on lines +354 to +359
Copy link

Copilot AI Dec 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The error handling after kubectl wait fails attempts to get logs from a provider pod, but the variable POD_NAME assignment doesn't quote the command substitution properly for the subsequent check. If no pod is found, POD_NAME will be an empty string, and the '-n' test will correctly handle it. However, if multiple pods match, only the first will be captured. Consider adding error handling for the case where multiple pods exist or no pods are found before the timeout.

Suggested change
echo "Logs for $POD_NAME:"
"${KUBECTL}" logs -n crossplane-system "$POD_NAME" -c package-runtime
else
echo "No pod found for provider ${PACKAGE_NAME}"
fi
exit 1
PODS_OUTPUT="$("${KUBECTL}" get pods -n crossplane-system -l "pkg.crossplane.io/provider=${PACKAGE_NAME}" -o name || true)"
if [ -z "$PODS_OUTPUT" ]; then
echo "No pod found for provider ${PACKAGE_NAME}"
else
POD_COUNT="$(printf "%s\n" "$PODS_OUTPUT" | sed '/^[[:space:]]*$/d' | wc -l | tr -d '[:space:]')"
if [ "$POD_COUNT" -gt 1 ]; then
echo_error "Multiple pods found for provider ${PACKAGE_NAME}:"
printf "%s\n" "$PODS_OUTPUT"
else
POD_NAME="$PODS_OUTPUT"
echo "Logs for $POD_NAME:"
"${KUBECTL}" logs -n crossplane-system "$POD_NAME" -c package-runtime
fi

Copilot uses AI. Check for mistakes.
fi
}

cleanup_provider() {
Expand Down Expand Up @@ -331,29 +481,29 @@ cleanup_provider_config() {
setup_mariadb_no_tls() {
echo_step "installing MariaDB with no TLS"
"${KUBECTL}" create secret generic mariadb-creds \
--from-literal username="root" \
--from-literal password="${MARIADB_ROOT_PW}" \
--from-literal endpoint="mariadb.default.svc.cluster.local" \
--from-literal port="3306"
--from-literal=username="root" \
--from-literal=password="${MARIADB_ROOT_PW}" \
--from-literal=endpoint="mariadb.default.svc.cluster.local" \
--from-literal=port="3306"

"${HELM}" repo add bitnami https://charts.bitnami.com/bitnami >/dev/null
"${HELM}" repo update
"${HELM}" install mariadb bitnami/mariadb \
--version 11.3.0 \
--version 24.0.2 \
--set auth.rootPassword="${MARIADB_ROOT_PW}" \
--wait
}

setup_mariadb_tls() {
echo_step "installing MariaDB with TLS"
"${KUBECTL}" create secret generic mariadb-creds \
--from-literal username="test" \
--from-literal password="${MARIADB_TEST_PW}" \
--from-literal endpoint="mariadb.default.svc.cluster.local" \
--from-literal port="3306" \
--from-file=ca-cert.pem \
--from-file=client-cert.pem \
--from-file=client-key.pem
--from-literal=username="test" \
--from-literal=password="${MARIADB_TEST_PW}" \
--from-literal=endpoint="mariadb.default.svc.cluster.local" \
--from-literal=port="3306" \
--from-file=ca-cert.pem \
--from-file=client-cert.pem \
--from-file=client-key.pem

local values=$(cat <<EOF
auth:
Expand All @@ -380,7 +530,7 @@ EOF
"${HELM}" repo add bitnami https://charts.bitnami.com/bitnami >/dev/null
"${HELM}" repo update
"${HELM}" install mariadb bitnami/mariadb \
--version 11.3.0 \
--version 24.0.2 \
--values <(echo "$values") \
--wait
}
Expand Down Expand Up @@ -459,6 +609,8 @@ cleanup_test_resources() {

setup_cluster
setup_crossplane
setup_local_registry
push_xpkg_to_registry
setup_provider

echo_step "--- INTEGRATION TESTS - NO TLS ---"
Expand Down
Loading