Skip to content

Commit ac64028

Browse files
williamlardierwilliamlardier
committed
Setup CTST in BeforeAll
1 parent b996f4f commit ac64028

File tree

15 files changed

+1787
-211
lines changed

15 files changed

+1787
-211
lines changed

‎.github/scripts/end2end/configure-e2e-ctst.sh‎

Lines changed: 0 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,5 @@
11
#!/bin/bash
22
set -exu
33

4-
# Setup test environment variables
5-
export ZENKO_NAME=${1:-"end2end"}
6-
# Getting kafka host from backbeat's config
7-
KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
8-
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
9-
KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
10-
# Removing the port
11-
export NOTIF_KAFKA_HOST=${KAFKA_HOST_PORT%:*}
12-
export NOTIF_KAFKA_PORT=${KAFKA_HOST_PORT#*:}
13-
14-
UUID=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
15-
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .extensions.replication.topic)
16-
UUID=${UUID%.*}
17-
UUID=${UUID:1}
18-
194
echo "127.0.0.1 iam.zenko.local ui.zenko.local s3-local-file.zenko.local keycloak.zenko.local \
205
sts.zenko.local management.zenko.local s3.zenko.local website.mywebsite.com utilization.zenko.local" | sudo tee -a /etc/hosts
21-
22-
# Add bucket notification target
23-
envsubst < ./configs/notification_destinations.yaml | kubectl apply -f -
24-
# Wait for service stabilization
25-
kubectl wait --for condition=DeploymentInProgress=true --timeout 10m zenko/${ZENKO_NAME}
26-
kubectl wait --for condition=DeploymentFailure=false --timeout 10m zenko/${ZENKO_NAME}
27-
kubectl wait --for condition=DeploymentInProgress=false --timeout 10m zenko/${ZENKO_NAME}
28-
29-
# Get kafka image name and tag
30-
KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yaml)
31-
KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
32-
KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
33-
KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
34-
35-
# Cold location topic
36-
AZURE_ARCHIVE_STATUS_TOPIC="${UUID}.cold-status-e2e-azure-archive"
37-
AZURE_ARCHIVE_STATUS_TOPIC_2_NV="${UUID}.cold-status-e2e-azure-archive-2-non-versioned"
38-
AZURE_ARCHIVE_STATUS_TOPIC_2_V="${UUID}.cold-status-e2e-azure-archive-2-versioned"
39-
AZURE_ARCHIVE_STATUS_TOPIC_2_S="${UUID}.cold-status-e2e-azure-archive-2-suspended"
40-
41-
# Creating bucket notification topic in kafka
42-
kubectl run kafka-topics \
43-
--image=$KAFKA_IMAGE \
44-
--pod-running-timeout=5m \
45-
--rm \
46-
--restart=Never \
47-
--attach=True \
48-
--command -- bash -c \
49-
"kafka-topics.sh --create --topic $NOTIF_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
50-
kafka-topics.sh --create --topic $NOTIF_ALT_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
51-
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
52-
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_NV --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
53-
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_V --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
54-
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_S --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists"

‎.github/scripts/end2end/run-e2e-ctst.sh‎

Lines changed: 17 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -13,145 +13,37 @@ JUNIT_REPORT_PATH=${JUNIT_REPORT_PATH:-"ctst-junit.xml"}
1313
# Zenko Version
1414
VERSION=$(cat ../../../VERSION | grep -Po 'VERSION="\K[^"]*')
1515

16-
# Zenko Environment
17-
ZENKO_ACCOUNT_NAME="zenko-ctst"
18-
ADMIN_ACCESS_KEY_ID=$(kubectl get secret end2end-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
19-
ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-management-vault-admin-creds.v1 -o jsonpath='{.data.secretKey}' | base64 -d)
20-
ADMIN_PRA_ACCESS_KEY_ID=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
21-
ADMIN_PRA_SECRET_ACCESS_KEY=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.secretKey}' | base64 -d)
22-
STORAGE_MANAGER_USER_NAME="ctst_storage_manager"
23-
STORAGE_ACCOUNT_OWNER_USER_NAME="ctst_storage_account_owner"
24-
DATA_CONSUMER_USER_NAME="ctst_data_consumer"
25-
DATA_ACCESSOR_USER_NAME="ctst_data_accessor"
26-
VAULT_AUTH_HOST="${ZENKO_NAME}-connector-vault-auth-api.default.svc.cluster.local"
27-
ZENKO_PORT="80"
28-
KEYCLOAK_TEST_USER=${OIDC_USERNAME}
29-
KEYCLOAK_TEST_PASSWORD=${OIDC_PASSWORD}
30-
KEYCLOAK_TEST_HOST=${OIDC_HOST}
31-
KEYCLOAK_TEST_PORT="80"
32-
KEYCLOAK_TEST_REALM_NAME=${OIDC_REALM}
33-
KEYCLOAK_TEST_CLIENT_ID=${OIDC_CLIENT_ID}
34-
KEYCLOAK_TEST_GRANT_TYPE="password"
16+
# Minimal environment setup - CTST will handle all Kubernetes discovery
3517

36-
# get Zenko service users credentials
37-
BACKBEAT_LCBP_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-lcbp-user-creds,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.backbeat-lifecycle-bp-1\.json}' | base64 -d)
38-
BACKBEAT_LCC_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-lcc-user-creds,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.backbeat-lifecycle-conductor-1\.json}' | base64 -d)
39-
BACKBEAT_LCOP_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-lcop-user-creds,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.backbeat-lifecycle-op-1\.json}' | base64 -d)
40-
BACKBEAT_QP_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-qp-user-creds,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.backbeat-qp-1\.json}' | base64 -d)
41-
SORBET_FWD_2_ACCESSKEY=$(kubectl get secret -l app.kubernetes.io/name=sorbet-fwd-creds,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.accessKey}' | base64 -d)
42-
SORBET_FWD_2_SECRETKEY=$(kubectl get secret -l app.kubernetes.io/name=sorbet-fwd-creds,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.secretKey}' | base64 -d)
43-
SERVICE_USERS_CREDENTIALS=$(echo '{"backbeat-lifecycle-bp-1":'${BACKBEAT_LCBP_1_CREDS}',"backbeat-lifecycle-conductor-1":'${BACKBEAT_LCC_1_CREDS}',"backbeat-lifecycle-op-1":'${BACKBEAT_LCOP_1_CREDS}',"backbeat-qp-1":'${BACKBEAT_QP_1_CREDS}',"sorbet-fwd-2":{"accessKey":"'${SORBET_FWD_2_ACCESSKEY}'","secretKey":"'${SORBET_FWD_2_SECRETKEY}'"}}' | jq -R)
44-
45-
# Get KAFKA topics for sorbet
46-
KAFKA_DEAD_LETTER_TOPIC=$(kubectl get secret -l app.kubernetes.io/name=cold-sorbet-config-e2e-azure-archive,app.kubernetes.io/instance=end2end \
47-
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq '."kafka-dead-letter-topic"' | cut -d "\"" -f 2)
48-
49-
KAFKA_OBJECT_TASK_TOPIC=$(kubectl get secret -l app.kubernetes.io/name=cold-sorbet-config-e2e-azure-archive,app.kubernetes.io/instance=end2end \
50-
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq '."kafka-object-task-topic"' | cut -d "\"" -f 2)
51-
52-
KAFKA_GC_REQUEST_TOPIC=$(kubectl get secret -l app.kubernetes.io/name=cold-sorbet-config-e2e-azure-archive,app.kubernetes.io/instance=end2end \
53-
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq '."kafka-gc-request-topic"' | cut -d "\"" -f 2)
54-
55-
DR_ADMIN_ACCESS_KEY_ID=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
56-
DR_ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.secretKey}' | base64 -d)
57-
58-
# Extracting kafka host from bacbeat's config
59-
KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
60-
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
61-
KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
62-
63-
TIME_PROGRESSION_FACTOR=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath="{.metadata.annotations.zenko\.io/time-progression-factor}")
64-
INSTANCE_ID=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.status.instanceID}')
65-
66-
# Azure archive tests
67-
AZURE_ARCHIVE_ACCESS_TIER="Hot"
68-
AZURE_ARCHIVE_MANIFEST_ACCESS_TIER="Hot"
69-
70-
BACKBEAT_API_HOST=$(kubectl get secret -l app.kubernetes.io/name=connector-cloudserver-config,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .backbeat.host)
71-
BACKBEAT_API_HOST=${BACKBEAT_API_HOST:1:-1}
72-
BACKBEAT_API_PORT=$(kubectl get secret -l app.kubernetes.io/name=connector-cloudserver-config,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .backbeat.port)
73-
74-
KAFKA_CLEANER_INTERVAL=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.kafkaCleaner.interval}')
75-
SORBETD_RESTORE_TIMEOUT=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.sorbet.server.azure.restoreTimeout}')
76-
77-
# Utilization service
78-
UTILIZATION_SERVICE_HOST=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.scuba.api.ingress.hostname}')
79-
UTILIZATION_SERVICE_PORT="80"
80-
81-
# Setting CTST world params
18+
# Minimal CTST world params - CTST handles all Kubernetes discovery
8219
WORLD_PARAMETERS="$(jq -c <<EOF
8320
{
21+
"Namespace":"default",
8422
"subdomain":"${SUBDOMAIN}",
8523
"DRSubdomain":"${DR_SUBDOMAIN}",
86-
"ssl":false,
87-
"port":"${ZENKO_PORT}",
88-
"AccountName":"${ZENKO_ACCOUNT_NAME}",
89-
"AdminAccessKey":"${ADMIN_ACCESS_KEY_ID}",
90-
"AdminSecretKey":"${ADMIN_SECRET_ACCESS_KEY}",
91-
"VaultAuthHost":"${VAULT_AUTH_HOST}",
92-
"NotificationDestination":"${NOTIF_DEST_NAME}",
93-
"NotificationDestinationTopic":"${NOTIF_DEST_TOPIC}",
94-
"NotificationDestinationAlt":"${NOTIF_ALT_DEST_NAME}",
95-
"NotificationDestinationTopicAlt":"${NOTIF_ALT_DEST_TOPIC}",
96-
"KafkaExternalIps": "${KAFKA_EXTERNAL_IP:-}",
97-
"PrometheusService":"${PROMETHEUS_NAME}-operated.default.svc.cluster.local",
98-
"KafkaHosts":"${KAFKA_HOST_PORT}",
99-
"KeycloakUsername":"${KEYCLOAK_TEST_USER}",
100-
"KeycloakPassword":"${KEYCLOAK_TEST_PASSWORD}",
101-
"KeycloakHost":"${KEYCLOAK_TEST_HOST}",
102-
"KeycloakPort":"${KEYCLOAK_TEST_PORT}",
103-
"keycloakRealm":"${KEYCLOAK_TEST_REALM_NAME}",
104-
"keycloakClientId":"${KEYCLOAK_TEST_CLIENT_ID}",
105-
"keycloakGrantType":"${KEYCLOAK_TEST_GRANT_TYPE}",
106-
"StorageManagerUsername":"${STORAGE_MANAGER_USER_NAME}",
107-
"StorageAccountOwnerUsername":"${STORAGE_ACCOUNT_OWNER_USER_NAME}",
108-
"DataConsumerUsername":"${DATA_CONSUMER_USER_NAME}",
109-
"DataAccessorUsername":"${DATA_ACCESSOR_USER_NAME}",
110-
"ServiceUsersCredentials":${SERVICE_USERS_CREDENTIALS},
111-
"AzureAccountName":"${AZURE_ACCOUNT_NAME}",
112-
"AzureAccountKey":"${AZURE_SECRET_KEY}",
113-
"AzureArchiveContainer":"${AZURE_ARCHIVE_BUCKET_NAME}",
114-
"AzureArchiveContainer2":"${AZURE_ARCHIVE_BUCKET_NAME_2}",
115-
"AzureArchiveAccessTier":"${AZURE_ARCHIVE_ACCESS_TIER}",
116-
"AzureArchiveManifestTier":"${AZURE_ARCHIVE_MANIFEST_ACCESS_TIER}",
117-
"AzureArchiveQueue":"${AZURE_ARCHIVE_QUEUE_NAME}",
118-
"TimeProgressionFactor":"${TIME_PROGRESSION_FACTOR}",
119-
"KafkaObjectTaskTopic":"${KAFKA_OBJECT_TASK_TOPIC}",
120-
"KafkaGCRequestTopic":"${KAFKA_GC_REQUEST_TOPIC}",
121-
"KafkaDeadLetterQueueTopic":"${KAFKA_DEAD_LETTER_TOPIC}",
122-
"InstanceID":"${INSTANCE_ID}",
123-
"BackbeatApiHost":"${BACKBEAT_API_HOST}",
124-
"BackbeatApiPort":"${BACKBEAT_API_PORT}",
125-
"KafkaCleanerInterval":"${KAFKA_CLEANER_INTERVAL}",
126-
"SorbetdRestoreTimeout":"${SORBETD_RESTORE_TIMEOUT}",
127-
"TimeProgressionFactor":"${TIME_PROGRESSION_FACTOR}",
128-
"DRAdminAccessKey":"${DR_ADMIN_ACCESS_KEY_ID}",
129-
"DRAdminSecretKey":"${DR_ADMIN_SECRET_ACCESS_KEY}",
130-
"UtilizationServiceHost":"${UTILIZATION_SERVICE_HOST}",
131-
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}"
24+
"KeycloakUsername":"${OIDC_USERNAME:-testuser}",
25+
"KeycloakPassword":"${OIDC_PASSWORD:-testpass}",
26+
"KeycloakHost":"${OIDC_HOST:-keycloak.zenko.local}",
27+
"KeycloakRealm":"${OIDC_REALM:-zenko}",
28+
"KeycloakClientId":"${OIDC_CLIENT_ID:-zenko-ui}",
29+
"AzureAccountName":"${AZURE_ACCOUNT_NAME:-devstoreaccount1}",
30+
"AzureAccountKey":"${AZURE_SECRET_KEY:-Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==}",
31+
"AzureArchiveContainer":"${AZURE_ARCHIVE_BUCKET_NAME:-archive-container}",
32+
"AzureArchiveContainer2":"${AZURE_ARCHIVE_BUCKET_NAME_2:-archive-container-2}",
33+
"AzureArchiveQueue":"${AZURE_ARCHIVE_QUEUE_NAME:-archive-queue}"
13234
}
13335
EOF
13436
)"
13537

136-
# Set up environment variables for testing
137-
kubectl set env deployment end2end-connector-cloudserver SCUBA_HEALTHCHECK_FREQUENCY=100
138-
kubectl rollout status deployment end2end-connector-cloudserver
139-
14038
E2E_IMAGE=$E2E_CTST_IMAGE_NAME:$E2E_IMAGE_TAG
14139
POD_NAME="${ZENKO_NAME}-ctst-tests"
14240
CTST_VERSION=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ../../../tests/ctst/package.json)
14341

144-
# Configure keycloak
145-
docker run \
146-
--rm \
147-
--network=host \
148-
"${E2E_IMAGE}" /bin/bash \
149-
-c "SUBDOMAIN=${SUBDOMAIN} CONTROL_PLANE_INGRESS_ENDPOINT=${OIDC_ENDPOINT} ACCOUNT=${ZENKO_ACCOUNT_NAME} KEYCLOAK_REALM=${KEYCLOAK_TEST_REALM_NAME} STORAGE_MANAGER=${STORAGE_MANAGER_USER_NAME} STORAGE_ACCOUNT_OWNER=${STORAGE_ACCOUNT_OWNER_USER_NAME} DATA_CONSUMER=${DATA_CONSUMER_USER_NAME} DATA_ACCESSOR=${DATA_ACCESSOR_USER_NAME} /ctst/bin/seedKeycloak.sh"; [[ $? -eq 1 ]] && exit 1 || echo 'Keycloak Configured!'
150-
151-
# Grant access to Kube API (insecure, only for testing)
152-
kubectl create clusterrolebinding serviceaccounts-cluster-admin \
42+
# Grant CTST cluster-admin permissions (test environment only)
43+
kubectl create clusterrolebinding ctst-cluster-admin \
15344
--clusterrole=cluster-admin \
154-
--group=system:serviceaccounts
45+
--serviceaccount=default:default \
46+
--dry-run=client -o yaml | kubectl apply -f -
15547

15648
# Running end2end ctst tests
15749
# Using overrides as we need to attach a local folder to the pod

‎.github/scripts/end2end/setup-ctst-local.sh‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/bash
2+
set -exu
3+
4+
# CTST Local Development Setup
5+
# This script sets up minimal prerequisites for running CTST locally
6+
# In the future, when CTST is the only test suite, this may be integrated into CTST itself
7+
8+
NAMESPACE=${1:-default}
9+
10+
echo "Setting up CTST local development environment..."
11+
12+
# 1. Grant CTST cluster-admin permissions (test environment only)
13+
echo "Setting up CTST permissions..."
14+
kubectl create clusterrolebinding ctst-cluster-admin \
15+
--clusterrole=cluster-admin \
16+
--serviceaccount=default:default \
17+
--dry-run=client -o yaml | kubectl apply -f -
18+
19+
# 2. Check if CoreDNS needs patching for mock service resolution
20+
echo "Checking CoreDNS configuration..."
21+
if ! kubectl get configmap coredns -n kube-system -o yaml | grep -q "azure-mock.zenko.local" 2>/dev/null; then
22+
echo "Patching CoreDNS for mock service resolution..."
23+
bash patch-coredns.sh
24+
else
25+
echo "CoreDNS already configured"
26+
fi
27+
28+
# 3. Setup /etc/hosts for local development (requires sudo)
29+
echo "Checking /etc/hosts configuration..."
30+
if ! grep -q "zenko.local" /etc/hosts 2>/dev/null; then
31+
echo "Setting up /etc/hosts (requires sudo)..."
32+
echo "127.0.0.1 iam.zenko.local ui.zenko.local s3-local-file.zenko.local keycloak.zenko.local \
33+
sts.zenko.local management.zenko.local s3.zenko.local website.mywebsite.com utilization.zenko.local" | sudo tee -a /etc/hosts
34+
else
35+
echo "/etc/hosts already configured"
36+
fi
37+
38+
# 4. Wait for Zenko to be ready
39+
echo "Waiting for Zenko deployment to be ready..."
40+
kubectl wait --for condition=DeploymentFailure=false --timeout 10m zenko/end2end -n $NAMESPACE 2>/dev/null || echo "Zenko wait failed or not found"
41+
kubectl wait --for condition=DeploymentInProgress=false --timeout 10m zenko/end2end -n $NAMESPACE 2>/dev/null || echo "Zenko wait failed or not found"
42+
43+
echo "CTST local environment ready!"
44+
echo ""
45+
echo "Usage:"
46+
echo " cd tests/ctst"
47+
echo " npm test # Run all CTST tests"
48+
echo " npm run test -- --tags @PRA # Run specific test tags"
49+
echo ""
50+
echo "Note: CTST will handle all Kubernetes setup (mocks, topics, deployments, etc.) automatically"

0 commit comments

Comments
 (0)