Add CRR E2E tests

.github/scripts/end2end/configure-e2e.sh

@@ -11,7 +11,8 @@ NAMESPACE=${3:-default}
 SERVICE_ACCOUNT="${ZENKO_NAME}-config"
 POD_NAME="${ZENKO_NAME}-config"
 MANAGEMENT_ENDPOINT="http://${ZENKO_NAME}-management-orbit-api:5001"
-VAULT_ENDPOINT="http://${ZENKO_NAME}-connector-vault-sts-api"
+IAM_ENDPOINT="http://${ZENKO_NAME}-management-vault-iam-admin-api"
+STS_ENDPOINT="http://${ZENKO_NAME}-connector-vault-sts-api"
 UUID=$(kubectl get zenko ${ZENKO_NAME} --namespace ${NAMESPACE} -o jsonpath='{.status.instanceID}')
 TOKEN=$(get_token)
 
@@ -80,7 +81,8 @@ kubectl run ${POD_NAME} \
   --env="TOKEN=${TOKEN}" \
   --env="UUID=${UUID}" \
   --env="MANAGEMENT_ENDPOINT=${MANAGEMENT_ENDPOINT}" \
-  --env="VAULT_ENDPOINT=${VAULT_ENDPOINT}" \
+  --env="IAM_ENDPOINT=${IAM_ENDPOINT}" \
+  --env="STS_ENDPOINT=${STS_ENDPOINT}" \
   --env="NAMESPACE=${NAMESPACE}" \
   --env=VERIFY_CERTIFICATES=false \
   --env=ENABLE_RING_TESTS=${ENABLE_RING_TESTS} \
@@ -106,6 +108,12 @@ kubectl run ${POD_NAME} \
   --env=AZURE_ARCHIVE_BUCKET_NAME=${AZURE_ARCHIVE_BUCKET_NAME} \
   --env=AZURE_ARCHIVE_BUCKET_NAME_2=${AZURE_ARCHIVE_BUCKET_NAME_2} \
   --env=AZURE_ARCHIVE_QUEUE_NAME=${AZURE_ARCHIVE_QUEUE_NAME} \
+  --env=CRR_SOURCE_LOCATION_NAME=${CRR_SOURCE_LOCATION_NAME} \
+  --env=CRR_DESTINATION_LOCATION_NAME=${CRR_DESTINATION_LOCATION_NAME} \
+  --env=CRR_SOURCE_ACCOUNT_NAME=${CRR_SOURCE_ACCOUNT_NAME} \
+  --env=CRR_DESTINATION_ACCOUNT_NAME=${CRR_DESTINATION_ACCOUNT_NAME} \
+  --env=CRR_ROLE_NAME=${CRR_ROLE_NAME} \
+  --env=DEPLOY_CRR_LOCATIONS=${DEPLOY_CRR_LOCATIONS} \
   --command -- python3 configuration.py
 
 ## wait for updates to trigger zenko upgrades

.github/scripts/end2end/run-e2e-test.sh

@@ -37,6 +37,16 @@ ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-management-vault-admin-cred
 ZENKO_ACCESS_KEY=$(kubectl get secret end2end-account-zenko -o jsonpath='{.data.AccessKeyId}' | base64 -d)
 ZENKO_SECRET_KEY=$(kubectl get secret end2end-account-zenko -o jsonpath='{.data.SecretAccessKey}' | base64 -d)
 ZENKO_SESSION_TOKEN=$(kubectl get secret end2end-account-zenko -o jsonpath='{.data.SessionToken}' | base64 -d)
+SOURCE_ACCESS_KEY=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.AccessKeyId}' | base64 -d)
+SOURCE_SECRET_KEY=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d)
+SOURCE_SESSION_TOKEN=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.SessionToken}' | base64 -d)
+SOURCE_ACCOUNT_ID=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.AccountId}' | base64 -d)
+CRR_SOURCE_INFO="{\"AccessKeyId\":\"${SOURCE_ACCESS_KEY}\",\"SecretAccessKey\":\"${SOURCE_SECRET_KEY}\",\"SessionToken\":\"${SOURCE_SESSION_TOKEN}\",\"AccountId\":\"${SOURCE_ACCOUNT_ID}\"}"
+DESTINATION_ACCESS_KEY=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.AccessKeyId}' | base64 -d)
+DESTINATION_SECRET_KEY=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d)
+DESTINATION_SESSION_TOKEN=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.SessionToken}' | base64 -d)
+DESTINATION_ACCOUNT_ID=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.AccountId}' | base64 -d)
+CRR_DESTINATION_INFO="{\"AccessKeyId\":\"${DESTINATION_ACCESS_KEY}\",\"SecretAccessKey\":\"${DESTINATION_SECRET_KEY}\",\"SessionToken\":\"${DESTINATION_SESSION_TOKEN}\",\"AccountId\":\"${DESTINATION_ACCOUNT_ID}\"}"
 OIDC_FULLNAME="${OIDC_FIRST_NAME} ${OIDC_LAST_NAME}"
 KEYCLOAK_TEST_USER="${OIDC_USERNAME}-norights"
 KEYCLOAK_TEST_PASSWORD=${OIDC_PASSWORD}
@@ -124,6 +134,11 @@ run_e2e_test() {
         --env=MONGO_WRITE_CONCERN=${MONGO_WRITE_CONCERN} \
         --env=MONGO_AUTH_USERNAME=${MONGO_AUTH_USERNAME} \
         --env=MONGO_AUTH_PASSWORD=${MONGO_AUTH_PASSWORD} \
+        --env=CRR_SOURCE_LOCATION_NAME=${CRR_SOURCE_LOCATION_NAME} \
+        --env=CRR_SOURCE_INFO=${CRR_SOURCE_INFO} \
+        --env=CRR_DESTINATION_LOCATION_NAME=${CRR_DESTINATION_LOCATION_NAME} \
+        --env=CRR_DESTINATION_INFO=${CRR_DESTINATION_INFO} \
+        --env=CRR_ROLE_NAME=${CRR_ROLE_NAME} \
         --env=MOCHA_FILE=${MOCHA_FILE} \
         --override-type strategic \
         --overrides='

.github/workflows/end2end.yaml

@@ -74,6 +74,12 @@ env:
   GCP_SECRET_KEY: ${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}
   GCP_BACKEND_SERVICE_KEY: ${{ secrets.GCP_BACKEND_SERVICE_KEY }}
   GCP_BACKEND_SERVICE_EMAIL: ${{ secrets.GCP_BACKEND_SERVICE_EMAIL }}
+  DEPLOY_CRR_LOCATIONS: "true"
+  CRR_SOURCE_LOCATION_NAME: crr-source-location
+  CRR_DESTINATION_LOCATION_NAME: crr-destination-location
+  CRR_SOURCE_ACCOUNT_NAME: crr-source-account
+  CRR_DESTINATION_ACCOUNT_NAME: crr-destination-account
+  CRR_ROLE_NAME: crr-role
   # Enable this for Ring tests
   ENABLE_RING_TESTS: "false"
   RING_S3C_ACCESS_KEY: accessKey1
@@ -433,6 +439,8 @@ jobs:
   end2end-pra:
     needs: [build-kafka, lint-and-build-ctst]
     runs-on: ubuntu-24.04-16core
+    env:
+      DEPLOY_CRR_LOCATIONS: "false"
     steps:
       - name: Checkout
         uses: actions/checkout@v4

solution/deps.yaml

@@ -16,7 +16,7 @@ cloudserver:
   sourceRegistry: ghcr.io/scality
   dashboard: cloudserver/cloudserver-dashboards
   image: cloudserver
-  tag: 9.1.0
+  tag: 9.1.1
   envsubst: CLOUDSERVER_TAG
 drctl:
   sourceRegistry: ghcr.io/scality
@@ -136,7 +136,7 @@ vault:
 zenko-operator:
   sourceRegistry: ghcr.io/scality
   image: zenko-operator
-  tag: v1.8.0-preview.1
+  tag: v1.8.0-preview.2
   envsubst: ZENKO_OPERATOR_TAG
 zookeeper:
   sourceRegistry: pravega

tests/zenko_tests/configuration.py

@@ -125,9 +125,11 @@ def main():
         create_buckets.create_azure_containers()
         create_buckets.create_azure_queues()
 
+        accounts_creds = {}
+
         # create zenko resources
         for account in e2e_config["accounts"]:
-            accounts.create_account(client,
+            accounts_creds[account] = accounts.create_account(client,
                                     TOKEN,
                                     UUID,
                                     account,
@@ -138,7 +140,7 @@ def main():
                                       endpoint["locationName"])
 
         for location in e2e_config["locations"]:
-            locations.create_location(client, UUID, location)
+            locations.create_location(client, UUID, location, accounts_creds)
 
         for wf in e2e_config["workflows"]["replication"]:
             workflows.create_replication_workflow(client, UUID, wf)

tests/zenko_tests/e2e-config.yaml.template

@@ -1,5 +1,7 @@
 accounts:
   - "zenko"
+  - "${CRR_SOURCE_ACCOUNT_NAME}"
+  - "${CRR_DESTINATION_ACCOUNT_NAME}"
 endpoints: []
 locations:
   - name: "${AWS_BACKEND_DESTINATION_LOCATION}"
@@ -113,6 +115,24 @@ locations:
       password: "pass1"
       repoId:
         - 65f9fd61-42fe-4a68-9ac0-6ba25311cc85
+  - name: "${CRR_SOURCE_LOCATION_NAME}"
+    locationType: "location-scality-crr-v1"
+    details:
+      endpoint: "http://s3.${SUBDOMAIN}:80"
+      stsEndpoint: "http://sts.${SUBDOMAIN}:80"
+      accessKey: ""
+      secretKey: ""
+      bucketMatch: no
+      repoId: []
+  - name: "${CRR_DESTINATION_LOCATION_NAME}"
+    locationType: "location-scality-crr-v1"
+    details:
+      endpoint: "http://s3.${SUBDOMAIN}:80"
+      stsEndpoint: "http://sts.${SUBDOMAIN}:80"
+      accessKey: ""
+      secretKey: ""
+      bucketMatch: no
+      repoId: []
 workflows:
   replication: []
   lifecycle: []

tests/zenko_tests/e2e_config/accounts.py

@@ -31,12 +31,13 @@ def get_credentials(token, account_id):
     return res
 
 
-def create_account_secret(name, credentials, namespace="default"):
+def create_account_secret(name, credentials, account_id, namespace="default"):
     """
     Create a k8s secret resource for account
 
     :param name: secret name
     :param credentials: sts assume role credentials
+    :param account_id: account id
     :param namespace: k8s namespace
     """
     _log.info("creating account secret")
@@ -50,7 +51,12 @@ def create_account_secret(name, credentials, namespace="default"):
                 "type": "end2end",
             },
         ),
-        string_data=credentials,
+        string_data={
+            "AccessKeyId": credentials["AccessKeyId"],
+            "SecretAccessKey": credentials["SecretAccessKey"],
+            "SessionToken": credentials["SessionToken"],
+            "AccountId": account_id,
+        },
     )
 
     try:
@@ -88,9 +94,12 @@ def create_account(client, token, uuid, account_name, namespace="default"):
         creds = get_credentials(token, res.id)
         create_account_secret(name="end2end-account-%s" % (res.userName),
                               credentials=creds["Credentials"],
+                              account_id=res.id,
                               namespace=namespace)
 
         _log.info("created account")
+
+        return creds["Credentials"]
     except Exception as e:
         raise Exception(
             "Failed to create account '%s': %s" % (account_name, e))

tests/zenko_tests/e2e_config/clients.py

@@ -1,10 +1,10 @@
 import os
 import boto3
 
-VAULT_ENDPOINT = os.getenv("VAULT_ENDPOINT")
+STS_ENDPOINT = os.getenv("STS_ENDPOINT")
 session = boto3.session.Session()
 
 stsclient = session.client(
     service_name='sts',
-    endpoint_url=VAULT_ENDPOINT,
+    endpoint_url=STS_ENDPOINT,
 )

tests/zenko_tests/e2e_config/locations.py

@@ -1,22 +1,98 @@
 #!/usr/bin/env python
 
+from boto3 import Session
 import logging
 import os
 
 _log = logging.getLogger("end2end configuration")
 
-def create_location(client, uuid, location):
+IAM_ENDPOINT = os.getenv("IAM_ENDPOINT", "http://iam.zenko.local")
+CRR_ROLE_NAME = os.getenv("CRR_ROLE_NAME", "crr-role")
+
+def _setup_crr_iam_resources(account_creds):
+    """
+    Sets up a CRR site by creating the user, role and policy.
+    :param account_creds: credentials of the crr site account
+    :return: accessKey and secretKey of the created crr user
+    """
+    try:
+        iam_client = Session(
+            aws_access_key_id=account_creds["AccessKeyId"],
+            aws_secret_access_key=account_creds["SecretAccessKey"],
+            aws_session_token=account_creds["SessionToken"],
+        ).client('iam', endpoint_url=IAM_ENDPOINT, region_name='us-east-1')
+
+        user = iam_client.create_user(UserName="crr-user")
+        credentials = iam_client.create_access_key(UserName="crr-user")
+
+        iam_client.create_role(
+            RoleName=CRR_ROLE_NAME,
+            AssumeRolePolicyDocument='''{
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Principal": {
+                            "AWS": "''' + user["User"]["Arn"] + '''"
+                        },
+                        "Action": "sts:AssumeRole"
+                    }
+                ]
+            }'''
+        )
+
+        policy = iam_client.create_policy(
+            PolicyName="crr-policy",
+            PolicyDocument='''{
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": "s3:ReplicateObject",
+                        "Resource": "arn:aws:s3:::*/*"
+                    }
+                ]
+            }'''
+        )
+
+        iam_client.attach_role_policy(
+            RoleName=CRR_ROLE_NAME,
+            PolicyArn=policy["Policy"]["Arn"]
+        )
+
+        return {
+            "accessKey": credentials["AccessKey"]["AccessKeyId"],
+            "secretKey": credentials["AccessKey"]["SecretAccessKey"],
+        }
+    except Exception as e:
+        raise Exception("Failed to setup CRR site: %s" % e)
+
+def create_location(client, uuid, location, accounts_creds):
     """
     Creates a location
     :param client: swagger client
     :param uuid: zenko instance uuid
     :param location: location details
+    :param account_credentials: credentials of the accounts created
     """
 
     ENABLE_RING_TESTS = os.environ['ENABLE_RING_TESTS']
     if ENABLE_RING_TESTS == "false" and location["locationType"] == "location-scality-ring-s3-v1":
         return
-
+
+    DEPLOY_CRR_LOCATIONS = os.getenv('DEPLOY_CRR_LOCATIONS', 'true')
+    if location["locationType"] == "location-scality-crr-v1":
+        if DEPLOY_CRR_LOCATIONS == "false":
+            return
+
+        account_name = os.environ['CRR_SOURCE_ACCOUNT_NAME']
+        if location["name"] == os.environ['CRR_DESTINATION_LOCATION_NAME']:
+            account_name = os.environ['CRR_DESTINATION_ACCOUNT_NAME']
+
+        user_creds = _setup_crr_iam_resources(accounts_creds[account_name])
+        location["details"]["accessKey"] = user_creds["accessKey"]
+        location["details"]["secretKey"] = user_creds["secretKey"]
+
     try:
         Location_V1 = client.get_model('location-v1')
         if "bootstrapList" not in location["details"]:
@@ -32,7 +108,7 @@ def create_location(client, uuid, location):
             .result
         )
 
-        _log.info("location created")
+        _log.info("location %s created", location["name"])
     except Exception as e:
         raise Exception(
             "Failed to create location '%s': %s" % (location["name"], e))