diff --git a/.github/scripts/end2end/configure-e2e.sh b/.github/scripts/end2end/configure-e2e.sh index 7a41195494..26a18fd732 100755 --- a/.github/scripts/end2end/configure-e2e.sh +++ b/.github/scripts/end2end/configure-e2e.sh @@ -11,7 +11,8 @@ NAMESPACE=${3:-default} SERVICE_ACCOUNT="${ZENKO_NAME}-config" POD_NAME="${ZENKO_NAME}-config" MANAGEMENT_ENDPOINT="http://${ZENKO_NAME}-management-orbit-api:5001" -VAULT_ENDPOINT="http://${ZENKO_NAME}-connector-vault-sts-api" +IAM_ENDPOINT="http://${ZENKO_NAME}-management-vault-iam-admin-api" +STS_ENDPOINT="http://${ZENKO_NAME}-connector-vault-sts-api" UUID=$(kubectl get zenko ${ZENKO_NAME} --namespace ${NAMESPACE} -o jsonpath='{.status.instanceID}') TOKEN=$(get_token) @@ -80,7 +81,8 @@ kubectl run ${POD_NAME} \ --env="TOKEN=${TOKEN}" \ --env="UUID=${UUID}" \ --env="MANAGEMENT_ENDPOINT=${MANAGEMENT_ENDPOINT}" \ - --env="VAULT_ENDPOINT=${VAULT_ENDPOINT}" \ + --env="IAM_ENDPOINT=${IAM_ENDPOINT}" \ + --env="STS_ENDPOINT=${STS_ENDPOINT}" \ --env="NAMESPACE=${NAMESPACE}" \ --env=VERIFY_CERTIFICATES=false \ --env=ENABLE_RING_TESTS=${ENABLE_RING_TESTS} \ @@ -106,6 +108,12 @@ kubectl run ${POD_NAME} \ --env=AZURE_ARCHIVE_BUCKET_NAME=${AZURE_ARCHIVE_BUCKET_NAME} \ --env=AZURE_ARCHIVE_BUCKET_NAME_2=${AZURE_ARCHIVE_BUCKET_NAME_2} \ --env=AZURE_ARCHIVE_QUEUE_NAME=${AZURE_ARCHIVE_QUEUE_NAME} \ + --env=CRR_SOURCE_LOCATION_NAME=${CRR_SOURCE_LOCATION_NAME} \ + --env=CRR_DESTINATION_LOCATION_NAME=${CRR_DESTINATION_LOCATION_NAME} \ + --env=CRR_SOURCE_ACCOUNT_NAME=${CRR_SOURCE_ACCOUNT_NAME} \ + --env=CRR_DESTINATION_ACCOUNT_NAME=${CRR_DESTINATION_ACCOUNT_NAME} \ + --env=CRR_ROLE_NAME=${CRR_ROLE_NAME} \ + --env=DEPLOY_CRR_LOCATIONS=${DEPLOY_CRR_LOCATIONS} \ --command -- python3 configuration.py ## wait for updates to trigger zenko upgrades diff --git a/.github/scripts/end2end/run-e2e-test.sh b/.github/scripts/end2end/run-e2e-test.sh index 16037f4a3f..ff596a1407 100755 --- a/.github/scripts/end2end/run-e2e-test.sh +++ b/.github/scripts/end2end/run-e2e-test.sh @@ -37,6 +37,16 @@ ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-management-vault-admin-cred ZENKO_ACCESS_KEY=$(kubectl get secret end2end-account-zenko -o jsonpath='{.data.AccessKeyId}' | base64 -d) ZENKO_SECRET_KEY=$(kubectl get secret end2end-account-zenko -o jsonpath='{.data.SecretAccessKey}' | base64 -d) ZENKO_SESSION_TOKEN=$(kubectl get secret end2end-account-zenko -o jsonpath='{.data.SessionToken}' | base64 -d) +SOURCE_ACCESS_KEY=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.AccessKeyId}' | base64 -d) +SOURCE_SECRET_KEY=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d) +SOURCE_SESSION_TOKEN=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.SessionToken}' | base64 -d) +SOURCE_ACCOUNT_ID=$(kubectl get secret "end2end-account-${CRR_SOURCE_ACCOUNT_NAME}" -o jsonpath='{.data.AccountId}' | base64 -d) +CRR_SOURCE_INFO="{\"AccessKeyId\":\"${SOURCE_ACCESS_KEY}\",\"SecretAccessKey\":\"${SOURCE_SECRET_KEY}\",\"SessionToken\":\"${SOURCE_SESSION_TOKEN}\",\"AccountId\":\"${SOURCE_ACCOUNT_ID}\"}" +DESTINATION_ACCESS_KEY=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.AccessKeyId}' | base64 -d) +DESTINATION_SECRET_KEY=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d) +DESTINATION_SESSION_TOKEN=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.SessionToken}' | base64 -d) +DESTINATION_ACCOUNT_ID=$(kubectl get secret "end2end-account-${CRR_DESTINATION_ACCOUNT_NAME}" -o jsonpath='{.data.AccountId}' | base64 -d) +CRR_DESTINATION_INFO="{\"AccessKeyId\":\"${DESTINATION_ACCESS_KEY}\",\"SecretAccessKey\":\"${DESTINATION_SECRET_KEY}\",\"SessionToken\":\"${DESTINATION_SESSION_TOKEN}\",\"AccountId\":\"${DESTINATION_ACCOUNT_ID}\"}" OIDC_FULLNAME="${OIDC_FIRST_NAME} ${OIDC_LAST_NAME}" KEYCLOAK_TEST_USER="${OIDC_USERNAME}-norights" KEYCLOAK_TEST_PASSWORD=${OIDC_PASSWORD} @@ -124,6 +134,11 @@ run_e2e_test() { --env=MONGO_WRITE_CONCERN=${MONGO_WRITE_CONCERN} \ --env=MONGO_AUTH_USERNAME=${MONGO_AUTH_USERNAME} \ --env=MONGO_AUTH_PASSWORD=${MONGO_AUTH_PASSWORD} \ + --env=CRR_SOURCE_LOCATION_NAME=${CRR_SOURCE_LOCATION_NAME} \ + --env=CRR_SOURCE_INFO=${CRR_SOURCE_INFO} \ + --env=CRR_DESTINATION_LOCATION_NAME=${CRR_DESTINATION_LOCATION_NAME} \ + --env=CRR_DESTINATION_INFO=${CRR_DESTINATION_INFO} \ + --env=CRR_ROLE_NAME=${CRR_ROLE_NAME} \ --env=MOCHA_FILE=${MOCHA_FILE} \ --override-type strategic \ --overrides=' diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index 8ac9f6eb25..26d38d213d 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -74,6 +74,12 @@ env: GCP_SECRET_KEY: ${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }} GCP_BACKEND_SERVICE_KEY: ${{ secrets.GCP_BACKEND_SERVICE_KEY }} GCP_BACKEND_SERVICE_EMAIL: ${{ secrets.GCP_BACKEND_SERVICE_EMAIL }} + DEPLOY_CRR_LOCATIONS: "true" + CRR_SOURCE_LOCATION_NAME: crr-source-location + CRR_DESTINATION_LOCATION_NAME: crr-destination-location + CRR_SOURCE_ACCOUNT_NAME: crr-source-account + CRR_DESTINATION_ACCOUNT_NAME: crr-destination-account + CRR_ROLE_NAME: crr-role # Enable this for Ring tests ENABLE_RING_TESTS: "false" RING_S3C_ACCESS_KEY: accessKey1 @@ -433,6 +439,8 @@ jobs: end2end-pra: needs: [build-kafka, lint-and-build-ctst] runs-on: ubuntu-24.04-16core + env: + DEPLOY_CRR_LOCATIONS: "false" steps: - name: Checkout uses: actions/checkout@v4 diff --git a/solution/deps.yaml b/solution/deps.yaml index ec50c4dd52..e9d93b8b85 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -16,7 +16,7 @@ cloudserver: sourceRegistry: ghcr.io/scality dashboard: cloudserver/cloudserver-dashboards image: cloudserver - tag: 9.1.0 + tag: 9.1.4 envsubst: CLOUDSERVER_TAG drctl: sourceRegistry: ghcr.io/scality @@ -136,7 +136,7 @@ vault: zenko-operator: sourceRegistry: ghcr.io/scality image: zenko-operator - tag: v1.8.0-preview.1 + tag: v1.8.0-preview.2 envsubst: ZENKO_OPERATOR_TAG zookeeper: sourceRegistry: pravega diff --git a/tests/zenko_tests/configuration.py b/tests/zenko_tests/configuration.py index 6dd94dcfaa..64556d8367 100644 --- a/tests/zenko_tests/configuration.py +++ b/tests/zenko_tests/configuration.py @@ -125,9 +125,11 @@ def main(): create_buckets.create_azure_containers() create_buckets.create_azure_queues() + accounts_creds = {} + # create zenko resources for account in e2e_config["accounts"]: - accounts.create_account(client, + accounts_creds[account] = accounts.create_account(client, TOKEN, UUID, account, @@ -138,7 +140,7 @@ def main(): endpoint["locationName"]) for location in e2e_config["locations"]: - locations.create_location(client, UUID, location) + locations.create_location(client, UUID, location, accounts_creds) for wf in e2e_config["workflows"]["replication"]: workflows.create_replication_workflow(client, UUID, wf) diff --git a/tests/zenko_tests/e2e-config.yaml.template b/tests/zenko_tests/e2e-config.yaml.template index ded5c90396..8d3f33f08a 100644 --- a/tests/zenko_tests/e2e-config.yaml.template +++ b/tests/zenko_tests/e2e-config.yaml.template @@ -1,5 +1,7 @@ accounts: - "zenko" + - "${CRR_SOURCE_ACCOUNT_NAME}" + - "${CRR_DESTINATION_ACCOUNT_NAME}" endpoints: [] locations: - name: "${AWS_BACKEND_DESTINATION_LOCATION}" @@ -113,6 +115,24 @@ locations: password: "pass1" repoId: - 65f9fd61-42fe-4a68-9ac0-6ba25311cc85 + - name: "${CRR_SOURCE_LOCATION_NAME}" + locationType: "location-scality-crr-v1" + details: + endpoint: "http://s3.${SUBDOMAIN}:80" + stsEndpoint: "http://sts.${SUBDOMAIN}:80" + accessKey: "" + secretKey: "" + bucketMatch: no + repoId: [] + - name: "${CRR_DESTINATION_LOCATION_NAME}" + locationType: "location-scality-crr-v1" + details: + endpoint: "http://s3.${SUBDOMAIN}:80" + stsEndpoint: "http://sts.${SUBDOMAIN}:80" + accessKey: "" + secretKey: "" + bucketMatch: no + repoId: [] workflows: replication: [] lifecycle: [] diff --git a/tests/zenko_tests/e2e_config/accounts.py b/tests/zenko_tests/e2e_config/accounts.py index b484a16b0b..08fd597b69 100644 --- a/tests/zenko_tests/e2e_config/accounts.py +++ b/tests/zenko_tests/e2e_config/accounts.py @@ -31,12 +31,13 @@ def get_credentials(token, account_id): return res -def create_account_secret(name, credentials, namespace="default"): +def create_account_secret(name, credentials, account_id, namespace="default"): """ Create a k8s secret resource for account :param name: secret name :param credentials: sts assume role credentials + :param account_id: account id :param namespace: k8s namespace """ _log.info("creating account secret") @@ -50,7 +51,12 @@ def create_account_secret(name, credentials, namespace="default"): "type": "end2end", }, ), - string_data=credentials, + string_data={ + "AccessKeyId": credentials["AccessKeyId"], + "SecretAccessKey": credentials["SecretAccessKey"], + "SessionToken": credentials["SessionToken"], + "AccountId": account_id, + }, ) try: @@ -88,9 +94,12 @@ def create_account(client, token, uuid, account_name, namespace="default"): creds = get_credentials(token, res.id) create_account_secret(name="end2end-account-%s" % (res.userName), credentials=creds["Credentials"], + account_id=res.id, namespace=namespace) _log.info("created account") + + return creds["Credentials"] except Exception as e: raise Exception( "Failed to create account '%s': %s" % (account_name, e)) diff --git a/tests/zenko_tests/e2e_config/clients.py b/tests/zenko_tests/e2e_config/clients.py index 17b6152e39..dd142f5469 100644 --- a/tests/zenko_tests/e2e_config/clients.py +++ b/tests/zenko_tests/e2e_config/clients.py @@ -1,10 +1,10 @@ import os import boto3 -VAULT_ENDPOINT = os.getenv("VAULT_ENDPOINT") +STS_ENDPOINT = os.getenv("STS_ENDPOINT") session = boto3.session.Session() stsclient = session.client( service_name='sts', - endpoint_url=VAULT_ENDPOINT, + endpoint_url=STS_ENDPOINT, ) diff --git a/tests/zenko_tests/e2e_config/locations.py b/tests/zenko_tests/e2e_config/locations.py index 578f4e5057..1635e056b2 100644 --- a/tests/zenko_tests/e2e_config/locations.py +++ b/tests/zenko_tests/e2e_config/locations.py @@ -1,22 +1,98 @@ #!/usr/bin/env python +from boto3 import Session import logging import os _log = logging.getLogger("end2end configuration") -def create_location(client, uuid, location): +IAM_ENDPOINT = os.getenv("IAM_ENDPOINT", "http://iam.zenko.local") +CRR_ROLE_NAME = os.getenv("CRR_ROLE_NAME", "crr-role") + +def _setup_crr_iam_resources(account_creds): + """ + Sets up a CRR site by creating the user, role and policy. + :param account_creds: credentials of the crr site account + :return: accessKey and secretKey of the created crr user + """ + try: + iam_client = Session( + aws_access_key_id=account_creds["AccessKeyId"], + aws_secret_access_key=account_creds["SecretAccessKey"], + aws_session_token=account_creds["SessionToken"], + ).client('iam', endpoint_url=IAM_ENDPOINT, region_name='us-east-1') + + user = iam_client.create_user(UserName="crr-user") + credentials = iam_client.create_access_key(UserName="crr-user") + + iam_client.create_role( + RoleName=CRR_ROLE_NAME, + AssumeRolePolicyDocument='''{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "''' + user["User"]["Arn"] + '''" + }, + "Action": "sts:AssumeRole" + } + ] + }''' + ) + + policy = iam_client.create_policy( + PolicyName="crr-policy", + PolicyDocument='''{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:ReplicateObject", + "Resource": "arn:aws:s3:::*/*" + } + ] + }''' + ) + + iam_client.attach_role_policy( + RoleName=CRR_ROLE_NAME, + PolicyArn=policy["Policy"]["Arn"] + ) + + return { + "accessKey": credentials["AccessKey"]["AccessKeyId"], + "secretKey": credentials["AccessKey"]["SecretAccessKey"], + } + except Exception as e: + raise Exception("Failed to setup CRR site: %s" % e) + +def create_location(client, uuid, location, accounts_creds): """ Creates a location :param client: swagger client :param uuid: zenko instance uuid :param location: location details + :param account_credentials: credentials of the accounts created """ ENABLE_RING_TESTS = os.environ['ENABLE_RING_TESTS'] if ENABLE_RING_TESTS == "false" and location["locationType"] == "location-scality-ring-s3-v1": return - + + DEPLOY_CRR_LOCATIONS = os.getenv('DEPLOY_CRR_LOCATIONS', 'true') + if location["locationType"] == "location-scality-crr-v1": + if DEPLOY_CRR_LOCATIONS == "false": + return + + account_name = os.environ['CRR_SOURCE_ACCOUNT_NAME'] + if location["name"] == os.environ['CRR_DESTINATION_LOCATION_NAME']: + account_name = os.environ['CRR_DESTINATION_ACCOUNT_NAME'] + + user_creds = _setup_crr_iam_resources(accounts_creds[account_name]) + location["details"]["accessKey"] = user_creds["accessKey"] + location["details"]["secretKey"] = user_creds["secretKey"] + try: Location_V1 = client.get_model('location-v1') if "bootstrapList" not in location["details"]: @@ -32,7 +108,7 @@ def create_location(client, uuid, location): .result ) - _log.info("location created") + _log.info("location %s created", location["name"]) except Exception as e: raise Exception( "Failed to create location '%s': %s" % (location["name"], e)) diff --git a/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js b/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js index bb15c2bf49..4fbf2272a3 100644 --- a/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js +++ b/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js @@ -496,7 +496,7 @@ class ReplicationUtility { }, cb); } - putBucketReplicationMultipleBackend( + putBucketReplication( srcBucket, destBucket, roleArn, @@ -521,6 +521,12 @@ class ReplicationUtility { }, cb); } + deleteBucketReplication(bucketName, cb) { + this.s3.deleteBucketReplication({ + Bucket: bucketName, + }, cb); + } + getHeadObject(bucketName, key, cb) { this.s3.headObject({ Bucket: bucketName, @@ -752,6 +758,40 @@ class ReplicationUtility { }); } + compareObjectsCRR(srcBucket, destClient, destBucket, key, userMetadataField, cb) { + return async.series([ + next => this.waitUntilReplicated(srcBucket, key, undefined, next), + next => this.getObject(srcBucket, key, next), + next => destClient.getObject(destBucket, key, next), + ], (err, data) => { + if (err) { + return cb(err); + } + const srcData = data[1]; + const destData = data[2]; + assert.strictEqual(srcData.ReplicationStatus, 'COMPLETED'); + assert.strictEqual(destData.ReplicationStatus, 'REPLICA'); + assert.strictEqual( + srcData.ContentLength, + destData.ContentLength, + ); + this._compareObjectBody(srcData.Body, destData.Body); + const srcUserMD = srcData.Metadata; + assert.strictEqual( + srcData.VersionId, + destData.VersionId, + ); + if (userMetadataField) { + const destUserMD = destData.Metadata; + assert.strictEqual( + srcUserMD[userMetadataField], + destUserMD[userMetadataField], + ); + } + return cb(); + }); + } + compareObjectsOneToMany( srcBucket, awsDestBucket, @@ -968,6 +1008,23 @@ class ReplicationUtility { }); } + compareACLsCRR(srcBucket, destClient, destBucket, key, cb) { + return async.series([ + next => this.waitUntilReplicated(srcBucket, key, undefined, next), + next => this.getObjectACL(srcBucket, key, next), + next => destClient.getObjectACL(destBucket, key, next), + ], (err, data) => { + if (err) { + return cb(err); + } + assert.strictEqual( + data[1].Grants[0].Permission, + data[2].Grants[0].Permission, + ); + return cb(); + }); + } + compareObjectTagsAWS( srcBucket, destBucket, @@ -1004,6 +1061,43 @@ class ReplicationUtility { }); } + compareObjectTagCRR( + srcBucket, + destClient, + destBucket, + key, + cb, + ) { + return async.series([ + next => this.waitUntilReplicated( + srcBucket, + key, + undefined, + next, + ), + next => this.getObjectTagging( + srcBucket, + key, + undefined, + next, + ), + next => destClient.getObjectTagging( + destBucket, + key, + null, + next, + ), + ], (err, data) => { + if (err) { + return cb(err); + } + const srcData = data[1]; + const destData = data[2]; + assert.deepStrictEqual(srcData.TagSet, destData.TagSet); + return cb(); + }); + } + compareObjectTagsAzure( srcBucket, destContainer, @@ -1095,6 +1189,19 @@ class ReplicationUtility { return cb(); }); } + + assertVersionCount(bucketName, expectedCount, cb) { + this.s3.listObjectVersions({ + Bucket: bucketName, + }, (err, data) => { + if (err) { + return cb(err); + } + const totalCount = data.Versions.length + data.DeleteMarkers.length; + assert.strictEqual(totalCount, expectedCount); + return cb(); + }); + } } module.exports = ReplicationUtility; diff --git a/tests/zenko_tests/node_tests/backbeat/tests/api/metrics.js b/tests/zenko_tests/node_tests/backbeat/tests/api/metrics.js index 22f0f70f50..6b1745c23b 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/api/metrics.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/api/metrics.js @@ -123,7 +123,7 @@ describe('Backbeat replication metrics data', function dF() { beforeEach(done => series([ next => scalityUtils.createVersionedBucket(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, 'placeholder', roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/api/objectMonitor.js b/tests/zenko_tests/node_tests/backbeat/tests/api/objectMonitor.js index a2dc42e396..7b5db563da 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/api/objectMonitor.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/api/objectMonitor.js @@ -59,7 +59,7 @@ describe('Backbeat object monitor CRR metrics', function () { beforeEach(done => series([ next => scalityUtils.createVersionedBucket(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, destBucket, roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/crr-pause-resume/awsBackend.js b/tests/zenko_tests/node_tests/backbeat/tests/crr-pause-resume/awsBackend.js index 15dbe600de..eb771b5640 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/crr-pause-resume/awsBackend.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/crr-pause-resume/awsBackend.js @@ -35,7 +35,7 @@ describe('Replication Pause-Resume with AWS backend', function () { beforeEach(done => series([ next => scalityUtils.createVersionedBucket(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, destBucket, roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/crr/awsBackend.js b/tests/zenko_tests/node_tests/backbeat/tests/crr/awsBackend.js index d96544949d..0a9017ce33 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/crr/awsBackend.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/crr/awsBackend.js @@ -28,7 +28,7 @@ describe('Replication with AWS backend', function () { beforeEach(done => series([ next => scalityUtils.createVersionedBucket(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, destBucket, roleArn, @@ -984,7 +984,7 @@ describe.skip('Replication with AWS backend: source AWS location', function () { beforeEach(done => series([ next => scalityUtils.createVersionedBucketAWS(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, destBucket, roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/crr/azureBackend.js b/tests/zenko_tests/node_tests/backbeat/tests/crr/azureBackend.js index dd952244e4..ae8be2b572 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/crr/azureBackend.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/crr/azureBackend.js @@ -29,7 +29,7 @@ describe('Replication with Azure backend', function () { beforeEach(done => series([ next => utils.createVersionedBucket(srcBucket, next), - next => utils.putBucketReplicationMultipleBackend( + next => utils.putBucketReplication( srcBucket, destContainer, roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/crr/crr.js b/tests/zenko_tests/node_tests/backbeat/tests/crr/crr.js new file mode 100644 index 0000000000..9736731618 --- /dev/null +++ b/tests/zenko_tests/node_tests/backbeat/tests/crr/crr.js @@ -0,0 +1,415 @@ +const async = require('async'); + +const { getS3Client } = require('../../../s3SDK'); +const ReplicationUtility = require('../../ReplicationUtility'); + +const sourceInfo = JSON.parse(process.env.CRR_SOURCE_INFO); +const sourceS3 = getS3Client( + sourceInfo.AccessKeyId, + sourceInfo.SecretAccessKey, + sourceInfo.SessionToken, +); + +const destinationInfo = JSON.parse(process.env.CRR_DESTINATION_INFO); +const destinationS3 = getS3Client( + destinationInfo.AccessKeyId, + destinationInfo.SecretAccessKey, + destinationInfo.SessionToken, +); + +const srcUtil = new ReplicationUtility(sourceS3); +const destUtil = new ReplicationUtility(destinationS3); + +const sourceLocation = process.env.CRR_SOURCE_LOCATION_NAME; +const destinationLocation = process.env.CRR_DESTINATION_LOCATION_NAME; + +const roleName = process.env.CRR_ROLE_NAME; +const sourceRole = `arn:aws:iam::${sourceInfo.AccountId}:role/${roleName}`; +const destinationRole = `arn:aws:iam::${destinationInfo.AccountId}:role/${roleName}`; + +describe('CRR', function () { + this.timeout(300000); + this.retries(3); + + let srcBucket; + let destBucket; + let key; + + beforeEach(done => { + key = `crr-key-${Date.now()}`; + srcBucket = `crr-source-bucket-${Date.now()}`; + destBucket = `crr-destination-bucket-${Date.now()}`; + async.series([ + next => srcUtil.createVersionedBucket(srcBucket, next), + next => destUtil.createVersionedBucket(destBucket, next), + next => srcUtil.putBucketReplication( + srcBucket, + destBucket, + `${sourceRole},${destinationRole}`, + destinationLocation, + next, + ), + ], done); + }); + + afterEach(done => async.series([ + next => srcUtil.deleteBucketReplication(srcBucket, next), + next => srcUtil.deleteVersionedBucket(srcBucket, next), + next => destUtil.deleteVersionedBucket(destBucket, next), + ], done)); + + it('should replicate an object', done => async.series([ + next => srcUtil.putObjectWithUserMetadata( + srcBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + 'customKey', + next, + ), + ], done)); + + it('should replicate a zero byte object', done => async.series([ + next => srcUtil.putObjectWithUserMetadata( + srcBucket, + key, + undefined, + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + 'customKey', + next, + ), + ], done)); + + it('should replicate delete markers', done => async.series([ + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + next, + ), + next => srcUtil.deleteObject( + srcBucket, + key, + null, + next, + ), + next => destUtil.waitUntilDeleted( + destBucket, + key, + 's3', + next, + ), + ], done)); + + it('should not replicate hard deletes', done => async.waterfall([ + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(1), + (err, data) => next(err, data.VersionId), + ), + (versionID, next) => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + err => next(err, versionID), + ), + (versionID, next) => srcUtil.deleteObject( + srcBucket, + key, + versionID, + err => next(err), + ), + next => setTimeout(() => next(), 30000), + next => destUtil.getHeadObject( + destBucket, + key, + next, + ), + ], done)); + + it('should replicate object tags', done => async.series([ + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + next, + ), + next => srcUtil.putObjectTagging( + srcBucket, + key, + undefined, + next, + ), + next => srcUtil.compareObjectTagCRR( + srcBucket, + destUtil, + destBucket, + key, + next, + ), + ], done)); + + it('should replicate object ACL', done => async.series([ + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.compareACLsCRR( + srcBucket, + destUtil, + destBucket, + key, + next, + ), + next => srcUtil.putObjectACL( + srcBucket, + key, + next, + ), + next => srcUtil.compareACLsCRR( + srcBucket, + destUtil, + destBucket, + key, + next, + ), + ], done)); + + it('should replicate a version on top of an existing version', done => async.series([ + next => destUtil.putObject( + destBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(2), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + next, + ), + next => destUtil.assertVersionCount( + destBucket, + 2, + next, + ), + ], done)); + + // TODO: Unskip after CLDSRV-632 + it.skip('should replicate version on top of a null version', done => async.series([ + next => destUtil.putBucketVersioning( + destBucket, + 'Suspended', + next, + ), + next => destUtil.putObject( + destBucket, + key, + Buffer.alloc(1), + next, + ), + next => destUtil.putBucketVersioning( + destBucket, + 'Enabled', + next, + ), + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(2), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + next, + ), + next => destUtil.assertVersionCount( + destBucket, + 2, + next, + ), + ], done)); +}); + +describe('CRR Active-Active', function () { + this.timeout(300000); + this.retries(3); + + let srcBucket; + let destBucket; + let key; + + beforeEach(done => { + key = `crr-active-active-key-${Date.now()}`; + srcBucket = `crr-active-active-source-bucket-${Date.now()}`; + destBucket = `crr-active-active-destination-bucket-${Date.now()}`; + async.series([ + next => srcUtil.createVersionedBucket(srcBucket, next), + next => destUtil.createVersionedBucket(destBucket, next), + next => srcUtil.putBucketReplication( + srcBucket, + destBucket, + `${sourceRole},${destinationRole}`, + destinationLocation, + next, + ), + next => destUtil.putBucketReplication( + destBucket, + srcBucket, + `${destinationRole},${sourceRole}`, + sourceLocation, + next, + ), + ], done); + }); + + afterEach(done => async.series([ + next => srcUtil.deleteBucketReplication(srcBucket, next), + next => destUtil.deleteBucketReplication(destBucket, next), + next => srcUtil.deleteVersionedBucket(srcBucket, next), + next => destUtil.deleteVersionedBucket(destBucket, next), + ], done)); + + it('should replicate back version from destination to source', done => async.series([ + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + next, + ), + next => destUtil.putObject( + destBucket, + key, + Buffer.alloc(2), + next, + ), + next => destUtil.compareObjectsCRR( + destBucket, + srcUtil, + srcBucket, + key, + undefined, + next, + ), + ], done)); + + it('should replicate tags to source bucket when they get updated on the destination bucket', done => async.series([ + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + next, + ), + next => destUtil.putObjectTagging( + destBucket, + key, + undefined, + next, + ), + next => destUtil.compareObjectsCRR( + destBucket, + srcUtil, + srcBucket, + key, + undefined, + next, + ), + next => destUtil.compareObjectTagCRR( + destBucket, + srcUtil, + srcBucket, + key, + next, + ), + ], done)); + + it('should replicate ACLs to source bucket when they get updated on the destination bucket', done => async.series([ + next => srcUtil.putObject( + srcBucket, + key, + Buffer.alloc(1), + next, + ), + next => srcUtil.compareObjectsCRR( + srcBucket, + destUtil, + destBucket, + key, + undefined, + next, + ), + next => destUtil.putObjectACL( + destBucket, + key, + next, + ), + next => srcUtil.compareACLsCRR( + srcBucket, + destUtil, + destBucket, + key, + next, + ), + ], done)); +}); diff --git a/tests/zenko_tests/node_tests/backbeat/tests/crr/gcpBackend.js b/tests/zenko_tests/node_tests/backbeat/tests/crr/gcpBackend.js index fd111b5df4..3ab3f47c97 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/crr/gcpBackend.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/crr/gcpBackend.js @@ -28,7 +28,7 @@ describe('Replication with GCP backend', function () { beforeEach(done => series([ next => utils.createVersionedBucket(srcBucket, next), - next => utils.putBucketReplicationMultipleBackend( + next => utils.putBucketReplication( srcBucket, destBucket, roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/crr/oneToMany.js b/tests/zenko_tests/node_tests/backbeat/tests/crr/oneToMany.js index 05ef5b7e4f..c89a7d4716 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/crr/oneToMany.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/crr/oneToMany.js @@ -35,7 +35,7 @@ describe( beforeEach(done => series([ next => utils.createVersionedBucket(srcBucket, next), - next => utils.putBucketReplicationMultipleBackend( + next => utils.putBucketReplication( srcBucket, 'placeholder', roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/retry/pendingMetrics.js b/tests/zenko_tests/node_tests/backbeat/tests/retry/pendingMetrics.js index a9100f0bdd..14295d9738 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/retry/pendingMetrics.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/retry/pendingMetrics.js @@ -67,7 +67,7 @@ describe('Backbeat API pending metrics', function () { before(done => series([ next => scalityUtils.createVersionedBucket(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, 'placeholder', roleArn, @@ -114,7 +114,7 @@ describe('Backbeat API pending metrics', function () { before(done => series([ next => scalityUtils.createVersionedBucket(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, destFailBucket, roleArn, diff --git a/tests/zenko_tests/node_tests/backbeat/tests/retry/retry.js b/tests/zenko_tests/node_tests/backbeat/tests/retry/retry.js index bcc5dd0b0f..d27c434847 100644 --- a/tests/zenko_tests/node_tests/backbeat/tests/retry/retry.js +++ b/tests/zenko_tests/node_tests/backbeat/tests/retry/retry.js @@ -131,7 +131,7 @@ describe('Backbeat replication retry', function () { beforeEach(done => series([ next => scalityUtils.createVersionedBucket(srcBucket, next), - next => scalityUtils.putBucketReplicationMultipleBackend( + next => scalityUtils.putBucketReplication( srcBucket, destFailBucket, roleArn, diff --git a/tests/zenko_tests/node_tests/package.json b/tests/zenko_tests/node_tests/package.json index 95227a37ca..4ce9d1fb92 100644 --- a/tests/zenko_tests/node_tests/package.json +++ b/tests/zenko_tests/node_tests/package.json @@ -39,7 +39,7 @@ "test_azure_crr": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json backbeat/tests/crr/azureBackend.js", "test_gcp_crr": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json backbeat/tests/crr/gcpBackend.js", "test_one_to_many": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json backbeat/tests/crr/oneToMany.js", - "test_crr": "npm-run-all test_aws_crr test_azure_crr test_gcp_crr test_one_to_many", + "test_crr": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json backbeat/tests/crr/crr.js", "test_api": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json --recursive backbeat/tests/api", "test_retry": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json --recursive backbeat/tests/retry", "test_crr_pause_resume": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json --recursive backbeat/tests/crr-pause-resume", @@ -53,7 +53,7 @@ "test_operator": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json ./init_test.js", "test_smoke": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json --recursive smoke_tests", "test_iam_policies": "mocha --tags ${MOCHA_TAGS} --exit -t 15000 --reporter mocha-multi-reporters --reporter-options configFile=config.json --recursive iam_policies", - "test_all_extensions": "run-p --aggregate-output test_aws_crr test_expiration test_transition test_ingestion_oob_s3c", + "test_all_extensions": "run-p --aggregate-output test_crr test_aws_crr test_expiration test_transition test_ingestion_oob_s3c", "test_object_api": "mocha --tags ${MOCHA_TAGS} --exit -t 10000 --reporter mocha-multi-reporters --reporter-options configFile=config.json --recursive cloudserver/keyFormatVersion/tests", "lint": "eslint $(find . -name '*.js' -not -path '*/node_modules/*')" },