diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 2b77984e2..929d6b366 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -45,7 +45,8 @@ variables: - community_private_preview_pullsecret_dockerconfigjson - RELEASE_INITIAL_VERSION - RELEASE_INITIAL_COMMIT_SHA - - RELEASE_OPERATOR_VERSION + - OPERATOR_VERSION + - BUILD_SCENARIO functions: @@ -509,23 +510,45 @@ functions: params: shell: bash working_dir: src/github.com/mongodb/mongodb-kubernetes - include_expansions_in_env: - - version_id add_to_path: - ${workdir}/bin binary: scripts/evergreen/e2e/build_tests_image_ibm.sh - pipeline_migrate_agents: + pipeline: - command: subprocess.exec - retry_on_failure: false + retry_on_failure: true type: setup params: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/release/pipeline_migrate_agent.sh + binary: scripts/release/pipeline.sh + env: + IMAGE_NAME: ${IMAGE_NAME} - pipeline: + pipeline_agent: + - command: subprocess.exec + retry_on_failure: true + type: setup + params: + shell: bash + <<: *e2e_include_expansions_in_env + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/release/pipeline_agent.sh + env: + FLAGS: ${FLAGS} + + pipeline_ops_manager: + - command: subprocess.exec + retry_on_failure: true + type: setup + params: + shell: bash + <<: *e2e_include_expansions_in_env + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/release/pipeline_ops_manager.sh + + pipeline_readiness_probe: - command: subprocess.exec retry_on_failure: true type: setup @@ -533,10 +556,9 @@ functions: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents} ${build_scenario} + binary: scripts/release/pipeline_readiness_probe.sh - # TODO: this function is very similar to pipeline and it will joined with it in the future - release_operator_pipeline: + pipeline_version_upgrade_hook: - command: subprocess.exec retry_on_failure: true type: setup @@ -544,8 +566,7 @@ functions: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes - # By default, use the git tag that triggered the task which can be overridden with RELEASE_OPERATOR_VERSION - binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py ${image_name} --build-scenario release --version ${RELEASE_OPERATOR_VERSION|*triggered_by_git_tag} + binary: scripts/release/pipeline_version_upgrade_hook.sh teardown_cloud_qa_all: - command: shell.exec @@ -846,7 +867,7 @@ functions: - macos_notary_secret - workdir - triggered_by_git_tag - - RELEASE_OPERATOR_VERSION + - OPERATOR_VERSION env: XDG_CONFIG_HOME: ${go_base_path}${workdir} GO111MODULE: "on" @@ -856,7 +877,7 @@ functions: # shell.exec EVG Task doesn't have add_to_path, so we need to explicitly add the path export below. script: | set -Eeu pipefail - export GORELEASER_CURRENT_TAG=${RELEASE_OPERATOR_VERSION|*triggered_by_git_tag} + export GORELEASER_CURRENT_TAG=${OPERATOR_VERSION|*triggered_by_git_tag} export PATH=$GOROOT/bin:$PATH export GITHUB_TOKEN=${generated_token} ${workdir}/goreleaser release --clean diff --git a/.evergreen-release.yml b/.evergreen-release.yml index 8306a7ec9..4010c3240 100644 --- a/.evergreen-release.yml +++ b/.evergreen-release.yml @@ -12,8 +12,7 @@ tasks: - func: build_multi_cluster_binary - func: pipeline vars: - image_name: meko-tests - build_scenario: --build-scenario patch + IMAGE_NAME: meko-tests - name: release_operator tags: [ "image_release" ] @@ -22,9 +21,9 @@ tasks: - func: clone - func: setup_building_host - func: quay_login - - func: release_operator_pipeline + - func: pipeline vars: - image_name: operator + IMAGE_NAME: operator # Releases init images to Quay - name: release_init_appdb @@ -34,9 +33,9 @@ tasks: - func: clone - func: setup_building_host - func: quay_login - - func: release_operator_pipeline + - func: pipeline vars: - image_name: init-appdb + IMAGE_NAME: init-appdb - name: release_init_database tags: [ "image_release" ] @@ -45,9 +44,9 @@ tasks: - func: clone - func: setup_building_host - func: quay_login - - func: release_operator_pipeline + - func: pipeline vars: - image_name: init-database + IMAGE_NAME: init-database - name: release_init_ops_manager tags: [ "image_release" ] @@ -56,9 +55,9 @@ tasks: - func: clone - func: setup_building_host - func: quay_login - - func: release_operator_pipeline + - func: pipeline vars: - image_name: init-ops-manager + IMAGE_NAME: init-ops-manager - name: release_database tags: [ "image_release" ] @@ -67,9 +66,9 @@ tasks: - func: clone - func: setup_building_host - func: quay_login - - func: release_operator_pipeline + - func: pipeline vars: - image_name: database + IMAGE_NAME: database - name: prepare_and_upload_openshift_bundles tags: [ "openshift_bundles" ] diff --git a/.evergreen.yml b/.evergreen.yml index 419c9d605..d3744d4ab 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1,3 +1,5 @@ +# For .evergreen.yml configuration and usage reference, see EVERGREEN.md file + # 2h timeout for all the tasks exec_timeout_secs: 7200 @@ -61,7 +63,7 @@ variables: variant: init_test_run - name: build_readiness_probe_image variant: init_test_run - - name: build_upgrade_hook_image + - name: build_version_upgrade_hook_image variant: init_test_run - name: build_mco_test_image variant: init_test_run @@ -195,20 +197,6 @@ parameters: value: "false" description: set this to true if you would like the pipeline to automatically push a branch with updated snippets outputs -# Each variant needs to be tagged with one or more tags referencing related build scenario: -# - pr_patch: for patches created by GitHub PRs -# - staging: for builds triggered when merging to master or release branch -# - release: for builds triggered on git tags -# For variants that are **only** triggered manually (patch) or by PCT we should use "manual_patch" tag. -# Examples: `migrate_all_agents`, `e2e_operator_perf` or `publish_om80_images`. -# -# This configuration allows us to run all the associated tasks for each scenario from evergreen command line. -# This is especially helpful when making changes to `staging` or `release` variants and testing them using Evergreen -# command line patches. For example there is no other way to trigger tasks that are run on merges to master other than -# combining them together using aliases. The same applies for tasks being run on git tags. -# -# See https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-and-Distro-Settings#project-aliases - # Triggered manually or by PCT. patch_aliases: - alias: "periodic_teardowns" @@ -292,10 +280,9 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: pipeline_agent vars: - image_name: agent - build_scenario: --build-scenario manual_release + BUILD_SCENARIO: manual_release - name: migrate_all_agents # this enables us to run this variant manually to build all the agents for the new registry @@ -304,7 +291,10 @@ tasks: - func: clone - func: setup_building_host - func: quay_login - - func: pipeline_migrate_agents + - func: pipeline_agent + vars: + BUILD_SCENARIO: manual_release + FLAGS: "--all-agents -r quay.io/mongodb/mongodb-agent" - name: run_precommit_and_push tags: [ "patch-run" ] @@ -332,10 +322,9 @@ tasks: commands: - func: clone - func: setup_building_host - - func: pipeline + - func: pipeline_agent vars: - image_name: agent - all_agents: "--all-agents" + FLAGS: "--all-agents" - name: release_all_currently_used_agents_on_ecr # this enables us to run this manually (patch) and release all agent versions to ECR to verify @@ -344,10 +333,9 @@ tasks: commands: - func: clone - func: setup_building_host - - func: pipeline + - func: pipeline_agent vars: - image_name: agent - all_agents: "--current-agents" + FLAGS: "--current-agents" - name: build_test_image commands: @@ -356,7 +344,7 @@ tasks: - func: build_multi_cluster_binary - func: pipeline vars: - image_name: meko-tests + IMAGE_NAME: meko-tests - name: build_test_image_ibm commands: @@ -379,7 +367,7 @@ tasks: - func: setup_building_host - func: pipeline vars: - image_name: mco-tests + IMAGE_NAME: mco-tests - name: build_operator_ubi commands: @@ -387,7 +375,7 @@ tasks: - func: setup_building_host - func: pipeline vars: - image_name: operator + IMAGE_NAME: operator - name: build_operator_race_ubi commands: @@ -395,7 +383,7 @@ tasks: - func: setup_building_host - func: pipeline vars: - image_name: operator-race + IMAGE_NAME: operator-race - name: build_init_om_images_ubi commands: @@ -403,7 +391,7 @@ tasks: - func: setup_building_host - func: pipeline vars: - image_name: init-ops-manager + IMAGE_NAME: init-ops-manager - name: build_init_appdb_images_ubi commands: @@ -411,15 +399,13 @@ tasks: - func: setup_building_host - func: pipeline vars: - image_name: init-appdb + IMAGE_NAME: init-appdb - name: build_agent_images_ubi commands: - func: clone - func: setup_building_host - - func: pipeline - vars: - image_name: agent + - func: pipeline_agent - name: build_init_database_image_ubi commands: @@ -427,7 +413,7 @@ tasks: - func: setup_building_host - func: pipeline vars: - image_name: init-database + IMAGE_NAME: init-database - name: build_database_image_ubi commands: @@ -435,23 +421,19 @@ tasks: - func: setup_building_host - func: pipeline vars: - image_name: database + IMAGE_NAME: database - name: build_readiness_probe_image commands: - func: clone - func: setup_building_host - - func: pipeline - vars: - image_name: readiness-probe + - func: pipeline_readiness_probe - - name: build_upgrade_hook_image + - name: build_version_upgrade_hook_image commands: - func: clone - func: setup_building_host - - func: pipeline - vars: - image_name: upgrade-hook + - func: pipeline_version_upgrade_hook - name: prepare_aws priority: 59 @@ -498,9 +480,7 @@ tasks: - func: clone - func: switch_context - func: setup_building_host - - func: pipeline - vars: - image_name: ops-manager + - func: pipeline_ops_manager - name: publish_ops_manager commands: @@ -509,10 +489,9 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: pipeline_ops_manager vars: - image_name: ops-manager - build_scenario: --build-scenario manual_release + BUILD_SCENARIO: manual_release - name: prepare_and_upload_openshift_bundles_for_e2e commands: @@ -590,7 +569,7 @@ task_groups: - e2e_search_external_basic - e2e_search_external_tls - # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant + # This is the task group that contains all the tests run in the e2e_mdb_kind_ubi_cloudqa build variant - name: e2e_mdb_kind_cloudqa_task_group max_hosts: -1 <<: *setup_group @@ -1195,7 +1174,7 @@ buildvariants: # e2e___[_] # where is any of mdb|om|operator # where is any of kind|openshift - # where is any of ubuntu|ubi + # where is only ubi # where denotes the OM version tested (e.g. om50, om60, cloudqa) - used only for MDB tests # MongoDBCommunity build variant @@ -1490,7 +1469,7 @@ buildvariants: - name: e2e_smoke_arm_task_group - name: e2e_static_smoke_arm - display_name: e2e_smoke_arm + display_name: e2e_static_smoke_arm tags: [ "staging", "e2e_smoke_test_suite", "static" ] run_on: - ubuntu2404-arm64-large @@ -1701,7 +1680,7 @@ buildvariants: - name: build_database_image_ubi - name: build_agent_images_ubi - name: build_readiness_probe_image - - name: build_upgrade_hook_image + - name: build_version_upgrade_hook_image - name: prepare_aws - name: init_test_run_ibm_power diff --git a/EVERGREEN.md b/EVERGREEN.md new file mode 100644 index 000000000..ac3aa18ff --- /dev/null +++ b/EVERGREEN.md @@ -0,0 +1,43 @@ +# Evergreen CI/CD Configuration Guide + +Each variant needs to be tagged with one or more tags referencing related build scenario: + - pr_patch: for patches created by GitHub PRs + - staging: for builds triggered when merging to master or release branch + - release: for builds triggered on git tags + +For variants that are **only** triggered manually (patch) or by PCT we should use "manual_patch" tag. +Examples: `migrate_all_agents`, `e2e_operator_perf` or `publish_om80_images`. + +This configuration allows us to run all the associated tasks for each scenario from evergreen command line. +This is especially helpful when making changes to `staging` or `release` variants and testing them using Evergreen +command line patches. For example there is no other way to trigger tasks that are run on merges to master other than +combining them together using aliases. The same applies for tasks being run on git tags. + +See https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-and-Distro-Settings#project-aliases + +# Running Evergreen build scenarios + +Use `BUILD_SCENARIO` parameter to select the build scenario you want to run. Based on the scenario selected the +appropriate `REGISTRY` and `OPERATOR_VERSION` values will be selected. `REGISTRY` and `OPERATOR_VERSION` can be +individually overridden as well. Overriding `OPERATOR_VERSION` is mostly applicable for `release` scenario, where +there is no git tag to pick up. + +## Example commands + +### Full patch scenario: + +```shell +evergreen patch -p mongodb-kubernetes -a pr_patch -d "Test PR patch build" -f -y -u --path .evergreen.yml +``` + +### Staging scenario: + +```shell +evergreen patch -p mongodb-kubernetes -a staging -d "Test staging build" -f -y -u --path .evergreen.yml --param BUILD_SCENARIO=staging +``` + +### Release scenario: + +```shell +evergreen patch -p mongodb-kubernetes -a release -d "Test release build" -f -y -u --path .evergreen.yml --param BUILD_SCENARIO=release --param OPERATOR_VERSION=1.3.0-rc +``` diff --git a/Makefile b/Makefile index b9fffd98d..42b428a98 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ usage: @ echo " full: ('make' is an alias for this command) ensures K8s cluster is up, cleans Kubernetes" @ echo " resources, build-push-deploy operator, push-deploy database, create secrets, " @ echo " config map, resources etc" - @ echo " appdb: build and push AppDB image. Specify 'om_version' in format '4.2.1' to provide the already released Ops Manager" + @ echo " appdb: build and push AppDB image. Specify 'OM_VERSION' in format '4.2.1' to provide the already released Ops Manager" @ echo " version which will be used to find the matching tag and find the Automation Agent version. Add 'om_branch' " @ echo " if Ops Manager is not released yet and you want to have some git branch as the source " @ echo " parameters in ~/operator-dev/om" @@ -75,13 +75,13 @@ operator: configure-operator build-and-push-operator-image # build-push, (todo) restart database database: aws_login - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py database + @ scripts/dev/run_python.sh scripts/release/pipeline.py database readiness_probe: aws_login - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py readiness-probe + @ scripts/dev/run_python.sh scripts/release/pipeline.py readiness-probe upgrade_hook: aws_login - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py upgrade-hook + @ scripts/dev/run_python.sh scripts/release/pipeline.py upgrade-hook # ensures cluster is up, cleans Kubernetes + OM, build-push-deploy operator, # push-deploy database, create secrets, config map, resources etc @@ -90,7 +90,7 @@ full: build-and-push-images # build-push appdb image appdb: aws_login - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py --include appdb + @ scripts/dev/run_python.sh scripts/release/pipeline.py --include appdb # runs the e2e test: make e2e test=e2e_sharded_cluster_pv. The Operator is redeployed before the test, the namespace is cleaned. # The e2e test image is built and pushed together with all main ones (operator, database, init containers) @@ -154,19 +154,19 @@ aws_cleanup: @ scripts/evergreen/prepare_aws.sh build-and-push-operator-image: aws_login - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py operator + @ scripts/dev/run_python.sh scripts/release/pipeline.py operator build-and-push-database-image: aws_login @ scripts/dev/build_push_database_image build-and-push-test-image: aws_login build-multi-cluster-binary @ if [[ -z "$(local)" ]]; then \ - scripts/dev/run_python.sh scripts/release/pipeline_main.py test; \ + scripts/dev/run_python.sh scripts/release/pipeline.py test; \ fi build-and-push-mco-test-image: aws_login @ if [[ -z "$(local)" ]]; then \ - scripts/dev/run_python.sh scripts/release/pipeline_main.py mco-test; \ + scripts/dev/run_python.sh scripts/release/pipeline.py mco-test; \ fi build-multi-cluster-binary: @@ -181,27 +181,27 @@ build-and-push-images: build-and-push-operator-image appdb-init-image om-init-im build-and-push-init-images: appdb-init-image om-init-image database-init-image database-init-image: - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py init-database + @ scripts/dev/run_python.sh scripts/release/pipeline.py init-database appdb-init-image: - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py init-appdb + @ scripts/dev/run_python.sh scripts/release/pipeline.py init-appdb # Not setting a parallel-factor will default to 0 which will lead to using all CPUs, that can cause docker to die. # Here we are defaulting to 6, a higher value might work for you. agent-image: - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel --parallel-factor 6 agent + @ scripts/dev/run_python.sh scripts/release/pipeline.py --parallel --parallel-factor 6 agent agent-image-slow: - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel-factor 1 agent + @ scripts/dev/run_python.sh scripts/release/pipeline.py --parallel-factor 1 agent operator-image: - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py operator + @ scripts/dev/run_python.sh scripts/release/pipeline.py operator om-init-image: - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py init-ops-manager + @ scripts/dev/run_python.sh scripts/release/pipeline.py init-ops-manager om-image: - @ scripts/dev/run_python.sh scripts/release/pipeline_main.py ops-manager + @ scripts/dev/run_python.sh scripts/release/pipeline.py ops-manager configure-operator: @ scripts/dev/configure_operator.sh diff --git a/build_info.json b/build_info.json index f7f903f7e..e57265f84 100644 --- a/build_info.json +++ b/build_info.json @@ -178,6 +178,12 @@ "linux/arm64", "linux/amd64" ] + }, + "release": { + "repositories": ["quay.io/mongodb/mongodb-kubernetes-tests"], + "platforms": [ + "linux/amd64" + ] } }, "mco-tests": { @@ -213,7 +219,6 @@ ] }, "release": { - "version": "1.0.22", "sign": true, "olm-tag": true, "repositories": [ @@ -243,7 +248,6 @@ ] }, "release": { - "version": "1.0.9", "sign": true, "olm-tag": true, "repositories": ["quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook"], @@ -286,14 +290,12 @@ "ops-manager": { "dockerfile-path": "docker/mongodb-enterprise-ops-manager/Dockerfile", "patch": { - "version": "om-version-from-release.json", "repositories": ["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-enterprise-ops-manager-ubi"], "platforms": [ "linux/amd64" ] }, "staging": { - "version": "om-version-from-release.json", "sign": true, "repositories": ["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-enterprise-ops-manager-ubi"], "platforms": [ diff --git a/docker/mongodb-kubernetes-tests/tests/clusterwideoperator/om_multiple.py b/docker/mongodb-kubernetes-tests/tests/clusterwideoperator/om_multiple.py index f58910e71..4ef6401d1 100644 --- a/docker/mongodb-kubernetes-tests/tests/clusterwideoperator/om_multiple.py +++ b/docker/mongodb-kubernetes-tests/tests/clusterwideoperator/om_multiple.py @@ -13,13 +13,11 @@ from tests.conftest import ( get_central_cluster_client, get_evergreen_task_id, - get_member_cluster_api_client, get_member_cluster_clients, get_multi_cluster_operator_clustermode, get_multi_cluster_operator_installation_config, get_operator_clusterwide, get_operator_installation_config, - get_version_id, is_multi_cluster, ) from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment diff --git a/docker/mongodb-kubernetes-tests/tests/olm/olm_meko_operator_upgrade_with_resources.py b/docker/mongodb-kubernetes-tests/tests/olm/olm_meko_operator_upgrade_with_resources.py index 6adc8ade5..a9e7bc5c3 100644 --- a/docker/mongodb-kubernetes-tests/tests/olm/olm_meko_operator_upgrade_with_resources.py +++ b/docker/mongodb-kubernetes-tests/tests/olm/olm_meko_operator_upgrade_with_resources.py @@ -123,7 +123,6 @@ def latest_released_meko_version(): @pytest.mark.e2e_olm_meko_operator_upgrade_with_resources def test_meko_install_stable_operator_version( namespace: str, - version_id: str, latest_released_meko_version: str, catalog_source: CustomObject, meko_subscription: CustomObject, @@ -464,7 +463,6 @@ def test_connectivity_after_meko_uninstall( @pytest.mark.e2e_olm_meko_operator_upgrade_with_resources def test_install_mck_operator( namespace: str, - version_id: str, catalog_source: CustomObject, ): current_operator_version = get_current_operator_version() diff --git a/docker/mongodb-kubernetes-tests/tests/olm/olm_operator_upgrade_with_resources.py b/docker/mongodb-kubernetes-tests/tests/olm/olm_operator_upgrade_with_resources.py index f0aafacf5..70373e5f7 100644 --- a/docker/mongodb-kubernetes-tests/tests/olm/olm_operator_upgrade_with_resources.py +++ b/docker/mongodb-kubernetes-tests/tests/olm/olm_operator_upgrade_with_resources.py @@ -87,7 +87,6 @@ def current_operator_version(): @pytest.mark.e2e_olm_operator_upgrade_with_resources def test_install_stable_operator_version( namespace: str, - version_id: str, current_operator_version: str, catalog_source: CustomObject, subscription: CustomObject, @@ -338,7 +337,6 @@ def test_resources_in_running_state_before_upgrade( @pytest.mark.e2e_olm_operator_upgrade_with_resources def test_operator_upgrade_to_fast( namespace: str, - version_id: str, catalog_source: CustomObject, subscription: CustomObject, ): @@ -356,8 +354,8 @@ def update_subscription() -> bool: except kubernetes.client.ApiException as e: if e.status == 409: return False - else: - raise e + else: + raise e run_periodically(update_subscription, timeout=100, msg="Subscription to be updated") diff --git a/mongodb-community-operator/scripts/dev/e2e.py b/mongodb-community-operator/scripts/dev/e2e.py index 137f946fd..a3c585649 100755 --- a/mongodb-community-operator/scripts/dev/e2e.py +++ b/mongodb-community-operator/scripts/dev/e2e.py @@ -81,7 +81,7 @@ def create_test_pod(args: argparse.Namespace, namespace: str) -> None: "containers": [ { "name": TEST_POD_NAME, - "image": f"{os.getenv('BASE_REPO_URL')}/mongodb-community-tests:{os.getenv('VERSION_ID')}", + "image": f"{os.getenv('REGISTRY')}/mongodb-community-tests:{os.getenv('OPERATOR_VERSION')}", "imagePullPolicy": "Always", "env": [ { @@ -89,12 +89,12 @@ def create_test_pod(args: argparse.Namespace, namespace: str) -> None: "value": f"{args.cluster_wide}", }, { - "name": "VERSION_ID", - "value": f"{os.getenv('VERSION_ID')}", + "name": "OPERATOR_VERSION", + "value": f"{os.getenv('OPERATOR_VERSION')}", }, { - "name": "BASE_REPO_URL", - "value": f"{os.getenv('BASE_REPO_URL')}", + "name": "OPERATOR_REGISTRY", + "value": f"{os.getenv('OPERATOR_REGISTRY')}", }, { "name": "MDB_COMMUNITY_AGENT_IMAGE", diff --git a/mongodb-community-operator/test/e2e/setup/test_config.go b/mongodb-community-operator/test/e2e/setup/test_config.go index 399155a12..9cd526d34 100644 --- a/mongodb-community-operator/test/e2e/setup/test_config.go +++ b/mongodb-community-operator/test/e2e/setup/test_config.go @@ -9,11 +9,11 @@ const ( testNamespaceEnvName = "WATCH_NAMESPACE" testCertManagerNamespaceEnvName = "TEST_CERT_MANAGER_NAMESPACE" testCertManagerVersionEnvName = "TEST_CERT_MANAGER_VERSION" - operatorImageRepoEnvName = "BASE_REPO_URL" + operatorImageRepoEnvName = "OPERATOR_REGISTRY" clusterWideEnvName = "CLUSTER_WIDE" performCleanupEnvName = "PERFORM_CLEANUP" LocalOperatorEnvName = "LOCAL_OPERATOR" - versionIdEnv = "VERSION_ID" + operatorVersionEnvName = "OPERATOR_VERSION" ) type TestConfig struct { @@ -39,7 +39,7 @@ func LoadTestConfigFromEnv() TestConfig { OperatorImage: "mongodb-kubernetes", Namespace: envvar.GetEnvOrDefault(testNamespaceEnvName, "mongodb-test"), // nolint:forbidigo // The operator version is based on the versionID, which context sets either locally manually or evg per patch - OperatorVersion: envvar.GetEnvOrDefault(versionIdEnv, ""), // nolint:forbidigo + OperatorVersion: envvar.GetEnvOrDefault(operatorVersionEnvName, ""), // nolint:forbidigo CertManagerNamespace: envvar.GetEnvOrDefault(testCertManagerNamespaceEnvName, "cert-manager"), // nolint:forbidigo CertManagerVersion: envvar.GetEnvOrDefault(testCertManagerVersionEnvName, "v1.5.3"), // nolint:forbidigo OperatorImageRepoUrl: envvar.GetEnvOrDefault(operatorImageRepoEnvName, "quay.io/mongodb"), // nolint:forbidigo diff --git a/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa index f230f6d2a..923fa5c33 100644 --- a/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa +++ b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa @@ -10,7 +10,7 @@ source "${script_dir}/root-context" export ops_manager_version="cloud_qa" # This is required to be able to rebuild the om image and use that image which has been rebuilt -export OPS_MANAGER_REGISTRY="${BASE_REPO_URL}" +export OPS_MANAGER_REGISTRY="${REGISTRY}" CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') export CUSTOM_OM_VERSION diff --git a/scripts/dev/contexts/e2e_smoke b/scripts/dev/contexts/e2e_smoke index d34badad6..8c5438743 100644 --- a/scripts/dev/contexts/e2e_smoke +++ b/scripts/dev/contexts/e2e_smoke @@ -7,43 +7,3 @@ script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" source "${script_dir}/variables/om70" - -export DATABASE_REGISTRY="${QUAY_REGISTRY}" -export INIT_OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" -export OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" -export OPERATOR_REGISTRY="${QUAY_REGISTRY}" -export INIT_IMAGES_REGISTRY="${QUAY_REGISTRY}" -export INIT_APPDB_REGISTRY="${QUAY_REGISTRY}" -export INIT_DATABASE_REGISTRY="${QUAY_REGISTRY}" -# Since we're sourcing this as an initial step, the jq might not be there. That's why we need bash magic here. -# TODO: provide version overrides from the root-context based on the scenario -# https://docs.google.com/document/d/1eJ8iKsI0libbpcJakGjxcPfbrTn8lmcZDbQH1UqMR_g/edit?tab=t.p76ry15gwmkk#bookmark=id.j8iox5gg46rp -OPERATOR_VERSION="$(grep -o '"mongodbOperator": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - OPERATOR_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export OPERATOR_VERSION - -INIT_DATABASE_VERSION="$(grep -o '"initDatabaseVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - INIT_DATABASE_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export INIT_DATABASE_VERSION - -INIT_APPDB_VERSION="$(grep -o '"initAppDbVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - INIT_APPDB_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export INIT_APPDB_VERSION - -INIT_OPS_MANAGER_VERSION="$(grep -o '"initOpsManagerVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - INIT_OPS_MANAGER_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export INIT_OPS_MANAGER_VERSION - -DATABASE_VERSION="$(grep -o '"databaseImageVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - DATABASE_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export DATABASE_VERSION diff --git a/scripts/dev/contexts/e2e_smoke_arm b/scripts/dev/contexts/e2e_smoke_arm index f23426d9b..caab9ef79 100644 --- a/scripts/dev/contexts/e2e_smoke_arm +++ b/scripts/dev/contexts/e2e_smoke_arm @@ -9,7 +9,6 @@ source "${script_dir}/root-context" export ops_manager_version="cloud_qa" - export CUSTOM_MDB_VERSION=8.0.12-ent export KUBE_ENVIRONMENT_NAME=kind export CLUSTER_TYPE=kind diff --git a/scripts/dev/contexts/e2e_smoke_ibm_power b/scripts/dev/contexts/e2e_smoke_ibm_power index 50047fbdc..a22c6162e 100644 --- a/scripts/dev/contexts/e2e_smoke_ibm_power +++ b/scripts/dev/contexts/e2e_smoke_ibm_power @@ -8,7 +8,5 @@ script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" source "${script_dir}/variables/e2e_ibm_smoke_base" -# This is required to be able to rebuild the om image and use that image which has been rebuilt -export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') export CUSTOM_OM_VERSION diff --git a/scripts/dev/contexts/e2e_smoke_ibm_z b/scripts/dev/contexts/e2e_smoke_ibm_z index 50047fbdc..a22c6162e 100644 --- a/scripts/dev/contexts/e2e_smoke_ibm_z +++ b/scripts/dev/contexts/e2e_smoke_ibm_z @@ -8,7 +8,5 @@ script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" source "${script_dir}/variables/e2e_ibm_smoke_base" -# This is required to be able to rebuild the om image and use that image which has been rebuilt -export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') export CUSTOM_OM_VERSION diff --git a/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa index 9188ba8af..2c3016a4e 100644 --- a/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa +++ b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa @@ -11,7 +11,7 @@ export ops_manager_version="cloud_qa" export MDB_DEFAULT_ARCHITECTURE=static # This is required to be able to rebuild the om image and use that image which has been rebuilt -export OPS_MANAGER_REGISTRY="${BASE_REPO_URL}" +export OPS_MANAGER_REGISTRY="${REGISTRY}" CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') export CUSTOM_OM_VERSION diff --git a/scripts/dev/contexts/e2e_static_smoke b/scripts/dev/contexts/e2e_static_smoke index baae4ce10..2d73c4376 100644 --- a/scripts/dev/contexts/e2e_static_smoke +++ b/scripts/dev/contexts/e2e_static_smoke @@ -8,46 +8,6 @@ script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" source "${script_dir}/variables/om70" -export DATABASE_REGISTRY="${QUAY_REGISTRY}" -export INIT_OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" -export OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" -export OPERATOR_REGISTRY="${QUAY_REGISTRY}" -export INIT_IMAGES_REGISTRY="${QUAY_REGISTRY}" -export INIT_APPDB_REGISTRY="${QUAY_REGISTRY}" -export INIT_DATABASE_REGISTRY="${QUAY_REGISTRY}" -# Since we're sourcing this as an initial step, the jq might not be there. That's why we need bash magic here. -# TODO: provide version overrides from the root-context based on the scenario -# https://docs.google.com/document/d/1eJ8iKsI0libbpcJakGjxcPfbrTn8lmcZDbQH1UqMR_g/edit?tab=t.p76ry15gwmkk#bookmark=id.j8iox5gg46rp -OPERATOR_VERSION="$(grep -o '"mongodbOperator": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - OPERATOR_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export OPERATOR_VERSION - -INIT_DATABASE_VERSION="$(grep -o '"initDatabaseVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - INIT_DATABASE_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export INIT_DATABASE_VERSION - -INIT_APPDB_VERSION="$(grep -o '"initAppDbVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - INIT_APPDB_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export INIT_APPDB_VERSION - -INIT_OPS_MANAGER_VERSION="$(grep -o '"initOpsManagerVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - INIT_OPS_MANAGER_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export INIT_OPS_MANAGER_VERSION - -DATABASE_VERSION="$(grep -o '"databaseImageVersion": "[^"]*' release.json | grep -o '[^"]*$')" -if [[ "${RELEASE_OPERATOR_VERSION:-}" != "" ]]; then - DATABASE_VERSION="${RELEASE_OPERATOR_VERSION}" -fi -export DATABASE_VERSION - export MDB_DEFAULT_ARCHITECTURE=static # For static smoke tests we need the previous MDB version to have a ubi9 binary export CUSTOM_MDB_PREV_VERSION=6.0.5 diff --git a/scripts/dev/contexts/e2e_static_smoke_arm b/scripts/dev/contexts/e2e_static_smoke_arm index 00976ea0d..bad52a3ed 100644 --- a/scripts/dev/contexts/e2e_static_smoke_arm +++ b/scripts/dev/contexts/e2e_static_smoke_arm @@ -9,8 +9,6 @@ source "${script_dir}/root-context" export ops_manager_version="cloud_qa" -# This is required to be able to rebuild the om image and use that image which has been rebuilt -export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') export CUSTOM_OM_VERSION diff --git a/scripts/dev/contexts/e2e_static_smoke_ibm_power b/scripts/dev/contexts/e2e_static_smoke_ibm_power index bb4d2a8ca..e6664f409 100644 --- a/scripts/dev/contexts/e2e_static_smoke_ibm_power +++ b/scripts/dev/contexts/e2e_static_smoke_ibm_power @@ -8,10 +8,9 @@ script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" source "${script_dir}/variables/e2e_ibm_smoke_base" -# This is required to be able to rebuild the om image and use that image which has been rebuilt -export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') export CUSTOM_OM_VERSION + export MDB_DEFAULT_ARCHITECTURE=static # MDB supports ppc64le on RHEL9 only from 8.0.12 onwards export CUSTOM_MDB_VERSION=8.0.12-ent diff --git a/scripts/dev/contexts/e2e_static_smoke_ibm_z b/scripts/dev/contexts/e2e_static_smoke_ibm_z index 9acff493a..6844f7c0e 100644 --- a/scripts/dev/contexts/e2e_static_smoke_ibm_z +++ b/scripts/dev/contexts/e2e_static_smoke_ibm_z @@ -8,10 +8,9 @@ script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" source "${script_dir}/variables/e2e_ibm_smoke_base" -# This is required to be able to rebuild the om image and use that image which has been rebuilt -export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') export CUSTOM_OM_VERSION + export MDB_DEFAULT_ARCHITECTURE=static # MDB supports s390x on RHEL9 only from 8.0.12 onwards export CUSTOM_MDB_VERSION=8.0.12-ent diff --git a/scripts/dev/contexts/evg-private-context b/scripts/dev/contexts/evg-private-context index 1eb2ef0f5..93374a409 100644 --- a/scripts/dev/contexts/evg-private-context +++ b/scripts/dev/contexts/evg-private-context @@ -33,26 +33,6 @@ else fi export NAMESPACE -export BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" - -export REGISTRY="${BASE_REPO_URL}" -export QUAY_REGISTRY=quay.io/mongodb -export OPERATOR_REGISTRY=${REGISTRY} -export INIT_IMAGES_REGISTRY=${INIT_IMAGES_REGISTRY:-${REGISTRY}} -export INIT_APPDB_REGISTRY=${INIT_IMAGES_REGISTRY} -export INIT_OPS_MANAGER_REGISTRY=${INIT_IMAGES_REGISTRY-"${QUAY_REGISTRY}"} -export INIT_DATABASE_REGISTRY=${INIT_IMAGES_REGISTRY:-"${QUAY_REGISTRY}"} -export DATABASE_REGISTRY=${INIT_IMAGES_REGISTRY:-"${QUAY_REGISTRY}"} -export OPS_MANAGER_REGISTRY=${QUAY_REGISTRY} -export MONGODB_ENTERPRISE_DATABASE_IMAGE="${INIT_IMAGES_REGISTRY}/mongodb-kubernetes-database" -export INIT_DATABASE_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-kubernetes-init-database" -export MDB_AGENT_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-agent" - -# these are needed to deploy OM -export INIT_APPDB_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-kubernetes-init-appdb" -export OPS_MANAGER_IMAGE_REPOSITORY="${QUAY_REGISTRY}/mongodb-enterprise-ops-manager-ubi" -export INIT_OPS_MANAGER_IMAGE_REPOSITORY=${INIT_IMAGES_REGISTRY}/mongodb-kubernetes-init-ops-manager - export CLUSTER_TYPE="kind" # Empty sting means that we're not using it. export EVG_HOST_NAME="" @@ -93,6 +73,8 @@ export IS_PATCH="${is_patch:-"unknown"}" export BUILD_ID="${build_id:-"unknown"}" export EXECUTION="${execution:-"unknown"}" export BUILD_VARIANT="${build_variant:-"unknown"}" +export VERSION_ID="${version_id:-"unknown"}" +export GIT_TAG="${triggered_by_git_tag:-}" # var used in pipeline.py to determine if we're running on EVG host # used to discern between local pipeline image build and the build in pipeline @@ -104,11 +86,6 @@ export DELETE_KIND_NETWORK="true" # it's also set explicitly in public_*_snippets variants export CODE_SNIPPETS_COMMIT_OUTPUT=${code_snippets_commit_output:-"false"} -# MCO -# shellcheck disable=SC2154 -export READINESS_PROBE_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-readinessprobe:${version_id}" -export VERSION_UPGRADE_HOOK_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-operator-version-upgrade-post-start-hook:${version_id}" - # shellcheck disable=SC2154 export PRERELEASE_PULLSECRET_DOCKERCONFIGJSON="${community_private_preview_pullsecret_dockerconfigjson}" @@ -121,3 +98,51 @@ export cognito_workload_url="${cognito_workload_url}" export cognito_workload_user_id="${cognito_workload_user_id}" export MDB_UPDATE_LICENSES=true + +# Set BUILD_SCENARIO based on evergreen variables +# BUILD_SCENARIO can be overridden externally if needed +BUILD_SCENARIO_RELEASE="release" # Official release triggered by a git tag +BUILD_SCENARIO_STAGING="staging" # CI build from a merge to the master +BUILD_SCENARIO_PATCH="patch" # CI build for a patch/pull request +if [ -n "${BUILD_SCENARIO-}" ]; then + echo "Build scenario has been set externally, not overriding it" +elif [ "${GIT_TAG}" ]; then + BUILD_SCENARIO=$BUILD_SCENARIO_RELEASE +elif [ "${IS_PATCH}" == "true" ]; then + BUILD_SCENARIO=$BUILD_SCENARIO_PATCH +else + BUILD_SCENARIO=$BUILD_SCENARIO_STAGING +fi +echo "Setting BUILD_SCENARIO=${BUILD_SCENARIO}" +export BUILD_SCENARIO="${BUILD_SCENARIO}" + +# Set REGISTRY and OPERATOR_VERSION based on build scenario +# REGISTRY and OPERATOR_VERSION can be overridden externally if needed +PATCH_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" +STAGING_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" +RELEASE_REGISTRY="quay.io/mongodb" +case $BUILD_SCENARIO in + "${BUILD_SCENARIO_RELEASE}") + REGISTRY="${REGISTRY:-$RELEASE_REGISTRY}" + OPERATOR_VERSION="${OPERATOR_VERSION:-$GIT_TAG}" + ;; + + "${BUILD_SCENARIO_STAGING}") + COMMIT_SHA_SHORT=$(git rev-parse --short=8 HEAD) + REGISTRY="${REGISTRY:-$STAGING_REGISTRY}" + OPERATOR_VERSION="${OPERATOR_VERSION:-$COMMIT_SHA_SHORT}" + ;; + + "${BUILD_SCENARIO_PATCH}") + REGISTRY="${REGISTRY:-$PATCH_REGISTRY}" + OPERATOR_VERSION="${OPERATOR_VERSION:-$VERSION_ID}" + ;; + + *) + echo "Unknown BUILD_SCENARIO=${BUILD_SCENARIO}, exiting" + exit 1 + ;; +esac +echo "Setting REGISTRY=${REGISTRY} and OPERATOR_VERSION=${OPERATOR_VERSION} for ${BUILD_SCENARIO} build scenario" +export OPERATOR_VERSION="${OPERATOR_VERSION}" +export REGISTRY="${REGISTRY}" diff --git a/scripts/dev/contexts/gha-private-context b/scripts/dev/contexts/gha-private-context index c5a189fb8..61770da79 100644 --- a/scripts/dev/contexts/gha-private-context +++ b/scripts/dev/contexts/gha-private-context @@ -5,4 +5,4 @@ set -Eeou pipefail export NAMESPACE="" export CLUSTER_NAME="" export LOCAL_OPERATOR="false" -export BASE_REPO_URL="" +export REGISTRY="" diff --git a/scripts/dev/contexts/local-defaults-context b/scripts/dev/contexts/local-defaults-context index e0c3b292c..485585f43 100644 --- a/scripts/dev/contexts/local-defaults-context +++ b/scripts/dev/contexts/local-defaults-context @@ -20,33 +20,13 @@ export workdir # The main registry to use for local test runs # - global: /dev is the shared registry # - dedicated: / -# Note: in our root-context we default the image tag to latest, if VERSION_ID is not preset on your machine +# Note: in our root-context we default the image tag to latest, if OPERATOR_VERSION is not preset on your machine # Note2: For quick local development defaults we only use shared images export QUAY_REGISTRY=quay.io/mongodb -export BASE_REPO_URL_SHARED="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" +export BASE_REPO_URL_SHARED="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" # Note: the Operator registry (OPERATOR_REGISTRY variable) is defined in the private context export REGISTRY="${BASE_REPO_URL_SHARED}" -export INIT_IMAGES_REGISTRY=${INIT_IMAGES_REGISTRY:-"${REGISTRY}"} - -export INIT_APPDB_REGISTRY="${BASE_REPO_URL_SHARED}" -export INIT_OPS_MANAGER_REGISTRY=${BASE_REPO_URL_SHARED:-"${QUAY_REGISTRY}"} -export INIT_DATABASE_REGISTRY=${BASE_REPO_URL_SHARED:-"${QUAY_REGISTRY}"} -export INIT_DATABASE_IMAGE_REPOSITORY="${BASE_REPO_URL_SHARED}/mongodb-kubernetes-init-database" -export DATABASE_REGISTRY=${BASE_REPO_URL_SHARED:-"${QUAY_REGISTRY}"} -export OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" -export MONGODB_REPO_URL="${QUAY_REGISTRY}" -export MONGODB_ENTERPRISE_DATABASE_IMAGE="${INIT_IMAGES_REGISTRY}/mongodb-kubernetes-database" -export MDB_AGENT_IMAGE_OPERATOR_VERSION=latest -export MDB_AGENT_IMAGE_REPOSITORY="${BASE_REPO_URL_SHARED}/mongodb-agent" -export AGENT_BASE_REGISTRY=${BASE_REPO_URL_SHARED} -export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:12.0.35.7911-1" - -# these are needed to deploy OM -export INIT_APPDB_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-kubernetes-init-appdb" -export OPS_MANAGER_IMAGE_REPOSITORY="${QUAY_REGISTRY}/mongodb-enterprise-ops-manager-ubi" -export INIT_OPS_MANAGER_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-kubernetes-init-ops-manager" - # Environment variable needed for local development # Used in controllers/operator/mongodbopsmanager_controller.go # The file is created by scripts/dev/prepare_local_e2e_run.sh diff --git a/scripts/dev/contexts/prerelease_kind_code_snippets b/scripts/dev/contexts/prerelease_kind_code_snippets index ee8ad3601..4043785d8 100644 --- a/scripts/dev/contexts/prerelease_kind_code_snippets +++ b/scripts/dev/contexts/prerelease_kind_code_snippets @@ -13,4 +13,6 @@ export CODE_SNIPPETS_FLAVOR=e2e_prerelease export CODE_SNIPPETS_COMMIT_OUTPUT=true export ops_manager_version="cloud_qa" +export ops_manager_version="cloud_qa" + export PRERELEASE_IMAGE_PULLSECRET="${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON}" diff --git a/scripts/dev/contexts/private-context-template b/scripts/dev/contexts/private-context-template index 2a95eaf42..ff49a1ab9 100644 --- a/scripts/dev/contexts/private-context-template +++ b/scripts/dev/contexts/private-context-template @@ -26,7 +26,7 @@ export EVG_HOST_NAME="" # - 268558157000.dkr.ecr.us-east-1.amazonaws.com/ export BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/${USER}" # The operator image should -in general- be pulled from the user repository -# In certain case, the /dev repo can be used as well (BASE_REPO_URL_SHARED) +# In certain case, the /staging repo can be used as well (BASE_REPO_URL_SHARED) export OPERATOR_REGISTRY="${BASE_REPO_URL}" # Set to true if running operator locally and not in a pod diff --git a/scripts/dev/contexts/root-context b/scripts/dev/contexts/root-context index 300f84e30..714aa15cf 100644 --- a/scripts/dev/contexts/root-context +++ b/scripts/dev/contexts/root-context @@ -39,7 +39,6 @@ export OPERATOR_ENV=${OPERATOR_ENV:-"dev"} AGENT_VERSION="$(jq -r '.agentVersion' "${PROJECT_DIR}/release.json")" export AGENT_VERSION -export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" # Ops Manager export OPS_MANAGER_NAMESPACE="operator-testing-50-current" @@ -47,19 +46,50 @@ export OPS_MANAGER_NAMESPACE="operator-testing-50-current" # Moved from the old set_env_context.sh export LOCAL_RUN=true -# version_id is similar to version_id from Evergreen. Used to differentiate different builds. Can be constant -# for local run -version_id=${version_id:-"latest"} +OPERATOR_VERSION=${OPERATOR_VERSION:-"latest"} +READINESS_PROBE_VERSION=${READINESS_PROBE_VERSION:-"latest"} +VERSION_UPGRADE_HOOK_VERSION=${VERSION_UPGRADE_HOOK_VERSION:-"latest"} if [[ "${OVERRIDE_VERSION_ID:-}" != "" ]]; then - version_id="${OVERRIDE_VERSION_ID}" + OPERATOR_VERSION="${OVERRIDE_VERSION_ID}" + READINESS_PROBE_VERSION="${OVERRIDE_VERSION_ID}" + VERSION_UPGRADE_HOOK_VERSION="${OVERRIDE_VERSION_ID}" fi -export version_id -export VERSION_ID="${version_id}" -export INIT_APPDB_VERSION="${VERSION_ID}" -export INIT_DATABASE_VERSION="${VERSION_ID}" -export INIT_OPS_MANAGER_VERSION="${VERSION_ID}" -export DATABASE_VERSION="${VERSION_ID}" +export OPERATOR_VERSION="${OPERATOR_VERSION}" +export INIT_APPDB_VERSION="${INIT_APPDB_VERSION:-${OPERATOR_VERSION}}" +export INIT_DATABASE_VERSION="${INIT_DATABASE_VERSION:-${OPERATOR_VERSION}}" +export INIT_OPS_MANAGER_VERSION="${INIT_OPS_MANAGER_VERSION:-${OPERATOR_VERSION}}" +export DATABASE_VERSION="${DATABASE_VERSION:-${OPERATOR_VERSION}}" +export READINESS_PROBE_VERSION +export VERSION_UPGRADE_HOOK_VERSION + +### Registries and images +export QUAY_REGISTRY="quay.io/mongodb" +export OPERATOR_REGISTRY=${OPERATOR_REGISTRY:-${REGISTRY}} +export INIT_IMAGES_REGISTRY=${INIT_IMAGES_REGISTRY:-${REGISTRY}} + +# Specific registries for different images, can be overridden if needed +export INIT_APPDB_REGISTRY=${INIT_APPDB_REGISTRY:-${INIT_IMAGES_REGISTRY}} +export INIT_APPDB_IMAGE_REPOSITORY="${INIT_APPDB_REGISTRY}/mongodb-kubernetes-init-appdb" +export INIT_OPS_MANAGER_REGISTRY=${INIT_OPS_MANAGER_REGISTRY:-${INIT_IMAGES_REGISTRY}} +export INIT_OPS_MANAGER_IMAGE_REPOSITORY="${INIT_OPS_MANAGER_REGISTRY}/mongodb-kubernetes-init-ops-manager" +export INIT_DATABASE_REGISTRY=${INIT_DATABASE_REGISTRY:-${INIT_IMAGES_REGISTRY}} +export INIT_DATABASE_IMAGE_REPOSITORY="${INIT_DATABASE_REGISTRY}/mongodb-kubernetes-init-database" +export DATABASE_REGISTRY=${DATABASE_REGISTRY:-${INIT_IMAGES_REGISTRY}} +export MONGODB_ENTERPRISE_DATABASE_IMAGE="${DATABASE_REGISTRY}/mongodb-kubernetes-database" +export OPS_MANAGER_REGISTRY=${OPS_MANAGER_REGISTRY:-${REGISTRY}} +export OPS_MANAGER_IMAGE_REPOSITORY="${OPS_MANAGER_REGISTRY}/mongodb-enterprise-ops-manager-ubi" +export MDB_AGENT_REGISTRY=${MDB_AGENT_REGISTRY:-${REGISTRY}} +export MDB_AGENT_IMAGE_REPOSITORY="${MDB_AGENT_REGISTRY}/mongodb-agent" +export MONGODB_REPO_URL="${MONGODB_REPO_URL:-${QUAY_REGISTRY}}" +export MEKO_TESTS_REGISTRY=${MEKO_TESTS_REGISTRY:-${REGISTRY}} +export MEKO_TESTS_IMAGE_REPOSITORY="${MEKO_TESTS_REGISTRY}/mongodb-kubernetes-tests" +export READINESS_PROBE_REGISTRY=${READINESS_PROBE_REGISTRY:-${REGISTRY}} +export VERSION_UPGRADE_HOOK_REGISTRY=${VERSION_UPGRADE_HOOK_REGISTRY:-${REGISTRY}} + +export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" +export READINESS_PROBE_IMAGE="${READINESS_PROBE_REGISTRY}/mongodb-kubernetes-readinessprobe:${READINESS_PROBE_VERSION}" +export VERSION_UPGRADE_HOOK_IMAGE="${VERSION_UPGRADE_HOOK_REGISTRY}/mongodb-kubernetes-operator-version-upgrade-post-start-hook:${VERSION_UPGRADE_HOOK_VERSION}" export KUBE_ENVIRONMENT_NAME=kind @@ -75,8 +105,6 @@ if [[ "$(uname)" == "Linux" ]]; then export GOROOT=/opt/golang/go1.24 fi -export MONGODB_REPO_URL="quay.io/mongodb" - export SIGNING_PUBLIC_KEY_URL="https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem" export CLUSTER_DOMAIN="cluster.local" @@ -96,23 +124,13 @@ export OLM_VERSION=v0.31.0 # Python version we use locally and in CI export PYTHON_VERSION=3.13.7 - ## MCO - export MDB_COMMUNITY_IMAGE=mongodb-community-server export MDB_COMMUNITY_REPO_URL=quay.io/mongodb export MDB_COMMUNITY_AGENT_IMAGE=${AGENT_IMAGE} export MDB_COMMUNITY_IMAGE_TYPE=ubi8 -export READINESS_PROBE_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-readinessprobe:${version_id}" -export VERSION_UPGRADE_HOOK_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-operator-version-upgrade-post-start-hook:${version_id}" - # Community Search -export MDB_COMMUNITY_IMAGE=mongodb-community-server -export MDB_COMMUNITY_REPO_URL=quay.io/mongodb -export MDB_COMMUNITY_AGENT_IMAGE=${AGENT_IMAGE} -export MDB_COMMUNITY_IMAGE_TYPE=ubi8 - MDB_SEARCH_VERSION="$(jq -r '.search.version' "${PROJECT_DIR}/release.json")" export MDB_SEARCH_VERSION diff --git a/scripts/dev/contexts/variables/e2e_smoke_base b/scripts/dev/contexts/variables/e2e_smoke_base new file mode 100644 index 000000000..3111a1194 --- /dev/null +++ b/scripts/dev/contexts/variables/e2e_smoke_base @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +if [[ "${BUILD_SCENARIO}" == "release" ]]; then + export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" +fi diff --git a/scripts/dev/contexts/variables/om60 b/scripts/dev/contexts/variables/om60 index 82927dd87..a15366b5e 100644 --- a/scripts/dev/contexts/variables/om60 +++ b/scripts/dev/contexts/variables/om60 @@ -12,4 +12,4 @@ export CUSTOM_MDB_PREV_VERSION=5.0.7 export CUSTOM_APPDB_VERSION=6.0.21-ent export TEST_MODE=opsmanager -export OPS_MANAGER_REGISTRY="${BASE_REPO_URL}" +export OPS_MANAGER_REGISTRY="${REGISTRY}" diff --git a/scripts/dev/contexts/variables/om60_image b/scripts/dev/contexts/variables/om60_image index 6a2488d51..53f87e821 100644 --- a/scripts/dev/contexts/variables/om60_image +++ b/scripts/dev/contexts/variables/om60_image @@ -2,5 +2,6 @@ set -Eeou pipefail -om_version=$(grep -E "^\s*-\s*&ops_manager_60_latest\s+(\S+)\s+#" < "${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') -export om_version + +OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_60_latest\s+(\S+)\s+#" < "${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export OM_VERSION diff --git a/scripts/dev/contexts/variables/om70 b/scripts/dev/contexts/variables/om70 index b4eec8d23..f2b94f280 100644 --- a/scripts/dev/contexts/variables/om70 +++ b/scripts/dev/contexts/variables/om70 @@ -15,4 +15,4 @@ export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" export CUSTOM_APPDB_VERSION=7.0.18-ent export TEST_MODE=opsmanager -export OPS_MANAGER_REGISTRY="${BASE_REPO_URL}" +export OPS_MANAGER_REGISTRY="${REGISTRY}" diff --git a/scripts/dev/contexts/variables/om70_image b/scripts/dev/contexts/variables/om70_image index aec42bb8a..3c93bf98f 100644 --- a/scripts/dev/contexts/variables/om70_image +++ b/scripts/dev/contexts/variables/om70_image @@ -2,5 +2,5 @@ set -Eeou pipefail -om_version=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" < "${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') -export om_version +OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" < "${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export OM_VERSION diff --git a/scripts/dev/contexts/variables/om80 b/scripts/dev/contexts/variables/om80 index abab46810..7053a52c2 100644 --- a/scripts/dev/contexts/variables/om80 +++ b/scripts/dev/contexts/variables/om80 @@ -15,4 +15,4 @@ export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" export CUSTOM_APPDB_VERSION=8.0.6-ent export TEST_MODE=opsmanager -export OPS_MANAGER_REGISTRY="${BASE_REPO_URL}" +export OPS_MANAGER_REGISTRY="${REGISTRY}" diff --git a/scripts/dev/contexts/variables/om80_image b/scripts/dev/contexts/variables/om80_image index 2931aa8f6..6ada1dcb8 100644 --- a/scripts/dev/contexts/variables/om80_image +++ b/scripts/dev/contexts/variables/om80_image @@ -2,7 +2,5 @@ set -Eeou pipefail -om_version=$(grep -E "^\s*-\s*&ops_manager_80_latest\s+(\S+)\s+#" < "${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') -export om_version - -export om_download_url +OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_80_latest\s+(\S+)\s+#" < "${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export OM_VERSION diff --git a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml index 1071e6770..2a3e37ff3 100644 --- a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml +++ b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml @@ -115,7 +115,7 @@ spec: - name: POD_NAME value: 'mongodb-enterprise-operator-testing-pod' - name: VERSION_ID - value: {{ .Values.tag }} + value: {{ .Values.versionId }} - name: REGISTRY value: {{ .Values.registry }} {{- if .Values.multiCluster.memberClusters }} @@ -190,7 +190,7 @@ spec: value: {{ .Values.cognito_workload_url }} - name: cognito_workload_user_id value: {{ .Values.cognito_workload_user_id }} - image: {{ .Values.repo }}/mongodb-kubernetes-tests:{{ .Values.tag }} + image: "{{ .Values.mekoTestsRegistry }}/mongodb-kubernetes-tests:{{ .Values.mekoTestsVersion }}" # Options to pytest command should go in the pytest.ini file. command: ["pytest"] {{ if .Values.otel_endpoint }} diff --git a/scripts/evergreen/deployments/test-app/values.yaml b/scripts/evergreen/deployments/test-app/values.yaml index fe171c27a..13aba3df8 100644 --- a/scripts/evergreen/deployments/test-app/values.yaml +++ b/scripts/evergreen/deployments/test-app/values.yaml @@ -7,7 +7,9 @@ managedSecurityContext: false apiKey: omApiKey orgId: "" projectId: omProjectId -tag: +mekoTestsRegistry: +mekoTestsVersion: +versionId: taskName: ${TASK_NAME} pytest: diff --git a/scripts/evergreen/e2e/build_tests_image_ibm.sh b/scripts/evergreen/e2e/build_tests_image_ibm.sh index 6acb880e4..d82ae9661 100755 --- a/scripts/evergreen/e2e/build_tests_image_ibm.sh +++ b/scripts/evergreen/e2e/build_tests_image_ibm.sh @@ -8,7 +8,7 @@ cp release.json docker/mongodb-kubernetes-tests/release.json cp requirements.txt docker/mongodb-kubernetes-tests/requirements.txt cp -rf helm_chart docker/mongodb-kubernetes-tests/helm_chart -echo "Building mongodb-kubernetes-tests image with tag: ${BASE_REPO_URL}/mongodb-kubernetes-tests:${version_id}-$(arch)" +echo "Building mongodb-kubernetes-tests image with tag: ${REGISTRY}/mongodb-kubernetes-tests:${OPERATOR_VERSION}-$(arch)" cd docker/mongodb-kubernetes-tests || exit -sudo podman buildx build --progress plain . -f Dockerfile -t "${BASE_REPO_URL}/mongodb-kubernetes-tests:${version_id}-$(arch)" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -sudo podman push --authfile="/root/.config/containers/auth.json" "${BASE_REPO_URL}/mongodb-kubernetes-tests:${version_id}-$(arch)" +sudo podman buildx build --progress plain . -f Dockerfile -t "${REGISTRY}/mongodb-kubernetes-tests:${OPERATOR_VERSION}-$(arch)" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" +sudo podman push --authfile="/root/.config/containers/auth.json" "${REGISTRY}/mongodb-kubernetes-tests:${OPERATOR_VERSION}-$(arch)" diff --git a/scripts/evergreen/e2e/single_e2e.sh b/scripts/evergreen/e2e/single_e2e.sh index f14cf9ab9..054470eef 100755 --- a/scripts/evergreen/e2e/single_e2e.sh +++ b/scripts/evergreen/e2e/single_e2e.sh @@ -22,23 +22,20 @@ deploy_test_app() { local context=${1} local helm_template_file helm_template_file=$(mktemp) - tag="${VERSION_ID:-latest}" - if [[ "${OVERRIDE_VERSION_ID:-}" != "" ]]; then - tag="${OVERRIDE_VERSION_ID}" - fi + meko_tests_version="${OPERATOR_VERSION}" local arch arch=$(uname -m) case "${arch}" in ppc64le) - tag="${tag}-ppc64le" + meko_tests_version="${meko_tests_version}-ppc64le" ;; s390x) - tag="${tag}-s390x" + meko_tests_version="${meko_tests_version}-s390x" ;; *) - echo "Not IBM host, using default tag" + echo "Not IBM host, using default meko_tests_version" ;; esac @@ -51,10 +48,11 @@ deploy_test_app() { # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager helm_params=( "--set" "taskId=${task_id:-'not-specified'}" - "--set" "repo=${BASE_REPO_URL:=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev}" "--set" "namespace=${NAMESPACE}" "--set" "taskName=${task_name}" - "--set" "tag=${tag}" + "--set" "mekoTestsRegistry=${MEKO_TESTS_REGISTRY}" + "--set" "mekoTestsVersion=${meko_tests_version}" + "--set" "versionId=${VERSION_ID}" "--set" "aws.accessKey=${AWS_ACCESS_KEY_ID}" "--set" "aws.secretAccessKey=${AWS_SECRET_ACCESS_KEY}" "--set" "skipExecution=${SKIP_EXECUTION:-'false'}" @@ -64,7 +62,7 @@ deploy_test_app() { "--set" "orgId=${OM_ORGID:-}" "--set" "imagePullSecrets=image-registries-secret" "--set" "managedSecurityContext=${MANAGED_SECURITY_CONTEXT:-false}" - "--set" "registry=${REGISTRY:-${BASE_REPO_URL}/${IMAGE_TYPE}}" + "--set" "registry=${REGISTRY}" "--set" "mdbDefaultArchitecture=${MDB_DEFAULT_ARCHITECTURE:-'non-static'}" "--set" "mdbImageType=${MDB_IMAGE_TYPE:-'ubi8'}" "--set" "clusterDomain=${CLUSTER_DOMAIN:-'cluster.local'}" diff --git a/scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh b/scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh index 4dea15c6c..7b5525794 100755 --- a/scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh +++ b/scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh @@ -20,8 +20,9 @@ set -Eeou pipefail # so this channel has an entry without the replaces field. # # Required env vars: -# - BASE_REPO_URL (or REGISTRY for EVG run) +# - REGISTRY (registry to push bundles to) # - VERSION_ID (set in EVG as patch-id) +# - OPERATOR_VERSION (version of the current operator being built) # for get_operator_helm_values function source scripts/funcs/operator_deployment @@ -34,12 +35,6 @@ increment_version() { echo "${major}.${minor}.${patch}" } -if [[ "${REGISTRY:-""}" != "" ]]; then - base_repo_url="${REGISTRY}" -else - base_repo_url="${BASE_REPO_URL:-"268558157000.dkr.ecr.us-east-1.amazonaws.com/dev"}" -fi - # Generates helm charts the same way they are generates part of pre-commit hook. # We also provide helm values override the same way it's done when installing the operator helm chart in e2e tests. generate_helm_charts() { @@ -202,16 +197,16 @@ meko_package_name="mongodb-enterprise" certified_operators_repo="https://github.com/redhat-openshift-ecosystem/certified-operators.git" current_operator_version_from_release_json=$(jq -r .mongodbOperator BuildInfo: +def load_build_info(scenario: BuildScenario) -> BuildInfo: f""" Load build information based on the specified scenario. :param scenario: BuildScenario enum value indicating the build scenario (e.g. "development", "patch", "staging", "release"). "development" scenario will return build info for "patch" scenario. - :param repository_path: Path to the Git repository. Default is the current directory `{DEFAULT_REPOSITORY_PATH}`. - :param changelog_sub_path: Path to the changelog directory relative to the repository root. Default is '{DEFAULT_CHANGELOG_PATH}'. - :param initial_commit_sha: SHA of the initial commit to start from if no previous version tag is found. If not provided, it will be determined based on `{RELEASE_INITIAL_VERSION_ENV_VAR} environment variable. - :param initial_version: Initial version to use if no previous version tag is found. If not provided, it will be determined based on `{RELEASE_INITIAL_VERSION_ENV_VAR}` environment variable. :return: BuildInfo object containing images, binaries, and helm charts information for specified scenario. """ - if not initial_commit_sha: - initial_commit_sha = get_initial_commit_sha() - if not initial_version: - initial_version = get_initial_version() - - version = scenario.get_version(repository_path, changelog_sub_path, initial_commit_sha, initial_version) - # For manual_release, version can be None and will be set by image-specific logic - with open("build_info.json", "r") as f: build_info = json.load(f) @@ -99,15 +71,9 @@ def load_build_info( # If no scenario_data is available for the scenario, skip this image continue - # Only update the image_version if it is not already set in the build_info.json file - image_version = scenario_data.get("version") - if not image_version and version is not None: - image_version = version - images[name] = ImageInfo( repositories=scenario_data["repositories"], platforms=scenario_data["platforms"], - version=image_version, dockerfile_path=data["dockerfile-path"], sign=scenario_data.get("sign", False), latest_tag=scenario_data.get("latest-tag", False), @@ -124,7 +90,6 @@ def load_build_info( binaries[name] = BinaryInfo( s3_store=scenario_data["s3-store"], platforms=scenario_data["platforms"], - version=version, sign=scenario_data.get("sign", False), ) @@ -137,7 +102,6 @@ def load_build_info( helm_charts[name] = HelmChartInfo( repositories=scenario_data["repositories"], - version=version, sign=scenario_data.get("sign", False), ) diff --git a/scripts/release/build/build_scenario.py b/scripts/release/build/build_scenario.py index a8cda4a72..5fdaf3344 100644 --- a/scripts/release/build/build_scenario.py +++ b/scripts/release/build/build_scenario.py @@ -1,14 +1,5 @@ from enum import StrEnum -from git import Repo - -from lib.base_logger import logger -from scripts.release.constants import triggered_by_git_tag, is_evg_patch, is_running_in_evg, get_version_id, \ - get_github_commit -from scripts.release.version import calculate_next_version - -COMMIT_SHA_LENGTH = 8 - class BuildScenario(StrEnum): RELEASE = "release" # Official release triggered by a git tag @@ -16,54 +7,3 @@ class BuildScenario(StrEnum): PATCH = "patch" # CI build for a patch/pull request STAGING = "staging" # CI build from a merge to the master DEVELOPMENT = "development" # Local build on a developer machine - - @classmethod - def infer_scenario_from_environment(cls) -> "BuildScenario": - """Infer the build scenario from environment variables.""" - git_tag = triggered_by_git_tag() - is_patch = is_evg_patch() - is_evg = is_running_in_evg() - patch_id = get_version_id() - commit_sha = get_github_commit() - - if git_tag: - # Release scenario and the git tag will be used for promotion process only - scenario = BuildScenario.RELEASE - logger.info(f"Build scenario: {scenario} (git_tag: {git_tag})") - elif is_patch or is_evg: - scenario = BuildScenario.PATCH - logger.info(f"Build scenario: {scenario} (patch_id: {patch_id})") - # TODO: Uncomment the following lines when starting to work on staging builds - # elif is_evg: - # scenario = BuildScenario.STAGING - # logger.info(f"Build scenario: {scenario} (commit_sha: {commit_sha[:COMMIT_SHA_LENGTH]})") - else: - scenario = BuildScenario.DEVELOPMENT - logger.info(f"Build scenario: {scenario}") - - return scenario - - def get_version(self, repository_path: str, changelog_sub_path: str, initial_commit_sha: str = None, - initial_version: str = None) -> str: - repo = Repo(repository_path) - - match self: - case BuildScenario.DEVELOPMENT: - # When working locally, "version_id" env variable is defined in the generated context file. It is "latest" by - # default, and can be overridden with OVERRIDE_VERSION_ID - return "latest" - case BuildScenario.PATCH: - patch_id = get_version_id() - if not patch_id: - raise ValueError(f"version_id environment variable is not set for `{self}` build scenario") - return patch_id - case BuildScenario.STAGING: - return repo.head.object.hexsha[:COMMIT_SHA_LENGTH] - case BuildScenario.RELEASE: - return calculate_next_version(repo, changelog_sub_path, initial_commit_sha, initial_version) - case BuildScenario.MANUAL_RELEASE: - # For manual releases, version must be provided externally (e.g., for ops-manager via om_version env var, - # for agent via release.json). Return None to indicate version will be set by image-specific logic. - return None - - raise ValueError(f"Unknown build scenario: {self}") diff --git a/scripts/release/constants.py b/scripts/release/constants.py index dd0e462c2..7fe0d4de4 100644 --- a/scripts/release/constants.py +++ b/scripts/release/constants.py @@ -1,62 +1,3 @@ -import os - -RELEASE_INITIAL_VERSION_ENV_VAR = "RELEASE_INITIAL_VERSION" -RELEASE_INITIAL_COMMIT_SHA_ENV_VAR = "RELEASE_INITIAL_COMMIT_SHA" - DEFAULT_RELEASE_INITIAL_VERSION = "1.0.0" DEFAULT_CHANGELOG_PATH = "changelog/" DEFAULT_REPOSITORY_PATH = "." - - -def get_initial_version() -> str | None: - return os.getenv(RELEASE_INITIAL_VERSION_ENV_VAR) - - -def get_initial_commit_sha() -> str | None: - return os.getenv(RELEASE_INITIAL_COMMIT_SHA_ENV_VAR) - - -def triggered_by_git_tag() -> str | None: - """ - "triggered_by_git_tag" is the name of the tag that triggered this version, if applicable. It will be None for - all patches except when tagging a commit on GitHub - :return: tag name if the build was triggered by a git tag, otherwise None. - """ - return os.getenv("triggered_by_git_tag") - - -def is_evg_patch() -> bool: - """ - A patch build is a version not triggered by a commit to a repository. "is_patch" is passed automatically by Evergreen. - It either runs tasks on a base commit plus some diff if submitted by the CLI or on a git branch if created by - a GitHub pull request. - :return: "true" if the running task is in a patch build, otherwise "false". - """ - return os.getenv("is_patch", "false").lower() == "true" - - -def is_running_in_evg() -> bool: - """ - "RUNNING_IN_EVG" is set by us in evg-private-context - :return: "true" if the script is running in Evergreen, otherwise "false". - """ - return os.getenv("RUNNING_IN_EVG", "false").lower() == "true" - - -def get_version_id() -> str | None: - """ - Get the version ID from the environment variable. This is typically used for patch builds in the Evergreen CI system. - It is generated automatically for each task run. For example: `6899b7e35bfaee00077db986` for a manual/PR patch, - or `mongodb_kubernetes_5c5a3accb47bb411682b8c67f225b61f7ad5a619` for a master merge - :return: version_id (patch ID) or None if not set - """ - return os.getenv("version_id") - - -def get_github_commit() -> str | None: - """ - ${github_commit} is the commit hash of the commit that triggered the patch run. - For non pull-request patches, it will be the same as ${revision}. - :return: github_commit or None if not set - """ - return os.getenv("github_commit") diff --git a/scripts/release/pipeline_main.py b/scripts/release/pipeline.py similarity index 88% rename from scripts/release/pipeline_main.py rename to scripts/release/pipeline.py index 010620284..254122283 100644 --- a/scripts/release/pipeline_main.py +++ b/scripts/release/pipeline.py @@ -98,7 +98,7 @@ def build_image(image_name: str, build_configuration: ImageBuildConfiguration): def image_build_config_from_args(args) -> ImageBuildConfiguration: image = args.image - build_scenario = get_scenario_from_arg(args.build_scenario) or BuildScenario.infer_scenario_from_environment() + build_scenario = get_scenario_from_arg(args.build_scenario) build_info = load_build_info(build_scenario) logger.info(f"image is {image}") logger.info(f"images are {build_info.images}") @@ -108,7 +108,7 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration: raise ValueError(f"Image '{image}' is not defined in the build info for scenario '{build_scenario}'") # Resolve final values with overrides - version = args.version or image_build_info.version + version = args.version latest_tag = image_build_info.latest_tag olm_tag = image_build_info.olm_tag if args.registry: @@ -119,9 +119,9 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration: sign = args.sign or image_build_info.sign dockerfile_path = image_build_info.dockerfile_path - # Validate version - only ops-manager and agent can have None version as the versions are managed by the agent - # and om methods themselves, which are externally retrieved - om_version env var and release.json respectively - if version is None and image not in ["ops-manager", "agent"]: + # Validate version - only agent can have None version as the versions are managed by the agent + # which are externally retrieved from release.json + if version is None and image != "agent": raise ValueError(f"Version cannot be empty for {image}.") return ImageBuildConfiguration( @@ -141,9 +141,6 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration: def get_scenario_from_arg(args_scenario: str) -> BuildScenario | None: - if not args_scenario: - return None - try: return BuildScenario(args_scenario) except ValueError as e: @@ -200,8 +197,7 @@ def main(): parser = argparse.ArgumentParser( description="""Builder tool for container images. It allows to push and sign images with multiple architectures using Docker Buildx. -By default build information is read from 'build_info.json' file in the project root directory based on the build scenario. -It can be inferred from the environment variables or overridden with the '--build-scenario' option.""", +By default build information is read from 'build_info.json' file in the project root directory based on the build scenario.""", ) parser.add_argument( "image", @@ -216,10 +212,11 @@ def main(): "--build-scenario", metavar="", action="store", + required=True, type=str, choices=supported_scenarios, - help=f"""Override the build scenario instead of inferring from environment. Options: {", ".join(supported_scenarios)}. -Default is to infer from environment variables. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", + help=f"""Build scenario when reading configuration from 'build_info.json'. +Options: {", ".join(supported_scenarios)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", ) parser.add_argument( "-p", @@ -235,7 +232,7 @@ def main(): metavar="", action="store", type=str, - help="Override the version/tag instead of resolving from build scenario. Default is to infer from environment variables based on the selected scenario.", + help="Version to use when building container image. Required for all images except for agent where we read it from release.json", ) parser.add_argument( "-r", diff --git a/scripts/release/pipeline.sh b/scripts/release/pipeline.sh new file mode 100755 index 000000000..703bbcac1 --- /dev/null +++ b/scripts/release/pipeline.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +scripts/dev/run_python.sh scripts/release/pipeline.py "${IMAGE_NAME}" --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}" diff --git a/scripts/release/pipeline_agent.sh b/scripts/release/pipeline_agent.sh new file mode 100755 index 000000000..962eceb2a --- /dev/null +++ b/scripts/release/pipeline_agent.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +if [[ "${FLAGS:-}" = "" ]]; then + FLAGS="--parallel" +else + FLAGS="--parallel ${FLAGS}" +fi + +# shellcheck disable=SC2086 +scripts/dev/run_python.sh scripts/release/pipeline.py agent \ + --build-scenario "${BUILD_SCENARIO}" \ + ${FLAGS} diff --git a/scripts/release/pipeline_migrate_agent.sh b/scripts/release/pipeline_migrate_agent.sh deleted file mode 100755 index 2433457bc..000000000 --- a/scripts/release/pipeline_migrate_agent.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -Eeou pipefail - -scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel agent \ - --all-agents \ - --build-scenario manual_release \ - -r quay.io/mongodb/mongodb-agent diff --git a/scripts/release/pipeline_ops_manager.sh b/scripts/release/pipeline_ops_manager.sh new file mode 100755 index 000000000..2c095a28a --- /dev/null +++ b/scripts/release/pipeline_ops_manager.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +scripts/dev/run_python.sh scripts/release/pipeline.py ops-manager \ + --build-scenario "${BUILD_SCENARIO}" \ + --version "${OM_VERSION}" diff --git a/scripts/release/pipeline_readiness_probe.sh b/scripts/release/pipeline_readiness_probe.sh new file mode 100755 index 000000000..a3aff54fa --- /dev/null +++ b/scripts/release/pipeline_readiness_probe.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +scripts/dev/run_python.sh scripts/release/pipeline.py readiness-probe \ + --build-scenario "${BUILD_SCENARIO}" \ + --version "${READINESS_PROBE_VERSION}" diff --git a/scripts/release/pipeline_version_upgrade_hook.sh b/scripts/release/pipeline_version_upgrade_hook.sh new file mode 100755 index 000000000..a8f135604 --- /dev/null +++ b/scripts/release/pipeline_version_upgrade_hook.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +scripts/dev/run_python.sh scripts/release/pipeline.py upgrade-hook \ + --build-scenario "${BUILD_SCENARIO}" \ + --version "${VERSION_UPGRADE_HOOK_VERSION}" diff --git a/scripts/release/release_info.py b/scripts/release/release_info.py index 487d5a66a..c5f1b2e24 100644 --- a/scripts/release/release_info.py +++ b/scripts/release/release_info.py @@ -30,17 +30,12 @@ UPGRADE_HOOK_IMAGE, ] +# TODO: this is dummy version, to be replaced with actual versioning logic https://docs.google.com/document/d/1eJ8iKsI0libbpcJakGjxcPfbrTn8lmcZDbQH1UqMR_g/edit?tab=t.45ig7xr3e3w4#bookmark=id.748ik8snxcyl +DUMMY_VERSION = "dummy_version" -def create_release_info_json( - repository_path: str, changelog_sub_path: str, initial_commit_sha: str = None, initial_version: str = None -) -> str: - build_info = load_build_info( - scenario=BuildScenario.RELEASE, - repository_path=repository_path, - changelog_sub_path=changelog_sub_path, - initial_commit_sha=initial_commit_sha, - initial_version=initial_version, - ) + +def create_release_info_json() -> str: + build_info = load_build_info(scenario=BuildScenario.RELEASE) release_info_json = convert_to_release_info_json(build_info) @@ -60,19 +55,19 @@ def convert_to_release_info_json(build_info: BuildInfo) -> dict: output["images"][name] = { "repositories": image.repositories, "platforms": image.platforms, - "version": image.version, + "version": DUMMY_VERSION, } for name, binary in build_info.binaries.items(): output["binaries"][name] = { "platforms": binary.platforms, - "version": binary.version, + "version": DUMMY_VERSION, } for name, chart in build_info.helm_charts.items(): output["helm-charts"][name] = { "repositories": chart.repositories, - "version": chart.version, + "version": DUMMY_VERSION, } return output @@ -83,53 +78,9 @@ def convert_to_release_info_json(build_info: BuildInfo) -> dict: description="Create relevant release artifacts information in JSON format.", formatter_class=argparse.RawTextHelpFormatter, ) - parser.add_argument( - "-p", - "--path", - default=DEFAULT_REPOSITORY_PATH, - metavar="", - action="store", - type=pathlib.Path, - help="Path to the Git repository. Default is the current directory '.'", - ) - parser.add_argument( - "-c", - "--changelog-path", - default=DEFAULT_CHANGELOG_PATH, - metavar="", - action="store", - type=str, - help=f"Path to the changelog directory relative to a current working directory. Default is '{DEFAULT_CHANGELOG_PATH}'", - ) - parser.add_argument( - "-s", - "--initial-commit-sha", - metavar="", - action="store", - type=str, - help="SHA of the initial commit to start from if no previous version tag is found.", - ) - parser.add_argument( - "-v", - "--initial-version", - default=DEFAULT_RELEASE_INITIAL_VERSION, - metavar="", - action="store", - type=str, - help=f"Version to use if no previous version tag is found. Default is '{DEFAULT_RELEASE_INITIAL_VERSION}'", - ) - parser.add_argument( - "--output", - "-o", - metavar="", - type=pathlib.Path, - help="Path to save the release information file. If not provided, prints to stdout.", - ) args = parser.parse_args() - release_info = create_release_info_json( - args.path, args.changelog_path, args.initial_commit_sha, args.initial_version - ) + release_info = create_release_info_json() if args.output is not None: with open(args.output, "w") as file: diff --git a/scripts/release/retag_images_to_staging.sh b/scripts/release/retag_images_to_staging.sh new file mode 100755 index 000000000..e9f4549c3 --- /dev/null +++ b/scripts/release/retag_images_to_staging.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# SINGLE SHOT SCRIPT, NOT USED IN CI/CD PIPELINES +# Utility used to retag and push container images from one repo to another +# Useful for migrating images between different repositories (e.g. dev -> staging) + + +image_exists() { + local image_tag="$1" + if docker manifest inspect "${image_tag}" > /dev/null 2>&1; then + return 0 # Image exists + else + return 1 # Image doesn't exist + fi +} + +get_arch_digest() { + local image="$1" + local arch="$2" + local manifest_json + manifest_json=$(docker buildx imagetools inspect --raw "${image}" 2>/dev/null || echo '{}') + + local media_type + media_type=$(echo "${manifest_json}" | jq -r '.mediaType // empty') + + if [[ "${media_type}" == *"manifest.list"* ]]; then + # this is a multi-arch manifest + local arch_digest + arch_digest=$(echo "${manifest_json}" | jq -r ".manifests[] | select(.platform.architecture == \"${arch}\" and .platform.os == \"linux\") | .digest") + + if [[ -n "${arch_digest}" && "${arch_digest}" != "null" ]]; then + echo "${arch_digest}" + return 0 + fi + elif [[ "${arch}" == "amd64" ]]; then + # otherwise it must be a single-arch (image) manifest, so we return it only if we ask for amd64 + local arch_digest + arch_digest="sha256:$(echo -n "${manifest_json}" | shasum --algorithm 256 | head -c 64)" + echo "${arch_digest}" + fi + + echo "" + return 0 +} + +process_image() { + local source_image="$1" + local target_image="$2" + + echo " Processing ${source_image}..." + + local digest_arm64 + local digest_amd64 + digest_arm64=$(get_arch_digest "${source_image}" arm64) + digest_amd64=$(get_arch_digest "${source_image}" amd64) + + if [[ -n "${digest_amd64}" ]]; then + docker pull "${source_image}@${digest_amd64}" + docker tag "${source_image}@${digest_amd64}" "${target_image}-amd64" + docker push "${target_image}-amd64" + fi + + if [[ -n "${digest_arm64}" ]]; then + docker pull "${source_image}@${digest_arm64}" + docker tag "${source_image}@${digest_arm64}" "${target_image}-arm64" + docker push "${target_image}-arm64" + fi + + docker manifest create "${target_image}" ${digest_amd64:+--amend ${target_image}-amd64} ${digest_arm64:+--amend ${target_image}-arm64} + docker manifest push "${target_image}" +} + +retag_container_image() { + local image_name="$1" + local version="$2" + local old_repo="$3" + local new_repo="$4" + + local old_tag="${old_repo}/${image_name}:${version}" + local new_tag="${new_repo}/${image_name}:${version}" + + if image_exists "${new_tag}"; then + echo "Image ${new_tag} exists, skipping" + else + process_image "${old_tag}" "${new_tag}" + fi + + return 0 +} + +versions=$(jq -r '.supportedImages."ops-manager".versions[]' release.json) + +# Iterate over each version +for version in ${versions}; do + echo "Retagging ops-manager version: ${version}" + retag_container_image "mongodb-enterprise-ops-manager-ubi" "${version}" "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" +done diff --git a/scripts/release/atomic_pipeline_test.py b/scripts/release/tests/atomic_pipeline_test.py similarity index 100% rename from scripts/release/atomic_pipeline_test.py rename to scripts/release/tests/atomic_pipeline_test.py diff --git a/scripts/release/tests/build_info_test.py b/scripts/release/tests/build_info_test.py index fd4e6ded0..d5f169f5e 100644 --- a/scripts/release/tests/build_info_test.py +++ b/scripts/release/tests/build_info_test.py @@ -1,7 +1,3 @@ -import os - -from git import Repo - from scripts.release.build.build_info import ( BinaryInfo, BuildInfo, @@ -12,63 +8,53 @@ from scripts.release.build.build_scenario import BuildScenario -def test_load_build_info_development(git_repo: Repo): - version = "latest" - +# TODO: Target testdata (test build_info.json file) and not the production build_info.json file from codebase +def test_load_build_info_development(): expected_build_info = BuildInfo( images={ "operator": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", ), "operator-race": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", ), "init-database": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-database"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-init-database/Dockerfile", ), "init-appdb": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-appdb"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-init-appdb/Dockerfile", ), "init-ops-manager": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-ops-manager"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-init-ops-manager/Dockerfile", ), "database": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-database"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-database/Dockerfile", ), "mco-tests": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-community-tests"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-community-tests/Dockerfile", ), "meko-tests": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", ), "readiness-probe": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-readinessprobe"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-readinessprobe/Dockerfile", ), "upgrade-hook": ImageInfo( @@ -76,19 +62,16 @@ def test_load_build_info_development(git_repo: Repo): "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-operator-version-upgrade-post-start-hook" ], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-upgrade-hook/Dockerfile", ), "agent": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-agent"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-agent/Dockerfile", ), "ops-manager": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-enterprise-ops-manager-ubi"], platforms=["linux/amd64"], - version="om-version-from-release.json", dockerfile_path="docker/mongodb-enterprise-ops-manager/Dockerfile", ), }, @@ -96,80 +79,66 @@ def test_load_build_info_development(git_repo: Repo): "kubectl-mongodb": BinaryInfo( s3_store="s3://kubectl-mongodb/dev", platforms=["linux/amd64"], - version=version, ) }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/helm-charts"], - version=version, ) }, ) - build_info = load_build_info(BuildScenario.DEVELOPMENT, git_repo.working_dir) + build_info = load_build_info(BuildScenario.DEVELOPMENT) assert build_info == expected_build_info -def test_load_build_info_patch(git_repo: Repo): - patch_id = "688364423f9b6c00072b3556" - os.environ["version_id"] = patch_id - +def test_load_build_info_patch(): expected_build_info = BuildInfo( images={ "operator": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", ), "operator-race": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", ), "init-database": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-database"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-init-database/Dockerfile", ), "init-appdb": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-appdb"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-init-appdb/Dockerfile", ), "init-ops-manager": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-ops-manager"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-init-ops-manager/Dockerfile", ), "database": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-database"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-database/Dockerfile", ), "mco-tests": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-community-tests"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-community-tests/Dockerfile", ), "meko-tests": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", ), "readiness-probe": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-readinessprobe"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-readinessprobe/Dockerfile", ), "upgrade-hook": ImageInfo( @@ -177,19 +146,16 @@ def test_load_build_info_patch(git_repo: Repo): "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-operator-version-upgrade-post-start-hook" ], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-kubernetes-upgrade-hook/Dockerfile", ), "agent": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-agent"], platforms=["linux/amd64"], - version=patch_id, dockerfile_path="docker/mongodb-agent/Dockerfile", ), "ops-manager": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-enterprise-ops-manager-ubi"], platforms=["linux/amd64"], - version="om-version-from-release.json", dockerfile_path="docker/mongodb-enterprise-ops-manager/Dockerfile", ), }, @@ -197,47 +163,38 @@ def test_load_build_info_patch(git_repo: Repo): "kubectl-mongodb": BinaryInfo( s3_store="s3://kubectl-mongodb/dev", platforms=["linux/amd64"], - version=patch_id, ) }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/helm-charts"], - version=patch_id, ) }, ) - build_info = load_build_info(BuildScenario.PATCH, git_repo.working_dir) + build_info = load_build_info(BuildScenario.PATCH) assert build_info == expected_build_info -def test_load_build_info_staging(git_repo: Repo): - initial_commit = list(git_repo.iter_commits(reverse=True))[4] - git_repo.git.checkout(initial_commit) - expected_commit_sha = initial_commit.hexsha[:8] - +def test_load_build_info_staging(): expected_build_info = BuildInfo( images={ "operator": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", sign=True, ), "operator-race": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes"], platforms=["linux/amd64"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", sign=True, ), "init-database": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-init-database"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-init-database/Dockerfile", latest_tag=True, sign=True, @@ -245,7 +202,6 @@ def test_load_build_info_staging(git_repo: Repo): "init-appdb": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-init-appdb"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-init-appdb/Dockerfile", latest_tag=True, sign=True, @@ -255,7 +211,6 @@ def test_load_build_info_staging(git_repo: Repo): "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-init-ops-manager" ], platforms=["linux/amd64"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-init-ops-manager/Dockerfile", latest_tag=True, sign=True, @@ -263,7 +218,6 @@ def test_load_build_info_staging(git_repo: Repo): "database": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-database"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-database/Dockerfile", latest_tag=True, sign=True, @@ -271,19 +225,16 @@ def test_load_build_info_staging(git_repo: Repo): "mco-tests": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-community-tests"], platforms=["linux/amd64"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-community-tests/Dockerfile", ), "meko-tests": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], platforms=["linux/arm64", "linux/amd64"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", ), "readiness-probe": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-readinessprobe"], platforms=["linux/arm64", "linux/amd64"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-readinessprobe/Dockerfile", latest_tag=True, sign=True, @@ -293,7 +244,6 @@ def test_load_build_info_staging(git_repo: Repo): "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-operator-version-upgrade-post-start-hook" ], platforms=["linux/arm64", "linux/amd64"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-kubernetes-upgrade-hook/Dockerfile", latest_tag=True, sign=True, @@ -301,7 +251,6 @@ def test_load_build_info_staging(git_repo: Repo): "agent": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-agent"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=expected_commit_sha, dockerfile_path="docker/mongodb-agent/Dockerfile", sign=True, ), @@ -310,7 +259,6 @@ def test_load_build_info_staging(git_repo: Repo): "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-enterprise-ops-manager-ubi" ], platforms=["linux/amd64"], - version="om-version-from-release.json", dockerfile_path="docker/mongodb-enterprise-ops-manager/Dockerfile", sign=True, ), @@ -319,36 +267,28 @@ def test_load_build_info_staging(git_repo: Repo): "kubectl-mongodb": BinaryInfo( s3_store="s3://kubectl-mongodb/staging", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], - version=expected_commit_sha, sign=True, ) }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/helm-charts"], - version=expected_commit_sha, sign=True, ) }, ) - build_info = load_build_info(BuildScenario.STAGING, git_repo.working_dir) + build_info = load_build_info(BuildScenario.STAGING) assert build_info == expected_build_info -def test_load_build_info_release( - git_repo: Repo, readinessprobe_version: str, operator_version_upgrade_post_start_hook_version: str -): - version = "1.2.0" - git_repo.git.checkout(version) - +def test_load_build_info_release(): expected_build_info = BuildInfo( images={ "operator": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=version, dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", olm_tag=True, sign=True, @@ -356,7 +296,6 @@ def test_load_build_info_release( "init-database": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes-init-database"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=version, dockerfile_path="docker/mongodb-kubernetes-init-database/Dockerfile", olm_tag=True, sign=True, @@ -364,7 +303,6 @@ def test_load_build_info_release( "init-appdb": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes-init-appdb"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=version, dockerfile_path="docker/mongodb-kubernetes-init-appdb/Dockerfile", olm_tag=True, sign=True, @@ -372,7 +310,6 @@ def test_load_build_info_release( "init-ops-manager": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes-init-ops-manager"], platforms=["linux/amd64"], - version=version, dockerfile_path="docker/mongodb-kubernetes-init-ops-manager/Dockerfile", olm_tag=True, sign=True, @@ -380,15 +317,18 @@ def test_load_build_info_release( "database": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes-database"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=version, dockerfile_path="docker/mongodb-kubernetes-database/Dockerfile", olm_tag=True, sign=True, ), + "meko-tests": ImageInfo( + repositories=["quay.io/mongodb/mongodb-kubernetes-tests"], + platforms=["linux/amd64"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + ), "readiness-probe": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes-readinessprobe"], platforms=["linux/arm64", "linux/amd64"], - version=readinessprobe_version, dockerfile_path="docker/mongodb-kubernetes-readinessprobe/Dockerfile", olm_tag=True, sign=True, @@ -396,7 +336,6 @@ def test_load_build_info_release( "upgrade-hook": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook"], platforms=["linux/arm64", "linux/amd64"], - version=operator_version_upgrade_post_start_hook_version, dockerfile_path="docker/mongodb-kubernetes-upgrade-hook/Dockerfile", olm_tag=True, sign=True, @@ -406,34 +345,28 @@ def test_load_build_info_release( "kubectl-mongodb": BinaryInfo( s3_store="s3://kubectl-mongodb/prod", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], - version=version, sign=True, ) }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( repositories=["quay.io/mongodb/helm-charts"], - version=version, sign=True, ) }, ) - build_info = load_build_info(BuildScenario.RELEASE, git_repo.working_dir) + build_info = load_build_info(BuildScenario.RELEASE) assert build_info == expected_build_info -def test_load_build_info_manual_release(git_repo: Repo): - version = "1.2.0" - git_repo.git.checkout(version) - +def test_load_build_info_manual_release(): expected_build_info = BuildInfo( images={ "agent": ImageInfo( repositories=["quay.io/mongodb/mongodb-agent-ubi", "quay.io/mongodb/mongodb-agent"], platforms=["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - version=None, # Version is None for manual_release scenario dockerfile_path="docker/mongodb-agent/Dockerfile", olm_tag=True, sign=True, @@ -441,7 +374,6 @@ def test_load_build_info_manual_release(git_repo: Repo): "ops-manager": ImageInfo( repositories=["quay.io/mongodb/mongodb-enterprise-ops-manager-ubi"], platforms=["linux/amd64"], - version=None, # Version is None for manual_release scenario dockerfile_path="docker/mongodb-enterprise-ops-manager/Dockerfile", olm_tag=True, sign=True, @@ -451,6 +383,6 @@ def test_load_build_info_manual_release(git_repo: Repo): helm_charts={}, ) - build_info = load_build_info(BuildScenario.MANUAL_RELEASE, git_repo.working_dir) + build_info = load_build_info(BuildScenario.MANUAL_RELEASE) assert build_info == expected_build_info diff --git a/scripts/release/tests/build_scenario_test.py b/scripts/release/tests/build_scenario_test.py deleted file mode 100644 index 9f142572b..000000000 --- a/scripts/release/tests/build_scenario_test.py +++ /dev/null @@ -1,39 +0,0 @@ -import os - -from git import Repo - -from scripts.release.build.build_scenario import BuildScenario -from scripts.release.constants import DEFAULT_CHANGELOG_PATH - - -class TestGetVersionForBuildScenario: - - def test_patch_build_scenario(self, git_repo: Repo): - os.environ["BUILD_ID"] = "688364423f9b6c00072b3556" - expected_version = os.environ["BUILD_ID"] - - version = BuildScenario.PATCH.get_version( - repository_path=git_repo.working_dir, changelog_sub_path=DEFAULT_CHANGELOG_PATH - ) - - assert version == expected_version - - def test_staging_build_scenario(self, git_repo: Repo): - initial_commit = list(git_repo.iter_commits(reverse=True))[4] - git_repo.git.checkout(initial_commit) - expected_version = initial_commit.hexsha[:8] - - version = BuildScenario.STAGING.get_version( - repository_path=git_repo.working_dir, changelog_sub_path=DEFAULT_CHANGELOG_PATH - ) - - assert version == expected_version - - def test_release_build_scenario(self, git_repo: Repo): - git_repo.git.checkout("1.2.0") - - version = BuildScenario.RELEASE.get_version( - repository_path=git_repo.working_dir, changelog_sub_path=DEFAULT_CHANGELOG_PATH - ) - - assert version == "1.2.0" diff --git a/scripts/release/tests/conftest.py b/scripts/release/tests/conftest.py index 4974a7ac8..5e1188add 100644 --- a/scripts/release/tests/conftest.py +++ b/scripts/release/tests/conftest.py @@ -164,23 +164,3 @@ def add_file(repo_path: str, src_file_path: str, dst_file_path: str | None = Non src_path = os.path.join("scripts/release/tests/testdata", src_file_path) return shutil.copy(src_path, dst_path) - - -@fixture(scope="module") -def readinessprobe_version() -> str: - return get_manually_upgradable_versions()["readiness-probe"] - - -@fixture(scope="module") -def operator_version_upgrade_post_start_hook_version() -> str: - return get_manually_upgradable_versions()["upgrade-hook"] - - -def get_manually_upgradable_versions() -> Dict[str, str]: - with open("build_info.json", "r") as f: - build_info = json.load(f) - - return { - "readiness-probe": build_info["images"]["readiness-probe"]["release"]["version"], - "upgrade-hook": build_info["images"]["upgrade-hook"]["release"]["version"], - } diff --git a/scripts/release/tests/release_info_test.py b/scripts/release/tests/release_info_test.py index add0c2dd4..bc59a2e83 100644 --- a/scripts/release/tests/release_info_test.py +++ b/scripts/release/tests/release_info_test.py @@ -1,65 +1,58 @@ import json -from git import Repo +from scripts.release.release_info import DUMMY_VERSION, create_release_info_json -from scripts.release.constants import DEFAULT_CHANGELOG_PATH -from scripts.release.release_info import create_release_info_json - - -def test_create_release_info_json( - git_repo: Repo, readinessprobe_version: str, operator_version_upgrade_post_start_hook_version: str -): - git_repo.git.checkout("1.2.0") +def test_create_release_info_json(): expected_json = { "images": { "operator": { "repositories": ["quay.io/mongodb/mongodb-kubernetes"], "platforms": ["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - "version": "1.2.0", + "version": DUMMY_VERSION, }, "init-database": { "repositories": ["quay.io/mongodb/mongodb-kubernetes-init-database"], "platforms": ["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - "version": "1.2.0", + "version": DUMMY_VERSION, }, "init-appdb": { "repositories": ["quay.io/mongodb/mongodb-kubernetes-init-appdb"], "platforms": ["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - "version": "1.2.0", + "version": DUMMY_VERSION, }, "init-ops-manager": { "repositories": ["quay.io/mongodb/mongodb-kubernetes-init-ops-manager"], "platforms": ["linux/amd64"], - "version": "1.2.0", + "version": DUMMY_VERSION, }, "database": { "repositories": ["quay.io/mongodb/mongodb-kubernetes-database"], "platforms": ["linux/arm64", "linux/amd64", "linux/s390x", "linux/ppc64le"], - "version": "1.2.0", + "version": DUMMY_VERSION, }, "readiness-probe": { "repositories": ["quay.io/mongodb/mongodb-kubernetes-readinessprobe"], "platforms": ["linux/arm64", "linux/amd64"], - "version": readinessprobe_version, + "version": DUMMY_VERSION, }, "upgrade-hook": { "repositories": ["quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook"], "platforms": ["linux/arm64", "linux/amd64"], - "version": operator_version_upgrade_post_start_hook_version, + "version": DUMMY_VERSION, }, }, "binaries": { "kubectl-mongodb": { "platforms": ["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], - "version": "1.2.0", + "version": DUMMY_VERSION, } }, - "helm-charts": {"mongodb-kubernetes": {"repositories": ["quay.io/mongodb/helm-charts"], "version": "1.2.0"}}, + "helm-charts": { + "mongodb-kubernetes": {"repositories": ["quay.io/mongodb/helm-charts"], "version": DUMMY_VERSION} + }, } expected_release_info_json = json.dumps(expected_json, indent=2) - release_info_json = create_release_info_json( - repository_path=git_repo.working_dir, changelog_sub_path=DEFAULT_CHANGELOG_PATH - ) + release_info_json = create_release_info_json() assert release_info_json == expected_release_info_json