Skip to content

Commit 4bf8109

Browse files
lsierantlsierant
committed
Publish prerelease artifacts
1 parent a4221cb commit 4bf8109

File tree

4 files changed

+273
-30
lines changed

4 files changed

+273
-30
lines changed

‎scripts/dev/configure_container_auth.sh‎

Lines changed: 60 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ fi
1515

1616
CONTAINER_RUNTIME="${CONTAINER_RUNTIME-"docker"}"
1717

18+
# Registry URLs
19+
ECR_EU_WEST="268558157000.dkr.ecr.eu-west-1.amazonaws.com"
20+
ECR_US_EAST="268558157000.dkr.ecr.us-east-1.amazonaws.com"
21+
ECR_SEARCH_US_EAST="901841024863.dkr.ecr.us-east-1.amazonaws.com"
22+
1823
setup_validate_container_runtime() {
1924
case "${CONTAINER_RUNTIME}" in
2025
"podman")
@@ -109,26 +114,44 @@ if [[ ! -f "${CONFIG_PATH}" ]]; then
109114
write_file '{}' "${CONFIG_PATH}"
110115
fi
111116

112-
if [[ -f "${CONFIG_PATH}" ]]; then
113-
if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then
114-
echo "Checking if container registry credentials are valid..."
115-
ecr_auth=$(exec_cmd jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}")
117+
check_if_login_required() {
118+
echo "Checking if container registry credentials are valid..."
119+
120+
check_registry_credentials() {
121+
registry_url=$1
122+
image_path=$1
123+
image_tag=$2
124+
# shellcheck disable=SC2016
125+
ecr_auth=$(exec_cmd jq -r --arg registry "${registry_url}" '.auths.[$registry].auth // empty' "${CONFIG_PATH}")
116126

117127
if [[ -n "${ecr_auth}" ]]; then
118-
http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://268558157000.dkr.ecr.us-east-1.amazonaws.com/v2/dev/mongodb-kubernetes/manifests/latest" \
128+
http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://${registry_url}/v2/${image_path}/manifest/${image_tag}" \
119129
-H "Authorization: Basic ${ecr_auth}" 2>/dev/null || echo "error/timeout")
120130

121131
if [[ "${http_status}" != "401" && "${http_status}" != "403" && "${http_status}" != "error/timeout" ]]; then
122132
echo "Container registry credentials are up to date - not performing the new login!"
123-
exit
133+
return 0
124134
fi
125-
echo "Container login required (HTTP status: ${http_status})"
135+
echo -e "${RED}Container login required (HTTP status: ${http_status})${NO_COLOR}"
126136
else
127-
echo "No ECR credentials found in container config - login required"
137+
echo -e "${RED}No ECR credentials found in container config - login required${NO_COLOR}"
128138
fi
139+
140+
return 0
141+
}
142+
143+
check_registry_credentials "${ECR_EU_WEST}" "mongot/community" "1.47.0" | prepend "${ECR_EU_WEST}" || return 1
144+
check_registry_credentials "${ECR_US_EAST}" "dev/mongodb-kubernetes" "latest" | prepend "${ECR_US_EAST}" || return 1
145+
if [[ "${MDB_SEARCH_AWS_SSO_LOGIN:-"false"}" == "true" ]]; then
146+
check_registry_credentials "${ECR_SEARCH_US_EAST}" "mongot-community/rapid-releases" "latest" | prepend "${ECR_SEARCH_US_EAST}" || return 1
129147
fi
130148

149+
return 0
150+
}
151+
152+
login_to_registries() {
131153
title "Performing container login to ECR registries"
154+
echo "$(aws --version)}"
132155

133156
# There could be some leftovers on Evergreen (Docker-specific, skip for Podman)
134157
if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then
@@ -139,34 +162,41 @@ if [[ -f "${CONFIG_PATH}" ]]; then
139162
remove_element "credHelpers"
140163
fi
141164
fi
142-
fi
143165

166+
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "${ECR_US_EAST}"
144167

145-
echo "$(aws --version)}"
168+
if [[ "${MDB_SEARCH_AWS_SSO_LOGIN:-"false"}" == "true" ]]; then
169+
aws sso login --profile devprod-platforms-ecr-user
170+
aws --profile devprod-platforms-ecr-user ecr get-login-password --region us-east-1 | registry_login "AWS" "${ECR_SEARCH_US_EAST}"
171+
fi
146172

147-
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com"
173+
# by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json
174+
# We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element
175+
# This is Docker-specific behavior, Podman stores credentials directly in auth.json
176+
if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && exec_cmd grep -q "credsStore" "${CONFIG_PATH}"; then
177+
remove_element "credsStore"
148178

149-
# by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json
150-
# We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element
151-
# This is Docker-specific behavior, Podman stores credentials directly in auth.json
152-
if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && exec_cmd grep -q "credsStore" "${CONFIG_PATH}"; then
153-
remove_element "credsStore"
179+
# login again to store the credentials into the config.json
180+
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "${ECR_US_EAST}"
181+
fi
154182

155-
# login again to store the credentials into the config.json
156-
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com"
157-
fi
183+
aws ecr get-login-password --region "eu-west-1" | registry_login "AWS" "${ECR_EU_WEST}"
184+
185+
if [[ -n "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then
186+
# log in to quay.io for the mongodb/mongodb-search-community private repo
187+
# TODO remove once we switch to the official repo in Public Preview
188+
quay_io_auth_file=$(mktemp)
189+
config_tmp=$(mktemp)
190+
echo "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}"
191+
exec_cmd jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
192+
exec_cmd mv "${config_tmp}" "${CONFIG_PATH}"
193+
rm "${quay_io_auth_file}"
194+
fi
195+
}
158196

159-
aws ecr get-login-password --region "eu-west-1" | registry_login "AWS" "268558157000.dkr.ecr.eu-west-1.amazonaws.com"
160-
161-
if [[ -n "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then
162-
# log in to quay.io for the mongodb/mongodb-search-community private repo
163-
# TODO remove once we switch to the official repo in Public Preview
164-
quay_io_auth_file=$(mktemp)
165-
config_tmp=$(mktemp)
166-
echo "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}"
167-
exec_cmd jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
168-
exec_cmd mv "${config_tmp}" "${CONFIG_PATH}"
169-
rm "${quay_io_auth_file}"
197+
if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then
198+
check_if_login_required
199+
login_to_registries
170200
fi
171201

172202
create_image_registries_secret

‎scripts/dev/contexts/private-context-template‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,7 @@ export PRERELEASE_PULLSECRET_DOCKERCONFIGJSON="<dockerconfigjson secret>"
101101

102102
# uncomment to enable license update with pre-commit script
103103
# export MDB_UPDATE_LICENSES=true
104+
105+
# enable only for getting mongodb search rapid-releases from
106+
# 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongot-community/rapid-releases
107+
export MDB_SEARCH_AWS_SSO_LOGIN="false"

‎scripts/dev/mirror_artifacts_to_staging_quay.sh‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#!/bin/bash
2+
#
3+
# Script to publish (mirror) container images and helm charts to staging registry.
4+
#
5+
6+
set -euo pipefail
7+
8+
test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x
9+
10+
source scripts/dev/set_env_context.sh
11+
source scripts/funcs/printing
12+
13+
BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev"
14+
STAGING_BASE_URL="quay.io/mongodb/staging"
15+
16+
if [[ $# -lt 2 ]]; then
17+
echo "The tool mirrors images built in any given evg patch id (or latest) from ${BASE_REPO_URL} to ${STAGING_BASE_URL}"
18+
echo "It publishes helm oci image of the helm chart with chart version \"<prerelease helm version>-<version_id>\""
19+
echo "Usage: $0 <prerelease helm version> <version_id>"
20+
echo "Example: $0 1.4.0-prerelease 68b1a853973bae0007d5eaa0"
21+
echo ""
22+
exit 1
23+
fi
24+
25+
helm_chart_version_prefix="$1"
26+
operator_version="$2"
27+
search_version=${3:-"latest"}
28+
29+
helm_chart_version="${helm_chart_version_prefix}-${operator_version}"
30+
31+
get_arch_digest() {
32+
local image="$1"
33+
local arch="$2"
34+
local manifest_json
35+
manifest_json=$(docker buildx imagetools inspect --raw "${image}" 2>/dev/null || echo '{}')
36+
37+
local media_type
38+
media_type=$(echo "$manifest_json" | jq -r '.mediaType // empty')
39+
40+
if [[ "$media_type" == *"manifest.list"* ]]; then
41+
# this is a multi-arch manifest
42+
local arch_digest
43+
arch_digest=$(echo "$manifest_json" | jq -r ".manifests[] | select(.platform.architecture == \"${arch}\" and .platform.os == \"linux\") | .digest")
44+
45+
if [[ -n "$arch_digest" && "$arch_digest" != "null" ]]; then
46+
echo "$arch_digest"
47+
return 0
48+
fi
49+
elif [[ "${arch}" == "amd64" ]]; then
50+
# otherwise it must be a single-arch (image) manifest, so we return it only if we ask for amd64
51+
local arch_digest
52+
arch_digest="sha256:$(echo -n "$manifest_json" | sha256)"
53+
echo "$arch_digest"
54+
fi
55+
56+
echo ""
57+
return 0
58+
}
59+
60+
process_image() {
61+
local source_image="$1"
62+
local target_image="$2"
63+
64+
echo " Processing ${source_image}..."
65+
66+
local digest_arm64
67+
local digest_amd64
68+
digest_arm64=$(get_arch_digest "${source_image}" arm64)
69+
digest_amd64=$(get_arch_digest "${source_image}" amd64)
70+
71+
if [[ -n "${digest_amd64}" ]]; then
72+
docker pull "${source_image}@${digest_amd64}"
73+
docker tag "${source_image}@${digest_amd64}" "${target_image}-amd64"
74+
docker push "${target_image}-amd64"
75+
fi
76+
77+
if [[ -n "${digest_arm64}" ]]; then
78+
docker pull "${source_image}@${digest_arm64}"
79+
docker tag "${source_image}@${digest_arm64}" "${target_image}-arm64"
80+
docker push "${target_image}-arm64"
81+
fi
82+
83+
docker manifest create "${target_image}" ${digest_amd64:+--amend ${target_image}-amd64} ${digest_arm64:+--amend ${target_image}-arm64}
84+
docker manifest push "${target_image}"
85+
}
86+
87+
publish_images() {
88+
local names=()
89+
local sources=()
90+
local destinations=()
91+
92+
operator_images=(
93+
"mongodb-kubernetes"
94+
"mongodb-kubernetes-database"
95+
"mongodb-kubernetes-init-appdb"
96+
"mongodb-kubernetes-init-database"
97+
"mongodb-kubernetes-init-ops-manager"
98+
"mongodb-kubernetes-readinessprobe"
99+
"mongodb-kubernetes-operator-version-upgrade-post-start-hook"
100+
)
101+
102+
if [[ -n "${search_version}" ]]; then
103+
names+=("mongodb-search")
104+
sources+=("901841024863.dkr.ecr.us-east-1.amazonaws.com/mongot-community/rapid-releases:${search_version}")
105+
destinations+=("${STAGING_BASE_URL}/mongodb-search:${search_version}")
106+
fi
107+
108+
for image in "${operator_images[@]}"; do
109+
names+=("${image}")
110+
sources+=("${BASE_REPO_URL}/${image}:${operator_version}")
111+
destinations+=("${STAGING_BASE_URL}/${image}:${helm_chart_version}")
112+
done
113+
114+
echo "Starting Docker image re-tagging and publishing to staging..."
115+
echo "Version ID: ${operator_version}"
116+
echo "Source repository: ${BASE_REPO_URL}"
117+
echo "Target repository: ${STAGING_BASE_URL}"
118+
echo ""
119+
120+
for i in "${!names[@]}"; do
121+
process_image "${sources[$i]}" "${destinations[$i]}"
122+
done
123+
124+
echo "=== SUMMARY ==="
125+
echo "All images have been successfully re-tagged and pushed to staging!"
126+
echo ""
127+
echo "Images processed:"
128+
for i in "${!names[@]}"; do
129+
echo " ${names[$i]}: ${sources[$i]} -> ${destinations[$i]}"
130+
done
131+
}
132+
133+
update_helm_values() {
134+
scripts/dev/run_python.sh scripts/evergreen/release/update_helm_values_files.py
135+
yq eval ".version = \"${helm_chart_version}\"" -i helm_chart/Chart.yaml
136+
echo "Updated helm_chart/Chart.yaml version to: ${helm_chart_version}"
137+
138+
yq eval \
139+
".registry.operator = \"${STAGING_BASE_URL}\" |
140+
.registry.database = \"${STAGING_BASE_URL}\" |
141+
.registry.initDatabase = \"${STAGING_BASE_URL}\" |
142+
.registry.initOpsManager = \"${STAGING_BASE_URL}\" |
143+
.registry.initAppDb = \"${STAGING_BASE_URL}\" |
144+
.registry.appDb = \"${STAGING_BASE_URL}\" |
145+
.registry.versionUpgradeHook = \"${STAGING_BASE_URL}\" |
146+
.registry.readinessProbe = \"${STAGING_BASE_URL}\"
147+
" -i helm_chart/values.yaml
148+
echo "Updated helm_chart/values.yaml registry to: ${STAGING_BASE_URL}"
149+
}
150+
151+
prepare_helm_oci_image() {
152+
mkdir -p tmp
153+
helm package helm_chart -d tmp/
154+
}
155+
156+
push_helm_oci_image() {
157+
export HELM_REGISTRY_CONFIG=~/.docker/config.json
158+
helm push "tmp/mongodb-kubernetes-${helm_chart_version}.tgz" "oci://${STAGING_BASE_URL}/helm-chart"
159+
}
160+
161+
update_release_json() {
162+
if [[ ! -f "release.json" ]]; then
163+
echo "Error: release.json file not found"
164+
exit 1
165+
fi
166+
167+
echo "Updating release.json with versions..."
168+
169+
# Update operator and init versions
170+
jq --arg version "${helm_chart_version}" \
171+
--arg registry "${STAGING_BASE_URL}" \
172+
'.mongodbOperator = $version |
173+
.initDatabaseVersion = $version |
174+
.initOpsManagerVersion = $version |
175+
.initAppDbVersion = $version |
176+
.databaseImageVersion = $version' \
177+
release.json > release.json.tmp && mv release.json.tmp release.json
178+
179+
# Update search community version
180+
jq --arg searchVersion "${search_version}" \
181+
--arg searchRepo "${STAGING_BASE_URL}" \
182+
--arg searchImageName "mongodb-search" \
183+
'.search.community.repo = $searchRepo |
184+
.search.community.name = $searchImageName |
185+
.search.community.version = $searchVersion' \
186+
release.json > release.json.tmp && mv release.json.tmp release.json
187+
188+
echo "Updated release.json with:"
189+
echo " - Operator versions: ${helm_chart_version}"
190+
echo " - Search community version: ${MDB_SEARCH_COMMUNITY_VERSION}"
191+
}
192+
193+
revert_changes_to_local_files() {
194+
echo "Reverting generated/updated files: helm_chart/ public/ config/ release.json"
195+
git checkout -- helm_chart/ public/ config/ release.json
196+
}
197+
198+
publish_images
199+
update_release_json
200+
update_helm_values
201+
prepare_helm_oci_image
202+
push_helm_oci_image
203+
revert_changes_to_local_files

‎scripts/evergreen/release/update_helm_values_files.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,12 @@ def update_helm_charts(operator_version, release):
8989
set_value_in_yaml_file(
9090
"helm_chart/values.yaml", "search.community.version", release["search"]["community"]["version"]
9191
)
92+
set_value_in_yaml_file(
93+
"helm_chart/values.yaml", "search.community.repo", release["search"]["community"]["repo"]
94+
)
95+
set_value_in_yaml_file(
96+
"helm_chart/values.yaml", "search.community.name", release["search"]["community"]["name"]
97+
)
9298

9399

94100
def update_cluster_service_version(operator_version):

0 commit comments

Comments
 (0)