Skip to content

Commit ce03769

Browse files
xeniapexeniape
committed
Merge branch 'main' into feat/hadoop-rework
2 parents 651c058 + 1965d50 commit ce03769

File tree

71 files changed

+3723
-2762
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+3723
-2762
lines changed

‎.github/actions/publish-image/action.yml‎

Lines changed: 83 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -59,68 +59,124 @@ runs:
5959
echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
6060
echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV
6161
62+
# TODO (@NickLarsenNZ): Make this a reusable action in .github/actions/push-and-sign/action.yml
6263
- name: Push Image to repo.stackable.tech and sign via cosign
6364
shell: bash
6465
run: |
6566
set -euo pipefail
66-
# Store the output of `docker image push` into a variable, so we can
67-
# parse it for the digest
68-
PUSH_OUTPUT=$(docker image push "$(< bake-target-tags)" 2>&1)
69-
echo "$PUSH_OUTPUT"
70-
# Obtain the digest of the pushed image from the output of `docker image
71-
# push`, because signing by tag is deprecated and will be removed from
72-
# cosign in the future
73-
DIGEST=$(echo "$PUSH_OUTPUT" | awk "/: digest: sha256:[a-f0-9]{64} size: [0-9]+$/ { print \$3 }")
74-
echo "DIGEST=$DIGEST" >> $GITHUB_ENV
67+
TARGET_TAG="$(< bake-target-tags)"
68+
docker image push "$TARGET_TAG"
69+
# Obtain the digest of the image, because signing by tag is deprecated and will be removed from cosign in the future
70+
# This must be done after a push, otherwise the repo digest is empty.
71+
# Find the digest for the image. Notice the architecture at the end of TAG
72+
# IMAGE_VERSION does not contain the architecture (otherwise we could have used ${IMAGE_NAME}:${IMAGE_VERSION})
73+
# Given:
74+
# IMAGE_NAME: docker.stackable.tech/stackable/hello-world
75+
# TARGET_TAG: docker.stackable.tech/stackable/hello-world:0.0.1-SNAPSHOT-stackable0.0.0-dev-arm64
76+
# Expect:
77+
# STDOUT: docker.stackable.tech/stackable/hello-world@sha256:917f800259ef4915f976e93987b752fd64debf347568610d7f685d20220fc88a
78+
REPO_DIGEST=$(
79+
docker inspect "$TARGET_TAG" --format json | \
80+
jq -r \
81+
--arg IMAGE_NAME "$IMAGE_NAME" \
82+
--arg TARGET_TAG "$TARGET_TAG" \
83+
'
84+
map(select(.RepoTags[] | contains($TARGET_TAG)))[0]
85+
| .RepoDigests[]
86+
| select(. | startswith($IMAGE_NAME))
87+
'
88+
)
89+
# Ensure REPO_DIGEST is not empty
90+
if [[ -z "$REPO_DIGEST" ]]; then
91+
>&2 echo "Repo Digest is empty, but is required for signing"
92+
exit 1
93+
fi
94+
95+
# Needed by future steps
96+
echo "REPO_DIGEST=$REPO_DIGEST" | tee -a $GITHUB_ENV
97+
7598
# Refer to image via its digest (docker.stackable.tech/stackable/airflow@sha256:0a1b2c...)
7699
# This generates a signature and publishes it to the registry, next to the image
77100
# Uses the keyless signing flow with Github Actions as identity provider
78-
cosign sign -y "$IMAGE_NAME@$DIGEST"
101+
cosign sign -y "${REPO_DIGEST}"
79102
80103
- name: Generate SBOM for the Nexus Image
81104
shell: bash
82105
run: |
83106
set -euo pipefail
84-
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "$IMAGE_NAME@$DIGEST";
107+
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "${REPO_DIGEST}";
108+
# The DIGEST is the right side of `REPO_DIGEST` (split by '@')
109+
DIGEST=${REPO_DIGEST#*@}
85110
# Determine the PURL for the image
86111
PURL="pkg:docker/stackable/${{ inputs.product }}@$DIGEST?repository_url=docker.stackable.tech";
87112
# Get metadata from the image
88-
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "$IMAGE_NAME@$DIGEST");
89-
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "$IMAGE_NAME@$DIGEST");
113+
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "${REPO_DIGEST}");
114+
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "${REPO_DIGEST}");
90115
# Merge the SBOM with the metadata for the image
91116
jq -s '{"metadata":{"component":{"description":"'"$IMAGE_METADATA_NAME. $IMAGE_METADATA_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;
92117
# Attest the SBOM to the image
93-
cosign attest -y --predicate sbom.merged.json --type cyclonedx "$IMAGE_NAME@$DIGEST"
118+
cosign attest -y --predicate sbom.merged.json --type cyclonedx "${REPO_DIGEST}"
94119
120+
# TODO (@NickLarsenNZ): Make this a reusable action in .github/actions/push-and-sign/action.yml
95121
- name: Push Image to oci.stackable.tech and sign via cosign
96122
shell: bash
97123
run: |
98124
set -euo pipefail
125+
# Update the registry
99126
IMAGE_NAME=oci.stackable.tech/sdp/${{ inputs.product }}
100-
echo "image: $IMAGE_NAME"
101-
docker tag "$(< bake-target-tags)" "$IMAGE_NAME:$TAG_NAME"
102-
# Store the output of `docker image push` into a variable, so we can parse it for the digest
103-
PUSH_OUTPUT=$(docker image push "$IMAGE_NAME:$TAG_NAME" 2>&1)
104-
echo "$PUSH_OUTPUT"
105-
# Obtain the digest of the pushed image from the output of `docker image push`, because signing by tag is deprecated and will be removed from cosign in the future
106-
DIGEST=$(echo "$PUSH_OUTPUT" | awk "/: digest: sha256:[a-f0-9]{64} size: [0-9]+$/ { print \$3 }")
107-
echo "DIGEST=$DIGEST" >> $GITHUB_ENV
127+
# IMAGE_NAME is needed by future steps
128+
echo "IMAGE_NAME=$IMAGE_NAME" | tee -a $GITHUB_ENV
129+
OLD_TARGET_TAG="$(< bake-target-tags)"
130+
TARGET_TAG="$IMAGE_NAME:$TAG_NAME"
131+
docker tag "$OLD_TARGET_TAG" "$TARGET_TAG"
132+
docker image push "$TARGET_TAG"
133+
# Obtain the digest of the image, because signing by tag is deprecated and will be removed from cosign in the future
134+
# This must be done after a push, otherwise the repo digest is empty.
135+
# Find the digest for the image. Notice the architecture at the end of TAG
136+
# IMAGE_VERSION does not contain the architecture (otherwise we could have used ${IMAGE_NAME}:${IMAGE_VERSION})
137+
# Given:
138+
# IMAGE_NAME: oci.stackable.tech/sdp/hello-world
139+
# TARGET_TAG: oci.stackable.tech/sdp/hello-world:0.0.1-SNAPSHOT-stackable0.0.0-dev-arm64
140+
# Expect:
141+
# STDOUT: oci.stackable.tech/sdp/hello-world@sha256:917f800259ef4915f976e93987b752fd64debf347568610d7f685d20220fc88a
142+
REPO_DIGEST=$(
143+
docker inspect "$TARGET_TAG" --format json | \
144+
jq -r \
145+
--arg IMAGE_NAME "$IMAGE_NAME" \
146+
--arg TARGET_TAG "$TARGET_TAG" \
147+
'
148+
map(select(.RepoTags[] | contains($TARGET_TAG)))[0]
149+
| .RepoDigests[]
150+
| select(. | startswith($IMAGE_NAME))
151+
'
152+
)
153+
# Ensure REPO_DIGEST is not empty
154+
if [[ -z "$REPO_DIGEST" ]]; then
155+
>&2 echo "Repo Digest is empty, but is required for signing"
156+
exit 1
157+
fi
158+
159+
# REPO_DIGEST is needed by future steps
160+
echo "REPO_DIGEST=$REPO_DIGEST" | tee -a $GITHUB_ENV
161+
108162
# Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)
109163
# This generates a signature and publishes it to the registry, next to the image
110164
# Uses the keyless signing flow with Github Actions as identity provider
111-
cosign sign -y "$IMAGE_NAME@$DIGEST"
165+
cosign sign -y "${REPO_DIGEST}"
112166
113167
- name: Generate SBOM for the Harbor Image
114168
shell: bash
115169
run: |
116170
set -euo pipefail
117-
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "$IMAGE_NAME@$DIGEST";
171+
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "${REPO_DIGEST}";
172+
# The DIGEST is the right side of `REPO_DIGEST` (split by '@')
173+
DIGEST=${REPO_DIGEST#*@}
118174
# Determine the PURL for the image
119175
PURL="pkg:docker/sdp/${{ inputs.product }}@$DIGEST?repository_url=oci.stackable.tech";
120176
# Get metadata from the image
121-
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "$IMAGE_NAME@$DIGEST");
122-
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "$IMAGE_NAME@$DIGEST");
177+
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "${REPO_DIGEST}");
178+
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "${REPO_DIGEST}");
123179
# Merge the SBOM with the metadata for the image
124180
jq -s '{"metadata":{"component":{"description":"'"$IMAGE_METADATA_NAME. $IMAGE_METADATA_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;
125181
# Attest the SBOM to the image
126-
cosign attest -y --predicate sbom.merged.json --type cyclonedx "$IMAGE_NAME@$DIGEST"
182+
cosign attest -y --predicate sbom.merged.json --type cyclonedx "${REPO_DIGEST}"

‎.github/actions/publish-manifest/action.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,25 @@ runs:
4545
shell: bash
4646
run: |
4747
set -euo pipefail
48+
49+
# Get the image index digest
50+
# Eg: index_digest docker.stackable.tech/stackable/hello-world:0.0.1-SNAPSHOT-stackable0.0.0-dev
51+
#
52+
# Note: `docker manifest push` currently outputs the same hash, but `manifest` is experimental and the
53+
# STDOUT is more likely to change than the structured output.
54+
function index_digest {
55+
docker buildx imagetools inspect --format '{{println .Manifest.Digest}}' "$1"
56+
}
57+
4858
MANIFEST_NAME="docker.stackable.tech/stackable/${{ inputs.product }}:$IMAGE_VERSION"
4959
# Create and push to Stackable Nexus
5060
# `docker manifest push` directly returns the digest of the manifest list
5161
# As it is an experimental feature, this might change in the future
5262
# Further reading: https://docs.docker.com/reference/cli/docker/manifest/push/
5363
# --amend because the manifest list would be updated since we use the same tag: 0.0.0-dev
5464
docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64"
55-
DIGEST=$(docker manifest push $MANIFEST_NAME)
65+
docker manifest push "$MANIFEST_NAME"
66+
DIGEST=$(index_digest "$MANIFEST_NAME")
5667
5768
# Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)
5869
# This generates a signature and publishes it to the registry, next to the image
@@ -62,5 +73,6 @@ runs:
6273
# Push to oci.stackable.tech as well
6374
MANIFEST_NAME="oci.stackable.tech/sdp/${{ inputs.product }}:$IMAGE_VERSION"
6475
docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64"
65-
DIGEST=$(docker manifest push $MANIFEST_NAME)
76+
docker manifest push $MANIFEST_NAME
77+
DIGEST=$(index_digest "$MANIFEST_NAME")
6678
cosign sign -y "$MANIFEST_NAME@$DIGEST"

0 commit comments

Comments
 (0)