Skip to content

Commit 984ca6c

Browse files
arturdzmarturdzm
authored
Merge pull request #411 from application-stacks/pipeline-modernize
RCO Pipeline modernize to WLO
2 parents b3bcb7b + 89cd2e3 commit 984ca6c

File tree

7 files changed

+453
-53
lines changed

7 files changed

+453
-53
lines changed

‎.one-pipeline.yaml‎

Lines changed: 95 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,18 @@ test:
1313
script: |
1414
#!/usr/bin/env bash
1515
# Download and configure golang
16-
wget --header "Accept: application/octet-stream" "https://golang.org/dl/go1.16.linux-amd64.tar.gz"
17-
rm -rf /usr/local/go && tar -C /usr/local -xf go1.16.linux-amd64.tar.gz
16+
GO_VERSION=$(get_env go-version)
17+
wget --header "Accept: application/octet-stream" "https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz"
18+
rm -rf /usr/local/go && tar -C /usr/local -xf go*.linux-amd64.tar.gz
1819
export PATH=$PATH:/usr/local/go/bin
1920
apt-get update
20-
apt-get -y install build-essential
21-
# Run unit tests
22-
#export DOCKER_USERNAME=$(get_env docker-username)
23-
#export DOCKER_PASSWORD=$(get_env docker-password)
21+
apt-get -y install build-essential
22+
2423
make unit-test
25-
#make docker-login
26-
#make build-image
2724
2825
static-scan:
2926
dind: true
27+
abort_on_failure: false
3028
image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
3129
script: |
3230
#!/usr/bin/env bash
@@ -47,7 +45,9 @@ static-scan:
4745
EOF
4846
chmod -x "$WORKSPACE"/runtime-component-operator/sonar-project.properties
4947
#echo "$SONAR_PASS" >> /tmp/sonarqube-token
50-
"${ONE_PIPELINE_PATH}"/internal/sonarqube/sonarqube_run
48+
"${COMMONS_PATH}"/static-scan/run.sh
49+
## Perform static lint
50+
./scripts/pipeline/static-linter-scan.sh --git-token $(get_env git-token) --static-linter-version $(get_env static-linter-version)
5151
containerize:
5252
dind: true
5353
abort_on_failure: true
@@ -79,7 +79,7 @@ containerize:
7979
fi
8080
# Build images
8181
export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
82-
export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key)
82+
export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key-staging)
8383
PIPELINE_REGISTRY=$(get_env pipeline-registry)
8484
PIPELINE_OPERATOR_IMAGE=$(get_env pipeline-operator-image)
8585
# Build amd64 image
@@ -111,6 +111,13 @@ containerize:
111111
save_artifact $i type=image name="$IMAGE" "digest=$DIGEST" "arch=$ARCH"
112112
done
113113
114+
sign-artifact:
115+
abort_on_failure: false
116+
image: icr.io/continuous-delivery/pipeline/image-signing:1.0.0@sha256:e9d8e354668ba3d40be2aaee08298d2aa7f0e1c8a1829cca4094ec93830e3e6a
117+
script: |
118+
#!/usr/bin/env bash
119+
echo "sign-artifact"
120+
114121
deploy:
115122
image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
116123
script: |
@@ -129,10 +136,6 @@ dynamic-scan:
129136
script: |
130137
#!/usr/bin/env bash
131138
echo "dynamic-scan"
132-
#export APP_URL=$(cat ../app-url)
133-
# feature preivew this until evidence locker v2 usage is full feature ready
134-
# can be triggered, and owasp will run for preview purposes
135-
#source scripts/zap/trigger_api_scan.sh
136139
137140
acceptance-test:
138141
dind: true
@@ -141,21 +144,27 @@ acceptance-test:
141144
script: |
142145
#!/usr/bin/env bash
143146
echo "acceptance-test"
144-
# Download and configure golang
145-
wget --header "Accept: application/octet-stream" "https://golang.org/dl/go1.16.linux-amd64.tar.gz"
146-
rm -rf /usr/local/go && tar -C /usr/local -xf go1.16.linux-amd64.tar.gz
147-
export PATH=$PATH:/usr/local/go/bin
148-
apt-get update
149-
apt-get -y install build-essential
150-
export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
151-
export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key)
152-
export DOCKER_USERNAME=$(get_env docker-username)
153-
export DOCKER_PASSWORD=$(get_env docker-password)
154-
export CLUSTER_URL=$(get_env test-cluster-url)
155-
export CLUSTER_TOKEN=$(get_env test-cluster-token)
156-
export TRAVIS_BUILD_NUMBER=$BUILD_NUMBER
157-
make setup
158-
make test-pipeline-e2e
147+
GO_VERSION=$(get_env go-version)
148+
export SKIP_KIND_E2E_TEST=$(get_env SKIP_KIND_E2E_TEST)
149+
if [[ $SKIP_KIND_E2E_TEST != "true" ]]; then
150+
# Download and configure golang
151+
wget --header "Accept: application/octet-stream" "https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz"
152+
rm -rf /usr/local/go && tar -C /usr/local -xf go1.16.linux-amd64.tar.gz
153+
export PATH=$PATH:/usr/local/go/bin
154+
apt-get update
155+
apt-get -y install build-essential
156+
export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
157+
export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key-staging)
158+
export DOCKER_USERNAME=$(get_env docker-username)
159+
export DOCKER_PASSWORD=$(get_env docker-password)
160+
export CLUSTER_URL=$(get_env test-cluster-url)
161+
export CLUSTER_TOKEN=$(get_env test-cluster-token)
162+
export TRAVIS_BUILD_NUMBER=$BUILD_NUMBER
163+
make setup
164+
make test-pipeline-e2e
165+
else
166+
echo "skipping Acceptance test"
167+
fi
159168
160169
scan-artifact:
161170
abort_on_failure: false
@@ -181,25 +190,64 @@ scan-artifact:
181190
done
182191
fi
183192
193+
echo "aqua scan"
194+
# install docker
195+
curl -fsSL https://get.docker.com -o get-docker.sh
196+
sudo sh get-docker.sh
197+
# get aqua scan executables
198+
git clone https://$(get_env git-token)@github.ibm.com/CICD-CPP/cpp-pipelines.git
199+
chmod -R +x cpp-pipelines
200+
# setup and execute aqua scan
201+
cd cpp-pipelines
202+
export CUSTOM_SCRIPTS_PATH=/workspace/app/one-pipeline-config-repo/cpp-pipelines
203+
./commons/aqua/aqua-local-scan
204+
205+
source "${COMMONS_PATH}/whitesource/whitesource_unified_agent_scan.sh"
206+
184207
release:
185208
abort_on_failure: false
186-
image: wcp-compliance-automation-team-docker-local.artifactory.swg-devops.com/ibm-compliance-automation:1.9.1@sha256:3f3e344a1efb160d83c48cf2ee878a39cbad058c8640c423472e0546316232fd
209+
image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
187210
script: |
188211
#!/usr/bin/env bash
189-
echo "release"
190-
set_env ibmcloud-api-key $(get_env ibmcloud-api-key-prod)
191-
192-
#owasp-zap-api:
193-
# dind: true
194-
# abort_on_failure: false
195-
# image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
196-
# script: |
197-
# !/usr/bin/env bash
198-
# if [[ "$PIPELINE_DEBUG" == 1 ]]; then
199-
# trap env EXIT
200-
# env
201-
# set -x
202-
# fi
203-
204-
# source scripts/zap/setup_api_scan.sh
205-
# source scripts/zap/run_api_scan.sh
212+
213+
RELEASE_FLAG=$(get_env release "false")
214+
215+
if [[ $RELEASE_FLAG != "true" ]]; then
216+
echo "Skipping release stage; environment property 'release' is set to $RELEASE_FLAG"
217+
exit 0
218+
fi
219+
220+
SKIP_ALL_CHECKS=$(get_env SKIP_ALL_CHECKS "false")
221+
./scripts/pipeline/evaluator.sh
222+
if [[ $? == 0 || $SKIP_ALL_CHECKS == "true" ]]; then
223+
if [[ $SKIP_ALL_CHECKS == "true" ]]; then
224+
echo "Skipping image scan checks"
225+
fi
226+
APP_REPO=$(pwd)
227+
echo "Application Repository: $APP_REPO"
228+
INVENTORY_REPO=$(get_env inventory-repo)
229+
echo "Cloning inventory repository: $INVENTORY_REPO"
230+
cd "$WORKSPACE"
231+
APP_TOKEN_PATH="$WORKSPACE/secrets/app-token"
232+
. "${ONE_PIPELINE_PATH}"/git/clone_repo \
233+
"$INVENTORY_REPO" \
234+
"master" \
235+
"" \
236+
"$APP_TOKEN_PATH"
237+
REPO=${INVENTORY_REPO##*/}
238+
NAME=${REPO%.*}
239+
echo "Inventory name: $NAME"
240+
cd $WORKSPACE/$NAME
241+
if [ "$(ls )" ]; then
242+
echo "Clearing inventory repository: $INVENTORY_REPO"
243+
git config --global user.email "tekton@example.com"
244+
git config --global user.name "Tekton"
245+
git rm *
246+
git commit -m "Delete contents of inventory repository - $PIPELINE_RUN_ID"
247+
git push origin master
248+
fi
249+
cd $APP_REPO
250+
./scripts/pipeline/release.sh
251+
else
252+
echo "Errors found. images will not be released"
253+
fi

‎scripts/pipeline/cd_finish‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/usr/bin/env bash
2+
3+
set -e -o pipefail
4+
5+
if [ "$PIPELINE_DEBUG" == 1 ]; then
6+
pwd
7+
env
8+
trap env EXIT
9+
set -x
10+
fi
11+
12+
export WORKSPACE
13+
14+
. "${ONE_PIPELINE_PATH}"/internal/pipeline/variables_config
15+
. "${ONE_PIPELINE_PATH}"/tools/retry
16+
17+
cd "$WORKSPACE"
18+
19+
pipeline_data="${WORKSPACE}/pipeline.data"
20+
source "$pipeline_data"
21+
22+
#
23+
# start
24+
#
25+
26+
INVENTORY_REPO="$(cat /config/inventory-url)"
27+
GHE_ORG=${INVENTORY_REPO%/*}
28+
GHE_ORG=${GHE_ORG##*/}
29+
GHE_REPO=${INVENTORY_REPO##*/}
30+
31+
curl -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $(get_env git-token)" https://github.ibm.com/api/v3/repos/${GHE_ORG}/${GHE_REPO}/releases -d "{\"tag_name\": \"$(get_env version)\",\"name\": \"$(get_env version)\",\"draft\": false,\"prerelease\": false}"
32+
33+
set_env ibmcloud-api "cloud.ibm.com"
34+
. "${ONE_PIPELINE_PATH}"/internal/security-compliance/scan
35+
36+
. "${ONE_PIPELINE_PATH}"/internal/doi/publish_acceptance_test
37+
38+
#publish_acceptance_test "$(get_env ACCEPTANCE_TESTS_TASK_NAME)" "$(get_env ACCEPTANCE_TESTS_STEP_NAME)" "com.ibm.acceptance_tests"
39+
40+
# collector_cd
41+
echo "Processing 'build-image-signing' ..."
42+
. "${ONE_PIPELINE_PATH}"/internal/evidence/collector \
43+
"$(get_env IMAGE_SIGNING_TASK_NAME)" `# task_name` \
44+
"$(get_env IMAGE_SIGNING_STEP_NAME)" `# step_name` \
45+
"sign-artifact" `# stage` \
46+
"success" `# excepted_status` \
47+
"$(get_env STAGE_SIGN_ARTIFACT_STATUS)" `# actual_status` \
48+
"image-signing" `# evidence_name` \
49+
"com.ibm.cloud.image_signing" `# evidence_type` \
50+
"1.0.0" `# evidence_type_version` \
51+
"cd" `# namespace` \
52+
"" `# artifacts`
53+
54+
echo "Processing 'acceptance-test' ..."
55+
. "${ONE_PIPELINE_PATH}"/internal/evidence/collector \
56+
"$(get_env ACCEPTANCE_TESTS_TASK_NAME)" `# task_name` \
57+
"$(get_env ACCEPTANCE_TESTS_STEP_NAME)" `# step_name` \
58+
"acceptance-test" `# stage` \
59+
"success" `# excepted_status` \
60+
"$(get_env STAGE_ACCEPTANCE_TEST_STATUS)" `# actual_status` \
61+
"acceptance-test" `# evidence_name` \
62+
"com.ibm.acceptance_tests" `# evidence_type` \
63+
"1.0.0" `# evidence_type_version` \
64+
"cd" `# namespace` \
65+
"" `# artifacts`
66+
67+
echo -e "\n"
68+
69+
#. "${ONE_PIPELINE_PATH}"/internal/evidence/create_summary "from-every-inventory-entry" "include_cd_evidence"
70+
71+
#. "${ONE_PIPELINE_PATH}"/internal/evidence/upload_summary
72+
73+
# collector_cd_final
74+
. "${ONE_PIPELINE_PATH}"/internal/evidence/upload_pipeline_data
75+
76+
printf "\n\nCollect and upload pipeline logs\n\n" >&2
77+
upload_pipeline_task_logs "cd"
78+
79+
printf "\n\nCollect and upload pipeline run data\n\n" >&2
80+
upload_pipeline_run_data "cd"
81+
82+
echo -e "\n"
83+
84+
"${ONE_PIPELINE_PATH}/internal/slack/generate_cd_end_message.py" | "${ONE_PIPELINE_PATH}/internal/slack/post_message.py" || true
85+
86+
#. "${ONE_PIPELINE_PATH}"/internal/pipeline/evaluator_cd

‎scripts/pipeline/evaluator.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/usr/bin/env bash
2+
3+
function evaluate() {
4+
name=$1
5+
expected_status=$2
6+
actual_status=$3
7+
task_name=$4
8+
skip_task=$5
9+
10+
if [[ $skip_task == 'true' ]]; then
11+
echo "Task '${name}' has been skipped."
12+
echo "Actual: '${actual_status}' | Expected: '${expected_status}'."
13+
elif [ "$expected_status" != "$actual_status" ]; then
14+
echo "Task '${name}' has failed"
15+
echo "The actual result value did not match expected value."
16+
echo "Actual: '${actual_status}' | Expected: '${expected_status}'."
17+
export PIPELINE_EXIT+=1
18+
else
19+
export PIPELINE_EXIT+=0
20+
fi
21+
}
22+
23+
printf "\n\nEvaluating Pipeline Task results \n\n" >&2
24+
25+
evaluate \
26+
"detect-secrets" `# name` \
27+
"success" `# expected_status` \
28+
"$(get_env DETECT_SECRETS_STATUS)" `# actual_status` \
29+
"$(get_env DETECT_SECRET_TASK_NAME)" `# task_name`
30+
31+
evaluate \
32+
"unit-tests" `# name` \
33+
"success" `# expected_status` \
34+
"$(get_env STAGE_TEST_STATUS)" `# actual_status` \
35+
"$(get_env UNIT_TESTS_TASK_NAME)" `# task_name`
36+
37+
enable_sonar=$(get_env opt-in-sonar "")
38+
enable_gosec=$(get_env opt-in-gosec "")
39+
if [[ -n $enable_sonar || -n $enable_gosec ]]; then
40+
evaluate \
41+
"static-scan" `# name` \
42+
"success" `# expected_status` \
43+
"$(get_env STAGE_STATIC_SCAN_STATUS)" `# actual_status` \
44+
"$(get_env STATIC_SCAN_TASK_NAME)" `# task_name`
45+
fi
46+
47+
evaluate \
48+
"vulnerability-scan" `# name` \
49+
"success" `# expected_status` \
50+
"$(get_env CRA_VULNERABILITY_RESULTS_STATUS)" `# actual_status` \
51+
"$(get_env CRA_VULNERABILITY_TASK_NAME)" `# task_name`
52+
53+
evaluate \
54+
"cis-check" `# name` \
55+
"success" `# expected_status` \
56+
"$(get_env CIS_CHECK_VULNERABILITY_RESULTS_STATUS)"`# actual_status` \
57+
"$(get_env CIS_CHECK_TASK_NAME)" `# task_name`
58+
59+
evaluate \
60+
"bom-check" `# name` \
61+
"success" `# expected_status` \
62+
"$(get_env CRA_BOM_CHECK_RESULTS_STATUS)" `# actual_status` \
63+
"$(get_env BOM_CHECK_TASK_NAME)" `# task_name`
64+
65+
evaluate \
66+
"branch-protection" `# name` \
67+
"success" `# expected_status` \
68+
"$(get_env BRANCH_PROTECTION_STATUS)" `# actual_status` \
69+
"$(get_env BRANCH_PROTECTION_TASK_NAME)" `# task_name` \
70+
"$(get_env SKIP_BRANCH_PROTECTION 'false')"
71+
72+
evaluate \
73+
"vulnerability-advisor" `# name` \
74+
"success" `# expected_status` \
75+
"$(get_env STAGE_SCAN_ARTIFACT_STATUS)" `# actual_status` \
76+
"$(get_env VULNERABILITY_ADVISOR_TASK_NAME)" `# task_name` \
77+
"$(get_env SKIP_VA 'false')"
78+
79+
evaluate \
80+
"acceptance-tests" `# name` \
81+
"success" `# expected_status` \
82+
"$(get_env STAGE_ACCEPTANCE_TEST_STATUS)" `# actual_status` \
83+
"$(get_env ACCEPTANCE_TESTS_TASK_NAME)" `# task_name`
84+
85+
if [[ "$PIPELINE_EXIT" == *"1"* ]]; then
86+
exit 1
87+
else
88+
echo "Every Task result passed the check!"
89+
exit 0
90+
fi

0 commit comments

Comments
 (0)