Build and pipeline updates (#470)

* Update pipeline

* Build only amd64

Author: arturdzm

.one-pipeline.yaml

@@ -1,13 +1,11 @@
 version: '1'
 
 setup:
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
   script: |
     #!/usr/bin/env bash
 
     echo "setup"
-    apt-get update
-    apt-get -y install build-essential
 
     # Download Go
     GO_VERSION=$(get_env go-version)
@@ -55,11 +53,11 @@ setup:
       echo "git push --prune https://$GHE_TOKEN@$WHITESOURCE_GHE_REPO $BRANCH_REFSPEC +refs/tags/*:refs/tags/*"
       git push --prune https://$GHE_TOKEN@$WHITESOURCE_GHE_REPO $BRANCH_REFSPEC +refs/tags/*:refs/tags/*
     fi
-  
+
 test:
   dind: true
   abort_on_failure: true
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
   script: |
     #!/usr/bin/env bash
 
@@ -89,17 +87,19 @@ test:
 static-scan:
   dind: true
   abort_on_failure: false
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
   script: |
     #!/usr/bin/env bash
+
     PERIODIC_SCAN=$(get_env periodic-rescan)
     PERIODIC_SCAN="$(echo "$PERIODIC_SCAN" | tr '[:upper:]' '[:lower:]')"
 
     if [[ ! -z "$PERIODIC_SCAN" && "$PERIODIC_SCAN" != "false" && "$PERIODIC_SCAN" != "no"  ]]; then
-      echo "Skipping static scan. This is a periodic run that is only meant to produce CVE information."
+      echo "Skipping static-scan. This is a periodic run that is only meant to produce CVE information."
       exit 0
     fi
 
+    BRANCH=$(get_env branch) 
     read -r SONAR_HOST_URL <<< "$(get_env sonarqube | jq -r '.parameters.dashboard_url' | sed 's:/*$::')"
     read -r SONAR_USER <<< "$(get_env sonarqube | jq -r '.parameters.user_login')"
     SONARQUBE_INSTANCE_ID=$(get_env sonarqube | jq -r '.instance_id')
@@ -109,6 +109,7 @@ static-scan:
     sonar.projectKey=runtime-component-operator
     sonar.host.url=$SONAR_HOST_URL
     sonar.sources=.
+    sonar.branch.name=$BRANCH
     sonar.login=$SONAR_USER
     sonar.password=$SONAR_PASS
     sonar.c.file.suffixes=-
@@ -118,6 +119,7 @@ static-scan:
     chmod -x "$WORKSPACE"/runtime-component-operator/sonar-project.properties
     #echo "$SONAR_PASS" >> /tmp/sonarqube-token
     "${COMMONS_PATH}"/static-scan/run.sh
+
     ## Perform static lint
     ./scripts/pipeline/static-linter-scan.sh --git-token $(get_env git-token) --static-linter-version $(get_env static-linter-version)
     
@@ -152,25 +154,19 @@ containerize:
     export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
     export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key-staging)  
     PIPELINE_REGISTRY=$(get_env pipeline-registry)
-    PIPELINE_OPERATOR_IMAGE=$(get_env pipeline-operator-image) 
-    
-    PERIODIC_SCAN=$(get_env periodic-rescan)
-    PERIODIC_SCAN="$(echo "$PERIODIC_SCAN" | tr '[:upper:]' '[:lower:]')"
-
-    if [[ ! -z "$PERIODIC_SCAN" && "$PERIODIC_SCAN" != "false" && "$PERIODIC_SCAN" != "no"  ]]; then
-      echo "Skipping containerize stage. This is a periodic run that is only meant to produce CVE information."
-    else 
-      # Build amd64 image
-      make build-pipeline-releases
-      # Build ppc64le and s390x images
-      ./scripts/pipeline/launch-travis.sh -t $(get_env travis-token) -r "https://github.com/application-stacks/runtime-component-operator" -b $(get_env branch) -l
-      # Build manifest
-      make build-pipeline-manifest    
-      # Build bundle image
-      ./scripts/pipeline/launch-catalog-build.sh -t $(get_env travis-token) -r "https://github.com/application-stacks/runtime-component-operator" -b $(get_env branch) -l
-    fi
+    PIPELINE_OPERATOR_IMAGE=$(get_env pipeline-operator-image)  
+    # Build amd64 image
+    make build-pipeline-releases
+    # Build ppc64le and s390x images
+    #./scripts/pipeline/launch-travis.sh -t $(get_env travis-token) -r "https://github.com/application-stacks/runtime-component-operator" -b $(get_env branch) -l
+    # Build manifest
+    make build-pipeline-manifest    
+    # Build bundle image
+    # ./scripts/pipeline/launch-catalog-build.sh -t $(get_env travis-token) -r "https://github.com/application-stacks/runtime-component-operator" -b $(get_env branch) -l
+    make install-opm
+    make bundle-pipeline-releases RELEASE_TARGET=${RELEASE_TARGET}
     # Save artifacts
-    declare -a tags=("daily-amd64" "daily-ppc64le" "daily-s390x") 
+    declare -a tags=("${RELEASE_TARGET}" "${RELEASE_TARGET}-amd64") 
     for i in "${tags[@]}"
     do
       IMAGE=$PIPELINE_REGISTRY/$PIPELINE_OPERATOR_IMAGE:$i
@@ -179,7 +175,7 @@ containerize:
       echo "Saving artifact $i name=$IMAGE digest=$DIGEST"
       save_artifact $i type=image name="$IMAGE" "digest=$DIGEST" "arch=$ARCH"   
     done
-    declare -a catalogs=("catalog-daily")
+    declare -a catalogs=("catalog-${RELEASE_TARGET}")
     for i in "${catalogs[@]}"
     do
       IMAGE=$PIPELINE_REGISTRY/$PIPELINE_OPERATOR_IMAGE:$i
@@ -190,9 +186,9 @@ containerize:
       save_artifact $i type=image name="$IMAGE" "digest=$DIGEST" "arch=$ARCH"   
     done
 
-    echo "whitesource scan"
-    chmod +x "${COMMONS_PATH}/whitesource/whitesource_unified_agent_scan.sh"
-    source "${COMMONS_PATH}/whitesource/whitesource_unified_agent_scan.sh"
+    # echo "whitesource scan"
+    # #source "${COMMONS_PATH}/whitesource/whitesource_unified_agent_scan.sh"
+    # source ./scripts/pipeline/whitesource_unified_agent_scan.sh
 
 sign-artifact:
   abort_on_failure: false
@@ -201,87 +197,151 @@ sign-artifact:
     #!/usr/bin/env bash
     echo "sign-artifact"
 
+    PERIODIC_SCAN=$(get_env periodic-rescan)
+    PERIODIC_SCAN="$(echo "$PERIODIC_SCAN" | tr '[:upper:]' '[:lower:]')"
+
+    if [[ ! -z "$PERIODIC_SCAN" && "$PERIODIC_SCAN" != "false" && "$PERIODIC_SCAN" != "no"  ]]; then
+      echo "Skipping sign-artifact. This is a periodic run that is only meant to produce CVE information."
+      exit 0
+    fi
+
 deploy:
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
+
   script: |
     #!/usr/bin/env bash
 
+
+    PERIODIC_SCAN=$(get_env periodic-rescan)
+    PERIODIC_SCAN="$(echo "$PERIODIC_SCAN" | tr '[:upper:]' '[:lower:]')"
+
+    if [[ ! -z "$PERIODIC_SCAN" && "$PERIODIC_SCAN" != "false" && "$PERIODIC_SCAN" != "no"  ]]; then
+      echo "Skipping deploy. This is a periodic run that is only meant to produce CVE information."
+      exit 0
+    fi
+
     if [[ "$PIPELINE_DEBUG" == 1 ]]; then
       trap env EXIT
       env
       set -x
     fi
     echo "deploy"
 
-dynamic-scan: 
+dynamic-scan:
   abort_on_failure: false
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
   script: |
     #!/usr/bin/env bash
     echo "dynamic-scan"
+    PERIODIC_SCAN=$(get_env periodic-rescan)
+    PERIODIC_SCAN="$(echo "$PERIODIC_SCAN" | tr '[:upper:]' '[:lower:]')"
+
+    if [[ ! -z "$PERIODIC_SCAN" && "$PERIODIC_SCAN" != "false" && "$PERIODIC_SCAN" != "no"  ]]; then
+      echo "Skipping dynamic-scan. This is a periodic run that is only meant to produce CVE information."
+      exit 0
+    fi
+
+    #export APP_URL=$(cat ../app-url)
+    # feature preview this until evidence locker v2 usage is full feature ready
+    # can be triggered, and owasp will run for preview purposes
+    #source scripts/zap/trigger_api_scan.sh
 
 acceptance-test:
-  dind: true  
+  dind: true
   abort_on_failure: true
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
   script: |
     #!/usr/bin/env bash
     echo "acceptance-test"
-
     PERIODIC_SCAN=$(get_env periodic-rescan)
     PERIODIC_SCAN="$(echo "$PERIODIC_SCAN" | tr '[:upper:]' '[:lower:]')"
-
     if [[ ! -z "$PERIODIC_SCAN" && "$PERIODIC_SCAN" != "false" && "$PERIODIC_SCAN" != "no"  ]]; then
-      echo "Skipping e2e-tests. This is a periodic run that is only meant to produce CVE information."
+      echo "Skipping acceptance-test. This is a periodic run that is only meant to produce CVE information."
       exit 0
     fi
-
+    SKIP_KIND_E2E_TEST=$(get_env SKIP_KIND_E2E_TEST)
+    SKIP_KIND_E2E_TEST="$(echo "$SKIP_KIND_E2E_TEST" | tr '[:upper:]' '[:lower:]')"
+    if [[ ! -z "$SKIP_KIND_E2E_TEST" && "$SKIP_KIND_E2E_TEST" != "false" && "$SKIP_KIND_E2E_TEST" != "no"  ]]; then
+      echo "Skipping acceptance-test, SKIP_KIND_E2E_TEST=$SKIP_KIND_E2E_TEST"
+      exit 0
+    fi
+    # Download and configure golang
     GO_VERSION=$(get_env go-version)
     if [[ -z "${GO_VERSION}" ]]; then
       GO_VERSION="$(grep '^go [0-9]\+.[0-9]\+' go.mod | cut -d ' ' -f 2)"
     fi
+    export GO_VERSION
+    # OCP test
+    export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
+    export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key-staging)  
+    export PIPELINE_REGISTRY=$(get_env pipeline-registry)
+    export PIPELINE_OPERATOR_IMAGE=$(get_env pipeline-operator-image)
+    export DOCKER_USERNAME=$(get_env docker-username)
+    export DOCKER_PASSWORD=$(get_env docker-password)    
+    export CLUSTER_URL=$(get_env test-cluster-url)
+    export CLUSTER_USER=$(get_env test-cluster-user kubeadmin)
+    export CLUSTER_TOKEN=$(get_env test-cluster-token)
+    export RELEASE_TARGET=$(get_env branch)
+    export DEBUG_FAILURE=$(get_env debug-failure)
+    # Kind test
+    export FYRE_USER=$(get_env fyre-user)
+    export FYRE_KEY=$(get_env fyre-key)
+    export FYRE_PASS=$(get_env fyre-pass)
+    export FYRE_PRODUCT_GROUP_ID=$(get_env fyre-product-group-id)
+    scripts/acceptance-test.sh
+
 
-    export SKIP_KIND_E2E_TEST=$(get_env SKIP_KIND_E2E_TEST)
-    if [[ $SKIP_KIND_E2E_TEST != "true" ]]; then
-
-      # Download and configure golang
-      rm -rf /usr/local/go && wget --no-verbose --header "Accept: application/octet-stream" "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" -O - | tar -xz -C /usr/local/
-      export PATH=$PATH:/usr/local/go/bin
-
-      apt-get update
-      apt-get -y install build-essential    
-      export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
-      export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key-staging)  
-      export DOCKER_USERNAME=$(get_env docker-username)
-      export DOCKER_PASSWORD=$(get_env docker-password)    
-      export CLUSTER_URL=$(get_env test-cluster-url)
-      export CLUSTER_TOKEN=$(get_env test-cluster-token) 
-      export TRAVIS_BUILD_NUMBER=$BUILD_NUMBER   
-      make setup
-      make test-pipeline-e2e
-    else
-      echo "skipping Acceptance test"
-    fi
 
 scan-artifact:
   abort_on_failure: false
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.15
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
   script: |
     #!/usr/bin/env bash
+    # echo "twistlock-scan"
+    # ./scripts/pipeline/twistlock-scan.sh
+    # echo "VA scan"
+    # . scripts/pipeline/va_scan
+    # if which list_artifacts >/dev/null; then
+    #   list_artifacts | while IFS= read -r artifact; do
+    #     image="$(load_artifact "$artifact" "name")"
+    #     type="$(load_artifact "$artifact" "type")"
+    #     digest="$(load_artifact "$artifact" "digest")"
+    #     name="$(echo "$artifact" | awk '{print $1}')"
+    #     if [[ "$type" == "image" ]]; then
+    #       if [[ "$image" == *"icr.io"* ]]; then
+    #         echo "Starting VA scan for $image"
+    #         start_va_scan "$name" "$image" "$digest"
+    #       else
+    #         echo "Skipping VA scan for $image"
+    #       fi
+    #     fi
+    #   done
+    # fi
+
+    # echo "aqua scan"
+    # # install docker
+    # curl -fsSL https://get.docker.com -o get-docker.sh
+    # sudo sh get-docker.sh
+    # # get aqua scan executables
+    # git clone https://$(get_env git-token)@github.ibm.com/CICD-CPP/cpp-pipelines.git
+    # chmod -R +x cpp-pipelines
+    # # setup and execute aqua scan
+    # cd cpp-pipelines
+    # export CUSTOM_SCRIPTS_PATH=/workspace/app/one-pipeline-config-repo/cpp-pipelines
+    # ./commons/aqua/aqua-local-scan
     # ========== Security Scanner ==========
     ./scripts/pipeline/ci_to_secure_pipeline_scan.sh
 
 release:
   abort_on_failure: false
-  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
   script: |
     #!/usr/bin/env bash
-
     PERIODIC_SCAN=$(get_env periodic-rescan)
     PERIODIC_SCAN="$(echo "$PERIODIC_SCAN" | tr '[:upper:]' '[:lower:]')"
 
     if [[ ! -z "$PERIODIC_SCAN" && "$PERIODIC_SCAN" != "false" && "$PERIODIC_SCAN" != "no"  ]]; then
-      echo "Skipping release stage. This is a periodic run that is only meant to produce CVE information."
+      echo "Skipping release. This is a periodic run that is only meant to produce CVE information."
       exit 0
     fi
 

Dockerfile.e2e

@@ -0,0 +1,17 @@
+FROM ubuntu:22.04
+
+ARG GO_VERSION
+
+env PATH=$PATH:/usr/local/go/bin
+
+WORKDIR /workspace
+
+RUN apt update -y && apt install -y curl sudo gpg make build-essential wget
+
+RUN wget --no-verbose --header "Accept: application/octet-stream" "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" -O - | tar -xz -C /usr/local/
+
+COPY . .
+
+RUN scripts/installers/install-operator-sdk.sh
+
+CMD [ "bash" ]

Makefile

@@ -266,11 +266,13 @@ bundle-build: ## Build the bundle image.
 
 .PHONY: bundle-push
 bundle-push: ## Push the bundle image.
-        $(CONTAINER_COMMAND) push $(PODMAN_SKIP_TLS_VERIFY)  "${PUBLISH_REGISTRY}/${BUNDLE_IMG}"
+	$(CONTAINER_COMMAND) push $(PODMAN_SKIP_TLS_VERIFY)  "${BUNDLE_IMG}"
 
 build-manifest: setup-manifest
 	./scripts/build-manifest.sh --image "${PUBLISH_REGISTRY}/${OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
 
+kind-e2e-test:
+	./scripts/e2e-kind.sh --test-tag "${TRAVIS_BUILD_NUMBER}"
 build-pipeline-manifest: setup-manifest
 	./scripts/build-manifest.sh -u "${PIPELINE_USERNAME}" -p "${PIPELINE_PASSWORD}" --registry "${PIPELINE_REGISTRY}" --image "${PIPELINE_REGISTRY}/${PIPELINE_OPERATOR_IMAGE}"	--target "${RELEASE_TARGET}"
 
@@ -292,10 +294,10 @@ test-e2e:
 
 test-pipeline-e2e:
 	./scripts/pipeline/fyre-e2e.sh -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" \
-                     --cluster-url "${CLUSTER_URL}" --cluster-token "${CLUSTER_TOKEN}" \
-                     --registry-name "${PIPELINE_REGISTRY}" --registry-namespace "${PIPELINE_REGISTRY_NAMESPACE}" \
-					 --registry-user "${PIPELINE_USERNAME}" --registry-password "${PIPELINE_PASSWORD}" \
-                     --test-tag "${TRAVIS_BUILD_NUMBER}" --release "${RELEASE_TARGET}"					 
+                     --cluster-url "${CLUSTER_URL}" --cluster-user "${CLUSTER_USER}" --cluster-token "${CLUSTER_TOKEN}" \
+                     --registry-name "${PIPELINE_REGISTRY}" --registry-image "${PIPELINE_OPERATOR_IMAGE}" \
+                     --registry-user "${PIPELINE_USERNAME}" --registry-password "${PIPELINE_PASSWORD}" \
+                     --test-tag "${TRAVIS_BUILD_NUMBER}" --release "${RELEASE_TARGET}" --channel "${DEFAULT_CHANNEL}"
 
 build-releases:
 	./scripts/build-releases.sh --image "${PUBLISH_REGISTRY}/${OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
@@ -316,13 +318,13 @@ bundle-push-podman:
 	podman push --format=docker "${BUNDLE_IMG}"
 
 build-catalog:
-	opm index add --bundles "${BUNDLE_IMG}" --tag "${CATALOG_IMG}"
+	opm index add --bundles "${BUNDLE_IMG}" --tag "${CATALOG_IMG}" -c docker
 
 push-catalog: 
-	podman push --format=docker "${CATALOG_IMG}"
+	docker push "${CATALOG_IMG}"
 
 push-pipeline-catalog: 
-	podman push --format=docker "${CATALOG_IMG}"
+	docker push "${CATALOG_IMG}"
 
 # Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'.
 # This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see:

bundle/tests/scorecard/kind-kuttl/ingress-certificate/00-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/tls
+metadata:
+  name: ingress-tls-secret
+data:
+  ca.crt: Y2FjcnQK
+  destCA.crt: ZGVzdENBY3J0Cg==
+  tls.crt: dGxzY3J0Cg==
+  tls.key: dGxza2V5Cg==