From 29e8c4b65ed042385706c4c0250dbce856c2f819 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 14:44:14 +0300 Subject: [PATCH 01/17] DEV-3379 testing workflow --- .../build-and-test-harbor-canary.yaml | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 .github/workflows/build-and-test-harbor-canary.yaml diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml new file mode 100644 index 00000000..e4b16fd1 --- /dev/null +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -0,0 +1,172 @@ +name: Build and test canary module + +on: + workflow_call: + inputs: + is_stage: + description: "Запускается ли план для публикации на стейдж" + required: true + type: boolean + project_path_mask: + description: "Маска пути к файлам проекта" + required: true + type: string + with_push: + description: "Пушить ли собранный образ" + required: true + type: boolean + runner: + description: "Указывает какой раннер будет использоваться для запуска флоу" + default: "sravni_dev" + required: false + type: string + image_version: + description: "Версия образа" + required: false + type: string + k8s_yaml_file_name: + description: "Имя файла {environment}.yaml в директории .k8s проекта для валидации его наличия и передачи следующим джобам" + required: true + type: string + canary: + description: "Включить канарейку" + required: false + type: boolean + default: false + + secrets: + GH_TOKEN: + required: true + GITHUB_NPM_TOKEN: + required: true + HARBOR_REGISTRY: + required: true + HARBOR_LOGIN: + required: true + HARBOR_PASS: + required: true + NEXTAUTH_URL: # Добавлено для команды MFO + required: false # + NEXTAUTH_SECRET: # Добавлено для команды MFO + required: false # + SLACK_CLIENT_SECRET: # Добавлено для команды MFO + required: false # + SLACK_CLIENT_ID: # Добавлено для команды MFO + required: false # + ALLURE_TOKEN: # Добавлено для команды Mobile + required: false # + SENTRY_SOURCEMAP_WEB_FRONTEND_AUTH_TOKEN: # AUTH токен для source map сентри, добавлено для команды OSAGO + required: false + + outputs: + projects: + description: "Собранные проекты" + value: ${{ jobs.get_projects.outputs.projects }} + +env: + CANARY: ${{ inputs.canary }} + +# Описание Джобов воркфлоу +jobs: + # Описание Джобы get_projects и ее шага, в котором происходит составление + # списка проектов для сборки. + # | project_name | dockerfile | project_path | k8s_yaml_path | build_number | image_name | image_version | + get_projects: + uses: ./.github/workflows/get-projects.yaml + with: + k8s_yaml_file_name: ${{ inputs.k8s_yaml_file_name }} + project_path_mask: ${{ inputs.project_path_mask }} + image_version: ${{ inputs.image_version }} + is_stage: ${{ inputs.is_stage }} + runner: ${{ inputs.runner }} + + build_and_test: + runs-on: ${{ inputs.runner }} + needs: [get_projects] + + strategy: + matrix: + project: ${{ fromJson(needs.get_projects.outputs.projects) }} + + steps: + - name: Set job data for metrics + id: data + run: | + echo "repository=${{ matrix.project.image_name }}" >> $GITHUB_OUTPUT + echo "env=${{ inputs.is_stage && 'stage' || 'production' }}" >> $GITHUB_OUTPUT + echo "start_timestamp=$(date +%s)" >> $GITHUB_OUTPUT + continue-on-error: true + + - name: Checkout source + uses: actions/checkout@v3 + + - name: Login to ACR + uses: docker/login-action@v2 + with: + registry: ${{ secrets.HARBOR_REGISTRY }} + username: ${{ secrets.HARBOR_LOGIN }} + password: ${{ secrets.HARBOR_PASS }} + + - name: Get build number and image version + run: | + if [[ "${{ inputs.canary }}" == 'true' ]] + then + canary_tag="canary-" + else + canary_tag="" + fi + echo "CANARY_TAG=$canary_tag" >> $GITHUB_ENV + + - name: Check if image exist + run: | + inspect_result=$(docker manifest inspect "${{ secrets.HARBOR_REGISTRY }}/${{ env.CANARY_TAG }}${{ matrix.project.image_name }}" > /dev/null 2>&1; echo $?) + if [ $inspect_result -eq 1 ] + then + echo "IMAGE_EXIST=false" >> $GITHUB_ENV + else + echo "IMAGE_EXIST=true" >> $GITHUB_ENV + echo "Образ ${{ matrix.project.image_name }} уже cуществует" + fi + + - name: Set up Docker Buildx + if: ${{ env.IMAGE_EXIST == 'false' }} + uses: docker/setup-buildx-action@v2 + with: + driver: docker + version: v0.10.0 + + - name: Docker build + if: ${{ env.IMAGE_EXIST == 'false' }} + uses: docker/build-push-action@v4 + with: + context: . + file: ${{ matrix.project.dockerfile }} + build-args: | + build_number=${{ matrix.project.build_number }} + ENVIROMENT=${{ inputs.is_stage && 'stage' || 'production' }} + GITHUB_TOKEN=${{ secrets.GH_TOKEN }} + GITHUB_USERNAME=sravni + GITHUB_NPM_TOKEN=${{ secrets.GH_TOKEN }} + NEXTAUTH_URL=${{ secrets.NEXTAUTH_URL }} + NEXTAUTH_SECRET=${{ secrets.NEXTAUTH_SECRET }} + SLACK_CLIENT_SECRET=${{ secrets.SLACK_CLIENT_SECRET }} + SLACK_CLIENT_ID=${{ secrets.SLACK_CLIENT_ID }} + ALLURE_TOKEN=${{ secrets.ALLURE_TOKEN }} + SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_SOURCEMAP_WEB_FRONTEND_AUTH_TOKEN }} + push: ${{ inputs.with_push }} + tags: "${{ secrets.HARBOR_REGISTRY }}/${{ matrix.project.image_name }}" + + + - name: Calculate job duration + if: always() + id: job_duration + run: | + echo "result=$(( $(date +%s) - ${{steps.data.outputs.start_timestamp}} ))" >> $GITHUB_OUTPUT + continue-on-error: true + + - name: Send metrics + if: always() + run : | + curl -d 'sravni_jobs{job="${{ github.job }}", result="${{ job.status }}", repository="${{ github.event.repository.name }}", env="${{ steps.data.outputs.env }}"} ${{steps.job_duration.outputs.result}}' -X POST https://vm-receive.yc.dev.infra.sravni-team.ru/insert/0/prometheus/api/v1/import/prometheus + curl -d 'sravni_jobs_event{job="${{ github.job }}", result="${{ job.status }}", repository="${{ github.event.repository.name }}", env="${{ steps.data.outputs.env }}"} 1' -X POST https://vm-receive.yc.dev.infra.sravni-team.ru/insert/0/prometheus/api/v1/import/prometheus + continue-on-error: true From 42440c450cc3cd45293368998189cd44f2a9ada4 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 15:05:26 +0300 Subject: [PATCH 02/17] DEV-3379 testing canary workflow --- .github/workflows/build-and-test-harbor-canary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index e4b16fd1..c639ddfe 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -107,7 +107,7 @@ jobs: username: ${{ secrets.HARBOR_LOGIN }} password: ${{ secrets.HARBOR_PASS }} - - name: Get build number and image version + - name: Check if image is a canary run: | if [[ "${{ inputs.canary }}" == 'true' ]] then From a6ee922243e082190e1b148e1779cb62ae23aff5 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 15:08:46 +0300 Subject: [PATCH 03/17] DEV-3379 testing canary workflow --- .github/workflows/build-and-test-harbor-canary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index c639ddfe..12d21b91 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -32,7 +32,7 @@ on: description: "Включить канарейку" required: false type: boolean - default: false + default: true secrets: GH_TOKEN: From 6923d8715bddbf964026fcd68bf830c95034c3ae Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 15:15:02 +0300 Subject: [PATCH 04/17] DEV-3379 testing canary workflow --- .github/workflows/build-and-test-harbor-canary.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 12d21b91..11432617 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -32,7 +32,7 @@ on: description: "Включить канарейку" required: false type: boolean - default: true + default: false secrets: GH_TOKEN: @@ -109,7 +109,7 @@ jobs: - name: Check if image is a canary run: | - if [[ "${{ inputs.canary }}" == 'true' ]] + if [[ "${{ env.CANARY }}" == 'true' ]] then canary_tag="canary-" else From ecc58e51e7dde9e6b00434c73556cb9d56cdc0a9 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 15:28:34 +0300 Subject: [PATCH 05/17] DEV-3379 testing canary workflow --- .github/workflows/build-and-test-harbor-canary.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 11432617..3b9ed6e9 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -108,8 +108,11 @@ jobs: password: ${{ secrets.HARBOR_PASS }} - name: Check if image is a canary + with: + canary: ${{ inputs.canary}} run: | - if [[ "${{ env.CANARY }}" == 'true' ]] + echo $canary + if [[ "${{ canary }}" == 'true' ]] then canary_tag="canary-" else From 1d11ca638226c873a9a3658a0d606500ad5c94d6 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 15:31:05 +0300 Subject: [PATCH 06/17] DEV-3379 testing canary workflow --- .github/workflows/build-and-test-harbor-canary.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 3b9ed6e9..0c4cf7ff 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -108,9 +108,9 @@ jobs: password: ${{ secrets.HARBOR_PASS }} - name: Check if image is a canary - with: - canary: ${{ inputs.canary}} run: | + echo ${{ inputs.canary }} + echo ${{ env.CANARY }} echo $canary if [[ "${{ canary }}" == 'true' ]] then From 53163789c0a6f7e221b07df5c01e892bd136a644 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 15:32:30 +0300 Subject: [PATCH 07/17] DEV-3379 testing canary workflow --- .github/workflows/build-and-test-harbor-canary.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 0c4cf7ff..674b2f3c 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -111,7 +111,6 @@ jobs: run: | echo ${{ inputs.canary }} echo ${{ env.CANARY }} - echo $canary if [[ "${{ canary }}" == 'true' ]] then canary_tag="canary-" From 82f63c349e4cd4d63b3490987cd0479262d946c4 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Tue, 2 Apr 2024 15:33:52 +0300 Subject: [PATCH 08/17] DEV-3379 testing canary workflow --- .github/workflows/build-and-test-harbor-canary.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 674b2f3c..4f47463f 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -109,9 +109,9 @@ jobs: - name: Check if image is a canary run: | - echo ${{ inputs.canary }} echo ${{ env.CANARY }} - if [[ "${{ canary }}" == 'true' ]] + echo ${{ inputs.CANARY }} + if [[ "${{ env.CANARY }}" == 'true' ]] then canary_tag="canary-" else From fdfcd51250ae681aa9b8a0f51c9b29f9dd3e84de Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 08:52:43 +0300 Subject: [PATCH 09/17] DEV-3379 fixing flow --- .github/workflows/build-and-test-harbor-canary.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 4f47463f..29cfc82d 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -31,7 +31,7 @@ on: canary: description: "Включить канарейку" required: false - type: boolean + type: string default: false secrets: @@ -111,6 +111,7 @@ jobs: run: | echo ${{ env.CANARY }} echo ${{ inputs.CANARY }} + echo $canary if [[ "${{ env.CANARY }}" == 'true' ]] then canary_tag="canary-" From 176f30a1f84fa57c4926042a82b16c54d02db910 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 09:00:42 +0300 Subject: [PATCH 10/17] DEV-3379 fixing flow --- .github/workflows/publish-to-prod-harbor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-prod-harbor.yaml b/.github/workflows/publish-to-prod-harbor.yaml index 19b4fef5..df8d0b5b 100644 --- a/.github/workflows/publish-to-prod-harbor.yaml +++ b/.github/workflows/publish-to-prod-harbor.yaml @@ -27,7 +27,7 @@ on: canary: required: false default: false - type: boolean + type: string secrets: TEST_CI_PAT_FOR_ACTIONS: required: true From 96fe5e3b13444451e4f9eb7dc8c4c3c0a07568e3 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 13:13:05 +0300 Subject: [PATCH 11/17] DEV-3379 testing canary flow --- .github/workflows/build-and-test-harbor-canary.yaml | 8 ++++---- .github/workflows/publish-to-prod-harbor.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 29cfc82d..e928a1a2 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -109,9 +109,9 @@ jobs: - name: Check if image is a canary run: | - echo ${{ env.CANARY }} - echo ${{ inputs.CANARY }} - echo $canary + echo env.CANARY ${{ env.CANARY }} + echo inputs.CANARY ${{ inputs.CANARY }} + echo canary $canary if [[ "${{ env.CANARY }}" == 'true' ]] then canary_tag="canary-" @@ -157,7 +157,7 @@ jobs: ALLURE_TOKEN=${{ secrets.ALLURE_TOKEN }} SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_SOURCEMAP_WEB_FRONTEND_AUTH_TOKEN }} push: ${{ inputs.with_push }} - tags: "${{ secrets.HARBOR_REGISTRY }}/${{ matrix.project.image_name }}" + tags: "${{ secrets.HARBOR_REGISTRY }}/${{ env.CANARY_TAG }}${{ matrix.project.image_name }}" - name: Calculate job duration diff --git a/.github/workflows/publish-to-prod-harbor.yaml b/.github/workflows/publish-to-prod-harbor.yaml index df8d0b5b..f1a24d16 100644 --- a/.github/workflows/publish-to-prod-harbor.yaml +++ b/.github/workflows/publish-to-prod-harbor.yaml @@ -140,7 +140,7 @@ jobs: if [ "${{ env.CANARY }}" = true ]; then yq -i '.global.image.tag = "${{ matrix.project.image_version }}" | - .global.image.repository = "registry.yc.prod.infra.sravni.market/${{ env.TEAM }}_prod/${{ matrix.project.project_name }}" | + .global.image.repository = "registry.yc.prod.infra.sravni.market/${{ env.TEAM }}_prod/canary-${{ matrix.project.project_name }}" | .global.imagePullSecrets[0] = "harbor-pull-secret"' ${{ env.REPO_PATH }}/${{ matrix.project.project_name }}-canary/values_prod.yaml; fi; From c716a548ce0e2835eb353d9755eb7fcaeb54806a Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 13:20:30 +0300 Subject: [PATCH 12/17] DEV-3379 testing canary flow --- .github/workflows/publish-to-prod-harbor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-prod-harbor.yaml b/.github/workflows/publish-to-prod-harbor.yaml index f1a24d16..aa9ef085 100644 --- a/.github/workflows/publish-to-prod-harbor.yaml +++ b/.github/workflows/publish-to-prod-harbor.yaml @@ -26,7 +26,7 @@ on: type: string canary: required: false - default: false + default: '1' type: string secrets: TEST_CI_PAT_FOR_ACTIONS: From 036a25967cd90d4a3701a79e8576f750c6739cc4 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 13:50:56 +0300 Subject: [PATCH 13/17] DEV-3379 fixing canary flow --- .github/workflows/build-and-test-harbor-canary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index e928a1a2..7b57750e 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -31,7 +31,7 @@ on: canary: description: "Включить канарейку" required: false - type: string + type: boolean default: false secrets: From fc0470110a035be39779262ddc76e6affd4d1a2d Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 14:01:09 +0300 Subject: [PATCH 14/17] DEV-3379 fixing canary flow --- .github/workflows/build-and-test-harbor-canary.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 7b57750e..6b904eaf 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -31,8 +31,8 @@ on: canary: description: "Включить канарейку" required: false - type: boolean - default: false + type: string + default: "false" secrets: GH_TOKEN: From 5814d1d11814c3d03fbd156adcd1468176cf18dd Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 14:05:46 +0300 Subject: [PATCH 15/17] DEV-3379 fixing canary flow --- .github/workflows/build-and-test-harbor-canary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 6b904eaf..e928a1a2 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -32,7 +32,7 @@ on: description: "Включить канарейку" required: false type: string - default: "false" + default: false secrets: GH_TOKEN: From bf3958c168abb66be933723b6d512563ee846333 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 14:07:58 +0300 Subject: [PATCH 16/17] DEV-3379 fixing canary flow --- .github/workflows/build-and-test-harbor-canary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index e928a1a2..6b904eaf 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -32,7 +32,7 @@ on: description: "Включить канарейку" required: false type: string - default: false + default: "false" secrets: GH_TOKEN: From 1ace11d701742c7948321203792e700cd2beae68 Mon Sep 17 00:00:00 2001 From: Ivan Lobov Date: Wed, 3 Apr 2024 14:17:42 +0300 Subject: [PATCH 17/17] DEV-3379 fixing canary flow --- .github/workflows/build-and-test-harbor-canary.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-harbor-canary.yaml b/.github/workflows/build-and-test-harbor-canary.yaml index 6b904eaf..7b57750e 100644 --- a/.github/workflows/build-and-test-harbor-canary.yaml +++ b/.github/workflows/build-and-test-harbor-canary.yaml @@ -31,8 +31,8 @@ on: canary: description: "Включить канарейку" required: false - type: string - default: "false" + type: boolean + default: false secrets: GH_TOKEN: