feat(integration-tests): add ATP Storage S3 integration

.github/workflows/build.yaml

@@ -71,13 +71,17 @@ jobs:
             components=$(jq -c '.components' .github/build-config.cfg)
           fi
           if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "push" ]]; then
-            all_changed_files=$(jq -c '.' .github/outputs/all_changed_files.json)
+            all_changed_files=$(jq -c '.' .github/outputs/all_changed_files.json 2>/dev/null || echo "null")
             if [ "$all_changed_files" != "null" ]; then
                 echo "all_changed_files=${all_changed_files}"
                 chmod +x .github/scripts/matrix.sh
                 components=$(./.github/scripts/matrix.sh ".github/build-config.cfg" ".github/outputs/all_changed_files.json")
             fi
           fi
+          if [ -z "$components" ] || [ "$components" = "[]" ]; then
+            echo "No components from matrix, using all from build-config.cfg"
+            components=$(jq -c '.components' .github/build-config.cfg)
+          fi
           echo "components=${components}"
           echo "components=${components}" >> "$GITHUB_OUTPUT"
           echo "platforms=$(jq -c '.platforms' .github/build-config.cfg)" >> "$GITHUB_OUTPUT"
@@ -122,6 +126,8 @@ jobs:
           password: ${{secrets.GITHUB_TOKEN}}
       - name: Prepare Tag
         run: echo "TAG_NAME=$(echo ${TAG_NAME} | sed 's@refs/tags/@@;s@refs/heads/@@;s@/@_@g')" >> $GITHUB_ENV
+      - name: Prepare registry (fork or upstream)
+        run: echo "GITHUB_GROUP=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
       - name: Get package IDs for delete
         id: get-ids-for-delete
         uses: Netcracker/get-package-ids@84bc8eb8bed50218be76e671b3a24c35a1300979
@@ -139,9 +145,10 @@ jobs:
           platforms: ${{ needs.prepare.outputs.platforms }}
           push: ${{ env.PUSH }}
           build-args: PG_VERSION=${{ matrix.component.pg_version }}
-          tags: ghcr.io/netcracker/${{ matrix.component.name }}:${{ env.TAG_NAME }}
+          tags: ghcr.io/${{ env.GITHUB_GROUP }}/${{ matrix.component.name }}:${{ env.TAG_NAME }}
           provenance: false
       - uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0
+        continue-on-error: true
         with:
           package-name: ${{ matrix.component.name }}
           package-type: 'container'

.github/workflows/upgrade_patroni_core_env_to_aws.yaml

@@ -0,0 +1,98 @@
+# Upgrade only the patroni-core Helm release to inject ATP-related settings into the CR (spec.integrationTests).
+# Use when the CR was installed without ATP/S3 values and you don't want a full reinstall.
+name: Upgrade PatroniCore env to AWS
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Environment Name (must have vars/secrets set)"
+        required: false
+        default: "dev"
+      repository:
+        description: "Repo to checkout. Empty = this repo."
+        required: false
+        default: ""
+      ref:
+        description: "Branch or ref to checkout"
+        required: false
+        default: "main"
+
+jobs:
+  Upgrade-PatroniCore-Env:
+    environment: "${{ github.event.inputs.environment }}"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.inputs.repository || github.repository }}
+          ref: ${{ github.event.inputs.ref || github.ref }}
+
+      - name: Validate required Environment variables
+        run: |
+          MISSING=""
+          [ -z "${{ vars.PGSKIPPER_INSTALL_NAMESPACE }}" ] && MISSING="${MISSING} PGSKIPPER_INSTALL_NAMESPACE"
+          [ -z "${{ vars.AWS_CLUSTERNAME }}" ] && MISSING="${MISSING} AWS_CLUSTERNAME"
+          [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] && MISSING="${MISSING} AWS_ACCESS_KEY_ID"
+          [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ] && MISSING="${MISSING} AWS_SECRET_ACCESS_KEY"
+          if [ -n "$MISSING" ]; then
+            echo "::error::Missing:$MISSING"
+            exit 1
+          fi
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v3
+        with:
+          version: 'v1.32.0'
+
+      - name: Setup Helm
+        uses: azure/setup-helm@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
+
+      - name: Configure kubeconfig for EKS
+        run: |
+          aws eks update-kubeconfig --region ${{ vars.AWS_REGION || 'us-east-1' }} --name ${{ vars.AWS_CLUSTERNAME }}
+
+      - name: Upgrade patroni-core (inject ATP Helm values)
+        run: |
+          NS="${{ vars.PGSKIPPER_INSTALL_NAMESPACE }}"
+          GITHUB_GROUP=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')
+          IMAGE_TAG=$(echo ${{ github.ref }} | sed 's@refs/tags/@@;s@refs/heads/@@;s@/@_@g')
+          ENV_NAME="${{ vars.ENVIRONMENT_NAME }}"
+          [ -z "$ENV_NAME" ] && ENV_NAME="${{ github.event.inputs.environment }}"
+          AWS_REGION="${{ vars.AWS_REGION }}"
+          [ -z "$AWS_REGION" ] && AWS_REGION="us-east-1"
+          ATP_PROVIDER="${{ vars.ATP_STORAGE_PROVIDER }}"
+          [ -z "$ATP_PROVIDER" ] && ATP_PROVIDER="aws"
+
+          TESTS_ATP_ARGS="--set tests.dockerImage=ghcr.io/${GITHUB_GROUP}/pgskipper-operator-tests:${IMAGE_TAG}"
+          TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpStorage.provider=${ATP_PROVIDER}"
+          TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpStorage.region=${AWS_REGION}"
+          [ -n "${{ secrets.S3_AWS_ACCESS_KEY_ID }}" ] && TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpStorage.username=${{ secrets.S3_AWS_ACCESS_KEY_ID }}"
+          [ -n "${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}" ] && TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpStorage.password=${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}"
+          [ -n "${{ vars.ATP_STORAGE_SERVER_URL }}" ] && TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpStorage.serverUrl=${{ vars.ATP_STORAGE_SERVER_URL }}"
+          [ -n "${{ vars.ATP_STORAGE_SERVER_UI_URL }}" ] && TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpStorage.serverUiUrl=${{ vars.ATP_STORAGE_SERVER_UI_URL }}"
+          [ -n "${{ vars.ATP_STORAGE_BUCKET }}" ] && TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpStorage.bucket=${{ vars.ATP_STORAGE_BUCKET }}"
+          [ -n "${{ vars.ATP_REPORT_VIEW_UI_URL }}" ] && TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.atpReportViewUiUrl=${{ vars.ATP_REPORT_VIEW_UI_URL }}"
+          [ -n "$ENV_NAME" ] && TESTS_ATP_ARGS="$TESTS_ATP_ARGS --set tests.environmentName=$ENV_NAME"
+
+          echo "Upgrading pgskipper-patroni in $NS with TESTS_ATP_ARGS..."
+          # -f values.yaml ensures chart defaults (tests.atpStorage, etc.) are in the merge; --reuse-values then overlay; --set overrides.
+          helm upgrade --namespace="$NS" --install \
+            -f ./operator/charts/patroni-core/values.yaml \
+            --reuse-values \
+            $TESTS_ATP_ARGS \
+            pgskipper-patroni ./operator/charts/patroni-core
+
+          echo "--- PatroniCore spec.integrationTests ---"
+          kubectl get patronicore -n "$NS" -o yaml | grep -A200 "integrationTests:" | head -80
+
+          echo "Delete the test pod so the operator recreates it with env: kubectl delete pod integration-robot-tests -n $NS"

README.md

@@ -14,6 +14,22 @@ Postgres-Operator provides PostgreSQL as a service on Kubernetes and OpenShift.
 
 Please refer to the [Quick Start Guide](/docs/public/quickstart.md)
 
+### Integration tests and ATP storage
+
+Integration test settings live under `tests` in the Helm values for **patroni-core** and **patroni-services** (see [`operator/charts/patroni-core/values.yaml`](operator/charts/patroni-core/values.yaml) and [`operator/charts/patroni-services/values.yaml`](operator/charts/patroni-services/values.yaml)). The test image is based on [qubership-docker-integration-tests](https://github.com/Netcracker/qubership-docker-integration-tests). Optional `tests.atpStorage`, `tests.atpReportViewUiUrl`, and `tests.environmentName` map to the same `ATP_*` / `ENVIRONMENT_NAME` variables as in other Qubership demos (Consul `integrationTests.*`, RabbitMQ `tests.*`). The Patroni Services chart renders these into the custom resource (`operator/charts/patroni-services/templates/cr.yaml`).
+
+| Value (Helm)                 | Description |
+|------------------------------|-------------|
+| `tests.atpStorage.provider`  | S3 provider (for example `aws`, `minio`, `s3`). When set, the chart can emit ATP storage environment variables for the test pod. |
+| `tests.atpStorage.serverUrl` | S3 API endpoint URL. |
+| `tests.atpStorage.serverUiUrl` | Optional storage UI URL. |
+| `tests.atpStorage.bucket`    | Bucket name; empty usually means no S3 upload in the base image flow. |
+| `tests.atpStorage.region`    | Region (for example for AWS). |
+| `tests.atpStorage.username`  | Access key (sensitive; prefer secrets / external overrides in real environments). |
+| `tests.atpStorage.password`  | Secret key (same as username). |
+| `tests.atpReportViewUiUrl`   | Optional Allure report UI base URL. |
+| `tests.environmentName`      | Optional logical name for paths or labels. |
+
 ### Smoke tests
 
 There is no smoke tests.

operator/charts/patroni-core/crds/netcracker.com_patronicores.yaml

@@ -2636,9 +2636,10 @@ spec:
                         operator:
                           description: |-
                             Operator represents a key's relationship to the value.
-                            Valid operators are Exists and Equal. Defaults to Equal.
+                            Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.
                             Exists is equivalent to wildcard for value, so that a pod can
                             tolerate all taints of a particular category.
+                            Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).
                           type: string
                         tolerationSeconds:
                           description: |-

operator/charts/patroni-core/templates/cr.yaml

@@ -40,9 +40,15 @@ spec:
     synchronousMode: {{ default "false" .Values.patroni.synchronousMode }}
     createEndpoint: {{ default "false" .Values.patroni.createEndpoint }}
     {{ end }}
-    {{ if .Values.patroni.fsGroup }}
-    fsGroup: {{ .Values.patroni.fsGroup }}
-    {{ end }}
+    {{- if or .Values.patroni.securityContext .Values.patroni.fsGroup }}
+    securityContext:
+      {{- if .Values.patroni.securityContext }}
+      {{- toYaml .Values.patroni.securityContext | nindent 6 }}
+      {{- end }}
+      {{- if .Values.patroni.fsGroup }}
+      fsGroup: {{ .Values.patroni.fsGroup }}
+      {{- end }}
+    {{- end }}
     {{ if .Values.patroni.affinity }}
     affinity: {{ .Values.patroni.affinity | toJson }}
     {{ else if .Values.affinity }}
@@ -243,6 +249,40 @@ spec:
         {{- end }}
 {{ end }}
     pgNodeQty: {{ ( include "postgres.replicasCount" . ) }}
+    {{- $atp := .Values.tests.atpStorage | default dict }}
+    env:
+    {{- if .Values.tests.environmentName }}
+    - name: ENVIRONMENT_NAME
+      value: {{ .Values.tests.environmentName | quote }}
+    {{- end }}
+    - name: ATP_STORAGE_PROVIDER
+      value: {{ ($atp.provider | default "aws") | quote }}
+    - name: ATP_STORAGE_REGION
+      value: {{ ($atp.region | default "us-east-1") | quote }}
+    {{- if $atp.serverUrl }}
+    - name: ATP_STORAGE_SERVER_URL
+      value: {{ $atp.serverUrl | quote }}
+    {{- end }}
+    {{- if $atp.serverUiUrl }}
+    - name: ATP_STORAGE_SERVER_UI_URL
+      value: {{ $atp.serverUiUrl | quote }}
+    {{- end }}
+    {{- if $atp.bucket }}
+    - name: ATP_STORAGE_BUCKET
+      value: {{ $atp.bucket | quote }}
+    {{- end }}
+    {{- if $atp.username }}
+    - name: ATP_STORAGE_USERNAME
+      value: {{ $atp.username | quote }}
+    {{- end }}
+    {{- if $atp.password }}
+    - name: ATP_STORAGE_PASSWORD
+      value: {{ $atp.password | quote }}
+    {{- end }}
+    {{- if .Values.tests.atpReportViewUiUrl }}
+    - name: ATP_REPORT_VIEW_UI_URL
+      value: {{ .Values.tests.atpReportViewUiUrl | quote }}
+    {{- end }}
 {{ end }}
 {{ if .Values.runTestsOnly }}
 runTestsTime: {{ now | unixEpoch | quote }}

operator/charts/patroni-core/templates/secrets/postgres-secret.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.patroni.install }}
+{{- if and .Values.patroni.install (ne (index .Values "createCredentials") false) }}
 apiVersion: v1
 kind: Secret
 metadata:

operator/charts/patroni-core/templates/secrets/replicator-secret.yaml

@@ -1,3 +1,4 @@
+{{- if ne (index .Values "createCredentials") false }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -10,3 +11,4 @@ data:
   password: {{ .Values.replicatorPassword | b64enc }}
   username: {{ "replicator" | b64enc }}
 type: Opaque
+{{- end }}

operator/charts/patroni-core/values.yaml

@@ -7,6 +7,9 @@ postgresUser: postgres
 # The password for the database superuser.
 postgresPassword: "p@ssWOrD1"
 replicatorPassword: "replicator"
+# Set to false when postgres-credentials and replicator-credentials are created outside the chart (e.g. by CI).
+# When false, the chart skips creating these secrets so Helm install does not conflict with existing resources.
+# createCredentials: true
 
 podLabels: {}
 
@@ -101,9 +104,8 @@ patroni:
   podLabels: {}
   podAnnotations: {}
   configMapAnnotations: {}
-  # patroni.fsGroup. Specifies fsGroup that will be used in Patroni Pod Spec.
-  # Useful in case of k8s installation and provisioned storage
-  #  fsGroup: 26
+  # patroni.fsGroup. Specifies fsGroup that will be used in Patroni Pod Spec (data volume writable by postgres GID 26).
+  fsGroup: 26
   #  affinity: {
   #    "podAffinity": {
   #      "preferredDuringSchedulingIgnoredDuringExecution": [
@@ -134,7 +136,6 @@ patroni:
   #      ]
   #    }
   #  }
-  #  fsGroup: 26
   # patroni.affinity parameter. Configures a `podAffinityTerm`. Possible values are required and preferred
 
   #  synchronousMode: true
@@ -176,7 +177,7 @@ patroni:
   patroniParams:
     - "failsafe_mode: true"
     - "primary_start_timeout: 30"
-    - "retry_timeout: 10"
+    - "retry_timeout: 600"
 
   # Storage section.
   storage:
@@ -274,6 +275,17 @@ tests:
   install: true
   dockerImage: ghcr.io/netcracker/pgskipper-operator-tests:main
   podLabels: {}
+  # S3/ATP storage for test results (defaults; override from GitHub vars/secrets in workflow)
+  atpStorage:
+    provider: "aws"
+    serverUrl: "https://s3.amazonaws.com"
+    serverUiUrl: "https://console.test.com"
+    bucket: ""
+    region: "us-east-1"
+    username: ""
+    password: ""
+  atpReportViewUiUrl: "https://test.com"
+  environmentName: "pgskipper-operator"
   # One of "full", "basic"or one from testScenarios
   runTestScenarios: "basic"
   testScenarios:

operator/charts/patroni-services/crds/netcracker.com_patroniservices.yaml

@@ -6139,9 +6139,10 @@ spec:
                         operator:
                           description: |-
                             Operator represents a key's relationship to the value.
-                            Valid operators are Exists and Equal. Defaults to Equal.
+                            Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.
                             Exists is equivalent to wildcard for value, so that a pod can
                             tolerate all taints of a particular category.
+                            Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).
                           type: string
                         tolerationSeconds:
                           description: |-

operator/charts/patroni-services/templates/_helpers.tpl

@@ -248,19 +248,20 @@ Get TLS secret name for services
 {{- end -}}
 
 {{/*
-Postgres host for backup daemon
+Postgres host for backup daemon. Uses FQDN (.svc.cluster.local) for reliable DNS in Kubernetes (e.g. EKS).
 */}}
 {{- define "backupDaemon.pgHost" -}}
+{{- $clusterDomain := default "cluster.local" .Values.backupDaemon.clusterDomain -}}
 {{- if .Values.connectionPooler.install  -}}
 pg-{{ default "patroni" .Values.patroni.clusterName }}-direct
 {{- else if .Values.externalDataBase }}
 {{- if or (eq (lower .Values.externalDataBase.type) "azure")  (eq (lower .Values.externalDataBase.type) "rds") }}
-{{- printf "%s.%s" "pg-patroni" .Release.Namespace }}
+{{- printf "pg-patroni.%s.svc.%s" .Release.Namespace $clusterDomain }}
 {{- else }}
-{{- default (printf "%s.%s" "pg-patroni" .Release.Namespace) .Values.backupDaemon.pgHost }}
+{{- default (printf "pg-patroni.%s.svc.%s" .Release.Namespace $clusterDomain) .Values.backupDaemon.pgHost }}
 {{- end }}
 {{- else }}
-{{- default (printf "%s.%s" "pg-patroni" .Release.Namespace) .Values.backupDaemon.pgHost }}
+{{- default (printf "pg-patroni.%s.svc.%s" .Release.Namespace $clusterDomain) .Values.backupDaemon.pgHost }}
 {{- end }}
 {{- end -}}