karimFin · karimFin · Mar 8, 2026 · Mar 8, 2026 · Mar 8, 2026 · Mar 8, 2026
diff --git a/.github/workflows/infra-dev.yml b/.github/workflows/infra-dev.yml
@@ -32,6 +32,9 @@ jobs:
     env:
       TF_IN_AUTOMATION: true
       TF_INPUT: 0
+      TF_TOKEN_app_terraform_io: ${{ secrets.TF_API_TOKEN }}
+      TF_CLOUD_ORGANIZATION: ${{ secrets.TF_CLOUD_ORGANIZATION }}
+      TF_WORKSPACE: gmp-dev
       TF_VAR_tenancy_ocid: ${{ secrets.OCI_CLI_TENANCY }}
       TF_VAR_user_ocid: ${{ secrets.OCI_CLI_USER }}
       TF_VAR_fingerprint: ${{ secrets.OCI_CLI_FINGERPRINT }}
@@ -41,6 +44,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Validate Terraform Cloud settings
+        run: |
+          if [ -z "$TF_CLOUD_ORGANIZATION" ] || [ -z "$TF_TOKEN_app_terraform_io" ]; then
+            echo "::error::Missing TF_CLOUD_ORGANIZATION or TF_API_TOKEN secrets for remote backend locking"
+            exit 1
+          fi
+
       - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.5.7

diff --git a/.github/workflows/preview-pr.yml b/.github/workflows/preview-pr.yml
@@ -0,0 +1,185 @@
+name: Preview PR
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled, closed]
+    branches: [dev]
+  schedule:
+    - cron: "0 */6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+  security-events: write
+  id-token: write
+  pull-requests: write
+
+concurrency:
+  group: preview-pr-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  deploy-preview:
+    if: ${{ github.event.action != 'closed' && github.event.pull_request.head.repo.full_name == github.repository && contains(github.event.pull_request.labels.*.name, 'preview') }}
+    uses: ./.github/workflows/reusable-cicd.yml
+    with:
+      environment_name: dev
+      namespace: pr-${{ github.event.pull_request.number }}
+      overlay: dev
+      deploy: true
+      build_images: true
+      push_images: true
+      sign_images: true
+      run_tests: false
+      run_frontend: false
+      validate_k8s: true
+      validate_k8s_all: false
+      policy_check: true
+      canary_checks: false
+      create_namespace: true
+      ref: ${{ github.event.pull_request.head.sha }}
+      preview_pr_number: ${{ github.event.pull_request.number }}
+    secrets: inherit
+
+  cleanup-preview:
+    if: ${{ github.event.action == 'closed' && github.event.pull_request.head.repo.full_name == github.repository }}
+    runs-on: ubuntu-latest
+    environment: dev
+    steps:
+      - name: Install OCI CLI
+        run: |
+          curl -L -O https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh
+          bash install.sh --accept-all-defaults
+          echo "$HOME/bin" >> "$GITHUB_PATH"
+
+      - name: Configure OCI CLI
+        env:
+          OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }}
+          OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }}
+          OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }}
+          OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }}
+          OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }}
+        run: |
+          for v in OCI_CLI_USER OCI_CLI_TENANCY OCI_CLI_FINGERPRINT OCI_CLI_KEY_CONTENT OCI_CLI_REGION; do
+            if [ -z "${!v}" ]; then
+              echo "::error::Missing required secret: $v"
+              exit 1
+            fi
+          done
+          mkdir -p ~/.oci
+          {
+            echo "[DEFAULT]"
+            echo "user=${OCI_CLI_USER}"
+            echo "fingerprint=${OCI_CLI_FINGERPRINT}"
+            echo "tenancy=${OCI_CLI_TENANCY}"
+            echo "region=${OCI_CLI_REGION}"
+            echo "key_file=~/.oci/oci_api_key.pem"
+          } > ~/.oci/config
+          RAW_KEY_STRIPPED=$(printf '%s' "${OCI_CLI_KEY_CONTENT}" | tr -d '\r')
+          if printf '%s' "$RAW_KEY_STRIPPED" | grep -q "BEGIN .*PRIVATE KEY"; then
+            printf '%s\n' "$RAW_KEY_STRIPPED" > ~/.oci/oci_api_key.pem
+          elif printf '%b' "$RAW_KEY_STRIPPED" | grep -q "BEGIN .*PRIVATE KEY"; then
+            printf '%b' "$RAW_KEY_STRIPPED" > ~/.oci/oci_api_key.pem
+          elif printf '%s' "$RAW_KEY_STRIPPED" | base64 -d > ~/.oci/oci_api_key.pem 2>/dev/null; then
+            :
+          else
+            printf '%b' "$RAW_KEY_STRIPPED" > ~/.oci/oci_api_key.pem
+          fi
+          chmod 600 ~/.oci/config ~/.oci/oci_api_key.pem
+
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: 'v1.30.2'
+
+      - name: Configure kubeconfig from OCI
+        run: |
+          set -euo pipefail
+          mkdir -p $HOME/.kube
+          CLUSTER_OCID=$(oci ce cluster list --compartment-id "${{ secrets.OCI_CLI_TENANCY }}" --all --query 'data[?name==`gmp-oke-dev` && "lifecycle-state"==`ACTIVE`] | [-1].id' --raw-output 2>/dev/null || true)
+          if [ -z "$CLUSTER_OCID" ] || [ "$CLUSTER_OCID" = "null" ]; then
+            echo "::error::No ACTIVE gmp-oke-dev cluster found"
+            exit 1
+          fi
+          oci ce cluster create-kubeconfig --cluster-id "$CLUSTER_OCID" --file "$HOME/.kube/config" --region "${{ secrets.OCI_CLI_REGION }}" --token-version 2.0.0 --kube-endpoint PUBLIC_ENDPOINT
+          chmod 600 "$HOME/.kube/config"
+
+      - name: Delete preview namespace
+        run: |
+          NS="pr-${{ github.event.pull_request.number }}"
+          kubectl delete namespace "$NS" --ignore-not-found=true --wait=true
+
+  cleanup-stale-preview:
+    if: ${{ github.event_name != 'pull_request' }}
+    runs-on: ubuntu-latest
+    environment: dev
+    steps:
+      - name: Install OCI CLI
+        run: |
+          curl -L -O https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh
+          bash install.sh --accept-all-defaults
+          echo "$HOME/bin" >> "$GITHUB_PATH"
+
+      - name: Configure OCI CLI
+        env:
+          OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }}
+          OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }}
+          OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }}
+          OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }}
+          OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }}
+        run: |
+          for v in OCI_CLI_USER OCI_CLI_TENANCY OCI_CLI_FINGERPRINT OCI_CLI_KEY_CONTENT OCI_CLI_REGION; do
+            if [ -z "${!v}" ]; then
+              echo "::error::Missing required secret: $v"
+              exit 1
+            fi
+          done
+          mkdir -p ~/.oci
+          {
+            echo "[DEFAULT]"
+            echo "user=${OCI_CLI_USER}"
+            echo "fingerprint=${OCI_CLI_FINGERPRINT}"
+            echo "tenancy=${OCI_CLI_TENANCY}"
+            echo "region=${OCI_CLI_REGION}"
+            echo "key_file=~/.oci/oci_api_key.pem"
+          } > ~/.oci/config
+          RAW_KEY_STRIPPED=$(printf '%s' "${OCI_CLI_KEY_CONTENT}" | tr -d '\r')
+          if printf '%s' "$RAW_KEY_STRIPPED" | grep -q "BEGIN .*PRIVATE KEY"; then
+            printf '%s\n' "$RAW_KEY_STRIPPED" > ~/.oci/oci_api_key.pem
+          elif printf '%b' "$RAW_KEY_STRIPPED" | grep -q "BEGIN .*PRIVATE KEY"; then
+            printf '%b' "$RAW_KEY_STRIPPED" > ~/.oci/oci_api_key.pem
+          elif printf '%s' "$RAW_KEY_STRIPPED" | base64 -d > ~/.oci/oci_api_key.pem 2>/dev/null; then
+            :
+          else
+            printf '%b' "$RAW_KEY_STRIPPED" > ~/.oci/oci_api_key.pem
+          fi
+          chmod 600 ~/.oci/config ~/.oci/oci_api_key.pem
+
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: 'v1.30.2'
+
+      - name: Configure kubeconfig from OCI
+        run: |
+          set -euo pipefail
+          mkdir -p $HOME/.kube
+          CLUSTER_OCID=$(oci ce cluster list --compartment-id "${{ secrets.OCI_CLI_TENANCY }}" --all --query 'data[?name==`gmp-oke-dev` && "lifecycle-state"==`ACTIVE`] | [-1].id' --raw-output 2>/dev/null || true)
+          if [ -z "$CLUSTER_OCID" ] || [ "$CLUSTER_OCID" = "null" ]; then
+            echo "::error::No ACTIVE gmp-oke-dev cluster found"
+            exit 1
+          fi
+          oci ce cluster create-kubeconfig --cluster-id "$CLUSTER_OCID" --file "$HOME/.kube/config" --region "${{ secrets.OCI_CLI_REGION }}" --token-version 2.0.0 --kube-endpoint PUBLIC_ENDPOINT
+          chmod 600 "$HOME/.kube/config"
+
+      - name: Delete stale preview namespaces by TTL
+        run: |
+          set -euo pipefail
+          NOW=$(date +%s)
+          kubectl get ns -o json | jq -r '.items[] | select(.metadata.labels["preview.gmp/enabled"]=="true") | [.metadata.name, (.metadata.labels["preview.gmp/created-at"] // "0"), (.metadata.labels["preview.gmp/ttl-hours"] // "24")] | @tsv' | while IFS=$'\t' read -r NS CREATED TTL; do
+            EXPIRE=$((CREATED + TTL * 3600))
+            if [ "$NOW" -ge "$EXPIRE" ]; then
+              kubectl delete namespace "$NS" --ignore-not-found=true --wait=true
+            fi
+          done
diff --git a/.github/workflows/reusable-cicd.yml b/.github/workflows/reusable-cicd.yml
@@ -77,6 +77,10 @@ on:
         required: false
         type: string
         default: ""
+      preview_pr_number:
+        required: false
+        type: string
+        default: ""
 
 env:
   REGISTRY: ghcr.io
@@ -362,8 +366,12 @@ jobs:
             oci ce cluster create-kubeconfig --cluster-id "$CLUSTER_OCID" --file "$CONFIG_FILE" --region "${OCI_CLI_REGION}" --token-version 2.0.0 --kube-endpoint PUBLIC_ENDPOINT
           }
 
+          PREVIEW_PR="${{ inputs.preview_pr_number }}"
           KCFG_OK=false
-          if [ -n "${KUBE_CONFIG_DEV}" ]; then
+          if [ -n "$PREVIEW_PR" ]; then
+            ensure_kubeconfig_from_oci
+            KCFG_OK=true
+          elif [ -n "${KUBE_CONFIG_DEV}" ]; then
             printf '%s' "${KUBE_CONFIG_DEV}" > "$RAW_FILE"
             if grep -q "apiVersion:" "$RAW_FILE"; then
               cp "$RAW_FILE" "$CONFIG_FILE"
@@ -450,6 +458,31 @@ jobs:
         if: ${{ inputs.create_namespace }}
         run: kubectl get namespace ${{ inputs.namespace }} || kubectl create namespace ${{ inputs.namespace }}
 
+      - name: Label preview namespace TTL
+        if: ${{ inputs.preview_pr_number != '' }}
+        run: |
+          NOW=$(date +%s)
+          kubectl label namespace ${{ inputs.namespace }} \
+            preview.gmp/enabled=true \
+            preview.gmp/pr=${{ inputs.preview_pr_number }} \
+            preview.gmp/created-at="$NOW" \
+            preview.gmp/ttl-hours=24 \
+            --overwrite=true
+
+      - name: Ensure preview data secrets
+        if: ${{ inputs.preview_pr_number != '' }}
+        run: |
+          kubectl create secret generic postgres-credentials \
+            -n ${{ inputs.namespace }} \
+            --from-literal=username=market \
+            --from-literal=password=marketpass \
+            --dry-run=client -o yaml | kubectl apply -f -
+          kubectl create secret generic minio-credentials \
+            -n ${{ inputs.namespace }} \
+            --from-literal=accesskey=admin \
+            --from-literal=secretkey=adminadmin \
+            --dry-run=client -o yaml | kubectl apply -f -
+
       - name: Configure GHCR pull secret
         env:
           GHCR_PASSWORD: ${{ secrets.GHCR_TOKEN || secrets.GITHUB_TOKEN }}
@@ -470,13 +503,33 @@ jobs:
       - name: Prepare kustomize images
         run: |
           cd platform/k8s/overlays/${{ inputs.overlay }}
+          kustomize edit set namespace ${{ inputs.namespace }}
           for img in api-gateway web identity seller catalog search pricing inventory cart checkout payments orders fulfillment notifications reviews analytics; do
             kustomize edit set image $img=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/$img:${{ needs.meta.outputs.sha }}
           done
 
+      - name: Verify rendered namespace
+        run: |
+          cat platform/k8s/overlays/${{ inputs.overlay }}/kustomization.yaml
+          kustomize build platform/k8s/overlays/${{ inputs.overlay }} | grep -m 40 '^  namespace:'
+
+      - name: Reset immutable bootstrap jobs
+        run: |
+          kubectl delete job \
+            minio-make-bucket \
+            s3-sink-register \
+            debezium-register \
+            kafka-topics-init \
+            -n ${{ inputs.namespace }} \
+            --ignore-not-found=true
+
       - name: Apply dev overlay
         run: kubectl apply -k platform/k8s/overlays/${{ inputs.overlay }}
 
+      - name: Limit preview load balancers
+        if: ${{ inputs.preview_pr_number != '' }}
+        run: kubectl patch svc web -n ${{ inputs.namespace }} -p '{"spec":{"type":"ClusterIP"}}' || true
+
       - name: Show external services
         run: |
           kubectl get svc web api-gateway -n ${{ inputs.namespace }} -o wide || true
@@ -506,10 +559,64 @@ jobs:
             echo "If IP is pending, wait 1-3 minutes and re-check service status."
           } >> "$GITHUB_STEP_SUMMARY"
 
+      - name: Comment preview URLs on PR
+        if: ${{ inputs.environment_name == 'dev' && inputs.preview_pr_number != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          WEB_IP=$(kubectl get svc web -n ${{ inputs.namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)
+          API_IP=$(kubectl get svc api-gateway -n ${{ inputs.namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)
+          WEB_URL="pending"
+          API_URL="pending"
+          if [ -n "$WEB_IP" ]; then WEB_URL="http://$WEB_IP"; fi
+          if [ -n "$API_IP" ]; then API_URL="http://$API_IP"; fi
+          MARKER="<!-- gmp-preview-${{ inputs.preview_pr_number }} -->"
+          BODY="$MARKER
+          ✅ Preview environment deployed
+
+          - Namespace: \`${{ inputs.namespace }}\`
+          - Web: $WEB_URL
+          - API Gateway: $API_URL
+
+          If URL is pending, wait a few minutes and re-run deployment."
+
+          EXISTING_ID=$(gh api repos/${{ github.repository }}/issues/${{ inputs.preview_pr_number }}/comments --paginate --jq '.[] | select(.body | contains("'"$MARKER"'")) | .id' | head -n 1 || true)
+          if [ -n "$EXISTING_ID" ]; then
+            gh api repos/${{ github.repository }}/issues/comments/$EXISTING_ID -X PATCH -f body="$BODY" >/dev/null
+          else
+            gh api repos/${{ github.repository }}/issues/${{ inputs.preview_pr_number }}/comments -X POST -f body="$BODY" >/dev/null
+          fi
+
       - name: Wait for rollouts
         run: |
-          for d in api-gateway web identity seller catalog search pricing inventory cart checkout payments orders fulfillment notifications reviews analytics; do
-            kubectl rollout status deployment/$d -n ${{ inputs.namespace }} --timeout=180s || exit 1
+          NS=${{ inputs.namespace }}
+          PREVIEW_PR="${{ inputs.preview_pr_number }}"
+          diagnose_ns() {
+            kubectl get pods -n "$NS" -o wide || true
+            kubectl get events -n "$NS" --sort-by='.lastTimestamp' | tail -n 150 || true
+          }
+          check_rollout() {
+            local kind="$1"
+            local name="$2"
+            echo "Checking rollout: ${kind}/${name}"
+            if ! kubectl rollout status "${kind}/${name}" -n "$NS" --timeout=600s; then
+              kubectl describe "${kind}/${name}" -n "$NS" || true
+              kubectl describe pods -n "$NS" -l app="$name" || true
+              kubectl logs -n "$NS" -l app="$name" --all-containers --tail=200 || true
+              diagnose_ns
+              exit 1
+            fi
+          }
+          if [ -z "$PREVIEW_PR" ]; then
+            check_rollout statefulset postgres
+            check_rollout statefulset kafka
+            check_rollout statefulset opensearch
+            for d in redis minio kafka-connect; do
+              check_rollout deployment "$d"
+            done
+          fi
+          for d in identity seller catalog search pricing inventory cart checkout payments orders fulfillment notifications reviews analytics api-gateway web transformer-api; do
+            check_rollout deployment "$d"
           done
 
   # ─────────────────────────────────────────────────────────────

diff --git a/README.md b/README.md
@@ -79,6 +79,7 @@ kubectl apply -k platform/k8s/base
 - `.github/workflows/ci.yml` runs service tests.
 - `.github/workflows/ci-extended.yml` runs lint, tests, image builds, SBOM, and vulnerability scans.
 - `.github/workflows/deploy.yml` deploys to dev on `dev` branch push; prod canary and promotion are manual dispatch.
+- `.github/workflows/preview-pr.yml` deploys PR previews to `pr-<number>` namespace when PR has `preview` label, comments URLs on the PR, and auto-cleans stale previews every 6 hours (24h TTL).
 
 **Secrets**
 - `KUBE_CONFIG_DEV` (base64 kubeconfig for dev)
@@ -101,6 +102,11 @@ Use one command entrypoints for learning lifecycle:
 - `make up-dev` → Apply infra + generate kubeconfig + update `KUBE_CONFIG_DEV` + trigger `Deploy Dev`
 - `make infra-status` → Show Terraform state and active OCI cluster/LB
 
+Required for remote backend locking:
+- `TF_CLOUD_ORGANIZATION` (local shell env)
+- `TF_WORKSPACE` (recommended: `gmp-dev`)
+- GitHub `dev` environment secrets: `TF_CLOUD_ORGANIZATION`, `TF_API_TOKEN`
+
 Script entrypoint: `scripts/devctl.sh`  
 Full quick commands: `cmd.md`