feat: 260 refactor database update

.github/actions/notify-slack/action.yml

@@ -6,7 +6,7 @@ inputs:
     required: true
     type: string
   PRIORITY:
-    description: 'Priority level of the alert (medium/high)'
+    description: 'Priority level of the alert (medium/high/test)'
     required: true
     type: string
   STEP:
@@ -31,19 +31,29 @@ runs:
       id: alert_message
       shell: bash
       run: |
-        if [[ "${{ inputs.PRIORITY }}" == "high" ]]; then
-          if [[ -n "${{ inputs.STEP }}" ]]; then
-            message="🚨 High Priority Alert: [${{ github.workflow }}] failed at step \"${{ inputs.STEP }}\". Immediate attention is required to avoid production impact."
-          else
-            message="🚨 High Priority Alert: [${{ github.workflow }}] failed. Immediate attention is required to avoid production impact."
-          fi
-        else
-          if [[ -n "${{ inputs.STEP }}" ]]; then
-            message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue at step \"${{ inputs.STEP }}\". This may affect ongoing integration processes."
-          else
-            message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue. This may affect ongoing integration processes."
-          fi
-        fi
+        case "${{ inputs.PRIORITY }}" in
+          high)
+            if [[ -n "${{ inputs.STEP }}" ]]; then
+              message="🚨 High Priority Alert: [${{ github.workflow }}] failed at step \"${{ inputs.STEP }}\". Immediate attention is required to avoid production impact."
+            else
+              message="🚨 High Priority Alert: [${{ github.workflow }}] failed. Immediate attention is required to avoid production impact."
+            fi
+            ;;
+          test)
+            if [[ -n "${{ inputs.STEP }}" ]]; then
+              message="🧪 Test Alert: [${{ github.workflow }}] reached step \"${{ inputs.STEP }}\". This is a test notification only."
+            else
+              message="🧪 Test Alert: [${{ github.workflow }}]. This is a test notification only."
+            fi
+            ;;
+          *)
+            if [[ -n "${{ inputs.STEP }}" ]]; then
+              message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue at step \"${{ inputs.STEP }}\". This may affect ongoing integration processes."
+            else
+              message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue. This may affect ongoing integration processes."
+            fi
+            ;;
+        esac
 
         # Construct the JSON payload and save it to a file
         jq -n --arg message "$message" --arg run_url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" '{
@@ -74,5 +84,3 @@ runs:
       env:
         SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }}
         SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
-
-

.github/actions/resolve-api-meta/action.yml

@@ -0,0 +1,80 @@
+name: Resolve API commit/version
+description: Resolve deployed API commit SHA and version from the metadata endpoint
+inputs:
+  api_base_url:
+    description: Base URL host for the API (e.g. api-dev.mobilitydatabase.org)
+    required: false
+    default: api.mobilitydatabase.org
+  api_refresh_token:
+    description: API refresh token
+    required: false
+outputs:
+  COMMIT_SHA:
+    description: Resolved commit SHA
+    value: ${{ steps.resolve.outputs.COMMIT_SHA }}
+  API_VERSION:
+    description: Resolved API version
+    value: ${{ steps.resolve.outputs.API_VERSION }}
+  RESOLVED:
+    description: Whether a commit SHA was resolved (true/false)
+    value: ${{ steps.resolve.outputs.RESOLVED }}
+runs:
+  using: composite
+  steps:
+    - id: resolve
+      name: Resolve via API and expose outputs
+      shell: bash
+      env:
+        API_BASE_URL: ${{ inputs.api_base_url }}
+        API_REFRESH_TOKEN: ${{ inputs.api_refresh_token }}
+      run: |
+        # Do not exit on failure; this action should never abort the caller workflow.
+        set -u
+        COMMIT_SHA=""
+        API_VERSION=""
+        RESOLVED="false"
+
+        if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then
+          echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..."
+
+          # Exchange refresh token -> access token (handle failures gracefully)
+          REPLY_JSON=$(curl --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \
+            --header 'Content-Type: application/json' \
+            --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }" ) || {
+              echo "Warning: token exchange failed; will fallback to 'main'" >&2
+              REPLY_JSON=""
+          }
+
+          if [[ -n "${REPLY_JSON}" ]]; then
+            ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token 2>/dev/null || echo "")
+            if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then
+              echo "Warning: Could not obtain access token from reply; will fallback to 'main'" >&2
+            else
+              META_JSON=$(curl --silent --show-error \
+                -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                -H 'accept: application/json' \
+                "https://${API_BASE_URL}/v1/metadata" ) || {
+                  echo "Warning: metadata request failed; will fallback to 'main'" >&2
+                  META_JSON=""
+              }
+
+              if [[ -n "${META_JSON}" ]]; then
+                COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash 2>/dev/null || echo "")
+                API_VERSION=$(echo "${META_JSON}" | jq -r .version 2>/dev/null || echo "")
+                if [[ -n "${COMMIT_SHA}" && "${COMMIT_SHA}" != "null" ]]; then
+                  RESOLVED="true"
+                  echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})"
+                else
+                  echo "Warning: commit_hash missing in metadata; will fallback to 'main'" >&2
+                fi
+              fi
+            fi
+          fi
+        else
+          echo "No API refresh token provided; skipping API metadata resolution and falling back to 'main'."
+        fi
+
+        # Expose outputs (empty COMMIT_SHA and RESOLVED=false indicate fallback to 'main')
+        echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
+        echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT"
+        echo "RESOLVED=${RESOLVED}" >> "$GITHUB_OUTPUT"

.github/workflows/catalog-update.yml

@@ -0,0 +1,144 @@
+# Called when the Mobility Catalog is updated.
+name: Mobility Catalog Update
+on:
+  workflow_dispatch:
+    inputs:
+      DRY_RUN:
+        description: Dry run. Skip applying schema and content updates
+        required: false
+        default: true
+        type: boolean
+  repository_dispatch: # Update on mobility-database-catalog repo dispatch
+    types: [ catalog-sources-updated, gbfs-systems-updated ]
+
+env:
+  python_version: '3.11'
+  liquibase_version: '4.33.0'
+
+jobs:
+  resolve-api-meta-qa:
+    name: QA Resolve API commit/version
+    runs-on: ubuntu-latest
+    outputs:
+      CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }}
+    steps:
+      - name: Checkout repo (for scripts and local action)
+        uses: actions/checkout@v4
+
+      - name: Resolve API commit/version
+        id: resolve
+        uses: ./.github/actions/resolve-api-meta
+        with:
+          api_base_url: api-qa.mobilitydatabase.org
+          api_refresh_token: ${{ secrets.QA_API_TEST_REFRESH_TOKEN }}
+
+  update-content-qa:
+    name: QA Update DB content
+    needs: [ resolve-api-meta-qa ]
+    uses: ./.github/workflows/db-update-content.yml
+    with:
+      PROJECT_ID: ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
+      REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
+      DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }}
+      ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
+      DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
+      DRY_RUN: ${{ github.event_name == 'repository_dispatch' || inputs.DRY_RUN }}
+      CHECKOUT_REF: ${{ needs.resolve-api-meta-qa.outputs.CHECKOUT_REF }}
+    secrets:
+      DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
+      DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
+      DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
+      GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
+      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+
+  resolve-api-meta-prod:
+    name: PROD Resolve API commit/version
+    runs-on: ubuntu-latest
+    outputs:
+      CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }}
+    steps:
+      - name: Checkout repo (for scripts and local action)
+        uses: actions/checkout@v4
+
+      - name: Resolve API commit/version
+        id: resolve
+        uses: ./.github/actions/resolve-api-meta
+        with:
+          api_base_url: api.mobilitydatabase.org
+          api_refresh_token: ${{ secrets.PROD_API_TEST_REFRESH_TOKEN }}
+
+  update-content-prod:
+    name: PROD Update DB content
+    needs: [ resolve-api-meta-prod ]
+    uses: ./.github/workflows/db-update-content.yml
+    with:
+      PROJECT_ID: ${{ vars.PROD_MOBILITY_FEEDS_PROJECT_ID }}
+      REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
+      DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }}
+      ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
+      DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
+      DRY_RUN: ${{ github.event_name == 'repository_dispatch' || inputs.DRY_RUN }}
+      CHECKOUT_REF: ${{ needs.resolve-api-meta-prod.outputs.CHECKOUT_REF }}
+    secrets:
+      DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
+      DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
+      DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
+      GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
+      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+
+  resolve-api-meta-dev:
+    name: DEV Resolve API commit/version
+    runs-on: ubuntu-latest
+    outputs:
+      CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }}
+    steps:
+      - name: Checkout repo (for scripts and local action)
+        uses: actions/checkout@v4
+
+      - name: Resolve API commit/version
+        id: resolve
+        uses: ./.github/actions/resolve-api-meta
+        with:
+          api_base_url: api-dev.mobilitydatabase.org
+          api_refresh_token: ${{ secrets.DEV_API_TEST_REFRESH_TOKEN }}
+
+  update-content-dev:
+    name: Dev Update DB content
+    needs: [ resolve-api-meta-dev ]
+    uses: ./.github/workflows/db-update-content.yml
+    with:
+      PROJECT_ID: ${{ vars.DEV_MOBILITY_FEEDS_PROJECT_ID }}
+      REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
+      DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }}
+      ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }}
+      # dev uses the QA sql instance
+      DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
+      DRY_RUN: ${{ github.event_name == 'repository_dispatch' || inputs.DRY_RUN }}
+      CHECKOUT_REF: ${{ needs.resolve-api-meta-dev.outputs.CHECKOUT_REF }}
+    secrets:
+      DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }}
+      DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }}
+      GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}
+      # dev uses the QA sql instance
+      DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
+      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+
+  notify-slack-on-failure:
+    # Run after all relevant jobs and notify if any failed
+    needs: [ resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ]
+    if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "high"