From 843f78202a69e60b4959989224b663faa9e8bc1c Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 29 Oct 2025 13:29:23 -0400 Subject: [PATCH 01/17] First draft --- .github/workflows/db-update-dev.yml | 2 + .github/workflows/db-update-prod.yml | 2 + .github/workflows/db-update-qa.yml | 2 + .github/workflows/db-update.yml | 64 +++++++++++++++++++++++++--- scripts/resolve_api_meta.sh | 58 +++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 7 deletions(-) create mode 100755 scripts/resolve_api_meta.sh diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index 542671489..f311d7dec 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -19,6 +19,7 @@ jobs: DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} + API_BASE_URL: api-dev.mobilitydatabase.org secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} @@ -28,6 +29,7 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + API_TEST_REFRESH_TOKEN: ${{ secrets.DEV_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index cfeff46cb..0f157ce92 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -14,6 +14,7 @@ jobs: DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} + API_BASE_URL: api.mobilitydatabase.org secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} @@ -23,6 +24,7 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + API_TEST_REFRESH_TOKEN: ${{ secrets.PROD_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index 62262e68c..38d2f6ae8 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -15,6 +15,7 @@ jobs: DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} + API_BASE_URL: api-qa.mobilitydatabase.org secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} @@ -24,6 +25,7 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + API_TEST_REFRESH_TOKEN: ${{ secrets.QA_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 3ce05c37a..a78377eeb 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -46,6 +46,9 @@ on: POSTGRE_SQL_INSTANCE_NAME: description: PostgreSQL Instance Name required: true + API_TEST_REFRESH_TOKEN: + description: API refresh token used to resolve deployed API commit (used on repository_dispatch) + required: false inputs: PROJECT_ID: description: GCP Project ID @@ -67,18 +70,57 @@ on: description: GCP region required: true type: string + API_BASE_URL: + description: Base URL host for the API used to resolve version/commit (e.g. api-dev.mobilitydatabase.org) + required: false + default: api.mobilitydatabase.org + type: string env: python_version: '3.11' liquibase_version: '4.33.0' jobs: + resolve-api-meta: + name: 'Resolve API commit/version' + runs-on: ubuntu-latest + if: ${{ github.event_name == 'repository_dispatch' && inputs.API_BASE_URL != '' && secrets.API_TEST_REFRESH_TOKEN != '' }} + outputs: + COMMIT_SHA: ${{ steps.resolve.outputs.COMMIT_SHA }} + API_VERSION: ${{ steps.resolve.outputs.API_VERSION }} + steps: + - name: Checkout repo (for scripts) + uses: actions/checkout@v4 + with: + fetch-depth: 1 + sparse-checkout: | + scripts + sparse-checkout-cone: true + - name: Resolve API commit/version + id: resolve + env: + API_BASE_URL: ${{ inputs.API_BASE_URL }} + API_REFRESH_TOKEN: ${{ secrets.API_TEST_REFRESH_TOKEN }} + EVENT: ${{ github.event_name }} + run: | + bash scripts/resolve_api_meta.sh + db-schema-update: name: 'Database Schema Update' permissions: write-all runs-on: ubuntu-latest + needs: [resolve-api-meta] + if: ${{ always() }} steps: - - name: Checkout code + - name: Checkout code at API commit + if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }} + uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve-api-meta.outputs.COMMIT_SHA }} + fetch-depth: 0 + + - name: Checkout code (default) + if: ${{ needs.resolve-api-meta.result != 'success' || needs.resolve-api-meta.outputs.COMMIT_SHA == '' }} uses: actions/checkout@v4 - name: Authenticate to Google Cloud QA/PROD @@ -139,10 +181,18 @@ jobs: name: 'Database Content Update' permissions: write-all runs-on: ubuntu-latest - needs: db-schema-update - if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} + needs: [resolve-api-meta, db-schema-update] + if: ${{ always() && (github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch') }} steps: - - name: Checkout code + - name: Checkout code at API commit + if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }} + uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve-api-meta.outputs.COMMIT_SHA }} + fetch-depth: 0 + + - name: Checkout code (default) + if: ${{ needs.resolve-api-meta.result != 'success' || needs.resolve-api-meta.outputs.COMMIT_SHA == '' }} uses: actions/checkout@v4 - name: Setup python @@ -159,6 +209,8 @@ jobs: uses: google-github-actions/setup-gcloud@v2 - name: Update .env file + env: + API_VERSION: ${{ needs.resolve-api-meta.outputs.API_VERSION }} run: | echo "PGUSER=${{ secrets.DB_USER_NAME }}" > config/.env.local echo "POSTGRES_USER=${{ secrets.DB_USER_NAME }}" >> config/.env.local @@ -168,6 +220,7 @@ jobs: echo "POSTGRES_PORT=5432" >> config/.env.local echo "POSTGRES_HOST=localhost" >> config/.env.local echo "ENV=${{ inputs.ENVIRONMENT }}" >> config/.env.local + if [[ -n "${API_VERSION}" ]]; then echo "API_VERSION=${API_VERSION}" >> config/.env.local; fi cat config/.env.local - name: Load secrets from 1Password @@ -283,6 +336,3 @@ jobs: echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..." echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID fi - - - diff --git a/scripts/resolve_api_meta.sh b/scripts/resolve_api_meta.sh new file mode 100755 index 000000000..2c67dc9dc --- /dev/null +++ b/scripts/resolve_api_meta.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Resolve API metadata and set GitHub Action step outputs COMMIT_SHA and API_VERSION +# Inputs (via env): +# EVENT - GitHub event name (e.g., repository_dispatch) +# API_BASE_URL - API host (e.g., api-dev.mobilitydatabase.org) +# API_REFRESH_TOKEN- Refresh token to obtain access token (only used on repository_dispatch) +# Outputs: +# Prints COMMIT_SHA and API_VERSION to stdout as NAME=VALUE lines +# Also exports to $GITHUB_ENV if defined so the script is runnable by hand + +set -euo pipefail +# QA +#export API_REFRESH_TOKEN="AMf-vBzv1rT8AQ0uZrzNDD5wuMxSvxLVbXRmViTaOVuP8eh-uDdWLDpsDHYMrGNqq2sqn1ya_-i8YXZdWh9GnLoPSLbbWY99hmb-JUrJ_NXz4pJ5v2ysm3kCpjy02zN2uI5csAi1YmGSrlJoUQazNO4ntkVHkgdpsyMSBwgIGDczwX4qANzEjsRjtRCQlCbm_MnWsLaBKrukn5qxFbyVszzOloG5piIivTL700I9cPslxlzirmYYrj3jsYIX00RKBF3pPvTpiiOGskRjeZi_UvI5spux0tkFuZJZGt-vNKWeLT9MSJp6S3Y5os4PQMjCJ-StCP4Qkqwja5EiDRdVhqJVCT7XYn_MAQBYGFUYvFOQu9S9siKlhXO9Mcc2NiH89eaJ1EUlkXRnzY9nahP82cmt8VOKHyu6GAT3-l4V9_9zJ7wjyQJ0wSf_5wZlLxFSScLaIADZdRya" +# https://beta.mobilitydatabase.org/account +#export API_REFRESH_TOKEN="AMf-vByLiUKvxTqH_4vkQTJ8aStgCvrNGDyMINJNpYVdRCQliF6q0FmZxy7Y29rgNce4HGnrr2La3lja5CsOPn8-Vx6RA0enAv5RVxnBC08G-c6ZAiwCMfvUT-vT49ZLeULmUIM6BlCQKGdXSxAOljg6QJNH0wXdvhHaLUGVGdhoeKW3pg692ZCCWTVCWOFSDhHhS-d-8ywSd7nsFwka0ZMBRAtOeY-lgvwsz7Wo21hCTaHyUYSoodKzWFjUUJ9W7nA33OB6lKBA9tZGap3pYlO-Vo47jl-M2GcZn77R-sgsgEbPXfHxc0NMS1ZbNXKeWmrcqPYcgcEoBFdFrnQcd1bD-b1O8_zTnDyjjRekLq8bGKLnDfRg-qZvUaNxtKPUqFnh-pmvO7gXMxwj7Tnoc5cMmQxHGZ2euuxaVRHNMoAxnEzqIzZ7nxIM4NX36eUZCuLLSTXvOnmsdMuhQ_7oI2wX-YKxRGHnrw" +# https://mobility-feeds-dev.web.app/account +export API_REFRESH_TOKEN="AMf-vBxJqJLukaDstkJl_Pi-GonY1suIN_3O5Fp7pHfYcD_XVZwSJczM_815UpEJFQN9ShzaE5KRqsyO4sRnFUBAA6KLQgcP-7Mx9yNzRBGtBi-e37X50CNwqUvsuUfOjFZNeKPfaO1ipuCA9LQLWf-5e29DmyxEgO1Fy9UguKm5KlDRrcIkUjTzSBHuSRu06j1_th4TR4l0X5OteNGsG6F6N2lrihj2Z5Idx6PoJ02_5fgLOhop8mlGd-ktNpQ3J46lAz7BX7_UgPkbugwIoD9YdYvnIlrSp5hl3ri7mmauN0rOV1HEMn-aynd7Zqkc0lmNWWL3BqQe-Ik0sOt23HZISK7Y2EITrXEtXxPN0-824BKUAoLN-_eutLk3NFRJW19jQc-RrZRP1LzXdWQETKhXPRBWCf33U1N0Yv2qzBKTzhvFjh3yULRVnZjUxh7CnZeYDKbxAWYj" + +COMMIT_SHA="" +API_VERSION="" + +if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then + echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..." + REPLY_JSON=$(curl --fail --silent --show-error --location "https://${mobility-feeds-dev}/v1/tokens" \ + --header 'Content-Type: application/json' \ + --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }") + ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token) + if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then + echo "Error: Could not obtain access token from reply" >&2 + exit 1 + fi + META_JSON=$(curl --fail --silent --show-error \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'accept: application/json' \ + "https://${API_BASE_URL}/v1/metadata") + COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash) + API_VERSION=$(echo "${META_JSON}" | jq -r .version) + if [[ -z "${COMMIT_SHA}" || "${COMMIT_SHA}" == "null" ]]; then + echo "Error: Could not extract commit_hash from metadata" >&2 + echo "Metadata reply: ${META_JSON}" >&2 + exit 1 + fi + echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" +else + echo "No token provided; skipping API metadata resolution." +fi + +# Output values to stdout in a parse-friendly format +echo "COMMIT_SHA=${COMMIT_SHA}" +echo "API_VERSION=${API_VERSION}" + +# Optionally export to $GITHUB_ENV for subsequent steps when available +if [[ -n "${GITHUB_ENV:-}" ]]; then + { + echo "COMMIT_SHA=${COMMIT_SHA}" + echo "API_VERSION=${API_VERSION}" + } >> "$GITHUB_ENV" +fi From f62dc88ee8c285c0a87d96eb40eed9b8cae60ac4 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 29 Oct 2025 17:59:41 -0400 Subject: [PATCH 02/17] Added a DRY_RUN parameter to db-update.yml to help with testing. --- .github/workflows/db-update-qa.yml | 8 ++++++ .github/workflows/db-update.yml | 40 ++++++++++++++++-------------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index 38d2f6ae8..d876d1ee7 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -2,7 +2,14 @@ name: Database Update - QA on: workflow_dispatch: + inputs: + DRY_RUN: + description: Skip applying schema and content updates + required: false + default: false + type: boolean workflow_call: + repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -16,6 +23,7 @@ jobs: ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-qa.mobilitydatabase.org + DRY_RUN: ${{ inputs.DRY_RUN }} secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index a78377eeb..c24e371d8 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -75,35 +75,35 @@ on: required: false default: api.mobilitydatabase.org type: string + DRY_RUN: + description: Skip applying schema and content updates + required: false + default: false + type: boolean env: python_version: '3.11' liquibase_version: '4.33.0' + # True when this run was triggered by a repository_dispatch or a manual workflow_dispatch + IS_DISPATCH: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} jobs: resolve-api-meta: name: 'Resolve API commit/version' runs-on: ubuntu-latest - if: ${{ github.event_name == 'repository_dispatch' && inputs.API_BASE_URL != '' && secrets.API_TEST_REFRESH_TOKEN != '' }} + if: ${{ env.IS_DISPATCH == 'true' && inputs.API_BASE_URL != '' && secrets.API_TEST_REFRESH_TOKEN != '' }} outputs: COMMIT_SHA: ${{ steps.resolve.outputs.COMMIT_SHA }} API_VERSION: ${{ steps.resolve.outputs.API_VERSION }} steps: - - name: Checkout repo (for scripts) + - name: Checkout repo (for scripts and local action) uses: actions/checkout@v4 - with: - fetch-depth: 1 - sparse-checkout: | - scripts - sparse-checkout-cone: true - name: Resolve API commit/version id: resolve - env: - API_BASE_URL: ${{ inputs.API_BASE_URL }} - API_REFRESH_TOKEN: ${{ secrets.API_TEST_REFRESH_TOKEN }} - EVENT: ${{ github.event_name }} - run: | - bash scripts/resolve_api_meta.sh + uses: ./.github/actions/resolve-api-meta + with: + api_base_url: ${{ inputs.API_BASE_URL }} + api_refresh_token: ${{ secrets.API_TEST_REFRESH_TOKEN }} db-schema-update: name: 'Database Schema Update' @@ -168,6 +168,7 @@ jobs: liquibase --version - name: Run Liquibase + if: ${{ !inputs.DRY_RUN }} working-directory: ${{ github.workspace }}/liquibase run: | export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml" @@ -182,7 +183,7 @@ jobs: permissions: write-all runs-on: ubuntu-latest needs: [resolve-api-meta, db-schema-update] - if: ${{ always() && (github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch') }} + if: ${{ always() && env.IS_DISPATCH == 'true' }} steps: - name: Checkout code at API commit if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }} @@ -265,11 +266,11 @@ jobs: run: echo "PATH=$(realpath sources.csv)" >> $GITHUB_OUTPUT - name: GTFS - Update Database Content - if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }} + if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} run: scripts/populate-db.sh ${{ steps.getpath.outputs.PATH }} > populate.log - name: GTFS - Upload log file for verification - if: ${{ always() && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} uses: actions/upload-artifact@v4 with: name: populate-${{ inputs.ENVIRONMENT }}.log @@ -285,11 +286,11 @@ jobs: run: echo "PATH=$(realpath systems.csv)" >> $GITHUB_OUTPUT - name: GBFS - Update Database Content - if: ${{ env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual' }} + if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} run: scripts/populate-db.sh ${{ steps.getsyspath.outputs.PATH }} gbfs >> populate-gbfs.log - name: GBFS - Upload log file for verification - if: ${{ always() && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} uses: actions/upload-artifact@v4 with: name: populate-gbfs-${{ inputs.ENVIRONMENT }}.log @@ -298,7 +299,7 @@ jobs: update-gcp-secret: name: Update GCP Secrets - if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} + if: ${{ env.IS_DISPATCH == 'true' && !inputs.DRY_RUN }} runs-on: ubuntu-latest steps: - name: Authenticate to Google Cloud @@ -336,3 +337,4 @@ jobs: echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..." echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID fi + From 10125551056106d1bdc6031bfce2fa507444da4c Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 29 Oct 2025 18:03:49 -0400 Subject: [PATCH 03/17] Corrected a syntax error in a workflow yml file. --- .github/workflows/db-update.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index c24e371d8..5d5e79302 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -81,17 +81,20 @@ on: default: false type: boolean +# Add workflow-level vars (these are available in job `if` expressions) +vars: + # True when this run was triggered by a repository_dispatch or a manual workflow_dispatch + IS_DISPATCH: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} + env: python_version: '3.11' liquibase_version: '4.33.0' - # True when this run was triggered by a repository_dispatch or a manual workflow_dispatch - IS_DISPATCH: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} jobs: resolve-api-meta: name: 'Resolve API commit/version' runs-on: ubuntu-latest - if: ${{ env.IS_DISPATCH == 'true' && inputs.API_BASE_URL != '' && secrets.API_TEST_REFRESH_TOKEN != '' }} + if: ${{ vars.IS_DISPATCH == 'true' && inputs.API_BASE_URL != '' && secrets.API_TEST_REFRESH_TOKEN != '' }} outputs: COMMIT_SHA: ${{ steps.resolve.outputs.COMMIT_SHA }} API_VERSION: ${{ steps.resolve.outputs.API_VERSION }} @@ -183,7 +186,7 @@ jobs: permissions: write-all runs-on: ubuntu-latest needs: [resolve-api-meta, db-schema-update] - if: ${{ always() && env.IS_DISPATCH == 'true' }} + if: ${{ always() && vars.IS_DISPATCH == 'true' }} steps: - name: Checkout code at API commit if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }} @@ -299,7 +302,7 @@ jobs: update-gcp-secret: name: Update GCP Secrets - if: ${{ env.IS_DISPATCH == 'true' && !inputs.DRY_RUN }} + if: ${{ vars.IS_DISPATCH == 'true' && !inputs.DRY_RUN }} runs-on: ubuntu-latest steps: - name: Authenticate to Google Cloud @@ -337,4 +340,3 @@ jobs: echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..." echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID fi - From df6b8a85936f5e4f8f282d469e9f57fba0963d6d Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 09:04:11 -0400 Subject: [PATCH 04/17] Added a missing action. --- .github/actions/resolve-api-meta/action.yml | 59 +++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/actions/resolve-api-meta/action.yml diff --git a/.github/actions/resolve-api-meta/action.yml b/.github/actions/resolve-api-meta/action.yml new file mode 100644 index 000000000..deb3c9fbf --- /dev/null +++ b/.github/actions/resolve-api-meta/action.yml @@ -0,0 +1,59 @@ +name: Resolve API commit/version +description: Resolve deployed API commit SHA and version from the metadata endpoint +inputs: + api_base_url: + description: Base URL host for the API (e.g. api-dev.mobilitydatabase.org) + required: false + default: api.mobilitydatabase.org + api_refresh_token: + description: API refresh token + required: false +outputs: + COMMIT_SHA: + description: Resolved commit SHA + value: ${{ steps.resolve.outputs.COMMIT_SHA }} + API_VERSION: + description: Resolved API version + value: ${{ steps.resolve.outputs.API_VERSION }} +runs: + using: composite + steps: + - id: resolve + name: Resolve via API and expose outputs + shell: bash + env: + API_BASE_URL: ${{ inputs.api_base_url }} + API_REFRESH_TOKEN: ${{ inputs.api_refresh_token }} + run: | + set -euo pipefail + COMMIT_SHA="" + API_VERSION="" + if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then + echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..." + REPLY_JSON=$(curl --fail --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \ + --header 'Content-Type: application/json' \ + --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }") + ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token) + if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then + echo "Error: Could not obtain access token from reply" >&2 + exit 1 + fi + META_JSON=$(curl --fail --silent --show-error \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'accept: application/json' \ + "https://${API_BASE_URL}/v1/metadata") + COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash) + API_VERSION=$(echo "${META_JSON}" | jq -r .version) + if [[ -z "${COMMIT_SHA}" || "${COMMIT_SHA}" == "null" ]]; then + echo "Error: Could not extract commit_hash from metadata" >&2 + echo "Metadata reply: ${META_JSON}" >&2 + exit 1 + fi + echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" + else + echo "No token provided; skipping API metadata resolution." + fi + echo "COMMIT_SHA=${COMMIT_SHA}" + echo "API_VERSION=${API_VERSION}" + echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT" From 7b34accee45746a0f483a73b83db78ab1e86ce42 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 09:07:21 -0400 Subject: [PATCH 05/17] Added a missing action. --- scripts/resolve_api_meta.sh | 58 ------------------------------------- 1 file changed, 58 deletions(-) delete mode 100755 scripts/resolve_api_meta.sh diff --git a/scripts/resolve_api_meta.sh b/scripts/resolve_api_meta.sh deleted file mode 100755 index 2c67dc9dc..000000000 --- a/scripts/resolve_api_meta.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# Resolve API metadata and set GitHub Action step outputs COMMIT_SHA and API_VERSION -# Inputs (via env): -# EVENT - GitHub event name (e.g., repository_dispatch) -# API_BASE_URL - API host (e.g., api-dev.mobilitydatabase.org) -# API_REFRESH_TOKEN- Refresh token to obtain access token (only used on repository_dispatch) -# Outputs: -# Prints COMMIT_SHA and API_VERSION to stdout as NAME=VALUE lines -# Also exports to $GITHUB_ENV if defined so the script is runnable by hand - -set -euo pipefail -# QA -#export API_REFRESH_TOKEN="AMf-vBzv1rT8AQ0uZrzNDD5wuMxSvxLVbXRmViTaOVuP8eh-uDdWLDpsDHYMrGNqq2sqn1ya_-i8YXZdWh9GnLoPSLbbWY99hmb-JUrJ_NXz4pJ5v2ysm3kCpjy02zN2uI5csAi1YmGSrlJoUQazNO4ntkVHkgdpsyMSBwgIGDczwX4qANzEjsRjtRCQlCbm_MnWsLaBKrukn5qxFbyVszzOloG5piIivTL700I9cPslxlzirmYYrj3jsYIX00RKBF3pPvTpiiOGskRjeZi_UvI5spux0tkFuZJZGt-vNKWeLT9MSJp6S3Y5os4PQMjCJ-StCP4Qkqwja5EiDRdVhqJVCT7XYn_MAQBYGFUYvFOQu9S9siKlhXO9Mcc2NiH89eaJ1EUlkXRnzY9nahP82cmt8VOKHyu6GAT3-l4V9_9zJ7wjyQJ0wSf_5wZlLxFSScLaIADZdRya" -# https://beta.mobilitydatabase.org/account -#export API_REFRESH_TOKEN="AMf-vByLiUKvxTqH_4vkQTJ8aStgCvrNGDyMINJNpYVdRCQliF6q0FmZxy7Y29rgNce4HGnrr2La3lja5CsOPn8-Vx6RA0enAv5RVxnBC08G-c6ZAiwCMfvUT-vT49ZLeULmUIM6BlCQKGdXSxAOljg6QJNH0wXdvhHaLUGVGdhoeKW3pg692ZCCWTVCWOFSDhHhS-d-8ywSd7nsFwka0ZMBRAtOeY-lgvwsz7Wo21hCTaHyUYSoodKzWFjUUJ9W7nA33OB6lKBA9tZGap3pYlO-Vo47jl-M2GcZn77R-sgsgEbPXfHxc0NMS1ZbNXKeWmrcqPYcgcEoBFdFrnQcd1bD-b1O8_zTnDyjjRekLq8bGKLnDfRg-qZvUaNxtKPUqFnh-pmvO7gXMxwj7Tnoc5cMmQxHGZ2euuxaVRHNMoAxnEzqIzZ7nxIM4NX36eUZCuLLSTXvOnmsdMuhQ_7oI2wX-YKxRGHnrw" -# https://mobility-feeds-dev.web.app/account -export API_REFRESH_TOKEN="AMf-vBxJqJLukaDstkJl_Pi-GonY1suIN_3O5Fp7pHfYcD_XVZwSJczM_815UpEJFQN9ShzaE5KRqsyO4sRnFUBAA6KLQgcP-7Mx9yNzRBGtBi-e37X50CNwqUvsuUfOjFZNeKPfaO1ipuCA9LQLWf-5e29DmyxEgO1Fy9UguKm5KlDRrcIkUjTzSBHuSRu06j1_th4TR4l0X5OteNGsG6F6N2lrihj2Z5Idx6PoJ02_5fgLOhop8mlGd-ktNpQ3J46lAz7BX7_UgPkbugwIoD9YdYvnIlrSp5hl3ri7mmauN0rOV1HEMn-aynd7Zqkc0lmNWWL3BqQe-Ik0sOt23HZISK7Y2EITrXEtXxPN0-824BKUAoLN-_eutLk3NFRJW19jQc-RrZRP1LzXdWQETKhXPRBWCf33U1N0Yv2qzBKTzhvFjh3yULRVnZjUxh7CnZeYDKbxAWYj" - -COMMIT_SHA="" -API_VERSION="" - -if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then - echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..." - REPLY_JSON=$(curl --fail --silent --show-error --location "https://${mobility-feeds-dev}/v1/tokens" \ - --header 'Content-Type: application/json' \ - --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }") - ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token) - if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then - echo "Error: Could not obtain access token from reply" >&2 - exit 1 - fi - META_JSON=$(curl --fail --silent --show-error \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'accept: application/json' \ - "https://${API_BASE_URL}/v1/metadata") - COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash) - API_VERSION=$(echo "${META_JSON}" | jq -r .version) - if [[ -z "${COMMIT_SHA}" || "${COMMIT_SHA}" == "null" ]]; then - echo "Error: Could not extract commit_hash from metadata" >&2 - echo "Metadata reply: ${META_JSON}" >&2 - exit 1 - fi - echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" -else - echo "No token provided; skipping API metadata resolution." -fi - -# Output values to stdout in a parse-friendly format -echo "COMMIT_SHA=${COMMIT_SHA}" -echo "API_VERSION=${API_VERSION}" - -# Optionally export to $GITHUB_ENV for subsequent steps when available -if [[ -n "${GITHUB_ENV:-}" ]]; then - { - echo "COMMIT_SHA=${COMMIT_SHA}" - echo "API_VERSION=${API_VERSION}" - } >> "$GITHUB_ENV" -fi From 1345cc3a0425478c03e4f268f963a6f5d8fab2c7 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 09:15:13 -0400 Subject: [PATCH 06/17] Put the IS_DISPATCH variable inline --- .github/workflows/db-update.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 5d5e79302..03d92d21f 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -81,11 +81,6 @@ on: default: false type: boolean -# Add workflow-level vars (these are available in job `if` expressions) -vars: - # True when this run was triggered by a repository_dispatch or a manual workflow_dispatch - IS_DISPATCH: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} - env: python_version: '3.11' liquibase_version: '4.33.0' @@ -94,7 +89,7 @@ jobs: resolve-api-meta: name: 'Resolve API commit/version' runs-on: ubuntu-latest - if: ${{ vars.IS_DISPATCH == 'true' && inputs.API_BASE_URL != '' && secrets.API_TEST_REFRESH_TOKEN != '' }} + if: ${{ contains('repository_dispatch,workflow_dispatch', github.event_name) && inputs.API_BASE_URL != '' }} outputs: COMMIT_SHA: ${{ steps.resolve.outputs.COMMIT_SHA }} API_VERSION: ${{ steps.resolve.outputs.API_VERSION }} @@ -103,6 +98,7 @@ jobs: uses: actions/checkout@v4 - name: Resolve API commit/version id: resolve + if: ${{ secrets.API_TEST_REFRESH_TOKEN != '' }} uses: ./.github/actions/resolve-api-meta with: api_base_url: ${{ inputs.API_BASE_URL }} @@ -186,7 +182,7 @@ jobs: permissions: write-all runs-on: ubuntu-latest needs: [resolve-api-meta, db-schema-update] - if: ${{ always() && vars.IS_DISPATCH == 'true' }} + if: ${{ always() && contains('repository_dispatch,workflow_dispatch', github.event_name) }} steps: - name: Checkout code at API commit if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }} @@ -302,7 +298,7 @@ jobs: update-gcp-secret: name: Update GCP Secrets - if: ${{ vars.IS_DISPATCH == 'true' && !inputs.DRY_RUN }} + if: ${{ contains('repository_dispatch,workflow_dispatch', github.event_name) && !inputs.DRY_RUN }} runs-on: ubuntu-latest steps: - name: Authenticate to Google Cloud @@ -340,3 +336,4 @@ jobs: echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..." echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID fi + From 0e7c145d2f53631248dda2907722dc2236572488 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 09:23:33 -0400 Subject: [PATCH 07/17] Put the IS_DISPATCH variable inline --- .github/workflows/db-update.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 03d92d21f..0cb476a48 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -98,7 +98,7 @@ jobs: uses: actions/checkout@v4 - name: Resolve API commit/version id: resolve - if: ${{ secrets.API_TEST_REFRESH_TOKEN != '' }} + # The composite action itself checks for a missing token and will skip resolution when none is provided. uses: ./.github/actions/resolve-api-meta with: api_base_url: ${{ inputs.API_BASE_URL }} @@ -336,4 +336,3 @@ jobs: echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..." echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID fi - From cee5de2b4394a360df25692b68f2723ad25ae95a Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 09:33:44 -0400 Subject: [PATCH 08/17] Modified to test on DEV --- .github/workflows/db-update-dev.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index f311d7dec..794138b09 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -10,6 +10,12 @@ on: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] workflow_dispatch: + inputs: + DRY_RUN: + description: Skip applying schema and content updates + required: false + default: false + type: boolean jobs: update: uses: ./.github/workflows/db-update.yml @@ -20,6 +26,7 @@ jobs: ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-dev.mobilitydatabase.org + DRY_RUN: ${{ inputs.DRY_RUN }} secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} From c2b8b92962b62d2b3494ea056ba4b1b5fd944d6f Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 13:21:24 -0400 Subject: [PATCH 09/17] Added an install mode. --- .github/actions/resolve-api-meta/action.yml | 7 +++ .github/workflows/db-update-dev.yml | 6 +++ .github/workflows/db-update-prod.yml | 13 +++++ .github/workflows/db-update-qa.yml | 12 +++++ .github/workflows/db-update.yml | 55 +++++++++++++++------ 5 files changed, 78 insertions(+), 15 deletions(-) diff --git a/.github/actions/resolve-api-meta/action.yml b/.github/actions/resolve-api-meta/action.yml index deb3c9fbf..f9ec04bab 100644 --- a/.github/actions/resolve-api-meta/action.yml +++ b/.github/actions/resolve-api-meta/action.yml @@ -15,6 +15,9 @@ outputs: API_VERSION: description: Resolved API version value: ${{ steps.resolve.outputs.API_VERSION }} + RESOLVED: + description: Whether a commit SHA was resolved (true/false) + value: ${{ steps.resolve.outputs.RESOLVED }} runs: using: composite steps: @@ -28,6 +31,7 @@ runs: set -euo pipefail COMMIT_SHA="" API_VERSION="" + RESOLVED="false" if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..." REPLY_JSON=$(curl --fail --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \ @@ -49,11 +53,14 @@ runs: echo "Metadata reply: ${META_JSON}" >&2 exit 1 fi + RESOLVED="true" echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" else echo "No token provided; skipping API metadata resolution." fi echo "COMMIT_SHA=${COMMIT_SHA}" echo "API_VERSION=${API_VERSION}" + echo "RESOLVED=${RESOLVED}" echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT" + echo "RESOLVED=${RESOLVED}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index 794138b09..81ba16953 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -16,6 +16,11 @@ on: required: false default: false type: boolean + INSTALL_MODE: + description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + required: false + default: latest + type: string jobs: update: uses: ./.github/workflows/db-update.yml @@ -27,6 +32,7 @@ jobs: DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-dev.mobilitydatabase.org DRY_RUN: ${{ inputs.DRY_RUN }} + INSTALL_MODE: ${{ inputs.INSTALL_MODE }} secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index 0f157ce92..414368da1 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -2,7 +2,19 @@ name: Database Update - PROD on: workflow_dispatch: + inputs: + INSTALL_MODE: + description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + required: false + default: installed + type: string workflow_call: + inputs: + INSTALL_MODE: + description: Which API version to install when called as a reusable workflow (defaults to latest) + required: false + default: latest + type: string repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] jobs: @@ -15,6 +27,7 @@ jobs: ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api.mobilitydatabase.org + INSTALL_MODE: ${{ inputs.INSTALL_MODE }} secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index d876d1ee7..f62a99d5f 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -8,7 +8,18 @@ on: required: false default: false type: boolean + INSTALL_MODE: + description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + required: false + default: installed + type: string workflow_call: + inputs: + INSTALL_MODE: + description: Which API version to install when called as a reusable workflow (defaults to latest) + required: false + default: latest + type: string repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -24,6 +35,7 @@ jobs: DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-qa.mobilitydatabase.org DRY_RUN: ${{ inputs.DRY_RUN }} + INSTALL_MODE: ${{ inputs.INSTALL_MODE }} secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 0cb476a48..b0353075a 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -75,6 +75,11 @@ on: required: false default: api.mobilitydatabase.org type: string + INSTALL_MODE: + description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + required: false + default: latest + type: string DRY_RUN: description: Skip applying schema and content updates required: false @@ -89,10 +94,12 @@ jobs: resolve-api-meta: name: 'Resolve API commit/version' runs-on: ubuntu-latest - if: ${{ contains('repository_dispatch,workflow_dispatch', github.event_name) && inputs.API_BASE_URL != '' }} + # Run this job for all triggers; the action itself will skip resolution when API_BASE_URL or token is not provided. + # Keeping it unconditional ensures CHECKOUT_REF is always set (defaults to 'main') for downstream jobs. outputs: COMMIT_SHA: ${{ steps.resolve.outputs.COMMIT_SHA }} API_VERSION: ${{ steps.resolve.outputs.API_VERSION }} + CHECKOUT_REF: ${{ steps.decide_ref.outputs.CHECKOUT_REF }} steps: - name: Checkout repo (for scripts and local action) uses: actions/checkout@v4 @@ -103,6 +110,34 @@ jobs: with: api_base_url: ${{ inputs.API_BASE_URL }} api_refresh_token: ${{ secrets.API_TEST_REFRESH_TOKEN }} + - name: Determine install target ref + id: decide_ref + run: | + set -euo pipefail + EVENT='${{ github.event_name }}' + INSTALL_MODE='${{ inputs.INSTALL_MODE }}' + RESOLVED='${{ steps.resolve.outputs.RESOLVED }}' + COMMIT_SHA='${{ steps.resolve.outputs.COMMIT_SHA }}' + echo "event=$EVENT, install_mode=$INSTALL_MODE, resolved=$RESOLVED, commit=$COMMIT_SHA" + if [ "$EVENT" = "repository_dispatch" ]; then + if [ "$RESOLVED" != "true" ] || [ -z "$COMMIT_SHA" ]; then + echo "ERROR: repository_dispatch requires a resolved API commit but none was found" >&2 + exit 1 + fi + echo "CHECKOUT_REF=$COMMIT_SHA" >> "$GITHUB_OUTPUT" + elif [ "$EVENT" = "workflow_dispatch" ]; then + if [ "$INSTALL_MODE" = "installed" ]; then + if [ "$RESOLVED" != "true" ] || [ -z "$COMMIT_SHA" ]; then + echo "ERROR: manual install requested 'installed' but no resolved commit available" >&2 + exit 1 + fi + echo "CHECKOUT_REF=$COMMIT_SHA" >> "$GITHUB_OUTPUT" + else + echo "CHECKOUT_REF=main" >> "$GITHUB_OUTPUT" + fi + else + echo "CHECKOUT_REF=main" >> "$GITHUB_OUTPUT" + fi db-schema-update: name: 'Database Schema Update' @@ -111,17 +146,12 @@ jobs: needs: [resolve-api-meta] if: ${{ always() }} steps: - - name: Checkout code at API commit - if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }} + - name: Checkout repo (shallow) uses: actions/checkout@v4 with: - ref: ${{ needs.resolve-api-meta.outputs.COMMIT_SHA }} + ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} fetch-depth: 0 - - name: Checkout code (default) - if: ${{ needs.resolve-api-meta.result != 'success' || needs.resolve-api-meta.outputs.COMMIT_SHA == '' }} - uses: actions/checkout@v4 - - name: Authenticate to Google Cloud QA/PROD uses: google-github-actions/auth@v2 with: @@ -184,17 +214,12 @@ jobs: needs: [resolve-api-meta, db-schema-update] if: ${{ always() && contains('repository_dispatch,workflow_dispatch', github.event_name) }} steps: - - name: Checkout code at API commit - if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }} + - name: Checkout repo (shallow) uses: actions/checkout@v4 with: - ref: ${{ needs.resolve-api-meta.outputs.COMMIT_SHA }} + ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} fetch-depth: 0 - - name: Checkout code (default) - if: ${{ needs.resolve-api-meta.result != 'success' || needs.resolve-api-meta.outputs.COMMIT_SHA == '' }} - uses: actions/checkout@v4 - - name: Setup python uses: actions/setup-python@v5 with: From 364cf7c78577a0321aaa934abc8e127de649f7f5 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 14:06:56 -0400 Subject: [PATCH 10/17] Added an install mode. --- .github/workflows/db-update-dev.yml | 12 +++++------ .github/workflows/db-update-prod.yml | 30 +++++++++++++++++----------- .github/workflows/db-update-qa.yml | 20 +++++++------------ .github/workflows/db-update.yml | 24 +++++++++++++--------- 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index 81ba16953..fb9b6f935 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -16,11 +16,11 @@ on: required: false default: false type: boolean - INSTALL_MODE: - description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + INSTALL_CURRENT: + description: Install the currently deployed API version when true; when false install main (latest). required: false - default: latest - type: string + default: true + type: boolean jobs: update: uses: ./.github/workflows/db-update.yml @@ -31,8 +31,8 @@ jobs: ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-dev.mobilitydatabase.org - DRY_RUN: ${{ inputs.DRY_RUN }} - INSTALL_MODE: ${{ inputs.INSTALL_MODE }} + DRY_RUN: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.DRY_RUN || 'false') || inputs.DRY_RUN || 'false' }} + INSTALL_CURRENT: ${{ github.event_name == 'repository_dispatch' && 'true' || inputs.INSTALL_CURRENT || 'true' }} secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index 414368da1..8bac2c48c 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -1,20 +1,19 @@ # Update the Mobility Database Schema name: Database Update - PROD on: - workflow_dispatch: + workflow_dispatch: # Manual trigger inputs: - INSTALL_MODE: - description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + DRY_RUN: + description: Skip applying schema and content updates required: false - default: installed - type: string - workflow_call: - inputs: - INSTALL_MODE: - description: Which API version to install when called as a reusable workflow (defaults to latest) + default: false + type: boolean + INSTALL_CURRENT: + description: Install the currently deployed API version when true; when false install main (latest). required: false - default: latest - type: string + default: true + type: boolean + workflow_call: [] repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] jobs: @@ -27,7 +26,14 @@ jobs: ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api.mobilitydatabase.org - INSTALL_MODE: ${{ inputs.INSTALL_MODE }} + DRY_RUN: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.DRY_RUN || 'false') || inputs.DRY_RUN || 'false' }} + # Algorithm for INSTALL_CURRENT (forwarded to db-update.yml): + # - If the trigger is repository_dispatch, force 'true' because we want to re-install the same repo-dispatch runs must + # install the currently deployed API version (fail if unknown). + # - If the trigger is from another workflow (workflow_call), currently from release.yml, force 'false' + # because it's a release, we want the latest not the current. + # - Otherwise (manual workflow_dispatch) use the wrapper's inputs.INSTALL_CURRENT. + INSTALL_CURRENT: ${{ github.event_name == 'repository_dispatch' && 'true' || github.event_name == 'workflow_call' && 'false' || inputs.INSTALL_CURRENT || 'false' }} secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index f62a99d5f..a1a619d3a 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -8,18 +8,12 @@ on: required: false default: false type: boolean - INSTALL_MODE: - description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + INSTALL_CURRENT: + description: Install the currently deployed API version when true; when false install main (latest). required: false - default: installed - type: string - workflow_call: - inputs: - INSTALL_MODE: - description: Which API version to install when called as a reusable workflow (defaults to latest) - required: false - default: latest - type: string + default: false + type: boolean + workflow_call: [] repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -34,8 +28,8 @@ jobs: ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-qa.mobilitydatabase.org - DRY_RUN: ${{ inputs.DRY_RUN }} - INSTALL_MODE: ${{ inputs.INSTALL_MODE }} + DRY_RUN: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.DRY_RUN || 'false') || inputs.DRY_RUN || 'false' }} + INSTALL_CURRENT: ${{ github.event_name == 'repository_dispatch' && 'true' || github.event_name == 'workflow_call' && 'false' || inputs.INSTALL_CURRENT || 'false' }} secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index b0353075a..5c2edeac0 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -75,11 +75,11 @@ on: required: false default: api.mobilitydatabase.org type: string - INSTALL_MODE: - description: Which API version to install when run manually. "installed" = install the currently deployed API version; "latest" = install latest from main. + INSTALL_CURRENT: + description: Install the currently deployed API version when true; when false install main (latest). required: false - default: latest - type: string + default: false + type: boolean DRY_RUN: description: Skip applying schema and content updates required: false @@ -115,20 +115,22 @@ jobs: run: | set -euo pipefail EVENT='${{ github.event_name }}' - INSTALL_MODE='${{ inputs.INSTALL_MODE }}' + INSTALL_CURRENT='${{ inputs.INSTALL_CURRENT }}' RESOLVED='${{ steps.resolve.outputs.RESOLVED }}' COMMIT_SHA='${{ steps.resolve.outputs.COMMIT_SHA }}' - echo "event=$EVENT, install_mode=$INSTALL_MODE, resolved=$RESOLVED, commit=$COMMIT_SHA" + echo "event=$EVENT, install_current=$INSTALL_CURRENT, resolved=$RESOLVED, commit=$COMMIT_SHA" if [ "$EVENT" = "repository_dispatch" ]; then + # For repository_dispatch we always install the currently deployed version; fail if missing if [ "$RESOLVED" != "true" ] || [ -z "$COMMIT_SHA" ]; then echo "ERROR: repository_dispatch requires a resolved API commit but none was found" >&2 exit 1 fi echo "CHECKOUT_REF=$COMMIT_SHA" >> "$GITHUB_OUTPUT" elif [ "$EVENT" = "workflow_dispatch" ]; then - if [ "$INSTALL_MODE" = "installed" ]; then + # For manual runs, INSTALL_CURRENT=true -> install deployed; false -> install main + if [ "$INSTALL_CURRENT" = "true" ]; then if [ "$RESOLVED" != "true" ] || [ -z "$COMMIT_SHA" ]; then - echo "ERROR: manual install requested 'installed' but no resolved commit available" >&2 + echo "ERROR: manual install requested current version but no resolved commit available" >&2 exit 1 fi echo "CHECKOUT_REF=$COMMIT_SHA" >> "$GITHUB_OUTPUT" @@ -136,6 +138,7 @@ jobs: echo "CHECKOUT_REF=main" >> "$GITHUB_OUTPUT" fi else + # push or other events: default to main echo "CHECKOUT_REF=main" >> "$GITHUB_OUTPUT" fi @@ -144,7 +147,10 @@ jobs: permissions: write-all runs-on: ubuntu-latest needs: [resolve-api-meta] - if: ${{ always() }} + # If we need to install the currently installed version Only run schema update when we're NOT installing the currently deployed API version. + # INSTALL_CURRENT is a boolean input: true means install the deployed commit (skip schema), + # false means install the latest from main (apply schema updates). + if: ${{ !inputs.INSTALL_CURRENT }} steps: - name: Checkout repo (shallow) uses: actions/checkout@v4 From 7191c49f12ef1de1b4520845bdc50ce3f0bb2cfb Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 16:22:45 -0400 Subject: [PATCH 11/17] Simplified --- .github/actions/resolve-api-meta/action.yml | 3 +- .github/workflows/db-update-dev.yml | 12 ++--- .github/workflows/db-update-prod.yml | 18 ++----- .github/workflows/db-update-qa.yml | 12 ++--- .github/workflows/db-update.yml | 56 ++++----------------- 5 files changed, 28 insertions(+), 73 deletions(-) diff --git a/.github/actions/resolve-api-meta/action.yml b/.github/actions/resolve-api-meta/action.yml index f9ec04bab..fc5f3444a 100644 --- a/.github/actions/resolve-api-meta/action.yml +++ b/.github/actions/resolve-api-meta/action.yml @@ -56,7 +56,8 @@ runs: RESOLVED="true" echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" else - echo "No token provided; skipping API metadata resolution." + echo "Error: no API refresh token provided; cannot resolve deployed commit." >&2 + exit 1 fi echo "COMMIT_SHA=${COMMIT_SHA}" echo "API_VERSION=${API_VERSION}" diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index fb9b6f935..d4c63c9ed 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -16,11 +16,6 @@ on: required: false default: false type: boolean - INSTALL_CURRENT: - description: Install the currently deployed API version when true; when false install main (latest). - required: false - default: true - type: boolean jobs: update: uses: ./.github/workflows/db-update.yml @@ -31,8 +26,11 @@ jobs: ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-dev.mobilitydatabase.org - DRY_RUN: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.DRY_RUN || 'false') || inputs.DRY_RUN || 'false' }} - INSTALL_CURRENT: ${{ github.event_name == 'repository_dispatch' && 'true' || inputs.INSTALL_CURRENT || 'true' }} + # DRY_RUN is only if requested by the user in a workflow_dispatch + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} + # We always want to install the latest except if we received a dispatch from mobility_database_catalog. + # In that case we want to keep the currently installed version. + INSTALL_LATEST: ${{ github.event_name != 'repository_dispatch' }} secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index 8bac2c48c..a72e084b6 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -8,11 +8,6 @@ on: required: false default: false type: boolean - INSTALL_CURRENT: - description: Install the currently deployed API version when true; when false install main (latest). - required: false - default: true - type: boolean workflow_call: [] repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -26,14 +21,11 @@ jobs: ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api.mobilitydatabase.org - DRY_RUN: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.DRY_RUN || 'false') || inputs.DRY_RUN || 'false' }} - # Algorithm for INSTALL_CURRENT (forwarded to db-update.yml): - # - If the trigger is repository_dispatch, force 'true' because we want to re-install the same repo-dispatch runs must - # install the currently deployed API version (fail if unknown). - # - If the trigger is from another workflow (workflow_call), currently from release.yml, force 'false' - # because it's a release, we want the latest not the current. - # - Otherwise (manual workflow_dispatch) use the wrapper's inputs.INSTALL_CURRENT. - INSTALL_CURRENT: ${{ github.event_name == 'repository_dispatch' && 'true' || github.event_name == 'workflow_call' && 'false' || inputs.INSTALL_CURRENT || 'false' }} + # DRY_RUN is only if requested by the user in a workflow_dispatch + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} + # We always want to install the latest except if we received a dispatch from mobility_database_catalog. + # In that case we want to keep the currently installed version. + INSTALL_LATEST: ${{ github.event_name != 'repository_dispatch' }} secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index a1a619d3a..9e7098ba5 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -8,11 +8,6 @@ on: required: false default: false type: boolean - INSTALL_CURRENT: - description: Install the currently deployed API version when true; when false install main (latest). - required: false - default: false - type: boolean workflow_call: [] repository_dispatch: # Update on mobility-database-catalog repo dispatch @@ -28,8 +23,11 @@ jobs: ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} API_BASE_URL: api-qa.mobilitydatabase.org - DRY_RUN: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.DRY_RUN || 'false') || inputs.DRY_RUN || 'false' }} - INSTALL_CURRENT: ${{ github.event_name == 'repository_dispatch' && 'true' || github.event_name == 'workflow_call' && 'false' || inputs.INSTALL_CURRENT || 'false' }} + # DRY_RUN is only if requested by the user in a workflow_dispatch + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} + # We always want to install the latest except if we received a dispatch from mobility_database_catalog. + # In that case we want to keep the currently installed version. + INSTALL_LATEST: ${{ github.event_name != 'repository_dispatch' }} secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 5c2edeac0..34ff27841 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -75,8 +75,8 @@ on: required: false default: api.mobilitydatabase.org type: string - INSTALL_CURRENT: - description: Install the currently deployed API version when true; when false install main (latest). + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false keep the currently deployed version. required: false default: false type: boolean @@ -97,64 +97,32 @@ jobs: # Run this job for all triggers; the action itself will skip resolution when API_BASE_URL or token is not provided. # Keeping it unconditional ensures CHECKOUT_REF is always set (defaults to 'main') for downstream jobs. outputs: - COMMIT_SHA: ${{ steps.resolve.outputs.COMMIT_SHA }} - API_VERSION: ${{ steps.resolve.outputs.API_VERSION }} - CHECKOUT_REF: ${{ steps.decide_ref.outputs.CHECKOUT_REF }} + # Use resolved commit when available; otherwise default to 'main'. + CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }} steps: - name: Checkout repo (for scripts and local action) uses: actions/checkout@v4 - name: Resolve API commit/version id: resolve - # The composite action itself checks for a missing token and will skip resolution when none is provided. + if: ${{ inputs.INSTALL_LATEST == false }} uses: ./.github/actions/resolve-api-meta with: api_base_url: ${{ inputs.API_BASE_URL }} api_refresh_token: ${{ secrets.API_TEST_REFRESH_TOKEN }} - - name: Determine install target ref - id: decide_ref - run: | - set -euo pipefail - EVENT='${{ github.event_name }}' - INSTALL_CURRENT='${{ inputs.INSTALL_CURRENT }}' - RESOLVED='${{ steps.resolve.outputs.RESOLVED }}' - COMMIT_SHA='${{ steps.resolve.outputs.COMMIT_SHA }}' - echo "event=$EVENT, install_current=$INSTALL_CURRENT, resolved=$RESOLVED, commit=$COMMIT_SHA" - if [ "$EVENT" = "repository_dispatch" ]; then - # For repository_dispatch we always install the currently deployed version; fail if missing - if [ "$RESOLVED" != "true" ] || [ -z "$COMMIT_SHA" ]; then - echo "ERROR: repository_dispatch requires a resolved API commit but none was found" >&2 - exit 1 - fi - echo "CHECKOUT_REF=$COMMIT_SHA" >> "$GITHUB_OUTPUT" - elif [ "$EVENT" = "workflow_dispatch" ]; then - # For manual runs, INSTALL_CURRENT=true -> install deployed; false -> install main - if [ "$INSTALL_CURRENT" = "true" ]; then - if [ "$RESOLVED" != "true" ] || [ -z "$COMMIT_SHA" ]; then - echo "ERROR: manual install requested current version but no resolved commit available" >&2 - exit 1 - fi - echo "CHECKOUT_REF=$COMMIT_SHA" >> "$GITHUB_OUTPUT" - else - echo "CHECKOUT_REF=main" >> "$GITHUB_OUTPUT" - fi - else - # push or other events: default to main - echo "CHECKOUT_REF=main" >> "$GITHUB_OUTPUT" - fi db-schema-update: name: 'Database Schema Update' permissions: write-all runs-on: ubuntu-latest needs: [resolve-api-meta] - # If we need to install the currently installed version Only run schema update when we're NOT installing the currently deployed API version. - # INSTALL_CURRENT is a boolean input: true means install the deployed commit (skip schema), - # false means install the latest from main (apply schema updates). - if: ${{ !inputs.INSTALL_CURRENT }} + # Only run the schema update when INSTALL_LATEST is true (we're installing main/latest). + # If not, we have nothing to do since we will use the version already installed. + if: ${{ inputs.INSTALL_LATEST == true }} steps: - name: Checkout repo (shallow) uses: actions/checkout@v4 with: + # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} fetch-depth: 0 @@ -218,11 +186,12 @@ jobs: permissions: write-all runs-on: ubuntu-latest needs: [resolve-api-meta, db-schema-update] - if: ${{ always() && contains('repository_dispatch,workflow_dispatch', github.event_name) }} + if: ${{ always() }} steps: - name: Checkout repo (shallow) uses: actions/checkout@v4 with: + # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} fetch-depth: 0 @@ -240,8 +209,6 @@ jobs: uses: google-github-actions/setup-gcloud@v2 - name: Update .env file - env: - API_VERSION: ${{ needs.resolve-api-meta.outputs.API_VERSION }} run: | echo "PGUSER=${{ secrets.DB_USER_NAME }}" > config/.env.local echo "POSTGRES_USER=${{ secrets.DB_USER_NAME }}" >> config/.env.local @@ -251,7 +218,6 @@ jobs: echo "POSTGRES_PORT=5432" >> config/.env.local echo "POSTGRES_HOST=localhost" >> config/.env.local echo "ENV=${{ inputs.ENVIRONMENT }}" >> config/.env.local - if [[ -n "${API_VERSION}" ]]; then echo "API_VERSION=${API_VERSION}" >> config/.env.local; fi cat config/.env.local - name: Load secrets from 1Password From c2738d1cdec4fdb0a4f29a6f31c1d270d547ed91 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 16:40:35 -0400 Subject: [PATCH 12/17] Added INSTALL_LATEST for manual trigger so it's somewhat testable. --- .github/workflows/db-update-dev.yml | 13 +++++++++---- .github/workflows/db-update-prod.yml | 20 ++++++++++++++++---- .github/workflows/db-update-qa.yml | 16 ++++++++++++---- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index d4c63c9ed..5bbff374c 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -12,7 +12,7 @@ on: workflow_dispatch: inputs: DRY_RUN: - description: Skip applying schema and content updates + description: Dry run. Skip applying schema and content updates required: false default: false type: boolean @@ -28,9 +28,14 @@ jobs: API_BASE_URL: api-dev.mobilitydatabase.org # DRY_RUN is only if requested by the user in a workflow_dispatch DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} - # We always want to install the latest except if we received a dispatch from mobility_database_catalog. - # In that case we want to keep the currently installed version. - INSTALL_LATEST: ${{ github.event_name != 'repository_dispatch' }} + # We want to use the currently installed version (not the latest) if we received a dispatch from + # mobility_database_catalog. + # For a workflow_dispatch (manual trigger), we use the value set by the user. + INSTALL_LATEST: ${{ + (github.event_name == 'repository_dispatch' && false) || + (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) + } } + secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index a72e084b6..0c529baf1 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -4,10 +4,15 @@ on: workflow_dispatch: # Manual trigger inputs: DRY_RUN: - description: Skip applying schema and content updates + description: Dry run. Skip applying schema and content updates required: false default: false type: boolean + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false install the currently deployed version. + required: false + default: true + type: boolean workflow_call: [] repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -23,9 +28,16 @@ jobs: API_BASE_URL: api.mobilitydatabase.org # DRY_RUN is only if requested by the user in a workflow_dispatch DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} - # We always want to install the latest except if we received a dispatch from mobility_database_catalog. - # In that case we want to keep the currently installed version. - INSTALL_LATEST: ${{ github.event_name != 'repository_dispatch' }} + # We want to use the currently installed version (not the latest) if we received a dispatch from + # mobility_database_catalog. + # We want to use the latest version if the trigger was workflow_call, because currently it's used for + # upgrading (e.g. release.yml) + # For a workflow_dispatch (manual trigger), we use the value set by the user. + INSTALL_LATEST: ${{ + (github.event_name == 'repository_dispatch' && false) || + (github.event_name == 'workflow_call' && true) || + (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) + }} secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index 9e7098ba5..c1d751269 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: DRY_RUN: - description: Skip applying schema and content updates + description: Dry run. Skip applying schema and content updates required: false default: false type: boolean @@ -25,9 +25,17 @@ jobs: API_BASE_URL: api-qa.mobilitydatabase.org # DRY_RUN is only if requested by the user in a workflow_dispatch DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} - # We always want to install the latest except if we received a dispatch from mobility_database_catalog. - # In that case we want to keep the currently installed version. - INSTALL_LATEST: ${{ github.event_name != 'repository_dispatch' }} + # We want to use the currently installed version (not the latest) if we received a dispatch from + # mobility_database_catalog. + # We want to use the latest version if the trigger was workflow_call, because currently it's used for + # upgrading (e.g. release.yml) + # For a workflow_dispatch (manual trigger), we use the value set by the user. + INSTALL_LATEST: ${{ + (github.event_name == 'repository_dispatch' && false) || + (github.event_name == 'workflow_call' && true) || + (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) + }} + secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} From fa2e15949a4462b065717b50a96dd1481be7dac2 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 22:06:44 -0400 Subject: [PATCH 13/17] Added INSTALL_LATEST for manual trigger so it's somewhat testable. --- .github/workflows/db-update-dev.yml | 5 +---- .github/workflows/db-update-prod.yml | 7 ++----- .github/workflows/db-update-qa.yml | 6 +----- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index 5bbff374c..f52d4ec18 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -31,10 +31,7 @@ jobs: # We want to use the currently installed version (not the latest) if we received a dispatch from # mobility_database_catalog. # For a workflow_dispatch (manual trigger), we use the value set by the user. - INSTALL_LATEST: ${{ - (github.event_name == 'repository_dispatch' && false) || - (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) - } } + INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index 0c529baf1..f6db72576 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -33,11 +33,8 @@ jobs: # We want to use the latest version if the trigger was workflow_call, because currently it's used for # upgrading (e.g. release.yml) # For a workflow_dispatch (manual trigger), we use the value set by the user. - INSTALL_LATEST: ${{ - (github.event_name == 'repository_dispatch' && false) || - (github.event_name == 'workflow_call' && true) || - (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) - }} + INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} + secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index c1d751269..54fa2d6e5 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -30,11 +30,7 @@ jobs: # We want to use the latest version if the trigger was workflow_call, because currently it's used for # upgrading (e.g. release.yml) # For a workflow_dispatch (manual trigger), we use the value set by the user. - INSTALL_LATEST: ${{ - (github.event_name == 'repository_dispatch' && false) || - (github.event_name == 'workflow_call' && true) || - (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) - }} + INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} From fb9910da8397b632382f68596bfae1745c76e08b Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 22:10:24 -0400 Subject: [PATCH 14/17] Added INSTALL_LATEST for manual trigger so it's somewhat testable. --- .github/workflows/db-update-dev.yml | 5 +++++ .github/workflows/db-update-prod.yml | 2 +- .github/workflows/db-update-qa.yml | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index f52d4ec18..b26cfdc1c 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -16,6 +16,11 @@ on: required: false default: false type: boolean + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false install the currently deployed version. + required: false + default: false + type: boolean jobs: update: uses: ./.github/workflows/db-update.yml diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index f6db72576..9c21d40db 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -11,7 +11,7 @@ on: INSTALL_LATEST: description: Install the latest (main) API version when true; when false install the currently deployed version. required: false - default: true + default: false type: boolean workflow_call: [] repository_dispatch: # Update on mobility-database-catalog repo dispatch diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index 54fa2d6e5..d87120a06 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -8,6 +8,11 @@ on: required: false default: false type: boolean + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false install the currently deployed version. + required: false + default: false + type: boolean workflow_call: [] repository_dispatch: # Update on mobility-database-catalog repo dispatch From 0140b453399161c8e46379e5a00d2d96c5691f3d Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 30 Oct 2025 23:00:15 -0400 Subject: [PATCH 15/17] Added INSTALL_LATEST for manual trigger so it's somewhat testable. --- .github/workflows/db-update-prod.yml | 3 ++- .github/workflows/db-update-qa.yml | 13 ++++++------- .github/workflows/db-update.yml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index 9c21d40db..2c539965b 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -13,9 +13,10 @@ on: required: false default: false type: boolean - workflow_call: [] + workflow_call: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] + jobs: update: uses: ./.github/workflows/db-update.yml diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index d87120a06..78a3084d7 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -8,13 +8,12 @@ on: required: false default: false type: boolean - INSTALL_LATEST: - description: Install the latest (main) API version when true; when false install the currently deployed version. - required: false - default: false - type: boolean - workflow_call: [] - + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false install the currently deployed version. + required: false + default: false + type: boolean + workflow_call: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 34ff27841..2c6e59eb9 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -119,7 +119,7 @@ jobs: # If not, we have nothing to do since we will use the version already installed. if: ${{ inputs.INSTALL_LATEST == true }} steps: - - name: Checkout repo (shallow) + - name: Checkout repo uses: actions/checkout@v4 with: # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') @@ -188,7 +188,7 @@ jobs: needs: [resolve-api-meta, db-schema-update] if: ${{ always() }} steps: - - name: Checkout repo (shallow) + - name: Checkout repo uses: actions/checkout@v4 with: # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') From ec8fdf5bfdf1a55eeace92a667a7776470b0b90f Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 31 Oct 2025 09:50:43 -0400 Subject: [PATCH 16/17] Used the main branch if for any reason we can't obtain the current commit hash. --- .github/actions/resolve-api-meta/action.yml | 63 +++++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/.github/actions/resolve-api-meta/action.yml b/.github/actions/resolve-api-meta/action.yml index fc5f3444a..72ef05154 100644 --- a/.github/actions/resolve-api-meta/action.yml +++ b/.github/actions/resolve-api-meta/action.yml @@ -28,40 +28,53 @@ runs: API_BASE_URL: ${{ inputs.api_base_url }} API_REFRESH_TOKEN: ${{ inputs.api_refresh_token }} run: | - set -euo pipefail + # Do not exit on failure; this action should never abort the caller workflow. + set -u COMMIT_SHA="" API_VERSION="" RESOLVED="false" + if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..." - REPLY_JSON=$(curl --fail --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \ + + # Exchange refresh token -> access token (handle failures gracefully) + REPLY_JSON=$(curl --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \ --header 'Content-Type: application/json' \ - --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }") - ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token) - if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then - echo "Error: Could not obtain access token from reply" >&2 - exit 1 + --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }" ) || { + echo "Warning: token exchange failed; will fallback to 'main'" >&2 + REPLY_JSON="" + } + + if [[ -n "${REPLY_JSON}" ]]; then + ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token 2>/dev/null || echo "") + if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then + echo "Warning: Could not obtain access token from reply; will fallback to 'main'" >&2 + else + META_JSON=$(curl --silent --show-error \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'accept: application/json' \ + "https://${API_BASE_URL}/v1/metadata" ) || { + echo "Warning: metadata request failed; will fallback to 'main'" >&2 + META_JSON="" + } + + if [[ -n "${META_JSON}" ]]; then + COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash 2>/dev/null || echo "") + API_VERSION=$(echo "${META_JSON}" | jq -r .version 2>/dev/null || echo "") + if [[ -n "${COMMIT_SHA}" && "${COMMIT_SHA}" != "null" ]]; then + RESOLVED="true" + echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" + else + echo "Warning: commit_hash missing in metadata; will fallback to 'main'" >&2 + fi + fi + fi fi - META_JSON=$(curl --fail --silent --show-error \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'accept: application/json' \ - "https://${API_BASE_URL}/v1/metadata") - COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash) - API_VERSION=$(echo "${META_JSON}" | jq -r .version) - if [[ -z "${COMMIT_SHA}" || "${COMMIT_SHA}" == "null" ]]; then - echo "Error: Could not extract commit_hash from metadata" >&2 - echo "Metadata reply: ${META_JSON}" >&2 - exit 1 - fi - RESOLVED="true" - echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" else - echo "Error: no API refresh token provided; cannot resolve deployed commit." >&2 - exit 1 + echo "No API refresh token provided; skipping API metadata resolution and falling back to 'main'." fi - echo "COMMIT_SHA=${COMMIT_SHA}" - echo "API_VERSION=${API_VERSION}" - echo "RESOLVED=${RESOLVED}" + + # Expose outputs (empty COMMIT_SHA and RESOLVED=false indicate fallback to 'main') echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT" echo "RESOLVED=${RESOLVED}" >> "$GITHUB_OUTPUT" From 3ddd09c1e52872cf272df7c148c52abc16b059f0 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 31 Oct 2025 10:06:37 -0400 Subject: [PATCH 17/17] Used the main branch if for any reason we can't obtain the current commit hash. --- .github/workflows/db-update.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 2c6e59eb9..584cd9729 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -115,9 +115,9 @@ jobs: permissions: write-all runs-on: ubuntu-latest needs: [resolve-api-meta] - # Only run the schema update when INSTALL_LATEST is true (we're installing main/latest). - # If not, we have nothing to do since we will use the version already installed. - if: ${{ inputs.INSTALL_LATEST == true }} + # Run the schema update when the resolved checkout target is 'main' (install latest/main). + # This covers both explicit INSTALL_LATEST runs and cases where resolution failed and CHECKOUT_REF fell back to 'main'. + if: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF == 'main' }} steps: - name: Checkout repo uses: actions/checkout@v4