diff --git a/.github/actions/resolve-api-meta/action.yml b/.github/actions/resolve-api-meta/action.yml new file mode 100644 index 000000000..72ef05154 --- /dev/null +++ b/.github/actions/resolve-api-meta/action.yml @@ -0,0 +1,80 @@ +name: Resolve API commit/version +description: Resolve deployed API commit SHA and version from the metadata endpoint +inputs: + api_base_url: + description: Base URL host for the API (e.g. api-dev.mobilitydatabase.org) + required: false + default: api.mobilitydatabase.org + api_refresh_token: + description: API refresh token + required: false +outputs: + COMMIT_SHA: + description: Resolved commit SHA + value: ${{ steps.resolve.outputs.COMMIT_SHA }} + API_VERSION: + description: Resolved API version + value: ${{ steps.resolve.outputs.API_VERSION }} + RESOLVED: + description: Whether a commit SHA was resolved (true/false) + value: ${{ steps.resolve.outputs.RESOLVED }} +runs: + using: composite + steps: + - id: resolve + name: Resolve via API and expose outputs + shell: bash + env: + API_BASE_URL: ${{ inputs.api_base_url }} + API_REFRESH_TOKEN: ${{ inputs.api_refresh_token }} + run: | + # Do not exit on failure; this action should never abort the caller workflow. + set -u + COMMIT_SHA="" + API_VERSION="" + RESOLVED="false" + + if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then + echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..." + + # Exchange refresh token -> access token (handle failures gracefully) + REPLY_JSON=$(curl --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \ + --header 'Content-Type: application/json' \ + --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }" ) || { + echo "Warning: token exchange failed; will fallback to 'main'" >&2 + REPLY_JSON="" + } + + if [[ -n "${REPLY_JSON}" ]]; then + ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token 2>/dev/null || echo "") + if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then + echo "Warning: Could not obtain access token from reply; will fallback to 'main'" >&2 + else + META_JSON=$(curl --silent --show-error \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'accept: application/json' \ + "https://${API_BASE_URL}/v1/metadata" ) || { + echo "Warning: metadata request failed; will fallback to 'main'" >&2 + META_JSON="" + } + + if [[ -n "${META_JSON}" ]]; then + COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash 2>/dev/null || echo "") + API_VERSION=$(echo "${META_JSON}" | jq -r .version 2>/dev/null || echo "") + if [[ -n "${COMMIT_SHA}" && "${COMMIT_SHA}" != "null" ]]; then + RESOLVED="true" + echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" + else + echo "Warning: commit_hash missing in metadata; will fallback to 'main'" >&2 + fi + fi + fi + fi + else + echo "No API refresh token provided; skipping API metadata resolution and falling back to 'main'." + fi + + # Expose outputs (empty COMMIT_SHA and RESOLVED=false indicate fallback to 'main') + echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT" + echo "RESOLVED=${RESOLVED}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index 542671489..b26cfdc1c 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -10,6 +10,17 @@ on: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] workflow_dispatch: + inputs: + DRY_RUN: + description: Dry run. Skip applying schema and content updates + required: false + default: false + type: boolean + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false install the currently deployed version. + required: false + default: false + type: boolean jobs: update: uses: ./.github/workflows/db-update.yml @@ -19,6 +30,14 @@ jobs: DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} + API_BASE_URL: api-dev.mobilitydatabase.org + # DRY_RUN is only if requested by the user in a workflow_dispatch + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} + # We want to use the currently installed version (not the latest) if we received a dispatch from + # mobility_database_catalog. + # For a workflow_dispatch (manual trigger), we use the value set by the user. + INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} + secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} @@ -28,6 +47,7 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + API_TEST_REFRESH_TOKEN: ${{ secrets.DEV_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index cfeff46cb..2c539965b 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -1,10 +1,22 @@ # Update the Mobility Database Schema name: Database Update - PROD on: - workflow_dispatch: + workflow_dispatch: # Manual trigger + inputs: + DRY_RUN: + description: Dry run. Skip applying schema and content updates + required: false + default: false + type: boolean + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false install the currently deployed version. + required: false + default: false + type: boolean workflow_call: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] + jobs: update: uses: ./.github/workflows/db-update.yml @@ -14,6 +26,16 @@ jobs: DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} + API_BASE_URL: api.mobilitydatabase.org + # DRY_RUN is only if requested by the user in a workflow_dispatch + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} + # We want to use the currently installed version (not the latest) if we received a dispatch from + # mobility_database_catalog. + # We want to use the latest version if the trigger was workflow_call, because currently it's used for + # upgrading (e.g. release.yml) + # For a workflow_dispatch (manual trigger), we use the value set by the user. + INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} + secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} @@ -23,6 +45,7 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + API_TEST_REFRESH_TOKEN: ${{ secrets.PROD_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index 62262e68c..78a3084d7 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -2,6 +2,17 @@ name: Database Update - QA on: workflow_dispatch: + inputs: + DRY_RUN: + description: Dry run. Skip applying schema and content updates + required: false + default: false + type: boolean + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false install the currently deployed version. + required: false + default: false + type: boolean workflow_call: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -15,6 +26,16 @@ jobs: DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} + API_BASE_URL: api-qa.mobilitydatabase.org + # DRY_RUN is only if requested by the user in a workflow_dispatch + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} + # We want to use the currently installed version (not the latest) if we received a dispatch from + # mobility_database_catalog. + # We want to use the latest version if the trigger was workflow_call, because currently it's used for + # upgrading (e.g. release.yml) + # For a workflow_dispatch (manual trigger), we use the value set by the user. + INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} + secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} @@ -24,6 +45,7 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + API_TEST_REFRESH_TOKEN: ${{ secrets.QA_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 3ce05c37a..584cd9729 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -46,6 +46,9 @@ on: POSTGRE_SQL_INSTANCE_NAME: description: PostgreSQL Instance Name required: true + API_TEST_REFRESH_TOKEN: + description: API refresh token used to resolve deployed API commit (used on repository_dispatch) + required: false inputs: PROJECT_ID: description: GCP Project ID @@ -67,19 +70,61 @@ on: description: GCP region required: true type: string + API_BASE_URL: + description: Base URL host for the API used to resolve version/commit (e.g. api-dev.mobilitydatabase.org) + required: false + default: api.mobilitydatabase.org + type: string + INSTALL_LATEST: + description: Install the latest (main) API version when true; when false keep the currently deployed version. + required: false + default: false + type: boolean + DRY_RUN: + description: Skip applying schema and content updates + required: false + default: false + type: boolean env: python_version: '3.11' liquibase_version: '4.33.0' jobs: + resolve-api-meta: + name: 'Resolve API commit/version' + runs-on: ubuntu-latest + # Run this job for all triggers; the action itself will skip resolution when API_BASE_URL or token is not provided. + # Keeping it unconditional ensures CHECKOUT_REF is always set (defaults to 'main') for downstream jobs. + outputs: + # Use resolved commit when available; otherwise default to 'main'. + CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }} + steps: + - name: Checkout repo (for scripts and local action) + uses: actions/checkout@v4 + - name: Resolve API commit/version + id: resolve + if: ${{ inputs.INSTALL_LATEST == false }} + uses: ./.github/actions/resolve-api-meta + with: + api_base_url: ${{ inputs.API_BASE_URL }} + api_refresh_token: ${{ secrets.API_TEST_REFRESH_TOKEN }} + db-schema-update: name: 'Database Schema Update' permissions: write-all runs-on: ubuntu-latest + needs: [resolve-api-meta] + # Run the schema update when the resolved checkout target is 'main' (install latest/main). + # This covers both explicit INSTALL_LATEST runs and cases where resolution failed and CHECKOUT_REF fell back to 'main'. + if: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF == 'main' }} steps: - - name: Checkout code + - name: Checkout repo uses: actions/checkout@v4 + with: + # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') + ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} + fetch-depth: 0 - name: Authenticate to Google Cloud QA/PROD uses: google-github-actions/auth@v2 @@ -126,6 +171,7 @@ jobs: liquibase --version - name: Run Liquibase + if: ${{ !inputs.DRY_RUN }} working-directory: ${{ github.workspace }}/liquibase run: | export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml" @@ -139,11 +185,15 @@ jobs: name: 'Database Content Update' permissions: write-all runs-on: ubuntu-latest - needs: db-schema-update - if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} + needs: [resolve-api-meta, db-schema-update] + if: ${{ always() }} steps: - - name: Checkout code + - name: Checkout repo uses: actions/checkout@v4 + with: + # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') + ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} + fetch-depth: 0 - name: Setup python uses: actions/setup-python@v5 @@ -212,11 +262,11 @@ jobs: run: echo "PATH=$(realpath sources.csv)" >> $GITHUB_OUTPUT - name: GTFS - Update Database Content - if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }} + if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} run: scripts/populate-db.sh ${{ steps.getpath.outputs.PATH }} > populate.log - name: GTFS - Upload log file for verification - if: ${{ always() && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} uses: actions/upload-artifact@v4 with: name: populate-${{ inputs.ENVIRONMENT }}.log @@ -232,11 +282,11 @@ jobs: run: echo "PATH=$(realpath systems.csv)" >> $GITHUB_OUTPUT - name: GBFS - Update Database Content - if: ${{ env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual' }} + if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} run: scripts/populate-db.sh ${{ steps.getsyspath.outputs.PATH }} gbfs >> populate-gbfs.log - name: GBFS - Upload log file for verification - if: ${{ always() && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} uses: actions/upload-artifact@v4 with: name: populate-gbfs-${{ inputs.ENVIRONMENT }}.log @@ -245,7 +295,7 @@ jobs: update-gcp-secret: name: Update GCP Secrets - if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} + if: ${{ contains('repository_dispatch,workflow_dispatch', github.event_name) && !inputs.DRY_RUN }} runs-on: ubuntu-latest steps: - name: Authenticate to Google Cloud @@ -283,6 +333,3 @@ jobs: echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..." echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID fi - - -