From d847cdff3b646db9085f1cedce6a74f3969b0dd1 Mon Sep 17 00:00:00 2001 From: Tyler Hill Date: Sat, 2 Aug 2025 13:53:16 -0700 Subject: [PATCH 1/3] Initial reworking --- .github/workflows/deploy.yml | 12 +++++++----- Dockerfile | 8 +++++--- deploy.sh => runners/daily.sh | 5 ++++- 3 files changed, 16 insertions(+), 9 deletions(-) rename deploy.sh => runners/daily.sh (88%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 833503d..925f323 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,6 +3,7 @@ name: Deploy on: + workflow_dispatch: push: branches: develop @@ -38,18 +39,19 @@ jobs: uses: docker/build-push-action@v6 with: push: true - tags: us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events + tags: us-central1-docker.pkg.dev/api-tools-451421/runners/daily + build-args: RUN_SCRIPT_PATH=runners/daily.sh - name: Upload to dev job uses: google-github-actions/deploy-cloudrun@v2 with: - job: 'daily-update-events-dev' - image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events' + job: 'daily-dev' + image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily' region: us-central1 - name: Upload to prod job uses: google-github-actions/deploy-cloudrun@v2 with: - job: 'daily-update-events-prod' - image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events' + job: 'daily-prod' + image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily' region: us-central1 diff --git a/Dockerfile b/Dockerfile index a4bdc4c..6a02383 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM golang:1.23 AS builder +ARG RUN_SCRIPT_PATH + WORKDIR /app COPY . . @@ -25,7 +27,7 @@ ENV GOOGLE_CHROME_BIN /usr/bin/chromium # Also set this for compatibility # Copy build file from builder COPY --from=builder /app/api-tools /app/api-tools -COPY deploy.sh /app/deploy.sh +COPY $RUN_SCRIPT_PATH /app/$RUN_SCRIPT_PATH -RUN chmod +x /app/deploy.sh -ENTRYPOINT ["/app/deploy.sh"] +RUN chmod +x /app/$RUN_SCRIPT_PATH +ENTRYPOINT ["/app/$RUN_SCRIPT_PATH"] diff --git a/deploy.sh b/runners/daily.sh similarity index 88% rename from deploy.sh rename to runners/daily.sh index eeffdfa..c624a0f 100644 --- a/deploy.sh +++ b/runners/daily.sh @@ -8,7 +8,10 @@ rm service_account.json # .env gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env -# Scrape, parse, and upload +# other daily activites +# ... + +# Scrape, parse, and upload events ./api-tools -scrape -mazevo -verbose ./api-tools -parse -mazevo -verbose ./api-tools -scrape -astra -verbose From ec8b9a94e8c253f6af5b20016f3397c9add209d1 Mon Sep 17 00:00:00 2001 From: Tyler Hill Date: Sun, 3 Aug 2025 00:51:16 -0700 Subject: [PATCH 2/3] Change based on env variable --- .github/workflows/deploy.yml | 7 +++---- Dockerfile | 8 +++----- runners/daily.sh | 8 -------- runners/setup.sh | 12 ++++++++++++ 4 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 runners/setup.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 925f323..7810cfe 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,19 +39,18 @@ jobs: uses: docker/build-push-action@v6 with: push: true - tags: us-central1-docker.pkg.dev/api-tools-451421/runners/daily - build-args: RUN_SCRIPT_PATH=runners/daily.sh + tags: us-central1-docker.pkg.dev/api-tools-451421/runners/runner - name: Upload to dev job uses: google-github-actions/deploy-cloudrun@v2 with: job: 'daily-dev' - image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily' + image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner' region: us-central1 - name: Upload to prod job uses: google-github-actions/deploy-cloudrun@v2 with: job: 'daily-prod' - image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily' + image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner' region: us-central1 diff --git a/Dockerfile b/Dockerfile index 6a02383..7c907c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,5 @@ FROM golang:1.23 AS builder -ARG RUN_SCRIPT_PATH - WORKDIR /app COPY . . @@ -27,7 +25,7 @@ ENV GOOGLE_CHROME_BIN /usr/bin/chromium # Also set this for compatibility # Copy build file from builder COPY --from=builder /app/api-tools /app/api-tools -COPY $RUN_SCRIPT_PATH /app/$RUN_SCRIPT_PATH +COPY runners /app/runners -RUN chmod +x /app/$RUN_SCRIPT_PATH -ENTRYPOINT ["/app/$RUN_SCRIPT_PATH"] +RUN chmod +x /app/runners/setup.sh +ENTRYPOINT ["/app/runners/setup.sh"] diff --git a/runners/daily.sh b/runners/daily.sh index c624a0f..f2a5edb 100644 --- a/runners/daily.sh +++ b/runners/daily.sh @@ -1,13 +1,5 @@ #!/bin/sh -# service account -gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json -gcloud auth activate-service-account --key-file=service_account.json -rm service_account.json - -# .env -gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env - # other daily activites # ... diff --git a/runners/setup.sh b/runners/setup.sh new file mode 100644 index 0000000..20cbb41 --- /dev/null +++ b/runners/setup.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# service account +gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json +gcloud auth activate-service-account --key-file=service_account.json +rm service_account.json + +# .env +gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env + +# run commands from file specified in GCP +sh "/app/runners/$RUNNER_SCRIPT_NAME" From c1e5df76c59254067f2ca00d5287b5796a64e24c Mon Sep 17 00:00:00 2001 From: Tyler Hill Date: Sun, 3 Aug 2025 00:56:16 -0700 Subject: [PATCH 3/3] Comments --- .github/workflows/deploy.yml | 4 ++-- runners/daily.sh | 5 ++--- runners/setup.sh | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7810cfe..6530729 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -41,14 +41,14 @@ jobs: push: true tags: us-central1-docker.pkg.dev/api-tools-451421/runners/runner - - name: Upload to dev job + - name: Upload to daily dev job uses: google-github-actions/deploy-cloudrun@v2 with: job: 'daily-dev' image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner' region: us-central1 - - name: Upload to prod job + - name: Upload to daily prod job uses: google-github-actions/deploy-cloudrun@v2 with: job: 'daily-prod' diff --git a/runners/daily.sh b/runners/daily.sh index f2a5edb..764ee13 100644 --- a/runners/daily.sh +++ b/runners/daily.sh @@ -1,9 +1,8 @@ #!/bin/sh -# other daily activites -# ... +# for daily tasks to run -# Scrape, parse, and upload events +# scrape, parse, and upload events ./api-tools -scrape -mazevo -verbose ./api-tools -parse -mazevo -verbose ./api-tools -scrape -astra -verbose diff --git a/runners/setup.sh b/runners/setup.sh index 20cbb41..d239dca 100644 --- a/runners/setup.sh +++ b/runners/setup.sh @@ -1,12 +1,12 @@ #!/bin/sh -# service account +# auth with service account gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json gcloud auth activate-service-account --key-file=service_account.json rm service_account.json -# .env +# use service account to access environment variables from GCP secrets, create .env gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env -# run commands from file specified in GCP +# run commands from the file path specified in the GCP run job's variable sh "/app/runners/$RUNNER_SCRIPT_NAME"