From 4880003d732be08f6158eeb24ed26d69c73ad9db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 00:11:47 +0000 Subject: [PATCH 1/9] Bump super-linter/super-linter from 8.0.0 to 8.1.0 Bumps [super-linter/super-linter](https://github.com/super-linter/super-linter) from 8.0.0 to 8.1.0. - [Release notes](https://github.com/super-linter/super-linter/releases) - [Changelog](https://github.com/super-linter/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/super-linter/super-linter/compare/v8.0.0...v8.1.0) --- updated-dependencies: - dependency-name: super-linter/super-linter dependency-version: 8.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7d33116..38425ca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,7 +26,7 @@ jobs: fetch-depth: 0 - name: Lint Code Base - uses: super-linter/super-linter/slim@v8.0.0 + uses: super-linter/super-linter/slim@v8.1.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e61490dea0ea9e3a43d1de7a0b8a52f7fb2be3b7 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 26 Aug 2025 22:42:49 +0200 Subject: [PATCH 2/9] Add health check endpoint and update Dockerfiles for non-root user --- .github/linters/.trivyignore | 3 +++ .github/linters/.zizmor.yml | 3 +++ .github/workflows/ci.yaml | 1 + infra/bigquery-export/Dockerfile | 12 ++++++++++++ infra/dataform-service/Dockerfile | 13 +++++++++++++ infra/dataform-service/index.js | 10 ++++++++-- 6 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 .github/linters/.trivyignore create mode 100644 .github/linters/.zizmor.yml diff --git a/.github/linters/.trivyignore b/.github/linters/.trivyignore new file mode 100644 index 0000000..a4124ee --- /dev/null +++ b/.github/linters/.trivyignore @@ -0,0 +1,3 @@ +# Ignore the dataplexAdmin role issue + +AVD-GCP-0007 \ No newline at end of file diff --git a/.github/linters/.zizmor.yml b/.github/linters/.zizmor.yml new file mode 100644 index 0000000..0e564a2 --- /dev/null +++ b/.github/linters/.zizmor.yml @@ -0,0 +1,3 @@ +rules: + unpinned-uses: + ignore: true \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 38425ca..421f34a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,7 @@ jobs: uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - name: Lint Code Base uses: super-linter/super-linter/slim@v8.1.0 diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/Dockerfile index ef7eaab..0960eb5 100644 --- a/infra/bigquery-export/Dockerfile +++ b/infra/bigquery-export/Dockerfile @@ -4,6 +4,9 @@ FROM node:22-slim # Set the working directory WORKDIR /app +# Create a non-root user +RUN groupadd -r appuser && useradd -r -g appuser appuser + # Copy package files first for better layer caching COPY package*.json ./ @@ -15,4 +18,13 @@ ENV EXPORT_CONFIG="" # Copy source code COPY . . +# Change ownership of the app directory to the non-root user +RUN chown -R appuser:appuser /app + +# Switch to non-root user +USER appuser + +# No healthcheck needed for one-time job containers +HEALTHCHECK NONE + CMD ["node", "index.js"] diff --git a/infra/dataform-service/Dockerfile b/infra/dataform-service/Dockerfile index 914849c..f2e7560 100644 --- a/infra/dataform-service/Dockerfile +++ b/infra/dataform-service/Dockerfile @@ -3,6 +3,9 @@ FROM node:22-slim # Set the working directory WORKDIR /app +# Create a non-root user +RUN groupadd -r appuser && useradd -r -g appuser appuser + # Copy package files first for better layer caching COPY package*.json ./ @@ -12,11 +15,21 @@ RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --f # Copy source code COPY . . +# Change ownership of the app directory to the non-root user +RUN chown -R appuser:appuser /app + +# Switch to non-root user +USER appuser + # Set default port (Cloud Run will override this) ENV PORT=8080 # Expose port for Cloud Run EXPOSE 8080 +# Add healthcheck +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD node -e "require('http').get('http://localhost:$PORT/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => { process.exit(1) })" || exit 1 + # Start the function CMD ["npm", "start"] diff --git a/infra/dataform-service/index.js b/infra/dataform-service/index.js index 515f24b..ae4e675 100644 --- a/infra/dataform-service/index.js +++ b/infra/dataform-service/index.js @@ -223,14 +223,20 @@ async function mainHandler (req, res) { console.info(`Received request for path: ${path}`) - if (path === '/trigger' || path.startsWith('/trigger/')) { + if (path === '/health') { + // Health check endpoint + res.status(200).json({ + status: 'healthy', + timestamp: new Date().toISOString() + }) + } else if (path === '/trigger' || path.startsWith('/trigger/')) { await handleTrigger(req, res) } else if (path === '/') { await handleExport(req, res) } else { res.status(404).json({ error: 'Not Found', - message: 'Available endpoints: /, /export' + message: 'Available endpoints: /, /trigger, /health' }) } } From ee66234780544817b7162d7f6a1ce3a25b4fe9df Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 26 Aug 2025 22:49:47 +0200 Subject: [PATCH 3/9] rename trivy and zizmor configuration files --- .github/linters/.trivyignore | 3 --- .github/linters/trivy.yaml | 2 ++ .github/linters/{.zizmor.yml => zizmor.yaml} | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 .github/linters/.trivyignore create mode 100644 .github/linters/trivy.yaml rename .github/linters/{.zizmor.yml => zizmor.yaml} (58%) diff --git a/.github/linters/.trivyignore b/.github/linters/.trivyignore deleted file mode 100644 index a4124ee..0000000 --- a/.github/linters/.trivyignore +++ /dev/null @@ -1,3 +0,0 @@ -# Ignore the dataplexAdmin role issue - -AVD-GCP-0007 \ No newline at end of file diff --git a/.github/linters/trivy.yaml b/.github/linters/trivy.yaml new file mode 100644 index 0000000..9847e9f --- /dev/null +++ b/.github/linters/trivy.yaml @@ -0,0 +1,2 @@ +ignore: + - AVD-GCP-0007 # Ignore the dataplexAdmin role issue diff --git a/.github/linters/.zizmor.yml b/.github/linters/zizmor.yaml similarity index 58% rename from .github/linters/.zizmor.yml rename to .github/linters/zizmor.yaml index 0e564a2..153e7e0 100644 --- a/.github/linters/.zizmor.yml +++ b/.github/linters/zizmor.yaml @@ -1,3 +1,3 @@ rules: unpinned-uses: - ignore: true \ No newline at end of file + ignore: true From e0ed2b337a21f59d240c87aafd44bc6d2d33285d Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 26 Aug 2025 23:00:13 +0200 Subject: [PATCH 4/9] Refactor Trivy and Zizmor config --- .github/linters/.trivyignore | 2 ++ .github/linters/trivy.yaml | 2 -- .github/linters/zizmor.yaml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 .github/linters/.trivyignore delete mode 100644 .github/linters/trivy.yaml diff --git a/.github/linters/.trivyignore b/.github/linters/.trivyignore new file mode 100644 index 0000000..2f83c65 --- /dev/null +++ b/.github/linters/.trivyignore @@ -0,0 +1,2 @@ +# Ignore the dataplexAdmin role issue +AVD-GCP-0007 diff --git a/.github/linters/trivy.yaml b/.github/linters/trivy.yaml deleted file mode 100644 index 9847e9f..0000000 --- a/.github/linters/trivy.yaml +++ /dev/null @@ -1,2 +0,0 @@ -ignore: - - AVD-GCP-0007 # Ignore the dataplexAdmin role issue diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yaml index 153e7e0..c2ffcae 100644 --- a/.github/linters/zizmor.yaml +++ b/.github/linters/zizmor.yaml @@ -1,3 +1,4 @@ rules: unpinned-uses: - ignore: true + ignore: + - ci.yaml From b35cf08107d5ec45ecad218c449a5837c2be274c Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 26 Aug 2025 23:07:55 +0200 Subject: [PATCH 5/9] rename Zizmor config --- .github/linters/{zizmor.yaml => zizmor.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/linters/{zizmor.yaml => zizmor.yml} (100%) diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yml similarity index 100% rename from .github/linters/zizmor.yaml rename to .github/linters/zizmor.yml From 98bb89a22615e3ced6ce1f7dd92fe45ffe491684 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 26 Aug 2025 23:55:06 +0200 Subject: [PATCH 6/9] Add comment to ignore unpinned uses in CI workflow --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 421f34a..acda800 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,6 +21,7 @@ jobs: steps: - name: Checkout Code + # zizmor: ignore[unpinned-uses] uses: actions/checkout@v5 with: fetch-depth: 0 From d8ec952ba0ed3e7673303d56315c6d5244337bab Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 27 Aug 2025 20:17:21 +0200 Subject: [PATCH 7/9] change super-linter configs path --- eslint.config.mjs => .github/linters/eslint.config.mjs | 0 .github/workflows/ci.yaml | 1 - package.json | 4 ++-- 3 files changed, 2 insertions(+), 3 deletions(-) rename eslint.config.mjs => .github/linters/eslint.config.mjs (100%) diff --git a/eslint.config.mjs b/.github/linters/eslint.config.mjs similarity index 100% rename from eslint.config.mjs rename to .github/linters/eslint.config.mjs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index acda800..b1c4732 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,7 +32,6 @@ jobs: env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - LINTER_RULES_PATH: . VALIDATE_JSCPD: false VALIDATE_JAVASCRIPT_PRETTIER: false VALIDATE_MARKDOWN_PRETTIER: false diff --git a/package.json b/package.json index 2be1e26..37ccdec 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,8 @@ "name": "crawl-data", "author": "@max-ostapenko", "scripts": { - "format": "npx eslint --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive", - "lint": "npx eslint .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile", + "format": "npx eslint -c .github/linters/eslint.config.mjs --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive", + "lint": "npx eslint -c .github/linters/eslint.config.mjs .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile", "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest" }, "dependencies": { From 142fb3b960429416e2a5557eaa8fbad3b7a97816 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 27 Aug 2025 20:43:23 +0200 Subject: [PATCH 8/9] trivy ignorefile location --- .github/linters/trivy.yaml | 1 + .github/linters/{zizmor.yml => zizmor.yaml} | 0 .github/workflows/ci.yaml | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .github/linters/trivy.yaml rename .github/linters/{zizmor.yml => zizmor.yaml} (100%) diff --git a/.github/linters/trivy.yaml b/.github/linters/trivy.yaml new file mode 100644 index 0000000..d90ba4d --- /dev/null +++ b/.github/linters/trivy.yaml @@ -0,0 +1 @@ +ignorefile: ".trivyignore" diff --git a/.github/linters/zizmor.yml b/.github/linters/zizmor.yaml similarity index 100% rename from .github/linters/zizmor.yml rename to .github/linters/zizmor.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b1c4732..d42df9b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,6 @@ jobs: steps: - name: Checkout Code - # zizmor: ignore[unpinned-uses] uses: actions/checkout@v5 with: fetch-depth: 0 From 87e6c7804b030d74f0ef4dc497f3e3eb0332b550 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 27 Aug 2025 20:46:47 +0200 Subject: [PATCH 9/9] fix path --- .github/linters/trivy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/linters/trivy.yaml b/.github/linters/trivy.yaml index d90ba4d..b0421bd 100644 --- a/.github/linters/trivy.yaml +++ b/.github/linters/trivy.yaml @@ -1 +1 @@ -ignorefile: ".trivyignore" +ignorefile: ".github/linters/.trivyignore"