From bf41a1e5ea79fe065ebb3e79b019e293216dfe88 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 8 Jun 2025 21:57:49 +0200 Subject: [PATCH 1/8] validate params --- src/__tests__/routes.test.js | 6 ++++-- src/controllers/categoriesController.js | 11 +++++++++++ src/controllers/reportController.js | 11 +++++++++++ src/controllers/technologiesController.js | 11 +++++++++++ src/controllers/versionsController.js | 11 +++++++++++ src/utils/controllerHelpers.js | 9 +++++++-- test-api.sh | 1 + 7 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/__tests__/routes.test.js b/src/__tests__/routes.test.js index 90ac268..b26fe47 100644 --- a/src/__tests__/routes.test.js +++ b/src/__tests__/routes.test.js @@ -335,8 +335,10 @@ describe('API Routes', () => { it('should handle invalid query parameters gracefully', async () => { const res = await request(app).get('/v1/technologies?invalid=parameter'); - expect(res.statusCode).toEqual(200); - expect(Array.isArray(res.body)).toBe(true); + expect(res.statusCode).toEqual(400); + expect(res.body).toHaveProperty('errors'); + expect(res.body.errors[0]).toHaveProperty('error'); + expect(res.body.errors[0].error).toContain('Unsupported parameters: '); }); }); diff --git a/src/controllers/categoriesController.js b/src/controllers/categoriesController.js index 6d46972..f7d1148 100644 --- a/src/controllers/categoriesController.js +++ b/src/controllers/categoriesController.js @@ -6,6 +6,17 @@ import { executeQuery, validateArrayParameter } from '../utils/controllerHelpers */ const listCategories = async (req, res) => { const queryBuilder = async (params) => { + // Validate parameters + const supportedParams = ['category', 'onlyname', 'fields']; + const providedParams = Object.keys(params); + const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); + + if (unsupportedParams.length > 0) { + const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); + error.statusCode = 400; + throw error; + } + const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; const hasCustomFields = params.fields && !isOnlyNames; diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js index fc42a58..7cf9f67 100644 --- a/src/controllers/reportController.js +++ b/src/controllers/reportController.js @@ -49,6 +49,17 @@ const createReportController = (reportType) => { try { const params = req.query; + // Validate supported parameters + const supportedParams = ['technology', 'geo', 'rank', 'start', 'end']; + const providedParams = Object.keys(params); + const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); + + if (unsupportedParams.length > 0) { + const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); + error.statusCode = 400; + throw error; + } + // Validate required parameters using shared utility const errors = validateRequiredParams(params, [ REQUIRED_PARAMS.GEO, diff --git a/src/controllers/technologiesController.js b/src/controllers/technologiesController.js index 23b4029..e5171b3 100644 --- a/src/controllers/technologiesController.js +++ b/src/controllers/technologiesController.js @@ -6,6 +6,17 @@ import { executeQuery, validateTechnologyArray, validateArrayParameter, FIRESTOR */ const listTechnologies = async (req, res) => { const queryBuilder = async (params) => { + // Validate parameters + const supportedParams = ['technology', 'category', 'onlyname', 'fields']; + const providedParams = Object.keys(params); + const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); + + if (unsupportedParams.length > 0) { + const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); + error.statusCode = 400; + throw error; + } + const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; const hasCustomFields = params.fields && !isOnlyNames; diff --git a/src/controllers/versionsController.js b/src/controllers/versionsController.js index 17cd8f7..aa19d55 100644 --- a/src/controllers/versionsController.js +++ b/src/controllers/versionsController.js @@ -6,6 +6,17 @@ import { executeQuery, validateTechnologyArray, FIRESTORE_IN_LIMIT } from '../ut */ const listVersions = async (req, res) => { const queryBuilder = async (params) => { + // Validate parameters + const supportedParams = ['version', 'technology', 'category', 'onlyname', 'fields']; + const providedParams = Object.keys(params); + const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); + + if (unsupportedParams.length > 0) { + const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); + error.statusCode = 400; + throw error; + } + let query = firestore.collection('versions'); // Apply technology filter with validation diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js index 047f7bf..56aa46e 100644 --- a/src/utils/controllerHelpers.js +++ b/src/utils/controllerHelpers.js @@ -230,9 +230,14 @@ const getCacheStats = () => { */ const handleControllerError = (res, error, operation) => { console.error(`Error ${operation}:`, error); - res.statusCode = 500; + const statusCode = error.statusCode || 500; + res.statusCode = statusCode; + + // Use custom error message for client errors (4xx), generic message for server errors (5xx) + const errorMessage = statusCode >= 400 && statusCode < 500 ? error.message : `Failed to ${operation}`; + res.end(JSON.stringify({ - errors: [{ error: `Failed to ${operation}` }] + errors: [{ error: errorMessage }] })); }; diff --git a/test-api.sh b/test-api.sh index ea98db3..c16bfe4 100755 --- a/test-api.sh +++ b/test-api.sh @@ -85,6 +85,7 @@ test_endpoint "/" "" # Test technologies endpoint test_cors_preflight "/v1/technologies" +test_endpoint "/v1/technologies" "?onlyname=true" test_endpoint "/v1/technologies" "?technology=WordPress&onlyname=true" test_endpoint "/v1/technologies" "?technology=WordPress&onlyname=true&fields=technology,icon" test_endpoint "/v1/technologies" "?technology=WordPress&category=CMS&fields=technology,icon" From 3a005b8ecba2eaa559ceab33e2e98480b1d3598d Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 8 Jun 2025 23:04:54 +0200 Subject: [PATCH 2/8] skip xss --- src/__tests__/routes.test.js | 10 ++++++---- src/controllers/categoriesController.js | 2 ++ src/controllers/reportController.js | 2 ++ src/controllers/technologiesController.js | 2 ++ src/controllers/versionsController.js | 2 ++ src/index.js | 8 ++++++++ 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/__tests__/routes.test.js b/src/__tests__/routes.test.js index b26fe47..b88c66d 100644 --- a/src/__tests__/routes.test.js +++ b/src/__tests__/routes.test.js @@ -335,10 +335,12 @@ describe('API Routes', () => { it('should handle invalid query parameters gracefully', async () => { const res = await request(app).get('/v1/technologies?invalid=parameter'); - expect(res.statusCode).toEqual(400); - expect(res.body).toHaveProperty('errors'); - expect(res.body.errors[0]).toHaveProperty('error'); - expect(res.body.errors[0].error).toContain('Unsupported parameters: '); + expect(res.statusCode).toEqual(200); + expect(Array.isArray(res.body)).toBe(true); + + //expect(res.body).toHaveProperty('errors'); + //expect(res.body.errors[0]).toHaveProperty('error'); + //expect(res.body.errors[0].error).toContain('Unsupported parameters: '); }); }); diff --git a/src/controllers/categoriesController.js b/src/controllers/categoriesController.js index f7d1148..6a76918 100644 --- a/src/controllers/categoriesController.js +++ b/src/controllers/categoriesController.js @@ -6,6 +6,7 @@ import { executeQuery, validateArrayParameter } from '../utils/controllerHelpers */ const listCategories = async (req, res) => { const queryBuilder = async (params) => { + /* // Validate parameters const supportedParams = ['category', 'onlyname', 'fields']; const providedParams = Object.keys(params); @@ -16,6 +17,7 @@ const listCategories = async (req, res) => { error.statusCode = 400; throw error; } + */ const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; const hasCustomFields = params.fields && !isOnlyNames; diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js index 7cf9f67..1e297f0 100644 --- a/src/controllers/reportController.js +++ b/src/controllers/reportController.js @@ -49,6 +49,7 @@ const createReportController = (reportType) => { try { const params = req.query; + /* // Validate supported parameters const supportedParams = ['technology', 'geo', 'rank', 'start', 'end']; const providedParams = Object.keys(params); @@ -59,6 +60,7 @@ const createReportController = (reportType) => { error.statusCode = 400; throw error; } + */ // Validate required parameters using shared utility const errors = validateRequiredParams(params, [ diff --git a/src/controllers/technologiesController.js b/src/controllers/technologiesController.js index e5171b3..5d75aba 100644 --- a/src/controllers/technologiesController.js +++ b/src/controllers/technologiesController.js @@ -6,6 +6,7 @@ import { executeQuery, validateTechnologyArray, validateArrayParameter, FIRESTOR */ const listTechnologies = async (req, res) => { const queryBuilder = async (params) => { + /* // Validate parameters const supportedParams = ['technology', 'category', 'onlyname', 'fields']; const providedParams = Object.keys(params); @@ -16,6 +17,7 @@ const listTechnologies = async (req, res) => { error.statusCode = 400; throw error; } + */ const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; const hasCustomFields = params.fields && !isOnlyNames; diff --git a/src/controllers/versionsController.js b/src/controllers/versionsController.js index aa19d55..8af580a 100644 --- a/src/controllers/versionsController.js +++ b/src/controllers/versionsController.js @@ -6,6 +6,7 @@ import { executeQuery, validateTechnologyArray, FIRESTORE_IN_LIMIT } from '../ut */ const listVersions = async (req, res) => { const queryBuilder = async (params) => { + /* // Validate parameters const supportedParams = ['version', 'technology', 'category', 'onlyname', 'fields']; const providedParams = Object.keys(params); @@ -16,6 +17,7 @@ const listVersions = async (req, res) => { error.statusCode = 400; throw error; } + */ let query = firestore.collection('versions'); diff --git a/src/index.js b/src/index.js index 46a01d4..1c553b6 100644 --- a/src/index.js +++ b/src/index.js @@ -105,6 +105,14 @@ const handleRequest = async (req, res) => { return; } + // Validate URL to skip XSS attacks + const unsafe = /onerror|onload|javascript:/i; + if (unsafe.test(req.url)) { + res.statusCode = 400 + res.end(JSON.stringify({ error: 'Invalid input' })); + return; + } + // Parse URL const parsedUrl = url.parse(req.url, true); const pathname = parsedUrl.pathname; From 256f07f58f1ba8ef89723f2f6adda444da552cdc Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Mon, 9 Jun 2025 00:37:45 +0200 Subject: [PATCH 3/8] bq init --- src/controllers/geosBQController.js | 17 + src/controllers/ranksBQController.js | 17 + src/index.js | 4 +- src/package-lock.json | 467 +++++++++++++++++++++++++++ src/package.json | 1 + src/utils/bigquery.js | 89 +++++ src/utils/controllerHelpers.js | 45 +++ 7 files changed, 638 insertions(+), 2 deletions(-) create mode 100644 src/controllers/geosBQController.js create mode 100644 src/controllers/ranksBQController.js create mode 100644 src/utils/bigquery.js diff --git a/src/controllers/geosBQController.js b/src/controllers/geosBQController.js new file mode 100644 index 0000000..9a6b417 --- /dev/null +++ b/src/controllers/geosBQController.js @@ -0,0 +1,17 @@ +import { getGeosFromBQ } from '../utils/bigquery.js'; +import { executeBigQuery } from '../utils/controllerHelpers.js'; + +/** + * List all geographic locations from BigQuery + */ +const listGeos = async (req, res) => { + const queryExecutor = async () => { + return await getGeosFromBQ(); + }; + + await executeBigQuery(req, res, 'geos', queryExecutor); +}; + +export { + listGeos +}; diff --git a/src/controllers/ranksBQController.js b/src/controllers/ranksBQController.js new file mode 100644 index 0000000..182fa73 --- /dev/null +++ b/src/controllers/ranksBQController.js @@ -0,0 +1,17 @@ +import { getRanksFromBQ } from '../utils/bigquery.js'; +import { executeBigQuery } from '../utils/controllerHelpers.js'; + +/** + * List all rank options from BigQuery + */ +const listRanks = async (req, res) => { + const queryExecutor = async () => { + return await getRanksFromBQ(); + }; + + await executeBigQuery(req, res, 'ranks', queryExecutor); +}; + +export { + listRanks +}; diff --git a/src/index.js b/src/index.js index 1c553b6..c886f73 100644 --- a/src/index.js +++ b/src/index.js @@ -33,10 +33,10 @@ const getController = async (name) => { controllers[name] = await import('./controllers/reportController.js'); break; case 'ranks': - controllers[name] = await import('./controllers/ranksController.js'); + controllers[name] = await import('./controllers/ranksBQController.js'); break; case 'geos': - controllers[name] = await import('./controllers/geosController.js'); + controllers[name] = await import('./controllers/geosBQController.js'); break; case 'versions': controllers[name] = await import('./controllers/versionsController.js'); diff --git a/src/package-lock.json b/src/package-lock.json index d4e94bb..62391c7 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -8,6 +8,7 @@ "name": "tech-report-api", "version": "1.0.0", "dependencies": { + "@google-cloud/bigquery": "^8.1.0", "@google-cloud/firestore": "7.11.1", "@google-cloud/functions-framework": "4.0.0" }, @@ -568,6 +569,310 @@ "dev": true, "license": "MIT" }, + "node_modules/@google-cloud/bigquery": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-8.1.0.tgz", + "integrity": "sha512-eDleD/IHKQIRm4GmMnwJvPkx4PgSaK8m8DCmDmVOf0gIhqPLSdvOAEeM4QjyyZGUGjV4yHyJfEJxzULTzl22Aw==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/common": "^6.0.0", + "@google-cloud/paginator": "^6.0.0", + "@google-cloud/precise-date": "^5.0.0", + "@google-cloud/promisify": "^5.0.0", + "arrify": "^3.0.0", + "big.js": "^6.2.2", + "duplexify": "^4.1.3", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "teeny-request": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/bigquery/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/@google-cloud/bigquery/node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@google-cloud/bigquery/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@google-cloud/bigquery/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@google-cloud/bigquery/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/@google-cloud/bigquery/node_modules/teeny-request": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", + "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^3.3.2", + "stream-events": "^1.0.5" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-6.0.0.tgz", + "integrity": "sha512-IXh04DlkLMxWgYLIUYuHHKXKOUwPDzDgke1ykkkJPe48cGIS9kkL2U/o0pm4ankHLlvzLF/ma1eO86n/bkumIA==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.0", + "duplexify": "^4.1.3", + "extend": "^3.0.2", + "google-auth-library": "^10.0.0-rc.1", + "html-entities": "^2.5.2", + "retry-request": "^8.0.0", + "teeny-request": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/@google-cloud/promisify": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.1.0.tgz", + "integrity": "sha512-G/FQx5cE/+DqBbOpA5jKsegGwdPniU6PuIEMt+qxWgFxvxuFOzVmp6zYchtYuwAWV5/8Dgs0yAmjvNZv3uXLQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/@google-cloud/common/node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@google-cloud/common/node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@google-cloud/common/node_modules/gaxios": { + "version": "7.0.0-rc.6", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.0.0-rc.6.tgz", + "integrity": "sha512-osVFpgeBiwTM2AVI9MXvb8iWzM6oSMbTVWc65Gm5BgBlE+nUA6PBHFMaYpqjZx1AhUH7aPOZq78WcRAM6hhAwA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/gcp-metadata": { + "version": "7.0.0-rc.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.0-rc.1.tgz", + "integrity": "sha512-E6c+AdIaK1LNA839OyotiTca+B2IG1nDlMjnlcck8JjXn3fVgx57Ib9i6iL1/iqN7bA3EUQdcRRu+HqOCOABIg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0-rc.1", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/google-auth-library": { + "version": "10.0.0-rc.3", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.0.0-rc.3.tgz", + "integrity": "sha512-WC9wfEKK0bk3seWKsDn2loduLth6JWKTsrbWftzrhPuzpwnVXb5oi2+aa0JDBxLBDdkGesLvTQ67F2nZ7leq1Q==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.0.0-rc.4", + "gcp-metadata": "^7.0.0-rc.1", + "google-logging-utils": "^1.0.0", + "gtoken": "^8.0.0-rc.1", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/google-logging-utils": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz", + "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/common/node_modules/gtoken": { + "version": "8.0.0-rc.1", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0-rc.1.tgz", + "integrity": "sha512-UjE/egX6ixArdcCKOkheuFQ4XN4/0gX92nd2JPVEYuRU2sWHAWuOVGnowm1fQUdQtaxqn1n8H0hOb2LCaUhJ3A==", + "license": "MIT", + "dependencies": { + "gaxios": "^7.0.0-rc.1", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@google-cloud/common/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/@google-cloud/common/node_modules/retry-request": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.0.tgz", + "integrity": "sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==", + "license": "MIT", + "dependencies": { + "@types/request": "^2.48.12", + "extend": "^3.0.2", + "teeny-request": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/teeny-request": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", + "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^3.3.2", + "stream-events": "^1.0.5" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/common/node_modules/teeny-request/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/@google-cloud/firestore": { "version": "7.11.1", "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.11.1.tgz", @@ -619,6 +924,45 @@ "node": ">=10" } }, + "node_modules/@google-cloud/paginator": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz", + "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-5.0.0.tgz", + "integrity": "sha512-9h0Gvw92EvPdE8AK8AgZPbMnH5ftDyPtKm7/KUfcJVaPEPjwGDsJd1QV0H8esBDV4II41R/2lDWH1epBqIoKUw==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz", + "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.3.tgz", @@ -1824,6 +2168,18 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, + "node_modules/arrify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", + "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", @@ -2036,6 +2392,19 @@ ], "license": "MIT" }, + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, "node_modules/bignumber.js": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.0.tgz", @@ -2488,6 +2857,15 @@ "node": ">= 8" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -2943,6 +3321,29 @@ "bser": "2.1.1" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -3031,6 +3432,18 @@ "node": ">= 0.12" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/formidable": { "version": "3.5.4", "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz", @@ -3405,6 +3818,22 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "license": "ISC" }, + "node_modules/html-entities": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -3604,6 +4033,15 @@ "node": ">= 0.10" } }, + "node_modules/is": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", + "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/is-arguments": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", @@ -4916,6 +5354,26 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -6416,6 +6874,15 @@ "makeerror": "1.0.12" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/src/package.json b/src/package.json index b129f94..1286c68 100644 --- a/src/package.json +++ b/src/package.json @@ -13,6 +13,7 @@ "test:live": "bash ../test-api.sh" }, "dependencies": { + "@google-cloud/bigquery": "^8.1.0", "@google-cloud/firestore": "7.11.1", "@google-cloud/functions-framework": "4.0.0" }, diff --git a/src/utils/bigquery.js b/src/utils/bigquery.js new file mode 100644 index 0000000..753e00a --- /dev/null +++ b/src/utils/bigquery.js @@ -0,0 +1,89 @@ +import { BigQuery } from '@google-cloud/bigquery'; + +// Initialize BigQuery client +const bigquery = new BigQuery({ + projectId: process.env.PROJECT || 'httparchive' +}); + +// BigQuery configuration optimizations +const BQ_CONFIG = { + // Optimize for BI Engine + location: 'US', // Use the same location as your BI Engine + // Use maximum parallelism for BI Engine + maximumBytesBilled: '100000000', // 100MB limit for safety + // Labels for monitoring + labels: { + 'app': 'tech-report-api', + 'source': 'bigquery-direct', + } +}; + +/** + * Execute a BigQuery query with caching support + * @param {string} query - SQL query string + * @param {Object} options - Query options + * @returns {Array} - Query results + */ +const executeBigQueryQuery = async (query, options = {}) => { + try { + const queryOptions = { + query, + + jobCreationMode: 'JOB_CREATION_OPTIONAL', // Returning immediate results is prioritized. + timeoutMs: 10000, // 10 seconds + // Use query cache when possible + useQueryCache: true, + // Apply BI Engine optimizations + ...BQ_CONFIG, + ...options + }; + + console.log('Executing BigQuery:', query); + const [rows, , metadata] = await bigquery.query(queryOptions); + if(metadata.jobReference) { + console.log(`BigQuery job ${metadata.jobReference.jobId} completed. Rows: ${rows.length}`); + } + + return rows; + } catch (error) { + console.error('BigQuery execution error:', error); + throw error; + } +}; + +/** + * Get ranks from BigQuery + * @returns {Array} - Array of rank objects + */ +const getRanksFromBQ = async () => { + const query = ` + SELECT rank + FROM \`httparchive.reports.tech_report_ranks\` + ORDER BY mobile_origins DESC + `; + + const rows = await executeBigQueryQuery(query); + return rows.map(row => ({ rank: row.rank })); +}; + +/** + * Get geos from BigQuery + * @returns {Array} - Array of geo objects + */ +const getGeosFromBQ = async () => { + const query = ` + SELECT geo + FROM \`httparchive.reports.tech_report_geos\` + ORDER BY mobile_origins DESC + `; + + const rows = await executeBigQueryQuery(query); + return rows.map(row => ({ geo: row.geo })); +}; + +export { + bigquery, + executeBigQueryQuery, + getRanksFromBQ, + getGeosFromBQ +}; diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js index 56aa46e..8f0a132 100644 --- a/src/utils/controllerHelpers.js +++ b/src/utils/controllerHelpers.js @@ -318,6 +318,50 @@ const validateTechnologyArray = (technologyParam) => { } }; +/** + * Generic BigQuery-enabled query executor + * Handles caching, query execution, and response for BigQuery queries + * @param {Object} req - Request object + * @param {Object} res - Response object + * @param {string} queryName - Query name for caching and error handling + * @param {Function} queryExecutor - Function that executes BigQuery and returns results + * @param {Function} dataProcessor - Optional function to process results + */ +const executeBigQuery = async (req, res, queryName, queryExecutor, dataProcessor = null) => { + try { + const params = req.query; + + // Generate cache key + const cacheKey = generateQueryCacheKey(`bq_${queryName}`, params); + + // Check cache first + const cachedResult = getCachedQueryResult(cacheKey); + if (cachedResult) { + res.statusCode = 200; + res.end(JSON.stringify(cachedResult)); + return; + } + + // Execute BigQuery + let data = await queryExecutor(params); + + // Process data if processor provided + if (dataProcessor) { + data = dataProcessor(data, params); + } + + // Cache the result + setCachedQueryResult(cacheKey, data); + + // Send response + res.statusCode = 200; + res.end(JSON.stringify(data)); + + } catch (error) { + handleControllerError(res, error, `executing BigQuery ${queryName}`); + } +}; + export { REQUIRED_PARAMS, FIRESTORE_IN_LIMIT, @@ -331,5 +375,6 @@ export { setCachedQueryResult, getCacheStats, executeQuery, + executeBigQuery, validateTechnologyArray }; From d9f1a0c880984f6f4af3480be997b4a019e798c2 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 18 Jun 2025 19:24:18 +0200 Subject: [PATCH 4/8] audits endpoint --- README.md | 74 +++++++++++++++++++++++++-- src/controllers/reportController.js | 11 +++- src/index.js | 5 ++ terraform/modules/run-service/main.tf | 2 +- test-api.sh | 3 ++ 5 files changed, 87 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 500b339..f66526e 100644 --- a/README.md +++ b/README.md @@ -174,9 +174,9 @@ Lists available versions. #### Versions Parameters -- `version` (optional): Filter by version name(s) - comma-separated list - `technology` (optional): Filter by technology name(s) - comma-separated list - `category` (optional): Filter by category - comma-separated list +- `version` (optional): Filter by version name(s) - comma-separated list - `onlyname` (optional): If present, returns only version names - `fields` (optional): Comma-separated list of fields to include in the response (see [Field Selection API Documentation](#field-selection-api-documentation) for details) @@ -209,10 +209,10 @@ Provides technology adoption data. #### Adoption Parameters - `technology` (required): Filter by technology name(s) - comma-separated list +- `geo` (required): Filter by geographic location +- `rank` (required): Filter by rank - `start` (optional): Filter by date range start (YYYY-MM-DD or 'latest') - `end` (optional): Filter by date range end (YYYY-MM-DD) -- `geo` (optional): Filter by geographic location -- `rank` (optional): Filter by rank #### Adoption Response @@ -334,8 +334,8 @@ Provides Page Weight metrics for technologies. #### Page Weight Parameters - `technology` (required): Filter by technology name(s) - comma-separated list -- `geo` (optional): Filter by geographic location -- `rank` (optional): Filter by rank +- `geo` (required): Filter by geographic location +- `rank` (required): Filter by rank - `start` (optional): Filter by date range start (YYYY-MM-DD or 'latest') - `end` (optional): Filter by date range end (YYYY-MM-DD) @@ -387,6 +387,70 @@ Returns a JSON object with the following schema: ] ``` + +### `GET /audits` + +Provides Lighthouse audits for technologies. + +#### Audits Parameters + +- `technology` (required): Filter by technology name(s) - comma-separated list +- `geo` (required): Filter by geographic location +- `rank` (required): Filter by rank +- `start` (optional): Filter by date range start (YYYY-MM-DD or 'latest') +- `end` (optional): Filter by date range end (YYYY-MM-DD) + +#### Audits Response + +```bash +curl --request GET \ + --url 'https://{{HOST}}/v1/audits?start=latest&geo=ALL&technology=WordPress&rank=ALL' +``` + +Returns a JSON object with the following schema: + +```json +[ + { + "date": "2025-06-01", + "audits": [ + { + "desktop": { + "pass_origins": 2428028 + }, + "mobile": { + "pass_origins": 2430912 + }, + "id": "first-contentful-paint", + "category": "performance" + }, + { + "desktop": { + "pass_origins": 490451 + }, + "mobile": { + "pass_origins": 477218 + }, + "id": "largest-contentful-paint", + "category": "performance" + }, + { + "desktop": { + "pass_origins": 1221876 + }, + "mobile": { + "pass_origins": 1296673 + }, + "id": "cumulative-layout-shift", + "category": "performance" + } + ], + "technology": "WordPress" + }, + ... +] +``` + ### `GET /ranks` Lists all available ranks. diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js index 1e297f0..9f24123 100644 --- a/src/controllers/reportController.js +++ b/src/controllers/reportController.js @@ -32,6 +32,10 @@ const REPORT_CONFIGS = { cwv: { table: 'core_web_vitals', dataField: 'vitals' + }, + audits: { + table: 'audits', + dataField: 'audits' } }; @@ -146,7 +150,10 @@ const createReportController = (reportType) => { }; // Export individual controller functions +export const listAuditsData = createReportController('audits'); export const listAdoptionData = createReportController('adoption'); -export const listPageWeightData = createReportController('pageWeight'); -export const listLighthouseData = createReportController('lighthouse'); export const listCWVTechData = createReportController('cwv'); +export const listLighthouseData = createReportController('lighthouse'); +export const listPageWeightData = createReportController('pageWeight'); + + diff --git a/src/index.js b/src/index.js index c886f73..a775263 100644 --- a/src/index.js +++ b/src/index.js @@ -11,6 +11,7 @@ const controllers = { cwvtech: null, lighthouse: null, pageWeight: null, + audits: null, ranks: null, geos: null, versions: null @@ -30,6 +31,7 @@ const getController = async (name) => { case 'cwvtech': case 'lighthouse': case 'pageWeight': + case 'audits': controllers[name] = await import('./controllers/reportController.js'); break; case 'ranks': @@ -144,6 +146,9 @@ const handleRequest = async (req, res) => { } else if (pathname === '/v1/page-weight' && req.method === 'GET') { const { listPageWeightData } = await getController('pageWeight'); await listPageWeightData(req, res); + } else if (pathname === '/v1/audits' && req.method === 'GET') { + const { listAuditsData } = await getController('audits'); + await listAuditsData(req, res); } else if (pathname === '/v1/ranks' && req.method === 'GET') { const { listRanks } = await getController('ranks'); await listRanks(req, res); diff --git a/terraform/modules/run-service/main.tf b/terraform/modules/run-service/main.tf index e2c6760..3969282 100644 --- a/terraform/modules/run-service/main.tf +++ b/terraform/modules/run-service/main.tf @@ -1,5 +1,5 @@ locals { - bucketName = "tf-cloudfunctions-backingapi-20230314" + bucketName = "gcf-v2-uploads-226352634162-us-central1" } data "archive_file" "source" { type = "zip" diff --git a/test-api.sh b/test-api.sh index c16bfe4..d97d9ff 100755 --- a/test-api.sh +++ b/test-api.sh @@ -113,4 +113,7 @@ test_endpoint "/v1/lighthouse" "?technology=WordPress&geo=ALL&rank=ALL&start=lat # Test page-weight endpoint test_endpoint "/v1/page-weight" "?technology=WordPress&geo=ALL&rank=ALL&start=latest" +# Test audits endpoint +test_endpoint "/v1/audits" "?technology=WordPress&geo=ALL&rank=ALL&start=latest" + echo "API tests complete! All endpoints returned 200 status code and CORS is properly configured." From 5425aef2daf7f67e7155429621a1c08c77ed6074 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 18 Jun 2025 20:16:17 +0200 Subject: [PATCH 5/8] cleanup --- src/controllers/geosBQController.js | 17 - src/controllers/ranksBQController.js | 17 - src/index.js | 4 +- src/package-lock.json | 467 --------------------------- src/package.json | 1 - src/utils/bigquery.js | 89 ----- src/utils/controllerHelpers.js | 45 --- 7 files changed, 2 insertions(+), 638 deletions(-) delete mode 100644 src/controllers/geosBQController.js delete mode 100644 src/controllers/ranksBQController.js delete mode 100644 src/utils/bigquery.js diff --git a/src/controllers/geosBQController.js b/src/controllers/geosBQController.js deleted file mode 100644 index 9a6b417..0000000 --- a/src/controllers/geosBQController.js +++ /dev/null @@ -1,17 +0,0 @@ -import { getGeosFromBQ } from '../utils/bigquery.js'; -import { executeBigQuery } from '../utils/controllerHelpers.js'; - -/** - * List all geographic locations from BigQuery - */ -const listGeos = async (req, res) => { - const queryExecutor = async () => { - return await getGeosFromBQ(); - }; - - await executeBigQuery(req, res, 'geos', queryExecutor); -}; - -export { - listGeos -}; diff --git a/src/controllers/ranksBQController.js b/src/controllers/ranksBQController.js deleted file mode 100644 index 182fa73..0000000 --- a/src/controllers/ranksBQController.js +++ /dev/null @@ -1,17 +0,0 @@ -import { getRanksFromBQ } from '../utils/bigquery.js'; -import { executeBigQuery } from '../utils/controllerHelpers.js'; - -/** - * List all rank options from BigQuery - */ -const listRanks = async (req, res) => { - const queryExecutor = async () => { - return await getRanksFromBQ(); - }; - - await executeBigQuery(req, res, 'ranks', queryExecutor); -}; - -export { - listRanks -}; diff --git a/src/index.js b/src/index.js index a775263..4269c99 100644 --- a/src/index.js +++ b/src/index.js @@ -35,10 +35,10 @@ const getController = async (name) => { controllers[name] = await import('./controllers/reportController.js'); break; case 'ranks': - controllers[name] = await import('./controllers/ranksBQController.js'); + controllers[name] = await import('./controllers/ranksController.js'); break; case 'geos': - controllers[name] = await import('./controllers/geosBQController.js'); + controllers[name] = await import('./controllers/geosController.js'); break; case 'versions': controllers[name] = await import('./controllers/versionsController.js'); diff --git a/src/package-lock.json b/src/package-lock.json index 62391c7..d4e94bb 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -8,7 +8,6 @@ "name": "tech-report-api", "version": "1.0.0", "dependencies": { - "@google-cloud/bigquery": "^8.1.0", "@google-cloud/firestore": "7.11.1", "@google-cloud/functions-framework": "4.0.0" }, @@ -569,310 +568,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@google-cloud/bigquery": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-8.1.0.tgz", - "integrity": "sha512-eDleD/IHKQIRm4GmMnwJvPkx4PgSaK8m8DCmDmVOf0gIhqPLSdvOAEeM4QjyyZGUGjV4yHyJfEJxzULTzl22Aw==", - "license": "Apache-2.0", - "dependencies": { - "@google-cloud/common": "^6.0.0", - "@google-cloud/paginator": "^6.0.0", - "@google-cloud/precise-date": "^5.0.0", - "@google-cloud/promisify": "^5.0.0", - "arrify": "^3.0.0", - "big.js": "^6.2.2", - "duplexify": "^4.1.3", - "extend": "^3.0.2", - "is": "^3.3.0", - "stream-events": "^1.0.5", - "teeny-request": "^10.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@google-cloud/bigquery/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/@google-cloud/bigquery/node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/@google-cloud/bigquery/node_modules/teeny-request": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", - "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", - "license": "Apache-2.0", - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^3.3.2", - "stream-events": "^1.0.5" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-6.0.0.tgz", - "integrity": "sha512-IXh04DlkLMxWgYLIUYuHHKXKOUwPDzDgke1ykkkJPe48cGIS9kkL2U/o0pm4ankHLlvzLF/ma1eO86n/bkumIA==", - "license": "Apache-2.0", - "dependencies": { - "@google-cloud/projectify": "^4.0.0", - "@google-cloud/promisify": "^4.0.0", - "arrify": "^2.0.0", - "duplexify": "^4.1.3", - "extend": "^3.0.2", - "google-auth-library": "^10.0.0-rc.1", - "html-entities": "^2.5.2", - "retry-request": "^8.0.0", - "teeny-request": "^10.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/@google-cloud/promisify": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.1.0.tgz", - "integrity": "sha512-G/FQx5cE/+DqBbOpA5jKsegGwdPniU6PuIEMt+qxWgFxvxuFOzVmp6zYchtYuwAWV5/8Dgs0yAmjvNZv3uXLQg==", - "license": "Apache-2.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/@google-cloud/common/node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@google-cloud/common/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@google-cloud/common/node_modules/gaxios": { - "version": "7.0.0-rc.6", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.0.0-rc.6.tgz", - "integrity": "sha512-osVFpgeBiwTM2AVI9MXvb8iWzM6oSMbTVWc65Gm5BgBlE+nUA6PBHFMaYpqjZx1AhUH7aPOZq78WcRAM6hhAwA==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "node-fetch": "^3.3.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/gcp-metadata": { - "version": "7.0.0-rc.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.0-rc.1.tgz", - "integrity": "sha512-E6c+AdIaK1LNA839OyotiTca+B2IG1nDlMjnlcck8JjXn3fVgx57Ib9i6iL1/iqN7bA3EUQdcRRu+HqOCOABIg==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^7.0.0-rc.1", - "google-logging-utils": "^1.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/google-auth-library": { - "version": "10.0.0-rc.3", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.0.0-rc.3.tgz", - "integrity": "sha512-WC9wfEKK0bk3seWKsDn2loduLth6JWKTsrbWftzrhPuzpwnVXb5oi2+aa0JDBxLBDdkGesLvTQ67F2nZ7leq1Q==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^7.0.0-rc.4", - "gcp-metadata": "^7.0.0-rc.1", - "google-logging-utils": "^1.0.0", - "gtoken": "^8.0.0-rc.1", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/google-logging-utils": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz", - "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/common/node_modules/gtoken": { - "version": "8.0.0-rc.1", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0-rc.1.tgz", - "integrity": "sha512-UjE/egX6ixArdcCKOkheuFQ4XN4/0gX92nd2JPVEYuRU2sWHAWuOVGnowm1fQUdQtaxqn1n8H0hOb2LCaUhJ3A==", - "license": "MIT", - "dependencies": { - "gaxios": "^7.0.0-rc.1", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/@google-cloud/common/node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/@google-cloud/common/node_modules/retry-request": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.0.tgz", - "integrity": "sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==", - "license": "MIT", - "dependencies": { - "@types/request": "^2.48.12", - "extend": "^3.0.2", - "teeny-request": "^10.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/teeny-request": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", - "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", - "license": "Apache-2.0", - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^3.3.2", - "stream-events": "^1.0.5" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/common/node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@google-cloud/firestore": { "version": "7.11.1", "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.11.1.tgz", @@ -924,45 +619,6 @@ "node": ">=10" } }, - "node_modules/@google-cloud/paginator": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz", - "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/precise-date": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-5.0.0.tgz", - "integrity": "sha512-9h0Gvw92EvPdE8AK8AgZPbMnH5ftDyPtKm7/KUfcJVaPEPjwGDsJd1QV0H8esBDV4II41R/2lDWH1epBqIoKUw==", - "license": "Apache-2.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/projectify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", - "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", - "license": "Apache-2.0", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/promisify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz", - "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=18" - } - }, "node_modules/@grpc/grpc-js": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.3.tgz", @@ -2168,18 +1824,6 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, - "node_modules/arrify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", - "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", @@ -2392,19 +2036,6 @@ ], "license": "MIT" }, - "node_modules/big.js": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", - "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", - "license": "MIT", - "engines": { - "node": "*" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/bigjs" - } - }, "node_modules/bignumber.js": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.0.tgz", @@ -2857,15 +2488,6 @@ "node": ">= 8" } }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -3321,29 +2943,6 @@ "bser": "2.1.1" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -3432,18 +3031,6 @@ "node": ">= 0.12" } }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/formidable": { "version": "3.5.4", "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz", @@ -3818,22 +3405,6 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "license": "ISC" }, - "node_modules/html-entities": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", - "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/mdevils" - }, - { - "type": "patreon", - "url": "https://patreon.com/mdevils" - } - ], - "license": "MIT" - }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -4033,15 +3604,6 @@ "node": ">= 0.10" } }, - "node_modules/is": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", - "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==", - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/is-arguments": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", @@ -5354,26 +4916,6 @@ "node": ">= 0.6" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -6874,15 +6416,6 @@ "makeerror": "1.0.12" } }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/src/package.json b/src/package.json index 1286c68..b129f94 100644 --- a/src/package.json +++ b/src/package.json @@ -13,7 +13,6 @@ "test:live": "bash ../test-api.sh" }, "dependencies": { - "@google-cloud/bigquery": "^8.1.0", "@google-cloud/firestore": "7.11.1", "@google-cloud/functions-framework": "4.0.0" }, diff --git a/src/utils/bigquery.js b/src/utils/bigquery.js deleted file mode 100644 index 753e00a..0000000 --- a/src/utils/bigquery.js +++ /dev/null @@ -1,89 +0,0 @@ -import { BigQuery } from '@google-cloud/bigquery'; - -// Initialize BigQuery client -const bigquery = new BigQuery({ - projectId: process.env.PROJECT || 'httparchive' -}); - -// BigQuery configuration optimizations -const BQ_CONFIG = { - // Optimize for BI Engine - location: 'US', // Use the same location as your BI Engine - // Use maximum parallelism for BI Engine - maximumBytesBilled: '100000000', // 100MB limit for safety - // Labels for monitoring - labels: { - 'app': 'tech-report-api', - 'source': 'bigquery-direct', - } -}; - -/** - * Execute a BigQuery query with caching support - * @param {string} query - SQL query string - * @param {Object} options - Query options - * @returns {Array} - Query results - */ -const executeBigQueryQuery = async (query, options = {}) => { - try { - const queryOptions = { - query, - - jobCreationMode: 'JOB_CREATION_OPTIONAL', // Returning immediate results is prioritized. - timeoutMs: 10000, // 10 seconds - // Use query cache when possible - useQueryCache: true, - // Apply BI Engine optimizations - ...BQ_CONFIG, - ...options - }; - - console.log('Executing BigQuery:', query); - const [rows, , metadata] = await bigquery.query(queryOptions); - if(metadata.jobReference) { - console.log(`BigQuery job ${metadata.jobReference.jobId} completed. Rows: ${rows.length}`); - } - - return rows; - } catch (error) { - console.error('BigQuery execution error:', error); - throw error; - } -}; - -/** - * Get ranks from BigQuery - * @returns {Array} - Array of rank objects - */ -const getRanksFromBQ = async () => { - const query = ` - SELECT rank - FROM \`httparchive.reports.tech_report_ranks\` - ORDER BY mobile_origins DESC - `; - - const rows = await executeBigQueryQuery(query); - return rows.map(row => ({ rank: row.rank })); -}; - -/** - * Get geos from BigQuery - * @returns {Array} - Array of geo objects - */ -const getGeosFromBQ = async () => { - const query = ` - SELECT geo - FROM \`httparchive.reports.tech_report_geos\` - ORDER BY mobile_origins DESC - `; - - const rows = await executeBigQueryQuery(query); - return rows.map(row => ({ geo: row.geo })); -}; - -export { - bigquery, - executeBigQueryQuery, - getRanksFromBQ, - getGeosFromBQ -}; diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js index 8f0a132..56aa46e 100644 --- a/src/utils/controllerHelpers.js +++ b/src/utils/controllerHelpers.js @@ -318,50 +318,6 @@ const validateTechnologyArray = (technologyParam) => { } }; -/** - * Generic BigQuery-enabled query executor - * Handles caching, query execution, and response for BigQuery queries - * @param {Object} req - Request object - * @param {Object} res - Response object - * @param {string} queryName - Query name for caching and error handling - * @param {Function} queryExecutor - Function that executes BigQuery and returns results - * @param {Function} dataProcessor - Optional function to process results - */ -const executeBigQuery = async (req, res, queryName, queryExecutor, dataProcessor = null) => { - try { - const params = req.query; - - // Generate cache key - const cacheKey = generateQueryCacheKey(`bq_${queryName}`, params); - - // Check cache first - const cachedResult = getCachedQueryResult(cacheKey); - if (cachedResult) { - res.statusCode = 200; - res.end(JSON.stringify(cachedResult)); - return; - } - - // Execute BigQuery - let data = await queryExecutor(params); - - // Process data if processor provided - if (dataProcessor) { - data = dataProcessor(data, params); - } - - // Cache the result - setCachedQueryResult(cacheKey, data); - - // Send response - res.statusCode = 200; - res.end(JSON.stringify(data)); - - } catch (error) { - handleControllerError(res, error, `executing BigQuery ${queryName}`); - } -}; - export { REQUIRED_PARAMS, FIRESTORE_IN_LIMIT, @@ -375,6 +331,5 @@ export { setCachedQueryResult, getCacheStats, executeQuery, - executeBigQuery, validateTechnologyArray }; From d8244f955aab9bd91e6cab5f2b6058de27052e25 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 18 Jun 2025 21:40:51 +0200 Subject: [PATCH 6/8] tf backend migrated --- terraform/dev/main.tf | 4 ++-- terraform/prod/main.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 037840f..dc9c003 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -1,7 +1,7 @@ terraform { backend "gcs" { - bucket = "tf-state-backingapi-20230314" - prefix = "dev" + bucket = "tfstate-httparchive" + prefix = "tech-report-apis/dev" } } diff --git a/terraform/prod/main.tf b/terraform/prod/main.tf index 50c36a0..9439dac 100644 --- a/terraform/prod/main.tf +++ b/terraform/prod/main.tf @@ -1,7 +1,7 @@ terraform { backend "gcs" { - bucket = "tf-state-backingapi-20230314" - prefix = "prod" + bucket = "tfstate-httparchive" + prefix = "tech-report-apis/prod" } } From 5a0212b1db7f5d6fde317413c2cac4e895f806cc Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Thu, 19 Jun 2025 01:38:11 +0200 Subject: [PATCH 7/8] api config update --- terraform/dev/main.tf | 13 ++++++++++--- terraform/prod/main.tf | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index dc9c003..db15672 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -52,14 +52,14 @@ paths: /v1/adoption: get: summary: adoption - operationId: getadoptionReports + operationId: getAdoptionReports responses: 200: description: String /v1/page-weight: get: summary: pageWeight - operationId: getpageWeight + operationId: getPageWeightReports responses: 200: description: String @@ -73,7 +73,7 @@ paths: /v1/cwv: get: summary: cwv - operationId: getCwv + operationId: getCWVReports responses: 200: description: String @@ -98,6 +98,13 @@ paths: responses: 200: description: String + /v1/audits: + get: + summary: audits + operationId: getAuditReports + responses: + 200: + description: String EOF ) } diff --git a/terraform/prod/main.tf b/terraform/prod/main.tf index 9439dac..721b575 100644 --- a/terraform/prod/main.tf +++ b/terraform/prod/main.tf @@ -52,14 +52,14 @@ paths: /v1/adoption: get: summary: adoption - operationId: getadoptionReports + operationId: getAdoptionReports responses: 200: description: String /v1/page-weight: get: summary: pageWeight - operationId: getpageWeight + operationId: getPageWeightReports responses: 200: description: String @@ -73,7 +73,7 @@ paths: /v1/cwv: get: summary: cwv - operationId: getCwv + operationId: getCWVReports responses: 200: description: String @@ -98,6 +98,13 @@ paths: responses: 200: description: String + /v1/audits: + get: + summary: audits + operationId: getAuditReports + responses: + 200: + description: String EOF ) } From 2383e75ecf697cd2c7b370dbeb6e2718f689bae4 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Thu, 19 Jun 2025 01:55:41 +0200 Subject: [PATCH 8/8] cache reset --- README.md | 26 ++++++++++++++++++++++++++ src/__tests__/routes.test.js | 31 +++++++++++++++++++++++++++++++ src/index.js | 7 ++++++- src/utils/controllerHelpers.js | 28 +++++++++++++++++++++++++++- test-api.sh | 23 +++++++++++++++++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f66526e..2724719 100644 --- a/README.md +++ b/README.md @@ -520,6 +520,32 @@ Returns a JSON object with the following schema: } ``` +### `POST /v1/cache-reset` + +Resets all caches in the API. This endpoint requires a POST request. + +```bash +curl --request POST \ + --url 'https://{{HOST}}/v1/cache-reset' +``` + +Returns a JSON object with the following schema: + +```json +{ + "success": true, + "message": "All caches have been reset", + "before": { + "queryCache": 150, + "dateCache": 12 + }, + "after": { + "queryCache": 0, + "dateCache": 0 + } +} +``` + ## Testing ```bash diff --git a/src/__tests__/routes.test.js b/src/__tests__/routes.test.js index b88c66d..f84cd9b 100644 --- a/src/__tests__/routes.test.js +++ b/src/__tests__/routes.test.js @@ -362,4 +362,35 @@ describe('API Routes', () => { expect(res.headers['timing-allow-origin']).toEqual('*'); }); }); + + describe('Cache Management', () => { + it('should provide cache stats', async () => { + const res = await request(app) + .get('/v1/cache-stats') + .expect(200); + + expect(res.body).toHaveProperty('queryCache'); + expect(res.body).toHaveProperty('dateCache'); + expect(res.body).toHaveProperty('config'); + }); + + it('should reset cache on POST request', async () => { + const res = await request(app) + .post('/v1/cache-reset') + .expect(200); + + expect(res.body).toHaveProperty('success', true); + expect(res.body).toHaveProperty('message'); + expect(res.body).toHaveProperty('before'); + expect(res.body).toHaveProperty('after'); + }); + + it('should handle cache reset OPTIONS request', async () => { + const res = await request(app) + .options('/v1/cache-reset') + .expect(204); + + expect(res.headers['access-control-allow-methods']).toContain('POST'); + }); + }); }); diff --git a/src/index.js b/src/index.js index 4269c99..cf2baeb 100644 --- a/src/index.js +++ b/src/index.js @@ -51,7 +51,7 @@ const getController = async (name) => { // Helper function to set CORS headers const setCORSHeaders = (res) => { res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Timing-Allow-Origin'); res.setHeader('Access-Control-Max-Age', '86400'); }; @@ -163,6 +163,11 @@ const handleRequest = async (req, res) => { const { getCacheStats } = await import('./utils/controllerHelpers.js'); const stats = getCacheStats(); sendJSONResponse(res, stats); + } else if (pathname === '/v1/cache-reset' && req.method === 'POST') { + // Cache reset endpoint + const { resetCache } = await import('./utils/controllerHelpers.js'); + const result = resetCache(); + sendJSONResponse(res, result); } else { // 404 Not Found res.statusCode = 404; diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js index 56aa46e..b22313f 100644 --- a/src/utils/controllerHelpers.js +++ b/src/utils/controllerHelpers.js @@ -318,6 +318,31 @@ const validateTechnologyArray = (technologyParam) => { } }; +/** + * Reset all caches + * @returns {Object} Reset operation result + */ +const resetCache = () => { + const beforeStats = { + queryCache: queryResultCache.size, + dateCache: latestDateCache.size + }; + + // Clear both caches + queryResultCache.clear(); + latestDateCache.clear(); + + return { + success: true, + message: 'All caches have been reset', + before: beforeStats, + after: { + queryCache: queryResultCache.size, + dateCache: latestDateCache.size + } + }; +}; + export { REQUIRED_PARAMS, FIRESTORE_IN_LIMIT, @@ -331,5 +356,6 @@ export { setCachedQueryResult, getCacheStats, executeQuery, - validateTechnologyArray + validateTechnologyArray, + resetCache }; diff --git a/test-api.sh b/test-api.sh index d97d9ff..e73e836 100755 --- a/test-api.sh +++ b/test-api.sh @@ -116,4 +116,27 @@ test_endpoint "/v1/page-weight" "?technology=WordPress&geo=ALL&rank=ALL&start=la # Test audits endpoint test_endpoint "/v1/audits" "?technology=WordPress&geo=ALL&rank=ALL&start=latest" +# Test cache stats endpoint +echo "Testing cache stats endpoint..." +test_endpoint "/v1/cache-stats" "" + +# Test cache reset endpoint +echo "Testing cache reset endpoint..." +echo "Checking cache reset: http://localhost:3000/v1/cache-reset" +response=$(curl -s -w "\n%{http_code}" -X POST "http://localhost:3000/v1/cache-reset") +http_code=$(echo "$response" | tail -n1) +body=$(echo "$response" | sed '$d') + +echo "$body" | jq . +echo "Status code: $http_code" + +if [[ $http_code -ne 200 ]]; then + echo "Error: Cache reset endpoint returned non-200 status code" + exit 1 +fi + +echo "" +echo "----------------------" +echo "" + echo "API tests complete! All endpoints returned 200 status code and CORS is properly configured."