From 75b06c9b361ee83ad4ba8e0f033f935e9ff74d9a Mon Sep 17 00:00:00 2001 From: Rohland de Charmoy Date: Fri, 4 Apr 2025 11:33:08 +0200 Subject: [PATCH 1/2] :sparkles: updated dependencies + support for env vars for secrets --- .github/workflows/main.yml | 4 +- .gitignore | 3 +- .nvmrc | 2 +- README.md | 23 ++++++++-- index.js | 3 ++ mysql-service.js | 6 ++- package-lock.json | 94 ++++++++++++++++++++++++++------------ package.json | 9 ++-- sheet-uploader.js | 37 +++++++++++++-- 9 files changed, 135 insertions(+), 46 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 255dbfb..63627a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,10 +11,10 @@ jobs: - name: Begin release... uses: actions/checkout@v4 - - name: Use Node 18 + - name: Use Node 20 uses: actions/setup-node@v4 with: - node-version: 18.x + node-version: 20.x - name: Install dependencies run: npm install diff --git a/.gitignore b/.gitignore index 1269e90..5b8ea9b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ node_modules/ *.swp *.swo *.env -*.env.* \ No newline at end of file +*.env.* +test.js \ No newline at end of file diff --git a/.nvmrc b/.nvmrc index 25bf17f..2edeafb 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18 \ No newline at end of file +20 \ No newline at end of file diff --git a/README.md b/README.md index a180d97..7da085a 100644 --- a/README.md +++ b/README.md @@ -70,20 +70,22 @@ const tasks = [ ]; module.exports = { - tasks: tasks, + tasks: tasks settings: { mysql: { prodtime: { host: "localhost", user: "root", - password: "root", - port: "3306" + port: "3306", + // for secrets, rely on environment variables (see Settings) } }, google: { gcred: { "type": "service_account", // ... + // for secrets, rely on environment variables for each key, or rather set an env var with the credential + // json, see Settings below } } } @@ -91,9 +93,20 @@ module.exports = { ``` -#### MySql Connections and Google Credential Settings -Note how in the above example, `connection` and `googleCredential` refer to keys defined in the `settings` object. +## Settings + +The configuration settings for MySql + Google Sheets API are pulled from the environment, using conventions. Environment +variables are also loaded from any local `.env` file relative to location application is launched from. + +With a named MySql connection, it will look for: + +- host: `{name}_mysql_host` +- user: `{name}_mysql_user` +- password: `{name}_mysql_password` +- port: `{name}_mysql_port` + +With a named Google Sheet credential, it will expect an environment variable called `{name}_google` which contains the credential JSON. Alternatively, set individual keys using `{name}_google_{property}`. To set up your Google Credentials, see https://www.npmjs.com/package/googleapis#service-account-credentials Note: Make sure you share the sheet (with Editor rights) with the email address noted in the `client_email` param of the service account JSON. diff --git a/index.js b/index.js index f84a95f..67c9539 100644 --- a/index.js +++ b/index.js @@ -1,4 +1,5 @@ #!/usr/bin/env node +const dotenv = require('dotenv'); const yargs = require('yargs/yargs'); const {uploadDataToGoogleSheets} = require("./sheet-uploader"); const {hideBin} = require('yargs/helpers') @@ -6,6 +7,8 @@ const argv = yargs(hideBin(process.argv)).argv const {getConfig} = require('./config-provider'); const {executeTasks} = require('./mysql-service'); +dotenv.config(); + const configPath = argv.path; if (!configPath) { console.error(`error: param --path is required\n`); diff --git a/mysql-service.js b/mysql-service.js index 69e547e..cd593e5 100644 --- a/mysql-service.js +++ b/mysql-service.js @@ -7,7 +7,11 @@ function getConnection( if (!settings.mysql) { throw new Error(`expected mysql to be defined on settings object but wasn't`); } - const config = settings.mysql[name]; + const config = settings.mysql[name] ?? {}; + const keys = ["host", "user", "password", "port"]; + keys.forEach(key => { + config[key] = config[key] ?? process.env[`${name}_mysql_${key}`]; + }); if (config.host && config.user && config.password diff --git a/package-lock.json b/package-lock.json index 60f1654..f1cc5be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,24 +1,25 @@ { "name": "mysql2sheet", - "version": "1.0.6", + "version": "2.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mysql2sheet", - "version": "1.0.6", + "version": "2.0.0", "license": "BSD-3-Clause", "dependencies": { - "googleapis": "^135.0.0", + "dotenv": "^16.4.7", + "googleapis": "^148.0.0", "knex": "^3.1.0", - "mysql2": "^3.9.7", + "mysql2": "^3.14.0", "yargs": "^17.7.2" }, "bin": { "mysql2sheet": "index.js" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/agent-base": { @@ -54,6 +55,14 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -186,6 +195,17 @@ "node": ">=0.10" } }, + "node_modules/dotenv": { + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -338,9 +358,9 @@ } }, "node_modules/googleapis": { - "version": "135.0.0", - "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-135.0.0.tgz", - "integrity": "sha512-gIbhO1ptjgg74DEO5WIiyO1l4zLL7Lssab0XoIdZ+TYArib4SwQbKObpidRBdYAixf/fmQd00JIYYx6JtpZzew==", + "version": "148.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-148.0.0.tgz", + "integrity": "sha512-8PDG5VItm6E1TdZWDqtRrUJSlBcNwz0/MwCa6AL81y/RxPGXJRUwKqGZfCoVX1ZBbfr3I4NkDxBmeTyOAZSWqw==", "dependencies": { "google-auth-library": "^9.0.0", "googleapis-common": "^7.0.0" @@ -585,12 +605,18 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" }, - "node_modules/lru-cache": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", - "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==", + "node_modules/lru.min": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.2.tgz", + "integrity": "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg==", "engines": { - "node": ">=16.14" + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" } }, "node_modules/ms": { @@ -599,15 +625,16 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mysql2": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.7.tgz", - "integrity": "sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.0.tgz", + "integrity": "sha512-8eMhmG6gt/hRkU1G+8KlGOdQi2w+CgtNoD1ksXZq9gQfkfDsX4LHaBwTe1SY0Imx//t2iZA03DFnyYKPinxSRw==", "dependencies": { + "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", "long": "^5.2.1", - "lru-cache": "^8.0.0", + "lru.min": "^1.0.0", "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" @@ -958,6 +985,11 @@ "color-convert": "^2.0.1" } }, + "aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1041,6 +1073,11 @@ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, + "dotenv": { + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==" + }, "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -1157,9 +1194,9 @@ } }, "googleapis": { - "version": "135.0.0", - "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-135.0.0.tgz", - "integrity": "sha512-gIbhO1ptjgg74DEO5WIiyO1l4zLL7Lssab0XoIdZ+TYArib4SwQbKObpidRBdYAixf/fmQd00JIYYx6JtpZzew==", + "version": "148.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-148.0.0.tgz", + "integrity": "sha512-8PDG5VItm6E1TdZWDqtRrUJSlBcNwz0/MwCa6AL81y/RxPGXJRUwKqGZfCoVX1ZBbfr3I4NkDxBmeTyOAZSWqw==", "requires": { "google-auth-library": "^9.0.0", "googleapis-common": "^7.0.0" @@ -1324,10 +1361,10 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" }, - "lru-cache": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", - "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==" + "lru.min": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.2.tgz", + "integrity": "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg==" }, "ms": { "version": "2.1.2", @@ -1335,15 +1372,16 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mysql2": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.7.tgz", - "integrity": "sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.0.tgz", + "integrity": "sha512-8eMhmG6gt/hRkU1G+8KlGOdQi2w+CgtNoD1ksXZq9gQfkfDsX4LHaBwTe1SY0Imx//t2iZA03DFnyYKPinxSRw==", "requires": { + "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", "long": "^5.2.1", - "lru-cache": "^8.0.0", + "lru.min": "^1.0.0", "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" diff --git a/package.json b/package.json index c42127c..19f4a2b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mysql2sheet", - "version": "1.1.0", + "version": "2.0.0", "description": "Does what it says on the tin. Takes data in your MySql database and uploads it to a Google Sheet.", "main": "index.js", "bin": { @@ -32,13 +32,14 @@ "url": "https://github.com/rohland/mysql2sheet" }, "engines": { - "node": ">=18" + "node": ">=20" }, "homepage": "https://github.com/rohland/mysql2sheet#readme", "dependencies": { - "googleapis": "^135.0.0", + "dotenv": "^16.4.7", + "googleapis": "^148.0.0", "knex": "^3.1.0", - "mysql2": "^3.9.7", + "mysql2": "^3.14.0", "yargs": "^17.7.2" } } diff --git a/sheet-uploader.js b/sheet-uploader.js index 40e497b..36f187b 100644 --- a/sheet-uploader.js +++ b/sheet-uploader.js @@ -8,10 +8,7 @@ function getGoogleAuth( if (cachedAuthToken) { return cachedAuthToken; } - const credential = credentials[name]; - if (!credential) { - throw new Error(`could not find google credential with name '${name}`); - } + const credential = getGoogleCredentials(name, credentials); const auth = new google.auth.JWT( credential.client_email, null, @@ -25,6 +22,38 @@ function getGoogleAuth( return auth; } +function getGoogleCredentials(name, credentials) { + try { + let credential = (credentials ?? {})[name]; + if (credential) { + const keys = [ + "type", + "project_id", + "private_key_id", + "private_key", + "client_email", + "client_id", + "auth_uri", + "token_uri", + "auth_provider_x509_cert_url", + "client_x509_cert_url"]; + keys.forEach(key => { + credential[key] = credential[key] ?? process.env[`${name}_google_${key}`]; + }); + } else { + credential = process.env[`${name}_google`] + ? JSON.parse(process.env[`${name}_google`]) + : null; + } + if (!credential) { + throw new Error(`could not find google credential with name '${name}`); + } + return credential; + } catch (err) { + throw new Error(`could not parse google credential with name '${name}'`, { cause: err}); + } +} + function getResultType(task) { const resultType = task.type; if (!resultType) { From 662cd55d4b2abf4a9a53c15ee7dea5fecc2839dd Mon Sep 17 00:00:00 2001 From: Rohland de Charmoy Date: Fri, 4 Apr 2025 11:51:36 +0200 Subject: [PATCH 2/2] :loud_sound: added more information about missing keys --- index.js | 0 mysql-service.js | 9 ++++----- package.json | 2 +- sheet-uploader.js | 6 ++++-- 4 files changed, 9 insertions(+), 8 deletions(-) mode change 100644 => 100755 index.js diff --git a/index.js b/index.js old mode 100644 new mode 100755 diff --git a/mysql-service.js b/mysql-service.js index cd593e5..5d18918 100644 --- a/mysql-service.js +++ b/mysql-service.js @@ -12,13 +12,12 @@ function getConnection( keys.forEach(key => { config[key] = config[key] ?? process.env[`${name}_mysql_${key}`]; }); - if (config.host - && config.user - && config.password - && config.port) { + const missingKeys = keys.filter(key => !config[key]); + if (missingKeys.length === 0) { return config; } - throw new Error(`One or more mysql settings are missing for connection named '${name}'`); + console.error(`missing keys for mysql connection '${name}': ${missingKeys.join(", ")}`); + process.exit(-1); } function getKnexConnection( diff --git a/package.json b/package.json index 19f4a2b..3092343 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mysql2sheet", - "version": "2.0.0", + "version": "2.0.1", "description": "Does what it says on the tin. Takes data in your MySql database and uploads it to a Google Sheet.", "main": "index.js", "bin": { diff --git a/sheet-uploader.js b/sheet-uploader.js index 36f187b..d4af1e4 100644 --- a/sheet-uploader.js +++ b/sheet-uploader.js @@ -46,11 +46,13 @@ function getGoogleCredentials(name, credentials) { : null; } if (!credential) { - throw new Error(`could not find google credential with name '${name}`); + console.error(`missing env keys for google credential '${name}'`); + process.exit(-1); } return credential; } catch (err) { - throw new Error(`could not parse google credential with name '${name}'`, { cause: err}); + console.error(`could not parse google credential with name '${name}'`, err); + process.exit(-1); } }