From 26b8e871b1516669a234865c9e2bbd3b4c2b985e Mon Sep 17 00:00:00 2001 From: AceTheNinja Date: Mon, 20 Oct 2025 10:41:00 +0400 Subject: [PATCH 1/4] moved report email to core-mail --- README.md | 9 +- config/facs/redis.config.json.example | 7 ++ config/service.report.json.example | 3 +- package-lock.json | 100 ++++++++++++++++++ package.json | 3 +- setup-config.sh | 25 +++++ workers/api.service.report.wrk.js | 3 +- workers/loc.api/di/app.deps.js | 8 +- workers/loc.api/di/types.js | 3 +- workers/loc.api/queue/send-mail/index.js | 55 ++-------- .../loc.api/queue/send-mail/views/email.pug | 46 -------- 11 files changed, 154 insertions(+), 108 deletions(-) create mode 100644 config/facs/redis.config.json.example create mode 100644 setup-config.sh delete mode 100644 workers/loc.api/queue/send-mail/views/email.pug diff --git a/README.md b/README.md index 1f341381..b0699fff 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,8 @@ npm install ### Configure service -- As to configure the service copy the json.example files from config folder into new ones. Open a console on projects folder a copy the following codes : - -```console -cp config/common.json.example config/common.json -cp config/service.report.json.example config/service.report.json -cp config/schedule.json.example config/schedule.json -cp config/facs/grc.config.json.example config/facs/grc.config.json +```bash +bash setup-config.sh ``` - To set grenache client for express, edit common.json. If running locally, leave actual values skipping this step. diff --git a/config/facs/redis.config.json.example b/config/facs/redis.config.json.example new file mode 100644 index 00000000..c7cb5f16 --- /dev/null +++ b/config/facs/redis.config.json.example @@ -0,0 +1,7 @@ +{ + "sk0": { + "host": "127.0.0.1", + "port": 6379, + "auth": "" + } +} diff --git a/config/service.report.json.example b/config/service.report.json.example index 4f497869..e9a25fbf 100644 --- a/config/service.report.json.example +++ b/config/service.report.json.example @@ -16,5 +16,6 @@ "isAddedUniqueEndingToReportFileName": false, "isCompress": true, "isLoggerDisabled": false, - "isHosted": true + "isHosted": true, + "mailQueue": "core.mail" } diff --git a/package-lock.json b/package-lock.json index 57f68d7d..49909996 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "bfx-facs-grc-slack": "git+https://github.com/bitfinexcom/bfx-facs-grc-slack.git", "bfx-facs-interval": "git+https://github.com/bitfinexcom/bfx-facs-interval.git", "bfx-facs-lokue": "git+https://github.com:bitfinexcom/bfx-facs-lokue.git", + "bfx-facs-redis": "git+https://github.com/bitfinexcom/bfx-facs-redis.git", "bfx-svc-boot-js": "https://github.com/bitfinexcom/bfx-svc-boot-js.git", "bfx-wrk-api": "git+https://github.com/bitfinexcom/bfx-wrk-api.git", "bitfinex-api-node": "7.0.0", @@ -257,6 +258,12 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/@ioredis/commands": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz", + "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==", + "license": "MIT" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -478,6 +485,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1246,6 +1254,17 @@ "lodash": "^4.17.14" } }, + "node_modules/bfx-facs-redis": { + "version": "0.0.2", + "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-facs-redis.git#23ad81e1cc30bf244dd09c83705a0df9cb174249", + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.1", + "bfx-facs-base": "git+https://github.com/bitfinexcom/bfx-facs-base.git", + "ioredis": "^5.3.0", + "lodash": "^4.17.21" + } + }, "node_modules/bfx-hf-util": { "version": "1.0.12", "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-hf-util.git#82ce185e336d14c84037494e62e6c5d84434cdad", @@ -1656,6 +1675,7 @@ "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.4.tgz", "integrity": "sha512-yS5H68VYOCtN1cjfwumDSuzn/9c+yza4f3reKXlE5rUg7SFcCEy90gJvydNgOYtblyf4Zi6jIWRnXOgErta0KA==", "license": "MIT", + "peer": true, "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.2", @@ -1814,6 +1834,15 @@ "wrap-ansi": "^7.0.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", @@ -2297,6 +2326,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -2670,6 +2708,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -2874,6 +2913,7 @@ "integrity": "sha512-ixmkI62Rbc2/w8Vfxyh1jQRTdRTF52VxwRVHl/ykPAmqG+Nb7/kNn+byLP0LxPgI7zWA16Jt82SybJInmMia3A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.8", @@ -2941,6 +2981,7 @@ "integrity": "sha512-jDex9s7D/Qial8AGVIHq4W7NswpUD5DPDL2RH8Lzd9EloWUuvUkHfv4FRLMipH5q2UtyurorBkPeNi1wVWNh3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "builtins": "^5.0.1", "eslint-plugin-es": "^4.1.0", @@ -2967,6 +3008,7 @@ "integrity": "sha512-57Zzfw8G6+Gq7axm2Pdo3gW/Rx3h9Yywgn61uE/3elTCOePEHVrn2i5CdfBwA1BLK0Q0WqctICIUSqXZW/VprQ==", "dev": true, "license": "ISC", + "peer": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, @@ -2983,6 +3025,7 @@ "integrity": "sha512-BGP0jRmfYyvOyvMoRX/uoUeW+GqNj9y16bPQzqAHf3AYII/tDs+jMN0dBVkl88/OZwNGwrVFxE7riHsXVfy/LQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "array-includes": "^3.1.8", "array.prototype.findlast": "^1.2.5", @@ -4271,6 +4314,30 @@ "integrity": "sha512-B3ex30927698TJENHR++8FfEaJGqoWOgI6ZY5Ht/nLUsFCwHn6akbwtnUAPCgUepAnTpe2qHxhDNjoKLyz6rgQ==", "license": "MIT" }, + "node_modules/ioredis": { + "version": "5.8.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.1.tgz", + "integrity": "sha512-Qho8TgIamqEPdgiMadJwzRMW3TudIg6vpg4YONokGDudy4eqRIJtDbVX72pfLBcWxvbn3qm/40TyGUObdW4tLQ==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.4.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -5100,6 +5167,12 @@ "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", "license": "MIT" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -6471,6 +6544,27 @@ "b4a": "^1.3.1" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -7125,6 +7219,12 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/standard-engine": { "version": "15.1.0", "resolved": "https://registry.npmjs.org/standard-engine/-/standard-engine-15.1.0.tgz", diff --git a/package.json b/package.json index cba8cbaf..f9fff634 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "bfx-facs-grc-slack": "git+https://github.com/bitfinexcom/bfx-facs-grc-slack.git", "bfx-facs-interval": "git+https://github.com/bitfinexcom/bfx-facs-interval.git", "bfx-facs-lokue": "git+https://github.com:bitfinexcom/bfx-facs-lokue.git", - "bfx-svc-boot-js": "https://github.com/bitfinexcom/bfx-svc-boot-js.git", + "bfx-facs-redis": "git+https://github.com/bitfinexcom/bfx-facs-redis.git", + "bfx-svc-boot-js": "git+https://github.com/bitfinexcom/bfx-svc-boot-js.git", "bfx-wrk-api": "git+https://github.com/bitfinexcom/bfx-wrk-api.git", "bitfinex-api-node": "7.0.0", "colors": "1.4.0", diff --git a/setup-config.sh b/setup-config.sh new file mode 100644 index 00000000..c315c993 --- /dev/null +++ b/setup-config.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +include_test=false +while [ $# -ne 0 ]; do + arg="$1" + case "$arg" in + -t | --test ) include_test=true; break ;; + esac + shift +done + +for file in $(find config -type f -name "*.example"); do + new_name=$(echo "$file" | sed -E -e 's/.example//') + echo "$file -> $new_name" + cp "$file" "$new_name" + + if [ "$include_test" = true ]; then + dir_name=$(dirname $new_name) + base_name=$(basename $new_name) + test_name="$dir_name/test.$base_name" + echo "$file -> $test_name" + cp "$file" "$test_name" + fi +done diff --git a/workers/api.service.report.wrk.js b/workers/api.service.report.wrk.js index 01a2b573..7001ea46 100644 --- a/workers/api.service.report.wrk.js +++ b/workers/api.service.report.wrk.js @@ -180,7 +180,8 @@ class WrkReportServiceApi extends WrkApi { 's0', 's0', {} - ] + ], + ['fac', 'bfx-facs-redis', 'sk0', 'sk0', {}] ] this.setInitFacs(facs) diff --git a/workers/loc.api/di/app.deps.js b/workers/loc.api/di/app.deps.js index 91c69ec6..5569e8fb 100644 --- a/workers/loc.api/di/app.deps.js +++ b/workers/loc.api/di/app.deps.js @@ -41,9 +41,13 @@ module.exports = ({ deflateFac, grcSlackFac, link, + redisSk0, + conf, i18next }) => { return new ContainerModule((bind) => { + bind(TYPES.CONF).toConstantValue(conf) + bind(TYPES.RedisSk0).toConstantValue(redisSk0) bind(TYPES.RService).toConstantValue(rService) bind(TYPES.RootPath).toConstantValue(rService.ctx.rootPath) bind(TYPES.I18next).toConstantValue(i18next) @@ -173,8 +177,8 @@ module.exports = ({ bindDepsToFn( sendMail, [ - TYPES.GrcBfxReq, - TYPES.I18next + TYPES.CONF, + TYPES.RedisSk0 ] ) ) diff --git a/workers/loc.api/di/types.js b/workers/loc.api/di/types.js index 74be5ae9..a5c9f417 100644 --- a/workers/loc.api/di/types.js +++ b/workers/loc.api/di/types.js @@ -38,5 +38,6 @@ module.exports = { WeightedAveragesReport: Symbol.for('WeightedAveragesReport'), WeightedAveragesReportCsvWriter: Symbol.for('WeightedAveragesReportCsvWriter'), BfxApiRouter: Symbol.for('BfxApiRouter'), - PdfWriter: Symbol.for('PdfWriter') + PdfWriter: Symbol.for('PdfWriter'), + RedisSk0: Symbol.for('RedisSk0') } diff --git a/workers/loc.api/queue/send-mail/index.js b/workers/loc.api/queue/send-mail/index.js index 1dfadd91..dc0a05f3 100644 --- a/workers/loc.api/queue/send-mail/index.js +++ b/workers/loc.api/queue/send-mail/index.js @@ -1,15 +1,5 @@ 'use strict' -const path = require('path') -const pug = require('pug') - -const getTranslator = require('../../helpers/get-translator') -const TRANSLATION_NAMESPACES = require( - '../../i18next/translation.namespaces' -) - -const basePathToViews = path.join(__dirname, 'views') - const SENDGRID_WORKER_SUPPORTED_LNGS = { en: 'en', 'en-US': 'en', @@ -27,59 +17,26 @@ const _getSendgridLng = (lng) => { return SENDGRID_WORKER_SUPPORTED_LNGS[lng] ?? lng } -module.exports = (grcBfxReq, i18next) => { +module.exports = (conf, redisSk0) => { return async ( configs, to, - viewName, dataArr ) => { - const pathToView = path.join(basePathToViews, viewName) - const promises = dataArr.map(data => { const { presigned_url: url, language = 'en' } = data ?? {} - const translate = getTranslator( - { i18next }, - { - lng: language, - ns: TRANSLATION_NAMESPACES.EMAIL - } - ) - const subject = translate( - configs.subject, - 'template.subject' - ) - const text = pug.renderFile( - pathToView, - { - ...data, - filters: { translate } - } - ) - const button = { - url, - text: translate( - 'Download Report', - 'template.btnText' - ) - } - const mailOptions = { + + const payload = { ...configs, + reportUrl: url, to, - text, - subject, - button, - language: _getSendgridLng(language) + lang: _getSendgridLng(language) } - return grcBfxReq({ - service: 'rest:ext:sendgrid', - action: 'sendEmail', - args: [mailOptions] - }) + return redisSk0.cli_rw.lpush(conf.mailQueue, JSON.stringify({ type: 'report-download-ready', payload })) }) return Promise.all(promises) diff --git a/workers/loc.api/queue/send-mail/views/email.pug b/workers/loc.api/queue/send-mail/views/email.pug deleted file mode 100644 index 3d1387d0..00000000 --- a/workers/loc.api/queue/send-mail/views/email.pug +++ /dev/null @@ -1,46 +0,0 @@ -if isUnauth - p - :translate(prop='template.unauth') - Your file could not be completed, please try again -else - p(style='color: #606060;font-size: 15px;line-height: 150%;') - :translate(prop='template.readyForDownload') - The report you request is ready for download - p(style='color: #606060;font-size: 15px;line-height: 150%;') - :translate(prop='template.fileName') - File name - | : - b(style='margin-left: 5px;') - | #{fileName} - if signatureS3 - p(style='color: #606060;font-size: 15px;line-height: 150%;') - :translate(prop='template.youCan') - You can - a( - style=` - margin:0 5px;letter-spacing:.7;line-height:100%; - text-decoration:none;color:#4995c4;word-wrap:break-word; - ` - target='_blank' - href=signatureS3.presigned_url - ) - :translate(prop='template.download') - download - :translate(prop='template.pgpSignature') - PGP digital signature file - - p(style='color: #606060;font-size: 15px;line-height: 150%;') - :translate(prop='template.ifDidNotInitAction') - If you did not initiate this action and you suspect that your account may be compromised, please - - a( - href='https://www.bitfinex.com/freeze', - target='_blank', - style='text-decoration: none; color: #4995c4' - ) - |  - :translate(prop='template.freezeAccount') - freeze your account - |  - :translate(prop='template.contactSupport') - and contact support From c4ea5fbfd3ac48d60504e5709922bd96308fca62 Mon Sep 17 00:00:00 2001 From: AceTheNinja Date: Mon, 20 Oct 2025 10:46:02 +0400 Subject: [PATCH 2/4] passed redis params to service inti --- workers/api.service.report.wrk.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workers/api.service.report.wrk.js b/workers/api.service.report.wrk.js index 7001ea46..556a9659 100644 --- a/workers/api.service.report.wrk.js +++ b/workers/api.service.report.wrk.js @@ -206,6 +206,8 @@ class WrkReportServiceApi extends WrkApi { link: this.grc_bfx.link, deflateFac: this.deflate_gzip, grcSlackFac, + redisSk0: this.redis_sk0, + conf: this.conf.report, i18next, ...deps }) From 788577e67b3996af8cb27546e992a46215dd3ff4 Mon Sep 17 00:00:00 2001 From: AceTheNinja Date: Mon, 20 Oct 2025 11:23:26 +0400 Subject: [PATCH 3/4] Refactor dependency bindings --- package.json | 3 +-- workers/loc.api/di/app.deps.js | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index f9fff634..cfb62438 100644 --- a/package.json +++ b/package.json @@ -71,8 +71,7 @@ ".idea/", ".vscode/", "csv/", - "workers/loc.api/queue/temp/", - "workers/loc.api/queue/views/" + "workers/loc.api/queue/temp/" ] }, "betterScripts": { diff --git a/workers/loc.api/di/app.deps.js b/workers/loc.api/di/app.deps.js index 5569e8fb..9353d997 100644 --- a/workers/loc.api/di/app.deps.js +++ b/workers/loc.api/di/app.deps.js @@ -46,11 +46,10 @@ module.exports = ({ i18next }) => { return new ContainerModule((bind) => { - bind(TYPES.CONF).toConstantValue(conf) - bind(TYPES.RedisSk0).toConstantValue(redisSk0) bind(TYPES.RService).toConstantValue(rService) bind(TYPES.RootPath).toConstantValue(rService.ctx.rootPath) bind(TYPES.I18next).toConstantValue(i18next) + bind(TYPES.RedisSk0).toConstantValue(redisSk0) bind(TYPES.RServiceDepsSchema).toConstantValue([ ['_responder', TYPES.Responder], ['_getREST', TYPES.GetREST], From 0f5a5e02c16e33422a6b432e942ff5788a8138e2 Mon Sep 17 00:00:00 2001 From: AceTheNinja Date: Mon, 20 Oct 2025 11:35:22 +0400 Subject: [PATCH 4/4] Remove email template reference from mail sending function --- workers/loc.api/queue/aggregator.js | 1 - 1 file changed, 1 deletion(-) diff --git a/workers/loc.api/queue/aggregator.js b/workers/loc.api/queue/aggregator.js index 46a893f5..6b3e9fce 100644 --- a/workers/loc.api/queue/aggregator.js +++ b/workers/loc.api/queue/aggregator.js @@ -57,7 +57,6 @@ module.exports = ( await sendMail( emailConf, email, - 'email.pug', s3Data.map((item, i) => ({ ...item, isUnauth,