From 6e24f04069847186cc2688ec36dc8a3b921c3ca5 Mon Sep 17 00:00:00 2001 From: grizim Date: Wed, 21 Mar 2018 20:53:44 +0200 Subject: [PATCH 1/3] Goodness squad is here. Added ratelimitter using redis. The rate limit is over IP with rate limit threshold. In config.json under rateLimiter: redis host redis port ratelimit threshold rarelimit ttl seconds Whatever was done, the middleware is being requested 3 times when hitting the homepage. So keep in mind multiply by 3 any threshold is being configured. Mind that maybe other pages than the homepage is calling this middleware even more than 3 times. --- package.json | 214 +++++++++++++++--------------- server/config.development.json | 6 + server/middleware.json | 1 + server/middleware/rate-limiter.js | 25 ++++ 4 files changed, 140 insertions(+), 106 deletions(-) create mode 100644 server/middleware/rate-limiter.js diff --git a/package.json b/package.json index 1df5525..4d86baf 100644 --- a/package.json +++ b/package.json @@ -1,108 +1,110 @@ { - "name": "oe-cloud", - "version": "1.2.0", - "main": "server/server.js", - "engines": { - "node": ">=6.9.0" - }, - "scripts": { - "pretest": "npm install --no-optional", - "test": "./node_modules/.bin/mocha", - "grunt-test": "./node_modules/.bin/grunt mochaTest:test", - "precoverage": "npm install --no-optional", - "coverage": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- -R spec", - "grunt-cover": "./node_modules/.bin/grunt test-with-coverage", - "oracle": "grunt mochaTest:oracle", - "lint": "./node_modules/.bin/eslint .", - "fix-lint": "./node_modules/.bin/eslint --fix .", - "validate": "npm ls" - }, - "dependencies": { - "adm-zip": "^0.4.7", - "amqplib": "^0.5.1", - "async": "2.4.1", - "body-parser": "1.14.1", - "bunyan": "^1.5.1", - "bunyan-prettystream": "", - "camelcase": "^3.0.0", - "cfenv": "^1.0.4", - "compression": "^1.6.2", - "connect-ensure-login": "0.1.1", - "cookie-parser": "1.4.0", - "cors": "2.7.1", - "debug": "2.2.0", - "errorhandler": "^1.4.3", - "express-flash": "0.0.2", - "express-session": "1.12.1", - "fast-memoize": "^2.2.7", - "fs-extra": "^3.0.1", - "gelf-stream": "^1.1.1", - "helmet": "^2.1.1", - "inflection": "^1.10.0", - "js-feel": "^1.3.0", - "locks": "^0.2.2", - "lodash": "3.10.1", - "loopback": "2.25.0", - "loopback-boot": "git+https://github.com/EdgeVerve/loopback-boot.git#master", - "loopback-component-passport": "git+https://github.com/EdgeVerve/loopback-component-passport.git#master", - "loopback-component-push": "1.7.0", - "loopback-component-storage": "git+https://github.com/EdgeVerve/loopback-component-storage.git#master", - "loopback-connector-mongodb": "git+https://github.com/EdgeVerve/loopback-connector-mongodb.git#master", - "loopback-connector-nodes-for-Node-RED": "git+https://github.com/EdgeVerve/loopback-connector-nodes-for-Node-RED.git#master", - "loopback-datasource-juggler": "git+https://github.com/EdgeVerve/loopback-datasource-juggler.git#master", - "loopback-filters": "0.1.2", - "lru-cache": "^4.0.1", - "mail": "^0.2.3", - "mqtt": "^1.13.0", - "mustache": "^2.3.0", - "mv": "^2.1.1", - "node-async-locks": "^1.0.1", - "node-red": "0.16.2", - "oe-explorer": "^1.0.0", - "oe-jwt-generator": "^1.0.0", - "oe-logger": "^1.2.0", - "oe-swagger-ui": "^1.2.0", - "passport-google-oauth": "0.1.5", - "passport-local": "1.0.0", - "passport-saml": "^0.15.0", - "q": "^1.5.1", - "request": "^2.67.0", - "restler": "^3.4.0", - "strong-error-handler": "^2.1.0", - "ua-parser-js": "^0.7.10", - "uuid": "^3.1.0", - "uws": "^9.14.0", - "ws": "^3.0.0" - }, - "devDependencies": { - "app-root-path": "^1.0.0", - "babel-eslint": "^7.2.3", - "chai": "^3.4.1", - "chai-datetime": "^1.4.0", - "chai-things": "^0.2.0", - "chalk": "^1.1.1", - "eslint": "^4.10.0", - "grunt": "^0.4.5", - "grunt-banner": "^0.6.0", - "grunt-cli": "^0.1.13", - "grunt-contrib-clean": "^0.7.0", - "grunt-contrib-copy": "^1.0.0", - "grunt-jsbeautifier": "^0.2.12", - "grunt-mkdir": "^1.0.0", - "grunt-mocha-istanbul": "^5.0.2", - "grunt-mocha-test": "^0.13.2", - "istanbul": "0.4.1", - "loopback-connector-postgresql": "git+https://github.com/EdgeVerve/loopback-connector-postgresql.git#master", - "mocha": "^3.4.2", - "nock": "^7.2.2", - "passport-ldapauth": "^2.0.0", - "sinon": "^1.17.7", - "superagent-defaults": "^0.1.14", - "supertest": "^1.1.0" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/EdgeVerve/oe-cloud.git" - }, - "description": "oeCloud.io - Node.js framework for Enterprise application development" + "name": "oe-cloud", + "version": "1.2.0", + "main": "server/server.js", + "engines": { + "node": ">=6.9.0" + }, + "scripts": { + "pretest": "npm install --no-optional", + "test": "./node_modules/.bin/mocha", + "grunt-test": "./node_modules/.bin/grunt mochaTest:test", + "precoverage": "npm install --no-optional", + "coverage": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- -R spec", + "grunt-cover": "./node_modules/.bin/grunt test-with-coverage", + "oracle": "grunt mochaTest:oracle", + "lint": "./node_modules/.bin/eslint .", + "fix-lint": "./node_modules/.bin/eslint --fix .", + "validate": "npm ls" + }, + "dependencies": { + "adm-zip": "^0.4.7", + "amqplib": "^0.5.1", + "async": "2.4.1", + "body-parser": "1.14.1", + "bunyan": "^1.5.1", + "bunyan-prettystream": "", + "camelcase": "^3.0.0", + "cfenv": "^1.0.4", + "compression": "^1.6.2", + "connect-ensure-login": "0.1.1", + "cookie-parser": "1.4.0", + "cors": "2.7.1", + "debug": "2.2.0", + "errorhandler": "^1.4.3", + "express-flash": "0.0.2", + "express-session": "1.12.1", + "fast-memoize": "^2.2.7", + "fs-extra": "^3.0.1", + "gelf-stream": "^1.1.1", + "helmet": "^2.1.1", + "inflection": "^1.10.0", + "js-feel": "^1.3.0", + "locks": "^0.2.2", + "lodash": "3.10.1", + "loopback": "2.25.0", + "loopback-boot": "git+https://github.com/EdgeVerve/loopback-boot.git#master", + "loopback-component-passport": "git+https://github.com/EdgeVerve/loopback-component-passport.git#master", + "loopback-component-push": "1.7.0", + "loopback-component-storage": "git+https://github.com/EdgeVerve/loopback-component-storage.git#master", + "loopback-connector-mongodb": "git+https://github.com/EdgeVerve/loopback-connector-mongodb.git#master", + "loopback-connector-nodes-for-Node-RED": "git+https://github.com/EdgeVerve/loopback-connector-nodes-for-Node-RED.git#master", + "loopback-datasource-juggler": "git+https://github.com/EdgeVerve/loopback-datasource-juggler.git#master", + "loopback-filters": "0.1.2", + "lru-cache": "^4.0.1", + "mail": "^0.2.3", + "mqtt": "^1.13.0", + "mustache": "^2.3.0", + "mv": "^2.1.1", + "node-async-locks": "^1.0.1", + "node-red": "0.16.2", + "oe-explorer": "^1.0.0", + "oe-jwt-generator": "^1.0.0", + "oe-logger": "^1.2.0", + "oe-swagger-ui": "^1.2.0", + "passport-google-oauth": "0.1.5", + "passport-local": "1.0.0", + "passport-saml": "^0.15.0", + "q": "^1.5.1", + "redis": "^2.8.0", + "redis-client": "^0.3.5", + "request": "^2.67.0", + "restler": "^3.4.0", + "strong-error-handler": "^2.1.0", + "ua-parser-js": "^0.7.10", + "uuid": "^3.1.0", + "uws": "^9.14.0", + "ws": "^3.0.0" + }, + "devDependencies": { + "app-root-path": "^1.0.0", + "babel-eslint": "^7.2.3", + "chai": "^3.4.1", + "chai-datetime": "^1.4.0", + "chai-things": "^0.2.0", + "chalk": "^1.1.1", + "eslint": "^4.10.0", + "grunt": "^0.4.5", + "grunt-banner": "^0.6.0", + "grunt-cli": "^0.1.13", + "grunt-contrib-clean": "^0.7.0", + "grunt-contrib-copy": "^1.0.0", + "grunt-jsbeautifier": "^0.2.12", + "grunt-mkdir": "^1.0.0", + "grunt-mocha-istanbul": "^5.0.2", + "grunt-mocha-test": "^0.13.2", + "istanbul": "0.4.1", + "loopback-connector-postgresql": "git+https://github.com/EdgeVerve/loopback-connector-postgresql.git#master", + "mocha": "^3.4.2", + "nock": "^7.2.2", + "passport-ldapauth": "^2.0.0", + "sinon": "^1.17.7", + "superagent-defaults": "^0.1.14", + "supertest": "^1.1.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/EdgeVerve/oe-cloud.git" + }, + "description": "oeCloud.io - Node.js framework for Enterprise application development" } diff --git a/server/config.development.json b/server/config.development.json index 1f6bdbb..2079ab2 100644 --- a/server/config.development.json +++ b/server/config.development.json @@ -3,6 +3,12 @@ "host": "0.0.0.0", "port": 3000, "cookieSecret": "246bace2-9876-4321-1234-0ae8160b07c8", + "rateLimiter": { + "redisHost": "127.0.0.1", + "redisPort": "6379", + "rateLimitThreshold": 300, + "rateLimitTLLSeconds": 3 + }, "remoting": { "context": false, "rest": { diff --git a/server/middleware.json b/server/middleware.json index 1131ea9..7f33d93 100644 --- a/server/middleware.json +++ b/server/middleware.json @@ -1,5 +1,6 @@ { "initial:before": { + "./middleware/rate-limiter": {}, "loopback#favicon": {} }, "initial": { diff --git a/server/middleware/rate-limiter.js b/server/middleware/rate-limiter.js new file mode 100644 index 0000000..f5d522f --- /dev/null +++ b/server/middleware/rate-limiter.js @@ -0,0 +1,25 @@ +var config = require('../config').rateLimiter + +var redis = require("redis"), + client = redis.createClient({ host: config.redisHost, port: config.redisPort }) + +module.exports = function rateLimiterMiddleware(options) { + return function rateLimiter(req, res, next) { + var ip = req.connection.remoteAddress + client.get(ip, function (err, reply) { + + if (!reply) { + client.set(ip, '1', 'EX', config.rateLimitTLLSeconds) + } + if (Number(reply) >= config.rateLimitThreshold) { + res.status(500).send('Unauthorized. \n rate exceeded.'); + } + else { + client.incr(ip); + next(); + } + }) + }; +}; + + From a477309b621e43dda4405308ea35e4aa7e76913c Mon Sep 17 00:00:00 2001 From: grizim Date: Wed, 21 Mar 2018 20:57:35 +0200 Subject: [PATCH 2/3] #goodnessSquad --- server/config.development.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/config.development.json b/server/config.development.json index 2079ab2..9602e72 100644 --- a/server/config.development.json +++ b/server/config.development.json @@ -6,7 +6,7 @@ "rateLimiter": { "redisHost": "127.0.0.1", "redisPort": "6379", - "rateLimitThreshold": 300, + "rateLimitThreshold": 301, "rateLimitTLLSeconds": 3 }, "remoting": { From 4b56b31f19a3a30c6dc51661d2b137e03cdc901a Mon Sep 17 00:00:00 2001 From: grizim Date: Wed, 21 Mar 2018 23:16:47 +0200 Subject: [PATCH 3/3] support rateLimiter disable + support config file without rateLimiter. --- server/config.development.json | 1 + server/config.json | 7 +++++++ server/middleware/rate-limiter.js | 23 ++++++++++++----------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/server/config.development.json b/server/config.development.json index 9602e72..1cf01fc 100644 --- a/server/config.development.json +++ b/server/config.development.json @@ -4,6 +4,7 @@ "port": 3000, "cookieSecret": "246bace2-9876-4321-1234-0ae8160b07c8", "rateLimiter": { + "isEnable": false, "redisHost": "127.0.0.1", "redisPort": "6379", "rateLimitThreshold": 301, diff --git a/server/config.json b/server/config.json index 39c8698..8b8862f 100644 --- a/server/config.json +++ b/server/config.json @@ -3,6 +3,13 @@ "host": "0.0.0.0", "port": 3000, "cookieSecret": "cookie-secret", + "rateLimiter": { + "isEnable": false, + "redisHost": "127.0.0.1", + "redisPort": "6379", + "rateLimitThreshold": 301, + "rateLimitTLLSeconds": 3 + }, "remoting": { "context": false, "rest": { diff --git a/server/middleware/rate-limiter.js b/server/middleware/rate-limiter.js index f5d522f..4c81801 100644 --- a/server/middleware/rate-limiter.js +++ b/server/middleware/rate-limiter.js @@ -1,17 +1,18 @@ -var config = require('../config').rateLimiter +const config = require('../config') +const rateLimiterConfig = config.rateLimiter || { isEnable: false } +const { redisHost, redisPort, rateLimitTLLSeconds, rateLimitThreshold, isEnable } = rateLimiterConfig -var redis = require("redis"), - client = redis.createClient({ host: config.redisHost, port: config.redisPort }) +const redis = require("redis"), + client = redis.createClient({ host: redisHost, port: redisPort }) module.exports = function rateLimiterMiddleware(options) { - return function rateLimiter(req, res, next) { - var ip = req.connection.remoteAddress - client.get(ip, function (err, reply) { - - if (!reply) { - client.set(ip, '1', 'EX', config.rateLimitTLLSeconds) + return isEnable ? function rateLimiter(req, res, next) { + const ip = req.connection.remoteAddress; + client.get(ip, function (err, rateCount) { + if (!rateCount) { + client.set(ip, '1', 'EX', rateLimitTLLSeconds); } - if (Number(reply) >= config.rateLimitThreshold) { + if (Number(rateCount) >= rateLimitThreshold) { res.status(500).send('Unauthorized. \n rate exceeded.'); } else { @@ -19,7 +20,7 @@ module.exports = function rateLimiterMiddleware(options) { next(); } }) - }; + } : function rateLimiterDisabled(req, res, next) { next(); }; };