From 6457073597ea202986b07361e07536ef22af5383 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Fri, 7 Oct 2016 13:51:22 -0700 Subject: [PATCH 1/5] Added listener to exit process when db connection fails --- lib/models/mongo/mongoose-control.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/models/mongo/mongoose-control.js b/lib/models/mongo/mongoose-control.js index a44998afc..ce38288fe 100644 --- a/lib/models/mongo/mongoose-control.js +++ b/lib/models/mongo/mongoose-control.js @@ -52,6 +52,19 @@ mongooseControl.start = function (cb) { } mongoose.connect(process.env.MONGO, mongooseOptions, cb) + mongoose.connection.on('disconnected', function () { + if (!mongoose.connection._hasOpened) { + log.fatal({message: 'Failed to connect to ' + process.env.MONGO}, 'failed to establish a connection to mongodb') + process.exit(1) + } else { + log.error({message: 'Lost connection to ' + process.env.MONGO}) + setTimeout(function () { + if (!mongoose.connection.readyState) { + process.exit(1) + } + }, 10000) + } + }) } mongooseControl.stop = function (cb) { From f193650f1c435b60d681d23d542375a76b2b4324 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Sat, 8 Oct 2016 14:07:28 -0700 Subject: [PATCH 2/5] Separated functions, added tests --- lib/models/mongo/mongoose-control.js | 25 +++++++++++++++-------- unit/models/mongo/mongoose-control.js | 29 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/lib/models/mongo/mongoose-control.js b/lib/models/mongo/mongoose-control.js index ce38288fe..a35a8f8cf 100644 --- a/lib/models/mongo/mongoose-control.js +++ b/lib/models/mongo/mongoose-control.js @@ -52,17 +52,12 @@ mongooseControl.start = function (cb) { } mongoose.connect(process.env.MONGO, mongooseOptions, cb) + mongoose.connection.on('disconnected', function () { if (!mongoose.connection._hasOpened) { - log.fatal({message: 'Failed to connect to ' + process.env.MONGO}, 'failed to establish a connection to mongodb') - process.exit(1) + mongooseControl._exitIfFailedToOpen() } else { - log.error({message: 'Lost connection to ' + process.env.MONGO}) - setTimeout(function () { - if (!mongoose.connection.readyState) { - process.exit(1) - } - }, 10000) + mongooseControl._exitIfFailedToReconnect() } }) } @@ -76,3 +71,17 @@ mongooseControl.stop = function (cb) { }) }) } + +mongooseControl._exitIfFailedToOpen = function () { + log.fatal({message: 'Failed to connect to ' + process.env.MONGO}, 'failed to establish a connection to mongodb') + process.exit(1) +} + +mongooseControl._exitIfFailedToReconnect = function() { + log.error({message: 'Lost connection to ' + process.env.MONGO}) + setTimeout(function () { + if (!mongoose.connection.readyState) { + process.exit(1) + } + }, 10000) +} diff --git a/unit/models/mongo/mongoose-control.js b/unit/models/mongo/mongoose-control.js index 3d8b09279..77aec3588 100644 --- a/unit/models/mongo/mongoose-control.js +++ b/unit/models/mongo/mongoose-control.js @@ -103,5 +103,34 @@ describe('mongoose-control', function () { done() }) }) + + describe('handling mongodb disconnect events', function () { + + beforeEach(function (done) { + sinon.stub(mongoose.connection, 'on').yields() + sinon.stub(mongooseControl, '_exitIfFailedToReconnect') + sinon.stub(mongooseControl, '_exitIfFailedToOpen') + done() + }) + + it('should exit if it cannot connect', function (done) { + mongooseControl.start(function (err) { + expect(err).to.not.exist() + sinon.assert.notCalled(mongooseControl._exitIfFailedToReconnect) + sinon.assert.calledOnce(mongooseControl._exitIfFailedToOpen) + done() + }) + }) + + it('should attempt a retry if connection existed', function (done) { + mongoose.connection._hasOpened = true + mongooseControl.start(function (err) { + expect(err).to.not.exist() + sinon.assert.notCalled(mongooseControl._exitIfFailedToOpen) + sinon.assert.calledOnce(mongooseControl._exitIfFailedToReconnect) + done() + }) + }) + }) }) }) From bfa69b93a50cb8ca921b5ce3a5d1b952f270a226 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Mon, 10 Oct 2016 18:05:50 -0700 Subject: [PATCH 3/5] Added more unit tests and fixed linting --- lib/models/mongo/mongoose-control.js | 7 ++-- unit/models/mongo/mongoose-control.js | 54 +++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/lib/models/mongo/mongoose-control.js b/lib/models/mongo/mongoose-control.js index a35a8f8cf..0be7a840a 100644 --- a/lib/models/mongo/mongoose-control.js +++ b/lib/models/mongo/mongoose-control.js @@ -73,14 +73,15 @@ mongooseControl.stop = function (cb) { } mongooseControl._exitIfFailedToOpen = function () { - log.fatal({message: 'Failed to connect to ' + process.env.MONGO}, 'failed to establish a connection to mongodb') + log.fatal({message: 'Failed to connect to ' + process.env.MONGO + ' failed to establish a connection to mongodb'}) process.exit(1) } -mongooseControl._exitIfFailedToReconnect = function() { - log.error({message: 'Lost connection to ' + process.env.MONGO}) +mongooseControl._exitIfFailedToReconnect = function () { + log.error({message: 'Failed to connect to ' + process.env.MONGO + ' failed to establish a connection to mongodb'}) setTimeout(function () { if (!mongoose.connection.readyState) { + log.fatal({message: 'Exiting nodejs process due to mongodb connection failure'}) process.exit(1) } }, 10000) diff --git a/unit/models/mongo/mongoose-control.js b/unit/models/mongo/mongoose-control.js index 77aec3588..1ab158b94 100644 --- a/unit/models/mongo/mongoose-control.js +++ b/unit/models/mongo/mongoose-control.js @@ -11,6 +11,7 @@ var Code = require('code') var fs = require('fs') var mongoose = require('mongoose') var sinon = require('sinon') +var clock var mongooseControl = require('models/mongo/mongoose-control') @@ -105,14 +106,22 @@ describe('mongoose-control', function () { }) describe('handling mongodb disconnect events', function () { - beforeEach(function (done) { sinon.stub(mongoose.connection, 'on').yields() + sinon.stub(process, 'exit') sinon.stub(mongooseControl, '_exitIfFailedToReconnect') sinon.stub(mongooseControl, '_exitIfFailedToOpen') done() }) + afterEach(function (done) { + mongooseControl._exitIfFailedToReconnect.restore() + mongooseControl._exitIfFailedToOpen.restore() + mongoose.connection.on.restore() + process.exit.restore() + done() + }) + it('should exit if it cannot connect', function (done) { mongooseControl.start(function (err) { expect(err).to.not.exist() @@ -124,13 +133,44 @@ describe('mongoose-control', function () { it('should attempt a retry if connection existed', function (done) { mongoose.connection._hasOpened = true - mongooseControl.start(function (err) { - expect(err).to.not.exist() - sinon.assert.notCalled(mongooseControl._exitIfFailedToOpen) - sinon.assert.calledOnce(mongooseControl._exitIfFailedToReconnect) - done() - }) + mongooseControl.start(function (err) { + expect(err).to.not.exist() + sinon.assert.notCalled(mongooseControl._exitIfFailedToOpen) + sinon.assert.calledOnce(mongooseControl._exitIfFailedToReconnect) + done() }) + }) + }) + + describe('exiting node process when db disconnects', function () { + beforeEach(function (done) { + clock = sinon.useFakeTimers() + sinon.stub(mongoose.connection, 'on').yields() + sinon.stub(process, 'exit') + done() + }) + + afterEach(function (done) { + mongoose.connection.on.restore() + process.exit.restore() + clock.restore() + done() + }) + + it('should exit immediately if it cannot connect', function (done) { + mongooseControl._exitIfFailedToOpen() + sinon.assert.calledOnce(process.exit) + done() + }) + + it('should attempt to reconnect when it was connected once', function (done) { + mongooseControl._exitIfFailedToReconnect() + clock.tick(1000) + sinon.assert.notCalled(process.exit) + clock.tick(10000) + sinon.assert.calledOnce(process.exit) + done() + }) }) }) }) From a96af0e769a5c66fa3bb3364e35390d871fafcad Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Tue, 11 Oct 2016 10:23:49 -0700 Subject: [PATCH 4/5] Added env variable to hopefully pass functional tests --- configs/.env.development | 1 + configs/.env.production-delta | 1 + configs/.env.production-epsilon | 1 + configs/.env.production-gamma | 1 + configs/.env.staging | 1 + configs/.env.test | 1 + lib/models/mongo/mongoose-control.js | 2 +- unit/models/mongo/mongoose-control.js | 2 +- 8 files changed, 8 insertions(+), 2 deletions(-) diff --git a/configs/.env.development b/configs/.env.development index 6880f2dbb..437769cce 100644 --- a/configs/.env.development +++ b/configs/.env.development @@ -2,6 +2,7 @@ ASSERT_HTTPS=false AWS_ACCESS_KEY_ID=AKIAJXF6CNCWBWNX7JZQ AWS_SECRET_ACCESS_KEY=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 CONTAINER_STOP_LIMIT=10 +DB_CONNECTION_TIMEOUT=10000 DOCKER_IMAGE_BUILDER_CACHE=/git-cache DOMAIN=localhost:3001 FULL_API_DOMAIN=http://localhost:3030 diff --git a/configs/.env.production-delta b/configs/.env.production-delta index efcfa345e..9fffc2062 100644 --- a/configs/.env.production-delta +++ b/configs/.env.production-delta @@ -1,4 +1,5 @@ CONTAINER_STOP_LIMIT=10 +DB_CONNECTION_TIMEOUT=10000 DOCKER_IMAGE_BUILDER_CACHE=/git-cache DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache DOMAIN=runnable.io diff --git a/configs/.env.production-epsilon b/configs/.env.production-epsilon index 29cc124d8..9094007c6 100644 --- a/configs/.env.production-epsilon +++ b/configs/.env.production-epsilon @@ -1,6 +1,7 @@ ALLOW_ALL_CORS=true BUNYAN_LOG_USE_SRC=true CONTAINER_STOP_LIMIT=10 +DB_CONNECTION_TIMEOUT=10000 DOCKER_IMAGE_BUILDER_CACHE=/git-cache DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache DOMAIN=runnable-beta.com diff --git a/configs/.env.production-gamma b/configs/.env.production-gamma index d75c0caec..6f2d1ff77 100644 --- a/configs/.env.production-gamma +++ b/configs/.env.production-gamma @@ -1,6 +1,7 @@ ALLOW_ALL_CORS=true BUNYAN_LOG_USE_SRC=true CONTAINER_STOP_LIMIT=10 +DB_CONNECTION_TIMEOUT=10000 DOCKER_IMAGE_BUILDER_CACHE=/git-cache DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache DOMAIN=runnable-gamma.com diff --git a/configs/.env.staging b/configs/.env.staging index 0f488dabe..5f5be07cf 100644 --- a/configs/.env.staging +++ b/configs/.env.staging @@ -3,6 +3,7 @@ AWS_ACCESS_KEY_ID=AKIAJXF6CNCWBWNX7JZQ AWS_SECRET_ACCESS_KEY=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 BUNYAN_BATCH_LOG_COUNT=5 DATADOG_HOST=datadog-staging-codenow.runnableapp.com +DB_CONNECTION_TIMEOUT=10000 DEBUG=runnable-api:docker*,api:worker*,hermes* DOCKER_IMAGE_BUILDER_CACHE=/git-cache DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache diff --git a/configs/.env.test b/configs/.env.test index 71449a82f..cf47f95e7 100644 --- a/configs/.env.test +++ b/configs/.env.test @@ -9,6 +9,7 @@ BUNYAN_LOG_USE_SRC=true CONTAINER_STOP_LIMIT=10 COOKIE_SECRET=\$up3r,$3 Date: Tue, 11 Oct 2016 17:18:04 -0700 Subject: [PATCH 5/5] Did the sensible thing --- configs/.env | 1 + configs/.env.development | 1 - configs/.env.production-delta | 1 - configs/.env.production-epsilon | 1 - configs/.env.production-gamma | 1 - configs/.env.staging | 1 - 6 files changed, 1 insertion(+), 5 deletions(-) diff --git a/configs/.env b/configs/.env index 30d3fc2e7..9933e36f8 100644 --- a/configs/.env +++ b/configs/.env @@ -10,6 +10,7 @@ CONTAINER_HARD_MEMORY_LIMIT_BYTES=2048000000 CONTAINER_SOFT_MEMORY_LIMIT_BYTES=128000000 CONTAINER_STOP_LIMIT=10 COOKIE_SECRET=\$up3r,$3