From 2517c623c330697c5dc867c734c8bf1f31c1a8cb Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Wed, 20 Dec 2017 15:06:23 -0800 Subject: [PATCH 01/14] Fixed bug in service route --- package-lock.json | 208 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + server/index.js | 42 ++++++++-- 3 files changed, 243 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index b7585fd..a9e3cb0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,11 @@ } } }, + "after-all-results": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/after-all-results/-/after-all-results-2.0.0.tgz", + "integrity": "sha1-asL8ICtQD4jaj09VMM+hAPTGotA=" + }, "ajv": { "version": "5.5.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.1.tgz", @@ -187,6 +192,14 @@ "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", "integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=" }, + "async-cache": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/async-cache/-/async-cache-1.1.0.tgz", + "integrity": "sha1-SppaidBl7F2OUlS9nulrp2xTK1o=", + "requires": { + "lru-cache": "4.1.1" + } + }, "async-each": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.1.tgz", @@ -1027,6 +1040,11 @@ "xdg-basedir": "3.0.0" } }, + "console-log-level": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.0.tgz", + "integrity": "sha1-QDWBi+6jflhQoMA8jUUMpfLNEhc=" + }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -1258,6 +1276,50 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "elastic-apm-http-client": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/elastic-apm-http-client/-/elastic-apm-http-client-5.1.0.tgz", + "integrity": "sha512-ynOtHIqbYhh6IkFYG5IbxEM+wrA8pUN5dPNGKQFuFQ9bIIU5kzV7Vl6jd/w+YECUwfKwAnUdE8fI1D+WtQ+E6A==", + "requires": { + "fast-safe-stringify": "1.2.1" + } + }, + "elastic-apm-node": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/elastic-apm-node/-/elastic-apm-node-0.9.0.tgz", + "integrity": "sha512-KygBBQW9Z504usrORo153LacN93hRr/GxqWEgEmF25glhUlgbMNMRfEBD0b3i76SEm/1xsMcwrZnf9SKITfv6g==", + "requires": { + "after-all-results": "2.0.0", + "console-log-level": "1.4.0", + "cookie": "0.3.1", + "core-util-is": "1.0.2", + "debug": "3.1.0", + "elastic-apm-http-client": "5.1.0", + "end-of-stream": "1.4.0", + "fast-safe-stringify": "1.2.1", + "http-headers": "3.0.2", + "is-native": "1.0.1", + "normalize-bool": "1.0.0", + "object-assign": "4.1.1", + "redact-secrets": "1.0.0", + "require-in-the-middle": "2.1.2", + "semver": "5.4.1", + "sql-summary": "1.0.0", + "stackman": "2.0.1", + "unicode-byte-truncate": "1.0.0", + "uuid": "3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + } + } + }, "emoji-regex": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-6.5.1.tgz", @@ -1281,6 +1343,19 @@ "iconv-lite": "0.4.19" } }, + "end-of-stream": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.0.tgz", + "integrity": "sha1-epDYM+/abPpurA9JSduw+tOmMgY=", + "requires": { + "once": "1.4.0" + } + }, + "error-callsites": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/error-callsites/-/error-callsites-1.0.1.tgz", + "integrity": "sha1-QoYWmt+PwSSC9VYRFyTFrthzppI=" + }, "error-ex": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz", @@ -1745,6 +1820,11 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, + "fast-safe-stringify": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-1.2.1.tgz", + "integrity": "sha512-g2UqeO0yyYjTSpiH4zJQk+IycRxyYRABjSf+TpmeMOn9uByzFIoX0y/HnweCFhKb+uuPwjIvqXuK/LTteEBhow==" + }, "fbjs": { "version": "0.8.16", "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.16.tgz", @@ -2963,6 +3043,14 @@ } } }, + "http-headers": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/http-headers/-/http-headers-3.0.2.tgz", + "integrity": "sha512-87E1I+2Wg4dxxz4rcxElo3dxO/w1ZtgL1yA0Sb6vH3qU16vRKq1NjWQv9SCY3ly2OQROcoxHZOUpmelS+k6wOw==", + "requires": { + "next-line": "1.1.0" + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -2998,6 +3086,11 @@ "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=" }, + "in-publish": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/in-publish/-/in-publish-2.0.0.tgz", + "integrity": "sha1-4g/146KvwmkDILbcVSaCqcf631E=" + }, "inflection": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz", @@ -3200,6 +3293,28 @@ "is-path-inside": "1.0.1" } }, + "is-integer": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-integer/-/is-integer-1.0.7.tgz", + "integrity": "sha1-a96Bqs3feLZZtmKdYpytxRqIbVw=", + "requires": { + "is-finite": "1.0.2" + } + }, + "is-native": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-native/-/is-native-1.0.1.tgz", + "integrity": "sha1-zRjMFi6EUNaDtbq+eayZwUVElnU=", + "requires": { + "is-nil": "1.0.1", + "to-source-code": "1.0.2" + } + }, + "is-nil": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-nil/-/is-nil-1.0.1.tgz", + "integrity": "sha1-LauingtYUGOHXntTnQcfWxWTeWk=" + }, "is-npm": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", @@ -3303,6 +3418,11 @@ "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz", "integrity": "sha1-EaBgVotnM5REAz0BJaYaINVk+zQ=" }, + "is-secret": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-secret/-/is-secret-1.1.1.tgz", + "integrity": "sha1-KYig6bOU41YM1IBAbWHKz9dPH/k=" + }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -3569,6 +3689,16 @@ } } }, + "load-source-map": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/load-source-map/-/load-source-map-1.0.0.tgz", + "integrity": "sha1-MY9JkFzopwnft8w/FvPv47zx3QU=", + "requires": { + "in-publish": "2.0.0", + "semver": "5.4.1", + "source-map": "0.5.7" + } + }, "loader-utils": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.1.0.tgz", @@ -3815,6 +3945,11 @@ "minimist": "0.0.8" } }, + "module-details-from-path": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", + "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" + }, "moment": { "version": "2.19.4", "resolved": "https://registry.npmjs.org/moment/-/moment-2.19.4.tgz", @@ -3846,6 +3981,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, + "next-line": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", + "integrity": "sha1-/K5XhTBStqm66CCOQN19PC0wRgM=" + }, "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", @@ -3880,6 +4020,11 @@ "abbrev": "1.1.1" } }, + "normalize-bool": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/normalize-bool/-/normalize-bool-1.0.0.tgz", + "integrity": "sha1-RqVx7ZPqWrM3IfrM/FpZuGiQ2Fg=" + }, "normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -7950,6 +8095,15 @@ "resolve": "1.5.0" } }, + "redact-secrets": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/redact-secrets/-/redact-secrets-1.0.0.tgz", + "integrity": "sha1-YPHbVpJP6QogO6jMs5KDzbsNkHw=", + "requires": { + "is-secret": "1.1.1", + "traverse": "0.6.6" + } + }, "regenerate": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.3.3.tgz", @@ -8084,6 +8238,15 @@ } } }, + "require-in-the-middle": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-2.1.2.tgz", + "integrity": "sha1-vduJMW1FvNsI4sYYa9Lm6Bmo7q4=", + "requires": { + "module-details-from-path": "1.0.3", + "resolve": "1.5.0" + } + }, "require-uncached": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", @@ -8300,6 +8463,11 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, + "sql-summary": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/sql-summary/-/sql-summary-1.0.0.tgz", + "integrity": "sha1-OeOlHY2F5Gc5g2/H1n0GVLFzo58=" + }, "sshpk": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", @@ -8315,6 +8483,19 @@ "tweetnacl": "0.14.5" } }, + "stackman": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/stackman/-/stackman-2.0.1.tgz", + "integrity": "sha1-ztMJxmLpubZn79cYOxrjDFF8uqM=", + "requires": { + "after-all-results": "2.0.0", + "async-cache": "1.1.0", + "debug": "2.6.9", + "error-callsites": "1.0.1", + "load-source-map": "1.0.0", + "path-is-absolute": "1.0.1" + } + }, "statuses": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", @@ -8519,6 +8700,14 @@ "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc=" }, + "to-source-code": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/to-source-code/-/to-source-code-1.0.2.tgz", + "integrity": "sha1-3RNr2x4dvYC76s8IiZJnjpBwv+o=", + "requires": { + "is-nil": "1.0.1" + } + }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", @@ -8535,6 +8724,11 @@ "punycode": "1.4.1" } }, + "traverse": { + "version": "0.6.6", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz", + "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=" + }, "trim-right": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", @@ -8601,6 +8795,20 @@ "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-0.0.3.tgz", "integrity": "sha1-7Mo6A+VrmvFzhbqsgSrIO5lKli8=" }, + "unicode-byte-truncate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unicode-byte-truncate/-/unicode-byte-truncate-1.0.0.tgz", + "integrity": "sha1-qm8PNHUZP+IMMgrJIT425i6HZKc=", + "requires": { + "is-integer": "1.0.7", + "unicode-substring": "0.1.0" + } + }, + "unicode-substring": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/unicode-substring/-/unicode-substring-0.1.0.tgz", + "integrity": "sha1-YSDOPDkDhdvND2DDK5BlxBgdSzY=" + }, "unique-string": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", diff --git a/package.json b/package.json index a1378c8..ba4d7a6 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "babel-preset-es2015": "^6.24.1", "bookshelf": "^0.11.1", "chai": "^4.1.2", + "elastic-apm-node": "^0.9.0", "eslint": "^4.11.0", "eslint-config-airbnb": "^16.1.0", "eslint-plugin-import": "^2.8.0", diff --git a/server/index.js b/server/index.js index 8f76b45..4d0904d 100644 --- a/server/index.js +++ b/server/index.js @@ -8,6 +8,12 @@ if (cluster.isMaster) { } } else { +// var apm = require('elastic-apm-node').start({ +// appName: 'youtube', +// // Use if APM Server requires a token +// // Set custom APM Server URL (default: http://localhost:8200) +// serverUrl: 'http://localhost:8200' +// }); const express = require('express'); const bodyParser = require('body-parser'); const PORT = process.env.PORT || 3000; @@ -23,25 +29,27 @@ if (cluster.isMaster) { app.use(bodyParser.urlencoded({ extended: false })); app.patch('/service', (req, res) => { - let type; let targetVideo = req.body.videoId; return new Promise((resolve, reject) => { resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) .then((success) => { - knex('videos').where({ video_id: targetVideo }).first() + knex('videos').where({ video_id: targetVideo }) .then((video) => { - if (!video.ad && video.view_count === 65200) { + console.log('has an ad', !video[0].ad); + if (!video[0].ad && video[0].view_count === 339510) { + console.log('made it to the if'); let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) - .then((ad_Id) => { - knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad_Id.rows[0].ad_id }) + .then((ad) => { + knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) .then((testVideo) => { - res.status(200).send({ 'video': video, 'ad_Id': ad_Id.rows[0].ad_id, 'messsage': 'added an Ad' }); + res.status(200).send(video); }); }); } else { - res.status(200).send({ 'video': video, 'messsage': 'updated count only' }); + console.log('did not add an ad'); + res.status(200).send(video); } }); }) @@ -50,7 +58,25 @@ if (cluster.isMaster) { }); }); }); - + + app.post('/count', (req, res) => { + let targetVideo = req.body.videoId; + + return new Promise((resolve, reject) => { + resolve(knex('videos').where({ video_id: targetVideo }) + .then((video) => { + res.status(200).send(video); + }) + ).catch((err) => { + console.log(err); + }); + }); + }); + + app.get('/kue', (req, res) => { + res.status(200).end(); + }); + if (!module.parent) { // only listen to port if existing port is not in use app.listen(PORT, () => { console.log('Listening on port', PORT); From 0e64e85bcb517fc08265fb7d8207057546ccdedd Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Wed, 20 Dec 2017 16:30:35 -0800 Subject: [PATCH 02/14] Reddis is set up --- db/reddis_testing.js | 7 + db/redis.js | 5 + package-lock.json | 802 +++++++++++++++++++++++++++++++++++++++++++ package.json | 3 + redisfile.js | 10 + server/index.js | 30 +- 6 files changed, 849 insertions(+), 8 deletions(-) create mode 100644 db/reddis_testing.js create mode 100644 db/redis.js create mode 100644 redisfile.js diff --git a/db/reddis_testing.js b/db/reddis_testing.js new file mode 100644 index 0000000..6337d5f --- /dev/null +++ b/db/reddis_testing.js @@ -0,0 +1,7 @@ +// var redis = require('redis'); +// var redisClient = require('./redis.js'); + +// redisClient.set("language", "nodejs", function (err, reply) { +// console.log(err); +// console.log(reply); +// }); \ No newline at end of file diff --git a/db/redis.js b/db/redis.js new file mode 100644 index 0000000..113be9b --- /dev/null +++ b/db/redis.js @@ -0,0 +1,5 @@ +var environment = process.env.NODE_ENV || 'development'; +var config = require('../redisfile.js')[environment.connection]; +var redis = require('redis'); + +module.exports = redis.createClient(config); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index a9e3cb0..f9b4339 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,21 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.2.1.tgz", "integrity": "sha512-jG0u7c4Ly+3QkkW18V+NRDN+4bWHdln30NL1ZL2AvFZZmQe/BfopYCtghCKKVBUSetZ4QKcyA0pY6/4Gw8Pv8w==" }, + "acorn-globals": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-3.1.0.tgz", + "integrity": "sha1-/YJw9x+7SZawBPqIDuXUZXOnMb8=", + "requires": { + "acorn": "4.0.13" + }, + "dependencies": { + "acorn": { + "version": "4.0.13", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz", + "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c=" + } + } + }, "acorn-jsx": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz", @@ -59,6 +74,21 @@ "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-2.1.1.tgz", "integrity": "sha1-YXmX/F9gV2iUxDX5QNgZ4TW4B2I=" }, + "align-text": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", + "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", + "requires": { + "kind-of": "3.2.2", + "longest": "1.0.1", + "repeat-string": "1.6.1" + } + }, + "amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=" + }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -91,6 +121,15 @@ "normalize-path": "2.1.1" } }, + "apparatus": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.9.tgz", + "integrity": "sha1-N9zSWDStC2UQdllikduCPusZCL0=", + "optional": true, + "requires": { + "sylvester": "0.0.21" + } + }, "argparse": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz", @@ -897,6 +936,15 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "center-align": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", + "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=", + "requires": { + "align-text": "0.1.4", + "lazy-cache": "1.0.4" + } + }, "chai": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", @@ -922,6 +970,14 @@ "supports-color": "2.0.0" } }, + "character-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/character-parser/-/character-parser-2.2.0.tgz", + "integrity": "sha1-x84o821LzZdE5f/CxfzeHHMmH8A=", + "requires": { + "is-regex": "1.0.4" + } + }, "chardet": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz", @@ -953,6 +1009,33 @@ "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==" }, + "clean-css": { + "version": "3.4.28", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-3.4.28.tgz", + "integrity": "sha1-vxlF6C/ICPVWlebd6uwBQA79A/8=", + "requires": { + "commander": "2.8.1", + "source-map": "0.4.4" + }, + "dependencies": { + "commander": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", + "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=", + "requires": { + "graceful-readlink": "1.0.1" + } + }, + "source-map": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", + "requires": { + "amdefine": "1.0.1" + } + } + } + }, "cli-boxes": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", @@ -971,11 +1054,33 @@ "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=" }, + "cliui": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", + "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", + "requires": { + "center-align": "0.1.3", + "right-align": "0.1.3", + "wordwrap": "0.0.2" + }, + "dependencies": { + "wordwrap": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", + "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=" + } + } + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" + }, "color-convert": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz", @@ -1045,6 +1150,22 @@ "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.0.tgz", "integrity": "sha1-QDWBi+6jflhQoMA8jUUMpfLNEhc=" }, + "constantinople": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/constantinople/-/constantinople-3.1.0.tgz", + "integrity": "sha1-dWnKqKo/jVk11i4fqW+fcCzYHHk=", + "requires": { + "acorn": "3.3.0", + "is-expression": "2.1.0" + }, + "dependencies": { + "acorn": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", + "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=" + } + } + }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -1136,6 +1257,11 @@ "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=" }, + "css-parse": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/css-parse/-/css-parse-1.7.0.tgz", + "integrity": "sha1-Mh9s9zeCpv91ERE5D8BeLGV9jJs=" + }, "damerau-levenshtein": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.4.tgz", @@ -1157,6 +1283,11 @@ "ms": "2.0.0" } }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + }, "deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", @@ -1244,6 +1375,11 @@ "esutils": "2.0.2" } }, + "doctypes": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/doctypes/-/doctypes-1.1.0.tgz", + "integrity": "sha1-6oCxBqh1OHdOijpKWv4pPeSJ4Kk=" + }, "dot-prop": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", @@ -1252,6 +1388,11 @@ "is-obj": "1.0.1" } }, + "double-ended-queue": { + "version": "2.1.0-0", + "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", + "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" + }, "duplexer": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", @@ -2828,6 +2969,11 @@ "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.2.tgz", "integrity": "sha1-iGvFvwvrfblugby7oHiBjeWmJoM=" }, + "get-caller-file": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz", + "integrity": "sha1-9wLmMSfn4jHBYKgMFVSstw1QR+U=" + }, "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", @@ -2952,6 +3098,11 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=" }, + "graceful-readlink": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", + "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=" + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -3190,6 +3341,11 @@ "loose-envify": "1.3.1" } }, + "invert-kv": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", + "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" + }, "ipaddr.js": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.5.2.tgz", @@ -3253,6 +3409,22 @@ "is-primitive": "2.0.0" } }, + "is-expression": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-expression/-/is-expression-2.1.0.tgz", + "integrity": "sha1-kb6dR968/vB3l36XIr5tz7RGXvA=", + "requires": { + "acorn": "3.3.0", + "object-assign": "4.1.1" + }, + "dependencies": { + "acorn": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", + "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=" + } + } + }, "is-extendable": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", @@ -3446,6 +3618,11 @@ "unc-path-regex": "0.1.2" } }, + "is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=" + }, "is-windows": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-0.2.0.tgz", @@ -3488,6 +3665,11 @@ "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", "integrity": "sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8=" }, + "js-stringify": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/js-stringify/-/js-stringify-1.0.2.tgz", + "integrity": "sha1-Fzb939lyTyijaCrcYjCufk6Weds=" + }, "js-tokens": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", @@ -3549,6 +3731,15 @@ "verror": "1.10.0" } }, + "jstransformer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/jstransformer/-/jstransformer-1.0.0.tgz", + "integrity": "sha1-7Yvwkh4vPx7U1cGkT2hwntJHIsM=", + "requires": { + "is-promise": "2.1.0", + "promise": "7.3.1" + } + }, "jsx-ast-utils": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-1.4.1.tgz", @@ -3638,6 +3829,35 @@ } } }, + "kue": { + "version": "0.11.6", + "resolved": "https://registry.npmjs.org/kue/-/kue-0.11.6.tgz", + "integrity": "sha512-56Jic22qSqdJ3nNpkhVr6RUx/QKalfdBdU0m70hgBKEkhBAgdt6Qr74evec+bM+LGmNbEC6zGGDskX4mcgBYcQ==", + "requires": { + "body-parser": "1.18.2", + "express": "4.16.2", + "lodash": "4.17.4", + "nib": "1.1.2", + "node-redis-warlock": "0.2.0", + "pug": "2.0.0-rc.4", + "redis": "2.6.5", + "reds": "0.2.5", + "stylus": "0.54.5", + "yargs": "4.8.1" + }, + "dependencies": { + "redis": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/redis/-/redis-2.6.5.tgz", + "integrity": "sha1-h8Hv9KSJ+Utwhx89CLaYjyOpVoc=", + "requires": { + "double-ended-queue": "2.1.0-0", + "redis-commands": "1.3.1", + "redis-parser": "2.6.0" + } + } + } + }, "latest-version": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", @@ -3646,6 +3866,19 @@ "package-json": "4.0.1" } }, + "lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=" + }, + "lcid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", + "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", + "requires": { + "invert-kv": "1.0.0" + } + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -3826,6 +4059,11 @@ "resolved": "https://registry.npmjs.org/lodash.restparam/-/lodash.restparam-3.6.1.tgz", "integrity": "sha1-k2pOMJ7zMKdkXtQUWYbIWuWyCAU=" }, + "longest": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", + "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=" + }, "loose-envify": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.3.1.tgz", @@ -3971,6 +4209,17 @@ "integrity": "sha1-7XFfP+neArV6XmJS2QqWZ14fCFo=", "optional": true }, + "natural": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/natural/-/natural-0.2.1.tgz", + "integrity": "sha1-HrUVap2QtFkZSeIOlOvHe7Iznto=", + "optional": true, + "requires": { + "apparatus": "0.0.9", + "sylvester": "0.0.21", + "underscore": "1.8.3" + } + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3986,6 +4235,14 @@ "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", "integrity": "sha1-/K5XhTBStqm66CCOQN19PC0wRgM=" }, + "nib": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/nib/-/nib-1.1.2.tgz", + "integrity": "sha1-amnt5AgblcDe+L4CSkyK4MLLtsc=", + "requires": { + "stylus": "0.54.5" + } + }, "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", @@ -3995,6 +4252,43 @@ "is-stream": "1.1.0" } }, + "node-redis-scripty": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/node-redis-scripty/-/node-redis-scripty-0.0.5.tgz", + "integrity": "sha1-S/LTZattqyAswIt6xj+PVarcliU=", + "requires": { + "extend": "1.3.0", + "lru-cache": "2.7.3" + }, + "dependencies": { + "extend": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-1.3.0.tgz", + "integrity": "sha1-0VFvsP9WJNLr+RI+odrFoZlABPg=" + }, + "lru-cache": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz", + "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=" + } + } + }, + "node-redis-warlock": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/node-redis-warlock/-/node-redis-warlock-0.2.0.tgz", + "integrity": "sha1-VjlbmUyCjo4y9qrlO5O27fzZeZA=", + "requires": { + "node-redis-scripty": "0.0.5", + "uuid": "2.0.3" + }, + "dependencies": { + "uuid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" + } + } + }, "nodemon": { "version": "1.12.5", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.12.5.tgz", @@ -7632,6 +7926,14 @@ "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" }, + "os-locale": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", + "requires": { + "lcid": "1.0.0" + } + }, "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -7962,6 +8264,136 @@ "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, + "pug": { + "version": "2.0.0-rc.4", + "resolved": "https://registry.npmjs.org/pug/-/pug-2.0.0-rc.4.tgz", + "integrity": "sha512-SL7xovj6E2Loq9N0UgV6ynjMLW4urTFY/L/Fprhvz13Xc5vjzkjZjI1QHKq31200+6PSD8PyU6MqrtCTJj6/XA==", + "requires": { + "pug-code-gen": "2.0.0", + "pug-filters": "2.1.5", + "pug-lexer": "3.1.0", + "pug-linker": "3.0.3", + "pug-load": "2.0.9", + "pug-parser": "4.0.0", + "pug-runtime": "2.0.3", + "pug-strip-comments": "1.0.2" + } + }, + "pug-attrs": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/pug-attrs/-/pug-attrs-2.0.2.tgz", + "integrity": "sha1-i+KyIlVo/6ddG4Zpgr/59BEa/8s=", + "requires": { + "constantinople": "3.1.0", + "js-stringify": "1.0.2", + "pug-runtime": "2.0.3" + } + }, + "pug-code-gen": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pug-code-gen/-/pug-code-gen-2.0.0.tgz", + "integrity": "sha512-E4oiJT+Jn5tyEIloj8dIJQognbiNNp0i0cAJmYtQTFS0soJ917nlIuFtqVss3IXMEyQKUew3F4gIkBpn18KbVg==", + "requires": { + "constantinople": "3.1.0", + "doctypes": "1.1.0", + "js-stringify": "1.0.2", + "pug-attrs": "2.0.2", + "pug-error": "1.3.2", + "pug-runtime": "2.0.3", + "void-elements": "2.0.1", + "with": "5.1.1" + } + }, + "pug-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/pug-error/-/pug-error-1.3.2.tgz", + "integrity": "sha1-U659nSm7A89WRJOgJhCfVMR/XyY=" + }, + "pug-filters": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/pug-filters/-/pug-filters-2.1.5.tgz", + "integrity": "sha512-xkw71KtrC4sxleKiq+cUlQzsiLn8pM5+vCgkChW2E6oNOzaqTSIBKIQ5cl4oheuDzvJYCTSYzRaVinMUrV4YLQ==", + "requires": { + "clean-css": "3.4.28", + "constantinople": "3.1.0", + "jstransformer": "1.0.0", + "pug-error": "1.3.2", + "pug-walk": "1.1.5", + "resolve": "1.5.0", + "uglify-js": "2.8.29" + } + }, + "pug-lexer": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pug-lexer/-/pug-lexer-3.1.0.tgz", + "integrity": "sha1-/QhzdtSmdbT1n4/vQiiDQ06VgaI=", + "requires": { + "character-parser": "2.2.0", + "is-expression": "3.0.0", + "pug-error": "1.3.2" + }, + "dependencies": { + "acorn": { + "version": "4.0.13", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz", + "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c=" + }, + "is-expression": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-expression/-/is-expression-3.0.0.tgz", + "integrity": "sha1-Oayqa+f9HzRx3ELHQW5hwkMXrJ8=", + "requires": { + "acorn": "4.0.13", + "object-assign": "4.1.1" + } + } + } + }, + "pug-linker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pug-linker/-/pug-linker-3.0.3.tgz", + "integrity": "sha512-DCKczglCXOzJ1lr4xUj/lVHYvS+lGmR2+KTCjZjtIpdwaN7lNOoX2SW6KFX5X4ElvW+6ThwB+acSUg08UJFN5A==", + "requires": { + "pug-error": "1.3.2", + "pug-walk": "1.1.5" + } + }, + "pug-load": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/pug-load/-/pug-load-2.0.9.tgz", + "integrity": "sha512-BDdZOCru4mg+1MiZwRQZh25+NTRo/R6/qArrdWIf308rHtWA5N9kpoUskRe4H6FslaQujC+DigH9LqlBA4gf6Q==", + "requires": { + "object-assign": "4.1.1", + "pug-walk": "1.1.5" + } + }, + "pug-parser": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pug-parser/-/pug-parser-4.0.0.tgz", + "integrity": "sha512-ocEUFPdLG9awwFj0sqi1uiZLNvfoodCMULZzkRqILryIWc/UUlDlxqrKhKjAIIGPX/1SNsvxy63+ayEGocGhQg==", + "requires": { + "pug-error": "1.3.2", + "token-stream": "0.0.1" + } + }, + "pug-runtime": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pug-runtime/-/pug-runtime-2.0.3.tgz", + "integrity": "sha1-mBYmB7D86eJU1CfzOYelrucWi9o=" + }, + "pug-strip-comments": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pug-strip-comments/-/pug-strip-comments-1.0.2.tgz", + "integrity": "sha1-0xOvoBvMN0mA4TmeI+vy65vchRM=", + "requires": { + "pug-error": "1.3.2" + } + }, + "pug-walk": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/pug-walk/-/pug-walk-1.1.5.tgz", + "integrity": "sha512-rJlH1lXerCIAtImXBze3dtKq/ykZMA4rpO9FnPcIgsWcxZLOvd8zltaoeOVFyBSSqCkhhJWbEbTMga8UxWUUSA==" + }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -8104,6 +8536,44 @@ "traverse": "0.6.6" } }, + "redis": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", + "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", + "requires": { + "double-ended-queue": "2.1.0-0", + "redis-commands": "1.3.1", + "redis-parser": "2.6.0" + } + }, + "redis-commands": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.1.tgz", + "integrity": "sha1-gdgm9F+pyLIBH0zXoP5ZfSQdRCs=" + }, + "redis-parser": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", + "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" + }, + "reds": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/reds/-/reds-0.2.5.tgz", + "integrity": "sha1-OKdn92Y810kDaEhpfYLHT9KbwB8=", + "optional": true, + "requires": { + "natural": "0.2.1", + "redis": "0.12.1" + }, + "dependencies": { + "redis": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz", + "integrity": "sha1-ZN92rQ/IrOuuvSoGReikj6xJGF4=", + "optional": true + } + } + }, "regenerate": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.3.3.tgz", @@ -8238,6 +8708,11 @@ } } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, "require-in-the-middle": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-2.1.2.tgz", @@ -8247,6 +8722,11 @@ "resolve": "1.5.0" } }, + "require-main-filename": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", + "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" + }, "require-uncached": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", @@ -8287,6 +8767,14 @@ "signal-exit": "3.0.2" } }, + "right-align": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", + "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=", + "requires": { + "align-text": "0.1.4" + } + }, "rimraf": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", @@ -8321,6 +8809,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, + "sax": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/sax/-/sax-0.5.8.tgz", + "integrity": "sha1-1HLbIo6zMcJQaw6MFVJK25OdEsE=" + }, "semver": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.4.1.tgz", @@ -8365,6 +8858,11 @@ "send": "0.16.1" } }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, "set-immediate-shim": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", @@ -8569,6 +9067,42 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" }, + "stylus": { + "version": "0.54.5", + "resolved": "https://registry.npmjs.org/stylus/-/stylus-0.54.5.tgz", + "integrity": "sha1-QrlWCTHKcJDOhRWnmLqeaqPW3Hk=", + "requires": { + "css-parse": "1.7.0", + "debug": "2.6.9", + "glob": "7.0.6", + "mkdirp": "0.5.1", + "sax": "0.5.8", + "source-map": "0.1.43" + }, + "dependencies": { + "glob": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.0.6.tgz", + "integrity": "sha1-IRuvr0nlJbjNkyYNFKsTYVKz9Xo=", + "requires": { + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" + } + }, + "source-map": { + "version": "0.1.43", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", + "integrity": "sha1-wkvBRspRfBRx9drL4lcbK3+eM0Y=", + "requires": { + "amdefine": "1.0.1" + } + } + } + }, "superagent": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.2.tgz", @@ -8615,6 +9149,11 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, + "sylvester": { + "version": "0.0.21", + "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.21.tgz", + "integrity": "sha1-KYexzivS84sNzio0OIiEv6RADqc=" + }, "table": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/table/-/table-4.0.2.tgz", @@ -8708,6 +9247,11 @@ "is-nil": "1.0.1" } }, + "token-stream": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/token-stream/-/token-stream-0.0.1.tgz", + "integrity": "sha1-zu78cXp2xDFvEm0LnbqlXX598Bo=" + }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", @@ -8785,6 +9329,40 @@ "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.17.tgz", "integrity": "sha512-uRdSdu1oA1rncCQL7sCj8vSyZkgtL7faaw9Tc9rZ3mGgraQ7+Pdx7w5mnOSF3gw9ZNG6oc+KXfkon3bKuROm0g==" }, + "uglify-js": { + "version": "2.8.29", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", + "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=", + "requires": { + "source-map": "0.5.7", + "uglify-to-browserify": "1.0.2", + "yargs": "3.10.0" + }, + "dependencies": { + "camelcase": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", + "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=" + }, + "yargs": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", + "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", + "requires": { + "camelcase": "1.2.1", + "cliui": "2.1.0", + "decamelize": "1.2.0", + "window-size": "0.1.0" + } + } + } + }, + "uglify-to-browserify": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz", + "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=", + "optional": true + }, "unc-path-regex": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", @@ -8795,6 +9373,12 @@ "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-0.0.3.tgz", "integrity": "sha1-7Mo6A+VrmvFzhbqsgSrIO5lKli8=" }, + "underscore": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", + "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=", + "optional": true + }, "unicode-byte-truncate": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unicode-byte-truncate/-/unicode-byte-truncate-1.0.0.tgz", @@ -8926,6 +9510,11 @@ "extsprintf": "1.3.0" } }, + "void-elements": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/void-elements/-/void-elements-2.0.1.tgz", + "integrity": "sha1-wGavtYK7HLQSjWDqkjkulNXp2+w=" + }, "whatwg-fetch": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz", @@ -8939,6 +9528,11 @@ "isexe": "2.0.0" } }, + "which-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz", + "integrity": "sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8=" + }, "widest-line": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.0.tgz", @@ -8947,11 +9541,61 @@ "string-width": "2.1.1" } }, + "window-size": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz", + "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=" + }, + "with": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/with/-/with-5.1.1.tgz", + "integrity": "sha1-+k2qktrzLE6pTtRTyB8EaGtXXf4=", + "requires": { + "acorn": "3.3.0", + "acorn-globals": "3.1.0" + }, + "dependencies": { + "acorn": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", + "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=" + } + } + }, "wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" }, + "wrap-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "requires": { + "string-width": "1.0.2", + "strip-ansi": "3.0.1" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "1.0.1" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "1.1.0", + "is-fullwidth-code-point": "1.0.0", + "strip-ansi": "3.0.1" + } + } + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -8985,10 +9629,168 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" }, + "y18n": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", + "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=" + }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + }, + "yargs": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-4.8.1.tgz", + "integrity": "sha1-wMQpJMpKqmsObaFznfshZDn53cA=", + "requires": { + "cliui": "3.2.0", + "decamelize": "1.2.0", + "get-caller-file": "1.0.2", + "lodash.assign": "4.2.0", + "os-locale": "1.4.0", + "read-pkg-up": "1.0.1", + "require-directory": "2.1.1", + "require-main-filename": "1.0.1", + "set-blocking": "2.0.0", + "string-width": "1.0.2", + "which-module": "1.0.0", + "window-size": "0.2.0", + "y18n": "3.2.1", + "yargs-parser": "2.4.1" + }, + "dependencies": { + "cliui": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", + "requires": { + "string-width": "1.0.2", + "strip-ansi": "3.0.1", + "wrap-ansi": "2.1.0" + } + }, + "find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "requires": { + "path-exists": "2.1.0", + "pinkie-promise": "2.0.1" + } + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "1.0.1" + } + }, + "load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "requires": { + "graceful-fs": "4.1.11", + "parse-json": "2.2.0", + "pify": "2.3.0", + "pinkie-promise": "2.0.1", + "strip-bom": "2.0.0" + } + }, + "lodash.assign": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", + "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=" + }, + "path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "requires": { + "pinkie-promise": "2.0.1" + } + }, + "path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "requires": { + "graceful-fs": "4.1.11", + "pify": "2.3.0", + "pinkie-promise": "2.0.1" + } + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" + }, + "read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "requires": { + "load-json-file": "1.1.0", + "normalize-package-data": "2.4.0", + "path-type": "1.1.0" + } + }, + "read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "requires": { + "find-up": "1.1.2", + "read-pkg": "1.1.0" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "1.1.0", + "is-fullwidth-code-point": "1.0.0", + "strip-ansi": "3.0.1" + } + }, + "strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "requires": { + "is-utf8": "0.2.1" + } + }, + "window-size": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz", + "integrity": "sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU=" + } + } + }, + "yargs-parser": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-2.4.1.tgz", + "integrity": "sha1-hVaN488VD/SfpRgl8DqMiA3cxcQ=", + "requires": { + "camelcase": "3.0.0", + "lodash.assign": "4.2.0" + }, + "dependencies": { + "camelcase": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz", + "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=" + }, + "lodash.assign": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", + "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=" + } + } } } } diff --git a/package.json b/package.json index ba4d7a6..8766103 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "babel-core": "^6.26.0", "babel-loader": "^7.1.2", "babel-preset-es2015": "^6.24.1", + "bluebird": "^3.5.1", "bookshelf": "^0.11.1", "chai": "^4.1.2", "elastic-apm-node": "^0.9.0", @@ -40,10 +41,12 @@ "express": "^4.16.2", "faker": "^4.1.0", "knex": "^0.14.2", + "kue": "^0.11.6", "moment": "^2.19.4", "nodemon": "^1.12.5", "npm": "^5.5.1", "pg": "^4.4.0", + "redis": "^2.8.0", "request": "^2.83.0", "supertest": "^3.0.0" }, diff --git a/redisfile.js b/redisfile.js new file mode 100644 index 0000000..5c0a980 --- /dev/null +++ b/redisfile.js @@ -0,0 +1,10 @@ +module.exports = { + development: { + client: '', + connection: { + host: 'localhost', + port: '6379', //needed +// database: 'youtube', + }, + }, +}; \ No newline at end of file diff --git a/server/index.js b/server/index.js index 4d0904d..63da62a 100644 --- a/server/index.js +++ b/server/index.js @@ -8,19 +8,36 @@ if (cluster.isMaster) { } } else { +// =============== ELASTIC SEARCH + KIBANA ======================================================= // var apm = require('elastic-apm-node').start({ // appName: 'youtube', // // Use if APM Server requires a token // // Set custom APM Server URL (default: http://localhost:8200) // serverUrl: 'http://localhost:8200' // }); +// ============================================================================================================ + const express = require('express'); const bodyParser = require('body-parser'); const PORT = process.env.PORT || 3000; var knex = require('../db/knex.js'); - //var bookshelf = require('bookshelf')(knex); - //var Video = require('./models/videos.js'); +//===================== REDIS ============================ + var redis = require('redis'); + var redisClient = require('../db/redis.js'); + + redisClient.on('ready', function () { + console.log("Redis is ready"); + }); + redisClient.on('error', function () { + console.log("Error in Redis"); + }); + + redisClient.set("language", "nodejs", function (err, reply) { + console.log(err); + console.log(reply); + }); +//================================================= const request = require('request'); const Promise = require('bluebird'); var app = express(); @@ -36,25 +53,22 @@ if (cluster.isMaster) { .then((success) => { knex('videos').where({ video_id: targetVideo }) .then((video) => { - console.log('has an ad', !video[0].ad); if (!video[0].ad && video[0].view_count === 339510) { - console.log('made it to the if'); let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) .then((ad) => { knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) .then((testVideo) => { - res.status(200).send(video); + res.status(204).end(); }); }); } else { - console.log('did not add an ad'); - res.status(200).send(video); + res.status(204).end(); } }); }) ).catch((err) => { - console.log(err); + throw err; }); }); }); From e8c949e934925d7500270dd47b3b7bf6921c2a8c Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Wed, 20 Dec 2017 19:20:26 -0800 Subject: [PATCH 03/14] reddis queue is accepting jobs. - current queue.process log's the job --- server/index.js | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/server/index.js b/server/index.js index 63da62a..ec7b146 100644 --- a/server/index.js +++ b/server/index.js @@ -24,6 +24,8 @@ if (cluster.isMaster) { //===================== REDIS ============================ var redis = require('redis'); var redisClient = require('../db/redis.js'); + var redisClient = require('redis').createClient({host: 'localhost', + port: '6379'}); redisClient.on('ready', function () { console.log("Redis is ready"); @@ -37,6 +39,15 @@ if (cluster.isMaster) { console.log(err); console.log(reply); }); + + const queue = require('kue').createQueue(); + queue.watchStuckJobs(6000); + + queue.process('ad', (job, done) => { + console.log(job.data); + done(); + }); + //================================================= const request = require('request'); const Promise = require('bluebird'); @@ -47,7 +58,13 @@ if (cluster.isMaster) { app.patch('/service', (req, res) => { let targetVideo = req.body.videoId; - + + var job = queue.create('ad', { + video_id: targetVideo + }).save( (err) => { + if(!err) console.log( job.id ); + }); + return new Promise((resolve, reject) => { resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) .then((success) => { @@ -71,6 +88,7 @@ if (cluster.isMaster) { throw err; }); }); + }); app.post('/count', (req, res) => { From 8fd9f1c3d6cbc0934b5de752ffc99240ae249c97 Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Wed, 20 Dec 2017 20:20:10 -0800 Subject: [PATCH 04/14] Queue functioning - sending response back to HTTP request - updating count to DB --- server/index.js | 83 +++++++++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/server/index.js b/server/index.js index ec7b146..490a425 100644 --- a/server/index.js +++ b/server/index.js @@ -21,33 +21,12 @@ if (cluster.isMaster) { const bodyParser = require('body-parser'); const PORT = process.env.PORT || 3000; var knex = require('../db/knex.js'); -//===================== REDIS ============================ - var redis = require('redis'); - var redisClient = require('../db/redis.js'); - var redisClient = require('redis').createClient({host: 'localhost', - port: '6379'}); - - redisClient.on('ready', function () { - console.log("Redis is ready"); - }); - redisClient.on('error', function () { - console.log("Error in Redis"); - }); + var redis = require('redis'); - redisClient.set("language", "nodejs", function (err, reply) { - console.log(err); - console.log(reply); - }); - const queue = require('kue').createQueue(); queue.watchStuckJobs(6000); - queue.process('ad', (job, done) => { - console.log(job.data); - done(); - }); - //================================================= const request = require('request'); const Promise = require('bluebird'); @@ -57,14 +36,23 @@ if (cluster.isMaster) { app.use(bodyParser.urlencoded({ extended: false })); app.patch('/service', (req, res) => { - let targetVideo = req.body.videoId; - var job = queue.create('ad', { + let targetVideo = req.body.videoId; + + queue.create('ad', { video_id: targetVideo - }).save( (err) => { - if(!err) console.log( job.id ); - }); - + }).save((err) => { + if (!err) { + res.status(200).send(targetVideo); + } else { + throw err; + } + }); + }); + + queue.process('ad', (job, done) => { + var targetVideo = job.data.video_id; + return new Promise((resolve, reject) => { resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) .then((success) => { @@ -76,11 +64,13 @@ if (cluster.isMaster) { .then((ad) => { knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) .then((testVideo) => { - res.status(204).end(); + done(); + // res.status(200).send(video); }); }); } else { - res.status(204).end(); + done(); + //res.status(200).send(video); } }); }) @@ -88,8 +78,39 @@ if (cluster.isMaster) { throw err; }); }); + }); + + // var job = queue.create('ad', { + // video_id: targetVideo + // }).save( (err) => { + // if(!err) console.log( job.id ); + // }); + + // return new Promise((resolve, reject) => { + // resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) + // .then((success) => { + // knex('videos').where({ video_id: targetVideo }) + // .then((video) => { + // if (!video[0].ad && video[0].view_count === 339510) { + // let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; + // knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) + // .then((ad) => { + // knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) + // .then((testVideo) => { + // res.status(204).end(); + // }); + // }); + // } else { + // res.status(204).end(); + // } + // }); + // }) + // ).catch((err) => { + // throw err; + // }); + // }); - }); + // }); app.post('/count', (req, res) => { let targetVideo = req.body.videoId; From 7a6f54f4e31983cafa4d5d2793b6cb5c383705cf Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Wed, 20 Dec 2017 20:53:40 -0800 Subject: [PATCH 05/14] Cleaned up comments --- server/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/index.js b/server/index.js index 490a425..801ecee 100644 --- a/server/index.js +++ b/server/index.js @@ -27,7 +27,7 @@ if (cluster.isMaster) { const queue = require('kue').createQueue(); queue.watchStuckJobs(6000); -//================================================= +// ================================================= const request = require('request'); const Promise = require('bluebird'); var app = express(); @@ -41,9 +41,9 @@ if (cluster.isMaster) { queue.create('ad', { video_id: targetVideo - }).save((err) => { + }).save( (err) => { if (!err) { - res.status(200).send(targetVideo); + res.status(204).end(); } else { throw err; } From fe818db5138c06c02f3a7f4217c02692c5b0fe34 Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Thu, 21 Dec 2017 11:58:38 -0800 Subject: [PATCH 06/14] comments --- db/queue.js | 16 ++++++++++++++++ db/reddis_testing.js | 7 ------- redisfile.js | 11 +++++++++-- server/index.js | 40 ++++------------------------------------ 4 files changed, 29 insertions(+), 45 deletions(-) create mode 100644 db/queue.js delete mode 100644 db/reddis_testing.js diff --git a/db/queue.js b/db/queue.js new file mode 100644 index 0000000..f11bf9d --- /dev/null +++ b/db/queue.js @@ -0,0 +1,16 @@ +var redisClient = require('../db/redis.js'); +const queue = require('kue').createQueue(); + +queue.watchStuckJobs(6000); + +console.log('hello world'); + +queue.on('ready', () => { + console.info('Queue is ready!'); +}); + +queue.on('error', (err) => { + console.log('There was an error in the queue'); + console.log(err); + console.log(err.stack); +}); \ No newline at end of file diff --git a/db/reddis_testing.js b/db/reddis_testing.js deleted file mode 100644 index 6337d5f..0000000 --- a/db/reddis_testing.js +++ /dev/null @@ -1,7 +0,0 @@ -// var redis = require('redis'); -// var redisClient = require('./redis.js'); - -// redisClient.set("language", "nodejs", function (err, reply) { -// console.log(err); -// console.log(reply); -// }); \ No newline at end of file diff --git a/redisfile.js b/redisfile.js index 5c0a980..672c083 100644 --- a/redisfile.js +++ b/redisfile.js @@ -3,8 +3,15 @@ module.exports = { client: '', connection: { host: 'localhost', - port: '6379', //needed -// database: 'youtube', + port: '6379' }, }, + production: { + client: '', + connection: { + host: process.env.REDIS_PORT, + port: process.env.REDIS_HOST, + auth: process.env.REDIS_PASS, + }, + } }; \ No newline at end of file diff --git a/server/index.js b/server/index.js index 801ecee..92391ed 100644 --- a/server/index.js +++ b/server/index.js @@ -19,7 +19,7 @@ if (cluster.isMaster) { const express = require('express'); const bodyParser = require('body-parser'); - const PORT = process.env.PORT || 3000; + const PORT = process.env.PORT || 3000; var knex = require('../db/knex.js'); var redis = require('redis'); @@ -45,7 +45,7 @@ if (cluster.isMaster) { if (!err) { res.status(204).end(); } else { - throw err; + res.end(); } }); }); @@ -80,38 +80,6 @@ if (cluster.isMaster) { }); }); - // var job = queue.create('ad', { - // video_id: targetVideo - // }).save( (err) => { - // if(!err) console.log( job.id ); - // }); - - // return new Promise((resolve, reject) => { - // resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) - // .then((success) => { - // knex('videos').where({ video_id: targetVideo }) - // .then((video) => { - // if (!video[0].ad && video[0].view_count === 339510) { - // let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; - // knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) - // .then((ad) => { - // knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) - // .then((testVideo) => { - // res.status(204).end(); - // }); - // }); - // } else { - // res.status(204).end(); - // } - // }); - // }) - // ).catch((err) => { - // throw err; - // }); - // }); - - // }); - app.post('/count', (req, res) => { let targetVideo = req.body.videoId; @@ -126,8 +94,8 @@ if (cluster.isMaster) { }); }); - app.get('/kue', (req, res) => { - res.status(200).end(); + app.patch('/', (req, res) => { + res.status(204).end(); }); if (!module.parent) { // only listen to port if existing port is not in use From 8adb32f50fe5f80e7bc6dd9dcb2f7007526278da Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Fri, 22 Dec 2017 15:17:22 -0800 Subject: [PATCH 07/14] Added: - winston for error logging - removeOnComplete in job creation --- package-lock.json | 38 ++++++++++++++++++++++++++++++++++++++ package.json | 3 ++- server/index.js | 44 +++++++++++++++++++++++++++++++++----------- 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index f9b4339..3490d22 100644 --- a/package-lock.json +++ b/package-lock.json @@ -231,6 +231,11 @@ "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", "integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=" }, + "async": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=" + }, "async-cache": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/async-cache/-/async-cache-1.1.0.tgz", @@ -1094,6 +1099,11 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + }, "combined-stream": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", @@ -1262,6 +1272,11 @@ "resolved": "https://registry.npmjs.org/css-parse/-/css-parse-1.7.0.tgz", "integrity": "sha1-Mh9s9zeCpv91ERE5D8BeLGV9jJs=" }, + "cycle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" + }, "damerau-levenshtein": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.4.tgz", @@ -1941,6 +1956,11 @@ "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, + "eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" + }, "faker": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/faker/-/faker-4.1.0.tgz", @@ -8981,6 +9001,11 @@ "tweetnacl": "0.14.5" } }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "stackman": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/stackman/-/stackman-2.0.1.tgz", @@ -9546,6 +9571,19 @@ "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz", "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=" }, + "winston": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.0.tgz", + "integrity": "sha1-gIBQuT1SZh7Z+2wms/DIJnCLCu4=", + "requires": { + "async": "1.0.0", + "colors": "1.0.3", + "cycle": "1.0.3", + "eyes": "0.1.8", + "isstream": "0.1.2", + "stack-trace": "0.0.10" + } + }, "with": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/with/-/with-5.1.1.tgz", diff --git a/package.json b/package.json index 8766103..685abf5 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,8 @@ "pg": "^4.4.0", "redis": "^2.8.0", "request": "^2.83.0", - "supertest": "^3.0.0" + "supertest": "^3.0.0", + "winston": "^2.4.0" }, "homepage": "https://github.com/jonnylau/hrsf83-thesis#readme" } diff --git a/server/index.js b/server/index.js index 92391ed..0285640 100644 --- a/server/index.js +++ b/server/index.js @@ -1,3 +1,4 @@ +// ====================== CLUSTER ======================================================= const cluster = require('cluster'); if (cluster.isMaster) { @@ -15,8 +16,24 @@ if (cluster.isMaster) { // // Set custom APM Server URL (default: http://localhost:8200) // serverUrl: 'http://localhost:8200' // }); -// ============================================================================================================ - +// =============================================================================================== + const Winston = require('winston'); + const logger = new Winston.Logger({ + level: 'verbose', + transports: [ + new Winston.transports.Console({ + timestamp: true + }), + new Winston.transports.File({ + filename: 'error.log', + timestamp: true, + level: 'error', + }) + ] + }); + + logger.error('Holla'); + const express = require('express'); const bodyParser = require('body-parser'); const PORT = process.env.PORT || 3000; @@ -27,7 +44,8 @@ if (cluster.isMaster) { const queue = require('kue').createQueue(); queue.watchStuckJobs(6000); -// ================================================= + // ============================================================================================= + const request = require('request'); const Promise = require('bluebird'); var app = express(); @@ -41,16 +59,20 @@ if (cluster.isMaster) { queue.create('ad', { video_id: targetVideo - }).save( (err) => { - if (!err) { - res.status(204).end(); - } else { - res.end(); - } - }); + }) + .priority('low') + .removeOnComplete( true ) + .save( (err) => { + if (!err) { + res.status(204).end(); + } else { + logger.error(err); + res.end(); + } + }); }); - queue.process('ad', (job, done) => { + queue.process('ad', 100, (job, done) => { var targetVideo = job.data.video_id; return new Promise((resolve, reject) => { From 94e35511bbf5b8589dfdb179a261c3dac4b9a25b Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Fri, 22 Dec 2017 16:46:55 -0800 Subject: [PATCH 08/14] Implemented concurrent jobs in Redis Kue --- server/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/index.js b/server/index.js index 0285640..8869e32 100644 --- a/server/index.js +++ b/server/index.js @@ -72,7 +72,7 @@ if (cluster.isMaster) { }); }); - queue.process('ad', 100, (job, done) => { + queue.process('ad', (job, done) => { var targetVideo = job.data.video_id; return new Promise((resolve, reject) => { From 1ed00dbfe4b0126d403f8d9b088cda1509b9ebac Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Sun, 31 Dec 2017 18:42:11 -0800 Subject: [PATCH 09/14] Added AWS env file and SQS file --- .env | 5 +++ package-lock.json | 104 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + server/sqs.js | 35 ++++++++++++++++ 4 files changed, 145 insertions(+) create mode 100644 .env create mode 100644 server/sqs.js diff --git a/.env b/.env new file mode 100644 index 0000000..9fc988f --- /dev/null +++ b/.env @@ -0,0 +1,5 @@ +PORT=8000 +DB_HOST='127.0.0.1' +aws_access_key_id=AKIAIU4YXV4CXZ4EBNMA +aws_secret_access_key=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 +Queue='https://sqs.us-east-2.amazonaws.com/077422758482/thesis' \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 3490d22..37a2aa0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -254,6 +254,30 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "aws-sdk": { + "version": "2.176.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.176.0.tgz", + "integrity": "sha1-JyPBThxiOsRcW4kKP8c27ugIX2c=", + "requires": { + "buffer": "4.9.1", + "crypto-browserify": "1.0.9", + "events": "1.1.1", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.1.0", + "xml2js": "0.4.17", + "xmlbuilder": "4.2.1" + }, + "dependencies": { + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + } + } + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -774,6 +798,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "base64-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.1.tgz", + "integrity": "sha512-dwVUVIXsBZXwTuwnXI9RK8sBmgq09NDHzyR9SAph9eqk76gKK2JSQmZARC2zRC81JC2QTtxD0ARU5qTS25gIGw==" + }, "bcrypt-pbkdf": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", @@ -898,6 +927,16 @@ "repeat-element": "1.1.2" } }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "1.2.1", + "ieee754": "1.1.8", + "isarray": "1.0.0" + } + }, "buffer-writer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz", @@ -1262,6 +1301,11 @@ } } }, + "crypto-browserify": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-1.0.9.tgz", + "integrity": "sha1-zFRJaF37hesRyYKKzHy4erW7/MA=" + }, "crypto-random-string": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", @@ -1853,6 +1897,11 @@ "through": "2.3.8" } }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -3237,6 +3286,11 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" }, + "ieee754": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", + "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + }, "ignore": { "version": "3.3.7", "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.7.tgz", @@ -3680,6 +3734,11 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "js-string-escape": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", @@ -8424,6 +8483,11 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "randomatic": { "version": "1.1.7", "resolved": "https://registry.npmjs.org/randomatic/-/randomatic-1.1.7.tgz", @@ -9480,6 +9544,22 @@ } } }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, "url-parse-lax": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", @@ -9662,6 +9742,30 @@ "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=" }, + "xml2js": { + "version": "0.4.17", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.17.tgz", + "integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=", + "requires": { + "sax": "1.2.4", + "xmlbuilder": "4.2.1" + }, + "dependencies": { + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + } + } + }, + "xmlbuilder": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz", + "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=", + "requires": { + "lodash": "4.17.4" + } + }, "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", diff --git a/package.json b/package.json index 685abf5..920e85e 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "url": "https://github.com/jonnylau/hrsf83-thesis/issues" }, "dependencies": { + "aws-sdk": "^2.176.0", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", "babel-preset-es2015": "^6.24.1", diff --git a/server/sqs.js b/server/sqs.js new file mode 100644 index 0000000..fa46f9e --- /dev/null +++ b/server/sqs.js @@ -0,0 +1,35 @@ +// Load the AWS SDK for Node.js +var AWS = require('aws-sdk'); +// Set the region +AWS.config.update({ region: 'us-east-2' }); + +// Create an SQS service object +var sqs = new AWS.SQS({ apiVersion: '2012-11-05' }); + +var params = { + DelaySeconds: 10, + MessageAttributes: { + "Title": { + DataType: "String", + StringValue: "The Whistler" + }, + "Author": { + DataType: "String", + StringValue: "John Grisham" + }, + "WeeksOn": { + DataType: "Number", + StringValue: "6" + } + }, + MessageBody: "Information about current NY Times fiction bestseller for week of 12/11/2016.", + QueueUrl: "https://sqs.us-east-2.amazonaws.com/077422758482/thesis" +}; + +sqs.sendMessage(params, function (err, data) { + if (err) { + console.log("Error", err); + } else { + console.log("Success", data.MessageId); + } +}); \ No newline at end of file From 73ed77a26d01b33b889dc64f7614827e7eca06dd Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Tue, 2 Jan 2018 12:23:44 -0800 Subject: [PATCH 10/14] AWS SQS set up --- .env | 2 +- .gitignore | 5 ++++ aws/config.json | 5 ++++ aws/sqs-retrieve.js | 66 +++++++++++++++++++++++++++++++++++++++++++++ aws/sqs-send.js | 34 +++++++++++++++++++++++ lib/videos.js | 65 ++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 5 ++++ package.json | 1 + server/index.js | 41 +++++++++++----------------- server/sqs.js | 35 ------------------------ 10 files changed, 197 insertions(+), 62 deletions(-) create mode 100644 aws/config.json create mode 100644 aws/sqs-retrieve.js create mode 100644 aws/sqs-send.js delete mode 100644 server/sqs.js diff --git a/.env b/.env index 9fc988f..d81897c 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -PORT=8000 +PORT=3000 DB_HOST='127.0.0.1' aws_access_key_id=AKIAIU4YXV4CXZ4EBNMA aws_secret_access_key=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 diff --git a/.gitignore b/.gitignore index 66428dc..0662082 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,8 @@ dist/ # see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git node_modules/ bower_components/ + +# AWS +./env +./server/aws.config.json + diff --git a/aws/config.json b/aws/config.json new file mode 100644 index 0000000..95dd91a --- /dev/null +++ b/aws/config.json @@ -0,0 +1,5 @@ +{ + "accessKeyId": "AKIAIU4YXV4CXZ4EBNMA", + "secretAccessKey": "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", + "region": "us-east-2" +} \ No newline at end of file diff --git a/aws/sqs-retrieve.js b/aws/sqs-retrieve.js new file mode 100644 index 0000000..d586c96 --- /dev/null +++ b/aws/sqs-retrieve.js @@ -0,0 +1,66 @@ +const knex = require('../db/knex.js'); +const Promise = require('bluebird'); + +// Load the AWS SDK for Node.js +var AWS = require('aws-sdk'); +// config +AWS.config.loadFromPath('./config.json'); + +// Create an SQS service object +var sqs = new AWS.SQS({ apiVersion: '2012-11-05' }); + +var queueURL = "https://sqs.us-east-2.amazonaws.com/077422758482/thesis"; + +var params = { + AttributeNames: [ + "SentTimestamp" + ], + MaxNumberOfMessages: 10, + MessageAttributeNames: [ + "All" + ], + QueueUrl: queueURL, + VisibilityTimeout: 30, + WaitTimeSeconds: 0 +}; + +sqs.receiveMessage(params, function (err, data) { + if (err) { + console.log("Receive Error", err); + } else if (data.Messages) { + for (message of data.Messages) { + console.log(message.MessageAttributes.videoId.StringValue); + let targetVideo = parseInt(message.MessageAttributes.videoId.StringValue); + // Adding to the DB + knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) + .then((success) => { + knex('videos').where({ video_id: targetVideo }) + .then((video) => { + if (!video[0].ad && video[0].view_count === 200) { + let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; + knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) + .then((ad) => { + knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) + .then((testVideo) => { + //console.log('added an ad to a video'); + }); + }); + } else { + //console.log('added an ad to a video'); + } + }); + }); + let deleteParams = { + QueueUrl: queueURL, + ReceiptHandle: message.ReceiptHandle + }; + sqs.deleteMessage(deleteParams, function (err, data) { + if (err) { + throw err; + } else { + console.log("Message Deleted", data); + } + }); + } + } +}); \ No newline at end of file diff --git a/aws/sqs-send.js b/aws/sqs-send.js new file mode 100644 index 0000000..ffde351 --- /dev/null +++ b/aws/sqs-send.js @@ -0,0 +1,34 @@ +// Load the AWS SDK for Node.js +var AWS = require('aws-sdk'); + +// Config settings +// require('dotenv').config(); +AWS.config.loadFromPath('./config.json'); + +// Set the region +//AWS.config.update({ region: 'us-east-2' }); + +// Create an SQS service object +var sqs = new AWS.SQS({ apiVersion: '2012-11-05' }); + +for (let i = 0; i < 10; i++) { + var params = { + DelaySeconds: 10, + MessageAttributes: { + "videoId": { + DataType: "Number", + StringValue: "1000002" + } + }, + MessageBody: "None", + QueueUrl: "https://sqs.us-east-2.amazonaws.com/077422758482/thesis" + }; + + sqs.sendMessage(params, function (err, data) { + if (err) { + console.log("Error", err); + } else { + console.log("Success", data.MessageId); + } + }); +} \ No newline at end of file diff --git a/lib/videos.js b/lib/videos.js index e69de29..d79fe98 100644 --- a/lib/videos.js +++ b/lib/videos.js @@ -0,0 +1,65 @@ +const knex = require('../db/knex.js'); +const Promise = require('bluebird'); +const redis = require('redis'); +const queue = require('kue').createQueue(); + +const express = require('express'); +const bodyParser = require('body-parser'); + +const app = express(); +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ extended: false })); + +let createQueueItem = (targetVideo) => { + queue.create('ad', { + video_id: targetVideo + }) + .priority('medium') + .removeOnComplete(true) + .save((err) => { + if (!err) { + res.status(204).end(); + } else { + logger.error(err); + res.end(); + } + }); +}; + +let updateViewCount = (targetVideo) => { + queue.process('ad', (job, done) => { + var targetVideo = job.data.video_id; + + return new Promise((resolve, reject) => { + resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) + .then((success) => { + knex('videos').where({ video_id: targetVideo }) + .then((video) => { + if (!video[0].ad && video[0].view_count === 200) { + let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; + knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) + .then((ad) => { + knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) + .then((testVideo) => { + done(); + // res.status(200).send(video); + }); + }); + } else { + done(); + //res.status(200).send(video); + } + }); + }) + ).catch((err) => { + throw err; + }); + }); + + }); +}; + +module.exports = { + updateViewCount: updateViewCount, + createQueueItem: createQueueItem +}; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 37a2aa0..9a20fbf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1447,6 +1447,11 @@ "is-obj": "1.0.1" } }, + "dotenv": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-4.0.0.tgz", + "integrity": "sha1-hk7xN5rO1Vzm+V3r7NzhefegzR0=" + }, "double-ended-queue": { "version": "2.1.0-0", "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", diff --git a/package.json b/package.json index 920e85e..06cceb7 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "bluebird": "^3.5.1", "bookshelf": "^0.11.1", "chai": "^4.1.2", + "dotenv": "^4.0.0", "elastic-apm-node": "^0.9.0", "eslint": "^4.11.0", "eslint-config-airbnb": "^16.1.0", diff --git a/server/index.js b/server/index.js index 8869e32..8d78070 100644 --- a/server/index.js +++ b/server/index.js @@ -32,35 +32,34 @@ if (cluster.isMaster) { ] }); - logger.error('Holla'); - - const express = require('express'); - const bodyParser = require('body-parser'); const PORT = process.env.PORT || 3000; - var knex = require('../db/knex.js'); - - var redis = require('redis'); + const knex = require('../db/knex.js'); + const redis = require('redis'); const queue = require('kue').createQueue(); queue.watchStuckJobs(6000); // ============================================================================================= + const express = require('express'); + const bodyParser = require('body-parser'); const request = require('request'); const Promise = require('bluebird'); - var app = express(); + const app = express(); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: false })); + const updateViewCount = require('../lib/videos.js').updateViewCount; + const createQueueItem = require('../lib/videos.js').createQueueItem; + app.patch('/service', (req, res) => { let targetVideo = req.body.videoId; - queue.create('ad', { video_id: targetVideo }) - .priority('low') + .priority('medium') .removeOnComplete( true ) .save( (err) => { if (!err) { @@ -74,13 +73,13 @@ if (cluster.isMaster) { queue.process('ad', (job, done) => { var targetVideo = job.data.video_id; - + return new Promise((resolve, reject) => { resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) .then((success) => { knex('videos').where({ video_id: targetVideo }) .then((video) => { - if (!video[0].ad && video[0].view_count === 339510) { + if (!video[0].ad && video[0].view_count === 200) { let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) .then((ad) => { @@ -102,20 +101,10 @@ if (cluster.isMaster) { }); }); - app.post('/count', (req, res) => { - let targetVideo = req.body.videoId; - - return new Promise((resolve, reject) => { - resolve(knex('videos').where({ video_id: targetVideo }) - .then((video) => { - res.status(200).send(video); - }) - ).catch((err) => { - console.log(err); - }); - }); - }); - + // ================================================================== + // Load testing dummy function + // ================================================================== + app.patch('/', (req, res) => { res.status(204).end(); }); diff --git a/server/sqs.js b/server/sqs.js deleted file mode 100644 index fa46f9e..0000000 --- a/server/sqs.js +++ /dev/null @@ -1,35 +0,0 @@ -// Load the AWS SDK for Node.js -var AWS = require('aws-sdk'); -// Set the region -AWS.config.update({ region: 'us-east-2' }); - -// Create an SQS service object -var sqs = new AWS.SQS({ apiVersion: '2012-11-05' }); - -var params = { - DelaySeconds: 10, - MessageAttributes: { - "Title": { - DataType: "String", - StringValue: "The Whistler" - }, - "Author": { - DataType: "String", - StringValue: "John Grisham" - }, - "WeeksOn": { - DataType: "Number", - StringValue: "6" - } - }, - MessageBody: "Information about current NY Times fiction bestseller for week of 12/11/2016.", - QueueUrl: "https://sqs.us-east-2.amazonaws.com/077422758482/thesis" -}; - -sqs.sendMessage(params, function (err, data) { - if (err) { - console.log("Error", err); - } else { - console.log("Success", data.MessageId); - } -}); \ No newline at end of file From de0073cea0a1e2edbc98fe58ae137109df917f84 Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Tue, 2 Jan 2018 17:38:07 -0800 Subject: [PATCH 11/14] Added new relic --- aws/sqs-retrieve.js | 80 ++++++++++++++++++++++++--------------------- aws/sqs-send.js | 3 +- newrelic.js | 25 ++++++++++++++ package-lock.json | 48 +++++++++++++++++++++++++++ package.json | 1 + server/index.js | 4 +-- 6 files changed, 120 insertions(+), 41 deletions(-) create mode 100644 newrelic.js diff --git a/aws/sqs-retrieve.js b/aws/sqs-retrieve.js index d586c96..5aad45b 100644 --- a/aws/sqs-retrieve.js +++ b/aws/sqs-retrieve.js @@ -24,43 +24,47 @@ var params = { WaitTimeSeconds: 0 }; -sqs.receiveMessage(params, function (err, data) { - if (err) { - console.log("Receive Error", err); - } else if (data.Messages) { - for (message of data.Messages) { - console.log(message.MessageAttributes.videoId.StringValue); - let targetVideo = parseInt(message.MessageAttributes.videoId.StringValue); - // Adding to the DB - knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) - .then((success) => { - knex('videos').where({ video_id: targetVideo }) - .then((video) => { - if (!video[0].ad && video[0].view_count === 200) { - let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; - knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) - .then((ad) => { - knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) - .then((testVideo) => { - //console.log('added an ad to a video'); - }); - }); - } else { - //console.log('added an ad to a video'); - } - }); +const recieveMessage = function() { + sqs.receiveMessage(params, function (err, data) { + if (err) { + console.log("Receive Error", err); + } else if (data.Messages) { + for (message of data.Messages) { + console.log(message.MessageAttributes.videoId.StringValue); + let targetVideo = parseInt(message.MessageAttributes.videoId.StringValue); + // Adding to the DB + knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) + .then((success) => { + knex('videos').where({ video_id: targetVideo }) + .then((video) => { + if (!video[0].ad && video[0].view_count === 200) { + let category = Math.random() > 0.5 ? type = 'comedy' : 'informational'; + knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`) + .then((ad) => { + knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) + .then((testVideo) => { + //console.log('added an ad to a video'); + }); + }); + } else { + //console.log('added an ad to a video'); + } + }); + }); + let deleteParams = { + QueueUrl: queueURL, + ReceiptHandle: message.ReceiptHandle + }; + sqs.deleteMessage(deleteParams, function (err, data) { + if (err) { + throw err; + } else { + console.log("Message Deleted", data); + } }); - let deleteParams = { - QueueUrl: queueURL, - ReceiptHandle: message.ReceiptHandle - }; - sqs.deleteMessage(deleteParams, function (err, data) { - if (err) { - throw err; - } else { - console.log("Message Deleted", data); - } - }); + } } - } -}); \ No newline at end of file + }); +}; + +setInterval(recieveMessage, 1000); \ No newline at end of file diff --git a/aws/sqs-send.js b/aws/sqs-send.js index ffde351..e6cc2fc 100644 --- a/aws/sqs-send.js +++ b/aws/sqs-send.js @@ -12,12 +12,13 @@ AWS.config.loadFromPath('./config.json'); var sqs = new AWS.SQS({ apiVersion: '2012-11-05' }); for (let i = 0; i < 10; i++) { + let videoId = Math.ceil( Math.random() * 10000000).toString(); var params = { DelaySeconds: 10, MessageAttributes: { "videoId": { DataType: "Number", - StringValue: "1000002" + StringValue: "videoId" } }, MessageBody: "None", diff --git a/newrelic.js b/newrelic.js new file mode 100644 index 0000000..46cfb2e --- /dev/null +++ b/newrelic.js @@ -0,0 +1,25 @@ +'use strict' +/** + * New Relic agent configuration. + * + * See lib/config.defaults.js in the agent distribution for a more complete + * description of configuration variables and their potential values. + */ +exports.config = { + /** + * Array of application names. + */ + app_name: ['thesis'], + /** + * Your New Relic license key. + */ + license_key: '1f02309c4e0cfa198a6ac889c055354047119eea', + logging: { + /** + * Level at which to log. 'trace' is most useful to New Relic when diagnosing + * issues with the agent, 'info' and higher will impose the least overhead on + * production applications. + */ + level: 'info' + } +} diff --git a/package-lock.json b/package-lock.json index 9a20fbf..bad4305 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,15 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@newrelic/native-metrics": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@newrelic/native-metrics/-/native-metrics-2.1.2.tgz", + "integrity": "sha1-mlldxgJlS3FxiKKUUHCHpR55+6M=", + "optional": true, + "requires": { + "nan": "2.8.0" + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -58,6 +67,11 @@ "resolved": "https://registry.npmjs.org/after-all-results/-/after-all-results-2.0.0.tgz", "integrity": "sha1-asL8ICtQD4jaj09VMM+hAPTGotA=" }, + "agent-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-1.0.2.tgz", + "integrity": "sha1-aJDT+yFwBLYrcPiSjg+uX4lSpwY=" + }, "ajv": { "version": "5.5.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.1.tgz", @@ -3286,6 +3300,16 @@ "sshpk": "1.13.1" } }, + "https-proxy-agent": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-0.3.6.tgz", + "integrity": "sha1-cT+jjl01P1DrFKNC/r4pAz7RYZs=", + "requires": { + "agent-base": "1.0.2", + "debug": "2.6.9", + "extend": "3.0.1" + } + }, "iconv-lite": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", @@ -4314,6 +4338,30 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, + "newrelic": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/newrelic/-/newrelic-2.4.2.tgz", + "integrity": "sha1-zmMji5ifj6J9rmDqJP48L62Wnlw=", + "requires": { + "@newrelic/native-metrics": "2.1.2", + "async": "2.6.0", + "concat-stream": "1.6.0", + "https-proxy-agent": "0.3.6", + "json-stringify-safe": "5.0.1", + "readable-stream": "2.3.3", + "semver": "5.4.1" + }, + "dependencies": { + "async": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", + "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==", + "requires": { + "lodash": "4.17.4" + } + } + } + }, "next-line": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", diff --git a/package.json b/package.json index 06cceb7..80772f2 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "knex": "^0.14.2", "kue": "^0.11.6", "moment": "^2.19.4", + "newrelic": "^2.4.2", "nodemon": "^1.12.5", "npm": "^5.5.1", "pg": "^4.4.0", diff --git a/server/index.js b/server/index.js index 8d78070..b60c2da 100644 --- a/server/index.js +++ b/server/index.js @@ -1,4 +1,5 @@ // ====================== CLUSTER ======================================================= +require('newrelic'); const cluster = require('cluster'); if (cluster.isMaster) { @@ -40,7 +41,6 @@ if (cluster.isMaster) { queue.watchStuckJobs(6000); // ============================================================================================= - const express = require('express'); const bodyParser = require('body-parser'); const request = require('request'); @@ -102,7 +102,7 @@ if (cluster.isMaster) { }); // ================================================================== - // Load testing dummy function + // Load Testing Endpoint // ================================================================== app.patch('/', (req, res) => { From b684f863b1b3ae5c37b2dc9b748bb04897e70edb Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Wed, 3 Jan 2018 19:18:09 -0800 Subject: [PATCH 12/14] Docker server container and postgres container talking to each other --- .dockerignore | 2 ++ Dockerfile | 21 ++++++++++++++++ aws/sqs-retrieve.js | 2 +- bookshelf.js | 4 ---- db/knex.js | 2 +- docker-compose.yml | 15 ++++++++++++ knexfile.js | 42 ++++++++++++++++++-------------- server/index.js | 58 +++++++++++++++++++++++---------------------- 8 files changed, 94 insertions(+), 52 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile delete mode 100644 bookshelf.js create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5171c54 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +node_modules +npm-debug.log \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..881427b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# What image do you want to start building on? +FROM node:latest + +# Make a folder in your image where your app's source code can live +RUN mkdir -p /src/app + +# Tell your container where your app's source code will live +WORKDIR /src/app + +# What source code do you what to copy, and where to put it? +COPY . /src/app + +# Does your app have any dependencies that should be installed? +RUN npm install + +# What port will the container talk to the outside world with once created? +EXPOSE 80 + +# How do you start your app? +#CMD [ "npm", "run", "start" ] +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/aws/sqs-retrieve.js b/aws/sqs-retrieve.js index 5aad45b..5ece6c3 100644 --- a/aws/sqs-retrieve.js +++ b/aws/sqs-retrieve.js @@ -67,4 +67,4 @@ const recieveMessage = function() { }); }; -setInterval(recieveMessage, 1000); \ No newline at end of file +setInterval(recieveMessage, 5000); \ No newline at end of file diff --git a/bookshelf.js b/bookshelf.js deleted file mode 100644 index 9211e7b..0000000 --- a/bookshelf.js +++ /dev/null @@ -1,4 +0,0 @@ -// var knex = require('../db/knex'); -// var bookshelf = require('bookshelf')(knex); - -// module.exports = bookshelf; \ No newline at end of file diff --git a/db/knex.js b/db/knex.js index 264b5c2..61bf341 100644 --- a/db/knex.js +++ b/db/knex.js @@ -1,4 +1,4 @@ -var environment = process.env.NODE_ENV || 'development'; +var environment = 'production'; var config = require('../knexfile.js')[environment]; module.exports = require('knex')(config); diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5af6503 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + server: + build: . + depends_on: + - 'database' + ports: + - '80:80' + + database: + image: postgres:latest + environment: + POSTGRES_USER: jonlau + POSTGRES_DB: youtube \ No newline at end of file diff --git a/knexfile.js b/knexfile.js index 064a41a..7fd45db 100644 --- a/knexfile.js +++ b/knexfile.js @@ -1,23 +1,29 @@ module.exports = { + // development: { + // client: 'pg', + // connection: { + // hostname: 'postgres', + // host: 'localhost', + // port: '5432', + // user: 'postgres', + // password: '', + // database: 'youtube', + // }, + // migrations: { + // directory: __dirname + '/db/migrations', + // }, + // seeds: { + // directory: __dirname + '/db/seeds', + // }, + // }, development: { - client: 'pg', - connection: { - hostname: 'postgres', - host: 'localhost', - port: '5432', //needed - user: 'postgres', - password: '', - database: 'youtube', - }, + client: 'pg', + connection: 'postgres://postgres@172.17.03', migrations: { directory: __dirname + '/db/migrations', - }, - seeds: { - directory: __dirname + '/db/seeds', - }, - }, - production: { - client: 'pg', - connection: process.env.DATABASE_URL, + } } -}; \ No newline at end of file +}; + +// "IPv4Address": "172.17.0.3/16" +// process.env.DATABASE_URL \ No newline at end of file diff --git a/server/index.js b/server/index.js index b60c2da..7674325 100644 --- a/server/index.js +++ b/server/index.js @@ -34,11 +34,14 @@ if (cluster.isMaster) { }); const PORT = process.env.PORT || 3000; - const knex = require('../db/knex.js'); - - const redis = require('redis'); - const queue = require('kue').createQueue(); - queue.watchStuckJobs(6000); + //const knex = require('../db/knex.js'); + const knex = require('knex')({ + client: 'pg', + connection: 'postgres://postgres@172.17.0.3', + }); + // const redis = require('redis'); + // const queue = require('kue').createQueue(); + // queue.watchStuckJobs(6000); // ============================================================================================= const express = require('express'); @@ -50,30 +53,29 @@ if (cluster.isMaster) { app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: false })); - const updateViewCount = require('../lib/videos.js').updateViewCount; - const createQueueItem = require('../lib/videos.js').createQueueItem; - app.patch('/service', (req, res) => { let targetVideo = req.body.videoId; - queue.create('ad', { - video_id: targetVideo - }) - .priority('medium') - .removeOnComplete( true ) - .save( (err) => { - if (!err) { - res.status(204).end(); - } else { - logger.error(err); - res.end(); - } - }); - }); - - queue.process('ad', (job, done) => { - var targetVideo = job.data.video_id; + // ====================================================================== + // REDIS KUE + // queue.create('ad', { + // video_id: targetVideo + // }) + // .priority('medium') + // .removeOnComplete( true ) + // .save( (err) => { + // if (!err) { + // res.status(204).end(); + // } else { + // logger.error(err); + // res.end(); + // } + // }); + // }); + // queue.process('ad', (job, done) => { + // var targetVideo = job.data.video_id; + // ====================================================================== return new Promise((resolve, reject) => { resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) .then((success) => { @@ -85,12 +87,12 @@ if (cluster.isMaster) { .then((ad) => { knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) .then((testVideo) => { - done(); + //done(); // res.status(200).send(video); }); }); } else { - done(); + //done(); //res.status(200).send(video); } }); @@ -106,7 +108,7 @@ if (cluster.isMaster) { // ================================================================== app.patch('/', (req, res) => { - res.status(204).end(); + res.status(200).send({testMessage: 'hello world'}); }); if (!module.parent) { // only listen to port if existing port is not in use From b5152eeff264b430d751c7ebdda3102a211e1f81 Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Thu, 4 Jan 2018 11:09:26 -0800 Subject: [PATCH 13/14] New test route for batch updates --- db/knex.js | 2 +- server/index.js | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/db/knex.js b/db/knex.js index 61bf341..c9ed149 100644 --- a/db/knex.js +++ b/db/knex.js @@ -1,4 +1,4 @@ -var environment = 'production'; +var environment = 'development'; var config = require('../knexfile.js')[environment]; module.exports = require('knex')(config); diff --git a/server/index.js b/server/index.js index 7674325..4f2f666 100644 --- a/server/index.js +++ b/server/index.js @@ -1,4 +1,6 @@ // ====================== CLUSTER ======================================================= +const queue = {}; + require('newrelic'); const cluster = require('cluster'); @@ -88,12 +90,12 @@ if (cluster.isMaster) { knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) .then((testVideo) => { //done(); - // res.status(200).send(video); + res.status(204).end(); }); }); } else { //done(); - //res.status(200).send(video); + res.status(204).end(); } }); }) @@ -111,6 +113,42 @@ if (cluster.isMaster) { res.status(200).send({testMessage: 'hello world'}); }); + + + app.patch('/test', (req, res) => { + let targetVideo = req.body.videoId; + console.log(typeof targetVideo); + + if (Object.keys(queue).length < 5) { + + queue[targetVideo] = (queue[targetVideo] || 0) + 1; + //queue[targetVideo] += 1; + + console.log('keyCount --->', Object.keys(queue).length); + console.log('queue ---->', queue); + + res.status(200).send({message: 'updates recieved'}); + } else { + let videos = []; + let views = []; + + for (video of Object.keys(queue)) { + videos.push(video); + } + for (video in queue) { + views.push(queue[video]); + } + console.log(videos, views); + res.status(200).send({ message: 'over threshold' }); + //knex.raw() + // update tmp set age = data_table.age + // from + // (select unnest(array['keith', 'leslie', 'bexley', 'casey']) as name, + // unnest(array[44, 50, 10, 12]) as age) as data_table + // where tmp.name = data_table.name; + } + }); + if (!module.parent) { // only listen to port if existing port is not in use app.listen(PORT, () => { console.log('Listening on port', PORT); From d0bcb73e7dd05ba97d3cf3c274bdc9b07ceae6a3 Mon Sep 17 00:00:00 2001 From: Jon Lau Date: Mon, 5 Feb 2018 10:52:09 -0800 Subject: [PATCH 14/14] Cleaned up comments --- db/knex.js | 2 +- knexfile.js | 34 ++++++++--------- server/index.js | 99 ++++++++++++++++++++++--------------------------- 3 files changed, 63 insertions(+), 72 deletions(-) diff --git a/db/knex.js b/db/knex.js index c9ed149..264b5c2 100644 --- a/db/knex.js +++ b/db/knex.js @@ -1,4 +1,4 @@ -var environment = 'development'; +var environment = process.env.NODE_ENV || 'development'; var config = require('../knexfile.js')[environment]; module.exports = require('knex')(config); diff --git a/knexfile.js b/knexfile.js index 7fd45db..2b5d1f7 100644 --- a/knexfile.js +++ b/knexfile.js @@ -1,22 +1,22 @@ module.exports = { - // development: { - // client: 'pg', - // connection: { - // hostname: 'postgres', - // host: 'localhost', - // port: '5432', - // user: 'postgres', - // password: '', - // database: 'youtube', - // }, - // migrations: { - // directory: __dirname + '/db/migrations', - // }, - // seeds: { - // directory: __dirname + '/db/seeds', - // }, - // }, development: { + client: 'pg', + connection: { + hostname: 'postgres', + host: 'localhost', + port: '5432', + user: 'postgres', + password: '', + database: 'youtube', + }, + migrations: { + directory: __dirname + '/db/migrations', + }, + seeds: { + directory: __dirname + '/db/seeds', + }, + }, + production: { client: 'pg', connection: 'postgres://postgres@172.17.03', migrations: { diff --git a/server/index.js b/server/index.js index 4f2f666..ace16d9 100644 --- a/server/index.js +++ b/server/index.js @@ -1,7 +1,8 @@ -// ====================== CLUSTER ======================================================= -const queue = {}; +let queue = {}; require('newrelic'); +// ====================== CLUSTER ======================================================= + const cluster = require('cluster'); if (cluster.isMaster) { @@ -37,10 +38,7 @@ if (cluster.isMaster) { const PORT = process.env.PORT || 3000; //const knex = require('../db/knex.js'); - const knex = require('knex')({ - client: 'pg', - connection: 'postgres://postgres@172.17.0.3', - }); + const knex = require('../db/knex.js'); // const redis = require('redis'); // const queue = require('kue').createQueue(); // queue.watchStuckJobs(6000); @@ -55,29 +53,30 @@ if (cluster.isMaster) { app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: false })); - app.patch('/service', (req, res) => { + app.patch('/service-redis-queue', (req, res) => { let targetVideo = req.body.videoId; // ====================================================================== - // REDIS KUE - // queue.create('ad', { - // video_id: targetVideo - // }) - // .priority('medium') - // .removeOnComplete( true ) - // .save( (err) => { - // if (!err) { - // res.status(204).end(); - // } else { - // logger.error(err); - // res.end(); - // } - // }); - // }); - - // queue.process('ad', (job, done) => { - // var targetVideo = job.data.video_id; + // REDIS KUE - EndPoint // ====================================================================== + + queue.create('ad', { + video_id: targetVideo + }) + .priority('medium') + .removeOnComplete( true ) + .save( (err) => { + if (!err) { + res.status(204).end(); + } else { + logger.error(err); + res.end(); + } + }); + }); + + queue.process('ad', (job, done) => { + var targetVideo = job.data.video_id; return new Promise((resolve, reject) => { resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1) .then((success) => { @@ -89,12 +88,12 @@ if (cluster.isMaster) { .then((ad) => { knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id }) .then((testVideo) => { - //done(); + done(); res.status(204).end(); }); }); } else { - //done(); + done(); res.status(204).end(); } }); @@ -105,51 +104,43 @@ if (cluster.isMaster) { }); }); - // ================================================================== - // Load Testing Endpoint - // ================================================================== - - app.patch('/', (req, res) => { - res.status(200).send({testMessage: 'hello world'}); - }); + // ====================================================================== + // BATCH PATCH - EndPoint + // ====================================================================== - - - app.patch('/test', (req, res) => { + app.patch('/service-batch', (req, res) => { let targetVideo = req.body.videoId; - console.log(typeof targetVideo); - if (Object.keys(queue).length < 5) { + if (Object.keys(queue).length < 9) { queue[targetVideo] = (queue[targetVideo] || 0) + 1; - //queue[targetVideo] += 1; - - console.log('keyCount --->', Object.keys(queue).length); - console.log('queue ---->', queue); - res.status(200).send({message: 'updates recieved'}); + res.status(204).end(); } else { let videos = []; let views = []; for (video of Object.keys(queue)) { - videos.push(video); + videos.push(parseInt(video)); } for (video in queue) { views.push(queue[video]); } - console.log(videos, views); - res.status(200).send({ message: 'over threshold' }); - //knex.raw() - // update tmp set age = data_table.age - // from - // (select unnest(array['keith', 'leslie', 'bexley', 'casey']) as name, - // unnest(array[44, 50, 10, 12]) as age) as data_table - // where tmp.name = data_table.name; + knex.raw(`UPDATE videos SET view_count = view_count + data_table.views + FROM + (select unnest(array[${videos}]) as video_id, + unnest(array[${views}]) as views) as data_table + WHERE videos.video_id = data_table.video_id`) + .then((success) => { + for (key in queue) { + delete queue[key]; + } + res.status(204).end(); + }); } }); - if (!module.parent) { // only listen to port if existing port is not in use + if (!module.parent) { app.listen(PORT, () => { console.log('Listening on port', PORT); });