Added Redis Kue + Node Cluster + Winston Error logger

.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+npm-debug.log

.env

@@ -0,0 +1,5 @@
+PORT=3000
+DB_HOST='127.0.0.1'
+aws_access_key_id=AKIAIU4YXV4CXZ4EBNMA
+aws_secret_access_key=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
+Queue='https://sqs.us-east-2.amazonaws.com/077422758482/thesis'

.gitignore

@@ -35,3 +35,8 @@ dist/
 # see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git
 node_modules/
 bower_components/
+
+# AWS
+./env
+./server/aws.config.json
+

Dockerfile

@@ -0,0 +1,21 @@
+# What image do you want to start building on?
+FROM node:latest
+
+# Make a folder in your image where your app's source code can live
+RUN mkdir -p /src/app
+
+# Tell your container where your app's source code will live
+WORKDIR /src/app
+
+# What source code do you what to copy, and where to put it?
+COPY . /src/app
+
+# Does your app have any dependencies that should be installed?
+RUN npm install
+
+# What port will the container talk to the outside world with once created?
+EXPOSE 80
+
+# How do you start your app?
+#CMD [ "npm", "run", "start" ]
+CMD [ "npm", "start" ]

aws/config.json

@@ -0,0 +1,5 @@
+{
+  "accessKeyId": "AKIAIU4YXV4CXZ4EBNMA",
+  "secretAccessKey": "aRZMX5sgjbSHgh+4AsaZ/wMxf29QZEaDIfRelJyp",
+  "region": "us-east-2"
+}

aws/sqs-retrieve.js

@@ -0,0 +1,70 @@
+const knex = require('../db/knex.js');
+const Promise = require('bluebird');
+
+// Load the AWS SDK for Node.js
+var AWS = require('aws-sdk');
+// config
+AWS.config.loadFromPath('./config.json');
+
+// Create an SQS service object
+var sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
+
+var queueURL = "https://sqs.us-east-2.amazonaws.com/077422758482/thesis";
+
+var params = {
+  AttributeNames: [
+    "SentTimestamp"
+  ],
+  MaxNumberOfMessages: 10,
+  MessageAttributeNames: [
+    "All"
+  ],
+  QueueUrl: queueURL,
+  VisibilityTimeout: 30,
+  WaitTimeSeconds: 0
+};
+
+const recieveMessage = function() {
+  sqs.receiveMessage(params, function (err, data) {
+    if (err) {
+      console.log("Receive Error", err);
+    } else if (data.Messages) {
+      for (message of data.Messages) {
+        console.log(message.MessageAttributes.videoId.StringValue);
+        let targetVideo = parseInt(message.MessageAttributes.videoId.StringValue);
+        // Adding to the DB
+        knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1)
+          .then((success) => {
+            knex('videos').where({ video_id: targetVideo })
+              .then((video) => {
+                if (!video[0].ad && video[0].view_count === 200) {
+                  let category = Math.random() > 0.5 ? type = 'comedy' : 'informational';
+                  knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`)
+                    .then((ad) => {
+                      knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id })
+                        .then((testVideo) => {
+                          //console.log('added an ad to a video');
+                        });
+                    });
+                } else {
+                  //console.log('added an ad to a video');
+                }
+              });
+          });
+        let deleteParams = {
+          QueueUrl: queueURL,
+          ReceiptHandle: message.ReceiptHandle
+        };
+        sqs.deleteMessage(deleteParams, function (err, data) {
+          if (err) {
+            throw err;
+          } else {
+            console.log("Message Deleted", data);
+          }
+        });
+      }
+    }
+  });
+};
+
+setInterval(recieveMessage, 5000);

aws/sqs-send.js

@@ -0,0 +1,35 @@
+// Load the AWS SDK for Node.js
+var AWS = require('aws-sdk');
+
+// Config settings
+// require('dotenv').config();
+AWS.config.loadFromPath('./config.json');
+
+// Set the region 
+//AWS.config.update({ region: 'us-east-2' });
+
+// Create an SQS service object
+var sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
+
+for (let i = 0; i < 10; i++) {
+  let videoId = Math.ceil( Math.random() * 10000000).toString();
+  var params = {
+    DelaySeconds: 10,
+    MessageAttributes: {
+      "videoId": {
+        DataType: "Number",
+        StringValue: "videoId"
+      }
+    },
+    MessageBody: "None",
+    QueueUrl: "https://sqs.us-east-2.amazonaws.com/077422758482/thesis"
+  };
+
+  sqs.sendMessage(params, function (err, data) {
+    if (err) {
+      console.log("Error", err);
+    } else {
+      console.log("Success", data.MessageId);
+    }
+  });
+}

db/queue.js

@@ -0,0 +1,16 @@
+var redisClient = require('../db/redis.js');
+const queue = require('kue').createQueue();
+
+queue.watchStuckJobs(6000);
+
+console.log('hello world');
+
+queue.on('ready', () => {
+  console.info('Queue is ready!');
+});
+
+queue.on('error', (err) => {
+  console.log('There was an error in the queue');
+  console.log(err);
+  console.log(err.stack);
+});

db/redis.js

@@ -0,0 +1,5 @@
+var environment = process.env.NODE_ENV || 'development';
+var config = require('../redisfile.js')[environment.connection];
+var redis = require('redis');
+
+module.exports = redis.createClient(config);  

docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3'
+
+services:
+  server:
+    build: .
+    depends_on:
+      - 'database'
+    ports: 
+      - '80:80'
+
+  database:
+    image: postgres:latest
+    environment: 
+      POSTGRES_USER: jonlau
+      POSTGRES_DB: youtube

knexfile.js

@@ -4,7 +4,7 @@ module.exports = {
     connection: {
       hostname: 'postgres',
       host: 'localhost',
-      port: '5432', //needed
+      port: '5432',
       user: 'postgres',
       password: '',
       database: 'youtube',
@@ -18,6 +18,12 @@ module.exports = {
   },
   production: {
     client: 'pg',
-    connection: process.env.DATABASE_URL,
+    connection: 'postgres://postgres@172.17.03',
+    migrations: {
+      directory: __dirname + '/db/migrations',
+    }
   }
-};
+};
+
+// "IPv4Address": "172.17.0.3/16"
+// process.env.DATABASE_URL

lib/videos.js

@@ -0,0 +1,65 @@
+const knex = require('../db/knex.js');
+const Promise = require('bluebird');
+const redis = require('redis');
+const queue = require('kue').createQueue();
+
+const express = require('express');
+const bodyParser = require('body-parser');
+
+const app = express();
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: false }));
+
+let createQueueItem = (targetVideo) => {
+  queue.create('ad', {
+    video_id: targetVideo
+  })
+    .priority('medium')
+    .removeOnComplete(true)
+    .save((err) => {
+      if (!err) {
+        res.status(204).end();
+      } else {
+        logger.error(err);
+        res.end();
+      }
+    });
+};
+
+let updateViewCount = (targetVideo) => {
+  queue.process('ad', (job, done) => {
+    var targetVideo = job.data.video_id;
+
+    return new Promise((resolve, reject) => {
+      resolve(knex('videos').where('video_id', '=', targetVideo).increment('view_count', 1)
+        .then((success) => {
+          knex('videos').where({ video_id: targetVideo })
+            .then((video) => {
+              if (!video[0].ad && video[0].view_count === 200) {
+                let category = Math.random() > 0.5 ? type = 'comedy' : 'informational';
+                knex.raw(`SELECT ad_id FROM ads WHERE category = '${category}' ORDER BY RANDOM() LIMIT 1`)
+                  .then((ad) => {
+                    knex('videos').where('video_id', '=', targetVideo).update({ 'ad': ad.rows[0].ad_id })
+                      .then((testVideo) => {
+                        done();
+                        // res.status(200).send(video);
+                      });
+                  });
+              } else {
+                done();
+                //res.status(200).send(video);
+              }
+            });
+        })
+      ).catch((err) => {
+        throw err;
+      });
+    });
+
+  });
+};
+
+module.exports = {
+  updateViewCount: updateViewCount,
+  createQueueItem: createQueueItem
+};

newrelic.js

@@ -0,0 +1,25 @@
+'use strict'
+/**
+ * New Relic agent configuration.
+ *
+ * See lib/config.defaults.js in the agent distribution for a more complete
+ * description of configuration variables and their potential values.
+ */
+exports.config = {
+  /**
+   * Array of application names.
+   */
+  app_name: ['thesis'],
+  /**
+   * Your New Relic license key.
+   */
+  license_key: '1f02309c4e0cfa198a6ac889c055354047119eea',
+  logging: {
+    /**
+     * Level at which to log. 'trace' is most useful to New Relic when diagnosing
+     * issues with the agent, 'info' and higher will impose the least overhead on
+     * production applications.
+     */
+    level: 'info'
+  }
+}