Use stream for deploy route response to provide more info for client

Author: yamalight

src/docker/build.js

@@ -4,9 +4,9 @@ const tar = require('tar-fs');
 // our modules
 const logger = require('../logger');
 const docker = require('./docker');
-const {tempDockerDir, getProjectConfig, tagFromConfig} = require('../util');
+const {tempDockerDir, getProjectConfig, tagFromConfig, writeStatus} = require('../util');
 
-module.exports = ({username}) =>
+module.exports = ({username, resultStream}) =>
   new Promise(async (resolve, reject) => {
     // get packed stream
     const tarStream = tar.pack(tempDockerDir);
@@ -17,6 +17,7 @@ module.exports = ({username}) =>
     // construct image tag
     const tag = tagFromConfig({username, config});
     logger.debug('building with tag:', tag);
+    writeStatus(resultStream, {message: `Building image with tag: ${tag}`, level: 'verbose'});
 
     // deploy as docker
     const log = [];
@@ -33,17 +34,21 @@ module.exports = ({username}) =>
           // process log data
           if (data.stream && data.stream.length) {
             log.push(data.stream);
+            writeStatus(resultStream, {message: data.stream, level: 'verbose'});
           } else if (data.error && data.error.length) {
             // process error data
             log.push(data.error);
+            writeStatus(resultStream, {message: data.error, level: 'error'});
             hasErrors = true;
           } else {
             // push everything else as-is
             log.push(s);
+            writeStatus(resultStream, {message: s, level: 'verbose'});
           }
         } catch (e) {
           if (s && s.length) {
             log.push(s);
+            writeStatus(resultStream, {message: s, level: 'verbose'});
           }
         }
       });

src/docker/dockercompose/index.js

@@ -4,18 +4,18 @@ const fs = require('fs');
 const path = require('path');
 const yaml = require('js-yaml');
 const uuid = require('uuid');
-const {exec} = require('child_process');
+const {spawn} = require('child_process');
 
 // our packages
-const {tempDockerDir, getProjectConfig} = require('../../util');
+const {tempDockerDir, getProjectConfig, writeStatus} = require('../../util');
 
 // compose file path
 const composePath = path.join(tempDockerDir, 'docker-compose.yml');
 
 exports.hasCompose = () => {
   // if project already has docker-compose - just exit
   try {
-    fs.readFileSync(path.join(tempDockerDir, 'docker-compose.yml'));
+    fs.readFileSync(composePath);
     return true;
   } catch (e) {
     return false;
@@ -72,20 +72,22 @@ exports.updateCompose = ({username}) => {
   return compose;
 };
 
-exports.executeCompose = () =>
-  new Promise((resolve, reject) => {
-    exec(
-      'docker-compose up -d',
-      {
-        cwd: tempDockerDir,
-      },
-      (err, stdout, stderr) => {
-        if (err) {
-          reject(err);
-          return;
-        }
+exports.executeCompose = resultStream =>
+  new Promise(resolve => {
+    const dc = spawn('docker-compose', ['up', '-d'], {cwd: tempDockerDir});
 
-        resolve({log: `${stdout.toString()}\n${stderr.toString()}`});
-      }
-    );
+    dc.stdout.on('data', data => {
+      const message = data.toString().replace(/\n$/, '');
+      const hasError = message.toLowerCase().includes('error');
+      writeStatus(resultStream, {message, level: hasError ? 'error' : 'info'});
+    });
+    dc.stderr.on('data', data => {
+      const message = data.toString().replace(/\n$/, '');
+      const hasError = message.toLowerCase().includes('error');
+      writeStatus(resultStream, {message, level: hasError ? 'error' : 'info'});
+    });
+    dc.on('exit', code => {
+      writeStatus(resultStream, {message: `Docker-compose exited with code ${code.toString()}`, level: 'info'});
+      resolve(code.toString());
+    });
   });

src/docker/dockerfile/index.js

@@ -4,15 +4,16 @@ const path = require('path');
 
 // our packages
 const logger = require('../../logger');
-const {tempDockerDir} = require('../../util');
+const {tempDockerDir, writeStatus} = require('../../util');
 const nodeDockerfile = require('./node-docker');
 const nginxDockerfile = require('./nginx-dockerfile');
 
-module.exports = () => {
+module.exports = resultStream => {
   // if project already has dockerfile - just exit
   try {
     fs.readFileSync(path.join(tempDockerDir, 'Dockerfile'));
     logger.debug('Project already has dockerfile!');
+    writeStatus(resultStream, {message: 'Deploying Dockerfile project..', level: 'info'});
     return;
   } catch (e) {
     logger.debug('No dockerfile found, going to generate a new one..');
@@ -25,14 +26,18 @@ module.exports = () => {
   // if it's a node.js project
   if (filesList.includes('package.json')) {
     dockerfile = nodeDockerfile({hasYarn: filesList.includes('yarn.lock')});
+    writeStatus(resultStream, {message: 'Deploying Node.js project..', level: 'info'});
   } else if (filesList.includes('index.html')) {
     dockerfile = nginxDockerfile;
+    writeStatus(resultStream, {message: 'Deploying Static HTML project..', level: 'info'});
   }
 
   logger.debug('Using dockerfile:', dockerfile);
+  writeStatus(resultStream, {message: 'Using dockerfile..', dockerfile, level: 'verbose'});
 
   if (!dockerfile) {
     logger.error('No suitable dockerfile found');
+    writeStatus(resultStream, {message: 'Error! No suitable dockerfile found!', level: 'error'});
     throw new Error('No suitable dockerfile found');
   }
 

src/docker/start.js

@@ -1,9 +1,9 @@
 // our modules
 const docker = require('./docker');
-const {getProjectConfig, baseNameFromImage, nameFromImage, projectFromConfig} = require('../util');
+const {getProjectConfig, baseNameFromImage, nameFromImage, projectFromConfig, writeStatus} = require('../util');
 const {getConfig} = require('../config');
 
-module.exports = async ({image, username}) => {
+module.exports = async ({image, username, resultStream}) => {
   const baseName = baseNameFromImage(image);
   const name = nameFromImage(image);
 
@@ -70,6 +70,8 @@ module.exports = async ({image, username}) => {
     containerConfig.Labels['traefik.frontend.rule'] = `Host:${host}`;
   }
 
+  writeStatus(resultStream, {message: 'Starting container with following config:', containerConfig, level: 'verbose'});
+
   // create container
   const container = await docker.createContainer(containerConfig);
 
@@ -84,5 +86,7 @@ module.exports = async ({image, username}) => {
   // start container
   await container.start();
 
+  writeStatus(resultStream, {message: 'Container successfully started!', level: 'verbose'});
+
   return container.inspect();
 };

src/routes/deploy.js

@@ -1,32 +1,36 @@
 // npm modules
+const _ = require('highland');
+const {Readable} = require('stream');
 
 // our modules
 const logger = require('../logger');
 const {hasCompose, updateCompose, executeCompose} = require('../docker/dockercompose');
 const generateDockerfile = require('../docker/dockerfile');
 const build = require('../docker/build');
 const start = require('../docker/start');
-const {sleep, cleanTemp, unpack, getProjectConfig, projectFromConfig} = require('../util');
+const {sleep, cleanTemp, unpack, getProjectConfig, projectFromConfig, writeStatus} = require('../util');
 const docker = require('../docker/docker');
 const {removeContainer} = require('../docker/util');
 
 // time to wait before removing old projects on update
 const WAIT_TIME = 5000;
 
 // deployment from unpacked files
-const deploy = async ({username}) => {
+const deploy = async ({username, resultStream}) => {
   // check if it's a docker-compose project
   if (hasCompose()) {
     // if it does - run compose workflow
     logger.debug('Docker-compose file found, executing compose workflow..');
+    writeStatus(resultStream, {message: 'Deploying docker-compose project..', level: 'info'});
 
     // update compose file with project params
     const composeConfig = updateCompose({username});
     logger.debug('Compose modified:', composeConfig);
+    writeStatus(resultStream, {message: 'Compose file modified', data: composeConfig, level: 'verbose'});
 
     // execute compose
-    const {log} = await executeCompose();
-    logger.debug('Compose executed:', log);
+    const exitCode = await executeCompose(resultStream);
+    logger.debug('Compose executed, exit code:', exitCode);
 
     // get container infos
     const allContainers = await docker.listContainers({all: true});
@@ -38,36 +42,43 @@ const deploy = async ({username}) => {
         .map(container => container.inspect())
     );
     // return them
-    return [{status: 'success', deployments}, 200];
+    writeStatus(resultStream, {message: 'Deployment success!', deployments, level: 'info'});
+    resultStream.end('');
+    return;
   }
 
   // generate dockerfile
-  generateDockerfile();
+  generateDockerfile(resultStream);
 
   // build docker image
   try {
-    const buildRes = await build({username});
-    logger.debug('build result:', buildRes);
+    const buildRes = await build({username, resultStream});
+    logger.debug('Build result:', buildRes);
 
     // check for errors in build log
     if (buildRes.log.map(it => it.toLowerCase()).some(it => it.includes('error') || it.includes('failed'))) {
-      logger.debug('build log conains error!');
-      return [{status: 'error', result: buildRes}, 400];
+      logger.debug('Build log conains error!');
+      writeStatus(resultStream, {message: 'Build log contains errors!', level: 'error'});
+      resultStream.end('');
+      return;
     }
 
     // start image
-    const containerInfo = await start(Object.assign(buildRes, {username}));
+    const containerInfo = await start(Object.assign({}, buildRes, {username, resultStream}));
     logger.debug(containerInfo.Name);
 
     // clean temp folder
     await cleanTemp();
 
     const containerData = docker.getContainer(containerInfo.Id);
     const container = await containerData.inspect();
-    return [{status: 'success', deployments: [container]}, 200];
+    // return new deployments
+    writeStatus(resultStream, {message: 'Deployment success!', deployments: [container], level: 'info'});
+    resultStream.end('');
   } catch (e) {
     logger.debug('build failed!', e);
-    return [{status: 'error', result: e}, 400];
+    writeStatus(resultStream, {message: e.error, error: e.error, log: e.log, level: 'error'});
+    resultStream.end('');
   }
 };
 
@@ -89,10 +100,12 @@ module.exports = server => {
       await cleanTemp();
       // unpack to temp folder
       await unpack(request.payload.path);
+      // create new highland stream for results
+      const resultStream = _();
       // run deploy
-      const [response, code] = await deploy({username});
-      // respond
-      reply(response).code(code);
+      deploy({username, resultStream});
+      // reply with deploy stream
+      reply(new Readable().wrap(resultStream)).code(200);
     },
   });
 
@@ -125,9 +138,12 @@ module.exports = server => {
         c => c.Labels['exoframe.user'] === username && c.Labels['exoframe.project'] === project
       );
 
+      // create new highland stream for results
+      const resultStream = _();
       // deploy new versions
-      // run deploy
-      const [response, code] = await deploy({username, payload: request.payload});
+      deploy({username, payload: request.payload, resultStream});
+      // reply with deploy stream
+      reply(new Readable().wrap(resultStream)).code(200);
       // wait a bit for it to start
       await sleep(WAIT_TIME);
 
@@ -140,9 +156,6 @@ module.exports = server => {
           logger.error('Error removing old deployment:', e);
         }
       }
-
-      // respond
-      reply(response).code(code);
     },
   });
 };

src/util/index.js

@@ -49,3 +49,5 @@ exports.projectFromConfig = ({username, config}) => {
 };
 
 exports.sleep = time => new Promise(r => setTimeout(r, time));
+
+exports.writeStatus = (stream, data) => stream.write(`${JSON.stringify(data)}\n`);