Increment schema to latest 5

html-oriented manifest format
-----------------------------

In the new schema format, which is defined in
ember-fastboot/fastboot@3fd5bc9
the manifest is written into HTML and later extracted by fastboot on
server side instead of previously reading from dist/package.json

Note:
The new schema in fastboot does not handle fastboot config
https://github.com/ember-fastboot/ember-cli-fastboot/tree/e4d0b7c7bcdf82def0dc8726835b49d707673f41#providing-additional-config

This commit changes to read Fastboot.config from dist/package.json
instead of ignoring it

Allow to require module path from whitelisted dependency
-------------------------------------------------------

Incrementing schema to 5 also included the changes in schema 4
strictWhitelist
See ember-fastboot/fastboot#200

Revert back to put config in dist/package.json

add data-fastboot-ignore to unexpected files

properly ignore files that should not execute in fastboot

Author: xg-wang

packages/ember-cli-fastboot/index.js

@@ -11,6 +11,7 @@ const chalk = require('chalk');
 
 const fastbootAppBoot = require('./lib/utilities/fastboot-app-boot');
 const FastBootConfig = require('./lib/broccoli/fastboot-config');
+const HTMLWriter = require('./lib/broccoli/html-writer');
 const fastbootAppFactoryModule = require('./lib/utilities/fastboot-app-factory-module');
 const migrateInitializers = require('./lib/build-utilities/migrate-initializers');
 const SilentError = require('silent-error');
@@ -219,6 +220,7 @@ module.exports = {
     return finalFastbootTree;
   },
 
+  // Note: this hook is ignored when built with embroider
   treeForPublic(tree) {
     let fastbootTree = this._getFastbootTree();
     let trees = [];
@@ -229,7 +231,7 @@ module.exports = {
 
     let newTree = new MergeTrees(trees);
 
-    let fastbootConfigTree = this._buildFastbootConfigTree(newTree);
+    let fastbootConfigTree = (this._fastbootConfigTree = this._buildFastbootConfigTree(newTree));
 
     // Merge the package.json with the existing tree
     return new MergeTrees([newTree, fastbootConfigTree], { overwrite: true });
@@ -306,6 +308,29 @@ module.exports = {
     });
   },
 
+  /**
+   * Write fastboot-script tags to the html file
+   */
+  postprocessTree(type, tree) {
+    this._super(...arguments);
+    if (type === 'all') {
+      let { fastbootConfig, appName, manifest } = this._fastbootConfigTree;
+      let fastbootHTMLTree = new HTMLWriter(tree, {
+        annotation: 'FastBoot HTML Writer',
+        fastbootConfig,
+        appName,
+        manifest,
+        appJsPath: this.app.options.outputPaths.app.js,
+        outputPaths: this.app.options.outputPaths,
+      });
+
+      // Merge the package.json with the existing tree
+      return new MergeTrees([tree, fastbootHTMLTree], { overwrite: true });
+    }
+
+    return tree;
+  },
+
   serverMiddleware(options) {
     let emberCliVersion = this._getEmberCliVersion();
     let app = options.app;

packages/ember-cli-fastboot/lib/broccoli/fastboot-config.js

@@ -1,17 +1,17 @@
 /* eslint-env node */
 'use strict';
 
-const fs     = require('fs');
-const fmt    = require('util').format;
-const uniq   = require('ember-cli-lodash-subset').uniq;
-const merge  = require('ember-cli-lodash-subset').merge;
+const fs = require('fs');
+const fmt = require('util').format;
+const uniq = require('ember-cli-lodash-subset').uniq;
+const merge = require('ember-cli-lodash-subset').merge;
 const md5Hex = require('md5-hex');
-const path   = require('path');
+const path = require('path');
 const Plugin = require('broccoli-plugin');
 
 const stringify = require('json-stable-stringify');
 
-const LATEST_SCHEMA_VERSION = 3;
+const LATEST_SCHEMA_VERSION = 5;
 
 module.exports = class FastBootConfig extends Plugin {
   constructor(inputNode, options) {
@@ -38,24 +38,28 @@ module.exports = class FastBootConfig extends Plugin {
       this.htmlFile = 'index.html';
     }
 
+    this.prepareConfig();
+    this.prepareDependencies();
   }
 
-
   /**
    * The main hook called by Broccoli Plugin. Used to build or
    * rebuild the tree. In this case, we generate the configuration
    * and write it to `package.json`.
    */
   build() {
-    this.buildConfig();
-    this.buildDependencies();
-    this.buildManifest();
     this.buildHostWhitelist();
-
     let outputPath = path.join(this.outputPath, 'package.json');
     this.writeFileIfContentChanged(outputPath, this.toJSONString());
   }
 
+  get manifest() {
+    if (!this._manifest) {
+      this._manifest = this.buildManifest();
+    }
+    return this._manifest;
+  }
+
   writeFileIfContentChanged(outputPath, content) {
     let previous = this._fileToChecksumMap[outputPath];
     let next = md5Hex(content);
@@ -66,11 +70,11 @@ module.exports = class FastBootConfig extends Plugin {
     }
   }
 
-  buildConfig() {
+  prepareConfig() {
     // we only walk the host app's addons to grab the config since ideally
     // addons that have dependency on other addons would never define
     // this advance hook.
-    this.project.addons.forEach((addon) => {
+    this.project.addons.forEach(addon => {
       if (addon.fastbootConfigTree) {
         let configFromAddon = addon.fastbootConfigTree();
 
@@ -83,7 +87,7 @@ module.exports = class FastBootConfig extends Plugin {
     });
   }
 
-  buildDependencies() {
+  prepareDependencies() {
     let dependencies = {};
     let moduleWhitelist = [];
     let ui = this.ui;
@@ -97,7 +101,10 @@ module.exports = class FastBootConfig extends Plugin {
 
           if (dep in dependencies) {
             version = dependencies[dep];
-            ui.writeLine(fmt("Duplicate FastBoot dependency %s. Versions may mismatch. Using range %s.", dep, version), ui.WARNING);
+            ui.writeLine(
+              fmt('Duplicate FastBoot dependency %s. Versions may mismatch. Using range %s.', dep, version),
+              ui.WARNING
+            );
             return;
           }
 
@@ -129,7 +136,7 @@ module.exports = class FastBootConfig extends Plugin {
   }
 
   updateFastBootManifest(manifest) {
-    this.project.addons.forEach(addon =>{
+    this.project.addons.forEach(addon => {
       if (addon.updateFastBootManifest) {
         manifest = addon.updateFastBootManifest(manifest);
 
@@ -157,7 +164,7 @@ module.exports = class FastBootConfig extends Plugin {
       htmlFile: this.htmlFile
     };
 
-    this.manifest = this.updateFastBootManifest(manifest);
+    return this.updateFastBootManifest(manifest);
   }
 
   buildHostWhitelist() {
@@ -167,17 +174,21 @@ module.exports = class FastBootConfig extends Plugin {
   }
 
   toJSONString() {
-    return stringify({
-      dependencies: this.dependencies,
-      fastboot: {
-        moduleWhitelist: this.moduleWhitelist,
-        schemaVersion: LATEST_SCHEMA_VERSION,
-        manifest: this.manifest,
-        hostWhitelist: this.normalizeHostWhitelist(),
-        config: this.fastbootConfig,
-        appName: this.appName,
-      }
-    }, null, 2);
+    return stringify(
+      {
+        name: this.appName,
+        dependencies: this.dependencies,
+        fastboot: {
+          moduleWhitelist: this.moduleWhitelist,
+          schemaVersion: LATEST_SCHEMA_VERSION,
+          hostWhitelist: this.normalizeHostWhitelist(),
+          config: this.fastbootConfig,
+          htmlEntrypoint: this.manifest.htmlFile
+        }
+      },
+      null,
+      2
+    );
   }
 
   normalizeHostWhitelist() {
@@ -194,7 +205,7 @@ module.exports = class FastBootConfig extends Plugin {
       }
     });
   }
-}
+};
 
 function eachAddonPackage(project, cb) {
   project.addons.map(addon => cb(addon.pkg));
@@ -207,7 +218,11 @@ function getFastBootDependencies(pkg) {
   }
 
   if (addon.fastBootDependencies) {
-    throw new SilentError('ember-addon.fastBootDependencies has been replaced with ember-addon.fastbootDependencies [addon: ' + pkg.name + ']')
+    throw new SilentError(
+      'ember-addon.fastBootDependencies has been replaced with ember-addon.fastbootDependencies [addon: ' +
+        pkg.name +
+        ']'
+    );
   }
 
   return addon.fastbootDependencies;

packages/ember-cli-fastboot/lib/broccoli/html-writer.js

@@ -0,0 +1,123 @@
+'use strict';
+
+const Filter = require('broccoli-persistent-filter');
+const { JSDOM } = require('jsdom');
+
+module.exports = class BasePageWriter extends Filter {
+  constructor(inputNodes, { annotation, fastbootConfig, appName, manifest, outputPaths }) {
+    super(inputNodes, {
+      annotation,
+      extensions: ['html'],
+      targetExtension: 'html',
+    });
+    this._manifest = manifest;
+    this._rootURL = getRootURL(fastbootConfig, appName);
+    this._appJsPath = outputPaths.app.js;
+    this._expectedFiles = expectedFiles(outputPaths);
+  }
+
+  getDestFilePath() {
+    let filteredRelativePath = super.getDestFilePath(...arguments);
+
+    return filteredRelativePath === this._manifest.htmlFile ? filteredRelativePath : null;
+  }
+
+  processString(content) {
+    let dom = new JSDOM(content);
+    let scriptTags = dom.window.document.querySelectorAll('script');
+
+    this._ignoreUnexpectedScripts(scriptTags);
+
+    let fastbootScripts = this._findFastbootScriptToInsert(scriptTags);
+    let appJsTag = findAppJsTag(scriptTags, this._appJsPath, this._rootURL);
+    insertFastbootScriptsBeforeAppJsTags(fastbootScripts, appJsTag);
+
+    return dom.serialize();
+  }
+
+  _findFastbootScriptToInsert(scriptTags) {
+    let rootURL = this._rootURL;
+    let scriptSrcs = [];
+    for (let element of scriptTags) {
+      scriptSrcs.push(urlWithin(element.getAttribute('src'), rootURL));
+    }
+
+    return this._manifest.vendorFiles
+      .concat(this._manifest.appFiles)
+      .map(src => urlWithin(src, rootURL))
+      .filter(src => !scriptSrcs.includes(src));
+  }
+
+  _ignoreUnexpectedScripts(scriptTags) {
+    let expectedFiles = this._expectedFiles;
+    let rootURL = this._rootURL;
+    for (let element of scriptTags) {
+      if (!expectedFiles.includes(urlWithin(element.getAttribute('src'), rootURL))) {
+        element.setAttribute('data-fastboot-ignore', '');
+      }
+    }
+  }
+};
+
+function expectedFiles(outputPaths) {
+  function stripLeadingSlash(filePath) {
+    return filePath.replace(/^\//, '');
+  }
+
+  let appFilePath = stripLeadingSlash(outputPaths.app.js);
+  let appFastbootFilePath = appFilePath.replace(/\.js$/, '') + '-fastboot.js';
+  let vendorFilePath = stripLeadingSlash(outputPaths.vendor.js);
+  return [appFilePath, appFastbootFilePath, vendorFilePath];
+}
+
+function getRootURL(appName, config) {
+  let rootURL = (config[appName] && config[appName].rootURL) || '/';
+  if (!rootURL.endsWith('/')) {
+    rootURL = rootURL + '/';
+  }
+  return rootURL;
+}
+
+function urlWithin(candidate, root) {
+  let candidateURL = new URL(candidate, 'http://_the_current_origin_');
+  let rootURL = new URL(root, 'http://_the_current_origin_');
+  if (candidateURL.href.startsWith(rootURL.href)) {
+    return candidateURL.href.slice(rootURL.href.length);
+  }
+}
+
+function findAppJsTag(scriptTags, appJsPath, rootURL) {
+  appJsPath = urlWithin(appJsPath, rootURL);
+  for (let e of scriptTags) {
+    if (urlWithin(e.getAttribute('src'), rootURL) === appJsPath) {
+      return e;
+    }
+  }
+}
+
+function insertFastbootScriptsBeforeAppJsTags(fastbootScripts, appJsTag) {
+  let range = new NodeRange(appJsTag);
+
+  for (let src of fastbootScripts) {
+    range.insertAsScriptTag(src);
+  }
+}
+
+class NodeRange {
+  constructor(initial) {
+    this.start = initial.ownerDocument.createTextNode('');
+    initial.parentElement.insertBefore(this.start, initial);
+    this.end = initial;
+  }
+
+  insertAsScriptTag(src) {
+    let newTag = this.end.ownerDocument.createElement('fastboot-script');
+    newTag.setAttribute('src', src);
+    this.insertNode(newTag);
+    this.insertNode(this.end.ownerDocument.createTextNode('\n'));
+  }
+
+  insertNode(node) {
+    this.end.parentElement.insertBefore(node, this.end);
+  }
+}

packages/ember-cli-fastboot/package.json

@@ -43,6 +43,7 @@
     "fastboot-express-middleware": "3.2.0-beta.2",
     "fastboot-transform": "^0.1.3",
     "fs-extra": "^7.0.0",
+    "jsdom": "^16.2.2",
     "json-stable-stringify": "^1.0.1",
     "md5-hex": "^2.0.0",
     "recast": "^0.19.1",

packages/fastboot/src/fastboot-schema.js

@@ -79,7 +79,8 @@ function loadConfig(distPath) {
     ({ appName, config, html, scripts } = loadManifest(distPath, pkg.fastboot, schemaVersion));
   } else {
     appName = pkg.name;
-    ({ config, html, scripts } = htmlEntrypoint(appName, distPath, pkg.fastboot.htmlEntrypoint));
+    config = pkg.fastboot.config;
+    ({ html, scripts } = htmlEntrypoint(appName, distPath, pkg.fastboot.htmlEntrypoint, config));
   }
 
   let sandboxRequire = buildWhitelistedRequire(

packages/fastboot/src/html-entrypoint.js

@@ -4,21 +4,11 @@ const { JSDOM } = require('jsdom');
 const fs = require('fs');
 const path = require('path');
 
-function htmlEntrypoint(appName, distPath, htmlPath) {
+function htmlEntrypoint(appName, distPath, htmlPath, config = {}) {
   let html = fs.readFileSync(path.join(distPath, htmlPath), 'utf8');
   let dom = new JSDOM(html);
   let scripts = [];
 
-  let config = {};
-  for (let element of dom.window.document.querySelectorAll('meta')) {
-    let name = element.getAttribute('name');
-    if (name && name.endsWith('/config/environment')) {
-      let content = JSON.parse(decodeURIComponent(element.getAttribute('content')));
-      content.APP = Object.assign({ autoboot: false }, content.APP);
-      config[name.slice(0, -1 * '/config/environment'.length)] = content;
-    }
-  }
-
   let rootURL = getRootURL(appName, config);
 
   for (let element of dom.window.document.querySelectorAll('script,fastboot-script')) {
@@ -34,7 +24,7 @@ function htmlEntrypoint(appName, distPath, htmlPath) {
     }
   }
 
-  return { config, html: dom.serialize(), scripts };
+  return { html: dom.serialize(), scripts };
 }
 
 function extractSrc(element) {