Flatten support for >200k row arrays (#166)

* flatten support for > 200k row arrays

Author: hdwatts

src/utils.js

@@ -3,7 +3,8 @@
 let path = require('doc-path'),
     constants = require('./constants.json');
 
-const dateStringRegex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
+const dateStringRegex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/,
+    MAX_ARRAY_LENGTH = 100000;
 
 module.exports = {
     isStringRepresentation,
@@ -285,6 +286,13 @@ function unique(array) {
 }
 
 function flatten(array) {
+    if (array.length > MAX_ARRAY_LENGTH) {
+        let safeArray = [];
+        for (let a = 0; a < array.length; a += MAX_ARRAY_LENGTH) {
+            safeArray = safeArray.concat(...array.slice(a, a + MAX_ARRAY_LENGTH));
+        }
+        return safeArray;
+    }
     return [].concat(...array);
 }
 

test/tests.js

@@ -1,5 +1,6 @@
 let json2csvTests = require('./json2csv'),
     csv2jsonTests = require('./csv2json'),
+    utilsTests = require('./utilsTests'),
     jsonTestData = require('./config/testJsonFilesList'),
     csvTestData = require('./config/testCsvFilesList');
 
@@ -10,4 +11,7 @@ describe('json-2-csv Node Module', function() {
 
     // Run CSV to JSON Tests
     csv2jsonTests.runTests(jsonTestData, csvTestData);
+
+    // Run Utils Tests
+    utilsTests.runTests();
 });

test/utilsTests.js

@@ -0,0 +1,36 @@
+'use strict';
+
+/* eslint no-unused-vars: ["error", { "varsIgnorePattern": "should" }]*/
+
+let should = require('should'),
+    utils = require('../src/utils');
+
+function runTests() {
+    describe('utils', () => {
+        describe('flatten()', () => {
+            it('should flatten a nested array', (done) => {
+                const nested = [[1], [2], [3], [4], [5]],
+                    expected = [1, 2, 3, 4, 5],
+                    result = utils.flatten(nested);
+
+                result.should.deepEqual(expected);
+                done();
+            });
+
+            it('should handle extremely large arrays', (done) => {
+                const nested = [],
+                    expected = [];
+                for (let a = 0; a < 234567; a++) {
+                    nested.push([a]);
+                    expected.push(a);
+                }
+
+                const result = utils.flatten(nested);
+                result.should.deepEqual(expected);
+                done();
+            });
+        });
+    });
+}
+
+module.exports = { runTests };