Add performance tests

.circleci/config.yml

@@ -33,6 +33,38 @@ jobs:
           paths:
             - plotly.js
 
+  performance-jasmine:
+    docker:
+      # need '-browsers' version to test in real (xvfb-wrapped) browsers
+      - image: cimg/node:18.20.4-browsers
+    environment:
+      # Alaska time (arbitrary timezone to test date logic)
+      TZ: "America/Anchorage"
+    working_directory: ~/plotly.js
+    steps:
+      - run: sudo apt-get update
+      - browser-tools/install-browser-tools:
+          install-firefox: false
+          install-geckodriver: false
+          install-chrome: true
+          chrome-version: "132.0.6834.110"
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Run performance tests
+          command: .circleci/test.sh performance-jasmine
+      - run:
+          name: Display system information
+          command: npm run system-info > ~/Downloads/system_info.txt
+      - run:
+          name: Combine CSV files
+          command: |
+            head -n 1 `ls ~/Downloads/*.csv | head -n 1` > ~/Downloads/all.csv
+            tail -n+2 -q ~/Downloads/*.csv >> ~/Downloads/all.csv
+      - store_artifacts:
+          path: ~/Downloads
+          destination: /
+
   timezone-jasmine:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
@@ -500,6 +532,9 @@ workflows:
       - bundle-jasmine:
           requires:
             - install-and-cibuild
+      - performance-jasmine:
+          requires:
+            - install-and-cibuild
       - mathjax-firefoxLatest:
           requires:
             - install-and-cibuild

.circleci/test.sh

@@ -79,6 +79,11 @@ case $1 in
         exit $EXIT_STATE
         ;;
 
+    performance-jasmine)
+        npm run test-performance || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
     mathjax-firefox)
         ./node_modules/karma/bin/karma start test/jasmine/karma.conf.js --FF --bundleTest=mathjax --nowatch || EXIT_STATE=$?
         exit $EXIT_STATE

package.json

@@ -47,6 +47,8 @@
     "test-export": "node test/image/export_test.js",
     "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output",
     "test-bundle": "node tasks/test_bundle.js",
+    "test-performance": "node tasks/test_performance.js",
+    "system-info": "node tasks/system_info.js",
     "test-plain-obj": "node tasks/test_plain_obj.mjs",
     "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
     "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.mjs",

tasks/system_info.js

@@ -0,0 +1,77 @@
+var os = require('os');
+
+var logs = [];
+function addLog(str) {
+    logs.push(str)
+}
+
+var systemInfo = {
+    platform: os.platform(),
+    type: os.type(),
+    arch: os.arch(),
+    release: os.release(),
+    version: os.version ? os.version() : 'Unknown',
+    hostname: os.hostname(),
+    homedir: os.homedir(),
+    tmpdir: os.tmpdir(),
+    endianness: os.endianness(),
+};
+
+addLog('💻 SYSTEM:');
+addLog(`   Platform: ${systemInfo.platform}`);
+addLog(`   Type: ${systemInfo.type}`);
+addLog(`   Architecture: ${systemInfo.arch}`);
+addLog(`   Release: ${systemInfo.release}`);
+addLog(`   Hostname: ${systemInfo.hostname}`);
+
+
+var cpus = os.cpus();
+var loadAvg = os.loadavg();
+
+var cpuInfo = {
+    model: cpus[0].model,
+    speed: cpus[0].speed,
+    cores: cpus.length,
+    loadAverage: loadAvg,
+    cpuDetails: cpus
+};
+
+addLog('');
+addLog('🔧 CPU:');
+addLog(`   Model: ${cpuInfo.model}`);
+addLog(`   Speed: ${cpuInfo.speed} MHz`);
+addLog(`   Cores: ${cpuInfo.cores}${cpuInfo.physicalCores ? ` (${cpuInfo.physicalCores} physical)` : ''}`);
+addLog(`   Load Average: ${loadAvg.map(load => load.toFixed(2)).join(', ')}`);
+
+
+var totalMem = os.totalmem();
+var freeMem = os.freemem();
+var usedMem = totalMem - freeMem;
+
+var memoryInfo = {
+    total: totalMem,
+    free: freeMem,
+    used: usedMem,
+    usagePercent: (usedMem / totalMem) * 100
+};
+
+function formatBytes(bytes, decimals = 2) {
+    if (bytes === 0) return '0 Bytes';
+    if (!bytes) return 'Unknown';
+
+    var k = 1024;
+    var dm = decimals < 0 ? 0 : decimals;
+    var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB'];
+    var i = Math.floor(Math.log(bytes) / Math.log(k));
+
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+}
+
+addLog('');
+addLog('💾 MEMORY:');
+addLog(`   Total: ${formatBytes(memoryInfo.total)}`);
+addLog(`   Used: ${formatBytes(memoryInfo.used)} (${memoryInfo.usagePercent.toFixed(1)}%)`);
+addLog(`   Free: ${formatBytes(memoryInfo.free)}`);
+
+
+console.log(logs.join('\n'));

tasks/test_performance.js

@@ -0,0 +1,110 @@
+var fs = require('fs');
+var path = require('path');
+var exec = require('child_process').exec;
+var { glob } = require('glob');
+var runSeries = require('run-series');
+
+var constants = require('./util/constants');
+var pathToJasminePerformanceTests = constants.pathToJasminePerformanceTests;
+var pathToRoot = constants.pathToRoot;
+
+/**
+ * Run all jasmine 'performance' test in series
+ *
+ * To run specific performance tests, use
+ *
+ * $ npm run test-jasmine -- --performanceTest=<name-of-suite>
+ */
+
+var testCases = require('../test/jasmine/performance_tests/assets/test_cases').testCases;
+
+glob(pathToJasminePerformanceTests + '/*.js').then(function(files) {
+    var tasks = [];
+    for(let file of files) {
+        for(let testCase of testCases) {
+            tasks.push(function(cb) {
+                var cmd = [
+                    'karma', 'start',
+                    path.join(constants.pathToRoot, 'test', 'jasmine', 'karma.conf.js'),
+                    '--performanceTest=' + path.basename(file),
+                    '--nowatch',
+                    '--tracesType=' + testCase.traceType,
+                    '--tracesMode=' + testCase.mode,
+                    '--tracesCount=' + testCase.nTraces,
+                    '--tracesPoints=' + testCase.n,
+                ].join(' ');
+
+                console.log('Running: ' + cmd);
+
+                exec(cmd, function(err) {
+                    cb(null, err);
+                }).stdout.pipe(process.stdout);
+            });
+        }
+    }
+
+    runSeries(tasks, function(err, results) {
+        var failed = results.filter(function(r) { return r; });
+
+        if(failed.length) {
+            console.log('\ntest-performance summary:');
+            failed.forEach(function(r) { console.warn('- ' + r.cmd + ' failed'); });
+            console.log('');
+
+            // Create CSV file for failed cases
+            var str = [
+                'number of traces',
+                'chart type & mode',
+                'data points',
+                'run id',
+                'rendering time(ms)'
+            ].join(',') + '\n';
+
+            failed.forEach(function(r) {
+                // split command string frist by space then by equal to get
+                var cmdArgs = r.cmd.split(' ').map(part => {
+                    return part.split('=');
+                });
+
+                var test = {};
+
+                for(var i = 0; i < cmdArgs.length; i++) {
+                    if('--tracesCount' === cmdArgs[i][0]) {
+                        test.nTraces = cmdArgs[i][1];
+                    }
+                }
+
+                for(var i = 0; i < cmdArgs.length; i++) {
+                    if('--tracesType' === cmdArgs[i][0]) {
+                        test.traceType = cmdArgs[i][1];
+                    }
+                }
+
+                for(var i = 0; i < cmdArgs.length; i++) {
+                    if('--tracesMode' === cmdArgs[i][0]) {
+                        test.mode = cmdArgs[i][1];
+                    }
+                }
+
+                for(var i = 0; i < cmdArgs.length; i++) {
+                    if('--tracesPoints' === cmdArgs[i][0]) {
+                        test.n = cmdArgs[i][1];
+                    }
+                }
+
+                str += [
+                    (test.nTraces || 1),
+                    (test.traceType + (test.mode ? ' ' + test.mode : '')),
+                    test.n,
+                    'failed',
+                    ''
+                ].join(',') + '\n';
+            });
+
+            var failedCSV = pathToRoot + '../Downloads/failed.csv';
+            console.log('Saving:', failedCSV)
+            console.log(str);
+            fs.writeFileSync(failedCSV, str);
+        }
+    });
+});

tasks/util/constants.js

@@ -225,6 +225,7 @@ module.exports = {
 
     pathToJasmineTests: path.join(pathToRoot, 'test/jasmine/tests'),
     pathToJasmineBundleTests: path.join(pathToRoot, 'test/jasmine/bundle_tests'),
+    pathToJasminePerformanceTests: path.join(pathToRoot, 'test/jasmine/performance_tests'),
 
     // this mapbox access token is 'public', no need to hide it
     // more info: https://www.mapbox.com/help/define-access-token/

test/jasmine/karma.conf.js

@@ -8,7 +8,7 @@ var esbuildConfig = require('../../esbuild-config.js');
 var isCI = Boolean(process.env.CI);
 
 var argv = minimist(process.argv.slice(4), {
-    string: ['bundleTest', 'width', 'height'],
+    string: ['bundleTest', 'performanceTest', 'width', 'height'],
     boolean: [
         'mathjax3',
         'info',
@@ -21,6 +21,7 @@ var argv = minimist(process.argv.slice(4), {
         Chrome: 'chrome',
         Firefox: ['firefox', 'FF'],
         bundleTest: ['bundletest', 'bundle_test'],
+        performanceTest: ['performancetest', 'performance_test'],
         nowatch: 'no-watch',
         failFast: 'fail-fast',
     },
@@ -53,7 +54,8 @@ if(argv.info) {
         '  - All non-flagged arguments corresponds to the test suites in `test/jasmine/tests/` to be run.',
         '    No need to add the `_test.js` suffix, we expand them correctly here.',
         '  - `--bundleTest` set the bundle test suite `test/jasmine/bundle_tests/ to be run.',
-        '    Note that only one bundle test can be run at a time.',
+        '  - `--performanceTest` set the bundle test suite `test/jasmine/performance_tests/ to be run.',
+        '    Note that only one bundle/performance test can be run at a time.',
         '  - Use `--tags` to specify which `@` tags to test (if any) e.g `npm run test-jasmine -- --tags=gl`',
         '    will run only gl tests.',
         '  - Use `--skip-tags` to specify which `@` tags to skip (if any) e.g `npm run test-jasmine -- --skip-tags=gl`',
@@ -100,7 +102,8 @@ var glob = function(_) {
 };
 
 var isBundleTest = !!argv.bundleTest;
-var isFullSuite = !isBundleTest && argv._.length === 0;
+var isPerformanceTest = !!argv.performanceTest;
+var isFullSuite = !(isBundleTest || isPerformanceTest) && argv._.length === 0;
 var testFileGlob;
 
 if(isFullSuite) {
@@ -113,6 +116,14 @@ if(isFullSuite) {
     }
 
     testFileGlob = path.join(__dirname, 'bundle_tests', glob([basename(_[0])]));
+} else if(isPerformanceTest) {
+    var _ = merge(argv.performanceTest);
+
+    if(_.length > 1) {
+        console.warn('Can only run one performance test suite at a time, ignoring ', _.slice(1));
+    }
+
+    testFileGlob = path.join(__dirname, 'performance_tests', glob([basename(_[0])]));
 } else {
     testFileGlob = path.join(__dirname, 'tests', glob(merge(argv._).map(basename)));
 }
@@ -145,7 +156,7 @@ var hasSpecReporter = reporters.indexOf('spec') !== -1;
 if(!hasSpecReporter && argv.showSkipped) reporters.push('spec');
 if(argv.verbose) reporters.push('verbose');
 
-function func(config) {
+var func = function(config) {
     // level of logging
     // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
     //
@@ -161,8 +172,18 @@ function func(config) {
         level: 'debug'
     };
 
+    if(isPerformanceTest) {
+        func.defaultConfig.client = func.defaultConfig.client || {};
+        func.defaultConfig.client.testCase = {
+            tracesType: config.tracesType,
+            tracesMode: config.tracesMode,
+            tracesCount: config.tracesCount,
+            tracesPoints: config.tracesPoints,
+        };
+    }
+
     config.set(func.defaultConfig);
-}
+};
 
 func.defaultConfig = {
 
@@ -250,7 +271,7 @@ func.defaultConfig = {
                 '--touch-events',
                 '--window-size=' + argv.width + ',' + argv.height,
                 isCI ? '--ignore-gpu-blacklist' : '',
-                (isBundleTest && basename(testFileGlob) === 'no_webgl') ? '--disable-webgl' : ''
+                ((isBundleTest || isPerformanceTest) && basename(testFileGlob) === 'no_webgl') ? '--disable-webgl' : ''
             ]
         },
         _Firefox: {