diff --git a/.circleci/config.yml b/.circleci/config.yml index b5d94298735..38ce2d5842a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,6 +33,38 @@ jobs: paths: - plotly.js + performance-jasmine: + docker: + # need '-browsers' version to test in real (xvfb-wrapped) browsers + - image: cimg/node:18.20.4-browsers + environment: + # Alaska time (arbitrary timezone to test date logic) + TZ: "America/Anchorage" + working_directory: ~/plotly.js + steps: + - run: sudo apt-get update + - browser-tools/install-browser-tools: + install-firefox: false + install-geckodriver: false + install-chrome: true + chrome-version: "132.0.6834.110" + - attach_workspace: + at: ~/ + - run: + name: Run performance tests + command: .circleci/test.sh performance-jasmine + - run: + name: Display system information + command: npm run system-info > ~/Downloads/system_info.txt + - run: + name: Combine CSV files + command: | + head -n 1 `ls ~/Downloads/*.csv | head -n 1` > ~/Downloads/all.csv + tail -n+2 -q ~/Downloads/*.csv >> ~/Downloads/all.csv + - store_artifacts: + path: ~/Downloads + destination: / + timezone-jasmine: docker: # need '-browsers' version to test in real (xvfb-wrapped) browsers @@ -500,6 +532,9 @@ workflows: - bundle-jasmine: requires: - install-and-cibuild + - performance-jasmine: + requires: + - install-and-cibuild - mathjax-firefoxLatest: requires: - install-and-cibuild diff --git a/.circleci/test.sh b/.circleci/test.sh index b814c1dd3e8..4df91e8af8c 100755 --- a/.circleci/test.sh +++ b/.circleci/test.sh @@ -79,6 +79,11 @@ case $1 in exit $EXIT_STATE ;; + performance-jasmine) + npm run test-performance || EXIT_STATE=$? + exit $EXIT_STATE + ;; + mathjax-firefox) ./node_modules/karma/bin/karma start test/jasmine/karma.conf.js --FF --bundleTest=mathjax --nowatch || EXIT_STATE=$? exit $EXIT_STATE diff --git a/package.json b/package.json index 39f76f1a5dd..9d5930a4030 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,8 @@ "test-export": "node test/image/export_test.js", "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output", "test-bundle": "node tasks/test_bundle.js", + "test-performance": "node tasks/test_performance.js", + "system-info": "node tasks/system_info.js", "test-plain-obj": "node tasks/test_plain_obj.mjs", "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint", "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.mjs", diff --git a/tasks/system_info.js b/tasks/system_info.js new file mode 100644 index 00000000000..fd09c670100 --- /dev/null +++ b/tasks/system_info.js @@ -0,0 +1,77 @@ +var os = require('os'); + +var logs = []; +function addLog(str) { + logs.push(str) +} + +var systemInfo = { + platform: os.platform(), + type: os.type(), + arch: os.arch(), + release: os.release(), + version: os.version ? os.version() : 'Unknown', + hostname: os.hostname(), + homedir: os.homedir(), + tmpdir: os.tmpdir(), + endianness: os.endianness(), +}; + +addLog('💻 SYSTEM:'); +addLog(` Platform: ${systemInfo.platform}`); +addLog(` Type: ${systemInfo.type}`); +addLog(` Architecture: ${systemInfo.arch}`); +addLog(` Release: ${systemInfo.release}`); +addLog(` Hostname: ${systemInfo.hostname}`); + + +var cpus = os.cpus(); +var loadAvg = os.loadavg(); + +var cpuInfo = { + model: cpus[0].model, + speed: cpus[0].speed, + cores: cpus.length, + loadAverage: loadAvg, + cpuDetails: cpus +}; + +addLog(''); +addLog('🔧 CPU:'); +addLog(` Model: ${cpuInfo.model}`); +addLog(` Speed: ${cpuInfo.speed} MHz`); +addLog(` Cores: ${cpuInfo.cores}${cpuInfo.physicalCores ? ` (${cpuInfo.physicalCores} physical)` : ''}`); +addLog(` Load Average: ${loadAvg.map(load => load.toFixed(2)).join(', ')}`); + + +var totalMem = os.totalmem(); +var freeMem = os.freemem(); +var usedMem = totalMem - freeMem; + +var memoryInfo = { + total: totalMem, + free: freeMem, + used: usedMem, + usagePercent: (usedMem / totalMem) * 100 +}; + +function formatBytes(bytes, decimals = 2) { + if (bytes === 0) return '0 Bytes'; + if (!bytes) return 'Unknown'; + + var k = 1024; + var dm = decimals < 0 ? 0 : decimals; + var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; +} + +addLog(''); +addLog('💾 MEMORY:'); +addLog(` Total: ${formatBytes(memoryInfo.total)}`); +addLog(` Used: ${formatBytes(memoryInfo.used)} (${memoryInfo.usagePercent.toFixed(1)}%)`); +addLog(` Free: ${formatBytes(memoryInfo.free)}`); + + +console.log(logs.join('\n')); \ No newline at end of file diff --git a/tasks/test_performance.js b/tasks/test_performance.js new file mode 100644 index 00000000000..c029ae5fc61 --- /dev/null +++ b/tasks/test_performance.js @@ -0,0 +1,110 @@ +var fs = require('fs'); +var path = require('path'); +var exec = require('child_process').exec; +var { glob } = require('glob'); +var runSeries = require('run-series'); + +var constants = require('./util/constants'); +var pathToJasminePerformanceTests = constants.pathToJasminePerformanceTests; +var pathToRoot = constants.pathToRoot; + +/** + * Run all jasmine 'performance' test in series + * + * To run specific performance tests, use + * + * $ npm run test-jasmine -- --performanceTest= + */ + +var testCases = require('../test/jasmine/performance_tests/assets/test_cases').testCases; + +glob(pathToJasminePerformanceTests + '/*.js').then(function(files) { + var tasks = []; + for(let file of files) { + for(let testCase of testCases) { + tasks.push(function(cb) { + var cmd = [ + 'karma', 'start', + path.join(constants.pathToRoot, 'test', 'jasmine', 'karma.conf.js'), + '--performanceTest=' + path.basename(file), + '--nowatch', + '--tracesType=' + testCase.traceType, + '--tracesMode=' + testCase.mode, + '--tracesCount=' + testCase.nTraces, + '--tracesPoints=' + testCase.n, + ].join(' '); + + console.log('Running: ' + cmd); + + exec(cmd, function(err) { + cb(null, err); + }).stdout.pipe(process.stdout); + }); + } + } + + runSeries(tasks, function(err, results) { + var failed = results.filter(function(r) { return r; }); + + if(failed.length) { + console.log('\ntest-performance summary:'); + failed.forEach(function(r) { console.warn('- ' + r.cmd + ' failed'); }); + console.log(''); + + // Create CSV file for failed cases + var str = [ + 'number of traces', + 'chart type & mode', + 'data points', + 'run id', + 'rendering time(ms)' + ].join(',') + '\n'; + + failed.forEach(function(r) { + // split command string frist by space then by equal to get + var cmdArgs = r.cmd.split(' ').map(part => { + return part.split('='); + }); + + var test = {}; + + for(var i = 0; i < cmdArgs.length; i++) { + if('--tracesCount' === cmdArgs[i][0]) { + test.nTraces = cmdArgs[i][1]; + } + } + + for(var i = 0; i < cmdArgs.length; i++) { + if('--tracesType' === cmdArgs[i][0]) { + test.traceType = cmdArgs[i][1]; + } + } + + for(var i = 0; i < cmdArgs.length; i++) { + if('--tracesMode' === cmdArgs[i][0]) { + test.mode = cmdArgs[i][1]; + } + } + + for(var i = 0; i < cmdArgs.length; i++) { + if('--tracesPoints' === cmdArgs[i][0]) { + test.n = cmdArgs[i][1]; + } + } + + str += [ + (test.nTraces || 1), + (test.traceType + (test.mode ? ' ' + test.mode : '')), + test.n, + 'failed', + '' + ].join(',') + '\n'; + }); + + var failedCSV = pathToRoot + '../Downloads/failed.csv'; + console.log('Saving:', failedCSV) + console.log(str); + fs.writeFileSync(failedCSV, str); + } + }); +}); diff --git a/tasks/util/constants.js b/tasks/util/constants.js index 6442501d9aa..69c1e88d64d 100644 --- a/tasks/util/constants.js +++ b/tasks/util/constants.js @@ -225,6 +225,7 @@ module.exports = { pathToJasmineTests: path.join(pathToRoot, 'test/jasmine/tests'), pathToJasmineBundleTests: path.join(pathToRoot, 'test/jasmine/bundle_tests'), + pathToJasminePerformanceTests: path.join(pathToRoot, 'test/jasmine/performance_tests'), // this mapbox access token is 'public', no need to hide it // more info: https://www.mapbox.com/help/define-access-token/ diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js index f66235d7593..7f24674cd7a 100644 --- a/test/jasmine/karma.conf.js +++ b/test/jasmine/karma.conf.js @@ -8,7 +8,7 @@ var esbuildConfig = require('../../esbuild-config.js'); var isCI = Boolean(process.env.CI); var argv = minimist(process.argv.slice(4), { - string: ['bundleTest', 'width', 'height'], + string: ['bundleTest', 'performanceTest', 'width', 'height'], boolean: [ 'mathjax3', 'info', @@ -21,6 +21,7 @@ var argv = minimist(process.argv.slice(4), { Chrome: 'chrome', Firefox: ['firefox', 'FF'], bundleTest: ['bundletest', 'bundle_test'], + performanceTest: ['performancetest', 'performance_test'], nowatch: 'no-watch', failFast: 'fail-fast', }, @@ -53,7 +54,8 @@ if(argv.info) { ' - All non-flagged arguments corresponds to the test suites in `test/jasmine/tests/` to be run.', ' No need to add the `_test.js` suffix, we expand them correctly here.', ' - `--bundleTest` set the bundle test suite `test/jasmine/bundle_tests/ to be run.', - ' Note that only one bundle test can be run at a time.', + ' - `--performanceTest` set the bundle test suite `test/jasmine/performance_tests/ to be run.', + ' Note that only one bundle/performance test can be run at a time.', ' - Use `--tags` to specify which `@` tags to test (if any) e.g `npm run test-jasmine -- --tags=gl`', ' will run only gl tests.', ' - Use `--skip-tags` to specify which `@` tags to skip (if any) e.g `npm run test-jasmine -- --skip-tags=gl`', @@ -100,7 +102,8 @@ var glob = function(_) { }; var isBundleTest = !!argv.bundleTest; -var isFullSuite = !isBundleTest && argv._.length === 0; +var isPerformanceTest = !!argv.performanceTest; +var isFullSuite = !(isBundleTest || isPerformanceTest) && argv._.length === 0; var testFileGlob; if(isFullSuite) { @@ -113,6 +116,14 @@ if(isFullSuite) { } testFileGlob = path.join(__dirname, 'bundle_tests', glob([basename(_[0])])); +} else if(isPerformanceTest) { + var _ = merge(argv.performanceTest); + + if(_.length > 1) { + console.warn('Can only run one performance test suite at a time, ignoring ', _.slice(1)); + } + + testFileGlob = path.join(__dirname, 'performance_tests', glob([basename(_[0])])); } else { testFileGlob = path.join(__dirname, 'tests', glob(merge(argv._).map(basename))); } @@ -145,7 +156,7 @@ var hasSpecReporter = reporters.indexOf('spec') !== -1; if(!hasSpecReporter && argv.showSkipped) reporters.push('spec'); if(argv.verbose) reporters.push('verbose'); -function func(config) { +var func = function(config) { // level of logging // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG // @@ -161,8 +172,18 @@ function func(config) { level: 'debug' }; + if(isPerformanceTest) { + func.defaultConfig.client = func.defaultConfig.client || {}; + func.defaultConfig.client.testCase = { + tracesType: config.tracesType, + tracesMode: config.tracesMode, + tracesCount: config.tracesCount, + tracesPoints: config.tracesPoints, + }; + } + config.set(func.defaultConfig); -} +}; func.defaultConfig = { @@ -250,7 +271,7 @@ func.defaultConfig = { '--touch-events', '--window-size=' + argv.width + ',' + argv.height, isCI ? '--ignore-gpu-blacklist' : '', - (isBundleTest && basename(testFileGlob) === 'no_webgl') ? '--disable-webgl' : '' + ((isBundleTest || isPerformanceTest) && basename(testFileGlob) === 'no_webgl') ? '--disable-webgl' : '' ] }, _Firefox: { diff --git a/test/jasmine/performance_tests/all_test.js b/test/jasmine/performance_tests/all_test.js new file mode 100644 index 00000000000..f61039b0a6a --- /dev/null +++ b/test/jasmine/performance_tests/all_test.js @@ -0,0 +1,296 @@ +var createGraphDiv = require('../assets/create_graph_div'); +var delay = require('../assets/delay'); +var d3SelectAll = require('../../strict-d3').selectAll; +var Plotly = require('../../../lib/index'); +var downloadCSV = require('./assets/post_process').downloadCSV; +var tests = require('./assets/test_cases').testCases; +var nSamples = require('./assets/constants').nSamples; +var MAX_RENDERING_TIME = 4000; + +var gd = createGraphDiv(); + +const samples = Array.from({ length: nSamples }, (_, i) => i); + + +function generateMock(spec) { + var type = spec.traceType; + return ( + (type === 'image') ? makeImage(spec) : + (type === 'heatmap' || type === 'contour') ? makeHeatmap(spec) : + (type === 'box' || type === 'violin') ? makeBox(spec) : + (type === 'bar' || type === 'histogram') ? makeBar(spec) : + (type === 'scatter' || type === 'scattergl') ? makeScatter(spec) : + (type === 'scattergeo') ? makeScatterGeo(spec) : + {} + ); +} + + +function makeImage(spec) { + var A = spec.nx; + var B = spec.ny; + + var x = Array.from({ length: A }, (_, i) => i); + var y = Array.from({ length: B }, (_, i) => i); + var z = []; + for(var k = 0; k < B ; k++) { + z[k] = []; + for(var i = 0; i < A ; i++) { + z[k][i] = [ + Math.floor(127 * (1 + Math.cos(Math.sqrt(i)))), + 0, + Math.floor(127 * (1 + Math.cos(Math.sqrt(k)))), + ]; + } + } + + return { + data: [{ + type: 'image', + x: x, + y: y, + z: z + }], + layout: { + width: 900, + height: 400 + } + }; +} + +function makeHeatmap(spec) { + var A = spec.nx; + var B = spec.ny; + + var x = Array.from({ length: A }, (_, i) => i); + var y = Array.from({ length: B }, (_, i) => i); + var z = []; + for(var k = 0; k < B ; k++) { + z[k] = Array.from({ length: A }, (_, i) => k * Math.cos(Math.sqrt(i))); + } + + return { + data: [{ + type: spec.traceType, + x: x, + y: y, + z: z + }], + layout: { + width: 900, + height: 400 + } + }; +} + +function makeBox(spec) { + var y = Array.from({ length: spec.n }, (_, i) => i * Math.cos(Math.sqrt(i))); + var data = []; + var nPerTrace = Math.floor(spec.n / spec.nTraces); + for(var k = 0; k < spec.nTraces; k++) { + var trace = { + type: spec.traceType, + boxpoints: spec.mode === 'all_points' ? 'all' : false, + y: y.slice(k * nPerTrace, (k + 1) * nPerTrace), + x: Array.from({ length: nPerTrace }, (_, i) => k) + }; + + if(spec.traceType === 'box') { + trace.boxpoints = spec.mode === 'all_points' ? 'all' : false; + } + + if(spec.traceType === 'violin') { + trace.points = spec.mode === 'all_points' ? 'all' : false; + } + + data.push(trace); + } + + return { + data: data, + layout: { + showlegend: false, + width: 900, + height: 400 + } + }; +} + +function makeBar(spec) { + var z = Array.from({ length: spec.n }, (_, i) => i * Math.cos(Math.sqrt(i))); + var data = []; + var nPerTrace = Math.floor(spec.n / spec.nTraces); + for(var k = 0; k < spec.nTraces; k++) { + if(spec.traceType === 'bar') { + data.push({ + type: 'bar', + y: z.slice(k * nPerTrace, (k + 1) * nPerTrace), + x: Array.from({ length: nPerTrace }, (_, i) => i) + }); + } else if(spec.traceType === 'histogram') { + data.push({ + type: 'histogram', + x: z.slice(k * nPerTrace, (k + 1) * nPerTrace), + y: Array.from({ length: nPerTrace }, (_, i) => i) + }); + } + } + + return { + data: data, + layout: { + barmode: spec.mode, + showlegend: false, + width: 900, + height: 400 + } + }; +} + +function makeScatter(spec) { + var y = Array.from({ length: spec.n }, (_, i) => i * Math.cos(Math.sqrt(i))); + var data = []; + var nPerTrace = Math.floor(spec.n / spec.nTraces); + for(var k = 0; k < spec.nTraces; k++) { + data.push({ + type: spec.traceType, + mode: spec.mode, + y: y.slice(k * nPerTrace, (k + 1) * nPerTrace), + x: Array.from({ length: nPerTrace }, (_, i) => i + k * nPerTrace) + }); + } + + return { + data: data, + layout: { + showlegend: false, + width: 900, + height: 400 + } + }; +} + +function makeScatterGeo(spec) { + var y = Array.from({ length: spec.n }, (_, i) => 0.001 * i * Math.cos(Math.sqrt(i))); + + var data = []; + var nPerTrace = Math.floor(spec.n / spec.nTraces); + for(var k = 0; k < spec.nTraces; k++) { + data.push({ + type: 'scattergeo', + mode: spec.mode, + lat: y.slice(k * nPerTrace, (k + 1) * nPerTrace), + lon: Array.from({ length: nPerTrace }, (_, i) => -180 + 0.005 * (i + k * nPerTrace)) + }); + } + + return { + data: data, + layout: { + showlegend: false, + width: 900, + height: 400 + } + }; +} + + +describe('Performance test various traces', function() { + 'use strict'; + + var filename; + + afterAll(function(done) { + downloadCSV(tests, filename); + // delay for the download to be completed + delay(1000)().then(done) + }); + + tests.forEach(function(spec, index) { + var testIt = true; + + var testCase = __karma__.config.testCase; + + filename = ''; + + if(testCase) { + if(testCase.tracesType) { + filename += testCase.tracesType; + if(testCase.tracesType !== spec.traceType) testIt = false; + } + + if(testCase.tracesMode && testCase.tracesMode !== 'undefined') { + filename += '_' + testCase.tracesMode; + if(testCase.tracesMode !== spec.mode) testIt = false; + } + + if(testCase.tracesPoints) { + filename += '_' + testCase.tracesPoints; + if(testCase.tracesPoints !== spec.n) testIt = false; + } + + if(testCase.tracesCount) { + filename += '_' + testCase.tracesCount; + if(testCase.tracesCount !== spec.nTraces) testIt = false; + } + } + + if(testIt) { + samples.forEach(function(t) { + it( + 'All points:' + spec.n + ' | ' + + spec.nTraces + ' X ' + spec.traceType + + (spec.mode ? ' | mode: ' + spec.mode : '') + + ' | turn: ' + t, function(done) { + if(t === 0) { + tests[index].raw = []; + } + + var timerID; + var requestID1, requestID2; + + var startTime, endTime; + + requestID1 = requestAnimationFrame(() => { + // Wait for actual rendering instead of promise + requestID2 = requestAnimationFrame(() => { + endTime = performance.now(); + + var delta = endTime - startTime; + + if(tests[index].raw[t] === undefined) { + tests[index].raw[t] = delta; + } + + if(spec.selector) { + var nodes = d3SelectAll(spec.selector); + expect(nodes.size()).toEqual(spec.nTraces); + } + + clearTimeout(timerID); + + done(); + }); + }); + + var mock = generateMock(spec); + + timerID = setTimeout(() => { + endTime = performance.now(); + + tests[index].raw[t] = 'none'; + + cancelAnimationFrame(requestID2); + cancelAnimationFrame(requestID1); + + done.fail('Takes too much time: ' + (endTime - startTime)); + }, MAX_RENDERING_TIME); + + startTime = performance.now(); + + Plotly.newPlot(gd, mock); + }); + }); + } + }); +}); diff --git a/test/jasmine/performance_tests/assets/constants.js b/test/jasmine/performance_tests/assets/constants.js new file mode 100644 index 00000000000..0fe113801ce --- /dev/null +++ b/test/jasmine/performance_tests/assets/constants.js @@ -0,0 +1,5 @@ +'use strict'; + +module.exports = { + nSamples: 4 +}; diff --git a/test/jasmine/performance_tests/assets/post_process.js b/test/jasmine/performance_tests/assets/post_process.js new file mode 100644 index 00000000000..940dae12d67 --- /dev/null +++ b/test/jasmine/performance_tests/assets/post_process.js @@ -0,0 +1,37 @@ +exports.downloadCSV = function(allTests, filename) { + var str = [ + 'number of traces', + 'chart type & mode', + 'data points', + 'run id', + 'rendering time(ms)' + ].join(',') + '\n'; + + for(var k = 0; k < allTests.length; k++) { + var test = allTests[k]; + + var raw = test.raw || []; + + for(var i = 0; i < raw.length; i++) { + str += [ + (test.nTraces || 1), + (test.traceType + (test.mode ? ' ' + test.mode : '')), + test.n, + i, + raw[i] + ].join(',') + '\n'; + } + } + + // download file by browser + var a = document.createElement('a'); + var myBlob = new Blob([str], {type: 'text/plain'}) + var url = window.URL.createObjectURL(myBlob); + a.href = url; + a.download = (filename || 'results') + '.csv'; + a.style.display = 'none'; + document.body.append(a); + a.click(); + a.remove(); + window.URL.revokeObjectURL(url); +}; diff --git a/test/jasmine/performance_tests/assets/test_cases.js b/test/jasmine/performance_tests/assets/test_cases.js new file mode 100644 index 00000000000..27f86de551f --- /dev/null +++ b/test/jasmine/performance_tests/assets/test_cases.js @@ -0,0 +1,77 @@ +var tests = []; + +for(let traceType of ['image', 'heatmap', 'contour']) { + for(let m of [10, 20, 40, 80, 160, 320, 640, 1280]) { + let nx = 5 * m; + let ny = 2 * m; + tests.push({ + nx: nx, + ny: ny, + n: nx * ny, + nTraces: 1, + traceType: traceType, + selector: traceType === 'image' ? 'g.imagelayer.mlayer' : + 'g.' + traceType + 'layer' + }); + } +} + +var allN = [1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000]; +var allNTraces = [1, /*10, */100] + +for(let traceType of ['box', 'violin']) { + for(let mode of ['no_points', 'all_points']) { + for(let nTraces of allNTraces) { + for(let n of allN) { + tests.push({ + n:n, + nTraces: nTraces, + traceType: traceType, + mode: mode, + selector: ( + traceType === 'box' ? 'g.trace.boxes' : + traceType === 'violin' ? 'g.trace.violins' : + undefined + ) + }); + } + } + } +} + +for(let traceType of ['scatter', 'scattergl', 'scattergeo']) { + for(let mode of ['markers', 'lines', 'markers+lines']) { + for(let nTraces of allNTraces) { + for(let n of allN) { + tests.push({ + n:n, + nTraces: nTraces, + traceType: traceType, + mode: mode, + selector: ( + traceType === 'scatter' ? 'g.trace.scatter' : + undefined + ) + }); + } + } + } +} + +for(let traceType of ['bar', 'histogram']) { + for(let mode of ['group', 'stack', 'overlay']) { + for(let nTraces of allNTraces) { + for(let n of allN) { + tests.push({ + n:n, + nTraces: nTraces, + traceType: traceType, + mode: mode, + selector: 'g.trace.bars' + }); + } + } + } +} + +exports.testCases = tests;