From 7ea383c66b3589796dd372f6a6efc0105c8e6f37 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 16 Sep 2024 13:14:38 -0400 Subject: [PATCH 1/6] chore(NODE-6382): add benchmarking for helpers --- .evergreen/config.yml | 23 ++++++++++++ .evergreen/run-custom-benchmarks.sh | 6 ++++ .gitignore | 1 + package.json | 1 + test/bench/custom/main.mjs | 56 +++++++++++++++++++++++++++++ test/bench/custom/readme.md | 33 +++++++++++++++++ 6 files changed, 120 insertions(+) create mode 100644 .evergreen/run-custom-benchmarks.sh create mode 100644 test/bench/custom/main.mjs create mode 100644 test/bench/custom/readme.md diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6f83976f8..4a38e8d00 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -113,6 +113,15 @@ functions: add_expansions_to_env: true args: - .evergreen/run-granular-benchmarks.sh + run custom benchmarks: + - command: subprocess.exec + type: test + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - .evergreen/run-custom-benchmarks.sh run spec benchmarks: - command: subprocess.exec type: test @@ -246,6 +255,19 @@ tasks: - command: perf.send params: file: src/test/bench/etc/resultsCollectedMeans.json + - name: run-custom-benchmarks + commands: + - func: fetch source + vars: + # This needs to stay pinned at Node v18.16.0 for consistency across perf runs. + NODE_LTS_VERSION: v18.16.0 + - func: install dependencies + vars: + NPM_VERSION: 9 + - func: run custom benchmarks + - command: perf.send + params: + file: results.json - name: run-spec-benchmarks commands: - func: fetch source @@ -300,3 +322,4 @@ buildvariants: tasks: - run-granular-benchmarks - run-spec-benchmarks + - run-custom-benchmarks diff --git a/.evergreen/run-custom-benchmarks.sh b/.evergreen/run-custom-benchmarks.sh new file mode 100644 index 000000000..2984cb356 --- /dev/null +++ b/.evergreen/run-custom-benchmarks.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +set -o xtrace + +npm run check:custom-bench diff --git a/.gitignore b/.gitignore index 39b98b7bd..d5d9cd00a 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ docs/public .nvmrc benchmarks.json +results.json diff --git a/package.json b/package.json index fd1fa8900..6fed86fa5 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "check:web-no-bigint": "WEB=true NO_BIGINT=true mocha test/node", "check:granular-bench": "npm run build:bench && node ./test/bench/etc/run_granular_benchmarks.js", "check:spec-bench": "npm run build:bench && node ./test/bench/lib/spec/bsonBench.js", + "check:custom-bench": "npm run build && node ./test/bench/custom/main.mjs", "build:bench": "cd test/bench && npx tsc", "build:ts": "node ./node_modules/typescript/bin/tsc", "build:dts": "npm run build:ts && api-extractor run --typescript-compiler-folder node_modules/typescript --local && node etc/clean_definition_files.cjs", diff --git a/test/bench/custom/main.mjs b/test/bench/custom/main.mjs new file mode 100644 index 000000000..51a574387 --- /dev/null +++ b/test/bench/custom/main.mjs @@ -0,0 +1,56 @@ +/* eslint-disable strict */ + +import util from 'node:util'; +import fs from 'node:fs'; +import os from 'node:os'; +import benchmark from 'benchmark'; +import { BSON } from '../../../lib/bson.mjs'; + +const hw = os.cpus(); +const ram = os.totalmem() / 1024 ** 3; +const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` }; + +const systemInfo = () => + [ + `\n- cpu: ${platform.name}`, + `- cores: ${platform.cores}`, + `- arch: ${os.arch()}`, + `- os: ${process.platform} (${os.release()})`, + `- ram: ${platform.ram}\n` + ].join('\n'); +console.log(systemInfo()); + +const suite = new benchmark.Suite(); + +benchmark.options = { + async: false, + defer: false, + initCount: 1000, + minSamples: 1000 +}; + +suite + .add('objectid_isvalid_strlen', function objectid_isvalid_strlen() { + BSON.ObjectId.isValid('a'); + }) + .add('objectid_isvalid_bestcase_false', function objectid_isvalid_bestcase_false() { + BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); + }) + .add('objectid_isvalid_worstcase_false', function objectid_isvalid_worstcase_false() { + BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg'); + }) + .add('objectid_isvalid_true', function objectid_isvalid_true() { + BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf'); + }) + .on('cycle', function logBenchmark(event) { + console.log(String(event.target)); + }) + .on('complete', function outputPerfSend() { + const data = Array.from(this).map(bench => ({ + info: { test_name: bench.name }, + metrics: [{ name: 'ops_per_sec', value: bench.hz }] + })); + console.log(util.inspect(data, { depth: Infinity, colors: true })); + fs.writeFileSync('results.json', JSON.stringify(data), 'utf8'); + }) + .run(); diff --git a/test/bench/custom/readme.md b/test/bench/custom/readme.md new file mode 100644 index 000000000..3648ffc89 --- /dev/null +++ b/test/bench/custom/readme.md @@ -0,0 +1,33 @@ +# Custom Benchmark Tests + +In this directory are tests for code paths not covered by our spec or granular (de)serialization benchmarks. + +## How to write your own + +In `main.mjs` call the `.add` function and pass it an underscore concatenated descriptive title. +Try to fit the title into the format of: "subject area", "method or function" "test case that is being covered" (Ex. `objectid_isvalid_bestcase_false`). +Copy the title to the name of the function to assist with debugging and flamegraph capturing. + +### Example + +```js +.add('subject_function_testcase', function subject_function_testcase() { + BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); +}) +``` + +## Output + +The JSON emitted at the end of the benchmarks must follow our performance tracking format. + +The JSON must be an array of "`Test`"s: + +```ts +type Metric = { name: string, value: number } +type Test = { + info: { test_name: string }, + metrics: Metric[] +} +``` + +The metric collected is always "ops_per_sec" so higher is better. From adffe644039289be251df65fa7a5d3d083747def Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 16 Sep 2024 14:22:04 -0400 Subject: [PATCH 2/6] chore: move benchmarks to file --- test/bench/custom/benchmarks.mjs | 20 ++++++++++++++++++++ test/bench/custom/main.mjs | 21 ++------------------- test/bench/custom/readme.md | 17 +++++++++++------ 3 files changed, 33 insertions(+), 25 deletions(-) create mode 100644 test/bench/custom/benchmarks.mjs diff --git a/test/bench/custom/benchmarks.mjs b/test/bench/custom/benchmarks.mjs new file mode 100644 index 000000000..8fbbf9afc --- /dev/null +++ b/test/bench/custom/benchmarks.mjs @@ -0,0 +1,20 @@ +/* eslint-disable strict */ +import { BSON } from '../../../lib/bson.mjs'; + +const ObjectId_isValid = [ + function objectid_isvalid_strlen() { + BSON.ObjectId.isValid('a'); + }, + function objectid_isvalid_bestcase_false() { + BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); + }, + function objectid_isvalid_worstcase_false() { + BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg'); + }, + function objectid_isvalid_true() { + BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf'); + } +]; + +// Add benchmarks here: +export const benchmarks = [...ObjectId_isValid]; diff --git a/test/bench/custom/main.mjs b/test/bench/custom/main.mjs index 51a574387..accb78350 100644 --- a/test/bench/custom/main.mjs +++ b/test/bench/custom/main.mjs @@ -4,7 +4,7 @@ import util from 'node:util'; import fs from 'node:fs'; import os from 'node:os'; import benchmark from 'benchmark'; -import { BSON } from '../../../lib/bson.mjs'; +import { benchmarks } from './benchmarks.mjs'; const hw = os.cpus(); const ram = os.totalmem() / 1024 ** 3; @@ -22,26 +22,9 @@ console.log(systemInfo()); const suite = new benchmark.Suite(); -benchmark.options = { - async: false, - defer: false, - initCount: 1000, - minSamples: 1000 -}; +for (const bench of benchmarks) suite.add(bench.name, bench); suite - .add('objectid_isvalid_strlen', function objectid_isvalid_strlen() { - BSON.ObjectId.isValid('a'); - }) - .add('objectid_isvalid_bestcase_false', function objectid_isvalid_bestcase_false() { - BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); - }) - .add('objectid_isvalid_worstcase_false', function objectid_isvalid_worstcase_false() { - BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg'); - }) - .add('objectid_isvalid_true', function objectid_isvalid_true() { - BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf'); - }) .on('cycle', function logBenchmark(event) { console.log(String(event.target)); }) diff --git a/test/bench/custom/readme.md b/test/bench/custom/readme.md index 3648ffc89..ed69ae218 100644 --- a/test/bench/custom/readme.md +++ b/test/bench/custom/readme.md @@ -4,16 +4,21 @@ In this directory are tests for code paths not covered by our spec or granular ( ## How to write your own -In `main.mjs` call the `.add` function and pass it an underscore concatenated descriptive title. -Try to fit the title into the format of: "subject area", "method or function" "test case that is being covered" (Ex. `objectid_isvalid_bestcase_false`). -Copy the title to the name of the function to assist with debugging and flamegraph capturing. +In `benchmarks.mjs` add a new test to an existing array or make a new array for a new subject area. +Try to fit the name of the function into the format of: "subject area", "method or function" "test case that is being covered" (Ex. `objectid_isvalid_bestcase_false`). +Make sure your test is added to the `benchmarks` export. ### Example ```js -.add('subject_function_testcase', function subject_function_testcase() { - BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); -}) +const ObjectId_isValid = [ + function objectid_isvalid_strlen() { + BSON.ObjectId.isValid('a'); + }, + // ... +]; + +export const benchmarks = [...ObjectId_isValid]; ``` ## Output From e49e0241b9b428d2eac9afa05f55a262cdbe47b1 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 16 Sep 2024 14:28:52 -0400 Subject: [PATCH 3/6] chore: fix perf.send file --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4a38e8d00..6604414f4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -267,7 +267,7 @@ tasks: - func: run custom benchmarks - command: perf.send params: - file: results.json + file: src/results.json - name: run-spec-benchmarks commands: - func: fetch source From 5e1474a2e619c9c255a593b9c09dee402e407566 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 16 Sep 2024 14:40:13 -0400 Subject: [PATCH 4/6] chore: rename results file --- .evergreen/config.yml | 2 +- .gitignore | 2 +- test/bench/custom/main.mjs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6604414f4..a043c638f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -267,7 +267,7 @@ tasks: - func: run custom benchmarks - command: perf.send params: - file: src/results.json + file: src/customBenchmarkResults.json - name: run-spec-benchmarks commands: - func: fetch source diff --git a/.gitignore b/.gitignore index d5d9cd00a..b2ee050b8 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ docs/public .nvmrc benchmarks.json -results.json +customBenchmarkResults.json diff --git a/test/bench/custom/main.mjs b/test/bench/custom/main.mjs index accb78350..b375418f1 100644 --- a/test/bench/custom/main.mjs +++ b/test/bench/custom/main.mjs @@ -34,6 +34,6 @@ suite metrics: [{ name: 'ops_per_sec', value: bench.hz }] })); console.log(util.inspect(data, { depth: Infinity, colors: true })); - fs.writeFileSync('results.json', JSON.stringify(data), 'utf8'); + fs.writeFileSync('customBenchmarkResults.json', JSON.stringify(data), 'utf8'); }) .run(); From a7ea8d0ec6267f5dcc9c22fa3d8fd6311ad7be90 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 16 Sep 2024 15:58:32 -0400 Subject: [PATCH 5/6] chore: add better description for short circuiting --- test/bench/custom/benchmarks.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/bench/custom/benchmarks.mjs b/test/bench/custom/benchmarks.mjs index 8fbbf9afc..3ba160152 100644 --- a/test/bench/custom/benchmarks.mjs +++ b/test/bench/custom/benchmarks.mjs @@ -5,10 +5,12 @@ const ObjectId_isValid = [ function objectid_isvalid_strlen() { BSON.ObjectId.isValid('a'); }, - function objectid_isvalid_bestcase_false() { + /** wrong character at the start */ + function objectid_isvalid_most_short_circuit_false() { BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); }, - function objectid_isvalid_worstcase_false() { + /** wrong character at the end */ + function objectid_isvalid_least_short_circuit_false() { BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg'); }, function objectid_isvalid_true() { From 3789acb891a823369ce96df1a704a1d50be6b0b8 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 16 Sep 2024 16:16:18 -0400 Subject: [PATCH 6/6] chore: name improvements --- test/bench/custom/benchmarks.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/bench/custom/benchmarks.mjs b/test/bench/custom/benchmarks.mjs index 3ba160152..74170fb90 100644 --- a/test/bench/custom/benchmarks.mjs +++ b/test/bench/custom/benchmarks.mjs @@ -2,18 +2,18 @@ import { BSON } from '../../../lib/bson.mjs'; const ObjectId_isValid = [ - function objectid_isvalid_strlen() { + function objectid_isvalid_wrong_string_length() { BSON.ObjectId.isValid('a'); }, - /** wrong character at the start */ - function objectid_isvalid_most_short_circuit_false() { + /** wrong character at the start, could be the most short circuited code path */ + function objectid_isvalid_invalid_hex_at_start() { BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); }, - /** wrong character at the end */ - function objectid_isvalid_least_short_circuit_false() { + /** wrong character at the end, could be the least short circuited code path */ + function objectid_isvalid_invalid_hex_at_end() { BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg'); }, - function objectid_isvalid_true() { + function objectid_isvalid_valid_hex_string() { BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf'); } ];