From 04e1f10d35459f076dce5bd8210bd09ecf475132 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Wed, 26 Jun 2024 15:43:05 -0400 Subject: [PATCH 01/15] add readme --- benches/README.md | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/benches/README.md b/benches/README.md index 464090415..cbf86119a 100644 --- a/benches/README.md +++ b/benches/README.md @@ -1 +1,44 @@ -# TODO +# Benches +This directory contains 3 sub-directories: + +| Sub-directory | Purpose | +|---------------|---------| +| `micro/` | Micro-benchmarks for crates (e.g. timings for a single function) +| `macro/` | Macro-benchmarks for whole crates or sub-systems (using Cuprate's custom benchmarking harness) +| `harness/` | Cuprate's custom benchmarking harness + +## Harness +The harness is just another crate (that happens to be for benchmarking). + +Conceptually, it's purpose is very simple: +1. Set-up the benchmark +1. Start timer +1. Run benchmark +1. Output data + +This single harness runs the benchmarks found in `macro/`. + +The way benchmarks "plug-in" to the harness is simply by implementing `trait Benchmark`. + +See `cuprate-harness`' crate documentation for a user-guide: +```bash +cargo doc --open --package cuprate-harness +``` + +## Macro +Each sub-directory in here is a crate that plugs into the harness. + +Benchmarks in `macro/` are for testing sub-systems and/or sections of a sub-system, e.g. the block downloader, the RPC server, the database, etc. + + +See `macro/cuprate-database` for an example. + + +## Micro +Each sub-directory in here is a crate that uses [Criterion](https://bheisler.github.io/criterion.rs/book) for timing single functions, groups of functions. + +They are generally be small in scope. + + +See `macro/cuprate-json-rpc` for an example. + \ No newline at end of file From b538272f4d14c873d1fc4c2871ee12f4761117b5 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Wed, 26 Jun 2024 16:25:55 -0400 Subject: [PATCH 02/15] readme, basic examples --- Cargo.lock | 266 ++++++++++++++++++ Cargo.toml | 31 +- benches/README.md | 85 ++++-- benches/criterion/cuprate-json-rpc/Cargo.toml | 21 ++ .../cuprate-json-rpc/benches/main.rs | 81 ++++++ .../cuprate-json-rpc/benches/response.rs | 94 +++++++ benches/criterion/cuprate-json-rpc/src/lib.rs | 77 +++++ benches/harness/Cargo.toml | 23 ++ benches/harness/harness-test/Cargo.toml | 14 + benches/harness/harness-test/src/lib.rs | 16 ++ benches/harness/lib/Cargo.toml | 15 + benches/harness/lib/src/benchmark.rs | 16 ++ benches/harness/lib/src/lib.rs | 97 +++++++ benches/harness/src/main.rs | 120 ++++++++ 14 files changed, 921 insertions(+), 35 deletions(-) create mode 100644 benches/criterion/cuprate-json-rpc/Cargo.toml create mode 100644 benches/criterion/cuprate-json-rpc/benches/main.rs create mode 100644 benches/criterion/cuprate-json-rpc/benches/response.rs create mode 100644 benches/criterion/cuprate-json-rpc/src/lib.rs create mode 100644 benches/harness/Cargo.toml create mode 100644 benches/harness/harness-test/Cargo.toml create mode 100644 benches/harness/harness-test/src/lib.rs create mode 100644 benches/harness/lib/Cargo.toml create mode 100644 benches/harness/lib/src/benchmark.rs create mode 100644 benches/harness/lib/src/lib.rs create mode 100644 benches/harness/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 079972889..fa6ebcc94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.18" @@ -50,6 +59,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstyle" version = "1.0.7" @@ -275,6 +290,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.99" @@ -305,6 +326,33 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.7" @@ -377,6 +425,42 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -569,6 +653,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "cuprate-criterion-json-rpc" +version = "0.0.0" +dependencies = [ + "criterion", + "cuprate-json-rpc", + "function_name", + "serde_json", + "tempfile", +] + [[package]] name = "cuprate-cryptonight" version = "0.1.0" @@ -637,6 +732,27 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cuprate-harness" +version = "0.0.0" +dependencies = [ + "cfg-if", + "cuprate-database", + "cuprate-harness-lib", + "cuprate-harness-test", +] + +[[package]] +name = "cuprate-harness-lib" +version = "0.0.0" + +[[package]] +name = "cuprate-harness-test" +version = "0.0.0" +dependencies = [ + "cuprate-harness-lib", +] + [[package]] name = "cuprate-helper" version = "0.1.0" @@ -1006,6 +1122,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "function_name" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1ab577a896d09940b5fe12ec5ae71f9d8211fff62c919c03a3750a9901e98a7" +dependencies = [ + "function_name-proc-macro", +] + +[[package]] +name = "function_name-proc-macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673464e1e314dd67a0fd9544abc99e8eb28d0c7e3b69b033bcff9b2d00b87333" + [[package]] name = "funty" version = "2.0.0" @@ -1127,6 +1258,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1374,6 +1515,26 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1601,6 +1762,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1761,6 +1928,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" + +[[package]] +name = "plotters-svg" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1998,6 +2193,29 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.4" @@ -2125,6 +2343,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.23" @@ -2395,6 +2622,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2655,6 +2892,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2724,6 +2971,16 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2740,6 +2997,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 8891b83bf..5d77862e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,30 +1,40 @@ [workspace] resolver = "2" - members = [ + # Benchmarks + "benches/harness", + "benches/harness/lib", + "benches/harness/harness-test", + "benches/criterion/cuprate-json-rpc", + # Consensus "consensus", "consensus/fast-sync", "consensus/rules", - "cryptonight", - "helper", + # Net "net/epee-encoding", "net/fixed-bytes", "net/levin", "net/wire", + # P2P "p2p/p2p", "p2p/p2p-core", "p2p/dandelion-tower", "p2p/async-buffer", "p2p/address-book", + # Storage "storage/blockchain", "storage/txpool", "storage/database", - "pruning", - "test-utils", - "types", + # RPC "rpc/json-rpc", "rpc/rpc-types", "rpc/rpc-interface", + # Misc + "cryptonight", + "helper", + "pruning", + "test-utils", + "types", ] [profile.release] @@ -85,6 +95,8 @@ tracing-subscriber = { version = "0.3.17", default-features = false } tracing = { version = "0.1.40", default-features = false } ## workspace.dev-dependencies +criterion = { version = "0.5.1" } +function_name = { version = "0.3.0" } tempfile = { version = "3" } pretty_assertions = { version = "1.4.0" } proptest = { version = "1" } @@ -99,9 +111,4 @@ tokio-test = { version = "0.4.4" } # once_cell = { version = "1.18.0" } # Lazy/one-time initialization | https://github.com/matklad/once_cell # open = { version = "5.0.0" } # Open PATH/URL, probably for binaries | https://github.com/byron/open-rs # regex = { version = "1.10.2" } # Regular expressions | https://github.com/rust-lang/regex -# ryu = { version = "1.0.15" } # Fast float to string formatting | https://github.com/dtolnay/ryu - -# Maybe one day. -# disk = { version = "*" } # (De)serialization to/from disk with various file formats | https://github.com/hinto-janai/disk -# readable = { version = "*" } # Stack-based string formatting utilities | https://github.com/hinto-janai/readable -# json-rpc = { git = "https://github.com/hinto-janai/json-rpc" } # JSON-RPC 2.0 types +# ryu = { version = "1.0.15" } # Fast float to string formatting | https://github.com/dtolnay/ryu \ No newline at end of file diff --git a/benches/README.md b/benches/README.md index cbf86119a..c5c676717 100644 --- a/benches/README.md +++ b/benches/README.md @@ -1,44 +1,83 @@ # Benches -This directory contains 3 sub-directories: +This directory contains Cuprate's benchmarks and benchmarking utilities. + +- [1. File layout and purpose](#1-file-layout-and-purpose) +- [2. Harness](#2-harness) + - [2.1 Creating a harness benchmark](#21-creating-a-harness-benchmark) + - [2.2 Running a harness benchmark](#22-running-a-harness-benchmark) +- [3. Criterion](#3-criterion) + - [2.1 Creating a Criterion benchmark](#21-creating-a-criterion-benchmark) + - [2.2 Running a Criterion benchmark](#22-running-a-criterion-benchmark) + +## 1. File layout and purpose +This directory is sorted into 4 important categories: | Sub-directory | Purpose | |---------------|---------| -| `micro/` | Micro-benchmarks for crates (e.g. timings for a single function) -| `macro/` | Macro-benchmarks for whole crates or sub-systems (using Cuprate's custom benchmarking harness) -| `harness/` | Cuprate's custom benchmarking harness +| `harness/src` | Cuprate's custom benchmarking harness **binary** +| `harness/lib` | Cuprate's custom benchmarking harness **library** +| `harness/*` | Macro-benchmarks for whole crates or sub-systems (using Cuprate's custom benchmarking harness) +| `criterion/*` | Micro-benchmarks for crates (e.g. timings for a single function) -## Harness -The harness is just another crate (that happens to be for benchmarking). +## 2. Harness +The harness is: +- `cuprate-harness`; the actual binary crate ran +- `cuprate-harness-lib`; the library that other crates hook into -Conceptually, it's purpose is very simple: +The purpose of the harness is very simple: 1. Set-up the benchmark 1. Start timer 1. Run benchmark 1. Output data -This single harness runs the benchmarks found in `macro/`. +The harness runs the benchmarks found in `harness/`. -The way benchmarks "plug-in" to the harness is simply by implementing `trait Benchmark`. +The way benchmarks "plug-in" to the harness is simply by implementing `cuprate_harness_lib::Benchmark`. -See `cuprate-harness`' crate documentation for a user-guide: +See `cuprate-harness-lib` crate documentation for a user-guide: ```bash -cargo doc --open --package cuprate-harness +cargo doc --open --package cuprate-harness-lib ``` -## Macro -Each sub-directory in here is a crate that plugs into the harness. +### 2.1 Creating a harness benchmark +1. Create a new crate inside `benches/harness` (consider copying `benches/harness/test` as a base) +2. Pull in `cuprate_harness_lib` as a dependency +3. Implement `cuprate_harness_lib::Benchmark` +4. Add a feature inside `cuprate_harness` for your benchmark -Benchmarks in `macro/` are for testing sub-systems and/or sections of a sub-system, e.g. the block downloader, the RPC server, the database, etc. - - -See `macro/cuprate-database` for an example. - +### 2.2 Running a harness benchmark +After your benchmark is implemented, run this command: +```bash +cargo run --release --package cuprate-harness --features $YOUR_BENCHMARK_CRATE_FEATURE +``` +For example, to run the test benchmark: +```bash +cargo run --release --package cuprate-harness --features test +``` -## Micro -Each sub-directory in here is a crate that uses [Criterion](https://bheisler.github.io/criterion.rs/book) for timing single functions, groups of functions. +## 3. Criterion +Each sub-directory in here is a crate that uses [Criterion](https://bheisler.github.io/criterion.rs/book) for timing single functions and/or groups of functions. They are generally be small in scope. - -See `macro/cuprate-json-rpc` for an example. - \ No newline at end of file +See [`criterion/cuprate-json-rpc`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc) for an example. + +### 3.1 Creating a Criterion benchmark +1. Copy [`criterion/test`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion) as base +2. Read the `Getting Started` section of +3. Get started + +### 3.1 Running a Criterion benchmark +To run all Criterion benchmarks, run this from the repository root: +```bash +cargo bench +``` + +To run specific package(s), use: +```bash +cargo bench --package $CRITERION_BENCHMARK_CRATE_NAME +``` +For example: +```bash +cargo bench --package cuprate-criterion-json-rpc +``` \ No newline at end of file diff --git a/benches/criterion/cuprate-json-rpc/Cargo.toml b/benches/criterion/cuprate-json-rpc/Cargo.toml new file mode 100644 index 000000000..eaa264708 --- /dev/null +++ b/benches/criterion/cuprate-json-rpc/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "cuprate-criterion-json-rpc" +version = "0.0.0" +edition = "2021" +description = "Criterion benchmarking for cuprate-json-rpc" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/micro/cuprate-json-rpc" +keywords = ["cuprate", "json-rpc", "criterion", "benchmark"] + +[dependencies] +criterion = { workspace = true } +function_name = { workspace = true } +serde_json = { workspace = true, features = ["default"] } +tempfile = { workspace = true } + +cuprate-json-rpc = { path = "../../../rpc/json-rpc" } + +[[bench]] +name = "main" +harness = false \ No newline at end of file diff --git a/benches/criterion/cuprate-json-rpc/benches/main.rs b/benches/criterion/cuprate-json-rpc/benches/main.rs new file mode 100644 index 000000000..84585ee99 --- /dev/null +++ b/benches/criterion/cuprate-json-rpc/benches/main.rs @@ -0,0 +1,81 @@ +//! TODO + +//---------------------------------------------------------------------------------------------------- Lints +// Forbid lints. +// Our code, and code generated (e.g macros) cannot overrule these. +#![forbid( + // `unsafe` is allowed but it _must_ be + // commented with `SAFETY: reason`. + clippy::undocumented_unsafe_blocks, + // Never. + unused_unsafe, + redundant_semicolons, + unused_allocation, + coherence_leak_check, + while_true, + clippy::missing_docs_in_private_items, + // Maybe can be put into `#[deny]`. + unconditional_recursion, + for_loops_over_fallibles, + unused_braces, + unused_doc_comments, + unused_labels, + keyword_idents, + non_ascii_idents, + variant_size_differences, + single_use_lifetimes, + // Probably can be put into `#[deny]`. + future_incompatible, + let_underscore, + break_with_label_and_loop, + duplicate_macro_attributes, + exported_private_dependencies, + large_assignments, + overlapping_range_endpoints, + semicolon_in_expressions_from_macros, + noop_method_call, + unreachable_pub, +)] +// Deny lints. +// Some of these are `#[allow]`'ed on a per-case basis. +#![deny( + clippy::all, + clippy::correctness, + clippy::suspicious, + clippy::style, + clippy::complexity, + clippy::perf, + clippy::pedantic, + clippy::nursery, + clippy::cargo, + unused_mut, + missing_docs, + deprecated, + unused_comparisons, + nonstandard_style +)] +#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove +#![allow( + // FIXME: this lint affects crates outside of + // `database/` for some reason, allow for now. + clippy::cargo_common_metadata, + // FIXME: adding `#[must_use]` onto everything + // might just be more annoying than useful... + // although it is sometimes nice. + clippy::must_use_candidate, + // TODO: should be removed after all `todo!()`'s are gone. + clippy::diverging_sub_expression, + clippy::module_name_repetitions, + clippy::module_inception, + clippy::redundant_pub_crate, + clippy::option_if_let_else, + clippy::significant_drop_tightening, +)] +// Allow some lints when running in debug mode. +#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] + +mod response; + +criterion::criterion_main! { + response::benches, +} diff --git a/benches/criterion/cuprate-json-rpc/benches/response.rs b/benches/criterion/cuprate-json-rpc/benches/response.rs new file mode 100644 index 000000000..93e209af9 --- /dev/null +++ b/benches/criterion/cuprate-json-rpc/benches/response.rs @@ -0,0 +1,94 @@ +//! `trait Storable` benchmarks. + +//---------------------------------------------------------------------------------------------------- Import +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use function_name::named; +use serde_json::{from_str, to_string_pretty}; + +use cuprate_json_rpc::{Id, Response}; + +//---------------------------------------------------------------------------------------------------- Criterion +criterion_group! { + benches, + response_from_str_u8, + response_from_str_u64, + response_from_str_string_5_len, + response_from_str_string_10_len, + response_from_str_string_100_len, + response_from_str_string_500_len, + response_to_string_pretty_u8, + response_to_string_pretty_u64, + response_to_string_pretty_string_5_len, + response_to_string_pretty_string_10_len, + response_to_string_pretty_string_100_len, + response_to_string_pretty_string_500_len, +} +criterion_main!(benches); + +//---------------------------------------------------------------------------------------------------- Deserialization +/// TODO +macro_rules! impl_from_str_benchmark { + ( + $( + $fn_name:ident => $request_type:ty => $request_string:literal, + )* + ) => { + $( + /// TODO + #[named] + fn $fn_name(c: &mut Criterion) { + let request_string = $request_string; + + c.bench_function(function_name!(), |b| { + b.iter(|| { + let _r = from_str::>( + black_box(request_string) + ); + }); + }); + } + )* + }; +} + +impl_from_str_benchmark! { + response_from_str_u8 => u8 => r#"{"jsonrpc":"2.0","id":123,"result":0}"#, + response_from_str_u64 => u64 => r#"{"jsonrpc":"2.0","id":123,"result":0}"#, + response_from_str_string_5_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"hello"}"#, + response_from_str_string_10_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"hellohello"}"#, + response_from_str_string_100_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld"}"#, + response_from_str_string_500_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld"}"#, +} + +//---------------------------------------------------------------------------------------------------- Deserialization +/// TODO +macro_rules! impl_to_string_pretty_benchmark { + ( + $( + $fn_name:ident => $request_constructor:expr, + )* + ) => { + $( + /// TODO + #[named] + fn $fn_name(c: &mut Criterion) { + let request = $request_constructor; + + c.bench_function(function_name!(), |b| { + b.iter(|| { + let _s = to_string_pretty(black_box(&request)).unwrap(); + }); + }); + } + )* + }; +} + +impl_to_string_pretty_benchmark! { + response_to_string_pretty_u8 => Response::::ok(Id::Null, 0), + response_to_string_pretty_u64 => Response::::ok(Id::Null, 0), + response_to_string_pretty_string_5_len => Response::ok(Id::Null, String::from("hello")), + response_to_string_pretty_string_10_len => Response::ok(Id::Null, String::from("hellohello")), + response_to_string_pretty_string_100_len => Response::ok(Id::Null, String::from("helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld")), + response_to_string_pretty_string_500_len => Response::ok(Id::Null, String::from("helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld")), +} diff --git a/benches/criterion/cuprate-json-rpc/src/lib.rs b/benches/criterion/cuprate-json-rpc/src/lib.rs new file mode 100644 index 000000000..00561eee1 --- /dev/null +++ b/benches/criterion/cuprate-json-rpc/src/lib.rs @@ -0,0 +1,77 @@ +//! TODO + +//---------------------------------------------------------------------------------------------------- Lints +// Forbid lints. +// Our code, and code generated (e.g macros) cannot overrule these. +#![forbid( + // `unsafe` is allowed but it _must_ be + // commented with `SAFETY: reason`. + clippy::undocumented_unsafe_blocks, + // Never. + unused_unsafe, + redundant_semicolons, + unused_allocation, + coherence_leak_check, + while_true, + clippy::missing_docs_in_private_items, + // Maybe can be put into `#[deny]`. + unconditional_recursion, + for_loops_over_fallibles, + unused_braces, + unused_doc_comments, + unused_labels, + keyword_idents, + non_ascii_idents, + variant_size_differences, + single_use_lifetimes, + // Probably can be put into `#[deny]`. + future_incompatible, + let_underscore, + break_with_label_and_loop, + duplicate_macro_attributes, + exported_private_dependencies, + large_assignments, + overlapping_range_endpoints, + semicolon_in_expressions_from_macros, + noop_method_call, + unreachable_pub, +)] +// Deny lints. +// Some of these are `#[allow]`'ed on a per-case basis. +#![deny( + clippy::all, + clippy::correctness, + clippy::suspicious, + clippy::style, + clippy::complexity, + clippy::perf, + clippy::pedantic, + clippy::nursery, + clippy::cargo, + unused_mut, + missing_docs, + deprecated, + unused_comparisons, + nonstandard_style +)] +#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove +#![allow( + // FIXME: this lint affects crates outside of + // `database/` for some reason, allow for now. + clippy::cargo_common_metadata, + // FIXME: adding `#[must_use]` onto everything + // might just be more annoying than useful... + // although it is sometimes nice. + clippy::must_use_candidate, + // TODO: should be removed after all `todo!()`'s are gone. + clippy::diverging_sub_expression, + clippy::module_name_repetitions, + clippy::module_inception, + clippy::redundant_pub_crate, + clippy::option_if_let_else, + clippy::significant_drop_tightening, +)] +// Allow some lints when running in debug mode. +#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] + +//---------------------------------------------------------------------------------------------------- Modules diff --git a/benches/harness/Cargo.toml b/benches/harness/Cargo.toml new file mode 100644 index 000000000..30a852cb8 --- /dev/null +++ b/benches/harness/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "cuprate-harness" +version = "0.0.0" +edition = "2021" +description = "Cuprate's benchmarking harness binary" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/harness" +keywords = ["cuprate", "benchmarking", "harness", "binary"] + +[features] +default = ["test"] +test = ["dep:cuprate-harness-test"] +database = ["dep:cuprate-database"] + +[dependencies] +cuprate-harness-lib = { path = "lib" } +cuprate-harness-test = { path = "harness-test", optional = true } +cuprate-database = { path = "../../storage/database", optional = true } + +cfg-if = { workspace = true } + +[dev-dependencies] \ No newline at end of file diff --git a/benches/harness/harness-test/Cargo.toml b/benches/harness/harness-test/Cargo.toml new file mode 100644 index 000000000..278ddd64c --- /dev/null +++ b/benches/harness/harness-test/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "cuprate-harness-test" +version = "0.0.0" +edition = "2021" +description = "Test for Cuprate's benchmarking harness" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/macro/harness-test" +keywords = ["cuprate", "benchmarking", "harness", "test"] + +[dependencies] +cuprate-harness-lib = { path = "../lib" } + +[dev-dependencies] \ No newline at end of file diff --git a/benches/harness/harness-test/src/lib.rs b/benches/harness/harness-test/src/lib.rs new file mode 100644 index 000000000..beb360fd7 --- /dev/null +++ b/benches/harness/harness-test/src/lib.rs @@ -0,0 +1,16 @@ +//! TODO + +/// TODO +pub struct BenchmarkHarnessTest; + +// TODO +impl cuprate_harness_lib::Benchmark for BenchmarkHarnessTest { + /// TODO + type Input = (); + + /// TODO + const SETUP: fn() -> Self::Input = || {}; + + /// TODO + const MAIN: fn(Self::Input) = |_| {}; +} diff --git a/benches/harness/lib/Cargo.toml b/benches/harness/lib/Cargo.toml new file mode 100644 index 000000000..965117ee1 --- /dev/null +++ b/benches/harness/lib/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "cuprate-harness-lib" +version = "0.0.0" +edition = "2021" +description = "Cuprate's benchmarking harness library" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/harness/core" +keywords = ["cuprate", "benchmarking", "harness", "library"] + +[features] + +[dependencies] + +[dev-dependencies] \ No newline at end of file diff --git a/benches/harness/lib/src/benchmark.rs b/benches/harness/lib/src/benchmark.rs new file mode 100644 index 000000000..3f46ff66f --- /dev/null +++ b/benches/harness/lib/src/benchmark.rs @@ -0,0 +1,16 @@ +//! TODO + +//---------------------------------------------------------------------------------------------------- Use + +//---------------------------------------------------------------------------------------------------- trait Benchmark +/// A benchmarking function and its inputs. +pub trait Benchmark { + /// Input to the main benchmarking function. + type Input; + + /// Setup function to generate the input. + const SETUP: fn() -> Self::Input; + + /// The main function to benchmark. + const MAIN: fn(Self::Input); +} diff --git a/benches/harness/lib/src/lib.rs b/benches/harness/lib/src/lib.rs new file mode 100644 index 000000000..1f8edeeb9 --- /dev/null +++ b/benches/harness/lib/src/lib.rs @@ -0,0 +1,97 @@ +//! TODO + +//---------------------------------------------------------------------------------------------------- Lints +// Forbid lints. +// Our code, and code generated (e.g macros) cannot overrule these. +#![forbid( + // `unsafe` is allowed but it _must_ be + // commented with `SAFETY: reason`. + clippy::undocumented_unsafe_blocks, + + // Never. + unused_unsafe, + redundant_semicolons, + unused_allocation, + coherence_leak_check, + while_true, + clippy::missing_docs_in_private_items, + + // Maybe can be put into `#[deny]`. + unconditional_recursion, + for_loops_over_fallibles, + unused_braces, + unused_labels, + keyword_idents, + non_ascii_idents, + variant_size_differences, + single_use_lifetimes, + + // Probably can be put into `#[deny]`. + future_incompatible, + let_underscore, + break_with_label_and_loop, + duplicate_macro_attributes, + exported_private_dependencies, + large_assignments, + overlapping_range_endpoints, + semicolon_in_expressions_from_macros, + noop_method_call, + unreachable_pub, +)] +// Deny lints. +// Some of these are `#[allow]`'ed on a per-case basis. +#![deny( + clippy::all, + clippy::correctness, + clippy::suspicious, + clippy::style, + clippy::complexity, + clippy::perf, + clippy::pedantic, + clippy::nursery, + clippy::cargo, + unused_doc_comments, + unused_mut, + missing_docs, + deprecated, + unused_comparisons, + nonstandard_style +)] +#![allow( + // FIXME: this lint affects crates outside of + // `database/` for some reason, allow for now. + clippy::cargo_common_metadata, + + // FIXME: adding `#[must_use]` onto everything + // might just be more annoying than useful... + // although it is sometimes nice. + clippy::must_use_candidate, + + // FIXME: good lint but too many false positives + // with our `Env` + `RwLock` setup. + clippy::significant_drop_tightening, + + // FIXME: good lint but is less clear in most cases. + clippy::items_after_statements, + + clippy::module_name_repetitions, + clippy::module_inception, + clippy::redundant_pub_crate, + clippy::option_if_let_else, +)] +// Allow some lints when running in debug mode. +#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] +// Allow some lints in tests. +#![cfg_attr( + test, + allow( + clippy::cognitive_complexity, + clippy::needless_pass_by_value, + clippy::cast_possible_truncation, + clippy::too_many_lines + ) +)] + +//---------------------------------------------------------------------------------------------------- Modules +mod benchmark; +pub use benchmark::Benchmark; diff --git a/benches/harness/src/main.rs b/benches/harness/src/main.rs new file mode 100644 index 000000000..2987b7243 --- /dev/null +++ b/benches/harness/src/main.rs @@ -0,0 +1,120 @@ +//! TODO + +//---------------------------------------------------------------------------------------------------- Lints +// Forbid lints. +// Our code, and code generated (e.g macros) cannot overrule these. +#![forbid( + // `unsafe` is allowed but it _must_ be + // commented with `SAFETY: reason`. + clippy::undocumented_unsafe_blocks, + + // Never. + unused_unsafe, + redundant_semicolons, + unused_allocation, + coherence_leak_check, + while_true, + clippy::missing_docs_in_private_items, + + // Maybe can be put into `#[deny]`. + unconditional_recursion, + for_loops_over_fallibles, + unused_braces, + unused_labels, + keyword_idents, + non_ascii_idents, + variant_size_differences, + single_use_lifetimes, + + // Probably can be put into `#[deny]`. + future_incompatible, + let_underscore, + break_with_label_and_loop, + duplicate_macro_attributes, + exported_private_dependencies, + large_assignments, + overlapping_range_endpoints, + semicolon_in_expressions_from_macros, + noop_method_call, + unreachable_pub, +)] +// Deny lints. +// Some of these are `#[allow]`'ed on a per-case basis. +#![deny( + clippy::all, + clippy::correctness, + clippy::suspicious, + clippy::style, + clippy::complexity, + clippy::perf, + clippy::pedantic, + clippy::nursery, + clippy::cargo, + unused_doc_comments, + unused_mut, + missing_docs, + deprecated, + unused_comparisons, + nonstandard_style +)] +#![allow( + // FIXME: this lint affects crates outside of + // `database/` for some reason, allow for now. + clippy::cargo_common_metadata, + + // FIXME: adding `#[must_use]` onto everything + // might just be more annoying than useful... + // although it is sometimes nice. + clippy::must_use_candidate, + + // FIXME: good lint but too many false positives + // with our `Env` + `RwLock` setup. + clippy::significant_drop_tightening, + + // FIXME: good lint but is less clear in most cases. + clippy::items_after_statements, + + clippy::module_name_repetitions, + clippy::module_inception, + clippy::redundant_pub_crate, + clippy::option_if_let_else, +)] +// Allow some lints when running in debug mode. +#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] +// Allow some lints in tests. +#![cfg_attr( + test, + allow( + clippy::cognitive_complexity, + clippy::needless_pass_by_value, + clippy::cast_possible_truncation, + clippy::too_many_lines + ) +)] + +//---------------------------------------------------------------------------------------------------- Modules +cfg_if::cfg_if! { + if #[cfg(feature = "database")] { + use cuprate_harness_database::BenchmarkDatabase as B; + } else if #[cfg(feature = "test")] { + use cuprate_harness_test::BenchmarkHarnessTest as B; + } else { + compile_error!("cuprate_harness: no feature specified. Use `--features $YOUR_CRATE_BENCHMARK_NAME` when building."); + } +} + +//---------------------------------------------------------------------------------------------------- Main +use cuprate_harness_lib::Benchmark; + +//---------------------------------------------------------------------------------------------------- Main + +#[allow(clippy::let_unit_value)] +fn main() { + let input = B::SETUP(); + let name = std::any::type_name::(); + println!("[Cuprate Harness] {name}"); + + let now = std::time::Instant::now(); + B::MAIN(input); + println!("{}", now.elapsed().as_secs_f32()); +} From 7bac741d5f023bdaa1bf783edaa77345a45a4ff4 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Thu, 3 Oct 2024 21:36:32 -0400 Subject: [PATCH 03/15] name changes, bin impl --- Cargo.lock | 42 +++--- Cargo.toml | 6 +- benches/benchmark/bin/Cargo.toml | 28 ++++ benches/benchmark/bin/README.md | 1 + benches/benchmark/bin/src/main.rs | 66 ++++++++++ benches/benchmark/example/Cargo.toml | 17 +++ benches/benchmark/example/README.md | 1 + benches/benchmark/example/src/lib.rs | 10 ++ benches/benchmark/lib/Cargo.toml | 18 +++ benches/benchmark/lib/README.md | 1 + .../lib/src/benchmark.rs | 11 +- benches/benchmark/lib/src/lib.rs | 5 + benches/harness/Cargo.toml | 23 ---- benches/harness/harness-test/Cargo.toml | 14 -- benches/harness/harness-test/src/lib.rs | 16 --- benches/harness/lib/Cargo.toml | 15 --- benches/harness/lib/src/lib.rs | 97 -------------- benches/harness/src/main.rs | 120 ------------------ 18 files changed, 179 insertions(+), 312 deletions(-) create mode 100644 benches/benchmark/bin/Cargo.toml create mode 100644 benches/benchmark/bin/README.md create mode 100644 benches/benchmark/bin/src/main.rs create mode 100644 benches/benchmark/example/Cargo.toml create mode 100644 benches/benchmark/example/README.md create mode 100644 benches/benchmark/example/src/lib.rs create mode 100644 benches/benchmark/lib/Cargo.toml create mode 100644 benches/benchmark/lib/README.md rename benches/{harness => benchmark}/lib/src/benchmark.rs (56%) create mode 100644 benches/benchmark/lib/src/lib.rs delete mode 100644 benches/harness/Cargo.toml delete mode 100644 benches/harness/harness-test/Cargo.toml delete mode 100644 benches/harness/harness-test/src/lib.rs delete mode 100644 benches/harness/lib/Cargo.toml delete mode 100644 benches/harness/lib/src/lib.rs delete mode 100644 benches/harness/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 82fbf8dfb..ac74f503b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -632,6 +632,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "cuprate-benchmark" +version = "0.0.0" +dependencies = [ + "cfg-if", + "cuprate-benchmark-example", + "cuprate-benchmark-lib", + "cuprate-database", +] + +[[package]] +name = "cuprate-benchmark-example" +version = "0.0.0" +dependencies = [ + "cuprate-benchmark-lib", +] + +[[package]] +name = "cuprate-benchmark-lib" +version = "0.0.0" + [[package]] name = "cuprate-blockchain" version = "0.0.0" @@ -821,27 +842,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "cuprate-harness" -version = "0.0.0" -dependencies = [ - "cfg-if", - "cuprate-database", - "cuprate-harness-lib", - "cuprate-harness-test", -] - -[[package]] -name = "cuprate-harness-lib" -version = "0.0.0" - -[[package]] -name = "cuprate-harness-test" -version = "0.0.0" -dependencies = [ - "cuprate-harness-lib", -] - [[package]] name = "cuprate-helper" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0d93a285f..08fbf3f07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,9 +4,9 @@ members = [ # Binaries "binaries/cuprated", # Benchmarks - "benches/harness", - "benches/harness/lib", - "benches/harness/harness-test", + "benches/benchmark/bin", + "benches/benchmark/lib", + "benches/benchmark/example", "benches/criterion/cuprate-json-rpc", # Consensus "consensus", diff --git a/benches/benchmark/bin/Cargo.toml b/benches/benchmark/bin/Cargo.toml new file mode 100644 index 000000000..bee6745d2 --- /dev/null +++ b/benches/benchmark/bin/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "cuprate-benchmark" +version = "0.0.0" +edition = "2021" +description = "Cuprate's benchmarking binary" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin" +keywords = ["cuprate", "benchmarking", "binary"] + +[features] +default = ["example"] +example = ["dep:cuprate-benchmark-example"] +database = ["dep:cuprate-database"] + +[dependencies] +cuprate-benchmark-lib = { path = "../lib" } +cuprate-benchmark-example = { path = "../example", optional = true } +cuprate-database = { path = "../../../storage/database", optional = true } + +cfg-if = { workspace = true } +# serde = { workspace = true, features = ["derive"] } +# serde_json = { workspace = true, features = ["std"] } + +[dev-dependencies] + +[lints] +workspace = true \ No newline at end of file diff --git a/benches/benchmark/bin/README.md b/benches/benchmark/bin/README.md new file mode 100644 index 000000000..562330cd9 --- /dev/null +++ b/benches/benchmark/bin/README.md @@ -0,0 +1 @@ +## `cuprate-benchmark` diff --git a/benches/benchmark/bin/src/main.rs b/benches/benchmark/bin/src/main.rs new file mode 100644 index 000000000..fb635c365 --- /dev/null +++ b/benches/benchmark/bin/src/main.rs @@ -0,0 +1,66 @@ +#![doc = include_str!("../README.md")] +#![allow( + unused_crate_dependencies, + reason = "this crate imports many potentially unused dependencies" +)] + +use std::{collections::HashMap, io::Write}; + +use cfg_if::cfg_if; + +use cuprate_benchmark_lib::Benchmark; + +fn main() { + let mut timings = HashMap::new(); + + cfg_if! { + if #[cfg(not(any(feature = "database", feature = "example")))] { + compile_error!("[cuprate_benchmark]: no feature specified. Use `--features $BENCHMARK_FEATURE` when building."); + } + } + + cfg_if! { + if #[cfg(feature = "database")] { + run_benchmark::(&mut timings); + } + } + + cfg_if! { + if #[cfg(feature = "example")] { + run_benchmark::(&mut timings); + } + } + + print_timings(&timings); +} + +fn run_benchmark(timings: &mut HashMap<&'static str, f32>) { + let name = std::any::type_name::(); + + print!("{name:>34} ... "); + std::io::stdout().flush().unwrap(); + + let input = B::SETUP(); + + let now = std::time::Instant::now(); + B::MAIN(input); + let time = now.elapsed().as_secs_f32(); + + println!("{time}"); + assert!( + timings.insert(name, time).is_none(), + "[cuprate_benchmark]: there were 2 benchmarks with the same name - this collides the final output: {name}", + ); +} + +fn print_timings(timings: &HashMap<&'static str, f32>) { + let mut s = String::new(); + s.push_str("| Benchmark | Time (seconds) |\n"); + s.push_str("|------------------------------------|----------------|"); + #[expect(clippy::iter_over_hash_type)] + for (k, v) in timings { + s += &format!("\n| {k:<34} | {v:<14} |"); + } + + println!("\n{s}"); +} diff --git a/benches/benchmark/example/Cargo.toml b/benches/benchmark/example/Cargo.toml new file mode 100644 index 000000000..5728bcd4b --- /dev/null +++ b/benches/benchmark/example/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "cuprate-benchmark-example" +version = "0.0.0" +edition = "2021" +description = "Example showcasing Cuprate's benchmarking harness" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/example" +keywords = ["cuprate", "benchmarking", "example"] + +[dependencies] +cuprate-benchmark-lib = { path = "../lib" } + +[dev-dependencies] + +[lints] +workspace = true \ No newline at end of file diff --git a/benches/benchmark/example/README.md b/benches/benchmark/example/README.md new file mode 100644 index 000000000..65177df97 --- /dev/null +++ b/benches/benchmark/example/README.md @@ -0,0 +1 @@ +## `cuprate-benchmark-example` diff --git a/benches/benchmark/example/src/lib.rs b/benches/benchmark/example/src/lib.rs new file mode 100644 index 000000000..6a60beb6e --- /dev/null +++ b/benches/benchmark/example/src/lib.rs @@ -0,0 +1,10 @@ +#![doc = include_str!("../README.md")] + +/// TODO +pub struct Example; + +impl cuprate_benchmark_lib::Benchmark for Example { + type Input = (); + const SETUP: fn() -> Self::Input = || {}; + const MAIN: fn(Self::Input) = |()| {}; +} diff --git a/benches/benchmark/lib/Cargo.toml b/benches/benchmark/lib/Cargo.toml new file mode 100644 index 000000000..b0771f09f --- /dev/null +++ b/benches/benchmark/lib/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "cuprate-benchmark-lib" +version = "0.0.0" +edition = "2021" +description = "Cuprate's benchmarking library" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/lib" +keywords = ["cuprate", "benchmarking", "library"] + +[features] + +[dependencies] + +[dev-dependencies] + +[lints] +workspace = true \ No newline at end of file diff --git a/benches/benchmark/lib/README.md b/benches/benchmark/lib/README.md new file mode 100644 index 000000000..5ad92b704 --- /dev/null +++ b/benches/benchmark/lib/README.md @@ -0,0 +1 @@ +## `cuprate-benchmark-lib` diff --git a/benches/harness/lib/src/benchmark.rs b/benches/benchmark/lib/src/benchmark.rs similarity index 56% rename from benches/harness/lib/src/benchmark.rs rename to benches/benchmark/lib/src/benchmark.rs index 3f46ff66f..bbd6ae611 100644 --- a/benches/harness/lib/src/benchmark.rs +++ b/benches/benchmark/lib/src/benchmark.rs @@ -1,16 +1,21 @@ //! TODO -//---------------------------------------------------------------------------------------------------- Use - -//---------------------------------------------------------------------------------------------------- trait Benchmark /// A benchmarking function and its inputs. pub trait Benchmark { /// Input to the main benchmarking function. + /// + /// This is passed to [`Self::MAIN`]. type Input; /// Setup function to generate the input. + /// + /// This function is not timed. const SETUP: fn() -> Self::Input; /// The main function to benchmark. + /// + /// The start of the timer begins right before + /// this function is called and ends after the + /// function returns. const MAIN: fn(Self::Input); } diff --git a/benches/benchmark/lib/src/lib.rs b/benches/benchmark/lib/src/lib.rs new file mode 100644 index 000000000..a6bec8208 --- /dev/null +++ b/benches/benchmark/lib/src/lib.rs @@ -0,0 +1,5 @@ +#![doc = include_str!("../README.md")] + +mod benchmark; + +pub use benchmark::Benchmark; diff --git a/benches/harness/Cargo.toml b/benches/harness/Cargo.toml deleted file mode 100644 index 30a852cb8..000000000 --- a/benches/harness/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "cuprate-harness" -version = "0.0.0" -edition = "2021" -description = "Cuprate's benchmarking harness binary" -license = "MIT" -authors = ["hinto-janai"] -repository = "https://github.com/Cuprate/cuprate/tree/main/benches/harness" -keywords = ["cuprate", "benchmarking", "harness", "binary"] - -[features] -default = ["test"] -test = ["dep:cuprate-harness-test"] -database = ["dep:cuprate-database"] - -[dependencies] -cuprate-harness-lib = { path = "lib" } -cuprate-harness-test = { path = "harness-test", optional = true } -cuprate-database = { path = "../../storage/database", optional = true } - -cfg-if = { workspace = true } - -[dev-dependencies] \ No newline at end of file diff --git a/benches/harness/harness-test/Cargo.toml b/benches/harness/harness-test/Cargo.toml deleted file mode 100644 index 278ddd64c..000000000 --- a/benches/harness/harness-test/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "cuprate-harness-test" -version = "0.0.0" -edition = "2021" -description = "Test for Cuprate's benchmarking harness" -license = "MIT" -authors = ["hinto-janai"] -repository = "https://github.com/Cuprate/cuprate/tree/main/benches/macro/harness-test" -keywords = ["cuprate", "benchmarking", "harness", "test"] - -[dependencies] -cuprate-harness-lib = { path = "../lib" } - -[dev-dependencies] \ No newline at end of file diff --git a/benches/harness/harness-test/src/lib.rs b/benches/harness/harness-test/src/lib.rs deleted file mode 100644 index beb360fd7..000000000 --- a/benches/harness/harness-test/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! TODO - -/// TODO -pub struct BenchmarkHarnessTest; - -// TODO -impl cuprate_harness_lib::Benchmark for BenchmarkHarnessTest { - /// TODO - type Input = (); - - /// TODO - const SETUP: fn() -> Self::Input = || {}; - - /// TODO - const MAIN: fn(Self::Input) = |_| {}; -} diff --git a/benches/harness/lib/Cargo.toml b/benches/harness/lib/Cargo.toml deleted file mode 100644 index 965117ee1..000000000 --- a/benches/harness/lib/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "cuprate-harness-lib" -version = "0.0.0" -edition = "2021" -description = "Cuprate's benchmarking harness library" -license = "MIT" -authors = ["hinto-janai"] -repository = "https://github.com/Cuprate/cuprate/tree/main/benches/harness/core" -keywords = ["cuprate", "benchmarking", "harness", "library"] - -[features] - -[dependencies] - -[dev-dependencies] \ No newline at end of file diff --git a/benches/harness/lib/src/lib.rs b/benches/harness/lib/src/lib.rs deleted file mode 100644 index 1f8edeeb9..000000000 --- a/benches/harness/lib/src/lib.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! TODO - -//---------------------------------------------------------------------------------------------------- Lints -// Forbid lints. -// Our code, and code generated (e.g macros) cannot overrule these. -#![forbid( - // `unsafe` is allowed but it _must_ be - // commented with `SAFETY: reason`. - clippy::undocumented_unsafe_blocks, - - // Never. - unused_unsafe, - redundant_semicolons, - unused_allocation, - coherence_leak_check, - while_true, - clippy::missing_docs_in_private_items, - - // Maybe can be put into `#[deny]`. - unconditional_recursion, - for_loops_over_fallibles, - unused_braces, - unused_labels, - keyword_idents, - non_ascii_idents, - variant_size_differences, - single_use_lifetimes, - - // Probably can be put into `#[deny]`. - future_incompatible, - let_underscore, - break_with_label_and_loop, - duplicate_macro_attributes, - exported_private_dependencies, - large_assignments, - overlapping_range_endpoints, - semicolon_in_expressions_from_macros, - noop_method_call, - unreachable_pub, -)] -// Deny lints. -// Some of these are `#[allow]`'ed on a per-case basis. -#![deny( - clippy::all, - clippy::correctness, - clippy::suspicious, - clippy::style, - clippy::complexity, - clippy::perf, - clippy::pedantic, - clippy::nursery, - clippy::cargo, - unused_doc_comments, - unused_mut, - missing_docs, - deprecated, - unused_comparisons, - nonstandard_style -)] -#![allow( - // FIXME: this lint affects crates outside of - // `database/` for some reason, allow for now. - clippy::cargo_common_metadata, - - // FIXME: adding `#[must_use]` onto everything - // might just be more annoying than useful... - // although it is sometimes nice. - clippy::must_use_candidate, - - // FIXME: good lint but too many false positives - // with our `Env` + `RwLock` setup. - clippy::significant_drop_tightening, - - // FIXME: good lint but is less clear in most cases. - clippy::items_after_statements, - - clippy::module_name_repetitions, - clippy::module_inception, - clippy::redundant_pub_crate, - clippy::option_if_let_else, -)] -// Allow some lints when running in debug mode. -#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] -// Allow some lints in tests. -#![cfg_attr( - test, - allow( - clippy::cognitive_complexity, - clippy::needless_pass_by_value, - clippy::cast_possible_truncation, - clippy::too_many_lines - ) -)] - -//---------------------------------------------------------------------------------------------------- Modules -mod benchmark; -pub use benchmark::Benchmark; diff --git a/benches/harness/src/main.rs b/benches/harness/src/main.rs deleted file mode 100644 index 2987b7243..000000000 --- a/benches/harness/src/main.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! TODO - -//---------------------------------------------------------------------------------------------------- Lints -// Forbid lints. -// Our code, and code generated (e.g macros) cannot overrule these. -#![forbid( - // `unsafe` is allowed but it _must_ be - // commented with `SAFETY: reason`. - clippy::undocumented_unsafe_blocks, - - // Never. - unused_unsafe, - redundant_semicolons, - unused_allocation, - coherence_leak_check, - while_true, - clippy::missing_docs_in_private_items, - - // Maybe can be put into `#[deny]`. - unconditional_recursion, - for_loops_over_fallibles, - unused_braces, - unused_labels, - keyword_idents, - non_ascii_idents, - variant_size_differences, - single_use_lifetimes, - - // Probably can be put into `#[deny]`. - future_incompatible, - let_underscore, - break_with_label_and_loop, - duplicate_macro_attributes, - exported_private_dependencies, - large_assignments, - overlapping_range_endpoints, - semicolon_in_expressions_from_macros, - noop_method_call, - unreachable_pub, -)] -// Deny lints. -// Some of these are `#[allow]`'ed on a per-case basis. -#![deny( - clippy::all, - clippy::correctness, - clippy::suspicious, - clippy::style, - clippy::complexity, - clippy::perf, - clippy::pedantic, - clippy::nursery, - clippy::cargo, - unused_doc_comments, - unused_mut, - missing_docs, - deprecated, - unused_comparisons, - nonstandard_style -)] -#![allow( - // FIXME: this lint affects crates outside of - // `database/` for some reason, allow for now. - clippy::cargo_common_metadata, - - // FIXME: adding `#[must_use]` onto everything - // might just be more annoying than useful... - // although it is sometimes nice. - clippy::must_use_candidate, - - // FIXME: good lint but too many false positives - // with our `Env` + `RwLock` setup. - clippy::significant_drop_tightening, - - // FIXME: good lint but is less clear in most cases. - clippy::items_after_statements, - - clippy::module_name_repetitions, - clippy::module_inception, - clippy::redundant_pub_crate, - clippy::option_if_let_else, -)] -// Allow some lints when running in debug mode. -#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] -// Allow some lints in tests. -#![cfg_attr( - test, - allow( - clippy::cognitive_complexity, - clippy::needless_pass_by_value, - clippy::cast_possible_truncation, - clippy::too_many_lines - ) -)] - -//---------------------------------------------------------------------------------------------------- Modules -cfg_if::cfg_if! { - if #[cfg(feature = "database")] { - use cuprate_harness_database::BenchmarkDatabase as B; - } else if #[cfg(feature = "test")] { - use cuprate_harness_test::BenchmarkHarnessTest as B; - } else { - compile_error!("cuprate_harness: no feature specified. Use `--features $YOUR_CRATE_BENCHMARK_NAME` when building."); - } -} - -//---------------------------------------------------------------------------------------------------- Main -use cuprate_harness_lib::Benchmark; - -//---------------------------------------------------------------------------------------------------- Main - -#[allow(clippy::let_unit_value)] -fn main() { - let input = B::SETUP(); - let name = std::any::type_name::(); - println!("[Cuprate Harness] {name}"); - - let now = std::time::Instant::now(); - B::MAIN(input); - println!("{}", now.elapsed().as_secs_f32()); -} From 24d176ff147d27b82c32b4673efdb34b4bb1ab54 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Fri, 4 Oct 2024 17:37:38 -0400 Subject: [PATCH 04/15] example, docs --- Cargo.lock | 2 ++ benches/benchmark/bin/Cargo.toml | 5 +-- benches/benchmark/bin/README.md | 18 ++++++++++ benches/benchmark/bin/src/main.rs | 49 ++++++-------------------- benches/benchmark/bin/src/print.rs | 35 ++++++++++++++++++ benches/benchmark/bin/src/run.rs | 34 ++++++++++++++++++ benches/benchmark/bin/src/timings.rs | 5 +++ benches/benchmark/example/README.md | 2 ++ benches/benchmark/example/src/lib.rs | 42 +++++++++++++++++++--- benches/benchmark/lib/src/benchmark.rs | 15 +++++++- 10 files changed, 161 insertions(+), 46 deletions(-) create mode 100644 benches/benchmark/bin/src/print.rs create mode 100644 benches/benchmark/bin/src/run.rs create mode 100644 benches/benchmark/bin/src/timings.rs diff --git a/Cargo.lock b/Cargo.lock index ac74f503b..f7b24b83b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -640,6 +640,8 @@ dependencies = [ "cuprate-benchmark-example", "cuprate-benchmark-lib", "cuprate-database", + "serde", + "serde_json", ] [[package]] diff --git a/benches/benchmark/bin/Cargo.toml b/benches/benchmark/bin/Cargo.toml index bee6745d2..e7267d4c3 100644 --- a/benches/benchmark/bin/Cargo.toml +++ b/benches/benchmark/bin/Cargo.toml @@ -10,6 +10,7 @@ keywords = ["cuprate", "benchmarking", "binary"] [features] default = ["example"] +json = [] example = ["dep:cuprate-benchmark-example"] database = ["dep:cuprate-database"] @@ -19,8 +20,8 @@ cuprate-benchmark-example = { path = "../example", optional = true } cuprate-database = { path = "../../../storage/database", optional = true } cfg-if = { workspace = true } -# serde = { workspace = true, features = ["derive"] } -# serde_json = { workspace = true, features = ["std"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } [dev-dependencies] diff --git a/benches/benchmark/bin/README.md b/benches/benchmark/bin/README.md index 562330cd9..515f5bb17 100644 --- a/benches/benchmark/bin/README.md +++ b/benches/benchmark/bin/README.md @@ -1 +1,19 @@ ## `cuprate-benchmark` +This crate links all benchmarks together into a single binary that can be run as: `cuprate-benchmark`. + +`cuprate-benchmark` will run all enabled benchmarks sequentially and print data at the end. + +## Benchmarks +Benchmarks are opt-in and enabled via features. + +| Feature | Enables which benchmark crate? | +|----------|--------------------------------| +| example | cuprate-benchmark-example | +| database | cuprate-benchmark-database | + +## Features +These are features that aren't for enabling benchmarks, but rather for other things. + +| Features | Does what | +|----------|-----------| +| json | Prints JSON timings instead of a markdown table \ No newline at end of file diff --git a/benches/benchmark/bin/src/main.rs b/benches/benchmark/bin/src/main.rs index fb635c365..03c9696ee 100644 --- a/benches/benchmark/bin/src/main.rs +++ b/benches/benchmark/bin/src/main.rs @@ -4,14 +4,18 @@ reason = "this crate imports many potentially unused dependencies" )] -use std::{collections::HashMap, io::Write}; +mod print; +mod run; +mod timings; use cfg_if::cfg_if; -use cuprate_benchmark_lib::Benchmark; - +/// What `main()` does: +/// 1. Run all enabled benchmarks +/// 2. Record benchmark timings +/// 3. Print timing data fn main() { - let mut timings = HashMap::new(); + let mut timings = timings::Timings::new(); cfg_if! { if #[cfg(not(any(feature = "database", feature = "example")))] { @@ -21,46 +25,15 @@ fn main() { cfg_if! { if #[cfg(feature = "database")] { - run_benchmark::(&mut timings); + run::run_benchmark::(&mut timings); } } cfg_if! { if #[cfg(feature = "example")] { - run_benchmark::(&mut timings); + run::run_benchmark::(&mut timings); } } - print_timings(&timings); -} - -fn run_benchmark(timings: &mut HashMap<&'static str, f32>) { - let name = std::any::type_name::(); - - print!("{name:>34} ... "); - std::io::stdout().flush().unwrap(); - - let input = B::SETUP(); - - let now = std::time::Instant::now(); - B::MAIN(input); - let time = now.elapsed().as_secs_f32(); - - println!("{time}"); - assert!( - timings.insert(name, time).is_none(), - "[cuprate_benchmark]: there were 2 benchmarks with the same name - this collides the final output: {name}", - ); -} - -fn print_timings(timings: &HashMap<&'static str, f32>) { - let mut s = String::new(); - s.push_str("| Benchmark | Time (seconds) |\n"); - s.push_str("|------------------------------------|----------------|"); - #[expect(clippy::iter_over_hash_type)] - for (k, v) in timings { - s += &format!("\n| {k:<34} | {v:<14} |"); - } - - println!("\n{s}"); + print::print_timings(&timings); } diff --git a/benches/benchmark/bin/src/print.rs b/benches/benchmark/bin/src/print.rs new file mode 100644 index 000000000..070d2575d --- /dev/null +++ b/benches/benchmark/bin/src/print.rs @@ -0,0 +1,35 @@ +use cfg_if::cfg_if; + +use crate::timings::Timings; + +/// Print the final the final markdown table of benchmark timings. +pub(crate) fn print_timings(timings: &Timings) { + cfg_if! { + if #[cfg(feature = "json")] { + print_timings_json(timings); + } else { + print_timings_markdown(timings); + } + } +} + +/// Default timing formatting. +pub(crate) fn print_timings_markdown(timings: &Timings) { + let mut s = String::new(); + s.push_str("| Benchmark | Time (seconds) |\n"); + s.push_str("|------------------------------------|----------------|"); + + #[expect(clippy::iter_over_hash_type)] + for (k, v) in timings { + s += &format!("\n| {k:<34} | {v:<14} |"); + } + + println!("\n{s}"); +} + +/// Enabled via `json` feature. +#[expect(dead_code)] +pub(crate) fn print_timings_json(timings: &Timings) { + let json = serde_json::to_string_pretty(timings).unwrap(); + println!("\n{json}"); +} diff --git a/benches/benchmark/bin/src/run.rs b/benches/benchmark/bin/src/run.rs new file mode 100644 index 000000000..8119048d2 --- /dev/null +++ b/benches/benchmark/bin/src/run.rs @@ -0,0 +1,34 @@ +use std::io::Write; + +use cuprate_benchmark_lib::Benchmark; + +use crate::timings::Timings; + +/// Run a [`Benchmark`] and record its timing. +pub(crate) fn run_benchmark(timings: &mut Timings) { + // Print the benchmark name. + let name = std::any::type_name::(); + print!("{name:>34} ... "); + std::io::stdout().flush().unwrap(); + + // Setup the benchmark input. + let input = B::SETUP(); + + // Sleep before running the benchmark. + std::thread::sleep(B::PRE_SLEEP_DURATION); + + // Run/time the benchmark. + let now = std::time::Instant::now(); + B::MAIN(input); + let time = now.elapsed().as_secs_f32(); + + // Print the benchmark timings. + println!("{time}"); + assert!( + timings.insert(name, time).is_none(), + "[cuprate_benchmark]: there were 2 benchmarks with the same name - this collides the final output: {name}", + ); + + // Sleep for a cooldown period after the benchmark run. + std::thread::sleep(B::POST_SLEEP_DURATION); +} diff --git a/benches/benchmark/bin/src/timings.rs b/benches/benchmark/bin/src/timings.rs new file mode 100644 index 000000000..34a07952b --- /dev/null +++ b/benches/benchmark/bin/src/timings.rs @@ -0,0 +1,5 @@ +/// Benchmark timing data. +/// +/// - Key = benchmark name +/// - Value = benchmark time in seconds +pub(crate) type Timings = std::collections::HashMap<&'static str, f32>; diff --git a/benches/benchmark/example/README.md b/benches/benchmark/example/README.md index 65177df97..be6b71679 100644 --- a/benches/benchmark/example/README.md +++ b/benches/benchmark/example/README.md @@ -1 +1,3 @@ ## `cuprate-benchmark-example` +This crate contains a short example benchmark that shows how to implement and use +`cuprate-benchmark-lib` so that it can be ran by `cuprate-benchmark`. \ No newline at end of file diff --git a/benches/benchmark/example/src/lib.rs b/benches/benchmark/example/src/lib.rs index 6a60beb6e..cc704a7f4 100644 --- a/benches/benchmark/example/src/lib.rs +++ b/benches/benchmark/example/src/lib.rs @@ -1,10 +1,42 @@ #![doc = include_str!("../README.md")] -/// TODO +use std::hint::black_box; + +use cuprate_benchmark_lib::Benchmark; + +/// Marker struct that implements [`Benchmark`] pub struct Example; -impl cuprate_benchmark_lib::Benchmark for Example { - type Input = (); - const SETUP: fn() -> Self::Input = || {}; - const MAIN: fn(Self::Input) = |()| {}; +/// The input to our benchmark function. +pub type ExampleBenchmarkInput = u64; + +/// The setup function that creates the input. +pub const fn example_benchmark_setup() -> ExampleBenchmarkInput { + 1 +} + +/// The main benchmarking function. +#[expect(clippy::unit_arg)] +pub fn example_benchmark_main(input: ExampleBenchmarkInput) { + // In this case, we're simply benchmarking the + // performance of simple arithmetic on the input data. + + fn math(input: ExampleBenchmarkInput, number: u64) { + let x = input; + let x = black_box(x * number); + let x = black_box(x / number); + let x = black_box(x + number); + let _ = black_box(x - number); + } + + for number in 1..100_000_000 { + black_box(math(input, number)); + } +} + +// This implementation will be run by `cuprate-benchmark`. +impl Benchmark for Example { + type Input = ExampleBenchmarkInput; + const SETUP: fn() -> Self::Input = example_benchmark_setup; + const MAIN: fn(Self::Input) = example_benchmark_main; } diff --git a/benches/benchmark/lib/src/benchmark.rs b/benches/benchmark/lib/src/benchmark.rs index bbd6ae611..3e99964ab 100644 --- a/benches/benchmark/lib/src/benchmark.rs +++ b/benches/benchmark/lib/src/benchmark.rs @@ -1,4 +1,6 @@ -//! TODO +//! Benchmarking trait. + +use std::time::Duration; /// A benchmarking function and its inputs. pub trait Benchmark { @@ -18,4 +20,15 @@ pub trait Benchmark { /// this function is called and ends after the /// function returns. const MAIN: fn(Self::Input); + + /// `cuprate-benchmark` will sleep for this [`Duration`] after + /// creating the [`Self::Input`], but before starting [`Self::Main`]. + /// + /// 1 second by default. + const PRE_SLEEP_DURATION: Duration = Duration::from_secs(1); + + /// `cuprate-benchmark` will sleep for this [`Duration`] after [`Self::Main`]. + /// + /// 1 second by default. + const POST_SLEEP_DURATION: Duration = Duration::from_secs(1); } From 65253077444c2ea4a54c4eb80e2abeeedfc8eea7 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Mon, 7 Oct 2024 20:33:44 -0400 Subject: [PATCH 05/15] book --- Cargo.lock | 2 - benches/README.md | 82 +------------------ benches/benchmark/bin/Cargo.toml | 7 +- benches/benchmark/bin/src/main.rs | 13 ++- benches/criterion/cuprate-json-rpc/Cargo.toml | 6 +- .../cuprate-json-rpc/benches/main.rs | 77 +---------------- .../cuprate-json-rpc/benches/response.rs | 11 +-- benches/criterion/cuprate-json-rpc/src/lib.rs | 77 +---------------- books/architecture/src/SUMMARY.md | 13 ++- .../src/benchmarking/criterion.md | 1 - .../src/benchmarking/criterion/creating.md | 10 +++ .../src/benchmarking/criterion/intro.md | 6 ++ .../src/benchmarking/criterion/running.md | 15 ++++ .../src/benchmarking/cuprate/creating.md | 42 ++++++++++ .../src/benchmarking/cuprate/intro.md | 12 +++ .../src/benchmarking/cuprate/running.md | 16 ++++ .../architecture/src/benchmarking/harness.md | 1 - books/architecture/src/benchmarking/intro.md | 23 +++++- 18 files changed, 156 insertions(+), 258 deletions(-) delete mode 100644 books/architecture/src/benchmarking/criterion.md create mode 100644 books/architecture/src/benchmarking/criterion/creating.md create mode 100644 books/architecture/src/benchmarking/criterion/intro.md create mode 100644 books/architecture/src/benchmarking/criterion/running.md create mode 100644 books/architecture/src/benchmarking/cuprate/creating.md create mode 100644 books/architecture/src/benchmarking/cuprate/intro.md create mode 100644 books/architecture/src/benchmarking/cuprate/running.md delete mode 100644 books/architecture/src/benchmarking/harness.md diff --git a/Cargo.lock b/Cargo.lock index f7b24b83b..8842f5b2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -639,7 +639,6 @@ dependencies = [ "cfg-if", "cuprate-benchmark-example", "cuprate-benchmark-lib", - "cuprate-database", "serde", "serde_json", ] @@ -746,7 +745,6 @@ dependencies = [ "cuprate-json-rpc", "function_name", "serde_json", - "tempfile", ] [[package]] diff --git a/benches/README.md b/benches/README.md index c5c676717..af6bb932f 100644 --- a/benches/README.md +++ b/benches/README.md @@ -1,83 +1,5 @@ # Benches This directory contains Cuprate's benchmarks and benchmarking utilities. -- [1. File layout and purpose](#1-file-layout-and-purpose) -- [2. Harness](#2-harness) - - [2.1 Creating a harness benchmark](#21-creating-a-harness-benchmark) - - [2.2 Running a harness benchmark](#22-running-a-harness-benchmark) -- [3. Criterion](#3-criterion) - - [2.1 Creating a Criterion benchmark](#21-creating-a-criterion-benchmark) - - [2.2 Running a Criterion benchmark](#22-running-a-criterion-benchmark) - -## 1. File layout and purpose -This directory is sorted into 4 important categories: - -| Sub-directory | Purpose | -|---------------|---------| -| `harness/src` | Cuprate's custom benchmarking harness **binary** -| `harness/lib` | Cuprate's custom benchmarking harness **library** -| `harness/*` | Macro-benchmarks for whole crates or sub-systems (using Cuprate's custom benchmarking harness) -| `criterion/*` | Micro-benchmarks for crates (e.g. timings for a single function) - -## 2. Harness -The harness is: -- `cuprate-harness`; the actual binary crate ran -- `cuprate-harness-lib`; the library that other crates hook into - -The purpose of the harness is very simple: -1. Set-up the benchmark -1. Start timer -1. Run benchmark -1. Output data - -The harness runs the benchmarks found in `harness/`. - -The way benchmarks "plug-in" to the harness is simply by implementing `cuprate_harness_lib::Benchmark`. - -See `cuprate-harness-lib` crate documentation for a user-guide: -```bash -cargo doc --open --package cuprate-harness-lib -``` - -### 2.1 Creating a harness benchmark -1. Create a new crate inside `benches/harness` (consider copying `benches/harness/test` as a base) -2. Pull in `cuprate_harness_lib` as a dependency -3. Implement `cuprate_harness_lib::Benchmark` -4. Add a feature inside `cuprate_harness` for your benchmark - -### 2.2 Running a harness benchmark -After your benchmark is implemented, run this command: -```bash -cargo run --release --package cuprate-harness --features $YOUR_BENCHMARK_CRATE_FEATURE -``` -For example, to run the test benchmark: -```bash -cargo run --release --package cuprate-harness --features test -``` - -## 3. Criterion -Each sub-directory in here is a crate that uses [Criterion](https://bheisler.github.io/criterion.rs/book) for timing single functions and/or groups of functions. - -They are generally be small in scope. - -See [`criterion/cuprate-json-rpc`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc) for an example. - -### 3.1 Creating a Criterion benchmark -1. Copy [`criterion/test`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion) as base -2. Read the `Getting Started` section of -3. Get started - -### 3.1 Running a Criterion benchmark -To run all Criterion benchmarks, run this from the repository root: -```bash -cargo bench -``` - -To run specific package(s), use: -```bash -cargo bench --package $CRITERION_BENCHMARK_CRATE_NAME -``` -For example: -```bash -cargo bench --package cuprate-criterion-json-rpc -``` \ No newline at end of file +See the [`Benchmarking` section in the Architecture book](https://architecture.cuprate.org/benchmarking/intro.html) +to see how to create and run these benchmarks. \ No newline at end of file diff --git a/benches/benchmark/bin/Cargo.toml b/benches/benchmark/bin/Cargo.toml index e7267d4c3..86558d163 100644 --- a/benches/benchmark/bin/Cargo.toml +++ b/benches/benchmark/bin/Cargo.toml @@ -9,15 +9,16 @@ repository = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bi keywords = ["cuprate", "benchmarking", "binary"] [features] -default = ["example"] +# All new benchmarks should be added here! +all = ["example"] + +default = [] json = [] example = ["dep:cuprate-benchmark-example"] -database = ["dep:cuprate-database"] [dependencies] cuprate-benchmark-lib = { path = "../lib" } cuprate-benchmark-example = { path = "../example", optional = true } -cuprate-database = { path = "../../../storage/database", optional = true } cfg-if = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/benches/benchmark/bin/src/main.rs b/benches/benchmark/bin/src/main.rs index 03c9696ee..3c0946da0 100644 --- a/benches/benchmark/bin/src/main.rs +++ b/benches/benchmark/bin/src/main.rs @@ -14,21 +14,20 @@ use cfg_if::cfg_if; /// 1. Run all enabled benchmarks /// 2. Record benchmark timings /// 3. Print timing data +/// +/// To add a new benchmark to be ran here: +/// 1. Copy + paste a `cfg_if` block +/// 2. Change it to your benchmark's feature flag +/// 3. Change it to your benchmark's type fn main() { let mut timings = timings::Timings::new(); cfg_if! { - if #[cfg(not(any(feature = "database", feature = "example")))] { + if #[cfg(not(any(feature = "example")))] { compile_error!("[cuprate_benchmark]: no feature specified. Use `--features $BENCHMARK_FEATURE` when building."); } } - cfg_if! { - if #[cfg(feature = "database")] { - run::run_benchmark::(&mut timings); - } - } - cfg_if! { if #[cfg(feature = "example")] { run::run_benchmark::(&mut timings); diff --git a/benches/criterion/cuprate-json-rpc/Cargo.toml b/benches/criterion/cuprate-json-rpc/Cargo.toml index eaa264708..5d47e3ad7 100644 --- a/benches/criterion/cuprate-json-rpc/Cargo.toml +++ b/benches/criterion/cuprate-json-rpc/Cargo.toml @@ -12,10 +12,12 @@ keywords = ["cuprate", "json-rpc", "criterion", "benchmark"] criterion = { workspace = true } function_name = { workspace = true } serde_json = { workspace = true, features = ["default"] } -tempfile = { workspace = true } cuprate-json-rpc = { path = "../../../rpc/json-rpc" } [[bench]] name = "main" -harness = false \ No newline at end of file +harness = false + +[lints] +workspace = true \ No newline at end of file diff --git a/benches/criterion/cuprate-json-rpc/benches/main.rs b/benches/criterion/cuprate-json-rpc/benches/main.rs index 84585ee99..578dd8ed6 100644 --- a/benches/criterion/cuprate-json-rpc/benches/main.rs +++ b/benches/criterion/cuprate-json-rpc/benches/main.rs @@ -1,78 +1,5 @@ -//! TODO - -//---------------------------------------------------------------------------------------------------- Lints -// Forbid lints. -// Our code, and code generated (e.g macros) cannot overrule these. -#![forbid( - // `unsafe` is allowed but it _must_ be - // commented with `SAFETY: reason`. - clippy::undocumented_unsafe_blocks, - // Never. - unused_unsafe, - redundant_semicolons, - unused_allocation, - coherence_leak_check, - while_true, - clippy::missing_docs_in_private_items, - // Maybe can be put into `#[deny]`. - unconditional_recursion, - for_loops_over_fallibles, - unused_braces, - unused_doc_comments, - unused_labels, - keyword_idents, - non_ascii_idents, - variant_size_differences, - single_use_lifetimes, - // Probably can be put into `#[deny]`. - future_incompatible, - let_underscore, - break_with_label_and_loop, - duplicate_macro_attributes, - exported_private_dependencies, - large_assignments, - overlapping_range_endpoints, - semicolon_in_expressions_from_macros, - noop_method_call, - unreachable_pub, -)] -// Deny lints. -// Some of these are `#[allow]`'ed on a per-case basis. -#![deny( - clippy::all, - clippy::correctness, - clippy::suspicious, - clippy::style, - clippy::complexity, - clippy::perf, - clippy::pedantic, - clippy::nursery, - clippy::cargo, - unused_mut, - missing_docs, - deprecated, - unused_comparisons, - nonstandard_style -)] -#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove -#![allow( - // FIXME: this lint affects crates outside of - // `database/` for some reason, allow for now. - clippy::cargo_common_metadata, - // FIXME: adding `#[must_use]` onto everything - // might just be more annoying than useful... - // although it is sometimes nice. - clippy::must_use_candidate, - // TODO: should be removed after all `todo!()`'s are gone. - clippy::diverging_sub_expression, - clippy::module_name_repetitions, - clippy::module_inception, - clippy::redundant_pub_crate, - clippy::option_if_let_else, - clippy::significant_drop_tightening, -)] -// Allow some lints when running in debug mode. -#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] +//! Benchmarks for `cuprate-json-rpc`. +#![allow(unused_crate_dependencies)] mod response; diff --git a/benches/criterion/cuprate-json-rpc/benches/response.rs b/benches/criterion/cuprate-json-rpc/benches/response.rs index 93e209af9..c1a134119 100644 --- a/benches/criterion/cuprate-json-rpc/benches/response.rs +++ b/benches/criterion/cuprate-json-rpc/benches/response.rs @@ -1,13 +1,12 @@ -//! `trait Storable` benchmarks. +//! Benchmarks for [`Response`]. +#![allow(unused_attributes, unused_crate_dependencies)] -//---------------------------------------------------------------------------------------------------- Import use criterion::{black_box, criterion_group, criterion_main, Criterion}; use function_name::named; use serde_json::{from_str, to_string_pretty}; use cuprate_json_rpc::{Id, Response}; -//---------------------------------------------------------------------------------------------------- Criterion criterion_group! { benches, response_from_str_u8, @@ -25,8 +24,7 @@ criterion_group! { } criterion_main!(benches); -//---------------------------------------------------------------------------------------------------- Deserialization -/// TODO +/// Generate `from_str` deserialization benchmark functions for [`Response`]. macro_rules! impl_from_str_benchmark { ( $( @@ -60,8 +58,7 @@ impl_from_str_benchmark! { response_from_str_string_500_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld"}"#, } -//---------------------------------------------------------------------------------------------------- Deserialization -/// TODO +/// Generate `to_string_pretty` serialization benchmark functions for [`Response`]. macro_rules! impl_to_string_pretty_benchmark { ( $( diff --git a/benches/criterion/cuprate-json-rpc/src/lib.rs b/benches/criterion/cuprate-json-rpc/src/lib.rs index 00561eee1..8a892fcb5 100644 --- a/benches/criterion/cuprate-json-rpc/src/lib.rs +++ b/benches/criterion/cuprate-json-rpc/src/lib.rs @@ -1,77 +1,2 @@ //! TODO - -//---------------------------------------------------------------------------------------------------- Lints -// Forbid lints. -// Our code, and code generated (e.g macros) cannot overrule these. -#![forbid( - // `unsafe` is allowed but it _must_ be - // commented with `SAFETY: reason`. - clippy::undocumented_unsafe_blocks, - // Never. - unused_unsafe, - redundant_semicolons, - unused_allocation, - coherence_leak_check, - while_true, - clippy::missing_docs_in_private_items, - // Maybe can be put into `#[deny]`. - unconditional_recursion, - for_loops_over_fallibles, - unused_braces, - unused_doc_comments, - unused_labels, - keyword_idents, - non_ascii_idents, - variant_size_differences, - single_use_lifetimes, - // Probably can be put into `#[deny]`. - future_incompatible, - let_underscore, - break_with_label_and_loop, - duplicate_macro_attributes, - exported_private_dependencies, - large_assignments, - overlapping_range_endpoints, - semicolon_in_expressions_from_macros, - noop_method_call, - unreachable_pub, -)] -// Deny lints. -// Some of these are `#[allow]`'ed on a per-case basis. -#![deny( - clippy::all, - clippy::correctness, - clippy::suspicious, - clippy::style, - clippy::complexity, - clippy::perf, - clippy::pedantic, - clippy::nursery, - clippy::cargo, - unused_mut, - missing_docs, - deprecated, - unused_comparisons, - nonstandard_style -)] -#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove -#![allow( - // FIXME: this lint affects crates outside of - // `database/` for some reason, allow for now. - clippy::cargo_common_metadata, - // FIXME: adding `#[must_use]` onto everything - // might just be more annoying than useful... - // although it is sometimes nice. - clippy::must_use_candidate, - // TODO: should be removed after all `todo!()`'s are gone. - clippy::diverging_sub_expression, - clippy::module_name_repetitions, - clippy::module_inception, - clippy::redundant_pub_crate, - clippy::option_if_let_else, - clippy::significant_drop_tightening, -)] -// Allow some lints when running in debug mode. -#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] - -//---------------------------------------------------------------------------------------------------- Modules +#![allow(unused_crate_dependencies, reason = "used in benchmarks")] diff --git a/books/architecture/src/SUMMARY.md b/books/architecture/src/SUMMARY.md index bf6686097..ee4795bfd 100644 --- a/books/architecture/src/SUMMARY.md +++ b/books/architecture/src/SUMMARY.md @@ -143,9 +143,16 @@ --- -- [⚪️ Benchmarking](benchmarking/intro.md) - - [⚪️ Criterion](benchmarking/criterion.md) - - [⚪️ Harness](benchmarking/harness.md) +- [🟢 Benchmarking](benchmarking/intro.md) + - [🟢 Criterion](benchmarking/criterion/intro.md) + - [🟢 Creating](benchmarking/criterion/creating.md) + - [🟢 Running](benchmarking/criterion/running.md) + - [🟢 `cuprate-benchmark`](benchmarking/cuprate/intro.md) + - [🟢 Creating](benchmarking/cuprate/creating.md) + - [🟢 Running](benchmarking/cuprate/running.md) + +--- + - [⚪️ Testing](testing/intro.md) - [⚪️ Monero data](testing/monero-data.md) - [⚪️ RPC client](testing/rpc-client.md) diff --git a/books/architecture/src/benchmarking/criterion.md b/books/architecture/src/benchmarking/criterion.md deleted file mode 100644 index e9d61e657..000000000 --- a/books/architecture/src/benchmarking/criterion.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ Criterion diff --git a/books/architecture/src/benchmarking/criterion/creating.md b/books/architecture/src/benchmarking/criterion/creating.md new file mode 100644 index 000000000..faf78bbf6 --- /dev/null +++ b/books/architecture/src/benchmarking/criterion/creating.md @@ -0,0 +1,10 @@ +# Creating +Creating a new Criterion-based benchmarking crate for one of Cuprate's crates is relatively simple, +although, it requires knowledge of how to use Criterion first: + +1. Read the `Getting Started` section of +2. Copy [`benches/criterion/example`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/example) as base +3. Get started + +For a real example, see: +[`cuprate-criterion-json-rpc`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc). \ No newline at end of file diff --git a/books/architecture/src/benchmarking/criterion/intro.md b/books/architecture/src/benchmarking/criterion/intro.md new file mode 100644 index 000000000..ec9ef802e --- /dev/null +++ b/books/architecture/src/benchmarking/criterion/intro.md @@ -0,0 +1,6 @@ +# Criterion +Each sub-directory in [`benches/criterion/`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion) is a crate that uses [Criterion](https://bheisler.github.io/criterion.rs/book) for timing single functions and/or groups of functions. + +They are generally be small in scope. + +See [`benches/criterion/cuprate-json-rpc`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc) for an example. diff --git a/books/architecture/src/benchmarking/criterion/running.md b/books/architecture/src/benchmarking/criterion/running.md new file mode 100644 index 000000000..14067f67e --- /dev/null +++ b/books/architecture/src/benchmarking/criterion/running.md @@ -0,0 +1,15 @@ +# Running +To run all Criterion benchmarks, run this from the repository root: +```bash +cargo bench +``` + +To run specific package(s), use: +```bash +cargo bench --package $CRITERION_BENCHMARK_CRATE_NAME +``` + +For example: +```bash +cargo bench --package cuprate-criterion-json-rpc +``` \ No newline at end of file diff --git a/books/architecture/src/benchmarking/cuprate/creating.md b/books/architecture/src/benchmarking/cuprate/creating.md new file mode 100644 index 000000000..a8820849e --- /dev/null +++ b/books/architecture/src/benchmarking/cuprate/creating.md @@ -0,0 +1,42 @@ +# Creating +New benchmarks are plugged into `cuprate-benchmark` by: +1. Implementing `cuprate_benchmark_lib::Benchmark` +1. Registering the benchmark in the `cuprate_benchmark` binary + +See [`benches/benchmark/example`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/example) +for an example. For a real example, see: +[`cuprate-benchmark-database`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/cuprate-database). + +## Creating the benchmark crate +Before plugging into `cuprate-benchmark`, your actual benchmark crate must be created: + +1. Create a new crate inside `benches/benchmark` (consider copying `benches/benchmark/example` as a base) +1. Pull in `cuprate_benchmark_lib` as a dependency +1. Create a benchmark +1. Implement `cuprate_benchmark_lib::Benchmark` + +## `cuprate_benchmark_lib::Benchmark` +This is the trait that standarizes all benchmarks ran under `cuprate-benchmark`. + +It must be implemented by your benchmarking crate. + +See `cuprate-benchmark-lib` crate documentation for a user-guide: . + +## Adding a feature to `cuprate-benchmark` +After your benchmark's behavior is defined, it must be registered +in the binary that is actually ran: `cuprate-benchmark`. + +If your benchmark is new, add a new crate feature to [`cuprate-benchmark`'s Cargo.toml file](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin/Cargo.toml) with an optional dependency to your benchmarking crate. + +## Adding to `cuprate-benchmark`'s `main()` +After adding your crate's feature, add a conditional line that run the benchmark +if the feature is enabled to the `main()` function: + +For example, if your crate's name is `egg`: +```rust +cfg_if! { + if #[cfg(feature = "egg")] { + run::run_benchmark::(&mut timings); + } +} +``` \ No newline at end of file diff --git a/books/architecture/src/benchmarking/cuprate/intro.md b/books/architecture/src/benchmarking/cuprate/intro.md new file mode 100644 index 000000000..7a31c48bd --- /dev/null +++ b/books/architecture/src/benchmarking/cuprate/intro.md @@ -0,0 +1,12 @@ +# cuprate-benchmark +Cuprate has 2 custom crates for macro benchmarking: +- `cuprate-benchmark`; the actual binary crate ran +- `cuprate-benchmark-lib`; the library that other crates hook into + +The purpose of `cuprate-benchmark` is very simple: +1. Set-up the benchmark +1. Start timer +1. Run benchmark +1. Output data + +`cuprate-benchmark` runs the benchmarks found in [`benches/benchmark/cuprate-*`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark). diff --git a/books/architecture/src/benchmarking/cuprate/running.md b/books/architecture/src/benchmarking/cuprate/running.md new file mode 100644 index 000000000..22ee9a696 --- /dev/null +++ b/books/architecture/src/benchmarking/cuprate/running.md @@ -0,0 +1,16 @@ +# Running +`cuprate-benchmark` benchmarks are ran with this command: +```bash +cargo run --release --package cuprate-benchmark --features $YOUR_BENCHMARK_CRATE_FEATURE +``` + +For example, to run the example benchmark: +```bash +cargo run --release --package cuprate-benchmark --features example +``` + +Use the `all` feature to run all benchmarks: +```bash +# Run all benchmarks +cargo run --release --package cuprate-benchmark --features all +``` diff --git a/books/architecture/src/benchmarking/harness.md b/books/architecture/src/benchmarking/harness.md deleted file mode 100644 index 6f82b523e..000000000 --- a/books/architecture/src/benchmarking/harness.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ Harness diff --git a/books/architecture/src/benchmarking/intro.md b/books/architecture/src/benchmarking/intro.md index f043a0bac..77ad7e58b 100644 --- a/books/architecture/src/benchmarking/intro.md +++ b/books/architecture/src/benchmarking/intro.md @@ -1 +1,22 @@ -# ⚪️ Benchmarking +# Benchmarking +Cuprate has 2 types of benchmarks: +- Criterion benchmarks +- `cuprate-benchmark` benchmarks + +[Criterion](https://bheisler.github.io/criterion.rs/book/user_guide/advanced_configuration.html) is used for micro benchmarks; they time single functions, groups of functions, and generally are small in scope. + +`cuprate-benchmark` and `cuprate-benchmark-lib` are custom in-house crates Cuprate uses for macro benchmarks; these test sub-systems, sections of a sub-system, or otherwise larger or more complicated code that isn't suited for micro benchmarks. + +## File layout and purpose +All benchmarking related files are in the [`benches/`](https://github.com/Cuprate/cuprate/tree/main/benches) folder. + +This directory is organized like such: + +| Directory | Purpose | +|-------------------------------|---------| +| `benches/criterion/` | Criterion (micro) benchmarks +| `benches/criterion/cuprate-*` | Criterion benchmarks for the crate with the same name +| `benches/benchmark/` | Cuprate's custom benchmarking files +| `benches/benchmark/bin` | The `cuprate-benchmark` crate; the actual binary run that links all benchmarks +| `benches/benchmark/lib` | The `cuprate-benchmark-lib` crate; the benchmarking framework all benchmarks plug into +| `benches/benchmark/cuprate-*` | `cuprate-benchmark` benchmarks for the crate with the same name From 185c2ee25a79414f7df6c841591d2ed9d84ab42e Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Tue, 8 Oct 2024 17:42:21 -0400 Subject: [PATCH 06/15] add `cuprate-criterion-example` --- Cargo.lock | 9 ++++ Cargo.toml | 1 + benches/criterion/cuprate-json-rpc/Cargo.toml | 2 +- benches/criterion/example/Cargo.toml | 21 ++++++++ benches/criterion/example/README.md | 14 ++++++ benches/criterion/example/benches/example.rs | 48 +++++++++++++++++++ benches/criterion/example/benches/main.rs | 10 ++++ benches/criterion/example/src/lib.rs | 13 +++++ .../src/benchmarking/cuprate/creating.md | 2 +- 9 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 benches/criterion/example/Cargo.toml create mode 100644 benches/criterion/example/README.md create mode 100644 benches/criterion/example/benches/example.rs create mode 100644 benches/criterion/example/benches/main.rs create mode 100644 benches/criterion/example/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index aa7054b02..ea83921be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -737,6 +737,15 @@ dependencies = [ name = "cuprate-constants" version = "0.1.0" +[[package]] +name = "cuprate-criterion-example" +version = "0.0.0" +dependencies = [ + "criterion", + "function_name", + "serde_json", +] + [[package]] name = "cuprate-criterion-json-rpc" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 08fbf3f07..a402cbdc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "benches/benchmark/bin", "benches/benchmark/lib", "benches/benchmark/example", + "benches/criterion/example", "benches/criterion/cuprate-json-rpc", # Consensus "consensus", diff --git a/benches/criterion/cuprate-json-rpc/Cargo.toml b/benches/criterion/cuprate-json-rpc/Cargo.toml index 5d47e3ad7..08383255c 100644 --- a/benches/criterion/cuprate-json-rpc/Cargo.toml +++ b/benches/criterion/cuprate-json-rpc/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" description = "Criterion benchmarking for cuprate-json-rpc" license = "MIT" authors = ["hinto-janai"] -repository = "https://github.com/Cuprate/cuprate/tree/main/benches/micro/cuprate-json-rpc" +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc" keywords = ["cuprate", "json-rpc", "criterion", "benchmark"] [dependencies] diff --git a/benches/criterion/example/Cargo.toml b/benches/criterion/example/Cargo.toml new file mode 100644 index 000000000..43e601191 --- /dev/null +++ b/benches/criterion/example/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "cuprate-criterion-example" +version = "0.0.0" +edition = "2021" +description = "Criterion benchmarking example for Cuprate" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/benches/criterion/example" +keywords = ["cuprate", "criterion", "benchmark", "example"] + +[dependencies] +criterion = { workspace = true } +function_name = { workspace = true } +serde_json = { workspace = true, features = ["default"] } + +[[bench]] +name = "main" +harness = false + +[lints] +workspace = true \ No newline at end of file diff --git a/benches/criterion/example/README.md b/benches/criterion/example/README.md new file mode 100644 index 000000000..cf1983ffc --- /dev/null +++ b/benches/criterion/example/README.md @@ -0,0 +1,14 @@ +## `cuprate-criterion-example` +An example of using Criterion for benchmarking Cuprate crates. + +Consider copy+pasting this crate to use as a base when creating new Criterion benchmark crates. + +## `src/` +Benchmark crates have a `benches/` ran by `cargo bench`, but they are also crates themselves, +as in, they have a `src` folder that `benches/` can pull code from. + +The `src` directories in these benchmarking crates are usually filled with +helper functions, types, etc, that are used repeatedly in the benchmarks. + +## `benches/` +These are the actual benchmarks ran by `cargo bench`. diff --git a/benches/criterion/example/benches/example.rs b/benches/criterion/example/benches/example.rs new file mode 100644 index 000000000..7ea8e9a1d --- /dev/null +++ b/benches/criterion/example/benches/example.rs @@ -0,0 +1,48 @@ +//! Benchmarks. +#![allow(unused_attributes, unused_crate_dependencies)] + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use function_name::named; + +use cuprate_criterion_example::SomeHardToCreateObject; + +// This is how you register criterion benchmarks. +criterion_group! { + name = benches; + config = Criterion::default(); + targets = benchmark_1, benchmark_range, +} +criterion_main!(benches); + +/// Benchmark a single input. +/// +/// +#[named] +fn benchmark_1(c: &mut Criterion) { + // It is recommended to use `function_name!()` as a benchmark + // identifier instead of manually re-typing the function name. + c.bench_function(function_name!(), |b| { + b.iter(|| { + black_box(SomeHardToCreateObject::from(1)); + }); + }); +} + +/// Benchmark a range of inputs. +/// +/// +#[named] +fn benchmark_range(c: &mut Criterion) { + let mut group = c.benchmark_group(function_name!()); + + for i in 0..4 { + group.throughput(Throughput::Elements(i)); + group.bench_with_input(BenchmarkId::from_parameter(i), &i, |b, &i| { + b.iter(|| { + black_box(SomeHardToCreateObject::from(i)); + }); + }); + } + + group.finish(); +} diff --git a/benches/criterion/example/benches/main.rs b/benches/criterion/example/benches/main.rs new file mode 100644 index 000000000..d4f0bf80c --- /dev/null +++ b/benches/criterion/example/benches/main.rs @@ -0,0 +1,10 @@ +//! Benchmarks examples. +#![allow(unused_crate_dependencies)] + +// All modules within `benches/` are `mod`ed here. +mod example; + +// And all the Criterion benchmarks are registered like so: +criterion::criterion_main! { + example::benches, +} diff --git a/benches/criterion/example/src/lib.rs b/benches/criterion/example/src/lib.rs new file mode 100644 index 000000000..0f732a47d --- /dev/null +++ b/benches/criterion/example/src/lib.rs @@ -0,0 +1,13 @@ +#![doc = include_str!("../README.md")] // See the README for crate documentation. +#![allow(unused_crate_dependencies, reason = "used in benchmarks")] + +/// Shared type that all benchmarks can use. +#[expect(dead_code)] +pub struct SomeHardToCreateObject(u64); + +impl From for SomeHardToCreateObject { + /// Shared function that all benchmarks can use. + fn from(value: u64) -> Self { + Self(value) + } +} diff --git a/books/architecture/src/benchmarking/cuprate/creating.md b/books/architecture/src/benchmarking/cuprate/creating.md index a8820849e..3f1e91ff9 100644 --- a/books/architecture/src/benchmarking/cuprate/creating.md +++ b/books/architecture/src/benchmarking/cuprate/creating.md @@ -16,7 +16,7 @@ Before plugging into `cuprate-benchmark`, your actual benchmark crate must be cr 1. Implement `cuprate_benchmark_lib::Benchmark` ## `cuprate_benchmark_lib::Benchmark` -This is the trait that standarizes all benchmarks ran under `cuprate-benchmark`. +This is the trait that standardizes all benchmarks ran under `cuprate-benchmark`. It must be implemented by your benchmarking crate. From 9a0b6138b5ce7b58f6ce167bcb3703c0a4b80cbc Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Tue, 8 Oct 2024 20:19:22 -0400 Subject: [PATCH 07/15] docs, tracing --- Cargo.lock | 38 +++++++++++++++++-- benches/benchmark/bin/Cargo.toml | 25 +++++++++--- benches/benchmark/bin/README.md | 10 ++++- benches/benchmark/bin/src/log.rs | 29 ++++++++++++++ benches/benchmark/bin/src/main.rs | 5 ++- benches/benchmark/bin/src/print.rs | 2 + benches/benchmark/bin/src/run.rs | 16 +++++--- benches/benchmark/lib/src/benchmark.rs | 12 +++++- .../src/benchmarking/criterion/intro.md | 4 +- .../src/benchmarking/cuprate/creating.md | 3 ++ .../src/benchmarking/cuprate/intro.md | 29 +++++++++++++- .../src/benchmarking/cuprate/running.md | 2 +- books/architecture/src/benchmarking/intro.md | 14 +++---- 13 files changed, 156 insertions(+), 33 deletions(-) create mode 100644 benches/benchmark/bin/src/log.rs diff --git a/Cargo.lock b/Cargo.lock index ea83921be..655d12748 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -641,6 +641,8 @@ dependencies = [ "cuprate-benchmark-lib", "serde", "serde_json", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1862,6 +1864,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matchit" version = "0.7.3" @@ -2362,7 +2373,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax", + "regex-syntax 0.8.4", "rusty-fork", "tempfile", "unarray", @@ -2536,8 +2547,17 @@ checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2548,9 +2568,15 @@ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.4", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -3257,10 +3283,14 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] diff --git a/benches/benchmark/bin/Cargo.toml b/benches/benchmark/bin/Cargo.toml index 86558d163..7f8c8e5f9 100644 --- a/benches/benchmark/bin/Cargo.toml +++ b/benches/benchmark/bin/Cargo.toml @@ -12,17 +12,30 @@ keywords = ["cuprate", "benchmarking", "binary"] # All new benchmarks should be added here! all = ["example"] -default = [] -json = [] -example = ["dep:cuprate-benchmark-example"] +# Non-benchmark features. +default = [] +json = [] +trace = [] +debug = [] +warn = [] +info = [] +error = [] + +# Benchmark features. +# New benchmarks should be added here! +example = [ + "dep:cuprate-benchmark-example" +] [dependencies] cuprate-benchmark-lib = { path = "../lib" } cuprate-benchmark-example = { path = "../example", optional = true } -cfg-if = { workspace = true } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true, features = ["std"] } +cfg-if = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +tracing = { workspace = true, features = ["std", "attributes"] } +tracing-subscriber = { workspace = true, features = ["fmt", "std", "env-filter"] } [dev-dependencies] diff --git a/benches/benchmark/bin/README.md b/benches/benchmark/bin/README.md index 515f5bb17..ad0700fcc 100644 --- a/benches/benchmark/bin/README.md +++ b/benches/benchmark/bin/README.md @@ -14,6 +14,14 @@ Benchmarks are opt-in and enabled via features. ## Features These are features that aren't for enabling benchmarks, but rather for other things. +Since `cuprate-benchmark` is built right before it is ran, +these features almost act like command line arguments. + | Features | Does what | |----------|-----------| -| json | Prints JSON timings instead of a markdown table \ No newline at end of file +| json | Prints JSON timings instead of a markdown table +| trace | Use the `trace` log-level +| debug | Use the `debug` log-level +| warn | Use the `warn` log-level +| info | Use the `info` log-level (default) +| error | Use the `error` log-level \ No newline at end of file diff --git a/benches/benchmark/bin/src/log.rs b/benches/benchmark/bin/src/log.rs new file mode 100644 index 000000000..455f13091 --- /dev/null +++ b/benches/benchmark/bin/src/log.rs @@ -0,0 +1,29 @@ +use cfg_if::cfg_if; +use tracing::{info, instrument, Level}; +use tracing_subscriber::FmtSubscriber; + +/// Initializes the `tracing` logger. +#[instrument] +pub(crate) fn init_logger() { + const LOG_LEVEL: Level = { + cfg_if! { + if #[cfg(feature = "trace")] { + Level::TRACE + } else if #[cfg(feature = "debug")] { + Level::DEBUG + } else if #[cfg(feature = "warn")] { + Level::WARN + } else if #[cfg(feature = "info")] { + Level::INFO + } else if #[cfg(feature = "error")] { + Level::ERROR + } else { + Level::INFO + } + } + }; + + FmtSubscriber::builder().with_max_level(LOG_LEVEL).init(); + + info!("Log level: {LOG_LEVEL}"); +} diff --git a/benches/benchmark/bin/src/main.rs b/benches/benchmark/bin/src/main.rs index 3c0946da0..2e6272094 100644 --- a/benches/benchmark/bin/src/main.rs +++ b/benches/benchmark/bin/src/main.rs @@ -4,6 +4,7 @@ reason = "this crate imports many potentially unused dependencies" )] +mod log; mod print; mod run; mod timings; @@ -20,11 +21,13 @@ use cfg_if::cfg_if; /// 2. Change it to your benchmark's feature flag /// 3. Change it to your benchmark's type fn main() { + log::init_logger(); + let mut timings = timings::Timings::new(); cfg_if! { if #[cfg(not(any(feature = "example")))] { - compile_error!("[cuprate_benchmark]: no feature specified. Use `--features $BENCHMARK_FEATURE` when building."); + compile_error!("No feature specified. Use `--features $BENCHMARK_FEATURE` when building."); } } diff --git a/benches/benchmark/bin/src/print.rs b/benches/benchmark/bin/src/print.rs index 070d2575d..8fb489d2e 100644 --- a/benches/benchmark/bin/src/print.rs +++ b/benches/benchmark/bin/src/print.rs @@ -4,6 +4,8 @@ use crate::timings::Timings; /// Print the final the final markdown table of benchmark timings. pub(crate) fn print_timings(timings: &Timings) { + println!("\nFinished all benchmarks, printing results:"); + cfg_if! { if #[cfg(feature = "json")] { print_timings_json(timings); diff --git a/benches/benchmark/bin/src/run.rs b/benches/benchmark/bin/src/run.rs index 8119048d2..28ffc3583 100644 --- a/benches/benchmark/bin/src/run.rs +++ b/benches/benchmark/bin/src/run.rs @@ -1,20 +1,23 @@ use std::io::Write; +use tracing::{info, instrument, trace}; + use cuprate_benchmark_lib::Benchmark; use crate::timings::Timings; /// Run a [`Benchmark`] and record its timing. +#[instrument(skip_all)] pub(crate) fn run_benchmark(timings: &mut Timings) { - // Print the benchmark name. - let name = std::any::type_name::(); - print!("{name:>34} ... "); - std::io::stdout().flush().unwrap(); + // Get the benchmark name. + let name = B::name(); + trace!("Running benchmark: {name}"); // Setup the benchmark input. let input = B::SETUP(); // Sleep before running the benchmark. + trace!("Pre-benchmark, sleeping for: {:?}", B::POST_SLEEP_DURATION); std::thread::sleep(B::PRE_SLEEP_DURATION); // Run/time the benchmark. @@ -23,12 +26,13 @@ pub(crate) fn run_benchmark(timings: &mut Timings) { let time = now.elapsed().as_secs_f32(); // Print the benchmark timings. - println!("{time}"); + info!("{name:>34} ... {time}"); assert!( timings.insert(name, time).is_none(), - "[cuprate_benchmark]: there were 2 benchmarks with the same name - this collides the final output: {name}", + "There were 2 benchmarks with the same name - this collides the final output: {name}", ); // Sleep for a cooldown period after the benchmark run. + trace!("Post-benchmark, sleeping for: {:?}", B::POST_SLEEP_DURATION); std::thread::sleep(B::POST_SLEEP_DURATION); } diff --git a/benches/benchmark/lib/src/benchmark.rs b/benches/benchmark/lib/src/benchmark.rs index 3e99964ab..031767317 100644 --- a/benches/benchmark/lib/src/benchmark.rs +++ b/benches/benchmark/lib/src/benchmark.rs @@ -4,6 +4,14 @@ use std::time::Duration; /// A benchmarking function and its inputs. pub trait Benchmark { + /// The benchmark's name. + /// + /// This is automatically implemented + /// as the name of the [`Self`] type. + fn name() -> &'static str { + std::any::type_name::() + } + /// Input to the main benchmarking function. /// /// This is passed to [`Self::MAIN`]. @@ -22,12 +30,12 @@ pub trait Benchmark { const MAIN: fn(Self::Input); /// `cuprate-benchmark` will sleep for this [`Duration`] after - /// creating the [`Self::Input`], but before starting [`Self::Main`]. + /// creating the [`Self::Input`], but before starting [`Self::MAIN`]. /// /// 1 second by default. const PRE_SLEEP_DURATION: Duration = Duration::from_secs(1); - /// `cuprate-benchmark` will sleep for this [`Duration`] after [`Self::Main`]. + /// `cuprate-benchmark` will sleep for this [`Duration`] after [`Self::MAIN`]. /// /// 1 second by default. const POST_SLEEP_DURATION: Duration = Duration::from_secs(1); diff --git a/books/architecture/src/benchmarking/criterion/intro.md b/books/architecture/src/benchmarking/criterion/intro.md index ec9ef802e..b7a79b21a 100644 --- a/books/architecture/src/benchmarking/criterion/intro.md +++ b/books/architecture/src/benchmarking/criterion/intro.md @@ -1,6 +1,4 @@ # Criterion Each sub-directory in [`benches/criterion/`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion) is a crate that uses [Criterion](https://bheisler.github.io/criterion.rs/book) for timing single functions and/or groups of functions. -They are generally be small in scope. - -See [`benches/criterion/cuprate-json-rpc`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc) for an example. +They are generally be small in scope. \ No newline at end of file diff --git a/books/architecture/src/benchmarking/cuprate/creating.md b/books/architecture/src/benchmarking/cuprate/creating.md index 3f1e91ff9..805c576af 100644 --- a/books/architecture/src/benchmarking/cuprate/creating.md +++ b/books/architecture/src/benchmarking/cuprate/creating.md @@ -28,6 +28,9 @@ in the binary that is actually ran: `cuprate-benchmark`. If your benchmark is new, add a new crate feature to [`cuprate-benchmark`'s Cargo.toml file](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin/Cargo.toml) with an optional dependency to your benchmarking crate. +Please remember to edit the feature table in the +[`README.md`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin/README.md) as well! + ## Adding to `cuprate-benchmark`'s `main()` After adding your crate's feature, add a conditional line that run the benchmark if the feature is enabled to the `main()` function: diff --git a/books/architecture/src/benchmarking/cuprate/intro.md b/books/architecture/src/benchmarking/cuprate/intro.md index 7a31c48bd..25efb4608 100644 --- a/books/architecture/src/benchmarking/cuprate/intro.md +++ b/books/architecture/src/benchmarking/cuprate/intro.md @@ -1,12 +1,37 @@ # cuprate-benchmark -Cuprate has 2 custom crates for macro benchmarking: +Cuprate has 2 custom crates for general benchmarking: - `cuprate-benchmark`; the actual binary crate ran - `cuprate-benchmark-lib`; the library that other crates hook into -The purpose of `cuprate-benchmark` is very simple: +The abstract purpose of `cuprate-benchmark` is very simple: 1. Set-up the benchmark 1. Start timer 1. Run benchmark 1. Output data `cuprate-benchmark` runs the benchmarks found in [`benches/benchmark/cuprate-*`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark). + +`cuprate-benchmark-lib` defines the `Benchmark` trait that all +benchmark crates implement to "plug-in" to the benchmarking harness. + +## Diagram +A diagram displaying the relation between `cuprate-benchmark` and related crates. + +``` + ┌─────────────────────┐ + │ cuprate_benchmark │ + │ (actual binary ran) │ + └──────────┬──────────┘ + ┌──────────────────┴───────────────────┐ + │ cuprate_benchmark_lib │ + │ ┌───────────────────────────────────┐│ + │ │ trait Benchmark ││ + │ └───────────────────────────────────┘│ + └──────────────────┬───────────────────┘ +┌───────────────────────────┐ │ ┌───────────────────────────┐ +│ cuprate_benchmark_example ├──┼───┤ cuprate_benchmark_* │ +└───────────────────────────┘ │ └───────────────────────────┘ +┌───────────────────────────┐ │ ┌───────────────────────────┐ +│ cuprate_benchmark_* ├──┴───┤ cuprate_benchmark_* │ +└───────────────────────────┘ └───────────────────────────┘ +``` \ No newline at end of file diff --git a/books/architecture/src/benchmarking/cuprate/running.md b/books/architecture/src/benchmarking/cuprate/running.md index 22ee9a696..b776163ed 100644 --- a/books/architecture/src/benchmarking/cuprate/running.md +++ b/books/architecture/src/benchmarking/cuprate/running.md @@ -1,7 +1,7 @@ # Running `cuprate-benchmark` benchmarks are ran with this command: ```bash -cargo run --release --package cuprate-benchmark --features $YOUR_BENCHMARK_CRATE_FEATURE +cargo run --release --package cuprate-benchmark --features $BENCHMARK_CRATE_FEATURE ``` For example, to run the example benchmark: diff --git a/books/architecture/src/benchmarking/intro.md b/books/architecture/src/benchmarking/intro.md index 77ad7e58b..e6ab6b12d 100644 --- a/books/architecture/src/benchmarking/intro.md +++ b/books/architecture/src/benchmarking/intro.md @@ -1,11 +1,11 @@ # Benchmarking Cuprate has 2 types of benchmarks: -- Criterion benchmarks +- [Criterion](https://bheisler.github.io/criterion.rs/book/user_guide/advanced_configuration.html) benchmarks - `cuprate-benchmark` benchmarks -[Criterion](https://bheisler.github.io/criterion.rs/book/user_guide/advanced_configuration.html) is used for micro benchmarks; they time single functions, groups of functions, and generally are small in scope. +Criterion is used for micro benchmarks; they time single functions, groups of functions, and generally are small in scope. -`cuprate-benchmark` and `cuprate-benchmark-lib` are custom in-house crates Cuprate uses for macro benchmarks; these test sub-systems, sections of a sub-system, or otherwise larger or more complicated code that isn't suited for micro benchmarks. +`cuprate-benchmark` and [`cuprate-benchmark-lib`](https://doc.cuprate.org/cuprate_benchmark_lib) are custom in-house crates Cuprate uses for macro benchmarks; these test sub-systems, sections of a sub-system, or otherwise larger or more complicated code that isn't well-suited for micro benchmarks. ## File layout and purpose All benchmarking related files are in the [`benches/`](https://github.com/Cuprate/cuprate/tree/main/benches) folder. @@ -14,9 +14,9 @@ This directory is organized like such: | Directory | Purpose | |-------------------------------|---------| -| `benches/criterion/` | Criterion (micro) benchmarks +| [`benches/criterion/`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion) | Criterion (micro) benchmarks | `benches/criterion/cuprate-*` | Criterion benchmarks for the crate with the same name -| `benches/benchmark/` | Cuprate's custom benchmarking files -| `benches/benchmark/bin` | The `cuprate-benchmark` crate; the actual binary run that links all benchmarks -| `benches/benchmark/lib` | The `cuprate-benchmark-lib` crate; the benchmarking framework all benchmarks plug into +| [`benches/benchmark/`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark) | Cuprate's custom benchmarking files +| [`benches/benchmark/bin`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin) | The `cuprate-benchmark` crate; the actual binary run that links all benchmarks +| [`benches/benchmark/lib`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/lib) | The `cuprate-benchmark-lib` crate; the benchmarking framework all benchmarks plug into | `benches/benchmark/cuprate-*` | `cuprate-benchmark` benchmarks for the crate with the same name From 8c9d159124fa3eb2e40fde158c2d1f9da62f6f73 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Tue, 8 Oct 2024 20:35:00 -0400 Subject: [PATCH 08/15] fix clippy --- benches/benchmark/bin/src/print.rs | 3 ++- benches/benchmark/bin/src/run.rs | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/benches/benchmark/bin/src/print.rs b/benches/benchmark/bin/src/print.rs index 8fb489d2e..36a5f05a6 100644 --- a/benches/benchmark/bin/src/print.rs +++ b/benches/benchmark/bin/src/print.rs @@ -1,3 +1,5 @@ +#![expect(dead_code, reason = "code hidden behind feature flags")] + use cfg_if::cfg_if; use crate::timings::Timings; @@ -30,7 +32,6 @@ pub(crate) fn print_timings_markdown(timings: &Timings) { } /// Enabled via `json` feature. -#[expect(dead_code)] pub(crate) fn print_timings_json(timings: &Timings) { let json = serde_json::to_string_pretty(timings).unwrap(); println!("\n{json}"); diff --git a/benches/benchmark/bin/src/run.rs b/benches/benchmark/bin/src/run.rs index 28ffc3583..05a220f93 100644 --- a/benches/benchmark/bin/src/run.rs +++ b/benches/benchmark/bin/src/run.rs @@ -1,5 +1,3 @@ -use std::io::Write; - use tracing::{info, instrument, trace}; use cuprate_benchmark_lib::Benchmark; From 828cbbac27e55ae3e2976534a204fbf1ae83dd6e Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Tue, 8 Oct 2024 20:49:13 -0400 Subject: [PATCH 09/15] docs --- Cargo.toml | 2 +- .../criterion/cuprate-json-rpc/benches/main.rs | 4 +++- .../cuprate-json-rpc/benches/response.rs | 7 +++++-- benches/criterion/cuprate-json-rpc/src/lib.rs | 2 +- .../src/benchmarking/criterion/creating.md | 13 ++++++++++++- .../src/benchmarking/cuprate/creating.md | 18 +++++++++++++++--- 6 files changed, 37 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a402cbdc7..d756b08a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,7 +106,7 @@ tempfile = { version = "3.13.0" } monero-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" } monero-simple-request-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" } pretty_assertions = { version = "1.4.1" } -proptest = { version = "1" } +proptest = { version = "1.5.0" } proptest-derive = { version = "0.4.0" } tokio-test = { version = "0.4.4" } diff --git a/benches/criterion/cuprate-json-rpc/benches/main.rs b/benches/criterion/cuprate-json-rpc/benches/main.rs index 578dd8ed6..8bdc5f70a 100644 --- a/benches/criterion/cuprate-json-rpc/benches/main.rs +++ b/benches/criterion/cuprate-json-rpc/benches/main.rs @@ -1,8 +1,10 @@ //! Benchmarks for `cuprate-json-rpc`. +//! +//! TODO: this crate is not finished. #![allow(unused_crate_dependencies)] mod response; criterion::criterion_main! { - response::benches, + response::serde, } diff --git a/benches/criterion/cuprate-json-rpc/benches/response.rs b/benches/criterion/cuprate-json-rpc/benches/response.rs index c1a134119..3ac995318 100644 --- a/benches/criterion/cuprate-json-rpc/benches/response.rs +++ b/benches/criterion/cuprate-json-rpc/benches/response.rs @@ -7,8 +7,11 @@ use serde_json::{from_str, to_string_pretty}; use cuprate_json_rpc::{Id, Response}; +// `serde` benchmarks on `Response`. criterion_group! { - benches, + name = serde; + config = Criterion::default(); + targets = response_from_str_u8, response_from_str_u64, response_from_str_string_5_len, @@ -22,7 +25,7 @@ criterion_group! { response_to_string_pretty_string_100_len, response_to_string_pretty_string_500_len, } -criterion_main!(benches); +criterion_main!(serde); /// Generate `from_str` deserialization benchmark functions for [`Response`]. macro_rules! impl_from_str_benchmark { diff --git a/benches/criterion/cuprate-json-rpc/src/lib.rs b/benches/criterion/cuprate-json-rpc/src/lib.rs index 8a892fcb5..cf9ea54fa 100644 --- a/benches/criterion/cuprate-json-rpc/src/lib.rs +++ b/benches/criterion/cuprate-json-rpc/src/lib.rs @@ -1,2 +1,2 @@ -//! TODO +//! TODO: this crate is not finished. #![allow(unused_crate_dependencies, reason = "used in benchmarks")] diff --git a/books/architecture/src/benchmarking/criterion/creating.md b/books/architecture/src/benchmarking/criterion/creating.md index faf78bbf6..010090429 100644 --- a/books/architecture/src/benchmarking/criterion/creating.md +++ b/books/architecture/src/benchmarking/criterion/creating.md @@ -6,5 +6,16 @@ although, it requires knowledge of how to use Criterion first: 2. Copy [`benches/criterion/example`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/example) as base 3. Get started +## Naming +New benchmark crates using Criterion should: +- Be in [`benches/criterion/`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/) +- Be in the `cuprate-criterion-$CRATE_NAME` format + For a real example, see: -[`cuprate-criterion-json-rpc`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc). \ No newline at end of file +[`cuprate-criterion-json-rpc`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cuprate-json-rpc). + +## Workspace +Finally, make sure to add the benchmark crate to the workspace +[`Cargo.toml`](https://github.com/Cuprate/cuprate/blob/main/Cargo.toml) file. + +Your benchmark is now ready to be ran. \ No newline at end of file diff --git a/books/architecture/src/benchmarking/cuprate/creating.md b/books/architecture/src/benchmarking/cuprate/creating.md index 805c576af..76eab7890 100644 --- a/books/architecture/src/benchmarking/cuprate/creating.md +++ b/books/architecture/src/benchmarking/cuprate/creating.md @@ -4,8 +4,7 @@ New benchmarks are plugged into `cuprate-benchmark` by: 1. Registering the benchmark in the `cuprate_benchmark` binary See [`benches/benchmark/example`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/example) -for an example. For a real example, see: -[`cuprate-benchmark-database`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/cuprate-database). +for an example. ## Creating the benchmark crate Before plugging into `cuprate-benchmark`, your actual benchmark crate must be created: @@ -15,6 +14,13 @@ Before plugging into `cuprate-benchmark`, your actual benchmark crate must be cr 1. Create a benchmark 1. Implement `cuprate_benchmark_lib::Benchmark` +New benchmark crates using `cuprate-database` should: +- Be in [`benches/benchmark/`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/) +- Be in the `cuprate-benchmark-$CRATE_NAME` format + +For a real example, see: +[`cuprate-benchmark-database`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/cuprate-database). + ## `cuprate_benchmark_lib::Benchmark` This is the trait that standardizes all benchmarks ran under `cuprate-benchmark`. @@ -42,4 +48,10 @@ cfg_if! { run::run_benchmark::(&mut timings); } } -``` \ No newline at end of file +``` + +## Workspace +Finally, make sure to add the benchmark crate to the workspace +[`Cargo.toml`](https://github.com/Cuprate/cuprate/blob/main/Cargo.toml) file. + +Your benchmark is now ready to be ran. \ No newline at end of file From cab201161303cba849d14c7b8a808fea0d2d6b6d Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Wed, 9 Oct 2024 17:23:32 -0400 Subject: [PATCH 10/15] lib readme --- benches/benchmark/lib/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/benches/benchmark/lib/README.md b/benches/benchmark/lib/README.md index 5ad92b704..6abce6b01 100644 --- a/benches/benchmark/lib/README.md +++ b/benches/benchmark/lib/README.md @@ -1 +1,16 @@ ## `cuprate-benchmark-lib` +This crate is the glue between +[`cuprate-benchmark`](https://github.com/Cuprate/cuprate/tree/benches/benches/benchmark/bin) +and all the benchmark crates. + +It defines the [`cuprate_benchmark_lib::Benchmark`] trait, +which is the behavior of all benchmarks. + +See the [`cuprate-benchmark-example`](https://github.com/Cuprate/cuprate/tree/benches/benches/benchmark/example) +crate to see an example implementation of this trait. + +After implementing this trait, a few steps must +be done such that the `cuprate-benchmark` binary +can actually run your benchmark crate; see the +[`Benchmarking` section in the Architecture book](https://architecture.cuprate.org/benchmarking/intro.html) +to see how to do this. \ No newline at end of file From eb617b30f885afb43c2b4c35da45cbbd95b954b7 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Wed, 9 Oct 2024 17:42:56 -0400 Subject: [PATCH 11/15] json-rpc benchmarks --- .../cuprate-json-rpc/benches/main.rs | 2 -- .../cuprate-json-rpc/benches/response.rs | 20 +++++++++++++++++-- benches/criterion/cuprate-json-rpc/src/lib.rs | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/benches/criterion/cuprate-json-rpc/benches/main.rs b/benches/criterion/cuprate-json-rpc/benches/main.rs index 8bdc5f70a..a72494302 100644 --- a/benches/criterion/cuprate-json-rpc/benches/main.rs +++ b/benches/criterion/cuprate-json-rpc/benches/main.rs @@ -1,6 +1,4 @@ //! Benchmarks for `cuprate-json-rpc`. -//! -//! TODO: this crate is not finished. #![allow(unused_crate_dependencies)] mod response; diff --git a/benches/criterion/cuprate-json-rpc/benches/response.rs b/benches/criterion/cuprate-json-rpc/benches/response.rs index 3ac995318..908a9f49b 100644 --- a/benches/criterion/cuprate-json-rpc/benches/response.rs +++ b/benches/criterion/cuprate-json-rpc/benches/response.rs @@ -8,6 +8,8 @@ use serde_json::{from_str, to_string_pretty}; use cuprate_json_rpc::{Id, Response}; // `serde` benchmarks on `Response`. +// +// These are benchmarked as `Response` has a custom serde implementation. criterion_group! { name = serde; config = Criterion::default(); @@ -24,6 +26,11 @@ criterion_group! { response_to_string_pretty_string_10_len, response_to_string_pretty_string_100_len, response_to_string_pretty_string_500_len, + response_from_str_bad_field_1, + response_from_str_bad_field_5, + response_from_str_bad_field_10, + response_from_str_bad_field_100, + response_from_str_missing_field, } criterion_main!(serde); @@ -35,7 +42,6 @@ macro_rules! impl_from_str_benchmark { )* ) => { $( - /// TODO #[named] fn $fn_name(c: &mut Criterion) { let request_string = $request_string; @@ -59,6 +65,17 @@ impl_from_str_benchmark! { response_from_str_string_10_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"hellohello"}"#, response_from_str_string_100_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld"}"#, response_from_str_string_500_len => String => r#"{"jsonrpc":"2.0","id":123,"result":"helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld"}"#, + + // The custom serde currently looks at all fields. + // These are for testing the performance if the serde + // has to parse through a bunch of unrelated fields. + response_from_str_bad_field_1 => u8 => r#"{"bad_field":0,"jsonrpc":"2.0","id":123,"result":0}"#, + response_from_str_bad_field_5 => u8 => r#"{"bad_field_1":0,"bad_field_2":0,"bad_field_3":0,"bad_field_4":0,"bad_field_5":0,"jsonrpc":"2.0","id":123,"result":0}"#, + response_from_str_bad_field_10 => u8 => r#"{"bad_field_1":0,"bad_field_2":0,"bad_field_3":0,"bad_field_4":0,"bad_field_5":0,"bad_field_6":0,"bad_field_7":0,"bad_field_8":0,"bad_field_9":0,"bad_field_10":0,"jsonrpc":"2.0","id":123,"result":0}"#, + response_from_str_bad_field_100 => u8 => r#"{"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0,"8":0,"9":0,"10":0,"11":0,"12":0,"13":0,"14":0,"15":0,"16":0,"17":0,"18":0,"19":0,"20":0,"21":0,"22":0,"23":0,"24":0,"25":0,"26":0,"27":0,"28":0,"29":0,"30":0,"31":0,"32":0,"33":0,"34":0,"35":0,"36":0,"37":0,"38":0,"39":0,"40":0,"41":0,"42":0,"43":0,"44":0,"45":0,"46":0,"47":0,"48":0,"49":0,"50":0,"51":0,"52":0,"53":0,"54":0,"55":0,"56":0,"57":0,"58":0,"59":0,"60":0,"61":0,"62":0,"63":0,"64":0,"65":0,"66":0,"67":0,"68":0,"69":0,"70":0,"71":0,"72":0,"73":0,"74":0,"75":0,"76":0,"77":0,"78":0,"79":0,"80":0,"81":0,"82":0,"83":0,"84":0,"85":0,"86":0,"87":0,"88":0,"89":0,"90":0,"91":0,"92":0,"93":0,"94":0,"95":0,"96":0,"97":0,"98":0,"99":0,"100":0,"jsonrpc":"2.0","id":123,"result":0}"#, + + // These are missing the `jsonrpc` field. + response_from_str_missing_field => u8 => r#"{"id":123,"result":0}"#, } /// Generate `to_string_pretty` serialization benchmark functions for [`Response`]. @@ -69,7 +86,6 @@ macro_rules! impl_to_string_pretty_benchmark { )* ) => { $( - /// TODO #[named] fn $fn_name(c: &mut Criterion) { let request = $request_constructor; diff --git a/benches/criterion/cuprate-json-rpc/src/lib.rs b/benches/criterion/cuprate-json-rpc/src/lib.rs index cf9ea54fa..b29887aa7 100644 --- a/benches/criterion/cuprate-json-rpc/src/lib.rs +++ b/benches/criterion/cuprate-json-rpc/src/lib.rs @@ -1,2 +1,2 @@ -//! TODO: this crate is not finished. +//! Benchmark lib for `cuprate-json-rpc`. #![allow(unused_crate_dependencies, reason = "used in benchmarks")] From 2e3e36c840e697830cd95207f77d6d84f6298fed Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Wed, 9 Oct 2024 19:46:13 -0400 Subject: [PATCH 12/15] add to crates.md --- benches/benchmark/lib/README.md | 3 +-- books/architecture/src/appendix/crates.md | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/benches/benchmark/lib/README.md b/benches/benchmark/lib/README.md index 6abce6b01..9ea79ae4e 100644 --- a/benches/benchmark/lib/README.md +++ b/benches/benchmark/lib/README.md @@ -3,8 +3,7 @@ This crate is the glue between [`cuprate-benchmark`](https://github.com/Cuprate/cuprate/tree/benches/benches/benchmark/bin) and all the benchmark crates. -It defines the [`cuprate_benchmark_lib::Benchmark`] trait, -which is the behavior of all benchmarks. +It defines the [`crate::Benchmark`] trait, which is the behavior of all benchmarks. See the [`cuprate-benchmark-example`](https://github.com/Cuprate/cuprate/tree/benches/benches/benchmark/example) crate to see an example implementation of this trait. diff --git a/books/architecture/src/appendix/crates.md b/books/architecture/src/appendix/crates.md index 1993c47e7..c792250e4 100644 --- a/books/architecture/src/appendix/crates.md +++ b/books/architecture/src/appendix/crates.md @@ -61,3 +61,11 @@ cargo doc --open --package cuprate-blockchain | [`cuprate-helper`](https://doc.cuprate.org/cuprate_helper) | [`helper/`](https://github.com/Cuprate/cuprate/tree/main/helper) | Kitchen-sink helper crate for Cuprate | [`cuprate-test-utils`](https://doc.cuprate.org/cuprate_test_utils) | [`test-utils/`](https://github.com/Cuprate/cuprate/tree/main/test-utils) | Testing utilities for Cuprate | [`cuprate-types`](https://doc.cuprate.org/cuprate_types) | [`types/`](https://github.com/Cuprate/cuprate/tree/main/types) | Shared types across Cuprate + +## Benchmarks +| Crate | In-tree path | Purpose | +|-------|--------------|---------| +| [`cuprate-benchmark`](https://doc.cuprate.org/cuprate_benchmark) | [`benches/benchmark/bin/`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin) | Cuprate benchmarking binary +| [`cuprate-benchmark-lib`](https://doc.cuprate.org/cuprate_benchmark_lib) | [`benches/benchmark/lib/`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/lib) | Cuprate benchmarking library +| `cuprate-benchmark-*` | [`benches/benchmark/cuprate-*`](https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/) | Benchmark for a Cuprate crate that uses `cuprate-benchmark` +| `cuprate-criterion-*` | [`benches/criterion/cuprate-*`](https://github.com/Cuprate/cuprate/tree/main/benches/criterion) | Benchmark for a Cuprate crate that uses [Criterion](https://bheisler.github.io/criterion.rs/book) \ No newline at end of file From 1c462e3c957e1d01f07f8b304618a8087fbcd40b Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Wed, 9 Oct 2024 20:17:58 -0400 Subject: [PATCH 13/15] add `fixme` --- benches/benchmark/lib/src/benchmark.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benches/benchmark/lib/src/benchmark.rs b/benches/benchmark/lib/src/benchmark.rs index 031767317..4dca55045 100644 --- a/benches/benchmark/lib/src/benchmark.rs +++ b/benches/benchmark/lib/src/benchmark.rs @@ -8,6 +8,9 @@ pub trait Benchmark { /// /// This is automatically implemented /// as the name of the [`Self`] type. + // + // FIXME: use `const` instead of `fn` when stable + // fn name() -> &'static str { std::any::type_name::() } From 1cada331f030f5840fc8b2e9437d09be8b87c2ec Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Mon, 21 Oct 2024 20:12:22 -0400 Subject: [PATCH 14/15] fix `cargo b` failing this `cfg()` existing makes a regular workspace `cargo b` fail --- benches/benchmark/bin/src/main.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benches/benchmark/bin/src/main.rs b/benches/benchmark/bin/src/main.rs index 2e6272094..02c480a08 100644 --- a/benches/benchmark/bin/src/main.rs +++ b/benches/benchmark/bin/src/main.rs @@ -20,6 +20,13 @@ use cfg_if::cfg_if; /// 1. Copy + paste a `cfg_if` block /// 2. Change it to your benchmark's feature flag /// 3. Change it to your benchmark's type +#[allow( + clippy::allow_attributes, + unused_variables, + unused_mut, + unreachable_code, + reason = "clippy does not account for all cfg()s" +)] fn main() { log::init_logger(); @@ -27,7 +34,8 @@ fn main() { cfg_if! { if #[cfg(not(any(feature = "example")))] { - compile_error!("No feature specified. Use `--features $BENCHMARK_FEATURE` when building."); + println!("No feature specified. Use `--features $BENCHMARK_FEATURE` when building."); + return; } } From e4a10bff45f2d27239d8395a08145b33494240e5 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Mon, 25 Nov 2024 14:34:25 -0500 Subject: [PATCH 15/15] fix cargo.toml --- Cargo.toml | 58 ++++++++++--------- benches/benchmark/bin/Cargo.toml | 4 +- benches/criterion/cuprate-json-rpc/Cargo.toml | 4 +- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d7249a788..a507631ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,34 +74,36 @@ opt-level = 3 [workspace.dependencies] # Cuprate members -cuprate-fast-sync = { path = "consensus/fast-sync", default-features = false } -cuprate-consensus-rules = { path = "consensus/rules", default-features = false } -cuprate-constants = { path = "constants", default-features = false } -cuprate-consensus = { path = "consensus", default-features = false } -cuprate-consensus-context = { path = "consensus/context", default-features = false } -cuprate-cryptonight = { path = "cryptonight", default-features = false } -cuprate-helper = { path = "helper", default-features = false } -cuprate-epee-encoding = { path = "net/epee-encoding", default-features = false } -cuprate-fixed-bytes = { path = "net/fixed-bytes", default-features = false } -cuprate-levin = { path = "net/levin", default-features = false } -cuprate-wire = { path = "net/wire", default-features = false } -cuprate-p2p = { path = "p2p/p2p", default-features = false } -cuprate-p2p-core = { path = "p2p/p2p-core", default-features = false } -cuprate-p2p-bucket = { path = "p2p/p2p-bucket", default-features = false } -cuprate-dandelion-tower = { path = "p2p/dandelion-tower", default-features = false } -cuprate-async-buffer = { path = "p2p/async-buffer", default-features = false } -cuprate-address-book = { path = "p2p/address-book", default-features = false } -cuprate-blockchain = { path = "storage/blockchain", default-features = false } -cuprate-database = { path = "storage/database", default-features = false } -cuprate-database-service = { path = "storage/service", default-features = false } -cuprate-txpool = { path = "storage/txpool", default-features = false } -cuprate-pruning = { path = "pruning", default-features = false } -cuprate-test-utils = { path = "test-utils", default-features = false } -cuprate-types = { path = "types", default-features = false } -cuprate-json-rpc = { path = "rpc/json-rpc", default-features = false } -cuprate-rpc-types = { path = "rpc/types", default-features = false } -cuprate-rpc-interface = { path = "rpc/interface", default-features = false } -cuprate-zmq-types = { path = "zmq/types", default-features = false } +cuprate-benchmark-lib = { path = "benches/benchmark/lib", default-features = false } +cuprate-benchmark-example = { path = "benches/benchmark/example", default-features = false } +cuprate-fast-sync = { path = "consensus/fast-sync", default-features = false } +cuprate-consensus-rules = { path = "consensus/rules", default-features = false } +cuprate-constants = { path = "constants", default-features = false } +cuprate-consensus = { path = "consensus", default-features = false } +cuprate-consensus-context = { path = "consensus/context", default-features = false } +cuprate-cryptonight = { path = "cryptonight", default-features = false } +cuprate-helper = { path = "helper", default-features = false } +cuprate-epee-encoding = { path = "net/epee-encoding", default-features = false } +cuprate-fixed-bytes = { path = "net/fixed-bytes", default-features = false } +cuprate-levin = { path = "net/levin", default-features = false } +cuprate-wire = { path = "net/wire", default-features = false } +cuprate-p2p = { path = "p2p/p2p", default-features = false } +cuprate-p2p-core = { path = "p2p/p2p-core", default-features = false } +cuprate-p2p-bucket = { path = "p2p/p2p-bucket", default-features = false } +cuprate-dandelion-tower = { path = "p2p/dandelion-tower", default-features = false } +cuprate-async-buffer = { path = "p2p/async-buffer", default-features = false } +cuprate-address-book = { path = "p2p/address-book", default-features = false } +cuprate-blockchain = { path = "storage/blockchain", default-features = false } +cuprate-database = { path = "storage/database", default-features = false } +cuprate-database-service = { path = "storage/service", default-features = false } +cuprate-txpool = { path = "storage/txpool", default-features = false } +cuprate-pruning = { path = "pruning", default-features = false } +cuprate-test-utils = { path = "test-utils", default-features = false } +cuprate-types = { path = "types", default-features = false } +cuprate-json-rpc = { path = "rpc/json-rpc", default-features = false } +cuprate-rpc-types = { path = "rpc/types", default-features = false } +cuprate-rpc-interface = { path = "rpc/interface", default-features = false } +cuprate-zmq-types = { path = "zmq/types", default-features = false } # External dependencies anyhow = { version = "1", default-features = false } diff --git a/benches/benchmark/bin/Cargo.toml b/benches/benchmark/bin/Cargo.toml index 7f8c8e5f9..36d0b2c74 100644 --- a/benches/benchmark/bin/Cargo.toml +++ b/benches/benchmark/bin/Cargo.toml @@ -28,8 +28,8 @@ example = [ ] [dependencies] -cuprate-benchmark-lib = { path = "../lib" } -cuprate-benchmark-example = { path = "../example", optional = true } +cuprate-benchmark-lib = { workspace = true } +cuprate-benchmark-example = { workspace = true, optional = true } cfg-if = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/benches/criterion/cuprate-json-rpc/Cargo.toml b/benches/criterion/cuprate-json-rpc/Cargo.toml index 08383255c..a0cae64e6 100644 --- a/benches/criterion/cuprate-json-rpc/Cargo.toml +++ b/benches/criterion/cuprate-json-rpc/Cargo.toml @@ -9,12 +9,12 @@ repository = "https://github.com/Cuprate/cuprate/tree/main/benches/criterion/cu keywords = ["cuprate", "json-rpc", "criterion", "benchmark"] [dependencies] +cuprate-json-rpc = { workspace = true } + criterion = { workspace = true } function_name = { workspace = true } serde_json = { workspace = true, features = ["default"] } -cuprate-json-rpc = { path = "../../../rpc/json-rpc" } - [[bench]] name = "main" harness = false