benches: initial implementation

Cargo.toml

@@ -1,33 +1,45 @@
 [workspace]
 resolver = "2"
-
 members = [
+	# Binaries
 	"binaries/cuprated",
-	"constants",
+	# Benchmarks
+	"benches/benchmark/bin",
+	"benches/benchmark/lib",
+	"benches/benchmark/example",
+	"benches/criterion/example",
+	"benches/criterion/cuprate-json-rpc",
+	# Consensus
 	"consensus",
 	"consensus/fast-sync",
 	"consensus/rules",
-	"cryptonight",
-	"helper",
+	# Net
 	"net/epee-encoding",
 	"net/fixed-bytes",
 	"net/levin",
 	"net/wire",
+	# P2P
 	"p2p/p2p",
 	"p2p/p2p-core",
 	"p2p/dandelion-tower",
 	"p2p/async-buffer",
 	"p2p/address-book",
+	# Storage
 	"storage/blockchain",
 	"storage/service",
 	"storage/txpool",
 	"storage/database",
-	"pruning",
-	"test-utils",
-	"types",
+	# RPC
 	"rpc/json-rpc",
 	"rpc/types",
 	"rpc/interface",
+	# Misc
+	"constants",
+	"cryptonight",
+	"helper",
+	"pruning",
+	"test-utils",
+	"types",
 ]
 
 [profile.release]
@@ -88,11 +100,13 @@ tracing-subscriber    = { version = "0.3.18", default-features = false }
 tracing               = { version = "0.1.40", default-features = false }
 
 ## workspace.dev-dependencies
+criterion                 = { version = "0.5.1" }
+function_name             = { version = "0.3.0" }
+tempfile                  = { version = "3.13.0" }
 monero-rpc                = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" }
 monero-simple-request-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" }
-tempfile                  = { version = "3" }
 pretty_assertions         = { version = "1.4.1" }
-proptest                  = { version = "1" }
+proptest                  = { version = "1.5.0" }
 proptest-derive           = { version = "0.4.0" }
 tokio-test                = { version = "0.4.4" }
 
@@ -321,4 +335,4 @@ non_camel_case_types = "deny"
 # unused_results = "deny"
 # non_exhaustive_omitted_patterns = "deny"
 # missing_docs = "deny"
-# missing_copy_implementations = "deny"
+# missing_copy_implementations = "deny"

benches/README.md

@@ -1 +1,5 @@
-# TODO
+# Benches
+This directory contains Cuprate's benchmarks and benchmarking utilities.
+
+See the [`Benchmarking` section in the Architecture book](https://architecture.cuprate.org/benchmarking/intro.html)
+to see how to create and run these benchmarks.

benches/benchmark/bin/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name        = "cuprate-benchmark"
+version     = "0.0.0"
+edition     = "2021"
+description = "Cuprate's benchmarking binary"
+license     = "MIT"
+authors     = ["hinto-janai"]
+repository  = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin"
+keywords    = ["cuprate", "benchmarking", "binary"]
+
+[features]
+# All new benchmarks should be added here!
+all = ["example"]
+
+# Non-benchmark features.
+default = []
+json    = []
+trace   = []
+debug   = []
+warn    = []
+info    = []
+error   = []
+
+# Benchmark features.
+# New benchmarks should be added here!
+example = [
+	"dep:cuprate-benchmark-example"
+]
+
+[dependencies]
+cuprate-benchmark-lib     = { path = "../lib" }
+cuprate-benchmark-example = { path = "../example", optional = true }
+
+cfg-if             = { workspace = true }
+serde              = { workspace = true, features = ["derive"] }
+serde_json         = { workspace = true, features = ["std"] }
+tracing            = { workspace = true, features = ["std", "attributes"] }
+tracing-subscriber = { workspace = true, features = ["fmt", "std", "env-filter"] }
+
+[dev-dependencies]
+
+[lints]
+workspace = true

benches/benchmark/bin/README.md

@@ -0,0 +1,27 @@
+## `cuprate-benchmark`
+This crate links all benchmarks together into a single binary that can be run as: `cuprate-benchmark`.
+
+`cuprate-benchmark` will run all enabled benchmarks sequentially and print data at the end.
+
+## Benchmarks
+Benchmarks are opt-in and enabled via features.
+
+| Feature  | Enables which benchmark crate? |
+|----------|--------------------------------|
+| example  | cuprate-benchmark-example      |
+| database | cuprate-benchmark-database     |
+
+## Features
+These are features that aren't for enabling benchmarks, but rather for other things.
+
+Since `cuprate-benchmark` is built right before it is ran,
+these features almost act like command line arguments.
+
+| Features | Does what |
+|----------|-----------|
+| json     | Prints JSON timings instead of a markdown table
+| trace    | Use the `trace` log-level
+| debug    | Use the `debug` log-level
+| warn     | Use the `warn` log-level
+| info     | Use the `info` log-level (default)
+| error    | Use the `error` log-level

benches/benchmark/bin/src/log.rs

@@ -0,0 +1,29 @@
+use cfg_if::cfg_if;
+use tracing::{info, instrument, Level};
+use tracing_subscriber::FmtSubscriber;
+
+/// Initializes the `tracing` logger.
+#[instrument]
+pub(crate) fn init_logger() {
+    const LOG_LEVEL: Level = {
+        cfg_if! {
+            if #[cfg(feature = "trace")] {
+                Level::TRACE
+            } else if #[cfg(feature = "debug")] {
+                Level::DEBUG
+            } else if #[cfg(feature = "warn")] {
+                Level::WARN
+            } else if #[cfg(feature = "info")] {
+                Level::INFO
+            } else if #[cfg(feature = "error")] {
+                Level::ERROR
+            } else {
+                Level::INFO
+            }
+        }
+    };
+
+    FmtSubscriber::builder().with_max_level(LOG_LEVEL).init();
+
+    info!("Log level: {LOG_LEVEL}");
+}

benches/benchmark/bin/src/main.rs

@@ -0,0 +1,41 @@
+#![doc = include_str!("../README.md")]
+#![allow(
+    unused_crate_dependencies,
+    reason = "this crate imports many potentially unused dependencies"
+)]
+
+mod log;
+mod print;
+mod run;
+mod timings;
+
+use cfg_if::cfg_if;
+
+/// What `main()` does:
+/// 1. Run all enabled benchmarks
+/// 2. Record benchmark timings
+/// 3. Print timing data
+///
+/// To add a new benchmark to be ran here:
+/// 1. Copy + paste a `cfg_if` block
+/// 2. Change it to your benchmark's feature flag
+/// 3. Change it to your benchmark's type
+fn main() {
+    log::init_logger();
+
+    let mut timings = timings::Timings::new();
+
+    cfg_if! {
+        if #[cfg(not(any(feature = "example")))] {
+            compile_error!("No feature specified. Use `--features $BENCHMARK_FEATURE` when building.");
+        }
+    }
+
+    cfg_if! {
+        if #[cfg(feature = "example")] {
+            run::run_benchmark::<cuprate_benchmark_example::Example>(&mut timings);
+        }
+    }
+
+    print::print_timings(&timings);
+}

benches/benchmark/bin/src/print.rs

@@ -0,0 +1,38 @@
+#![expect(dead_code, reason = "code hidden behind feature flags")]
+
+use cfg_if::cfg_if;
+
+use crate::timings::Timings;
+
+/// Print the final the final markdown table of benchmark timings.
+pub(crate) fn print_timings(timings: &Timings) {
+    println!("\nFinished all benchmarks, printing results:");
+
+    cfg_if! {
+        if #[cfg(feature = "json")] {
+            print_timings_json(timings);
+        } else {
+            print_timings_markdown(timings);
+        }
+    }
+}
+
+/// Default timing formatting.
+pub(crate) fn print_timings_markdown(timings: &Timings) {
+    let mut s = String::new();
+    s.push_str("| Benchmark                          | Time (seconds) |\n");
+    s.push_str("|------------------------------------|----------------|");
+
+    #[expect(clippy::iter_over_hash_type)]
+    for (k, v) in timings {
+        s += &format!("\n| {k:<34} | {v:<14} |");
+    }
+
+    println!("\n{s}");
+}
+
+/// Enabled via `json` feature.
+pub(crate) fn print_timings_json(timings: &Timings) {
+    let json = serde_json::to_string_pretty(timings).unwrap();
+    println!("\n{json}");
+}

benches/benchmark/bin/src/run.rs

@@ -0,0 +1,36 @@
+use tracing::{info, instrument, trace};
+
+use cuprate_benchmark_lib::Benchmark;
+
+use crate::timings::Timings;
+
+/// Run a [`Benchmark`] and record its timing.
+#[instrument(skip_all)]
+pub(crate) fn run_benchmark<B: Benchmark>(timings: &mut Timings) {
+    // Get the benchmark name.
+    let name = B::name();
+    trace!("Running benchmark: {name}");
+
+    // Setup the benchmark input.
+    let input = B::SETUP();
+
+    // Sleep before running the benchmark.
+    trace!("Pre-benchmark, sleeping for: {:?}", B::POST_SLEEP_DURATION);
+    std::thread::sleep(B::PRE_SLEEP_DURATION);
+
+    // Run/time the benchmark.
+    let now = std::time::Instant::now();
+    B::MAIN(input);
+    let time = now.elapsed().as_secs_f32();
+
+    // Print the benchmark timings.
+    info!("{name:>34} ... {time}");
+    assert!(
+        timings.insert(name, time).is_none(),
+        "There were 2 benchmarks with the same name - this collides the final output: {name}",
+    );
+
+    // Sleep for a cooldown period after the benchmark run.
+    trace!("Post-benchmark, sleeping for: {:?}", B::POST_SLEEP_DURATION);
+    std::thread::sleep(B::POST_SLEEP_DURATION);
+}

benches/benchmark/bin/src/timings.rs

@@ -0,0 +1,5 @@
+/// Benchmark timing data.
+///
+/// - Key = benchmark name
+/// - Value = benchmark time in seconds
+pub(crate) type Timings = std::collections::HashMap<&'static str, f32>;

benches/benchmark/example/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name        = "cuprate-benchmark-example"
+version     = "0.0.0"
+edition     = "2021"
+description = "Example showcasing Cuprate's benchmarking harness"
+license     = "MIT"
+authors     = ["hinto-janai"]
+repository  = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/example"
+keywords    = ["cuprate", "benchmarking", "example"]
+
+[dependencies]
+cuprate-benchmark-lib = { path = "../lib" }
+
+[dev-dependencies]
+
+[lints]
+workspace = true

benches/benchmark/example/README.md

@@ -0,0 +1,3 @@
+## `cuprate-benchmark-example`
+This crate contains a short example benchmark that shows how to implement and use
+`cuprate-benchmark-lib` so that it can be ran by `cuprate-benchmark`.

benches/benchmark/example/src/lib.rs

@@ -0,0 +1,42 @@
+#![doc = include_str!("../README.md")]
+
+use std::hint::black_box;
+
+use cuprate_benchmark_lib::Benchmark;
+
+/// Marker struct that implements [`Benchmark`]
+pub struct Example;
+
+/// The input to our benchmark function.
+pub type ExampleBenchmarkInput = u64;
+
+/// The setup function that creates the input.
+pub const fn example_benchmark_setup() -> ExampleBenchmarkInput {
+    1
+}
+
+/// The main benchmarking function.
+#[expect(clippy::unit_arg)]
+pub fn example_benchmark_main(input: ExampleBenchmarkInput) {
+    // In this case, we're simply benchmarking the
+    // performance of simple arithmetic on the input data.
+
+    fn math(input: ExampleBenchmarkInput, number: u64) {
+        let x = input;
+        let x = black_box(x * number);
+        let x = black_box(x / number);
+        let x = black_box(x + number);
+        let _ = black_box(x - number);
+    }
+
+    for number in 1..100_000_000 {
+        black_box(math(input, number));
+    }
+}
+
+// This implementation will be run by `cuprate-benchmark`.
+impl Benchmark for Example {
+    type Input = ExampleBenchmarkInput;
+    const SETUP: fn() -> Self::Input = example_benchmark_setup;
+    const MAIN: fn(Self::Input) = example_benchmark_main;
+}

benches/benchmark/lib/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name        = "cuprate-benchmark-lib"
+version     = "0.0.0"
+edition     = "2021"
+description = "Cuprate's benchmarking library"
+license     = "MIT"
+authors     = ["hinto-janai"]
+repository  = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/lib"
+keywords    = ["cuprate", "benchmarking", "library"]
+
+[features]
+
+[dependencies]
+
+[dev-dependencies]
+
+[lints]
+workspace = true