diff --git a/crates/config/src/fuzz.rs b/crates/config/src/fuzz.rs index cb5290d54860f..0d85aa43b83d1 100644 --- a/crates/config/src/fuzz.rs +++ b/crates/config/src/fuzz.rs @@ -24,10 +24,11 @@ pub struct FuzzConfig { pub dictionary: FuzzDictionaryConfig, /// Number of runs to execute and include in the gas report. pub gas_report_samples: u32, + /// The fuzz corpus configuration. + #[serde(flatten)] + pub corpus: FuzzCorpusConfig, /// Path where fuzz failures are recorded and replayed. pub failure_persist_dir: Option, - /// Name of the file to record fuzz failures, defaults to `failures`. - pub failure_persist_file: Option, /// show `console.log` in fuzz test, defaults to `false` pub show_logs: bool, /// Optional timeout (in seconds) for each property test @@ -43,8 +44,8 @@ impl Default for FuzzConfig { seed: None, dictionary: FuzzDictionaryConfig::default(), gas_report_samples: 256, + corpus: FuzzCorpusConfig::default(), failure_persist_dir: None, - failure_persist_file: None, show_logs: false, timeout: None, } @@ -54,11 +55,7 @@ impl Default for FuzzConfig { impl FuzzConfig { /// Creates fuzz configuration to write failures in `{PROJECT_ROOT}/cache/fuzz` dir. pub fn new(cache_dir: PathBuf) -> Self { - Self { - failure_persist_dir: Some(cache_dir), - failure_persist_file: Some("failures".to_string()), - ..Default::default() - } + Self { failure_persist_dir: Some(cache_dir), ..Default::default() } } } @@ -97,3 +94,49 @@ impl Default for FuzzDictionaryConfig { } } } + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FuzzCorpusConfig { + // Path to corpus directory, enabled coverage guided fuzzing mode. + // If not set then sequences producing new coverage are not persisted and mutated. + pub corpus_dir: Option, + // Whether corpus to use gzip file compression and decompression. + pub corpus_gzip: bool, + // Number of mutations until entry marked as eligible to be flushed from in-memory corpus. + // Mutations will be performed at least `corpus_min_mutations` times. + pub corpus_min_mutations: usize, + // Number of corpus that won't be evicted from memory. + pub corpus_min_size: usize, + /// Whether to collect and display edge coverage metrics. + pub show_edge_coverage: bool, +} + +impl FuzzCorpusConfig { + pub fn with_test_name(&mut self, test_name: &String) { + if let Some(corpus_dir) = &self.corpus_dir { + self.corpus_dir = Some(corpus_dir.join(test_name)); + } + } + + /// Whether edge coverage should be collected and displayed. + pub fn collect_edge_coverage(&self) -> bool { + self.corpus_dir.is_some() || self.show_edge_coverage + } + + /// Whether coverage guided fuzzing is enabled. + pub fn is_coverage_guided(&self) -> bool { + self.corpus_dir.is_some() + } +} + +impl Default for FuzzCorpusConfig { + fn default() -> Self { + Self { + corpus_dir: None, + corpus_gzip: true, + corpus_min_mutations: 5, + corpus_min_size: 0, + show_edge_coverage: false, + } + } +} diff --git a/crates/config/src/invariant.rs b/crates/config/src/invariant.rs index 841b685d7ab89..140d5b2f5a796 100644 --- a/crates/config/src/invariant.rs +++ b/crates/config/src/invariant.rs @@ -1,6 +1,6 @@ //! Configuration for invariant testing -use crate::fuzz::FuzzDictionaryConfig; +use crate::fuzz::{FuzzCorpusConfig, FuzzDictionaryConfig}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -26,15 +26,9 @@ pub struct InvariantConfig { pub max_assume_rejects: u32, /// Number of runs to execute and include in the gas report. pub gas_report_samples: u32, - /// Path where invariant corpus is stored, enables coverage guided fuzzing and edge coverage - /// metrics. - pub corpus_dir: Option, - /// Whether corpus to use gzip file compression and decompression. - pub corpus_gzip: bool, - // Number of corpus mutations until marked as eligible to be flushed from memory. - pub corpus_min_mutations: usize, - // Number of corpus that won't be evicted from memory. - pub corpus_min_size: usize, + /// The fuzz corpus configuration. + #[serde(flatten)] + pub corpus: FuzzCorpusConfig, /// Path where invariant failures are recorded and replayed. pub failure_persist_dir: Option, /// Whether to collect and display fuzzed selectors metrics. @@ -43,8 +37,6 @@ pub struct InvariantConfig { pub timeout: Option, /// Display counterexample as solidity calls. pub show_solidity: bool, - /// Whether to collect and display edge coverage metrics. - pub show_edge_coverage: bool, } impl Default for InvariantConfig { @@ -58,15 +50,11 @@ impl Default for InvariantConfig { shrink_run_limit: 5000, max_assume_rejects: 65536, gas_report_samples: 256, - corpus_dir: None, - corpus_gzip: true, - corpus_min_mutations: 5, - corpus_min_size: 0, + corpus: FuzzCorpusConfig::default(), failure_persist_dir: None, show_metrics: true, timeout: None, show_solidity: false, - show_edge_coverage: false, } } } @@ -83,15 +71,11 @@ impl InvariantConfig { shrink_run_limit: 5000, max_assume_rejects: 65536, gas_report_samples: 256, - corpus_dir: None, - corpus_gzip: true, - corpus_min_mutations: 5, - corpus_min_size: 0, + corpus: FuzzCorpusConfig::default(), failure_persist_dir: Some(cache_dir), show_metrics: true, timeout: None, show_solidity: false, - show_edge_coverage: false, } } } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index f83385b1b3531..d5f5511234919 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -107,7 +107,7 @@ pub use providers::Remappings; use providers::*; mod fuzz; -pub use fuzz::{FuzzConfig, FuzzDictionaryConfig}; +pub use fuzz::{FuzzConfig, FuzzCorpusConfig, FuzzDictionaryConfig}; mod invariant; pub use invariant::InvariantConfig; @@ -1095,7 +1095,8 @@ impl Config { } }; remove_test_dir(&self.fuzz.failure_persist_dir); - remove_test_dir(&self.invariant.corpus_dir); + remove_test_dir(&self.fuzz.corpus.corpus_dir); + remove_test_dir(&self.invariant.corpus.corpus_dir); remove_test_dir(&self.invariant.failure_persist_dir); Ok(()) @@ -4614,7 +4615,6 @@ mod tests { runs: 512, depth: 10, failure_persist_dir: Some(PathBuf::from("cache/invariant")), - corpus_dir: None, ..Default::default() } ); diff --git a/crates/evm/core/src/constants.rs b/crates/evm/core/src/constants.rs index c69098ea8d992..e773a3f43e963 100644 --- a/crates/evm/core/src/constants.rs +++ b/crates/evm/core/src/constants.rs @@ -37,9 +37,6 @@ pub const MAGIC_ASSUME: &[u8] = b"FOUNDRY::ASSUME"; /// Magic return value returned by the `skip` cheatcode. Optionally appended with a reason. pub const MAGIC_SKIP: &[u8] = b"FOUNDRY::SKIP"; -/// Test timeout return value. -pub const TEST_TIMEOUT: &str = "FOUNDRY::TEST_TIMEOUT"; - /// The address that deploys the default CREATE2 deployer contract. pub const DEFAULT_CREATE2_DEPLOYER_DEPLOYER: Address = address!("0x3fAB184622Dc19b6109349B94811493BF2a45362"); diff --git a/crates/evm/evm/src/executors/invariant/corpus.rs b/crates/evm/evm/src/executors/corpus.rs similarity index 63% rename from crates/evm/evm/src/executors/invariant/corpus.rs rename to crates/evm/evm/src/executors/corpus.rs index 2614f8cb3da5b..7a4a18be8811e 100644 --- a/crates/evm/evm/src/executors/invariant/corpus.rs +++ b/crates/evm/evm/src/executors/corpus.rs @@ -1,14 +1,13 @@ -use crate::executors::{ - Executor, - invariant::{InvariantTest, InvariantTestRun}, -}; +use crate::executors::{Executor, RawCallResult}; use alloy_dyn_abi::JsonAbiExt; -use alloy_primitives::U256; +use alloy_json_abi::Function; +use alloy_primitives::{Bytes, U256}; use eyre::eyre; -use foundry_config::InvariantConfig; +use foundry_config::FuzzCorpusConfig; use foundry_evm_fuzz::{ - invariant::{BasicTxDetails, FuzzRunIdentifiedContracts}, - strategies::fuzz_param_from_state, + BasicTxDetails, + invariant::FuzzRunIdentifiedContracts, + strategies::{EvmFuzzState, mutate_param_value}, }; use proptest::{ prelude::{Just, Rng, Strategy}, @@ -27,6 +26,7 @@ use uuid::Uuid; const METADATA_SUFFIX: &str = "metadata.json"; const JSON_EXTENSION: &str = ".json"; const FAVORABILITY_THRESHOLD: f64 = 0.3; +const COVERAGE_MAP_SIZE: usize = 65536; /// Possible mutation strategies to apply on a call sequence. #[derive(Debug, Clone)] @@ -74,12 +74,12 @@ impl CorpusEntry { } /// New corpus with given call sequence and new uuid. - pub fn from_tx_seq(tx_seq: Vec) -> Self { + pub fn from_tx_seq(tx_seq: &[BasicTxDetails]) -> Self { Self { uuid: Uuid::new_v4(), total_mutations: 0, new_finds_produced: 0, - tx_seq, + tx_seq: tx_seq.into(), is_favored: false, } } @@ -128,21 +128,14 @@ impl CorpusMetrics { } } -/// Invariant corpus manager. -pub struct TxCorpusManager { +/// Fuzz corpus manager, used in coverage guided fuzzing mode by both stateless and stateful tests. +pub(crate) struct CorpusManager { // Fuzzed calls generator. tx_generator: BoxedStrategy, // Call sequence mutation strategy type generator. mutation_generator: BoxedStrategy, - // Path to invariant corpus directory. If None, sequences with new coverage are not persisted. - corpus_dir: Option, // TODO consolidate into config - // Whether corpus to use gzip file compression and decompression. - corpus_gzip: bool, - // Number of mutations until entry marked as eligible to be flushed from in-memory corpus. - // Mutations will be performed at least `corpus_min_mutations` times. - corpus_min_mutations: usize, - // Number of corpus that won't be evicted from memory. - corpus_min_size: usize, + // Corpus configuration. + config: FuzzCorpusConfig, // In-memory corpus, populated from persisted files and current runs. // Mutation is performed on these. in_memory_corpus: Vec, @@ -150,19 +143,24 @@ pub struct TxCorpusManager { current_mutated: Option, // Number of failed replays from persisted corpus. failed_replays: usize, + // History of binned hitcount of edges seen during fuzzing. + history_map: Vec, // Corpus metrics. pub(crate) metrics: CorpusMetrics, } -impl TxCorpusManager { +impl CorpusManager { pub fn new( - invariant_config: &InvariantConfig, - test_name: &String, - fuzzed_contracts: &FuzzRunIdentifiedContracts, + config: &FuzzCorpusConfig, + func_name: &String, tx_generator: BoxedStrategy, executor: &Executor, - history_map: &mut [u8], + fuzzed_function: Option<&Function>, + fuzzed_contracts: Option<&FuzzRunIdentifiedContracts>, ) -> eyre::Result { + let mut config = config.clone(); + config.with_test_name(func_name); + let mutation_generator = prop_oneof![ Just(MutationType::Splice), Just(MutationType::Repeat), @@ -172,48 +170,49 @@ impl TxCorpusManager { Just(MutationType::Abi), ] .boxed(); + let mut history_map = vec![0u8; COVERAGE_MAP_SIZE]; + let mut metrics = CorpusMetrics::default(); let mut in_memory_corpus = vec![]; - let corpus_gzip = invariant_config.corpus_gzip; - let corpus_min_mutations = invariant_config.corpus_min_mutations; - let corpus_min_size = invariant_config.corpus_min_size; let mut failed_replays = 0; // Early return if corpus dir / coverage guided fuzzing not configured. - let Some(corpus_dir) = &invariant_config.corpus_dir else { + let Some(corpus_dir) = &config.corpus_dir else { return Ok(Self { tx_generator, mutation_generator, - corpus_dir: None, - corpus_gzip, - corpus_min_mutations, - corpus_min_size, + config, in_memory_corpus, current_mutated: None, failed_replays, - metrics: CorpusMetrics::default(), + history_map, + metrics, }); }; - // Ensure corpus dir for invariant function is created. - let corpus_dir = corpus_dir.join(test_name); + // Ensure corpus dir for current test is created. if !corpus_dir.is_dir() { - foundry_common::fs::create_dir_all(&corpus_dir)?; + foundry_common::fs::create_dir_all(corpus_dir)?; } - let fuzzed_contracts = fuzzed_contracts.targets.lock(); - let mut metrics = CorpusMetrics::default(); + let can_replay_tx = |tx: &BasicTxDetails| -> bool { + fuzzed_contracts.is_some_and(|contracts| contracts.targets.lock().can_replay(tx)) + || fuzzed_function.is_some_and(|function| { + tx.call_details + .calldata + .get(..4) + .is_some_and(|selector| function.selector() == selector) + }) + }; - for entry in std::fs::read_dir(&corpus_dir)? { + 'corpus_replay: for entry in std::fs::read_dir(corpus_dir)? { let path = entry?.path(); if path.is_file() && let Some(name) = path.file_name().and_then(|s| s.to_str()) + && name.contains(METADATA_SUFFIX) { // Ignore metadata files - if name.contains(METADATA_SUFFIX) { - continue; - } + continue; } - metrics.corpus_count += 1; let read_corpus_result = match path.extension().and_then(|ext| ext.to_str()) { Some("gz") => foundry_common::fs::read_json_gzip_file::>(&path), @@ -229,27 +228,39 @@ impl TxCorpusManager { // Warm up history map from loaded sequences. let mut executor = executor.clone(); for tx in &tx_seq { - let mut call_result = executor - .call_raw( - tx.sender, - tx.call_details.target, - tx.call_details.calldata.clone(), - U256::ZERO, - ) - .map_err(|e| eyre!(format!("Could not make raw evm call: {e}")))?; - - if fuzzed_contracts.can_replay(tx) { - let (new_coverage, is_edge) = call_result.merge_edge_coverage(history_map); + if can_replay_tx(tx) { + let mut call_result = executor + .call_raw( + tx.sender, + tx.call_details.target, + tx.call_details.calldata.clone(), + U256::ZERO, + ) + .map_err(|e| eyre!(format!("Could not make raw evm call: {e}")))?; + + let (new_coverage, is_edge) = + call_result.merge_edge_coverage(&mut history_map); if new_coverage { metrics.update_seen(is_edge); } - executor.commit(&mut call_result); + // Commit only when running invariant / stateful tests. + if fuzzed_contracts.is_some() { + executor.commit(&mut call_result); + } } else { failed_replays += 1; + + // If the only input for fuzzed function cannot be replied, then move to + // next one without adding it in memory. + if fuzzed_function.is_some() { + continue 'corpus_replay; + } } } + metrics.corpus_count += 1; + trace!( target: "corpus", "load sequence with len {} from corpus file {}", @@ -257,7 +268,7 @@ impl TxCorpusManager { path.display() ); - // Populate in memory corpus with sequence from corpus file. + // Populate in memory corpus with the sequence from corpus file. in_memory_corpus.push(CorpusEntry::new(tx_seq, path)?); } } @@ -265,22 +276,21 @@ impl TxCorpusManager { Ok(Self { tx_generator, mutation_generator, - corpus_dir: Some(corpus_dir), - corpus_gzip, - corpus_min_mutations, - corpus_min_size, + config, in_memory_corpus, current_mutated: None, failed_replays, + history_map, metrics, }) } - /// Collects inputs from given invariant run, if new coverage produced. - /// Persists call sequence (if corpus directory is configured) and updates in-memory corpus. - pub fn collect_inputs(&mut self, test_run: &InvariantTestRun) { + /// Updates stats for the given call sequence, if new coverage produced. + /// Persists the call sequence (if corpus directory is configured and new coverage) and updates + /// in-memory corpus. + pub fn process_inputs(&mut self, inputs: &[BasicTxDetails], new_coverage: bool) { // Early return if corpus dir / coverage guided fuzzing is not configured. - let Some(corpus_dir) = &self.corpus_dir else { + let Some(corpus_dir) = &self.config.corpus_dir else { return; }; @@ -290,7 +300,7 @@ impl TxCorpusManager { self.in_memory_corpus.iter_mut().find(|corpus| corpus.uuid.eq(uuid)) { corpus.total_mutations += 1; - if test_run.new_coverage { + if new_coverage { corpus.new_finds_produced += 1 } let is_favored = (corpus.new_finds_produced as f64 / corpus.total_mutations as f64) @@ -309,15 +319,15 @@ impl TxCorpusManager { } // Collect inputs only if current run produced new coverage. - if !test_run.new_coverage { + if !new_coverage { return; } - let corpus = CorpusEntry::from_tx_seq(test_run.inputs.clone()); + let corpus = CorpusEntry::from_tx_seq(inputs); let corpus_uuid = corpus.uuid; // Persist to disk if corpus dir is configured. - let write_result = if self.corpus_gzip { + let write_result = if self.config.corpus_gzip { foundry_common::fs::write_json_gzip_file( corpus_dir.join(format!("{corpus_uuid}{JSON_EXTENSION}.gz")).as_path(), &corpus.tx_seq, @@ -346,50 +356,29 @@ impl TxCorpusManager { } /// Generates new call sequence from in memory corpus. Evicts oldest corpus mutated more than - /// configured max mutations value. - pub fn new_sequence(&mut self, test: &InvariantTest) -> eyre::Result> { + /// configured max mutations value. Used by invariant test campaigns. + pub fn new_inputs( + &mut self, + test_runner: &mut TestRunner, + fuzz_state: &EvmFuzzState, + targeted_contracts: &FuzzRunIdentifiedContracts, + ) -> eyre::Result> { let mut new_seq = vec![]; - let test_runner = &mut test.execution_data.borrow_mut().branch_runner; // Early return with first_input only if corpus dir / coverage guided fuzzing not // configured. - let Some(corpus_dir) = &self.corpus_dir else { + if !self.config.is_coverage_guided() { new_seq.push(self.new_tx(test_runner)?); return Ok(new_seq); }; if !self.in_memory_corpus.is_empty() { - // Flush oldest corpus mutated more than configured max mutations unless they are - // favored. - let should_evict = self.in_memory_corpus.len() > self.corpus_min_size.max(1); - if should_evict - && let Some(index) = self.in_memory_corpus.iter().position(|corpus| { - corpus.total_mutations > self.corpus_min_mutations && !corpus.is_favored - }) - { - let corpus = self.in_memory_corpus.get(index).unwrap(); - - let uuid = corpus.uuid; - debug!(target: "corpus", "evict corpus {uuid}"); - - // Flush to disk the seed metadata at the time of eviction. - let eviction_time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - foundry_common::fs::write_json_file( - corpus_dir.join(format!("{uuid}-{eviction_time}-{METADATA_SUFFIX}")).as_path(), - &corpus, - )?; - - // Remove corpus from memory. - self.in_memory_corpus.remove(index); - } + self.evict_oldest_corpus()?; let mutation_type = self .mutation_generator .new_tree(test_runner) - .expect("Could not generate mutation type") + .map_err(|err| eyre!("Could not generate mutation type {err}"))? .current(); let rng = test_runner.rng(); let corpus_len = self.in_memory_corpus.len(); @@ -463,7 +452,7 @@ impl TxCorpusManager { } } MutationType::Abi => { - let targets = test.targeted_contracts.targets.lock(); + let targets = targeted_contracts.targets.lock(); let corpus = if rng.random::() { primary } else { secondary }; trace!(target: "corpus", "ABI mutate args of {}", corpus.uuid); @@ -477,56 +466,14 @@ impl TxCorpusManager { // TODO add call_value to call details and mutate it as well as sender some // of the time if !function.inputs.is_empty() { - let mut new_function = function.clone(); - let mut arg_mutation_rounds = - rng.random_range(0..=function.inputs.len()).max(1); - let round_arg_idx: Vec = if function.inputs.len() <= 1 { - vec![0] - } else { - (0..arg_mutation_rounds) - .map(|_| { - test_runner.rng().random_range(0..function.inputs.len()) - }) - .collect() - }; - // TODO mutation strategy for individual ABI types - let mut prev_inputs = function - .abi_decode_input(&tx.call_details.calldata[4..]) - .expect("fuzzed_artifacts returned wrong sig"); - // For now, only new inputs are generated, no existing inputs are - // mutated. - let mut gen_input = |input: &alloy_json_abi::Param| { - fuzz_param_from_state( - &input.selector_type().parse().unwrap(), - &test.fuzz_state, - ) - .new_tree(test_runner) - .expect("Could not generate case") - .current() - }; - - while arg_mutation_rounds > 0 { - let idx = round_arg_idx[arg_mutation_rounds - 1]; - let input = new_function - .inputs - .get_mut(idx) - .expect("Could not get input to mutate"); - let new_input = gen_input(input); - prev_inputs[idx] = new_input; - arg_mutation_rounds -= 1; - } - - tx.call_details.calldata = new_function - .abi_encode_input(&prev_inputs) - .map_err(|e| eyre!(e.to_string()))? - .into(); + self.abi_mutate(tx, function, test_runner, fuzz_state)?; } } } } } - // Make sure sequence contains at least one tx to start fuzzing from. + // Make sure the new sequence contains at least one tx to start fuzzing from. if new_seq.is_empty() { new_seq.push(self.new_tx(test_runner)?); } @@ -535,6 +482,36 @@ impl TxCorpusManager { Ok(new_seq) } + /// Generates new input from in memory corpus. Evicts oldest corpus mutated more than + /// configured max mutations value. Used by fuzz test campaigns. + pub fn new_input( + &mut self, + test_runner: &mut TestRunner, + fuzz_state: &EvmFuzzState, + function: &Function, + ) -> eyre::Result { + // Early return if not running with coverage guided fuzzing. + if !self.config.is_coverage_guided() { + return Ok(self.new_tx(test_runner)?.call_details.calldata); + } + + let tx = if !self.in_memory_corpus.is_empty() { + self.evict_oldest_corpus()?; + + let corpus = &self.in_memory_corpus + [test_runner.rng().random_range(0..self.in_memory_corpus.len())]; + self.current_mutated = Some(corpus.uuid); + let new_seq = corpus.tx_seq.clone(); + let mut tx = new_seq.first().unwrap().clone(); + self.abi_mutate(&mut tx, function, test_runner, fuzz_state)?; + tx + } else { + self.new_tx(test_runner)? + }; + + Ok(tx.call_details.calldata) + } + /// Returns the next call to be used in call sequence. /// If coverage guided fuzzing is not configured or if previous input was discarded then this is /// a new tx from strategy. @@ -543,16 +520,14 @@ impl TxCorpusManager { /// sequence. pub fn generate_next_input( &mut self, - test: &InvariantTest, + test_runner: &mut TestRunner, sequence: &[BasicTxDetails], discarded: bool, depth: usize, ) -> eyre::Result { - let test_runner = &mut test.execution_data.borrow_mut().branch_runner; - // Early return with new input if corpus dir / coverage guided fuzzing not configured or if // call was discarded. - if self.corpus_dir.is_none() || discarded { + if self.config.corpus_dir.is_none() || discarded { return self.new_tx(test_runner); } @@ -566,7 +541,7 @@ impl TxCorpusManager { Ok(sequence[depth].clone()) } - /// Generates single call from invariant strategy. + /// Generates single call from corpus strategy. pub fn new_tx(&mut self, test_runner: &mut TestRunner) -> eyre::Result { Ok(self .tx_generator @@ -580,8 +555,91 @@ impl TxCorpusManager { self.failed_replays } - /// Updates seen edges or features metrics. - pub fn update_seen_metrics(&mut self, is_edge: bool) { - self.metrics.update_seen(is_edge); + /// Collects coverage from call result and updates metrics. + pub fn merge_edge_coverage(&mut self, call_result: &mut RawCallResult) -> bool { + if !self.config.collect_edge_coverage() { + return false; + } + + let (new_coverage, is_edge) = call_result.merge_edge_coverage(&mut self.history_map); + if new_coverage { + self.metrics.update_seen(is_edge); + } + new_coverage + } + + /// Flush the oldest corpus mutated more than configured max mutations unless they are + /// favored. + fn evict_oldest_corpus(&mut self) -> eyre::Result<()> { + if self.in_memory_corpus.len() > self.config.corpus_min_size.max(1) + && let Some(index) = self.in_memory_corpus.iter().position(|corpus| { + corpus.total_mutations > self.config.corpus_min_mutations && !corpus.is_favored + }) + { + let corpus = self.in_memory_corpus.get(index).unwrap(); + + let uuid = corpus.uuid; + debug!(target: "corpus", "evict corpus {uuid}"); + + // Flush to disk the seed metadata at the time of eviction. + let eviction_time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + foundry_common::fs::write_json_file( + self.config + .corpus_dir + .clone() + .unwrap() + .join(format!("{uuid}-{eviction_time}-{METADATA_SUFFIX}")) + .as_path(), + &corpus, + )?; + + // Remove corpus from memory. + self.in_memory_corpus.remove(index); + } + Ok(()) + } + + /// Mutates calldata of provided tx by abi decoding current values and randomly selecting the + /// inputs to change. + fn abi_mutate( + &self, + tx: &mut BasicTxDetails, + function: &Function, + test_runner: &mut TestRunner, + fuzz_state: &EvmFuzzState, + ) -> eyre::Result<()> { + // let rng = test_runner.rng(); + let mut arg_mutation_rounds = + test_runner.rng().random_range(0..=function.inputs.len()).max(1); + let round_arg_idx: Vec = if function.inputs.len() <= 1 { + vec![0] + } else { + (0..arg_mutation_rounds) + .map(|_| test_runner.rng().random_range(0..function.inputs.len())) + .collect() + }; + let mut prev_inputs = function + .abi_decode_input(&tx.call_details.calldata[4..]) + .map_err(|err| eyre!("failed to load previous inputs: {err}"))?; + + while arg_mutation_rounds > 0 { + let idx = round_arg_idx[arg_mutation_rounds - 1]; + prev_inputs[idx] = mutate_param_value( + &function + .inputs + .get(idx) + .expect("Could not get input to mutate") + .selector_type() + .parse()?, + prev_inputs[idx].clone(), + test_runner, + fuzz_state, + ); + arg_mutation_rounds -= 1; + } + + tx.call_details.calldata = + function.abi_encode_input(&prev_inputs).map_err(|e| eyre!(e.to_string()))?.into(); + Ok(()) } } diff --git a/crates/evm/evm/src/executors/fuzz/mod.rs b/crates/evm/evm/src/executors/fuzz/mod.rs index dc83ae94b712e..72fe796be7b7c 100644 --- a/crates/evm/evm/src/executors/fuzz/mod.rs +++ b/crates/evm/evm/src/executors/fuzz/mod.rs @@ -1,48 +1,58 @@ -use crate::executors::{Executor, FuzzTestTimer, RawCallResult}; +use crate::executors::{DURATION_BETWEEN_METRICS_REPORT, Executor, FuzzTestTimer, RawCallResult}; use alloy_dyn_abi::JsonAbiExt; use alloy_json_abi::Function; use alloy_primitives::{Address, Bytes, Log, U256, map::HashMap}; use eyre::Result; -use foundry_common::evm::Breakpoints; +use foundry_common::{evm::Breakpoints, sh_println}; use foundry_config::FuzzConfig; use foundry_evm_core::{ - constants::{CHEATCODE_ADDRESS, MAGIC_ASSUME, TEST_TIMEOUT}, + constants::{CHEATCODE_ADDRESS, MAGIC_ASSUME}, decode::{RevertDecoder, SkipReason}, }; use foundry_evm_coverage::HitMaps; use foundry_evm_fuzz::{ - BaseCounterExample, CounterExample, FuzzCase, FuzzError, FuzzFixtures, FuzzTestResult, + BaseCounterExample, BasicTxDetails, CallDetails, CounterExample, FuzzCase, FuzzError, + FuzzFixtures, FuzzTestResult, strategies::{EvmFuzzState, fuzz_calldata, fuzz_calldata_from_state}, }; use foundry_evm_traces::SparsedTraceArena; use indicatif::ProgressBar; -use proptest::test_runner::{TestCaseError, TestError, TestRunner}; -use std::{cell::RefCell, collections::BTreeMap}; +use proptest::{ + strategy::Strategy, + test_runner::{TestCaseError, TestRunner}, +}; +use serde_json::json; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; mod types; +use crate::executors::corpus::CorpusManager; pub use types::{CaseOutcome, CounterExampleOutcome, FuzzOutcome}; /// Contains data collected during fuzz test runs. #[derive(Default)] -pub struct FuzzTestData { +struct FuzzTestData { // Stores the first fuzz case. - pub first_case: Option, + first_case: Option, // Stored gas usage per fuzz case. - pub gas_by_case: Vec<(u64, u64)>, + gas_by_case: Vec<(u64, u64)>, // Stores the result and calldata of the last failed call, if any. - pub counterexample: (Bytes, RawCallResult), + counterexample: (Bytes, RawCallResult), // Stores up to `max_traces_to_collect` traces. - pub traces: Vec, + traces: Vec, // Stores breakpoints for the last fuzz case. - pub breakpoints: Option, + breakpoints: Option, // Stores coverage information for all fuzz cases. - pub coverage: Option, + coverage: Option, // Stores logs for all fuzz cases - pub logs: Vec, - // Stores gas snapshots for all fuzz cases - pub gas_snapshots: BTreeMap>, + logs: Vec, // Deprecated cheatcodes mapped to their replacements. - pub deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>, + deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>, + // Runs performed in fuzz test. + runs: u32, + // Current assume rejects of the fuzz run. + rejects: u32, + // Test failure. + failure: Option, } /// Wrapper around an [`Executor`] which provides fuzzing support using [`proptest`]. @@ -51,14 +61,16 @@ pub struct FuzzTestData { /// inputs, until it finds a counterexample. The provided [`TestRunner`] contains all the /// configuration which can be overridden via [environment variables](proptest::test_runner::Config) pub struct FuzzedExecutor { - /// The EVM executor + /// The EVM executor. executor: Executor, /// The fuzzer runner: TestRunner, - /// The account that calls tests + /// The account that calls tests. sender: Address, - /// The fuzz configuration + /// The fuzz configuration. config: FuzzConfig, + /// The persisted counterexample to be replayed, if any. + persisted_failure: Option, } impl FuzzedExecutor { @@ -68,155 +80,197 @@ impl FuzzedExecutor { runner: TestRunner, sender: Address, config: FuzzConfig, + persisted_failure: Option, ) -> Self { - Self { executor, runner, sender, config } + Self { executor, runner, sender, config, persisted_failure } } /// Fuzzes the provided function, assuming it is available at the contract at `address` /// If `should_fail` is set to `true`, then it will stop only when there's a success /// test case. /// - /// Returns a list of all the consumed gas and calldata of every fuzz case + /// Returns a list of all the consumed gas and calldata of every fuzz case. pub fn fuzz( - &self, + &mut self, func: &Function, fuzz_fixtures: &FuzzFixtures, deployed_libs: &[Address], address: Address, rd: &RevertDecoder, progress: Option<&ProgressBar>, - ) -> FuzzTestResult { + ) -> Result { // Stores the fuzz test execution data. - let execution_data = RefCell::new(FuzzTestData::default()); + let mut test_data = FuzzTestData::default(); let state = self.build_fuzz_state(deployed_libs); let dictionary_weight = self.config.dictionary.dictionary_weight.min(100); let strategy = proptest::prop_oneof![ 100 - dictionary_weight => fuzz_calldata(func.clone(), fuzz_fixtures), dictionary_weight => fuzz_calldata_from_state(func.clone(), &state), - ]; + ] + .prop_map(move |calldata| BasicTxDetails { + sender: Default::default(), + call_details: CallDetails { target: Default::default(), calldata }, + }); // We want to collect at least one trace which will be displayed to user. let max_traces_to_collect = std::cmp::max(1, self.config.gas_report_samples) as usize; - let show_logs = self.config.show_logs; + + let mut corpus_manager = CorpusManager::new( + &self.config.corpus, + &func.name, + strategy.boxed(), + &self.executor, + Some(func), + None, + )?; // Start timer for this fuzz test. let timer = FuzzTestTimer::new(self.config.timeout); + let mut last_metrics_report = Instant::now(); + let max_runs = self.config.runs; + let continue_campaign = |runs: u32| { + if timer.is_enabled() { !timer.is_timed_out() } else { runs < max_runs } + }; - let run_result = self.runner.clone().run(&strategy, |calldata| { - // Check if the timeout has been reached. - if timer.is_timed_out() { - return Err(TestCaseError::fail(TEST_TIMEOUT)); - } + 'stop: while continue_campaign(test_data.runs) { + // If counterexample recorded, replay it first, without incrementing runs. + let input = if let Some(failure) = self.persisted_failure.take() { + failure.calldata + } else { + // If running with progress, then increment current run. + if let Some(progress) = progress { + progress.inc(1); + // Display metrics in progress bar. + if self.config.corpus.collect_edge_coverage() { + progress.set_message(format!("{}", &corpus_manager.metrics)); + } + } else if self.config.corpus.collect_edge_coverage() + && last_metrics_report.elapsed() > DURATION_BETWEEN_METRICS_REPORT + { + // Display metrics inline. + let metrics = json!({ + "timestamp": SystemTime::now() + .duration_since(UNIX_EPOCH)? + .as_secs(), + "test": func.name, + "metrics": &corpus_manager.metrics, + }); + let _ = sh_println!("{}", serde_json::to_string(&metrics)?); + last_metrics_report = Instant::now(); + }; - let fuzz_res = self.single_fuzz(address, calldata)?; + test_data.runs += 1; - // If running with progress then increment current run. - if let Some(progress) = progress { - progress.inc(1); + match corpus_manager.new_input(&mut self.runner, &state, func) { + Ok(input) => input, + Err(err) => { + test_data.failure = Some(TestCaseError::fail(format!( + "failed to generate fuzzed input: {err}" + ))); + break 'stop; + } + } }; - match fuzz_res { - FuzzOutcome::Case(case) => { - let mut data = execution_data.borrow_mut(); - data.gas_by_case.push((case.case.gas, case.case.stipend)); - - if data.first_case.is_none() { - data.first_case.replace(case.case); - } + match self.single_fuzz(address, input, &mut corpus_manager) { + Ok(fuzz_outcome) => match fuzz_outcome { + FuzzOutcome::Case(case) => { + test_data.gas_by_case.push((case.case.gas, case.case.stipend)); - if let Some(call_traces) = case.traces { - if data.traces.len() == max_traces_to_collect { - data.traces.pop(); + if test_data.first_case.is_none() { + test_data.first_case.replace(case.case); } - data.traces.push(call_traces); - data.breakpoints.replace(case.breakpoints); - } - if show_logs { - data.logs.extend(case.logs); - } - - HitMaps::merge_opt(&mut data.coverage, case.coverage); + if let Some(call_traces) = case.traces { + if test_data.traces.len() == max_traces_to_collect { + test_data.traces.pop(); + } + test_data.traces.push(call_traces); + test_data.breakpoints.replace(case.breakpoints); + } - data.deprecated_cheatcodes = case.deprecated_cheatcodes; + if self.config.show_logs { + test_data.logs.extend(case.logs); + } - Ok(()) - } - FuzzOutcome::CounterExample(CounterExampleOutcome { - exit_reason: status, - counterexample: outcome, - .. - }) => { - // We cannot use the calldata returned by the test runner in `TestError::Fail`, - // since that input represents the last run case, which may not correspond with - // our failure - when a fuzz case fails, proptest will try to run at least one - // more case to find a minimal failure case. - let reason = rd.maybe_decode(&outcome.1.result, status); - execution_data.borrow_mut().logs.extend(outcome.1.logs.clone()); - execution_data.borrow_mut().counterexample = outcome; - // HACK: we have to use an empty string here to denote `None`. - Err(TestCaseError::fail(reason.unwrap_or_default())) + HitMaps::merge_opt(&mut test_data.coverage, case.coverage); + test_data.deprecated_cheatcodes = case.deprecated_cheatcodes; + } + FuzzOutcome::CounterExample(CounterExampleOutcome { + exit_reason: status, + counterexample: outcome, + .. + }) => { + let reason = rd.maybe_decode(&outcome.1.result, status); + test_data.logs.extend(outcome.1.logs.clone()); + test_data.counterexample = outcome; + test_data.failure = Some(TestCaseError::fail(reason.unwrap_or_default())); + break 'stop; + } + }, + Err(err) => { + match err { + TestCaseError::Fail(_) => { + test_data.failure = Some(err); + break 'stop; + } + TestCaseError::Reject(_) => { + // Apply max rejects only if configured, otherwise silently discard run. + if self.config.max_test_rejects > 0 { + test_data.rejects += 1; + if test_data.rejects >= self.config.max_test_rejects { + test_data.failure = Some(err); + break 'stop; + } + } + } + } } } - }); - - let fuzz_result = execution_data.into_inner(); - let (calldata, call) = fuzz_result.counterexample; + } - let mut traces = fuzz_result.traces; - let (last_run_traces, last_run_breakpoints) = if run_result.is_ok() { - (traces.pop(), fuzz_result.breakpoints) + let (calldata, call) = test_data.counterexample; + let mut traces = test_data.traces; + let (last_run_traces, last_run_breakpoints) = if test_data.failure.is_none() { + (traces.pop(), test_data.breakpoints) } else { (call.traces.clone(), call.cheatcodes.map(|c| c.breakpoints)) }; let mut result = FuzzTestResult { - first_case: fuzz_result.first_case.unwrap_or_default(), - gas_by_case: fuzz_result.gas_by_case, - success: run_result.is_ok(), + first_case: test_data.first_case.unwrap_or_default(), + gas_by_case: test_data.gas_by_case, + success: test_data.failure.is_none(), skipped: false, reason: None, counterexample: None, - logs: fuzz_result.logs, + logs: test_data.logs, labels: call.labels, traces: last_run_traces, breakpoints: last_run_breakpoints, gas_report_traces: traces.into_iter().map(|a| a.arena).collect(), - line_coverage: fuzz_result.coverage, - deprecated_cheatcodes: fuzz_result.deprecated_cheatcodes, + line_coverage: test_data.coverage, + deprecated_cheatcodes: test_data.deprecated_cheatcodes, + failed_corpus_replays: corpus_manager.failed_replays(), }; - match run_result { - Ok(()) => {} - Err(TestError::Abort(reason)) => { - let msg = reason.message(); - // Currently the only operation that can trigger proptest global rejects is the - // `vm.assume` cheatcode, thus we surface this info to the user when the fuzz test - // aborts due to too many global rejects, making the error message more actionable. - result.reason = if msg == "Too many global rejects" { - let error = FuzzError::TooManyRejects(self.runner.config().max_global_rejects); - Some(error.to_string()) + match test_data.failure { + Some(TestCaseError::Fail(reason)) => { + let reason = reason.to_string(); + result.reason = (!reason.is_empty()).then_some(reason); + let args = if let Some(data) = calldata.get(4..) { + func.abi_decode_input(data).unwrap_or_default() } else { - Some(msg.to_string()) + vec![] }; + result.counterexample = Some(CounterExample::Single( + BaseCounterExample::from_fuzz_call(calldata, args, call.traces), + )); } - Err(TestError::Fail(reason, _)) => { + Some(TestCaseError::Reject(reason)) => { let reason = reason.to_string(); - if reason == TEST_TIMEOUT { - // If the reason is a timeout, we consider the fuzz test successful. - result.success = true; - } else { - result.reason = (!reason.is_empty()).then_some(reason); - let args = if let Some(data) = calldata.get(4..) { - func.abi_decode_input(data).unwrap_or_default() - } else { - vec![] - }; - - result.counterexample = Some(CounterExample::Single( - BaseCounterExample::from_fuzz_call(calldata, args, call.traces), - )); - } + result.reason = (!reason.is_empty()).then_some(reason); } + None => {} } if let Some(reason) = &result.reason @@ -228,24 +282,35 @@ impl FuzzedExecutor { state.log_stats(); - result + Ok(result) } /// Granular and single-step function that runs only one fuzz and returns either a `CaseOutcome` /// or a `CounterExampleOutcome` - pub fn single_fuzz( - &self, + fn single_fuzz( + &mut self, address: Address, - calldata: alloy_primitives::Bytes, + calldata: Bytes, + coverage_metrics: &mut CorpusManager, ) -> Result { let mut call = self .executor .call_raw(self.sender, address, calldata.clone(), U256::ZERO) .map_err(|e| TestCaseError::fail(e.to_string()))?; + let new_coverage = coverage_metrics.merge_edge_coverage(&mut call); + coverage_metrics.process_inputs( + &[BasicTxDetails { + sender: self.sender, + call_details: CallDetails { target: address, calldata: calldata.clone() }, + }], + new_coverage, + ); // Handle `vm.assume`. if call.result.as_ref() == MAGIC_ASSUME { - return Err(TestCaseError::reject(FuzzError::AssumeReject)); + return Err(TestCaseError::reject(FuzzError::TooManyRejects( + self.config.max_test_rejects, + ))); } let (breakpoints, deprecated_cheatcodes) = diff --git a/crates/evm/evm/src/executors/invariant/error.rs b/crates/evm/evm/src/executors/invariant/error.rs index 061d9ed8d0ddc..9f48e9da82cbd 100644 --- a/crates/evm/evm/src/executors/invariant/error.rs +++ b/crates/evm/evm/src/executors/invariant/error.rs @@ -1,9 +1,9 @@ -use super::{BasicTxDetails, InvariantContract}; +use super::InvariantContract; use crate::executors::RawCallResult; use alloy_primitives::{Address, Bytes}; use foundry_config::InvariantConfig; use foundry_evm_core::decode::RevertDecoder; -use foundry_evm_fuzz::{Reason, invariant::FuzzRunIdentifiedContracts}; +use foundry_evm_fuzz::{BasicTxDetails, Reason, invariant::FuzzRunIdentifiedContracts}; use proptest::test_runner::TestError; /// Stores information about failures and reverts of the invariant tests. diff --git a/crates/evm/evm/src/executors/invariant/mod.rs b/crates/evm/evm/src/executors/invariant/mod.rs index 47c5b7daa4904..84ff2a29d00a9 100644 --- a/crates/evm/evm/src/executors/invariant/mod.rs +++ b/crates/evm/evm/src/executors/invariant/mod.rs @@ -14,10 +14,10 @@ use foundry_evm_core::{ precompiles::PRECOMPILES, }; use foundry_evm_fuzz::{ - FuzzCase, FuzzFixtures, FuzzedCases, + BasicTxDetails, FuzzCase, FuzzFixtures, FuzzedCases, invariant::{ - ArtifactFilters, BasicTxDetails, FuzzRunIdentifiedContracts, InvariantContract, - RandomCallGenerator, SenderFilters, TargetedContract, TargetedContracts, + ArtifactFilters, FuzzRunIdentifiedContracts, InvariantContract, RandomCallGenerator, + SenderFilters, TargetedContract, TargetedContracts, }, strategies::{EvmFuzzState, invariant_strat, override_call_strat}, }; @@ -29,10 +29,9 @@ use result::{assert_after_invariant, assert_invariants, can_continue}; use revm::state::Account; use shrink::shrink_sequence; use std::{ - cell::RefCell, collections::{HashMap as Map, btree_map::Entry}, sync::Arc, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + time::{Instant, SystemTime, UNIX_EPOCH}, }; mod error; @@ -48,10 +47,10 @@ pub use result::InvariantFuzzTestResult; use serde::{Deserialize, Serialize}; use serde_json::json; -mod corpus; - mod shrink; -use crate::executors::{EvmError, FuzzTestTimer, invariant::corpus::TxCorpusManager}; +use crate::executors::{ + DURATION_BETWEEN_METRICS_REPORT, EvmError, FuzzTestTimer, corpus::CorpusManager, +}; pub use shrink::check_sequence; sol! { @@ -108,8 +107,6 @@ sol! { } } -const DURATION_BETWEEN_METRICS_REPORT: Duration = Duration::from_secs(5); - /// Contains invariant metrics for a single fuzzed selector. #[derive(Default, Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub struct InvariantMetrics { @@ -122,42 +119,42 @@ pub struct InvariantMetrics { } /// Contains data collected during invariant test runs. -pub struct InvariantTestData { +struct InvariantTestData { // Consumed gas and calldata of every successful fuzz call. - pub fuzz_cases: Vec, + fuzz_cases: Vec, // Data related to reverts or failed assertions of the test. - pub failures: InvariantFailures, + failures: InvariantFailures, // Calldata in the last invariant run. - pub last_run_inputs: Vec, + last_run_inputs: Vec, // Additional traces for gas report. - pub gas_report_traces: Vec>, + gas_report_traces: Vec>, // Last call results of the invariant test. - pub last_call_results: Option, + last_call_results: Option, // Line coverage information collected from all fuzzed calls. - pub line_coverage: Option, + line_coverage: Option, // Metrics for each fuzzed selector. - pub metrics: Map, + metrics: Map, // Proptest runner to query for random values. // The strategy only comes with the first `input`. We fill the rest of the `inputs` // until the desired `depth` so we can use the evolving fuzz dictionary // during the run. - pub branch_runner: TestRunner, + branch_runner: TestRunner, } /// Contains invariant test data. -pub struct InvariantTest { +struct InvariantTest { // Fuzz state of invariant test. - pub fuzz_state: EvmFuzzState, + fuzz_state: EvmFuzzState, // Contracts fuzzed by the invariant test. - pub targeted_contracts: FuzzRunIdentifiedContracts, + targeted_contracts: FuzzRunIdentifiedContracts, // Data collected during invariant runs. - pub execution_data: RefCell, + test_data: InvariantTestData, } impl InvariantTest { /// Instantiates an invariant test. - pub fn new( + fn new( fuzz_state: EvmFuzzState, targeted_contracts: FuzzRunIdentifiedContracts, failures: InvariantFailures, @@ -168,7 +165,7 @@ impl InvariantTest { if last_call_results.is_none() { fuzz_cases.push(FuzzedCases::new(vec![])); } - let execution_data = RefCell::new(InvariantTestData { + let test_data = InvariantTestData { fuzz_cases, failures, last_run_inputs: vec![], @@ -177,48 +174,48 @@ impl InvariantTest { line_coverage: None, metrics: Map::default(), branch_runner, - }); - Self { fuzz_state, targeted_contracts, execution_data } + }; + Self { fuzz_state, targeted_contracts, test_data } } /// Returns number of invariant test reverts. - pub fn reverts(&self) -> usize { - self.execution_data.borrow().failures.reverts + fn reverts(&self) -> usize { + self.test_data.failures.reverts } /// Whether invariant test has errors or not. - pub fn has_errors(&self) -> bool { - self.execution_data.borrow().failures.error.is_some() + fn has_errors(&self) -> bool { + self.test_data.failures.error.is_some() } /// Set invariant test error. - pub fn set_error(&self, error: InvariantFuzzError) { - self.execution_data.borrow_mut().failures.error = Some(error); + fn set_error(&mut self, error: InvariantFuzzError) { + self.test_data.failures.error = Some(error); } /// Set last invariant test call results. - pub fn set_last_call_results(&self, call_result: Option) { - self.execution_data.borrow_mut().last_call_results = call_result; + fn set_last_call_results(&mut self, call_result: Option) { + self.test_data.last_call_results = call_result; } /// Set last invariant run call sequence. - pub fn set_last_run_inputs(&self, inputs: &Vec) { - self.execution_data.borrow_mut().last_run_inputs.clone_from(inputs); + fn set_last_run_inputs(&mut self, inputs: &Vec) { + self.test_data.last_run_inputs.clone_from(inputs); } /// Merge current collected line coverage with the new coverage from last fuzzed call. - pub fn merge_coverage(&self, new_coverage: Option) { - HitMaps::merge_opt(&mut self.execution_data.borrow_mut().line_coverage, new_coverage); + fn merge_line_coverage(&mut self, new_coverage: Option) { + HitMaps::merge_opt(&mut self.test_data.line_coverage, new_coverage); } /// Update metrics for a fuzzed selector, extracted from tx details. /// Always increments number of calls; discarded runs (through assume cheatcodes) are tracked /// separated from reverts. - pub fn record_metrics(&self, tx_details: &BasicTxDetails, reverted: bool, discarded: bool) { + fn record_metrics(&mut self, tx_details: &BasicTxDetails, reverted: bool, discarded: bool) { if let Some(metric_key) = self.targeted_contracts.targets.lock().fuzzed_metric_key(tx_details) { - let test_metrics = &mut self.execution_data.borrow_mut().metrics; + let test_metrics = &mut self.test_data.metrics; let invariant_metrics = test_metrics.entry(metric_key).or_default(); invariant_metrics.calls += 1; if discarded { @@ -231,17 +228,16 @@ impl InvariantTest { /// End invariant test run by collecting results, cleaning collected artifacts and reverting /// created fuzz state. - pub fn end_run(&self, run: InvariantTestRun, gas_samples: usize) { + fn end_run(&mut self, run: InvariantTestRun, gas_samples: usize) { // We clear all the targeted contracts created during this run. self.targeted_contracts.clear_created_contracts(run.created_contracts); - let mut invariant_data = self.execution_data.borrow_mut(); - if invariant_data.gas_report_traces.len() < gas_samples { - invariant_data + if self.test_data.gas_report_traces.len() < gas_samples { + self.test_data .gas_report_traces .push(run.run_traces.into_iter().map(|arena| arena.arena).collect()); } - invariant_data.fuzz_cases.push(FuzzedCases::new(run.fuzz_runs)); + self.test_data.fuzz_cases.push(FuzzedCases::new(run.fuzz_runs)); // Revert state to not persist values between runs. self.fuzz_state.revert(); @@ -249,28 +245,28 @@ impl InvariantTest { } /// Contains data for an invariant test run. -pub struct InvariantTestRun { +struct InvariantTestRun { // Invariant run call sequence. - pub inputs: Vec, + inputs: Vec, // Current invariant run executor. - pub executor: Executor, + executor: Executor, // Invariant run stat reports (eg. gas usage). - pub fuzz_runs: Vec, + fuzz_runs: Vec, // Contracts created during current invariant run. - pub created_contracts: Vec
, + created_contracts: Vec
, // Traces of each call of the invariant run call sequence. - pub run_traces: Vec, + run_traces: Vec, // Current depth of invariant run. - pub depth: u32, + depth: u32, // Current assume rejects of the invariant run. - pub assume_rejects_counter: u32, + rejects: u32, // Whether new coverage was discovered during this run. - pub new_coverage: bool, + new_coverage: bool, } impl InvariantTestRun { /// Instantiates an invariant test run. - pub fn new(first_input: BasicTxDetails, executor: Executor, depth: usize) -> Self { + fn new(first_input: BasicTxDetails, executor: Executor, depth: usize) -> Self { Self { inputs: vec![first_input], executor, @@ -278,7 +274,7 @@ impl InvariantTestRun { created_contracts: vec![], run_traces: vec![], depth: 0, - assume_rejects_counter: 0, + rejects: 0, new_coverage: false, } } @@ -303,10 +299,7 @@ pub struct InvariantExecutor<'a> { project_contracts: &'a ContractsByArtifact, /// Filters contracts to be fuzzed through their artifact identifiers. artifact_filters: ArtifactFilters, - /// History of binned hitcount of edges seen during fuzzing. - history_map: Vec, } -const COVERAGE_MAP_SIZE: usize = 65536; impl<'a> InvariantExecutor<'a> { /// Instantiates a fuzzed executor EVM given a testrunner @@ -324,7 +317,6 @@ impl<'a> InvariantExecutor<'a> { setup_contracts, project_contracts, artifact_filters: ArtifactFilters::default(), - history_map: vec![0u8; COVERAGE_MAP_SIZE], } } @@ -341,7 +333,7 @@ impl<'a> InvariantExecutor<'a> { return Err(eyre!("Invariant test function should have no inputs")); } - let (invariant_test, mut corpus_manager) = + let (mut invariant_test, mut corpus_manager) = self.prepare_test(&invariant_contract, fuzz_fixtures, deployed_libs)?; // Start timer for this invariant test. @@ -349,20 +341,18 @@ impl<'a> InvariantExecutor<'a> { let timer = FuzzTestTimer::new(self.config.timeout); let mut last_metrics_report = Instant::now(); let continue_campaign = |runs: u32| { - // If timeout is configured, then perform invariant runs until expires. - if self.config.timeout.is_some() { - return !timer.is_timed_out(); - } - // If no timeout configured then loop until configured runs. - runs < self.config.runs + if timer.is_enabled() { !timer.is_timed_out() } else { runs < self.config.runs } }; // Invariant runs with edge coverage if corpus dir is set or showing edge coverage. - let edge_coverage_enabled = - self.config.corpus_dir.is_some() || self.config.show_edge_coverage; + let edge_coverage_enabled = self.config.corpus.collect_edge_coverage(); 'stop: while continue_campaign(runs) { - let initial_seq = corpus_manager.new_sequence(&invariant_test)?; + let initial_seq = corpus_manager.new_inputs( + &mut invariant_test.test_data.branch_runner, + &invariant_test.fuzz_state, + &invariant_test.targeted_contracts, + )?; // Create current invariant run data. let mut current_run = InvariantTestRun::new( @@ -410,22 +400,16 @@ impl<'a> InvariantExecutor<'a> { } // Collect line coverage from last fuzzed call. - invariant_test.merge_coverage(call_result.line_coverage.clone()); - // If running with edge coverage then merge edge count with the current history - // map and set new coverage in current run. - if edge_coverage_enabled { - let (new_coverage, is_edge) = - call_result.merge_edge_coverage(&mut self.history_map); - if new_coverage { - current_run.new_coverage = true; - corpus_manager.update_seen_metrics(is_edge); - } + invariant_test.merge_line_coverage(call_result.line_coverage.clone()); + // Collect edge coverage and set the flag in the current run. + if corpus_manager.merge_edge_coverage(&mut call_result) { + current_run.new_coverage = true; } if discarded { current_run.inputs.pop(); - current_run.assume_rejects_counter += 1; - if current_run.assume_rejects_counter > self.config.max_assume_rejects { + current_run.rejects += 1; + if current_run.rejects > self.config.max_assume_rejects { invariant_test.set_error(InvariantFuzzError::MaxAssumeRejects( self.config.max_assume_rejects, )); @@ -475,7 +459,7 @@ impl<'a> InvariantExecutor<'a> { // Determine if test can continue or should exit. let result = can_continue( &invariant_contract, - &invariant_test, + &mut invariant_test, &mut current_run, &self.config, call_result, @@ -495,7 +479,7 @@ impl<'a> InvariantExecutor<'a> { } current_run.inputs.push(corpus_manager.generate_next_input( - &invariant_test, + &mut invariant_test.test_data.branch_runner, &initial_seq, discarded, current_run.depth as usize, @@ -503,13 +487,13 @@ impl<'a> InvariantExecutor<'a> { } // Extend corpus with current run data. - corpus_manager.collect_inputs(¤t_run); + corpus_manager.process_inputs(¤t_run.inputs, current_run.new_coverage); // Call `afterInvariant` only if it is declared and test didn't fail already. if invariant_contract.call_after_invariant && !invariant_test.has_errors() { assert_after_invariant( &invariant_contract, - &invariant_test, + &mut invariant_test, ¤t_run, &self.config, ) @@ -546,7 +530,7 @@ impl<'a> InvariantExecutor<'a> { trace!(?fuzz_fixtures); invariant_test.fuzz_state.log_stats(); - let result = invariant_test.execution_data.into_inner(); + let result = invariant_test.test_data; Ok(InvariantFuzzTestResult { error: result.failures.error, cases: result.fuzz_cases, @@ -567,7 +551,7 @@ impl<'a> InvariantExecutor<'a> { invariant_contract: &InvariantContract<'_>, fuzz_fixtures: &FuzzFixtures, deployed_libs: &[Address], - ) -> Result<(InvariantTest, TxCorpusManager)> { + ) -> Result<(InvariantTest, CorpusManager)> { // Finds out the chosen deployed contracts and/or senders. self.select_contract_artifacts(invariant_contract.address)?; let (targeted_senders, targeted_contracts) = @@ -632,15 +616,14 @@ impl<'a> InvariantExecutor<'a> { return Err(eyre!(error.revert_reason().unwrap_or_default())); } - let corpus_manager = TxCorpusManager::new( - &self.config, + let corpus_manager = CorpusManager::new( + &self.config.corpus, &invariant_contract.invariant_function.name, - &targeted_contracts, strategy.boxed(), &self.executor, - &mut self.history_map, + None, + Some(&targeted_contracts), )?; - let invariant_test = InvariantTest::new( fuzz_state, targeted_contracts, diff --git a/crates/evm/evm/src/executors/invariant/replay.rs b/crates/evm/evm/src/executors/invariant/replay.rs index 5aabfc6e16e19..dcae4334b5362 100644 --- a/crates/evm/evm/src/executors/invariant/replay.rs +++ b/crates/evm/evm/src/executors/invariant/replay.rs @@ -8,10 +8,7 @@ use alloy_primitives::{Log, U256, map::HashMap}; use eyre::Result; use foundry_common::{ContractsByAddress, ContractsByArtifact}; use foundry_evm_coverage::HitMaps; -use foundry_evm_fuzz::{ - BaseCounterExample, - invariant::{BasicTxDetails, InvariantContract}, -}; +use foundry_evm_fuzz::{BaseCounterExample, BasicTxDetails, invariant::InvariantContract}; use foundry_evm_traces::{TraceKind, TraceMode, Traces, load_contracts}; use indicatif::ProgressBar; use parking_lot::RwLock; diff --git a/crates/evm/evm/src/executors/invariant/result.rs b/crates/evm/evm/src/executors/invariant/result.rs index f1cd9a2174fb3..611f9fb6bfbc8 100644 --- a/crates/evm/evm/src/executors/invariant/result.rs +++ b/crates/evm/evm/src/executors/invariant/result.rs @@ -9,8 +9,8 @@ use foundry_config::InvariantConfig; use foundry_evm_core::utils::StateChangeset; use foundry_evm_coverage::HitMaps; use foundry_evm_fuzz::{ - FuzzedCases, - invariant::{BasicTxDetails, FuzzRunIdentifiedContracts, InvariantContract}, + BasicTxDetails, FuzzedCases, + invariant::{FuzzRunIdentifiedContracts, InvariantContract}, }; use revm_inspectors::tracing::CallTraceArena; use std::{borrow::Cow, collections::HashMap}; @@ -97,7 +97,7 @@ pub(crate) fn assert_invariants( /// function (if it can continue). pub(crate) fn can_continue( invariant_contract: &InvariantContract<'_>, - invariant_test: &InvariantTest, + invariant_test: &mut InvariantTest, invariant_run: &mut InvariantTestRun, invariant_config: &InvariantConfig, call_result: RawCallResult, @@ -128,14 +128,14 @@ pub(crate) fn can_continue( &invariant_test.targeted_contracts, &invariant_run.executor, &invariant_run.inputs, - &mut invariant_test.execution_data.borrow_mut().failures, + &mut invariant_test.test_data.failures, )?; if call_results.is_none() { return Ok(RichInvariantResults::new(false, None)); } } else { // Increase the amount of reverts. - let mut invariant_data = invariant_test.execution_data.borrow_mut(); + let invariant_data = &mut invariant_test.test_data; invariant_data.failures.reverts += 1; // If fail on revert is set, we must return immediately. if invariant_config.fail_on_revert { @@ -164,7 +164,7 @@ pub(crate) fn can_continue( /// If call fails then the invariant test is considered failed. pub(crate) fn assert_after_invariant( invariant_contract: &InvariantContract<'_>, - invariant_test: &InvariantTest, + invariant_test: &mut InvariantTest, invariant_run: &InvariantTestRun, invariant_config: &InvariantConfig, ) -> Result { diff --git a/crates/evm/evm/src/executors/invariant/shrink.rs b/crates/evm/evm/src/executors/invariant/shrink.rs index aa193d0baa3d9..5ae0c70e37a4b 100644 --- a/crates/evm/evm/src/executors/invariant/shrink.rs +++ b/crates/evm/evm/src/executors/invariant/shrink.rs @@ -6,7 +6,7 @@ use crate::executors::{ }; use alloy_primitives::{Address, Bytes, U256}; use foundry_evm_core::constants::MAGIC_ASSUME; -use foundry_evm_fuzz::invariant::BasicTxDetails; +use foundry_evm_fuzz::BasicTxDetails; use indicatif::ProgressBar; use proptest::bits::{BitSetLike, VarBitSet}; use std::cmp::min; diff --git a/crates/evm/evm/src/executors/mod.rs b/crates/evm/evm/src/executors/mod.rs index 13c918f2b8ab1..6e794855f32fa 100644 --- a/crates/evm/evm/src/executors/mod.rs +++ b/crates/evm/evm/src/executors/mod.rs @@ -56,9 +56,13 @@ pub use fuzz::FuzzedExecutor; pub mod invariant; pub use invariant::InvariantExecutor; +mod corpus; mod trace; + pub use trace::TracingExecutor; +const DURATION_BETWEEN_METRICS_REPORT: Duration = Duration::from_secs(5); + sol! { interface ITest { function setUp() external; @@ -1095,6 +1099,11 @@ impl FuzzTestTimer { Self { inner: timeout.map(|timeout| (Instant::now(), Duration::from_secs(timeout.into()))) } } + /// Whether the fuzz test timer is enabled. + pub fn is_enabled(&self) -> bool { + self.inner.is_some() + } + /// Whether the current fuzz test timed out and should be stopped. pub fn is_timed_out(&self) -> bool { self.inner.is_some_and(|(start, duration)| start.elapsed() > duration) diff --git a/crates/evm/fuzz/src/invariant/call_override.rs b/crates/evm/fuzz/src/invariant/call_override.rs index dddf591526f08..8cfe6a0e6e459 100644 --- a/crates/evm/fuzz/src/invariant/call_override.rs +++ b/crates/evm/fuzz/src/invariant/call_override.rs @@ -1,4 +1,4 @@ -use super::{BasicTxDetails, CallDetails}; +use crate::{BasicTxDetails, CallDetails}; use alloy_primitives::Address; use parking_lot::{Mutex, RwLock}; use proptest::{ diff --git a/crates/evm/fuzz/src/invariant/mod.rs b/crates/evm/fuzz/src/invariant/mod.rs index 82d5031932556..a773e8f04bb27 100644 --- a/crates/evm/fuzz/src/invariant/mod.rs +++ b/crates/evm/fuzz/src/invariant/mod.rs @@ -1,15 +1,15 @@ use alloy_json_abi::{Function, JsonAbi}; -use alloy_primitives::{Address, Bytes, Selector, map::HashMap}; +use alloy_primitives::{Address, Selector, map::HashMap}; use foundry_compilers::artifacts::StorageLayout; use itertools::Either; use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, sync::Arc}; mod call_override; pub use call_override::RandomCallGenerator; mod filters; +use crate::BasicTxDetails; pub use filters::{ArtifactFilters, SenderFilters}; use foundry_common::{ContractsByAddress, ContractsByArtifact}; use foundry_evm_core::utils::{StateChangeset, get_function}; @@ -252,24 +252,6 @@ impl TargetedContract { } } -/// Details of a transaction generated by invariant strategy for fuzzing a target. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct BasicTxDetails { - // Transaction sender address. - pub sender: Address, - // Transaction call details. - pub call_details: CallDetails, -} - -/// Call details of a transaction generated to fuzz invariant target. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CallDetails { - // Address of target contract. - pub target: Address, - // The data of the transaction. - pub calldata: Bytes, -} - /// Test contract which is testing its invariants. #[derive(Clone, Debug)] pub struct InvariantContract<'a> { diff --git a/crates/evm/fuzz/src/lib.rs b/crates/evm/fuzz/src/lib.rs index a97181dfffc37..7ac9a2b640e2a 100644 --- a/crates/evm/fuzz/src/lib.rs +++ b/crates/evm/fuzz/src/lib.rs @@ -31,6 +31,24 @@ pub mod strategies; mod inspector; pub use inspector::Fuzzer; +/// Details of a transaction generated by fuzz strategy for fuzzing a target. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BasicTxDetails { + // Transaction sender address. + pub sender: Address, + // Transaction call details. + pub call_details: CallDetails, +} + +/// Call details of a transaction generated to fuzz. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CallDetails { + // Address of target contract. + pub target: Address, + // The data of the transaction. + pub calldata: Bytes, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[expect(clippy::large_enum_variant)] pub enum CounterExample { @@ -226,6 +244,9 @@ pub struct FuzzTestResult { // Deprecated cheatcodes mapped to their replacements. pub deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>, + + /// NUmber of failed replays from persisted corpus. + pub failed_corpus_replays: usize, } impl FuzzTestResult { diff --git a/crates/evm/fuzz/src/strategies/invariants.rs b/crates/evm/fuzz/src/strategies/invariants.rs index f739b4f86dac1..340efde3b84a8 100644 --- a/crates/evm/fuzz/src/strategies/invariants.rs +++ b/crates/evm/fuzz/src/strategies/invariants.rs @@ -1,7 +1,7 @@ use super::{fuzz_calldata, fuzz_param_from_state}; use crate::{ - FuzzFixtures, - invariant::{BasicTxDetails, CallDetails, FuzzRunIdentifiedContracts, SenderFilters}, + BasicTxDetails, CallDetails, FuzzFixtures, + invariant::{FuzzRunIdentifiedContracts, SenderFilters}, strategies::{EvmFuzzState, fuzz_calldata_from_state, fuzz_param}, }; use alloy_json_abi::Function; diff --git a/crates/evm/fuzz/src/strategies/mod.rs b/crates/evm/fuzz/src/strategies/mod.rs index 1d8e647a52f3f..e49534e3dccd9 100644 --- a/crates/evm/fuzz/src/strategies/mod.rs +++ b/crates/evm/fuzz/src/strategies/mod.rs @@ -5,7 +5,7 @@ mod uint; pub use uint::UintStrategy; mod param; -pub use param::{fuzz_param, fuzz_param_from_state, fuzz_param_with_fixtures}; +pub use param::{fuzz_param, fuzz_param_from_state, fuzz_param_with_fixtures, mutate_param_value}; mod calldata; pub use calldata::{fuzz_calldata, fuzz_calldata_from_state}; @@ -15,3 +15,5 @@ pub use state::EvmFuzzState; mod invariants; pub use invariants::{fuzz_contract_with_calldata, invariant_strat, override_call_strat}; + +mod mutators; diff --git a/crates/evm/fuzz/src/strategies/mutators.rs b/crates/evm/fuzz/src/strategies/mutators.rs new file mode 100644 index 0000000000000..b9ab525acfc62 --- /dev/null +++ b/crates/evm/fuzz/src/strategies/mutators.rs @@ -0,0 +1,547 @@ +use alloy_dyn_abi::Word; +use alloy_primitives::{Address, I256, Sign, U256}; +use proptest::{prelude::*, test_runner::TestRunner}; +use rand::seq::IndexedRandom; +use std::fmt::Debug; + +// Interesting 8-bit values to inject. +static INTERESTING_8: [i8; 9] = [-128, -1, 0, 1, 16, 32, 64, 100, 127]; + +/// Interesting 16-bit values to inject. +static INTERESTING_16: [i16; 19] = [ + -128, -1, 0, 1, 16, 32, 64, 100, 127, -32768, -129, 128, 255, 256, 512, 1000, 1024, 4096, 32767, +]; + +/// Interesting 32-bit values to inject. +static INTERESTING_32: [i32; 27] = [ + -128, + -1, + 0, + 1, + 16, + 32, + 64, + 100, + 127, + -32768, + -129, + 128, + 255, + 256, + 512, + 1000, + 1024, + 4096, + 32767, + -2147483648, + -100663046, + -32769, + 32768, + 65535, + 65536, + 100663045, + 2147483647, +]; + +/// Mutator that randomly increments or decrements an uint or int. +pub(crate) trait IncrementDecrementMutator: Sized + Copy + Debug { + fn validate(old: Self, new: Self, size: usize) -> Option; + + #[instrument(name = "mutator::increment_decrement", skip(size, test_runner), ret)] + fn increment_decrement(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mutated = if test_runner.rng().random::() { + self.wrapping_add(Self::ONE) + } else { + self.wrapping_sub(Self::ONE) + }; + Self::validate(self, mutated, size) + } + + fn wrapping_add(self, rhs: Self) -> Self; + fn wrapping_sub(self, rhs: Self) -> Self; + const ONE: Self; +} + +macro_rules! impl_increment_decrement_mutator { + ($ty:ty, $validate_fn:path) => { + impl IncrementDecrementMutator for $ty { + fn validate(old: Self, new: Self, size: usize) -> Option { + $validate_fn(old, new, size) + } + + fn wrapping_add(self, rhs: Self) -> Self { + Self::wrapping_add(self, rhs) + } + + fn wrapping_sub(self, rhs: Self) -> Self { + Self::wrapping_sub(self, rhs) + } + + const ONE: Self = Self::ONE; + } + }; +} + +impl_increment_decrement_mutator!(U256, validate_uint_mutation); +impl_increment_decrement_mutator!(I256, validate_int_mutation); + +/// ABI mutator that changes current value by flipping a random bit and randomly injecting +/// interesting words - see . +/// Implemented for uint, int, address and fixed bytes. +pub(crate) trait AbiMutator: Sized + Copy + Debug { + fn flip_random_bit(self, size: usize, test_runner: &mut TestRunner) -> Option; + fn mutate_interesting_byte(self, size: usize, test_runner: &mut TestRunner) -> Option; + fn mutate_interesting_word(self, size: usize, test_runner: &mut TestRunner) -> Option; + fn mutate_interesting_dword(self, size: usize, test_runner: &mut TestRunner) -> Option; +} + +impl AbiMutator for U256 { + #[instrument(name = "U256::flip_random_bit", skip(size, test_runner), ret)] + fn flip_random_bit(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + flip_random_bit_in_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_uint_mutation(self, Self::from_be_bytes(bytes), size) + } + + #[instrument(name = "U256::mutate_interesting_byte", skip(size, test_runner), ret)] + fn mutate_interesting_byte(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + mutate_interesting_byte_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_uint_mutation(self, Self::from_be_bytes(bytes), size) + } + + #[instrument(name = "U256::mutate_interesting_word", skip(size, test_runner), ret)] + fn mutate_interesting_word(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + mutate_interesting_word_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_uint_mutation(self, Self::from_be_bytes(bytes), size) + } + + #[instrument(name = "U256::mutate_interesting_dword", skip(size, test_runner), ret)] + fn mutate_interesting_dword(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + mutate_interesting_dword_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_uint_mutation(self, Self::from_be_bytes(bytes), size) + } +} + +impl AbiMutator for I256 { + #[instrument(name = "I256::flip_random_bit", skip(size, test_runner), ret)] + fn flip_random_bit(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + flip_random_bit_in_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_int_mutation(self, Self::from_be_bytes(bytes), size) + } + + #[instrument(name = "I256::mutate_interesting_byte", skip(size, test_runner), ret)] + fn mutate_interesting_byte(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + mutate_interesting_byte_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_int_mutation(self, Self::from_be_bytes(bytes), size) + } + + #[instrument(name = "I256::mutate_interesting_word", skip(size, test_runner), ret)] + fn mutate_interesting_word(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + mutate_interesting_word_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_int_mutation(self, Self::from_be_bytes(bytes), size) + } + + #[instrument(name = "I256::mutate_interesting_dword", skip(size, test_runner), ret)] + fn mutate_interesting_dword(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes: [u8; 32] = self.to_be_bytes(); + mutate_interesting_dword_slice(&mut bytes[32 - size / 8..], test_runner)?; + validate_int_mutation(self, Self::from_be_bytes(bytes), size) + } +} + +impl AbiMutator for Address { + #[instrument(name = "Address::flip_random_bit", skip(_size, test_runner), ret)] + fn flip_random_bit(mut self, _size: usize, test_runner: &mut TestRunner) -> Option { + flip_random_bit_in_slice(self.as_mut_slice(), test_runner)?; + Some(self) + } + + #[instrument(name = "Address::mutate_interesting_byte", skip(_size, test_runner), ret)] + fn mutate_interesting_byte( + mut self, + _size: usize, + test_runner: &mut TestRunner, + ) -> Option { + mutate_interesting_byte_slice(self.as_mut_slice(), test_runner)?; + Some(self) + } + + #[instrument(name = "Address::mutate_interesting_word", skip(_size, test_runner), ret)] + fn mutate_interesting_word( + mut self, + _size: usize, + test_runner: &mut TestRunner, + ) -> Option { + mutate_interesting_word_slice(self.as_mut_slice(), test_runner)?; + Some(self) + } + + #[instrument(name = "Address::mutate_interesting_dword", skip(_size, test_runner), ret)] + fn mutate_interesting_dword( + mut self, + _size: usize, + test_runner: &mut TestRunner, + ) -> Option { + mutate_interesting_dword_slice(self.as_mut_slice(), test_runner)?; + Some(self) + } +} + +impl AbiMutator for Word { + #[instrument(name = "Word::flip_random_bit", skip(size, test_runner), ret)] + fn flip_random_bit(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes = self; + let slice = &mut bytes[..size]; + flip_random_bit_in_slice(slice, test_runner)?; + Some(bytes) + } + + #[instrument(name = "Word::mutate_interesting_byte", skip(size, test_runner), ret)] + fn mutate_interesting_byte(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes = self; + let slice = &mut bytes[..size]; + mutate_interesting_byte_slice(slice, test_runner)?; + Some(bytes) + } + + #[instrument(name = "Word::mutate_interesting_word", skip(size, test_runner), ret)] + fn mutate_interesting_word(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes = self; + let slice = &mut bytes[..size]; + mutate_interesting_word_slice(slice, test_runner)?; + Some(bytes) + } + + #[instrument(name = "Word::mutate_interesting_dword", skip(size, test_runner), ret)] + fn mutate_interesting_dword(self, size: usize, test_runner: &mut TestRunner) -> Option { + let mut bytes = self; + let slice = &mut bytes[..size]; + mutate_interesting_dword_slice(slice, test_runner)?; + Some(bytes) + } +} + +/// Flips a random bit in the given mutable byte slice. +fn flip_random_bit_in_slice(bytes: &mut [u8], test_runner: &mut TestRunner) -> Option<()> { + if bytes.is_empty() { + return None; + } + let bit_index = test_runner.rng().random_range(0..(bytes.len() * 8)); + bytes[bit_index / 8] ^= 1 << (bit_index % 8); + Some(()) +} + +/// Mutates a random byte in the given byte slice by replacing it with a randomly chosen +/// interesting 8-bit value. +fn mutate_interesting_byte_slice(bytes: &mut [u8], test_runner: &mut TestRunner) -> Option<()> { + let index = test_runner.rng().random_range(0..bytes.len()); + let val = *INTERESTING_8.choose(&mut test_runner.rng())? as u8; + bytes[index] = val; + Some(()) +} + +/// Mutates a random 2-byte (16-bit) region in the byte slice with a randomly chosen interesting +/// 16-bit value. +fn mutate_interesting_word_slice(bytes: &mut [u8], test_runner: &mut TestRunner) -> Option<()> { + if bytes.len() < 2 { + return None; + } + let index = test_runner.rng().random_range(0..=bytes.len() - 2); + let val = *INTERESTING_16.choose(&mut test_runner.rng())? as u16; + bytes[index..index + 2].copy_from_slice(&val.to_be_bytes()); + Some(()) +} + +/// Mutates a random 4-byte (32-bit) region in the byte slice with a randomly chosen interesting +/// 32-bit value. +fn mutate_interesting_dword_slice(bytes: &mut [u8], test_runner: &mut TestRunner) -> Option<()> { + if bytes.len() < 4 { + return None; + } + let index = test_runner.rng().random_range(0..=bytes.len() - 4); + let val = *INTERESTING_32.choose(&mut test_runner.rng())? as u32; + bytes[index..index + 4].copy_from_slice(&val.to_be_bytes()); + Some(()) +} + +/// Returns mutated uint value if different from the original value and if it fits in the given +/// size, otherwise None. +fn validate_uint_mutation(original: U256, mutated: U256, size: usize) -> Option { + // Early return if mutated value is the same as original value. + if mutated == original { + return None; + } + + // Check if mutated value fits the given size. + let max = if size < 256 { (U256::from(1) << size) - U256::from(1) } else { U256::MAX }; + (mutated < max).then_some(mutated) +} + +/// Returns mutated int value if different from the original value and if it fits in the given size, +/// otherwise None. +fn validate_int_mutation(original: I256, mutated: I256, size: usize) -> Option { + // Early return if mutated value is the same as original value. + if mutated == original { + return None; + } + + // Check if mutated value fits the given size. + let max_abs = (U256::from(1) << (size - 1)) - U256::from(1); + match mutated.sign() { + Sign::Positive => mutated < I256::overflowing_from_sign_and_abs(Sign::Positive, max_abs).0, + Sign::Negative => mutated > I256::overflowing_from_sign_and_abs(Sign::Negative, max_abs).0, + } + .then_some(mutated) +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::test_runner::Config; + + #[test] + fn test_increment_decrement_u256() { + let mut runner = TestRunner::new(Config::default()); + + let mut increment_decrement = |value: U256, expected: Vec| { + for _ in 0..100 { + let mutated = value.increment_decrement(8, &mut runner); + assert!( + mutated.is_none() || mutated.is_some_and(|mutated| expected.contains(&mutated)) + ); + } + }; + + increment_decrement(U256::ZERO, vec![U256::ONE]); + increment_decrement(U256::from(255), vec![U256::from(254)]); + increment_decrement(U256::from(64), vec![U256::from(63), U256::from(65)]); + } + + #[test] + fn test_increment_decrement_i256() { + let mut runner = TestRunner::new(Config::default()); + + let mut increment_decrement = |value: I256, expected: Vec| { + for _ in 0..100 { + let mutated = value.increment_decrement(8, &mut runner); + assert!( + mutated.is_none() || mutated.is_some_and(|mutated| expected.contains(&mutated)) + ); + } + }; + + increment_decrement( + I256::from_dec_str("-128").unwrap(), + vec![I256::from_dec_str("-127").unwrap()], + ); + increment_decrement( + I256::from_dec_str("127").unwrap(), + vec![I256::from_dec_str("126").unwrap()], + ); + increment_decrement( + I256::from_dec_str("-47").unwrap(), + vec![I256::from_dec_str("-48").unwrap(), I256::from_dec_str("-46").unwrap()], + ); + increment_decrement( + I256::from_dec_str("47").unwrap(), + vec![I256::from_dec_str("48").unwrap(), I256::from_dec_str("46").unwrap()], + ); + } + + #[test] + fn test_bit_flip_u256() { + let mut runner = TestRunner::new(Config::default()); + let size = 8; + + let mut test_bit_flip = |value: U256| { + for _ in 0..100 { + let flipped = U256::flip_random_bit(value, size, &mut runner); + assert!( + flipped.is_none() + || flipped.is_some_and( + |flipped| flipped != value && flipped < (U256::from(1) << size) + ) + ); + } + }; + + test_bit_flip(U256::ZERO); + test_bit_flip(U256::ONE); + test_bit_flip(U256::MAX); + test_bit_flip(U256::from(255)); + } + + #[test] + fn test_bit_flip_i256() { + let mut runner = TestRunner::new(Config::default()); + let size = 8; + + let mut test_bit_flip = |value: I256| { + for _ in 0..100 { + let flipped = I256::flip_random_bit(value, size, &mut runner); + assert!( + flipped.is_none() + || flipped.is_some_and(|flipped| { + flipped != value + && flipped.abs().unsigned_abs() < (U256::from(1) << (size - 1)) + }) + ); + } + }; + + test_bit_flip(I256::from_dec_str("-128").unwrap()); + test_bit_flip(I256::from_dec_str("127").unwrap()); + test_bit_flip(I256::MAX); + test_bit_flip(I256::MIN); + test_bit_flip(I256::MINUS_ONE); + } + + #[test] + fn test_mutate_interesting_byte_u256() { + let mut runner = TestRunner::new(Config::default()); + let value = U256::from(0); + let size = 8; + + for _ in 0..100 { + let mutated = U256::mutate_interesting_byte(value, size, &mut runner); + assert!( + mutated.is_none() + || mutated.is_some_and( + |mutated| mutated != value && mutated < (U256::from(1) << size) + ) + ); + } + } + + #[test] + fn test_mutate_interesting_word_u256() { + let mut runner = TestRunner::new(Config::default()); + let value = U256::from(0); + let size = 16; + + for _ in 0..100 { + let mutated = U256::mutate_interesting_word(value, size, &mut runner); + assert!( + mutated.is_none() + || mutated.is_some_and( + |mutated| mutated != value && mutated < (U256::from(1) << size) + ) + ); + } + } + + #[test] + fn test_mutate_interesting_dword_u256() { + let mut runner = TestRunner::new(Config::default()); + let value = U256::from(0); + let size = 32; + + for _ in 0..100 { + let mutated = U256::mutate_interesting_dword(value, size, &mut runner); + assert!( + mutated.is_none() + || mutated.is_some_and( + |mutated| mutated != value && mutated < (U256::from(1) << size) + ) + ); + } + } + + #[test] + fn test_mutate_interesting_byte_i256() { + let mut runner = TestRunner::new(Config::default()); + let value = I256::ZERO; + let size = 8; + + for _ in 0..100 { + let mutated = I256::mutate_interesting_byte(value, size, &mut runner); + assert!( + mutated.is_none() + || mutated.is_some_and(|mutated| mutated != value + && mutated.abs().unsigned_abs() < (U256::from(1) << (size - 1))) + ) + } + } + + #[test] + fn test_mutate_interesting_word_i256() { + let mut runner = TestRunner::new(Config::default()); + let value = I256::ZERO; + let size = 16; + + for _ in 0..100 { + let mutated = I256::mutate_interesting_word(value, size, &mut runner); + assert!( + mutated.is_none() + || mutated.is_some_and(|mutated| mutated != value + && mutated.abs().unsigned_abs() < (U256::from(1) << (size - 1))) + ) + } + } + + #[test] + fn test_mutate_interesting_dword_i256() { + let mut runner = TestRunner::new(Config::default()); + let value = I256::ZERO; + let size = 32; + + for _ in 0..100 { + let mutated = I256::mutate_interesting_dword(value, size, &mut runner); + assert!( + mutated.is_none() + || mutated.is_some_and(|mutated| mutated != value + && mutated.abs().unsigned_abs() < (U256::from(1) << (size - 1))) + ) + } + } + + #[test] + fn test_mutate_address() { + let mut runner = TestRunner::new(Config::default()); + for _ in 0..100 { + let value = Address::random(); + assert_ne!(value, Address::flip_random_bit(value, 20, &mut runner).unwrap()); + let value1 = Address::random(); + assert_ne!(value1, Address::mutate_interesting_byte(value1, 20, &mut runner).unwrap()); + let value2 = Address::random(); + assert_ne!(value2, Address::mutate_interesting_word(value2, 20, &mut runner).unwrap()); + let value3 = Address::random(); + assert_ne!(value3, Address::mutate_interesting_dword(value3, 20, &mut runner).unwrap()); + } + } + + #[test] + fn test_mutate_word() { + let mut runner = TestRunner::new(Config::default()); + for _ in 0..100 { + let value = Word::random(); + assert_ne!(value, Word::flip_random_bit(value, 32, &mut runner).unwrap()); + let value1 = Word::random(); + assert_ne!(value1, Word::mutate_interesting_byte(value1, 32, &mut runner).unwrap()); + let value2 = Word::random(); + assert_ne!(value2, Word::mutate_interesting_word(value2, 32, &mut runner).unwrap()); + let value3 = Word::random(); + assert_ne!(value3, Word::mutate_interesting_dword(value3, 32, &mut runner).unwrap()); + } + } + + #[test] + fn test_mutate_interesting_word_too_small_returns_none() { + let mut runner = TestRunner::new(Config::default()); + let value = U256::from(123); + assert!(U256::mutate_interesting_word(value, 8, &mut runner).is_none()); + } + + #[test] + fn test_mutate_interesting_dword_too_small_returns_none() { + let mut runner = TestRunner::new(Config::default()); + let value = I256::from_dec_str("123").unwrap(); + assert!(I256::mutate_interesting_dword(value, 16, &mut runner).is_none()); + } +} diff --git a/crates/evm/fuzz/src/strategies/param.rs b/crates/evm/fuzz/src/strategies/param.rs index 37bc521a0637c..d2efda8c254df 100644 --- a/crates/evm/fuzz/src/strategies/param.rs +++ b/crates/evm/fuzz/src/strategies/param.rs @@ -1,8 +1,10 @@ use super::state::EvmFuzzState; -use alloy_dyn_abi::{DynSolType, DynSolValue}; +use crate::strategies::mutators::{AbiMutator, IncrementDecrementMutator}; +use alloy_dyn_abi::{DynSolType, DynSolValue, Word}; use alloy_primitives::{Address, B256, I256, U256}; -use proptest::prelude::*; -use rand::{SeedableRng, rngs::StdRng}; +use proptest::{prelude::*, test_runner::TestRunner}; +use rand::{SeedableRng, prelude::IndexedMutRandom, rngs::StdRng}; +use std::mem::replace; /// The max length of arrays we fuzz for is 256. const MAX_ARRAY_LEN: usize = 256; @@ -224,6 +226,149 @@ pub fn fuzz_param_from_state( } } +/// Mutates the current value of the given parameter type and value. +pub fn mutate_param_value( + param: &DynSolType, + value: DynSolValue, + test_runner: &mut TestRunner, + state: &EvmFuzzState, +) -> DynSolValue { + let new_value = |param: &DynSolType, test_runner: &mut TestRunner| { + fuzz_param_from_state(param, state) + .new_tree(test_runner) + .expect("Could not generate case") + .current() + }; + + match value { + DynSolValue::Bool(val) => { + // flip boolean value + trace!(target: "mutator", "Bool flip {val}"); + Some(DynSolValue::Bool(!val)) + } + DynSolValue::Uint(val, size) => match test_runner.rng().random_range(0..=5) { + 0 => U256::increment_decrement(val, size, test_runner), + 1 => U256::flip_random_bit(val, size, test_runner), + 2 => U256::mutate_interesting_byte(val, size, test_runner), + 3 => U256::mutate_interesting_word(val, size, test_runner), + 4 => U256::mutate_interesting_dword(val, size, test_runner), + 5 => None, + _ => unreachable!(), + } + .map(|v| DynSolValue::Uint(v, size)), + DynSolValue::Int(val, size) => match test_runner.rng().random_range(0..=5) { + 0 => I256::increment_decrement(val, size, test_runner), + 1 => I256::flip_random_bit(val, size, test_runner), + 2 => I256::mutate_interesting_byte(val, size, test_runner), + 3 => I256::mutate_interesting_word(val, size, test_runner), + 4 => I256::mutate_interesting_dword(val, size, test_runner), + 5 => None, + _ => unreachable!(), + } + .map(|v| DynSolValue::Int(v, size)), + DynSolValue::Address(val) => match test_runner.rng().random_range(0..=4) { + 0 => Address::flip_random_bit(val, 20, test_runner), + 1 => Address::mutate_interesting_byte(val, 20, test_runner), + 2 => Address::mutate_interesting_word(val, 20, test_runner), + 3 => Address::mutate_interesting_dword(val, 20, test_runner), + 4 => None, + _ => unreachable!(), + } + .map(DynSolValue::Address), + DynSolValue::Array(mut values) => { + if let DynSolType::Array(param_type) = param + && !values.is_empty() + { + match test_runner.rng().random_range(0..=2) { + // Decrease array size by removing a random element. + 0 => { + values.remove(test_runner.rng().random_range(0..values.len())); + } + // Increase array size. + 1 => values.push(new_value(param_type, test_runner)), + // Mutate random array element. + 2 => mutate_random_array_value(&mut values, param_type, test_runner, state), + _ => unreachable!(), + } + Some(DynSolValue::Array(values)) + } else { + None + } + } + DynSolValue::FixedArray(mut values) => { + if let DynSolType::FixedArray(param_type, _size) = param + && !values.is_empty() + { + mutate_random_array_value(&mut values, param_type, test_runner, state); + Some(DynSolValue::FixedArray(values)) + } else { + None + } + } + DynSolValue::FixedBytes(word, size) => match test_runner.rng().random_range(0..=4) { + 0 => Word::flip_random_bit(word, size, test_runner), + 1 => Word::mutate_interesting_byte(word, size, test_runner), + 2 => Word::mutate_interesting_word(word, size, test_runner), + 3 => Word::mutate_interesting_dword(word, size, test_runner), + 4 => None, + _ => unreachable!(), + } + .map(|word| DynSolValue::FixedBytes(word, size)), + DynSolValue::CustomStruct { name, prop_names, tuple: mut values } => { + if let DynSolType::CustomStruct { name: _, prop_names: _, tuple: tuple_types } + | DynSolType::Tuple(tuple_types) = param + && !values.is_empty() + { + // Mutate random struct element. + mutate_random_tuple_value(&mut values, tuple_types, test_runner, state); + Some(DynSolValue::CustomStruct { name, prop_names, tuple: values }) + } else { + None + } + } + DynSolValue::Tuple(mut values) => { + if let DynSolType::Tuple(tuple_types) = param + && !values.is_empty() + { + // Mutate random tuple element. + mutate_random_tuple_value(&mut values, tuple_types, test_runner, state); + Some(DynSolValue::Tuple(values)) + } else { + None + } + } + _ => None, + } + .unwrap_or_else(|| new_value(param, test_runner)) +} + +/// Mutates random value from given tuples. +fn mutate_random_tuple_value( + tuple_values: &mut [DynSolValue], + tuple_types: &[DynSolType], + test_runner: &mut TestRunner, + state: &EvmFuzzState, +) { + let id = test_runner.rng().random_range(0..tuple_values.len()); + let param_type = &tuple_types[id]; + let old_val = replace(&mut tuple_values[id], DynSolValue::Bool(false)); + let new_val = mutate_param_value(param_type, old_val, test_runner, state); + tuple_values[id] = new_val; +} + +/// Mutates random value from given array. +fn mutate_random_array_value( + array_values: &mut [DynSolValue], + element_type: &DynSolType, + test_runner: &mut TestRunner, + state: &EvmFuzzState, +) { + let elem = array_values.choose_mut(&mut test_runner.rng()).unwrap(); + let old_val = replace(elem, DynSolValue::Bool(false)); + let new_val = mutate_param_value(element_type, old_val, test_runner, state); + *elem = new_val; +} + #[cfg(test)] mod tests { use crate::{ diff --git a/crates/evm/fuzz/src/strategies/state.rs b/crates/evm/fuzz/src/strategies/state.rs index 6a61775bc1bec..4371efc8bd064 100644 --- a/crates/evm/fuzz/src/strategies/state.rs +++ b/crates/evm/fuzz/src/strategies/state.rs @@ -1,4 +1,4 @@ -use crate::invariant::{BasicTxDetails, FuzzRunIdentifiedContracts}; +use crate::{BasicTxDetails, invariant::FuzzRunIdentifiedContracts}; use alloy_dyn_abi::{DynSolType, DynSolValue, EventExt, FunctionExt}; use alloy_json_abi::{Function, JsonAbi}; use alloy_primitives::{ diff --git a/crates/forge/src/cmd/snapshot.rs b/crates/forge/src/cmd/snapshot.rs index 43cf4a158dea7..4309a833f59ee 100644 --- a/crates/forge/src/cmd/snapshot.rs +++ b/crates/forge/src/cmd/snapshot.rs @@ -225,6 +225,7 @@ impl FromStr for GasSnapshotEntry { runs: runs.as_str().parse().unwrap(), median_gas: med.as_str().parse().unwrap(), mean_gas: avg.as_str().parse().unwrap(), + failed_corpus_replays: 0, }, }) } else { @@ -471,7 +472,12 @@ mod tests { GasSnapshotEntry { contract_name: "Test".to_string(), signature: "deposit()".to_string(), - gas_used: TestKindReport::Fuzz { runs: 256, median_gas: 200, mean_gas: 100 } + gas_used: TestKindReport::Fuzz { + runs: 256, + median_gas: 200, + mean_gas: 100, + failed_corpus_replays: 0 + } } ); } diff --git a/crates/forge/src/result.rs b/crates/forge/src/result.rs index 53aa3b22b6003..694d26782fd4e 100644 --- a/crates/forge/src/result.rs +++ b/crates/forge/src/result.rs @@ -572,6 +572,7 @@ impl TestResult { mean_gas: result.mean_gas(false), first_case: result.first_case, runs: result.gas_by_case.len(), + failed_corpus_replays: result.failed_corpus_replays, }; // Record logs, labels, traces and merge coverages. @@ -592,6 +593,19 @@ impl TestResult { self.deprecated_cheatcodes = result.deprecated_cheatcodes; } + /// Returns the fail result for fuzz test setup. + pub fn fuzz_setup_fail(&mut self, e: Report) { + self.kind = TestKind::Fuzz { + first_case: Default::default(), + runs: 0, + mean_gas: 0, + median_gas: 0, + failed_corpus_replays: 0, + }; + self.status = TestStatus::Failure; + self.reason = Some(format!("failed to set up fuzz testing environment: {e}")); + } + /// Returns the skipped result for invariant test. pub fn invariant_skip(&mut self, reason: SkipReason) { self.kind = TestKind::Invariant { @@ -701,6 +715,7 @@ pub enum TestKindReport { runs: usize, mean_gas: u64, median_gas: u64, + failed_corpus_replays: usize, }, Invariant { runs: usize, @@ -717,8 +732,15 @@ impl fmt::Display for TestKindReport { Self::Unit { gas } => { write!(f, "(gas: {gas})") } - Self::Fuzz { runs, mean_gas, median_gas } => { - write!(f, "(runs: {runs}, μ: {mean_gas}, ~: {median_gas})") + Self::Fuzz { runs, mean_gas, median_gas, failed_corpus_replays } => { + if *failed_corpus_replays != 0 { + write!( + f, + "(runs: {runs}, μ: {mean_gas}, ~: {median_gas}, failed corpus replays: {failed_corpus_replays})" + ) + } else { + write!(f, "(runs: {runs}, μ: {mean_gas}, ~: {median_gas})") + } } Self::Invariant { runs, calls, reverts, metrics: _, failed_corpus_replays } => { if *failed_corpus_replays != 0 { @@ -759,6 +781,7 @@ pub enum TestKind { runs: usize, mean_gas: u64, median_gas: u64, + failed_corpus_replays: usize, }, /// An invariant test. Invariant { @@ -781,8 +804,13 @@ impl TestKind { pub fn report(&self) -> TestKindReport { match self { Self::Unit { gas } => TestKindReport::Unit { gas: *gas }, - Self::Fuzz { first_case: _, runs, mean_gas, median_gas } => { - TestKindReport::Fuzz { runs: *runs, mean_gas: *mean_gas, median_gas: *median_gas } + Self::Fuzz { first_case: _, runs, mean_gas, median_gas, failed_corpus_replays } => { + TestKindReport::Fuzz { + runs: *runs, + mean_gas: *mean_gas, + median_gas: *median_gas, + failed_corpus_replays: *failed_corpus_replays, + } } Self::Invariant { runs, calls, reverts, metrics: _, failed_corpus_replays } => { TestKindReport::Invariant { diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index 4129f8938d793..f584545a08748 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -2,7 +2,7 @@ use crate::{ MultiContractRunner, TestFilter, - fuzz::{BaseCounterExample, invariant::BasicTxDetails}, + fuzz::BaseCounterExample, multi_runner::{TestContract, TestRunnerConfig, is_matching_test}, progress::{TestsProgress, start_fuzz_progress}, result::{SuiteResult, TestResult, TestSetup}, @@ -13,7 +13,7 @@ use alloy_primitives::{Address, Bytes, U256, address, map::HashMap}; use eyre::Result; use foundry_common::{TestFunctionExt, TestFunctionKind, contracts::ContractsByAddress}; use foundry_compilers::utils::canonicalized; -use foundry_config::{Config, InvariantConfig}; +use foundry_config::Config; use foundry_evm::{ constants::CALLER, decode::RevertDecoder, @@ -25,15 +25,13 @@ use foundry_evm::{ }, }, fuzz::{ - CounterExample, FuzzFixtures, fixture_name, - invariant::{CallDetails, InvariantContract}, + BasicTxDetails, CallDetails, CounterExample, FuzzFixtures, fixture_name, + invariant::InvariantContract, }, traces::{TraceKind, TraceMode, load_contracts}, }; use itertools::Itertools; -use proptest::test_runner::{ - FailurePersistence, FileFailurePersistence, RngAlgorithm, TestError, TestRng, TestRunner, -}; +use proptest::test_runner::{RngAlgorithm, TestError, TestRng, TestRunner}; use rayon::prelude::*; use serde::{Deserialize, Serialize}; use std::{ @@ -708,9 +706,9 @@ impl<'a> FunctionRunner<'a> { let mut executor = self.clone_executor(); // Enable edge coverage if running with coverage guided fuzzing or with edge coverage // metrics (useful for benchmarking the fuzzer). - executor.inspector_mut().collect_edge_coverage( - invariant_config.corpus_dir.is_some() || invariant_config.show_edge_coverage, - ); + executor + .inspector_mut() + .collect_edge_coverage(invariant_config.corpus.collect_edge_coverage()); let mut evm = InvariantExecutor::new( executor, @@ -726,8 +724,8 @@ impl<'a> FunctionRunner<'a> { abi: &self.cr.contract.abi, }; - let (failure_dir, failure_file) = invariant_failure_paths( - invariant_config, + let (failure_dir, failure_file) = test_failure_paths( + invariant_config.failure_persist_dir.clone().unwrap(), self.cr.name, &invariant_contract.invariant_function.name, ); @@ -929,17 +927,48 @@ impl<'a> FunctionRunner<'a> { fuzz_config.runs, ); + let (failure_dir, failure_file) = test_failure_paths( + fuzz_config.failure_persist_dir.clone().unwrap(), + self.cr.name, + &func.name, + ); + + let mut executor = self.executor.into_owned(); + // Enable edge coverage if running with coverage guided fuzzing or with edge coverage + // metrics (useful for benchmarking the fuzzer). + executor.inspector_mut().collect_edge_coverage(fuzz_config.corpus.collect_edge_coverage()); + // Load persisted counterexample, if any. + let persisted_failure = + foundry_common::fs::read_json_file::(failure_file.as_path()).ok(); // Run fuzz test. - let fuzzed_executor = - FuzzedExecutor::new(self.executor.into_owned(), runner, self.tcfg.sender, fuzz_config); - let result = fuzzed_executor.fuzz( + let mut fuzzed_executor = + FuzzedExecutor::new(executor, runner, self.tcfg.sender, fuzz_config, persisted_failure); + let result = match fuzzed_executor.fuzz( func, &self.setup.fuzz_fixtures, &self.setup.deployed_libs, self.address, &self.cr.mcr.revert_decoder, progress.as_ref(), - ); + ) { + Ok(x) => x, + Err(e) => { + self.result.fuzz_setup_fail(e); + return self.result; + } + }; + + // Record counterexample. + if let Some(CounterExample::Single(counterexample)) = &result.counterexample { + if let Err(err) = foundry_common::fs::create_dir_all(failure_dir) { + error!(%err, "Failed to create fuzz failure dir"); + } else if let Err(err) = + foundry_common::fs::write_json_file(failure_file.as_path(), counterexample) + { + error!(%err, "Failed to record call sequence"); + } + } + self.result.fuzz_result(result); self.result } @@ -995,25 +1024,12 @@ impl<'a> FunctionRunner<'a> { fn fuzz_runner(&self) -> TestRunner { let config = &self.config.fuzz; - let failure_persist_path = config - .failure_persist_dir - .as_ref() - .unwrap() - .join(config.failure_persist_file.as_ref().unwrap()) - .into_os_string() - .into_string() - .unwrap(); - fuzzer_with_cases( - config.seed, - config.runs, - config.max_test_rejects, - Some(Box::new(FileFailurePersistence::Direct(failure_persist_path.leak()))), - ) + fuzzer_with_cases(config.seed, config.runs, config.max_test_rejects) } fn invariant_runner(&self) -> TestRunner { let config = &self.config.invariant; - fuzzer_with_cases(self.config.fuzz.seed, config.runs, config.max_assume_rejects, None) + fuzzer_with_cases(self.config.fuzz.seed, config.runs, config.max_assume_rejects) } fn clone_executor(&self) -> Executor { @@ -1021,14 +1037,8 @@ impl<'a> FunctionRunner<'a> { } } -fn fuzzer_with_cases( - seed: Option, - cases: u32, - max_global_rejects: u32, - file_failure_persistence: Option>, -) -> TestRunner { +fn fuzzer_with_cases(seed: Option, cases: u32, max_global_rejects: u32) -> TestRunner { let config = proptest::test_runner::Config { - failure_persistence: file_failure_persistence, cases, max_global_rejects, // Disable proptest shrink: for fuzz tests we provide single counterexample, @@ -1077,18 +1087,13 @@ fn persisted_call_sequence(path: &Path, bytecode: &Bytes) -> Option (PathBuf, PathBuf) { - let dir = config - .failure_persist_dir - .clone() - .unwrap() - .join("failures") - .join(contract_name.split(':').next_back().unwrap()); + let dir = persist_dir.join("failures").join(contract_name.split(':').next_back().unwrap()); let dir = canonicalized(dir); let file = canonicalized(dir.join(invariant_name)); (dir, file) diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index 9c32f8d47c103..e2e55265f6924 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -7,8 +7,8 @@ use foundry_compilers::{ solc::Solc, }; use foundry_config::{ - CompilationRestrictions, Config, FsPermissions, FuzzConfig, InvariantConfig, SettingsOverrides, - SolcReq, + CompilationRestrictions, Config, FsPermissions, FuzzConfig, FuzzCorpusConfig, InvariantConfig, + SettingsOverrides, SolcReq, cache::{CachedChains, CachedEndpoints, StorageCachingConfig}, filter::GlobMatcher, fs_permissions::{FsAccessPermission, PathPermission}, @@ -85,14 +85,16 @@ forgetest!(can_extract_config_values, |prj, cmd| { max_test_rejects: 100203, seed: Some(U256::from(1000)), failure_persist_dir: Some("test-cache/fuzz".into()), - failure_persist_file: Some("failures".to_string()), show_logs: false, ..Default::default() }, invariant: InvariantConfig { runs: 256, failure_persist_dir: Some("test-cache/fuzz".into()), - corpus_dir: Some("cache/invariant/corpus".into()), + corpus: FuzzCorpusConfig { + corpus_dir: Some("cache/invariant/corpus".into()), + ..Default::default() + }, ..Default::default() }, ffi: true, @@ -1104,8 +1106,11 @@ include_push_bytes = true max_fuzz_dictionary_addresses = 15728640 max_fuzz_dictionary_values = 6553600 gas_report_samples = 256 +corpus_gzip = true +corpus_min_mutations = 5 +corpus_min_size = 0 +show_edge_coverage = false failure_persist_dir = "cache/fuzz" -failure_persist_file = "failures" show_logs = false [invariant] @@ -1124,10 +1129,10 @@ gas_report_samples = 256 corpus_gzip = true corpus_min_mutations = 5 corpus_min_size = 0 +show_edge_coverage = false failure_persist_dir = "cache/invariant" show_metrics = true show_solidity = false -show_edge_coverage = false [labels] @@ -1218,8 +1223,12 @@ exclude = [] "max_fuzz_dictionary_addresses": 15728640, "max_fuzz_dictionary_values": 6553600, "gas_report_samples": 256, + "corpus_dir": null, + "corpus_gzip": true, + "corpus_min_mutations": 5, + "corpus_min_size": 0, + "show_edge_coverage": false, "failure_persist_dir": "cache/fuzz", - "failure_persist_file": "failures", "show_logs": false, "timeout": null }, @@ -1240,11 +1249,11 @@ exclude = [] "corpus_gzip": true, "corpus_min_mutations": 5, "corpus_min_size": 0, + "show_edge_coverage": false, "failure_persist_dir": "cache/invariant", "show_metrics": true, "timeout": null, - "show_solidity": false, - "show_edge_coverage": false + "show_solidity": false }, "ffi": false, "allow_internal_expect_revert": false, diff --git a/crates/forge/tests/cli/test_cmd.rs b/crates/forge/tests/cli/test_cmd.rs index 6a446a381751b..8d21bacfcb3b9 100644 --- a/crates/forge/tests/cli/test_cmd.rs +++ b/crates/forge/tests/cli/test_cmd.rs @@ -793,14 +793,14 @@ contract CounterTest is Test { Compiler run successful! Ran 1 test for test/CounterFuzz.t.sol:CounterTest -[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=0xa76d58f5fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd args=[115792089237316195423570985008687907853269984665640564039457584007913129639933 [1.157e77]]] testAddOne(uint256) (runs: 84, [AVG_GAS]) +[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=0xa76d58f5fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe args=[115792089237316195423570985008687907853269984665640564039457584007913129639934 [1.157e77]]] testAddOne(uint256) (runs: 27, [AVG_GAS]) Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED] Ran 1 test suite [ELAPSED]: 0 tests passed, 1 failed, 0 skipped (1 total tests) Failing tests: Encountered 1 failing test in test/CounterFuzz.t.sol:CounterTest -[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=0xa76d58f5fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd args=[115792089237316195423570985008687907853269984665640564039457584007913129639933 [1.157e77]]] testAddOne(uint256) (runs: 84, [AVG_GAS]) +[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=0xa76d58f5fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe args=[115792089237316195423570985008687907853269984665640564039457584007913129639934 [1.157e77]]] testAddOne(uint256) (runs: 27, [AVG_GAS]) Encountered a total of 1 failing tests, 0 tests succeeded @@ -2456,7 +2456,7 @@ contract Dummy { forgetest_init!(test_assume_no_revert_with_data, |prj, cmd| { prj.update_config(|config| { - config.fuzz.seed = Some(U256::from(100)); + config.fuzz.seed = Some(U256::from(111)); }); prj.add_source( diff --git a/crates/forge/tests/it/fuzz.rs b/crates/forge/tests/it/fuzz.rs index a22470948a9e6..cc4b22411089d 100644 --- a/crates/forge/tests/it/fuzz.rs +++ b/crates/forge/tests/it/fuzz.rs @@ -150,10 +150,10 @@ async fn test_persist_fuzz_failure() { assert_eq!(initial_calldata, new_calldata, "run {i}"); } - // write new failure in different file - let new_calldata = match run_fail!(|config| { - config.fuzz.failure_persist_file = Some("failure1".to_string()); - }) { + // write new failure in different dir. + let persist_dir = tempfile::tempdir().unwrap().keep(); + let new_calldata = match run_fail!(|config| config.fuzz.failure_persist_dir = Some(persist_dir)) + { Some(CounterExample::Single(counterexample)) => counterexample.calldata, _ => Bytes::new(), }; diff --git a/crates/forge/tests/it/test_helpers.rs b/crates/forge/tests/it/test_helpers.rs index 408b0b4af251b..b891ebb057d04 100644 --- a/crates/forge/tests/it/test_helpers.rs +++ b/crates/forge/tests/it/test_helpers.rs @@ -11,8 +11,8 @@ use foundry_compilers::{ utils::RuntimeOrHandle, }; use foundry_config::{ - Config, FsPermissions, FuzzConfig, FuzzDictionaryConfig, InvariantConfig, RpcEndpointUrl, - RpcEndpoints, fs_permissions::PathPermission, + Config, FsPermissions, FuzzConfig, FuzzCorpusConfig, FuzzDictionaryConfig, InvariantConfig, + RpcEndpointUrl, RpcEndpoints, fs_permissions::PathPermission, }; use foundry_evm::{constants::CALLER, opts::EvmOpts}; use foundry_test_utils::{ @@ -126,8 +126,8 @@ impl ForgeTestProfile { max_fuzz_dictionary_values: 10_000, }, gas_report_samples: 256, + corpus: FuzzCorpusConfig::default(), failure_persist_dir: Some(tempfile::tempdir().unwrap().keep()), - failure_persist_file: Some("testfailure".to_string()), show_logs: false, timeout: None, }; @@ -146,10 +146,7 @@ impl ForgeTestProfile { shrink_run_limit: 5000, max_assume_rejects: 65536, gas_report_samples: 256, - corpus_dir: None, - corpus_gzip: true, - corpus_min_mutations: 5, - corpus_min_size: 0, + corpus: FuzzCorpusConfig::default(), failure_persist_dir: Some( tempfile::Builder::new() .prefix(&format!("foundry-{self}")) @@ -160,7 +157,6 @@ impl ForgeTestProfile { show_metrics: true, timeout: None, show_solidity: false, - show_edge_coverage: false, }; config.sanitized()