feat(forge): coverage guided fuzzing & time based campaigns for invariant mode (#10190)

* rename coverage to line coverage for clarity

* WIP: coverage guided fuzzing

* wip persist invariant corpus

* add binning and history map

* rm proptest runner, add corpus mutations

* fix: splice mutation, add some notes

* Clippy and more tests

* save

* use libafl_bolt's SIMD hitmap

* fix eyre issues

* add comments and psuedocode

* Revert libafl

* Typo

* Fix win config test

* cleanup, save corpus at the end of run, if new coverage

* consolidate corpus manager

* Consolidate tx manager corpus logic

* Review changes: do not stop fuzzing if corpus replay failures, report
number of failures, uuids for corpus file

* Default gzip corpus and config to toggle json/gzip

* Evict oldest corpus with more than x mutations

* Add min corpus size config, bump max mutations to default depth run

* Simplify corpus manager and corpus struct, enable prefix / suffix
mutation, manager to handle generate from strategy

* Fuzz arg from ABI

* Corpus max mutations default 5

* Save metadata on disk at eviction time

* Remove more than 2 branches branch, make sure we always have one

* Load gz and json seeds, ignore metadata files

* ABI mutation replaces subset of arguments sometimes

* prevent empty range but perform at least 1 round

* trim selector when using abi_decode_input

* Nit, remove clippy allow

* retain corpus items that are highly likely to produce new finds

* rename corpus_max_mutations to corpus_min_mutations

* update cli test expectations

* Stateless fuzz corpus config revert, add invariant time based campaigns

* Changes after review
- revert cache dir configs, invariant corpus can be external of cache
- save and load as json.gz
- comment update
- introduce mutation type enum

* Remove outdated comment

* Update crates/evm/evm/src/executors/mod.rs

Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>

* Changes after review: comment, update merge_edge_coverage, use rng.gen

* Fix docs

* Keep test assert, found faster than without guidance

* Fix

* Do not use in memory mutated corpus if coverage guided is disabled.

---------

Co-authored-by: grandizzy <grandizzy.the.egg@gmail.com>
Co-authored-by: grandizzy <38490174+grandizzy@users.noreply.github.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>

Author: 3 people

Cargo.lock

@@ -349,6 +349,8 @@ yansi = { version = "1.0", features = ["detect-tty", "detect-env"] }
 path-slash = "0.2"
 jiff = "0.2"
 heck = "0.5"
+uuid = "1.17.0"
+flate2 = "1.1"
 
 ## Pinned dependencies. Enabled for the workspace in crates/test-utils.
 

Cargo.toml

@@ -84,7 +84,7 @@ futures.workspace = true
 async-trait.workspace = true
 
 # misc
-flate2 = "1.1"
+flate2.workspace = true
 serde_json.workspace = true
 serde.workspace = true
 thiserror.workspace = true

crates/anvil/Cargo.toml

@@ -73,6 +73,8 @@ anstyle.workspace = true
 terminal_size.workspace = true
 ciborium.workspace = true
 
+flate2.workspace = true
+
 [build-dependencies]
 chrono.workspace = true
 vergen = { workspace = true, features = ["build", "git", "gitcl"] }

crates/common/Cargo.toml

@@ -1,10 +1,11 @@
 //! Contains various `std::fs` wrapper functions that also contain the target path in their errors.
 
 use crate::errors::FsPathError;
+use flate2::{read::GzDecoder, write::GzEncoder, Compression};
 use serde::{de::DeserializeOwned, Serialize};
 use std::{
     fs::{self, File},
-    io::{BufWriter, Write},
+    io::{BufReader, BufWriter, Write},
     path::{Component, Path, PathBuf},
 };
 
@@ -49,6 +50,15 @@ pub fn read_json_file<T: DeserializeOwned>(path: &Path) -> Result<T> {
     serde_json::from_str(&s).map_err(|source| FsPathError::ReadJson { source, path: path.into() })
 }
 
+/// Reads and decodes the json gzip file, then deserialize it into the provided type.
+pub fn read_json_gzip_file<T: DeserializeOwned>(path: &Path) -> Result<T> {
+    let file = open(path)?;
+    let reader = BufReader::new(file);
+    let decoder = GzDecoder::new(reader);
+    serde_json::from_reader(decoder)
+        .map_err(|source| FsPathError::ReadJson { source, path: path.into() })
+}
+
 /// Writes the object as a JSON object.
 pub fn write_json_file<T: Serialize>(path: &Path, obj: &T) -> Result<()> {
     let file = create_file(path)?;
@@ -67,6 +77,20 @@ pub fn write_pretty_json_file<T: Serialize>(path: &Path, obj: &T) -> Result<()>
     writer.flush().map_err(|e| FsPathError::write(e, path))
 }
 
+/// Writes the object as a gzip compressed file.
+pub fn write_json_gzip_file<T: Serialize>(path: &Path, obj: &T) -> Result<()> {
+    let file = create_file(path)?;
+    let writer = BufWriter::new(file);
+    let mut encoder = GzEncoder::new(writer, Compression::default());
+    serde_json::to_writer(&mut encoder, obj)
+        .map_err(|source| FsPathError::WriteJson { source, path: path.into() })?;
+    encoder
+        .finish()
+        .map_err(serde_json::Error::io)
+        .map_err(|source| FsPathError::WriteJson { source, path: path.into() })?;
+    Ok(())
+}
+
 /// Wrapper for `std::fs::write`
 pub fn write(path: impl AsRef<Path>, contents: impl AsRef<[u8]>) -> Result<()> {
     let path = path.as_ref();

crates/common/src/fs.rs

@@ -26,6 +26,15 @@ pub struct InvariantConfig {
     pub max_assume_rejects: u32,
     /// Number of runs to execute and include in the gas report.
     pub gas_report_samples: u32,
+    /// Path where invariant corpus is stored. If not configured then coverage guided fuzzing is
+    /// disabled.
+    pub corpus_dir: Option<PathBuf>,
+    /// Whether corpus to use gzip file compression and decompression.
+    pub corpus_gzip: bool,
+    // Number of corpus mutations until marked as eligible to be flushed from memory.
+    pub corpus_min_mutations: usize,
+    // Number of corpus that won't be evicted from memory.
+    pub corpus_min_size: usize,
     /// Path where invariant failures are recorded and replayed.
     pub failure_persist_dir: Option<PathBuf>,
     /// Whether to collect and display fuzzed selectors metrics.
@@ -47,6 +56,10 @@ impl Default for InvariantConfig {
             shrink_run_limit: 5000,
             max_assume_rejects: 65536,
             gas_report_samples: 256,
+            corpus_dir: None,
+            corpus_gzip: true,
+            corpus_min_mutations: 5,
+            corpus_min_size: 0,
             failure_persist_dir: None,
             show_metrics: true,
             timeout: None,
@@ -67,6 +80,10 @@ impl InvariantConfig {
             shrink_run_limit: 5000,
             max_assume_rejects: 65536,
             gas_report_samples: 256,
+            corpus_dir: None,
+            corpus_gzip: true,
+            corpus_min_mutations: 5,
+            corpus_min_size: 0,
             failure_persist_dir: Some(cache_dir),
             show_metrics: true,
             timeout: None,

crates/config/src/invariant.rs

@@ -1059,6 +1059,7 @@ impl Config {
             }
         };
         remove_test_dir(&self.fuzz.failure_persist_dir);
+        remove_test_dir(&self.invariant.corpus_dir);
         remove_test_dir(&self.invariant.failure_persist_dir);
 
         Ok(())
@@ -4537,6 +4538,7 @@ mod tests {
                     runs: 512,
                     depth: 10,
                     failure_persist_dir: Some(PathBuf::from("cache/invariant")),
+                    corpus_dir: None,
                     ..Default::default()
                 }
             );

crates/config/src/lib.rs

@@ -10,7 +10,7 @@ use std::ptr::NonNull;
 
 /// Inspector implementation for collecting coverage information.
 #[derive(Clone, Debug)]
-pub struct CoverageCollector {
+pub struct LineCoverageCollector {
     // NOTE: `current_map` is always a valid reference into `maps`.
     // It is accessed only through `get_or_insert_map` which guarantees that it's valid.
     // Both of these fields are unsafe to access directly outside of `*insert_map`.
@@ -21,10 +21,10 @@ pub struct CoverageCollector {
 }
 
 // SAFETY: See comments on `current_map`.
-unsafe impl Send for CoverageCollector {}
-unsafe impl Sync for CoverageCollector {}
+unsafe impl Send for LineCoverageCollector {}
+unsafe impl Sync for LineCoverageCollector {}
 
-impl Default for CoverageCollector {
+impl Default for LineCoverageCollector {
     fn default() -> Self {
         Self {
             current_map: NonNull::dangling(),
@@ -34,7 +34,7 @@ impl Default for CoverageCollector {
     }
 }
 
-impl<CTX> Inspector<CTX> for CoverageCollector
+impl<CTX> Inspector<CTX> for LineCoverageCollector
 where
     CTX: ContextTr<Journal: JournalExt>,
 {
@@ -50,7 +50,7 @@ where
     }
 }
 
-impl CoverageCollector {
+impl LineCoverageCollector {
     /// Finish collecting coverage information and return the [`HitMaps`].
     pub fn finish(self) -> HitMaps {
         self.maps

crates/evm/coverage/src/inspector.rs

@@ -29,7 +29,7 @@ pub mod analysis;
 pub mod anchors;
 
 mod inspector;
-pub use inspector::CoverageCollector;
+pub use inspector::LineCoverageCollector;
 
 /// A coverage report.
 ///

crates/evm/coverage/src/lib.rs

@@ -52,3 +52,4 @@ thiserror.workspace = true
 tracing.workspace = true
 indicatif.workspace = true
 serde.workspace = true
+uuid.workspace = true

crates/evm/evm/Cargo.toml

@@ -181,7 +181,7 @@ impl FuzzedExecutor {
             traces: last_run_traces,
             breakpoints: last_run_breakpoints,
             gas_report_traces: traces.into_iter().map(|a| a.arena).collect(),
-            coverage: fuzz_result.coverage,
+            line_coverage: fuzz_result.coverage,
             deprecated_cheatcodes: fuzz_result.deprecated_cheatcodes,
         };
 
@@ -258,7 +258,7 @@ impl FuzzedExecutor {
             Ok(FuzzOutcome::Case(CaseOutcome {
                 case: FuzzCase { calldata, gas: call.gas_used, stipend: call.stipend },
                 traces: call.traces,
-                coverage: call.coverage,
+                coverage: call.line_coverage,
                 breakpoints,
                 logs: call.logs,
                 deprecated_cheatcodes,