Skip to content

Commit 95ab8e8

Browse files
onbjerggakonstampcode-com
authored
feat(forge): generate random fuzz seed if none provided (#13309)
* feat(forge): generate random fuzz seed if none provided Generate a random seed for fuzz/invariant tests when no seed is explicitly configured. This ensures reproducibility by always having a seed available that can be passed via --fuzz-seed to reproduce test runs. * feat(forge): print fuzz seed on fuzz failure (#13310) * feat(forge): print fuzz seed on test failure When a fuzz or invariant test fails, print the seed used so users can reproduce the failure with --fuzz-seed. * test: update snapshots for fuzz seed output Add [SEED] redaction pattern to match 'Fuzz seed: 0x...' output. Update all test snapshots that have fuzz/invariant failures to include the new seed line. * fix: use [SEED] placeholder in issue_3055 test snapshot Amp-Thread-ID: https://ampcode.com/threads/T-019c26cb-9d21-74f9-9e49-7ea59885e827 Co-authored-by: Amp <[email protected]> --------- Co-authored-by: Georgios Konstantopoulos <[email protected]> Co-authored-by: Amp <[email protected]> --------- Co-authored-by: Georgios Konstantopoulos <[email protected]> Co-authored-by: Amp <[email protected]>
1 parent c4e99ce commit 95ab8e8

File tree

12 files changed

+102
-7
lines changed

12 files changed

+102
-7
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/forge/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ foundry-linking.workspace = true
3535
comfy-table.workspace = true
3636
eyre.workspace = true
3737
proptest.workspace = true
38+
rand.workspace = true
3839
rayon.workspace = true
3940
serde.workspace = true
4041
tracing.workspace = true

‎crates/forge/src/cmd/test/mod.rs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ use foundry_evm::{
4343
opts::EvmOpts,
4444
traces::{backtrace::BacktraceBuilder, identifier::TraceIdentifiers, prune_trace_depth},
4545
};
46+
use rand::Rng;
4647
use regex::Regex;
4748
use std::{
4849
collections::{BTreeMap, BTreeSet},
@@ -304,6 +305,12 @@ impl TestArgs {
304305
config.invariant.gas_report_samples = 0;
305306
}
306307

308+
// Generate a random fuzz seed if none provided, for reproducibility.
309+
config.fuzz.seed = config
310+
.fuzz
311+
.seed
312+
.or_else(|| Some(U256::from_be_bytes(rand::rng().random::<[u8; 32]>())));
313+
307314
// Create test options from general project settings and compiler output.
308315
let should_debug = self.debug;
309316
let should_draw = self.flamegraph || self.flamechart;
@@ -429,6 +436,7 @@ impl TestArgs {
429436
filter: &ProjectPathsAwareFilter,
430437
output: &ProjectCompileOutput,
431438
) -> eyre::Result<TestOutcome> {
439+
let fuzz_seed = config.fuzz.seed;
432440
if self.list {
433441
return list(runner, filter);
434442
}
@@ -504,13 +512,13 @@ impl TestArgs {
504512
}
505513
});
506514
sh_println!("{}", serde_json::to_string(&results)?)?;
507-
return Ok(TestOutcome::new(Some(runner), results, self.allow_failure));
515+
return Ok(TestOutcome::new(Some(runner), results, self.allow_failure, fuzz_seed));
508516
}
509517

510518
if self.junit {
511519
let results = runner.test_collect(filter)?;
512520
sh_println!("{}", junit_xml_report(&results, verbosity).to_string()?)?;
513-
return Ok(TestOutcome::new(Some(runner), results, self.allow_failure));
521+
return Ok(TestOutcome::new(Some(runner), results, self.allow_failure, fuzz_seed));
514522
}
515523

516524
let remote_chain =
@@ -567,6 +575,7 @@ impl TestArgs {
567575
let mut gas_snapshots = BTreeMap::<String, BTreeMap<String, String>>::new();
568576

569577
let mut outcome = TestOutcome::empty(None, self.allow_failure);
578+
outcome.fuzz_seed = fuzz_seed;
570579

571580
let mut any_test_failed = false;
572581
let mut backtrace_builder = None;

‎crates/forge/src/result.rs‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use crate::{
66
gas_report::GasReport,
77
};
88
use alloy_primitives::{
9-
Address, I256, Log,
9+
Address, I256, Log, U256,
1010
map::{AddressHashMap, HashMap},
1111
};
1212
use eyre::Report;
@@ -46,6 +46,8 @@ pub struct TestOutcome {
4646
pub gas_report: Option<GasReport>,
4747
/// The runner used to execute the tests.
4848
pub runner: Option<MultiContractRunner>,
49+
/// The fuzz seed used for the test run.
50+
pub fuzz_seed: Option<U256>,
4951
}
5052

5153
impl TestOutcome {
@@ -54,13 +56,14 @@ impl TestOutcome {
5456
runner: Option<MultiContractRunner>,
5557
results: BTreeMap<String, SuiteResult>,
5658
allow_failure: bool,
59+
fuzz_seed: Option<U256>,
5760
) -> Self {
58-
Self { results, allow_failure, last_run_decoder: None, gas_report: None, runner }
61+
Self { results, allow_failure, last_run_decoder: None, gas_report: None, runner, fuzz_seed }
5962
}
6063

6164
/// Creates a new empty test outcome.
6265
pub fn empty(runner: Option<MultiContractRunner>, allow_failure: bool) -> Self {
63-
Self::new(runner, BTreeMap::new(), allow_failure)
66+
Self::new(runner, BTreeMap::new(), allow_failure, None)
6467
}
6568

6669
/// Returns an iterator over all individual succeeding tests and their names.
@@ -130,6 +133,11 @@ impl TestOutcome {
130133
self.failures().count()
131134
}
132135

136+
/// Returns `true` if any fuzz or invariant test failed.
137+
pub fn has_fuzz_failures(&self) -> bool {
138+
self.failures().any(|(_, t)| t.kind.is_fuzz() || t.kind.is_invariant())
139+
}
140+
133141
/// Sums up all the durations of all individual test suites.
134142
///
135143
/// Note that this is not necessarily the wall clock time of the entire test run.
@@ -200,6 +208,17 @@ impl TestOutcome {
200208
test_word
201209
)?;
202210

211+
// Print seed for fuzz/invariant test failures to enable reproduction.
212+
if let Some(seed) = self.fuzz_seed
213+
&& outcome.has_fuzz_failures()
214+
{
215+
sh_println!(
216+
"\nFuzz seed: {} (use {} to reproduce)",
217+
format!("{seed:#x}").cyan(),
218+
"`--fuzz-seed`".cyan()
219+
)?;
220+
}
221+
203222
std::process::exit(1);
204223
}
205224

@@ -895,6 +914,16 @@ impl Default for TestKind {
895914
}
896915

897916
impl TestKind {
917+
/// Returns `true` if this is a fuzz test.
918+
pub fn is_fuzz(&self) -> bool {
919+
matches!(self, Self::Fuzz { .. })
920+
}
921+
922+
/// Returns `true` if this is an invariant test.
923+
pub fn is_invariant(&self) -> bool {
924+
matches!(self, Self::Invariant { .. })
925+
}
926+
898927
/// The gas consumed by this test
899928
pub fn report(&self) -> TestKindReport {
900929
match self {

‎crates/forge/tests/cli/test_cmd/fuzz.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,8 @@ Encountered a total of 1 failing tests, 2 tests succeeded
384384
385385
Tip: Run `forge test --rerun` to retry only the 1 failed test
386386
387+
[SEED] (use `--fuzz-seed` to reproduce)
388+
387389
"#]]);
388390
});
389391

‎crates/forge/tests/cli/test_cmd/invariant/common.rs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ Encountered a total of 2 failing tests, 1 tests succeeded
9898
9999
Tip: Run `forge test --rerun` to retry only the 2 failed tests
100100
101+
[SEED] (use `--fuzz-seed` to reproduce)
102+
101103
"#]]);
102104
});
103105

@@ -175,6 +177,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
175177
176178
Tip: Run `forge test --rerun` to retry only the 1 failed test
177179
180+
[SEED] (use `--fuzz-seed` to reproduce)
181+
178182
"#]]);
179183
});
180184

@@ -304,6 +308,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
304308
305309
Tip: Run `forge test --rerun` to retry only the 1 failed test
306310
311+
[SEED] (use `--fuzz-seed` to reproduce)
312+
307313
"#]]);
308314
});
309315

@@ -373,6 +379,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
373379
374380
Tip: Run `forge test --rerun` to retry only the 1 failed test
375381
382+
[SEED] (use `--fuzz-seed` to reproduce)
383+
376384
"#]]);
377385
});
378386

@@ -538,6 +546,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
538546
539547
Tip: Run `forge test --rerun` to retry only the 1 failed test
540548
549+
[SEED] (use `--fuzz-seed` to reproduce)
550+
541551
"#]]);
542552
});
543553

@@ -624,6 +634,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
624634
625635
Tip: Run `forge test --rerun` to retry only the 1 failed test
626636
637+
[SEED] (use `--fuzz-seed` to reproduce)
638+
627639
"#]]);
628640
});
629641

@@ -696,6 +708,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
696708
697709
Tip: Run `forge test --rerun` to retry only the 1 failed test
698710
711+
[SEED] (use `--fuzz-seed` to reproduce)
712+
699713
"#]]);
700714
});
701715

@@ -779,6 +793,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
779793
780794
Tip: Run `forge test --rerun` to retry only the 1 failed test
781795
796+
[SEED] (use `--fuzz-seed` to reproduce)
797+
782798
"#]]);
783799

784800
// `fuzz_seed` at 119 makes this sequence shrinkable from 4 to 2.
@@ -888,6 +904,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
888904
889905
Tip: Run `forge test --rerun` to retry only the 1 failed test
890906
907+
[SEED] (use `--fuzz-seed` to reproduce)
908+
891909
"#]]);
892910
});
893911

@@ -980,6 +998,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
980998
981999
Tip: Run `forge test --rerun` to retry only the 1 failed test
9821000
1001+
[SEED] (use `--fuzz-seed` to reproduce)
1002+
9831003
"#]]);
9841004
});
9851005

@@ -1092,6 +1112,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
10921112
10931113
Tip: Run `forge test --rerun` to retry only the 1 failed test
10941114
1115+
[SEED] (use `--fuzz-seed` to reproduce)
1116+
10951117
"#]]);
10961118
});
10971119

@@ -1191,6 +1213,8 @@ Encountered a total of 2 failing tests, 0 tests succeeded
11911213
11921214
Tip: Run `forge test --rerun` to retry only the 2 failed tests
11931215
1216+
[SEED] (use `--fuzz-seed` to reproduce)
1217+
11941218
"#]]);
11951219
});
11961220

@@ -1290,6 +1314,8 @@ Encountered a total of 2 failing tests, 0 tests succeeded
12901314
12911315
Tip: Run `forge test --rerun` to retry only the 2 failed tests
12921316
1317+
[SEED] (use `--fuzz-seed` to reproduce)
1318+
12931319
"#]]);
12941320
});
12951321

@@ -1589,6 +1615,8 @@ Encountered a total of 2 failing tests, 0 tests succeeded
15891615
15901616
Tip: Run `forge test --rerun` to retry only the 2 failed tests
15911617
1618+
[SEED] (use `--fuzz-seed` to reproduce)
1619+
15921620
"#]]);
15931621
});
15941622

‎crates/forge/tests/cli/test_cmd/invariant/mod.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
431431
432432
Tip: Run `forge test --rerun` to retry only the 1 failed test
433433
434+
[SEED] (use `--fuzz-seed` to reproduce)
435+
434436
"#]],
435437
);
436438

@@ -458,6 +460,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
458460
459461
Tip: Run `forge test --rerun` to retry only the 1 failed test
460462
463+
[SEED] (use `--fuzz-seed` to reproduce)
464+
461465
"#]],
462466
);
463467

@@ -481,6 +485,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
481485
482486
Tip: Run `forge test --rerun` to retry only the 1 failed test
483487
488+
[SEED] (use `--fuzz-seed` to reproduce)
489+
484490
"#]],
485491
);
486492
});

‎crates/forge/tests/cli/test_cmd/invariant/target.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -627,6 +627,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
627627
628628
Tip: Run `forge test --rerun` to retry only the 1 failed test
629629
630+
[SEED] (use `--fuzz-seed` to reproduce)
631+
630632
"#]]);
631633

632634
// Test TargetSelectors
@@ -670,6 +672,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
670672
671673
Tip: Run `forge test --rerun` to retry only the 1 failed test
672674
675+
[SEED] (use `--fuzz-seed` to reproduce)
676+
673677
"#]],
674678
);
675679

@@ -715,6 +719,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
715719
716720
Tip: Run `forge test --rerun` to retry only the 1 failed test
717721
722+
[SEED] (use `--fuzz-seed` to reproduce)
723+
718724
"#]]);
719725

720726
// Test TargetArtifactSelectors
@@ -763,6 +769,8 @@ Encountered a total of 1 failing tests, 1 tests succeeded
763769
764770
Tip: Run `forge test --rerun` to retry only the 1 failed test
765771
772+
[SEED] (use `--fuzz-seed` to reproduce)
773+
766774
"#]]);
767775
});
768776

‎crates/forge/tests/cli/test_cmd/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
915915
916916
Tip: Run `forge test --rerun` to retry only the 1 failed test
917917
918+
[SEED] (use `--fuzz-seed` to reproduce)
919+
918920
"#]]);
919921
});
920922

@@ -966,6 +968,8 @@ Encountered a total of 1 failing tests, 0 tests succeeded
966968
967969
Tip: Run `forge test --rerun` to retry only the 1 failed test
968970
971+
[SEED] (use `--fuzz-seed` to reproduce)
972+
969973
"#]]);
970974
});
971975

‎crates/forge/tests/cli/test_cmd/repros.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ Encountered a total of 3 failing tests, 0 tests succeeded
6363
6464
Tip: Run `forge test --rerun` to retry only the 3 failed tests
6565
66+
[SEED] (use `--fuzz-seed` to reproduce)
67+
6668
"#]]);
6769
});
6870

0 commit comments

Comments
 (0)