From baf4aa820622cb6e895fd1a4ba479d99109189de Mon Sep 17 00:00:00 2001 From: Nils Lehnen <30603423+iderex@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:47:26 +0200 Subject: [PATCH] Make the corpus replay's own rules fail, and prove each one bites (#86) The replay landed as four assertions over one fixed directory. Every one of them read the corpus this tree carries, that corpus is healthy, and a rule that has only ever run against a healthy subject cannot be told apart from a rule that refuses nothing. Two of #86's four conditions are statements about a run going RED - that an empty corpus reddens the build, and that a deliberately unhandled input in a seed reddens it - and neither of them had ever been watched happening. The rules now answer instead of asserting. `rules(root, named)` takes the corpus root and the target names as arguments and returns the refusals, so the gate hands it this tree's corpus and requires an empty answer while each proof hands it a root built for one defect and requires exactly that rule and no other. `replay_every_seed(root, call)` takes the target function the same way, so a proof can hand it a target that does not name what it did with a seed and watch the failure come out rather than be counted as an answer. The rules the gate runs and the rules the proofs trip are one function rather than two, which is what stops a proof passing against a second copy of the logic. What this prevents is a replay that reports a pass for a surface nobody reached. Every one of the four states - a root with no directory, a directory with no seed, a directory with no entry point behind it, and a target with no directory - exits zero and prints a page indistinguishable from a run that covered everything, and the same is true of a loop that catches a panic and counts it. Six deliberate removals were made against these rules and each reddened exactly one proof and left the other ten green. Signed-off-by: Nils Lehnen <30603423+iderex@users.noreply.github.com> --- tests/replay_the_seed_corpus.rs | 387 +++++++++++++++++++++++++++----- 1 file changed, 327 insertions(+), 60 deletions(-) diff --git a/tests/replay_the_seed_corpus.rs b/tests/replay_the_seed_corpus.rs index 137479f..27f81d7 100644 --- a/tests/replay_the_seed_corpus.rs +++ b/tests/replay_the_seed_corpus.rs @@ -28,6 +28,33 @@ //! 0116's change-notification connection and the response decoding - and neither //! has an entry point in this tree, so neither has a directory here. //! +//! # The rules answer, and what the gate asserts on is the answer +//! +//! THE RULES USED TO BE ASSERTIONS OVER ONE FIXED DIRECTORY, WHICH IS A GUARD +//! NOBODY HAS WATCHED FAIL. Each of them reads the corpus this tree carries, +//! that corpus is healthy, and a rule that has only ever run against a healthy +//! subject cannot be told apart from a rule that refuses nothing. Two of #86's +//! four conditions ask for the opposite: that an empty corpus REDDEN the build +//! and that a deliberately unhandled input in a seed REDDEN it, which are +//! statements about a run going red rather than about an assertion existing. +//! +//! So [`rules`] takes the root and the target names as arguments and ANSWERS +//! with the refusals rather than asserting them. The gate hands it this tree's +//! corpus and the names in `TARGETS` and requires the answer to be empty; each +//! proof below hands it a root built for one defect and requires the answer to +//! name exactly that rule and no other. The rules the gate runs and the rules +//! the proofs trip are one function rather than two, which is what stops a proof +//! passing against a second copy of the logic. +//! +//! [`replay_every_seed`] is parameterised the same way and for the same reason: +//! the target it calls is an argument, so a proof can hand it a target that does +//! not name what it did with a seed and watch the panic come out rather than be +//! counted as an answer. +//! +//! Every root a proof builds is under the build directory rather than in the +//! tree, so nothing a proof writes is tracked and no such root can be mistaken +//! for a corpus. +//! //! # What a seed asserts //! //! That the target ANSWERS. Every target below returns a value or a member of a @@ -69,10 +96,9 @@ fn corpus_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/corpus") } -/// The directory names under the corpus root, sorted. -fn corpus_directories() -> Vec { - let root = corpus_root(); - let entries = std::fs::read_dir(&root) +/// The directory names under one corpus root, sorted. +fn corpus_directories(root: &Path) -> Vec { + let entries = std::fs::read_dir(root) .unwrap_or_else(|e| panic!("cannot read the corpus root {}: {e}", root.display())); let mut names = Vec::new(); for entry in entries { @@ -85,9 +111,9 @@ fn corpus_directories() -> Vec { names } -/// The seeds in one target's directory, sorted, as (name, bytes). -fn seeds(target: &str) -> Vec<(String, Vec)> { - let directory = corpus_root().join(target); +/// The seeds in one target's directory under one root, sorted, as (name, bytes). +fn seeds(root: &Path, target: &str) -> Vec<(String, Vec)> { + let directory = root.join(target); let entries = std::fs::read_dir(&directory) .unwrap_or_else(|e| panic!("cannot read {}: {e}", directory.display())); let mut found = Vec::new(); @@ -105,6 +131,81 @@ fn seeds(target: &str) -> Vec<(String, Vec)> { found } +/// One reason a corpus root cannot be replayed as it stands. +/// +/// The rule is an identifier rather than a sentence, so a proof can require +/// exactly one of them and no other in a way a message could not. The sentence a +/// reader of a failing run needs is built beside the assertion that carries it. +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +struct Refusal { + rule: &'static str, + subject: String, +} + +impl Refusal { + fn new(rule: &'static str, subject: &str) -> Self { + Self { + rule, + subject: subject.to_owned(), + } + } +} + +/// Every rule over one corpus root, answering rather than asserting. +/// +/// `named` is the set of targets [`replay`] can call, passed in rather than read +/// from the constant above, so a proof can build a root of its own and still run +/// this function rather than a copy of it. +fn rules(root: &Path, named: &[&str]) -> Vec { + let mut refusals = Vec::new(); + let directories = corpus_directories(root); + + // Replaying nothing proves nothing and exits zero, which reads exactly like + // a replay that found no defect. + if directories.is_empty() { + refusals.push(Refusal::new( + "the-root-holds-no-directory", + "the corpus root", + )); + } + + let found: BTreeSet<&str> = directories.iter().map(String::as_str).collect(); + let declared: BTreeSet<&str> = named.iter().copied().collect(); + + // An empty corpus reddens the build rather than passing quietly, which is + // #86's third condition. + for directory in &directories { + if seeds(root, directory).is_empty() { + refusals.push(Refusal::new("a-directory-holds-no-seed", directory)); + } + } + + // A corpus with no entry point behind it is replayed by nothing. + for directory in found.difference(&declared) { + refusals.push(Refusal::new("a-directory-names-no-target", directory)); + } + + // A target with no seeds is a target nothing replays, and the run above it + // would have passed without ever reaching it. + for target in declared.difference(&found) { + refusals.push(Refusal::new("a-target-has-no-directory", target)); + } + + refusals.sort(); + refusals +} + +/// The rule identifiers in one answer, sorted and deduplicated. +/// +/// A proof compares this against the single rule its root was built to trip, so +/// a root that trips a second rule fails the proof rather than passing it. +fn rule_ids(refusals: &[Refusal]) -> Vec<&'static str> { + let mut ids: Vec<&'static str> = refusals.iter().map(|refusal| refusal.rule).collect(); + ids.sort_unstable(); + ids.dedup(); + ids +} + /// Hands one seed to one target and returns what it answered, as text. /// /// The text is what the report prints, so a reader of a failing run sees which @@ -157,66 +258,51 @@ fn replay(target: &str, bytes: &[u8]) -> String { } } -#[test] -fn the_corpus_root_is_not_empty() { - let directories = corpus_directories(); - assert!( - !directories.is_empty(), - "there are no corpus directories under {}. Replaying nothing proves nothing and \ - exits zero, which reads exactly like a replay that found no defect.", - corpus_root().display() - ); +/// Every seed of every directory under one root, through one target function. +/// +/// Returns how many seeds were replayed. What it asserts per seed is that the +/// target answered at all: anything the code does not name could only leave +/// through a panic, and a panic is deliberately not caught here. Catching one +/// and counting it would turn #86's fourth condition into a number in a report, +/// which is the one-line change this arrangement exists against. +fn replay_every_seed(root: &Path, call: fn(&str, &[u8]) -> String) -> usize { + let mut replayed = 0; + for target in corpus_directories(root) { + for (name, bytes) in seeds(root, &target) { + let answer = call(&target, &bytes); + assert!( + !answer.is_empty(), + "{target}/{name} produced no answer at all" + ); + replayed += 1; + } + } + replayed } -#[test] -fn every_corpus_directory_names_a_target_and_every_target_has_one() { - let found: BTreeSet = corpus_directories().into_iter().collect(); - let named: BTreeSet = TARGETS.iter().map(|t| (*t).to_owned()).collect(); +// -------------------------------------------------------------------------- +// The gate: the rules and the replay, over the corpus this tree carries. +// -------------------------------------------------------------------------- - let without_a_target: Vec<&String> = found.difference(&named).collect(); - assert!( - without_a_target.is_empty(), - "corpus directories naming no target this file can replay: {without_a_target:?}. \ - A corpus with no entry point behind it is replayed by nothing." - ); - - let without_a_corpus: Vec<&String> = named.difference(&found).collect(); +#[test] +fn the_corpus_this_tree_carries_breaks_no_rule() { + let root = corpus_root(); + let refusals = rules(&root, TARGETS); assert!( - without_a_corpus.is_empty(), - "targets with no corpus directory: {without_a_corpus:?}. A target with no seeds is a \ - target nothing replays, and the run above would have passed without reaching it." + refusals.is_empty(), + "the corpus under {} breaks {} rule(s): {refusals:?}. Each one is a state in which a \ + replay exits zero having covered less than the run reads as having covered.", + root.display(), + refusals.len() ); } -#[test] -fn no_corpus_directory_is_empty() { - for target in corpus_directories() { - assert!( - !seeds(&target).is_empty(), - "the corpus directory {target} holds no seed. #86 asks that an empty corpus redden \ - the build rather than passing quietly, because replaying nothing proves nothing." - ); - } -} - -/// The replay itself. Every seed of every target, and the assertion is that each -/// one produced an answer at all: a target that did anything else would have -/// panicked, and a panic here is this test failing. #[test] fn every_seed_is_replayed_and_every_target_answers() { - let mut replayed = 0; - for target in corpus_directories() { - for (name, bytes) in seeds(&target) { - let answer = replay(&target, &bytes); - assert!( - !answer.is_empty(), - "{target}/{name} produced no answer at all" - ); - replayed += 1; - } - } + let root = corpus_root(); + let replayed = replay_every_seed(&root, replay); assert!( - replayed >= corpus_directories().len(), + replayed >= corpus_directories(&root).len(), "fewer seeds were replayed than there are targets, so at least one target ran nothing" ); } @@ -230,9 +316,10 @@ fn every_seed_is_replayed_and_every_target_answers() { /// seed for on the day it lands. #[test] fn the_named_refusals_are_each_reached_by_a_seed() { + let root = corpus_root(); let mut image_refusals = BTreeSet::new(); let mut image_accepted = BTreeSet::new(); - for (_, bytes) in seeds("artwork-format") { + for (_, bytes) in seeds(&root, "artwork-format") { match admitted(&bytes) { Ok(found) => { image_accepted.insert(format!("{:?}", found.format())); @@ -269,7 +356,7 @@ fn the_named_refusals_are_each_reached_by_a_seed() { let mut envelope_drops = BTreeSet::new(); let mut envelope_opened = 0; - for (_, bytes) in seeds("cache-envelope") { + for (_, bytes) in seeds(&root, "cache-envelope") { match open(EntryKind::LibraryQueryResults, &bytes) { Ok(_) => envelope_opened += 1, Err(failed) => { @@ -292,7 +379,7 @@ fn the_named_refusals_are_each_reached_by_a_seed() { let mut addresses_parsed = 0; let mut addresses_refused = 0; - for (_, bytes) in seeds("server-address") { + for (_, bytes) in seeds(&root, "server-address") { match BaseAddress::parse(&String::from_utf8_lossy(&bytes)) { Ok(_) => addresses_parsed += 1, Err(_) => addresses_refused += 1, @@ -304,3 +391,183 @@ fn the_named_refusals_are_each_reached_by_a_seed() { parsed and {addresses_refused} refused." ); } + +// -------------------------------------------------------------------------- +// The proofs. Each builds a root outside the tree, hands it to the same +// function the gate above runs, and requires the rule it was built to trip. +// -------------------------------------------------------------------------- + +/// A root under the build directory, emptied first so a run is never read +/// against what the run before it left behind. +fn a_root_for(case: &str) -> PathBuf { + let root = Path::new(env!("CARGO_TARGET_TMPDIR")) + .join("corpus-rules") + .join(case); + if root.exists() { + std::fs::remove_dir_all(&root) + .unwrap_or_else(|e| panic!("cannot clear {}: {e}", root.display())); + } + std::fs::create_dir_all(&root) + .unwrap_or_else(|e| panic!("cannot create {}: {e}", root.display())); + root +} + +fn a_directory(root: &Path, target: &str) { + let directory = root.join(target); + std::fs::create_dir_all(&directory) + .unwrap_or_else(|e| panic!("cannot create {}: {e}", directory.display())); +} + +fn a_seed(root: &Path, target: &str, name: &str, bytes: &[u8]) { + a_directory(root, target); + let path = root.join(target).join(name); + std::fs::write(&path, bytes).unwrap_or_else(|e| panic!("cannot write {}: {e}", path.display())); +} + +/// The healthy root every one-change neighbour below is a change to. +/// +/// Two directories rather than one, each holding one seed, and both named. With +/// a single directory a rule that fires on the LAST directory it reads would be +/// indistinguishable from one that fires on the only directory there is. +fn a_healthy_root(case: &str) -> PathBuf { + let root = a_root_for(case); + a_seed(&root, "first-target", "a-seed", b"one"); + a_seed(&root, "second-target", "a-seed", b"two"); + root +} + +const HEALTHY: &[&str] = &["first-target", "second-target"]; + +#[test] +fn the_healthy_root_breaks_no_rule() { + let root = a_healthy_root("the-healthy-root"); + assert_eq!( + rules(&root, HEALTHY), + Vec::new(), + "the root each proof below is a one-change neighbour of already breaks a rule, so \ + nothing those proofs report could be attributed to their own change." + ); +} + +#[test] +fn a_root_with_no_directory_is_refused() { + // The one proof that is not a neighbour of the healthy root. A root holding + // no directory names no target either, so the target list goes with the + // directories: both halves are the same absence rather than two changes. + let root = a_root_for("a-root-with-no-directory"); + let refusals = rules(&root, &[]); + assert_eq!( + rule_ids(&refusals), + vec!["the-root-holds-no-directory"], + "an empty corpus root is what a replay of nothing looks like from the inside, and it \ + exits zero. Refusals were {refusals:?}." + ); +} + +#[test] +fn a_directory_with_no_seed_is_refused() { + let root = a_healthy_root("a-directory-with-no-seed"); + // The one change: the second target keeps its directory and loses its seed. + std::fs::remove_file(root.join("second-target").join("a-seed")).expect("the seed is there"); + let refusals = rules(&root, HEALTHY); + assert_eq!( + rule_ids(&refusals), + vec!["a-directory-holds-no-seed"], + "this is #86's `an empty corpus reddens the build`. Refusals were {refusals:?}." + ); + assert_eq!( + refusals[0].subject, "second-target", + "the refusal names the directory rather than only a count, so a corpus that lost one \ + seed is repaired without reading every directory in it." + ); +} + +#[test] +fn a_directory_naming_no_target_is_refused() { + let root = a_healthy_root("a-directory-naming-no-target"); + // The one change: a third directory, with a seed and nothing behind it. + a_seed(&root, "a-surface-nobody-can-call", "a-seed", b"three"); + let refusals = rules(&root, HEALTHY); + assert_eq!( + rule_ids(&refusals), + vec!["a-directory-names-no-target"], + "a corpus with no entry point behind it is replayed by nothing, and the run that walks \ + past it exits zero. Refusals were {refusals:?}." + ); + assert_eq!(refusals[0].subject, "a-surface-nobody-can-call"); +} + +#[test] +fn a_target_with_no_directory_is_refused() { + let root = a_healthy_root("a-target-with-no-directory"); + // The one change: a third target is named and acquires no corpus. + let named = &["first-target", "second-target", "a-target-with-no-seeds"]; + let refusals = rules(&root, named); + assert_eq!( + rule_ids(&refusals), + vec!["a-target-has-no-directory"], + "a target nothing replays is the half of #86's derivation that a directory listing \ + cannot see on its own. Refusals were {refusals:?}." + ); + assert_eq!(refusals[0].subject, "a-target-with-no-seeds"); +} + +/// A target that does not name what it did with one of its seeds. +/// +/// It exists only to be handed to [`replay_every_seed`], and the input it fails +/// on is a literal rather than a shape, so the two proofs below differ by that +/// seed and by nothing else. +fn a_target_that_does_not_handle_one_seed(_target: &str, bytes: &[u8]) -> String { + assert!( + bytes != b"the input this target does not name", + "this target does not name what it did with the seed it was handed" + ); + format!("answered {} byte(s)", bytes.len()) +} + +#[test] +#[should_panic(expected = "this target does not name what it did with the seed it was handed")] +fn a_seed_the_target_does_not_handle_reddens_the_replay() { + let root = a_root_for("a-seed-the-target-does-not-handle"); + // Seeds are read in sorted order, so the ordinary one is replayed first and + // the failure cannot be read as a loop that never started. + a_seed(&root, "first-target", "a-handled-seed", b"one"); + a_seed( + &root, + "first-target", + "b-unhandled-seed", + b"the input this target does not name", + ); + replay_every_seed(&root, a_target_that_does_not_handle_one_seed); +} + +#[test] +fn the_same_root_without_that_seed_replays_green() { + // The one-change neighbour of the proof above. Without it that proof shows a + // target failing at everything, which says nothing about the seed. + let root = a_root_for("without-the-unhandled-seed"); + a_seed(&root, "first-target", "a-handled-seed", b"one"); + assert_eq!( + replay_every_seed(&root, a_target_that_does_not_handle_one_seed), + 1, + "the target fails on one literal and answers everything else, so a green run here is \ + what makes the red run beside it about the seed rather than about the target." + ); +} + +/// A target that returns, and returns nothing. +/// +/// The other way a target can fail to answer. A count of seeds replayed is +/// identical either way, which is why the count is not what the replay asserts +/// on. +fn a_target_that_answers_nothing(_target: &str, _bytes: &[u8]) -> String { + String::new() +} + +#[test] +#[should_panic(expected = "produced no answer at all")] +fn a_target_that_answers_nothing_reddens_the_replay() { + let root = a_root_for("a-target-that-answers-nothing"); + a_seed(&root, "first-target", "a-seed", b"one"); + replay_every_seed(&root, a_target_that_answers_nothing); +}