From 9147ebab5dbc119f4d26c862e92f6c36254a25df Mon Sep 17 00:00:00 2001 From: Chandra Pratap Date: Wed, 12 Aug 2026 17:55:12 +0000 Subject: [PATCH 1/2] scripts/smite-orchestrator: Add explicit execution and hang timeouts The IR programs take longer to execute than standard AFL++ inputs, so using the default AFL++ values causes false positive hangs and timeouts. Add explicit timeout durations to mitigate this. --- scripts/smite-orchestrator.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/smite-orchestrator.py b/scripts/smite-orchestrator.py index f867cadd..e5f0ed04 100755 --- a/scripts/smite-orchestrator.py +++ b/scripts/smite-orchestrator.py @@ -47,6 +47,8 @@ --afl-dir AFL_DIR \ [--trials N | --trial-ids ID[,ID...]] \ [--timeout SECONDS] \ + [--exec-timeout MS] \ + [--hang-timeout MS] \ [--seed-dir SEED_DIR] Examples: @@ -145,6 +147,8 @@ class TrialConfig: smite_dir: Path afl_dir: Path timeout: int + exec_timeout: int + hang_timeout: int seed_dir: Path @property @@ -203,6 +207,8 @@ def build_afl_cmd(self) -> list[str]: POWER_SCHEDULE, "-V", str(self.timeout), + "-t", + str(self.exec_timeout), "--", str(self.sharedir), ] @@ -215,6 +221,7 @@ def build_afl_env(self) -> dict: "AFL_NO_UI": "1", "AFL_NO_COLOR": "1", "AFL_FORKSRV_INIT_TMOUT": "1800000", + "AFL_HANG_TMOUT": str(self.hang_timeout), } ) testcache = testcache_size_mb() @@ -836,6 +843,8 @@ def worker_thread(core: int, work: Queue, args, state: CampaignState, smite_dirs smite_dir=smite_dirs[label], afl_dir=args.afl_dir, timeout=args.timeout, + exec_timeout=args.exec_timeout, + hang_timeout=args.hang_timeout, seed_dir=args.seed_dir, ) @@ -882,6 +891,15 @@ def parse_args(): help="Comma-separated trial numbers to run, e.g. '1,5,15'. Overrides --trials.", ) p.add_argument("--timeout", type=int, default=86400) + p.add_argument( + "--exec-timeout", type=int, default=2000, help="AFL++ exec timeout in ms (-t)" + ) + p.add_argument( + "--hang-timeout", + type=int, + default=4000, + help="AFL++ hang timeout in ms (AFL_HANG_TMOUT)", + ) p.add_argument("--seed-dir", type=Path) args = p.parse_args() From 9a99b7db7b642e9c4337aca060bf16275cd6eb5b Mon Sep 17 00:00:00 2001 From: Chandra Pratap Date: Wed, 12 Aug 2026 18:20:06 +0000 Subject: [PATCH 2/2] scripts/smite-evaluation: Display the latest commit info in evaluation reports For each configuration, record the latest Git commit metadata corresponding to that configuration and display it in the final report. This ensures evaluation data lineage and reproducibility. --- scripts/smite-evaluation.py | 38 ++++++++++++++++++++++++++++------- scripts/smite-orchestrator.py | 27 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/scripts/smite-evaluation.py b/scripts/smite-evaluation.py index 034a2184..3ad2a106 100755 --- a/scripts/smite-evaluation.py +++ b/scripts/smite-evaluation.py @@ -93,9 +93,13 @@ def validate_and_find_data(root_dir, config_a, config_b): f"[*] Configurations: Baseline (A) = {config_a}, Experimental (B) = {config_b}" ) - # Find targets - targets_a = set(os.listdir(path_a)) - targets_b = set(os.listdir(path_b)) + # Find targets (excluding files like latest_commit.txt) + targets_a = { + d for d in os.listdir(path_a) if os.path.isdir(os.path.join(path_a, d)) + } + targets_b = { + d for d in os.listdir(path_b) if os.path.isdir(os.path.join(path_b, d)) + } if targets_a != targets_b: raise ValueError( @@ -177,6 +181,15 @@ def parse_fuzzer_stats(filepath): return 0.0 +def get_commit_info(root_dir, config_name): + """Reads the saved commit metadata from the orchestrator.""" + commit_file = os.path.join(root_dir, config_name, "latest_commit.txt") + if os.path.exists(commit_file): + with open(commit_file, "r") as f: + return f.read().strip() + return "Unknown commit" + + def calculate_union_coverage(trial_paths): """ Performs bitwise-AND across all trial bitmaps to calculate multi-core union coverage. @@ -331,7 +344,9 @@ def generate_plots( plt.close() -def write_evaluation_report(report_path, df_results, config_a, config_b, targets): +def write_evaluation_report( + report_path, df_results, config_a, config_b, commit_a, commit_b, targets +): """Writes the final comprehensive Markdown evaluation report.""" # Order columns for clean Markdown display @@ -357,8 +372,12 @@ def write_evaluation_report(report_path, df_results, config_a, config_b, targets with open(report_path, "w") as f: f.write("# Fuzzing Evaluation Report\n\n") - f.write(f"**Configuration A (Baseline):** `{config_a}`\n") - f.write(f"**Configuration B (Experimental):** `{config_b}`\n\n") + f.write( + f"**Configuration A (Baseline):** `{config_a}` *(Commit: {commit_a})*\n" + ) + f.write( + f"**Configuration B (Experimental):** `{config_b}` *(Commit: {commit_b})*\n\n" + ) f.write("## 1. Summary Statistics\n\n") pd.set_option("display.float_format", lambda x: "%.3f" % x) @@ -458,6 +477,9 @@ def process_data( results_dir = os.path.join(root_dir, "results") os.makedirs(results_dir, exist_ok=True) + commit_a = get_commit_info(root_dir, config_a) + commit_b = get_commit_info(root_dir, config_b) + summary_stats = [] p_values_cov_raw = [] p_values_auc_raw = [] @@ -613,7 +635,9 @@ def process_data( df_results.to_csv(csv_path, index=False) report_path = os.path.join(results_dir, "evaluation_report.md") - write_evaluation_report(report_path, df_results, config_a, config_b, targets) + write_evaluation_report( + report_path, df_results, config_a, config_b, commit_a, commit_b, targets + ) print(f"\n[*] Evaluation complete. Results saved to {results_dir}") print(f" - Open {report_path} to interpret the campaign.") diff --git a/scripts/smite-orchestrator.py b/scripts/smite-orchestrator.py index e5f0ed04..232a48ee 100755 --- a/scripts/smite-orchestrator.py +++ b/scripts/smite-orchestrator.py @@ -874,6 +874,31 @@ def ensure_seed_dir(args, console: Console): ) +def save_commit_metadata(out_dir: Path, smite_dirs: dict, console: Console): + """Extracts the latest git commit hash and date, saving them to the output directory.""" + for label, smite_dir in smite_dirs.items(): + config_out = out_dir / label + config_out.mkdir(parents=True, exist_ok=True) + + try: + # %h = abbreviated hash, %cd = commit date + res = subprocess.run( + ["git", "log", "-1", "--format=%h (%cd)", "--date=short"], + cwd=smite_dir, + capture_output=True, + text=True, + check=True, + ) + commit_info = res.stdout.strip() + except Exception: + commit_info = "Unknown commit" + console.print( + f"[yellow]Warning: Could not get git info for '{label}' in {smite_dir}[/]" + ) + + (config_out / "latest_commit.txt").write_text(commit_info + "\n") + + def parse_args(): """Parse CLI args and resolve all filesystem paths to absolute up front.""" p = argparse.ArgumentParser( @@ -931,6 +956,8 @@ def main(): except ValueError: sys.exit("ERROR: --configs must use 'label:smite_dir' format") + save_commit_metadata(args.out_dir, smite_dirs, console) + EnvironmentManager.validate(args.afl_dir, smite_dirs, console) EnvironmentManager.validate_paths(args.afl_dir, smite_dirs, console)