diff --git a/scripts/smite-evaluation.py b/scripts/smite-evaluation.py index 034a2184..3ad2a106 100755 --- a/scripts/smite-evaluation.py +++ b/scripts/smite-evaluation.py @@ -93,9 +93,13 @@ def validate_and_find_data(root_dir, config_a, config_b): f"[*] Configurations: Baseline (A) = {config_a}, Experimental (B) = {config_b}" ) - # Find targets - targets_a = set(os.listdir(path_a)) - targets_b = set(os.listdir(path_b)) + # Find targets (excluding files like latest_commit.txt) + targets_a = { + d for d in os.listdir(path_a) if os.path.isdir(os.path.join(path_a, d)) + } + targets_b = { + d for d in os.listdir(path_b) if os.path.isdir(os.path.join(path_b, d)) + } if targets_a != targets_b: raise ValueError( @@ -177,6 +181,15 @@ def parse_fuzzer_stats(filepath): return 0.0 +def get_commit_info(root_dir, config_name): + """Reads the saved commit metadata from the orchestrator.""" + commit_file = os.path.join(root_dir, config_name, "latest_commit.txt") + if os.path.exists(commit_file): + with open(commit_file, "r") as f: + return f.read().strip() + return "Unknown commit" + + def calculate_union_coverage(trial_paths): """ Performs bitwise-AND across all trial bitmaps to calculate multi-core union coverage. @@ -331,7 +344,9 @@ def generate_plots( plt.close() -def write_evaluation_report(report_path, df_results, config_a, config_b, targets): +def write_evaluation_report( + report_path, df_results, config_a, config_b, commit_a, commit_b, targets +): """Writes the final comprehensive Markdown evaluation report.""" # Order columns for clean Markdown display @@ -357,8 +372,12 @@ def write_evaluation_report(report_path, df_results, config_a, config_b, targets with open(report_path, "w") as f: f.write("# Fuzzing Evaluation Report\n\n") - f.write(f"**Configuration A (Baseline):** `{config_a}`\n") - f.write(f"**Configuration B (Experimental):** `{config_b}`\n\n") + f.write( + f"**Configuration A (Baseline):** `{config_a}` *(Commit: {commit_a})*\n" + ) + f.write( + f"**Configuration B (Experimental):** `{config_b}` *(Commit: {commit_b})*\n\n" + ) f.write("## 1. Summary Statistics\n\n") pd.set_option("display.float_format", lambda x: "%.3f" % x) @@ -458,6 +477,9 @@ def process_data( results_dir = os.path.join(root_dir, "results") os.makedirs(results_dir, exist_ok=True) + commit_a = get_commit_info(root_dir, config_a) + commit_b = get_commit_info(root_dir, config_b) + summary_stats = [] p_values_cov_raw = [] p_values_auc_raw = [] @@ -613,7 +635,9 @@ def process_data( df_results.to_csv(csv_path, index=False) report_path = os.path.join(results_dir, "evaluation_report.md") - write_evaluation_report(report_path, df_results, config_a, config_b, targets) + write_evaluation_report( + report_path, df_results, config_a, config_b, commit_a, commit_b, targets + ) print(f"\n[*] Evaluation complete. Results saved to {results_dir}") print(f" - Open {report_path} to interpret the campaign.") diff --git a/scripts/smite-orchestrator.py b/scripts/smite-orchestrator.py index f867cadd..232a48ee 100755 --- a/scripts/smite-orchestrator.py +++ b/scripts/smite-orchestrator.py @@ -47,6 +47,8 @@ --afl-dir AFL_DIR \ [--trials N | --trial-ids ID[,ID...]] \ [--timeout SECONDS] \ + [--exec-timeout MS] \ + [--hang-timeout MS] \ [--seed-dir SEED_DIR] Examples: @@ -145,6 +147,8 @@ class TrialConfig: smite_dir: Path afl_dir: Path timeout: int + exec_timeout: int + hang_timeout: int seed_dir: Path @property @@ -203,6 +207,8 @@ def build_afl_cmd(self) -> list[str]: POWER_SCHEDULE, "-V", str(self.timeout), + "-t", + str(self.exec_timeout), "--", str(self.sharedir), ] @@ -215,6 +221,7 @@ def build_afl_env(self) -> dict: "AFL_NO_UI": "1", "AFL_NO_COLOR": "1", "AFL_FORKSRV_INIT_TMOUT": "1800000", + "AFL_HANG_TMOUT": str(self.hang_timeout), } ) testcache = testcache_size_mb() @@ -836,6 +843,8 @@ def worker_thread(core: int, work: Queue, args, state: CampaignState, smite_dirs smite_dir=smite_dirs[label], afl_dir=args.afl_dir, timeout=args.timeout, + exec_timeout=args.exec_timeout, + hang_timeout=args.hang_timeout, seed_dir=args.seed_dir, ) @@ -865,6 +874,31 @@ def ensure_seed_dir(args, console: Console): ) +def save_commit_metadata(out_dir: Path, smite_dirs: dict, console: Console): + """Extracts the latest git commit hash and date, saving them to the output directory.""" + for label, smite_dir in smite_dirs.items(): + config_out = out_dir / label + config_out.mkdir(parents=True, exist_ok=True) + + try: + # %h = abbreviated hash, %cd = commit date + res = subprocess.run( + ["git", "log", "-1", "--format=%h (%cd)", "--date=short"], + cwd=smite_dir, + capture_output=True, + text=True, + check=True, + ) + commit_info = res.stdout.strip() + except Exception: + commit_info = "Unknown commit" + console.print( + f"[yellow]Warning: Could not get git info for '{label}' in {smite_dir}[/]" + ) + + (config_out / "latest_commit.txt").write_text(commit_info + "\n") + + def parse_args(): """Parse CLI args and resolve all filesystem paths to absolute up front.""" p = argparse.ArgumentParser( @@ -882,6 +916,15 @@ def parse_args(): help="Comma-separated trial numbers to run, e.g. '1,5,15'. Overrides --trials.", ) p.add_argument("--timeout", type=int, default=86400) + p.add_argument( + "--exec-timeout", type=int, default=2000, help="AFL++ exec timeout in ms (-t)" + ) + p.add_argument( + "--hang-timeout", + type=int, + default=4000, + help="AFL++ hang timeout in ms (AFL_HANG_TMOUT)", + ) p.add_argument("--seed-dir", type=Path) args = p.parse_args() @@ -913,6 +956,8 @@ def main(): except ValueError: sys.exit("ERROR: --configs must use 'label:smite_dir' format") + save_commit_metadata(args.out_dir, smite_dirs, console) + EnvironmentManager.validate(args.afl_dir, smite_dirs, console) EnvironmentManager.validate_paths(args.afl_dir, smite_dirs, console)