From 4610caef311f6d42eab3911605804fa9656e74af Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Fri, 10 Jul 2026 16:40:27 -0700 Subject: [PATCH 1/2] remove fom when not converging --- ScaFFold/worker.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ScaFFold/worker.py b/ScaFFold/worker.py index 61b1c41..654643d 100644 --- a/ScaFFold/worker.py +++ b/ScaFFold/worker.py @@ -284,8 +284,6 @@ def main(kwargs_dict: dict = {}): outfile_path = trainer.outfile_path train_data = np.genfromtxt(outfile_path, dtype=float, delimiter=",", names=True) total_train_time = train_data["epoch_duration"].sum() - fom = 1.0 / total_train_time - adiak_value("FOM", fom) if "total_optimizer_steps" in train_data.dtype.names: optimizer_steps = np.atleast_1d(train_data["total_optimizer_steps"]) total_optimizer_steps = int(optimizer_steps[-1]) @@ -296,16 +294,21 @@ def main(kwargs_dict: dict = {}): else: total_optimizer_steps = int(getattr(trainer, "total_optimizer_steps", 0)) adiak_value("total_optimizer_steps", total_optimizer_steps) - log.info( - f"FOM = {fom} (1 / total_train_time={total_train_time:.6f} seconds). " - f"This FOM is specific to problem_scale={config.problem_scale}, " - f"target_dice={config.target_dice}, seed={config.seed}, " - f"total_optimizer_steps={total_optimizer_steps}." - ) epochs = np.atleast_1d(train_data["epoch"]) total_epochs = int(epochs[-1]) if config.epochs == -1: extra_msg = f"Trained to >= {config.target_dice} validation dice score in {total_train_time:.2f} seconds, {total_epochs} epochs, {total_optimizer_steps} optimizer steps." + epochs = np.atleast_1d(train_data["epoch"]) + total_epochs = int(epochs[-1]) + if config.epochs == -1: + fom = 1.0 / total_train_time + adiak_value("FOM", fom) + log.info( + f"FOM = {fom} (1 / total_train_time={total_train_time:.6f} seconds). " + f"This FOM is specific to problem_scale={config.problem_scale}, " + f"target_dice={config.target_dice}, seed={config.seed}." + ) + extra_msg = f"Trained to >= {config.target_dice} validation dice score in {total_train_time:.2f} seconds, {total_epochs} epochs." else: extra_msg = f"Completed in {total_train_time:.2f} seconds, {total_epochs} epochs, {total_optimizer_steps} optimizer steps." From 5e0c39f133f4ca0080ca51558f8f879217653961 Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Thu, 16 Jul 2026 12:58:25 -0700 Subject: [PATCH 2/2] Fix artifact --- ScaFFold/worker.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ScaFFold/worker.py b/ScaFFold/worker.py index 654643d..a51c51e 100644 --- a/ScaFFold/worker.py +++ b/ScaFFold/worker.py @@ -296,10 +296,6 @@ def main(kwargs_dict: dict = {}): adiak_value("total_optimizer_steps", total_optimizer_steps) epochs = np.atleast_1d(train_data["epoch"]) total_epochs = int(epochs[-1]) - if config.epochs == -1: - extra_msg = f"Trained to >= {config.target_dice} validation dice score in {total_train_time:.2f} seconds, {total_epochs} epochs, {total_optimizer_steps} optimizer steps." - epochs = np.atleast_1d(train_data["epoch"]) - total_epochs = int(epochs[-1]) if config.epochs == -1: fom = 1.0 / total_train_time adiak_value("FOM", fom) @@ -308,7 +304,7 @@ def main(kwargs_dict: dict = {}): f"This FOM is specific to problem_scale={config.problem_scale}, " f"target_dice={config.target_dice}, seed={config.seed}." ) - extra_msg = f"Trained to >= {config.target_dice} validation dice score in {total_train_time:.2f} seconds, {total_epochs} epochs." + extra_msg = f"Trained to >= {config.target_dice} validation dice score in {total_train_time:.2f} seconds, {total_epochs} epochs, {total_optimizer_steps} optimizer steps." else: extra_msg = f"Completed in {total_train_time:.2f} seconds, {total_epochs} epochs, {total_optimizer_steps} optimizer steps."