From fadcc5c68b7263c0018bd61c881d76cfb77291fd Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Fri, 31 Jul 2026 15:28:45 +0530 Subject: [PATCH 1/8] Support for XCalibur executor --- scripts/performance/argument_parser.py | 86 +++++++++++++++++++++++++ scripts/performance/setup_experiment.py | 58 ++++++++++++++++- scripts/performance/utils/executors.py | 37 +++++++++++ 3 files changed, 178 insertions(+), 3 deletions(-) diff --git a/scripts/performance/argument_parser.py b/scripts/performance/argument_parser.py index 2edd4a683e..59f07a023b 100644 --- a/scripts/performance/argument_parser.py +++ b/scripts/performance/argument_parser.py @@ -681,6 +681,92 @@ def parse_cli_args(): default=None, ) + # XCalibur + xcalibur_args = parser.add_argument_group("XCalibur arguments") + xcalibur_args.add_argument( + "--xcalibur_namespace", + type=str, + help="Kubernetes namespace for XCalibur WorkloadRun. When set, uses the XCalibur executor instead of Slurm.", + required=False, + ) + xcalibur_args.add_argument( + "--xcalibur_image_pull_secret", + type=str, + help="Kubernetes image pull secret name for pulling the container image.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_workdir_pvc", + type=str, + help="PVC name for syncing job workdir to the cluster before launch.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_workdir_pvc_path", + type=str, + help="Mount path for the workdir PVC inside the training pod.", + default="/nemo_run", + required=False, + ) + xcalibur_args.add_argument( + "--xcalibur_workdir_local_path", + type=str, + help="Local directory rsynced into the workdir PVC before launch.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_node_selector_json", + type=str, + help="JSON-encoded dict of node selector labels for targeting specific nodes.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_volumes_json", + type=str, + help="JSON-encoded list of Kubernetes Volume dicts.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_volume_mounts_json", + type=str, + help="JSON-encoded list of Kubernetes VolumeMount dicts.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_timeout_per_job", + type=str, + help="Per-job timeout passed to XCalibur orchestration (e.g. '24h').", + required=False, + default="24h", + ) + xcalibur_args.add_argument( + "--xcalibur_test_scale", + type=str, + help="XCalibur test scale: 'intra-node', 'intra-rack', or 'full-scale'.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_kubeconfig", + type=str, + help="Path to kubeconfig file for xcalctl/kubectl.", + required=False, + default=None, + ) + xcalibur_args.add_argument( + "--xcalibur_kube_context", + type=str, + help="Kubernetes context to use with xcalctl/kubectl.", + required=False, + default=None, + ) + # For performance performance_args = parser.add_argument_group("Performance arguments") performance_args.add_argument( diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index fe0288d015..9bc059fba8 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -36,6 +36,7 @@ _kubeflow_numa_binding_script, kubeflow_executor, slurm_executor, + xcalibur_executor, ) from utils.utils import configure_slurm_gpu_tuning, select_config_variant_interactive except (ImportError, ModuleNotFoundError): @@ -45,9 +46,15 @@ _kubeflow_numa_binding_script, kubeflow_executor, slurm_executor, + xcalibur_executor, ) from .utils.utils import configure_slurm_gpu_tuning, select_config_variant_interactive +try: + from nemo_run.core.execution.xcalibur import XCaliburExecutor as _XCaliburExecutor +except ImportError: + _XCaliburExecutor = None # type: ignore[assignment,misc] + try: import wandb @@ -101,7 +108,7 @@ def _is_launcher_only(flag: str) -> bool: "--enable_vboost", "--lock_gpu_freq", "--peak_mem_clk", - ) or flag.startswith("--kubeflow_") + ) or flag.startswith("--kubeflow_") or flag.startswith("--xcalibur_") filtered_args = [] skip_next = False @@ -540,6 +547,19 @@ def main( kubeflow_container_kwargs_json: Optional[str], kubeflow_labels_json: Optional[str], kubeflow_pod_annotations_json: Optional[str], + xcalibur_namespace: Optional[str] = None, + xcalibur_image_pull_secret: Optional[str] = None, + xcalibur_workdir_pvc: Optional[str] = None, + xcalibur_workdir_pvc_path: str = "/nemo_run", + xcalibur_workdir_local_path: Optional[str] = None, + xcalibur_node_selector_json: Optional[str] = None, + xcalibur_volumes_json: Optional[str] = None, + xcalibur_volume_mounts_json: Optional[str] = None, + xcalibur_timeout_per_job: str = "24h", + xcalibur_test_scale: Optional[str] = None, + xcalibur_kubeconfig: Optional[str] = None, + xcalibur_kube_context: Optional[str] = None, + deterministic: bool = False, config_variant: str | None = None, gres: Optional[str] = None, packager: str = "git", @@ -630,7 +650,7 @@ def main( # Kubeflow the trainer pod runs the image — which ships Megatron-Bridge at # /opt/Megatron-Bridge — and custom_mounts do not apply, so the launcher's # /tmp path does not exist in the pod; use the image's script path instead. - if kubeflow_namespace: + if kubeflow_namespace or xcalibur_namespace: in_container_script_dir = "/opt/Megatron-Bridge/scripts/performance" in_container_script_path = f"{in_container_script_dir}/{script_name}" else: @@ -686,6 +706,25 @@ def main( labels=json.loads(kubeflow_labels_json) if kubeflow_labels_json else None, pod_annotations=(json.loads(kubeflow_pod_annotations_json) if kubeflow_pod_annotations_json else None), ) + elif xcalibur_namespace is not None: + executor = xcalibur_executor( + namespace=xcalibur_namespace, + image=container_image, + num_nodes=num_gpus // gpus_per_node, + gpus_per_node=gpus_per_node, + image_pull_secret=xcalibur_image_pull_secret, + workdir_pvc=xcalibur_workdir_pvc, + workdir_pvc_path=xcalibur_workdir_pvc_path, + workdir_local_path=xcalibur_workdir_local_path, + node_selector=json.loads(xcalibur_node_selector_json) if xcalibur_node_selector_json else None, + volumes=json.loads(xcalibur_volumes_json) if xcalibur_volumes_json else None, + volume_mounts=json.loads(xcalibur_volume_mounts_json) if xcalibur_volume_mounts_json else None, + timeout_per_job=xcalibur_timeout_per_job, + test_scale=xcalibur_test_scale, + kubeconfig=xcalibur_kubeconfig, + kube_context=xcalibur_kube_context, + ) + executor.env_vars = custom_env_vars else: executor = slurm_executor( gpu=gpu, @@ -1008,7 +1047,7 @@ def main( task=args.task, compute_dtype=args.compute_dtype, gpu=args.gpu, - hf_token=args.hf_token, + hf_token=args.hf_token or os.environ.get('HF_TOKEN'), offline=args.offline, detach=args.detach, dryrun=args.dryrun, @@ -1087,6 +1126,19 @@ def main( kubeflow_container_kwargs_json=args.kubeflow_container_kwargs_json, kubeflow_labels_json=args.kubeflow_labels_json, kubeflow_pod_annotations_json=args.kubeflow_pod_annotations_json, + xcalibur_namespace=args.xcalibur_namespace, + xcalibur_image_pull_secret=args.xcalibur_image_pull_secret, + xcalibur_workdir_pvc=args.xcalibur_workdir_pvc, + xcalibur_workdir_pvc_path=args.xcalibur_workdir_pvc_path, + xcalibur_workdir_local_path=args.xcalibur_workdir_local_path, + xcalibur_node_selector_json=args.xcalibur_node_selector_json, + xcalibur_volumes_json=args.xcalibur_volumes_json, + xcalibur_volume_mounts_json=args.xcalibur_volume_mounts_json, + xcalibur_timeout_per_job=args.xcalibur_timeout_per_job, + xcalibur_test_scale=args.xcalibur_test_scale, + xcalibur_kubeconfig=args.xcalibur_kubeconfig, + xcalibur_kube_context=args.xcalibur_kube_context, + deterministic=args.deterministic, config_variant=config_variant, gres=args.gres, packager=args.packager, diff --git a/scripts/performance/utils/executors.py b/scripts/performance/utils/executors.py index 072bf3b56d..6a76663386 100644 --- a/scripts/performance/utils/executors.py +++ b/scripts/performance/utils/executors.py @@ -21,6 +21,7 @@ import nemo_run as run from nemo_run.config import get_nemorun_home, set_nemorun_home from nemo_run.core.execution.launcher import SlurmTemplate +from nemo_run.core.execution.xcalibur import XCaliburExecutor DEFAULT_NEMO_CACHE_HOME = Path.home() / ".cache" / "nemo" @@ -361,3 +362,39 @@ def kubeflow_executor( packager=run.GitArchivePackager(include_submodules=True), ) return executor + + +def xcalibur_executor( + namespace: str, + image: str, + num_nodes: int, + gpus_per_node: int, + image_pull_secret: Optional[str] = None, + workdir_pvc: Optional[str] = None, + workdir_pvc_path: str = "/nemo_run", + workdir_local_path: Optional[str] = None, + node_selector: Optional[dict] = None, + volumes: Optional[list] = None, + volume_mounts: Optional[list] = None, + timeout_per_job: str = "24h", + test_scale: Optional[str] = None, + kubeconfig: Optional[str] = None, + kube_context: Optional[str] = None, +) -> XCaliburExecutor: + return XCaliburExecutor( + namespace=namespace, + container_image=image, + num_nodes=num_nodes, + gpus_per_node=gpus_per_node, + image_pull_secret=image_pull_secret, + node_selector=node_selector or {}, + workdir_pvc=workdir_pvc, + workdir_pvc_path=workdir_pvc_path, + workdir_local_path=workdir_local_path, + volumes=volumes or [], + volume_mounts=volume_mounts or [], + timeout_per_job=timeout_per_job, + test_scale=test_scale, + kubeconfig=kubeconfig, + kube_context=kube_context, + ) From c7da4d081eaace51e575e54b2354ffd86efd6e5b Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Fri, 31 Jul 2026 16:06:00 +0530 Subject: [PATCH 2/8] Support for XCalibur executor --- scripts/performance/setup_experiment.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index 9bc059fba8..a54c2a422f 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -724,7 +724,14 @@ def main( kubeconfig=xcalibur_kubeconfig, kube_context=xcalibur_kube_context, ) - executor.env_vars = custom_env_vars + xcal_env = custom_env_vars.copy() + if hf_token: + # Always allow the pod to reach HF to download gated model files + # (tokenizer configs, etc.) — the pod has no access to the host + # HF cache so offline mode must not be forced here even when + # --offline was passed for the launcher-side setup. + xcal_env.update({"HF_TOKEN": hf_token, "HF_HUB_OFFLINE": "0", "TRANSFORMERS_OFFLINE": "0"}) + executor.env_vars = xcal_env else: executor = slurm_executor( gpu=gpu, From 113962e714d638f28d328da848e702c2088912f9 Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Fri, 31 Jul 2026 16:30:56 +0530 Subject: [PATCH 3/8] Support for XCalibur executor --- scripts/performance/setup_experiment.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index a54c2a422f..22a561c0ee 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -93,10 +93,14 @@ def _filter_run_script_args(argv: List[str]) -> List[str]: carry JSON values whose ``{}`` / ``[]`` are brace/glob-expanded by the shell in the generated launch command, corrupting argv and leaking tokens into the training entrypoint's Hydra override parser. + * ``--offline`` — controls HF Hub access on the launcher node only; offline + behaviour in the rank-local script is governed by the ``HF_HUB_OFFLINE`` + environment variable, not by this flag. - All of these take a value, passed either as ``--flag value`` (two tokens) or - ``--flag=value`` (one token). + Value-taking flags are passed as ``--flag value`` or ``--flag=value``. + Boolean flags (no following value) are listed in ``_LAUNCHER_ONLY_BOOL``. """ + _LAUNCHER_ONLY_BOOL = {"--offline", "--dryrun"} def _is_launcher_only(flag: str) -> bool: return flag in ( @@ -117,6 +121,8 @@ def _is_launcher_only(flag: str) -> bool: if skip_next: skip_next = False continue + if arg in _LAUNCHER_ONLY_BOOL: + continue if _is_launcher_only(arg.split("=", 1)[0]): skip_next = "=" not in arg continue From 74c7cd9ca0792b1bee60a57aa6eec215f534ef66 Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Fri, 31 Jul 2026 23:38:51 +0530 Subject: [PATCH 4/8] Fixed the container entrypoint to use run_script.py instead of bootstrap.py --- scripts/performance/setup_experiment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index 22a561c0ee..0c8de43148 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -613,7 +613,8 @@ def main( if export_nsys_sqlite and not enable_nsys: logger.warning("--export_nsys_sqlite was set without --enable_nsys; no Nsys SQLite export will be generated.") - script_name = ENTRYPOINT_BOOTSTRAP + # XCalibur uses run_script.py directly; all other executors use bootstrap.py. + script_name = "run_script.py" if xcalibur_namespace else ENTRYPOINT_BOOTSTRAP # Keep the historical W&B-name behavior for CI. The lightweight fallback # deliberately avoids resolving a recipe: effective parallelism, batches, # and process environment are finalized by bootstrap.py in the container. From e6437697a332d2462196999b5a87235393b4372f Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Tue, 4 Aug 2026 17:15:37 +0530 Subject: [PATCH 5/8] Made the xcalibur_executor import as conditional --- scripts/performance/setup_experiment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index 0c8de43148..4dd0ed2554 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -36,7 +36,6 @@ _kubeflow_numa_binding_script, kubeflow_executor, slurm_executor, - xcalibur_executor, ) from utils.utils import configure_slurm_gpu_tuning, select_config_variant_interactive except (ImportError, ModuleNotFoundError): @@ -46,7 +45,6 @@ _kubeflow_numa_binding_script, kubeflow_executor, slurm_executor, - xcalibur_executor, ) from .utils.utils import configure_slurm_gpu_tuning, select_config_variant_interactive @@ -714,6 +712,10 @@ def main( pod_annotations=(json.loads(kubeflow_pod_annotations_json) if kubeflow_pod_annotations_json else None), ) elif xcalibur_namespace is not None: + try: + from utils.executors import xcalibur_executor + except ImportError: + from .utils.executors import xcalibur_executor executor = xcalibur_executor( namespace=xcalibur_namespace, image=container_image, From 4698698f715f40999fc0fa6e66a99186768e79f3 Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Thu, 6 Aug 2026 18:46:49 +0530 Subject: [PATCH 6/8] Fix code for review comments --- scripts/performance/setup_experiment.py | 9 ++------- scripts/performance/utils/executors.py | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index 4dd0ed2554..bf7945bd40 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -48,11 +48,6 @@ ) from .utils.utils import configure_slurm_gpu_tuning, select_config_variant_interactive -try: - from nemo_run.core.execution.xcalibur import XCaliburExecutor as _XCaliburExecutor -except ImportError: - _XCaliburExecutor = None # type: ignore[assignment,misc] - try: import wandb @@ -636,7 +631,7 @@ def main( # runs. Creating the dir from the launcher would either fail (PVC # not present) or create a useless dir on the launcher's local FS. # Let the trainer container create its own dirs on first write. - if kubeflow_namespace is None: + if kubeflow_namespace is None and xcalibur_namespace is None: save_dir_path.mkdir(parents=True, exist_ok=True) save_dir_mount = f"{save_dir_path}:{save_dir_path}" if save_dir_mount not in custom_mounts: @@ -719,7 +714,7 @@ def main( executor = xcalibur_executor( namespace=xcalibur_namespace, image=container_image, - num_nodes=num_gpus // gpus_per_node, + num_nodes=-(num_gpus // -gpus_per_node), gpus_per_node=gpus_per_node, image_pull_secret=xcalibur_image_pull_secret, workdir_pvc=xcalibur_workdir_pvc, diff --git a/scripts/performance/utils/executors.py b/scripts/performance/utils/executors.py index 6a76663386..a733c71955 100644 --- a/scripts/performance/utils/executors.py +++ b/scripts/performance/utils/executors.py @@ -21,7 +21,6 @@ import nemo_run as run from nemo_run.config import get_nemorun_home, set_nemorun_home from nemo_run.core.execution.launcher import SlurmTemplate -from nemo_run.core.execution.xcalibur import XCaliburExecutor DEFAULT_NEMO_CACHE_HOME = Path.home() / ".cache" / "nemo" @@ -380,7 +379,8 @@ def xcalibur_executor( test_scale: Optional[str] = None, kubeconfig: Optional[str] = None, kube_context: Optional[str] = None, -) -> XCaliburExecutor: +): + from nemo_run.core.execution.xcalibur import XCaliburExecutor return XCaliburExecutor( namespace=namespace, container_image=image, From 0809ad9c38f065817b6f18fb179ac9755c6d64c2 Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Tue, 11 Aug 2026 17:38:02 +0530 Subject: [PATCH 7/8] Support for KAI scheduler --- scripts/performance/argument_parser.py | 7 +++++++ scripts/performance/perf_plugins.py | 7 +++++++ scripts/performance/setup_experiment.py | 3 +++ scripts/performance/utils/executors.py | 2 ++ 4 files changed, 19 insertions(+) diff --git a/scripts/performance/argument_parser.py b/scripts/performance/argument_parser.py index 59f07a023b..d2a2fc19b2 100644 --- a/scripts/performance/argument_parser.py +++ b/scripts/performance/argument_parser.py @@ -766,6 +766,13 @@ def parse_cli_args(): required=False, default=None, ) + xcalibur_args.add_argument( + "--xcalibur_gang_scheduler_name", + type=str, + help="Gang scheduler name for the WorkloadRun (e.g. 'kai-scheduler').", + required=False, + default=None, + ) # For performance performance_args = parser.add_argument_group("Performance arguments") diff --git a/scripts/performance/perf_plugins.py b/scripts/performance/perf_plugins.py index 2c0f12451b..68e75322fc 100644 --- a/scripts/performance/perf_plugins.py +++ b/scripts/performance/perf_plugins.py @@ -33,6 +33,10 @@ import nemo_run as run from nemo_run import Plugin, Script, SlurmExecutor +try: + from nemo_run.core.execution.xcalibur import XCaliburExecutor +except ImportError: + XCaliburExecutor = None # type: ignore[assignment,misc] logger: logging.Logger = logging.getLogger(__name__) @@ -185,6 +189,9 @@ def setup(self, task: Union["run.Partial", "run.Script"], executor: "run.Executo if isinstance(executor, SlurmExecutor): # NOTE: DO NOT change to f-string, `%q{}` is Slurm placeholder launcher.nsys_filename = "profile_%p_%q{SLURM_JOB_ID}_node%q{SLURM_NODEID}_rank%q{SLURM_PROCID}" + elif XCaliburExecutor is not None and isinstance(executor, XCaliburExecutor): + # XCalibur pods use PET_* env vars for distributed rank info. + launcher.nsys_filename = "profile_node${PET_NODE_RANK}_rank%p" if self.nsys_gpu_metrics: if hasattr(launcher, "nsys_gpu_metrics"): diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index bf7945bd40..ebc58ae7d4 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -558,6 +558,7 @@ def main( xcalibur_test_scale: Optional[str] = None, xcalibur_kubeconfig: Optional[str] = None, xcalibur_kube_context: Optional[str] = None, + xcalibur_gang_scheduler_name: Optional[str] = None, deterministic: bool = False, config_variant: str | None = None, gres: Optional[str] = None, @@ -727,6 +728,7 @@ def main( test_scale=xcalibur_test_scale, kubeconfig=xcalibur_kubeconfig, kube_context=xcalibur_kube_context, + gang_scheduler_name=xcalibur_gang_scheduler_name, ) xcal_env = custom_env_vars.copy() if hf_token: @@ -1149,6 +1151,7 @@ def main( xcalibur_test_scale=args.xcalibur_test_scale, xcalibur_kubeconfig=args.xcalibur_kubeconfig, xcalibur_kube_context=args.xcalibur_kube_context, + xcalibur_gang_scheduler_name=args.xcalibur_gang_scheduler_name, deterministic=args.deterministic, config_variant=config_variant, gres=args.gres, diff --git a/scripts/performance/utils/executors.py b/scripts/performance/utils/executors.py index a733c71955..e748e3479d 100644 --- a/scripts/performance/utils/executors.py +++ b/scripts/performance/utils/executors.py @@ -379,6 +379,7 @@ def xcalibur_executor( test_scale: Optional[str] = None, kubeconfig: Optional[str] = None, kube_context: Optional[str] = None, + gang_scheduler_name: Optional[str] = None, ): from nemo_run.core.execution.xcalibur import XCaliburExecutor return XCaliburExecutor( @@ -397,4 +398,5 @@ def xcalibur_executor( test_scale=test_scale, kubeconfig=kubeconfig, kube_context=kube_context, + gang_scheduler_name=gang_scheduler_name, ) From a1dbbb4d6b8a2a0f4dc60a5398b6e76e0d305c4e Mon Sep 17 00:00:00 2001 From: Smitha Jayaram Date: Thu, 13 Aug 2026 11:00:46 +0530 Subject: [PATCH 8/8] Fix detached runs for XCalibur jobs --- scripts/performance/setup_experiment.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index ebc58ae7d4..28770db062 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -858,15 +858,17 @@ def main( logger.info("dryrun requested: exiting") return - job_dir, job_status = get_job_dir_and_status_from_run(exp_name) - - terminal_failure = job_status not in ["SUCCEEDED", "SUBMITTED", "PENDING", "RUNNING"] - if detach: + # For detached runs (e.g. XCalibur), skip status polling — the + # caller (llmb-run) polls for completion via xcalctl. is_finished_experiment = True is_testing_passed = True break + job_dir, job_status = get_job_dir_and_status_from_run(exp_name) + + terminal_failure = job_status not in ["SUCCEEDED", "SUBMITTED", "PENDING", "RUNNING"] + log_file_paths = list(Path(f"{job_dir}").glob("log*.out")) ensure_logs_where_written(log_file_paths) @@ -1062,7 +1064,8 @@ def main( gpu=args.gpu, hf_token=args.hf_token or os.environ.get('HF_TOKEN'), offline=args.offline, - detach=args.detach, + # Force detach for XCalibur — llmb-run polls for completion via xcalctl + detach=True if args.xcalibur_namespace else args.detach, dryrun=args.dryrun, enable_vboost=args.enable_vboost, lock_gpu_freq=args.lock_gpu_freq,