PyTorch Full Inductor Unit Tests

[naromero77amd]

gfx1250 PyTorch Inductor Merged Outcome

Updated: 2026-08-05

Important

This is the latest exact-node merge, not a fully completed targeted rerun.
All seven high-MISS full suites completed, but the targeted phase completed
27 / 1,848 nodes and left 1,821 reruns pending.

Overall Result

• Discovered nodes: 22,962
• Explicit PASSED outcomes: 17,580
• Pass rate (passed + skipped + xfailed) / total: 21,096 / 22,962 = 91.87%
• Failing outcomes (failed + error + missed) / total: 1,866 / 22,962 = 8.13%
• Merge rule: the latest terminal result overrides by exact pytest node ID.

Suite Summary

Suite	Total	Passed	Skipped	Xfailed	Failed	Error	Timed Out	Missed
test_alignment.py	12	12	0	0	0	0	0	0
test_analysis.py	28	28	0	0	0	0	0	0
test_aot_inductor.py	950	455	480	3	7	5	0	0
test_aot_inductor_arrayref.py	314	128	182	3	1	0	0	0
test_aot_inductor_custom_ops.py	35	35	0	0	0	0	0	0
test_aot_inductor_package.py	88	59	25	0	4	0	0	0
test_async_compile.py	8	8	0	0	0	0	0	0
test_augmented_graph_helper.py	20	20	0	0	0	0	0	0
test_auto_chunker.py	9	9	0	0	0	0	0	0
test_auto_functionalize.py	39	38	1	0	0	0	0	0
test_benchmark_fusion.py	16	10	4	0	0	2	0	0
test_benchmarking.py	14	8	0	6	0	0	0	0
test_best_config.py	1	1	0	0	0	0	0	0
test_binary_folding.py	6	5	1	0	0	0	0	0
test_block_analysis.py	10	10	0	0	0	0	0	0
test_cache.py	725	725	0	0	0	0	0	0
test_caching.py	212	212	0	0	0	0	0	0
test_ck_backend.py	34	0	34	0	0	0	0	0
test_codecache.py	257	193	59	1	3	1	0	0
test_codegen_triton.py	1	1	0	0	0	0	0	0
test_collective_autotuning.py	2	0	2	0	0	0	0	0
test_combo_kernels.py	77	70	5	0	0	2	0	0
test_compile.py	10	10	0	0	0	0	0	0
test_compile_subprocess.py	936	868	40	0	8	20	0	0
test_compile_worker.py	16	16	0	0	0	0	0	0
test_compiled_autograd.py	875	865	4	5	1	0	0	0
test_compiled_optimizers.py	682	677	3	0	0	2	0	0
test_config.py	14	14	0	0	0	0	0	0
test_control_deps.py	4	4	0	0	0	0	0	0
test_control_flow.py	741	739	2	0	0	0	0	0
test_cooperative_reductions.py	163	160	0	0	0	3	0	0
test_coordinate_descent_tuner.py	5	5	0	0	0	0	0	0
test_cpp_wrapper_hipify.py	3	3	0	0	0	0	0	0
test_cpu_repro.py	752	747	5	0	0	0	0	0
test_cuda_repro.py	98	78	3	1	3	13	0	0
test_cudacodecache.py	3	0	0	0	3	0	0	0
test_cudagraph_trees.py	189	180	8	0	0	1	0	0
test_cudagraph_trees_expandable_segments.py	158	151	6	0	0	1	0	0
test_custom_lowering.py	6	4	1	0	1	0	0	0
test_custom_op_autotune.py	4	3	0	0	1	0	0	0
test_custom_partitioner_fn.py	1	1	0	0	0	0	0	0
test_custom_post_grad_passes.py	6	6	0	0	0	0	0	0
test_cutedsl_grouped_mm.py	24	0	24	0	0	0	0	0
test_cutedsl_template.py	13	0	13	0	0	0	0	0
test_cutlass_backend.py	181	0	181	0	0	0	0	0
test_cutlass_evt.py	8	0	8	0	0	0	0	0
test_debug_trace.py	3	3	0	0	0	0	0	0
test_decompose_mem_bound_mm.py	37	35	2	0	0	0	0	0
test_dependencies.py	5	5	0	0	0	0	0	0
test_deterministic.py	32	7	0	0	24	1	0	0
test_device_assert.py	8	8	0	0	0	0	0	0
test_distributed_patterns.py	20	20	0	0	0	0	0	0
test_efficient_conv_bn_eval.py	2	1	0	0	0	1	0	0
test_exc_lowering_stack_trace.py	2	2	0	0	0	0	0	0
test_extension_backend.py	1	0	0	0	1	0	0	0
test_external_callables.py	3	2	0	0	0	1	0	0
test_flex_attention.py	786	750	27	1	2	6	0	0
test_flex_decoding.py	557	554	1	2	0	0	0	0
test_flex_flash.py	210	6	204	0	0	0	0	0
test_foreach.py	607	581	26	0	0	0	0	0
test_fp8.py	202	98	55	0	44	5	0	0
test_fused_attention.py	116	114	2	0	0	0	0	0
test_fusion_regions.py	6	6	0	0	0	0	0	0
test_fuzzer.py	11	10	1	0	0	0	0	0
test_fx_fusion.py	4	4	0	0	0	0	0	0
test_fxir_backend.py	75	74	0	0	0	1	0	0
test_gpu_cpp_wrapper.py	297	289	1	0	3	4	0	0
test_gpu_select_algorithm.py	58	58	0	0	0	0	0	0
test_graph_transform_observer.py	1	1	0	0	0	0	0	0
test_group_batch_fusion.py	13	11	2	0	0	0	0	0
test_halide.py	4	0	4	0	0	0	0	0
test_helion_kernels.py	2	0	2	0	0	0	0	0
test_indexing.py	22	22	0	0	0	0	0	0
test_inductor_annotations.py	2	2	0	0	0	0	0	0
test_inductor_freezing.py	48	33	1	0	1	2	0	11
test_inductor_scheduler.py	10	10	0	0	0	0	0	0
test_inductor_utils.py	2	2	0	0	0	0	0	0
test_inplace_padding.py	9	9	0	0	0	0	0	0
test_inplacing_pass.py	23	23	0	0	0	0	0	0
test_kernel_benchmark.py	19	16	0	0	0	3	0	0
test_kernel_optimization.py	1	1	0	0	0	0	0	0
test_lookup_table.py	37	29	4	0	0	0	0	4
test_loop_ordering.py	70	68	0	0	0	2	0	0
test_max_autotune.py	472	182	173	0	17	95	0	5
test_max_autotune_blackwell.py	106	2	104	0	0	0	0	0
test_mem_estimation.py	4	4	0	0	0	0	0	0
test_memory.py	8	8	0	0	0	0	0	0
test_memory_planning.py	4	4	0	0	0	0	0	0
test_metrics.py	6	6	0	0	0	0	0	0
test_minifier.py	14	14	0	0	0	0	0	0
test_minifier_isolate.py	2	1	1	0	0	0	0	0
test_minifier_utils.py	3	3	0	0	0	0	0	0
test_mix_order_reduction.py	485	248	219	0	16	1	0	1
test_mkldnn_pattern_matcher.py	20	16	4	0	0	0	0	0
test_mmdecomp.py	28	28	0	0	0	0	0	0
test_move_constructors_to_gpu.py	7	6	1	0	0	0	0	0
test_mps_basic.py	46	24	0	0	20	2	0	0
test_multi_kernel.py	19	16	2	0	0	1	0	0
test_native_matmul.py	14	2	0	0	4	8	0	0
test_needs_exact_strides.py	2	2	0	0	0	0	0	0
test_nv_universal_gemm.py	23	3	20	0	0	0	0	0
test_online_softmax.py	31	31	0	0	0	0	0	0
test_op_completeness.py	5	4	1	0	0	0	0	0
test_op_dtype_prop.py	581	581	0	0	0	0	0	0
test_ordered_set.py	401	386	15	0	0	0	0	0
test_pad_mm.py	19	8	1	0	0	10	0	0
test_padding.py	55	46	9	0	0	0	0	0
test_pattern_matcher.py	63	56	0	0	0	7	0	0
test_perf.py	66	66	0	0	0	0	0	0
test_profiler.py	8	8	0	0	0	0	0	0
test_provenance_tracing.py	16	16	0	0	0	0	0	0
test_quantization.py	2	2	0	0	0	0	0	0
test_remote_cache.py	3	3	0	0	0	0	0	0
test_scatter_optimization.py	8	8	0	0	0	0	0	0
test_segmented_tree.py	12	12	0	0	0	0	0	0
test_select_algorithm.py	27	14	1	0	2	9	0	1
test_selective_lowering.py	2	2	0	0	0	0	0	0
test_smoke.py	3	3	0	0	0	0	0	0
test_snode_runtime.py	22	22	0	0	0	0	0	0
test_split_cat_fx_aten_passes.py	5	5	0	0	0	0	0	0
test_split_cat_fx_passes.py	11	11	0	0	0	0	0	0
test_static_triton_launcher.py	17	17	0	0	0	0	0	0
test_subgraph_choice.py	2	2	0	0	0	0	0	0
test_template_heuristics_registry.py	7	7	0	0	0	0	0	0
test_torchbind.py	16	16	0	0	0	0	0	0
test_torchinductor.py	1041	962	42	0	3	22	0	12
test_torchinductor_codegen_config_overrides.py	4	4	0	0	0	0	0	0
test_torchinductor_codegen_dynamic_shapes.py	1864	1466	200	195	3	0	0	0
test_torchinductor_dynamic_shapes.py	1933	1808	112	3	8	2	0	0
test_torchinductor_opinfo.py	3674	1555	691	38	0	1390	0	0
test_torchinductor_strided_blocks.py	304	86	194	0	1	1	0	22
test_triton_extension_backend.py	3	3	0	0	0	0	0	0
test_triton_helpers.py	2	2	0	0	0	0	0	0
test_triton_heuristics.py	13	11	1	0	1	0	0	0
test_triton_kernels.py	372	326	44	0	0	2	0	0
test_triton_syntax.py	1	1	0	0	0	0	0	0
test_triton_wrapper.py	2	2	0	0	0	0	0	0
test_unbacked_symints.py	34	32	0	0	0	2	0	0
test_utils.py	11	11	0	0	0	0	0	0
test_xpu_basic.py	4	4	0	0	0	0	0	0
Total	22962	17580	3258	258	182	1628	0	56

Execution and Merge Notes

Continuation coverage

• Full-suite phase: 6,254 / 6,254 completed.
• Targeted phase: 27 / 1,848 completed; 1,821 pending.
• Total continuation executions: 6,281 / 8,102.
• Merged provenance: 16,681 latest outcomes from ROCm/HIP 7.15.0 and 6,281 from ROCm/HIP 7.15.26305.
• Pending targeted nodes retain their earlier terminal state in the table.

Stop-condition correction

• The targeted continuation stopped under the previous monitor after it accumulated 10 distinct SIGABRT-producing nodes.
• Those were not 10 adjacent executed tests. Replaying the completed result sequence gives a maximum adjacent SIGABRT streak of 2.
• The monitor has been corrected to stop only after 5 immediately adjacent executed tests trigger SIGABRT; any other completed test resets the streak.

Notable outcomes

• The previously GPU-loss-associated FlexAttention node test_builtin_score_mods_different_block_size_score_mod6_BLOCK_SIZE3_cuda_float16 passed in the continuation.
• The prior hard-wedge node test_layer_norm_bwd_with_dynamic_shape_dynamic_dims2 remains MISSED and is still pending in the targeted phase.
• After the stop, no test/compile processes survived and the gfx1250 tensor correctness smoke test passed.
• The merged artifacts do not retain comparable per-node timing for every source run, so recorded-time totals are intentionally omitted.

Largest unresolved clusters

• Errors: test_torchinductor_opinfo.py (1,390) and test_max_autotune.py (95).
• Failures: test_fp8.py (44), test_deterministic.py (24), and test_mps_basic.py (20).
• Misses: test_torchinductor_strided_blocks.py (22), test_torchinductor.py (12), and test_inductor_freezing.py (11).

Environment

• GPU: AMD Radeon Graphics, gfx1250
• Baseline: PyTorch 2.11.0+rocm7.15.0a20260721, ROCm/HIP 7.15.0, Triton 3.8.0
• Continuation: PyTorch 2.11.0+rocm10.1.0a20260803, ROCm/HIP 7.15.26305, Triton 3.8.0

Detailed reproduction steps and high-risk test notes are in this issue comment.

The comments below are intermediate checkpoints and can be ignored.

[naromero77amd]

Inductor full-suite checkpoint

• Status: Stopped immediately on invalid-device error
• Stopped: 2026-08-03T20:38:52+00:00
• Selected nodes: 1,192
• Completed: 1
• Triggering node: test/inductor/test_flex_attention.py::TestFlexAttentionCUDA::test_block_mask_non_divisible_cuda
• Triggering message: torch.AcceleratorError: CUDA error: invalid device function
• Failing operation: torch.argsort inside FlexAttention block-mask construction
• Retry attempts: 0
• Runner exit: 5 (matched stop message)
• SIGABRT streak: 0
• Timeout streak: 0

The TorchInductor cache had been deleted before launch, and the GPU tensor smoke test passed. The first test reproduced the same invalid-device-function failure, so the runner stopped exactly as requested. The runner, monitor, watchdog, abort tracker, and tmux session are now stopped; full output and state are preserved.

[naromero77amd]

Agent Run State Backup

Date: 2026-07-27T05:02:31+00:00
Host/container: 9030359f6382
GPU architecture: gfx1250 (AMD Radeon Graphics)
PyTorch: 2.11.0+rocm7.15.0a20260721
PyTorch package git: 44d3283f6350c35f7e372f306b0d8854f510a6ee
PyTorch checkout: release/2.11 at 7c3ad2a7418e6a2a7f323fc355291191572aad96
ROCm/HIP: 7.15.0
Triton: 3.8.0
framework_scripts: main at 033a1eba3b7877c300a78698bb06d37a64aa8a62

Original Goal

Run all registered PyTorch Inductor unit tests on the new AMD gfx1250 architecture, checkpoint progress to this issue, rerun all MISSED tests in shards of 50, then publish a suite-by-suite final report in the issue description.

Authoritative Primary-Run Progress

The persisted runner checkpoint and generated analysis are authoritative; the earlier live monitor overcounted some pytest alias lines.

• Collected: 22,962
• Recorded: 10,023 (43.65%)
• Primary tests remaining: 12,939
• Passed: 5,941
• Skipped: 1,014
• Xfailed: 19
• Failed: 65
• Error: 111
• Timed out: 1
• Missed: 2,872
• Minimum future executions: 15,811 (12,939 primary + 2,872 missed reruns)
• Last recorded: test/inductor/test_foreach.py::ForeachTests::test_zero_elems
• Resume point: test/inductor/test_fp8.py::TestCvtE8M0Rceil::test_correctness

Current Status: Blocked

• The original tmux server and test processes are no longer present.
• The checkpoint, logs, analysis, and helper scripts survived.
• /dev/kfd and render nodes exist and have no active users.
• torch.cuda.is_available() is False; device count is 0.
• rocminfo fails with Unable to open /dev/kfd read-write: Invalid argument.
• A fresh two-tensor GPU smoke test times out after 30 seconds.
• No resume attempt was started after this failed preflight.

A GPU/runtime, container, or host reset is required before resuming.

Preserved Artifacts

• Main log: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.log
• Monitor log: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.monitor.log
• Analysis: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.analysis.md
• Checkpoint: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.log.checkpoint
• Metadata: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.env
• Run wrapper: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.run.sh
• Monitor: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.monitor.py
• GPU core dumps: 183 files totaling approximately 248.6 GB under /workspace/pytorch/gpucore.*.gpu

The /workspace/pytorch artifacts are in the container writable overlay. They should be captured by docker commit, but the large GPU core dumps will substantially increase the image size.

Repository State and Reproducibility Notes

• The execution plan is committed and pushed at cursor/plans/run-all-inductor-tests-new-amd-gpu/PLAN.md.
• Relevant plan commits: f8e3b06, 5a266ca, and 033a1eb.
• framework_scripts/pytorch/run_tests.py has an intentionally uncommitted local line adding HIP_FORCE_DEV_KERNARG=0; this exact local runner was used for the primary run.
• framework_scripts/pytorch/__pycache__/ is untracked and not relevant.
• The PyTorch checkout has a pre-existing modified .ci/pytorch/test.sh, which was not changed by this work.
• pytest-timeout was installed in the isolated dependency directory /workspace/inductor-test-deps and supplied through PYTHONPATH.

Resume Procedure

1. Reset/restart the GPU runtime, container, or host.
2. Require all of the following before resuming:
   • no process using /dev/kfd or the AMD render nodes;
   • rocminfo succeeds;
   • torch.cuda.is_available() is true with one visible device;
   • the two-tensor addition smoke test succeeds.
3. Start a new tmux session and rerun the same full-suite command with the same log path plus --resume; it will continue from the FP8 node stored in the checkpoint.
4. Recreate rolling terminal/GitHub checkpoint monitoring.

Remaining Plan After Primary Completion

1. Implement a crash-safe per-test JSONL result journal in framework_scripts/pytorch/run_tests.py.
2. Add unit tests and a forced-SIGSEGV integration test.
3. If those tests pass, commit and push only the journal implementation and tests.
4. Extract exact MISSED node IDs and rerun them in file-scoped shards of 50 until none remain or a complete round makes no progress.
5. Merge latest results by exact node ID.
6. Publish and verify the reference-style suite summary, reproduction instructions, and failure analysis in this issue's description.

Immediate Next Step

Restore GPU/runtime health, then resume the primary suite from the persisted checkpoint. Do not start tests while /dev/kfd returns EINVAL or the tensor smoke test hangs.

[naromero77amd]

Hard-Hang Test Exclusion

The exact node that wedged the gfx1250 GPU/kernel path is:

test/inductor/test_mix_order_reduction.py::MixOrderReductionTest::test_layer_norm_bwd_with_dynamic_shape_dynamic_dims2

Evidence correction: SkipPatternTest::test_dimension_too_close and the tests preceding the node above completed successfully. The log stops while dynamic_dims2 is active, and its pytest process plus worker remained alive after both SIGTERM and SIGKILL.

Actions recorded:

• Added the exact node to /workspace/pytorch/inductor_all_gfx1250_20260724_210412.skip_nodes.txt.
• Updated the resume wrapper to pass it through PYTEST_ADDOPTS=--deselect=<node-id>.
• Updated the official execution plan so primary resumes and shard-50 missed reruns exclude it.
• The final report must list it as an intentional unresolved hard-hang exclusion; it must not be rerun on this system.

The runner and monitors remain paused pending the host reboot. The checkpoint and logs are intact.

[naromero77amd]

Additional GPU-Loss Test Exclusion

The node active when the GPU disappeared and required a host driver reload was:

test/inductor/test_flex_attention.py::TestFlexAttentionCUDA::test_builtin_score_mods_different_block_size_score_mod6_BLOCK_SIZE3_cuda_float16

Actions taken:

• Added it to /workspace/pytorch/inductor_all_gfx1250_20260724_210412.skip_nodes.txt.
• Removed it from the persisted MISSED rerun selection before resume.
• Added it to PYTEST_ADDOPTS deselection in the rerun wrapper.
• Added it to the official plan's hard-hang exclusion section.
• It must not be rerun on this gfx1250 system and must be reported as an intentional unresolved exclusion.

The MISSED rerun has resumed from its persisted 810 completed outcomes using 500-test file-scoped shards.

[naromero77amd]

Abort And Fresh-Process Restart Tracking Enabled

A persistent live tracker now records every test that causes a file/shard subprocess failure followed by a fresh-process restart.

Tracked fields per pytest node:

• Restart count
• Exit codes
• Signal names (SIGABRT, SIGSEGV, SIGTERM, etc.)
• Signal-triggered restart count
• Successful recovery count
• First/last observed log position

Initial scan of the current rerun log found:

• Distinct restarted nodes: 46
• Total fresh-process restarts: 59
• Signal-triggered restarts: 58
• Recovered-in-fresh-process events: 2

Artifacts:

• JSON: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.rerun_abort_restarts.json
• Markdown: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.rerun_abort_restarts.md
• Tracker log: /workspace/pytorch/inductor_all_gfx1250_20260724_210412.rerun_abort_tracker.log

The tracker runs in the aborts tmux window and will be incorporated into the final suite failure analysis.

[naromero77amd]

Current Pass Rates Before Full Host Reboot

The GPU reset did not recover the device reliably. All test processes are stopped, and the persisted logs/state are intact.

Primary Full-Suite Results

• Executable tests: 22,961 / 22,961
• Passed: 12,098 (52.69%)
• Skipped: 2,659
• Xfailed: 103
• Non-failing outcomes (passed + skipped + xfailed): 14,860 (64.72%)
• Failed: 185
• Errors: 1,715
• Timed out: 1
• Missed: 6,200

MISSED-Test Rerun Results Preserved So Far

• Completed with definitive outcomes: 1,315 / 6,200 (21.21% of the original MISSED set)
• Passed: 816 (62.05% of completed reruns)
• Skipped: 101
• Xfailed: 3
• Non-failing outcomes: 920 (69.96%)
• Failed: 394
• Errors: 1
• Still MISSED among completed reruns: 0
• Pending non-Flex nodes: 4,141
• Deferred nodes: 743 (439 FlexAttention + 304 FlexDecoding)
• Intentional GPU-loss hard-hang exclusion: 1

Key Suite Rerun Rates

• test_compile_subprocess.py: 474 passed, 26 skipped, 1 failed, 1 error out of 502; non-failing rate 99.60%.
• test_flex_attention.py: 94 passed, 14 skipped, 1 xfailed, 143 failed out of 252; non-failing rate 43.25%.
• test_flex_decoding.py: 1 passed, 2 xfailed, 250 failed out of 253; non-failing rate 1.19%.

Abort/Restart Scope

• Distinct tests with fresh-process restarts: 399
• Total fresh-process restarts: 412
• Signal-triggered restarts: 411
  • SIGABRT: 408 restart events across 395 distinct tests
  • Watchdog SIGTERM: 3
• Recovered after fresh-process restart: 2

Environment Experiments Before Stopping

• HIP_FORCE_DEV_KERNARG=1 alone: 6 / 557 FlexDecoding nodes completed; all 6 failed.
• HIP_FORCE_DEV_KERNARG=1 with HSA_TOOLS_DISABLE_REGISTER unset: 3 / 250 targeted SIGABRT nodes completed; all 3 failed.
• These very small samples did not show improvement before cancellation.

Post-Reboot Plan

1. Perform a full host reboot.
2. Revalidate /dev/kfd, GPU idleness, rocminfo, PyTorch visibility, and tensor-add smoke test.
3. Resume the 4,141 pending non-Flex MISSED nodes with JSONL journaling and file-scoped shards.
4. Rerun the 395 distinct SIGABRT-affected tests after reboot for a clean comparison.
5. Keep the two documented GPU-loss hard-hang nodes excluded unless explicitly approved.
6. Merge latest outcomes by exact pytest node ID and publish the final issue-description report.

Artifacts remain under /workspace/pytorch/inductor_all_gfx1250_20260724_210412*. The abort/restart inventory is in rerun_abort_restarts.json and rerun_abort_restarts.md.

[naromero77amd]

gfx1250 PyTorch Inductor Failure Reproduction

Use a dedicated test host where a GPU reset or full reboot is acceptable.
Some tests below have left processes unkillable inside the GPU/kernel path.

Replace <PRIVATE_IMAGE_URI> with the private container image URI.

Known Problem Areas

Confirmed GPU/kernel wedges

• test/inductor/test_mix_order_reduction.py::MixOrderReductionTest::test_layer_norm_bwd_with_dynamic_shape_dynamic_dims2
  • The pytest process and a worker remained alive after SIGTERM and SIGKILL.
• test/inductor/test_flex_attention.py::TestFlexAttentionCUDA::test_builtin_score_mods_different_block_size_score_mod6_BLOCK_SIZE3_cuda_float16
  • The GPU disappeared and required a host driver reload.

Confirmed long-running tests requiring watchdog termination

• test/inductor/test_compile_subprocess.py::GPUTests::test_avg_pool3d_backward2_cuda
• test/inductor/test_compile_subprocess.py::GPUTests::test_slice_scatter_reinplace_cuda
• test/inductor/test_torchinductor.py::GPUTests::test_avg_pool3d_backward2_cuda
• test/inductor/test_torchinductor_codegen_dynamic_shapes.py::DynamicShapesCodegenGPUTests::test_slice_scatter_reinplace_dynamic_shapes_cuda
• test/inductor/test_flex_attention.py::TestFlexAttentionCUDA::test_create_block_mask_recompile_persistent_reduction_oob_cuda

SIGABRT scope observed during reruns

• 395 distinct tests triggered SIGABRT.
• 250 were in test_flex_decoding.py.
• 144 were in test_flex_attention.py.
• 1 was in test_compile_subprocess.py.

The full node list is recorded in:

• /workspace/pytorch/inductor_all_gfx1250_20260724_210412.rerun_abort_restarts.json
• /workspace/pytorch/inductor_all_gfx1250_20260724_210412.rerun_abort_restarts.md

Simple Reproduction Steps

1. Pull the container

docker pull <PRIVATE_IMAGE_URI>

2. Start the container with GPU access

docker run --rm -it \
  --name gfx1250-inductor-repro \
  --device=/dev/kfd \
  --device=/dev/dri \
  --group-add video \
  --ipc=host \
  --security-opt seccomp=unconfined \
  <PRIVATE_IMAGE_URI> bash

3. Disable core dumps

This avoids hundreds of gigabytes of gpucore.*.gpu files.

ulimit -c 0

4. Get the exact test helper version

mkdir -p /home/niromero/docker_workspace
git clone https://github.com/naromero77amd/framework_scripts.git \
  /home/niromero/docker_workspace/framework_scripts
git -C /home/niromero/docker_workspace/framework_scripts checkout 2b1702f

5. Install the test dependencies

PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install \
  pytest pytest-timeout pytest-rerunfailures expecttest

6. Confirm the GPU is visible

cd /tmp
python3 -c 'import torch; print(torch.__version__, torch.version.hip, torch.cuda.get_device_name(0), torch.cuda.get_device_properties(0).gcnArchName)'

Expected architecture: gfx1250.

7. Set the baseline environment

export PYTORCH_TEST_WITH_ROCM=1
export PYTORCH_TESTING_DEVICE_ONLY_FOR=cuda
export HSA_FORCE_FINE_GRAIN_PCIE=1
export HSA_TOOLS_DISABLE_REGISTER=1
export HSA_ENABLE_SDMA=0
export HSA_OVERRIDE_GFX_VERSION=12.5.0
export HIP_FORCE_DEV_KERNARG=0
export TORCHINDUCTOR_USE_STATIC_CUDA_LAUNCHER=0
export ROCM_HOME=/usr/local/lib/python3.12/dist-packages/_rocm_sdk_devel
cd /workspace/pytorch

Fast SIGABRT Reproduction

8. Run one FlexDecoding SIGABRT test

set -o pipefail
pytest -vv -s --timeout=120 \
  test/inductor/test_flex_decoding.py::TestFlexDecodingCUDA::test_builtin_score_mods_bfloat16_score_mod0_head_dims0_cuda_bfloat16 \
  2>&1 | tee /tmp/flexdecoding-sigabrt.log

Expected failure:

• The pytest process exits from SIGABRT.
• A shell may report exit status 134.
• The log contains a fatal Python/extension stack and no complete JUnit result.

9. Run one FlexAttention SIGABRT/GPU-loss test

Only run this if a host reboot is acceptable.

pytest -vv -s --timeout=120 \
  test/inductor/test_flex_attention.py::TestFlexAttentionCUDA::test_builtin_score_mods_different_block_size_score_mod6_BLOCK_SIZE3_cuda_float16 \
  2>&1 | tee /tmp/flexattention-gpu-loss.log

GPU/Kernel Wedge Reproduction

10. Run the confirmed kernel-wedge test

pytest -vv -s --timeout=120 \
  test/inductor/test_mix_order_reduction.py::MixOrderReductionTest::test_layer_norm_bwd_with_dynamic_shape_dynamic_dims2 \
  2>&1 | tee /tmp/mix-order-kernel-wedge.log

11. If the test does not exit, inspect it from a host terminal

sudo fuser -v /dev/kfd /dev/dri/renderD*
docker top gfx1250-inductor-repro -eo pid,ppid,stat,etime,args

12. Try to terminate only the pytest host PID

sudo kill -9 <HOST_PYTEST_PID>

The wedge is reproduced if the process remains in D state or remains alive
with a fatal signal pending.

13. Save host diagnostics before rebooting

sudo dmesg -T > /tmp/gfx1250-inductor-dmesg.txt
sudo fuser -v /dev/kfd /dev/dri/renderD* \
  > /tmp/gfx1250-inductor-device-users.txt 2>&1
docker top gfx1250-inductor-repro -eo pid,ppid,stat,etime,args \
  > /tmp/gfx1250-inductor-processes.txt

14. Reset the GPU or reboot the host

If the process survives SIGKILL, a container restart may not be sufficient.
Use the supported host GPU-reset procedure or reboot the host.

Environment Comparison

To test whether the kernarg/HSA tool settings affect SIGABRT, change only
these variables and repeat step 8:

export HIP_FORCE_DEV_KERNARG=1
unset HSA_TOOLS_DISABLE_REGISTER

In the initial small comparison, 3 / 3 completed tests still failed, so this
combination had not shown an improvement before the run was stopped.

Files To Return To Engineering

• /tmp/flexdecoding-sigabrt.log
• /tmp/flexattention-gpu-loss.log, if step 9 was run
• /tmp/mix-order-kernel-wedge.log
• /tmp/gfx1250-inductor-dmesg.txt
• /tmp/gfx1250-inductor-device-users.txt
• /tmp/gfx1250-inductor-processes.txt
• Container image URI and digest
• Host kernel version from uname -a

[naromero77amd]

Five-Line Full Inductor Reproduction

Replace <PRIVATE_IMAGE_URI>, save this as a shell script, and run it from the host:

IMAGE='<PRIVATE_IMAGE_URI>'; docker pull "$IMAGE"
mkdir -p "$PWD/inductor-repro-results"
docker run --rm --name gfx1250-inductor-repro --user root --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --security-opt seccomp=unconfined -v "$PWD/inductor-repro-results:/results" "$IMAGE" bash -lc '
ulimit -c 0; mkdir -p /home/niromero/docker_workspace; git clone https://github.com/naromero77amd/framework_scripts.git /home/niromero/docker_workspace/framework_scripts; git -C /home/niromero/docker_workspace/framework_scripts checkout 2b1702f; PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install pytest pytest-timeout pytest-rerunfailures expecttest; cd /workspace/pytorch
HSA_OVERRIDE_GFX_VERSION=12.5.0 HSA_ENABLE_SDMA=0 HIP_FORCE_DEV_KERNARG=0 python3 /home/niromero/docker_workspace/framework_scripts/pytorch/run_tests.py --include-inductor-all-tests --pytorch-path /workspace/pytorch --batch-mode file --retry-attempts 2 --per-test-timeout 1200 --per-file-timeout 43200 --log-file /results/full_inductor.log'

The result log will be on the host at ./inductor-repro-results/full_inductor.log. Run only on a host that can be rebooted if the GPU/kernel wedges.