From 891bb7586b1b6b37cb5dd45c2bc5d4292445956c Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 17:31:57 +0000 Subject: [PATCH 1/2] [Fix] conv3d fp8: guard padded M rows in the epilogue store The FP8 conv3d epilogue masked stores on the column bound only. With the n==1 fast-path output addressing `off = col * dhw + row`, the M rows added to pad npq up to a whole TILE_M are not out of bounds -- when dhw == npq they alias real elements of the next column: col=0, row=288 -> off = 288 == col=1, row=0 Those padded rows carry a zero accumulator (A is zeroed by the im2col mask), so they race legitimate stores from another block and silently zero part of the output. Whichever block writes last wins, which makes it intermittent. Shape (1,96,4,8,9,96) padding=1 has npq=288, TILE_M=128 -> 96 padded rows and dhw == npq, so 32.99% of the output is clobberable. Measured 12/200 failures locally on gfx950, matching the intermittent CI failures of tests/kernels/test_conv3d_implicit_fp8.py on linux-flydsl-mi355-1 (both observed failures reported rel_err 3.075e-01, the deterministic value for a fully-clobbered result). The set of zeroed offsets was verified to be exactly the predicted padded-row alias set. After the fix: 0/200 failures, worst rel_err back to 1.11e-05. tests/kernels/test_conv3d_implicit_fp8.py and tests/kernels/test_conv3d_implicit.py pass (33 tests). The BF16 kernel already guards this via `_row_chk` in kernels/conv/conv3d_implicit.py; the split-K path of the FP8 kernel also already checks `row < npq` for its atomics. Only the non-atomic FP8 store was missing the check. --- kernels/conv/conv3d_implicit_fp8.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kernels/conv/conv3d_implicit_fp8.py b/kernels/conv/conv3d_implicit_fp8.py index 5a2414f56..85e5a17d7 100644 --- a/kernels/conv/conv3d_implicit_fp8.py +++ b/kernels/conv/conv3d_implicit_fp8.py @@ -456,7 +456,13 @@ def store_half_pair(acc0, acc1, m_half): n_idx = row // dhw sp = row % dhw off_ncdhw = n_idx * (k * dhw) + col * dhw + sp - buffer_ops.buffer_store(out.to(fx.BFloat16), y_rsrc, off_ncdhw, mask=col_valid) + # Padded M rows (npq..grid_m*TILE_M) must not store: with the + # n==1 layout col*dhw+row they alias real elements of a later + # column and race legitimate stores from another block. + row_valid = row < fx.Index(npq) + buffer_ops.buffer_store( + out.to(fx.BFloat16), y_rsrc, off_ncdhw, mask=col_valid & row_valid + ) store_half_pair(acc00, acc01, 0) store_half_pair(acc10, acc11, 1) From f86a007860647e89102fc63094e662fb2865da57 Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 17:48:58 +0000 Subject: [PATCH 2/2] conv3d fp8: make the padded-row guard compile-time conditional Gate the row check on `row_chk = npq % TILE_M != 0`, mirroring `_row_chk` in the BF16 kernel, so tile-aligned shapes (which have no padded rows) emit no extra compare. Verified by dumping final ISA for an aligned shape (npq=768): the generated assembly is byte-identical to the pre-fix kernel, so aligned shapes pay nothing. The unaligned shape (npq=288) gains 2 v_cmp for the guard with unchanged VGPR/SGPR (120/60), zero spills, and identical mfma, ds_read, buffer_store_short and s_barrier counts. --- kernels/conv/conv3d_implicit_fp8.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kernels/conv/conv3d_implicit_fp8.py b/kernels/conv/conv3d_implicit_fp8.py index 85e5a17d7..ebdb2f3df 100644 --- a/kernels/conv/conv3d_implicit_fp8.py +++ b/kernels/conv/conv3d_implicit_fp8.py @@ -216,6 +216,9 @@ def compile_conv3d_implicit_fp8(n, c, d, h, width, k, kt, kh, kw, st, sh, sw, pt grid_m = (npq + TILE_M - 1) // TILE_M grid_n = (k + TILE_N - 1) // TILE_N + # Padded M rows only exist when npq is not tile-aligned; mirrors _row_chk in + # the BF16 kernel so aligned shapes emit no extra compare. + row_chk = npq % TILE_M != 0 elem_ty = fx.Float8E4M3FN @flyc.kernel(known_block_size=[BLOCK_THREADS, 1, 1]) @@ -459,10 +462,10 @@ def store_half_pair(acc0, acc1, m_half): # Padded M rows (npq..grid_m*TILE_M) must not store: with the # n==1 layout col*dhw+row they alias real elements of a later # column and race legitimate stores from another block. - row_valid = row < fx.Index(npq) - buffer_ops.buffer_store( - out.to(fx.BFloat16), y_rsrc, off_ncdhw, mask=col_valid & row_valid - ) + store_mask = col_valid + if const_expr(row_chk): + store_mask = col_valid & (row < fx.Index(npq)) + buffer_ops.buffer_store(out.to(fx.BFloat16), y_rsrc, off_ncdhw, mask=store_mask) store_half_pair(acc00, acc01, 0) store_half_pair(acc10, acc11, 1)