From 7dd84885e296762df659ba2e11b612fc40599f15 Mon Sep 17 00:00:00 2001 From: ziipo Date: Sun, 9 Aug 2026 13:27:35 -0700 Subject: [PATCH] dashboard: serialize mp3 decode to avoid libsndfile SIGBUS The dashboard intermittently died with "Bus error: 10" ~1.4s after launch, before training started. The faulting thread was inside libsndfile: mpeg_init <- mpeg_open <- psf_open_file <- sf_open <- ffi_call with pc = lr = far = 0x1 and ESR "PC alignment" -- the CPU jumped to 0x1 and faulted on the instruction fetch. Register x2 pointed at a symbol named `jmpbuf`, and three threads were inside libsndfile at the same moment (psf_fread, mpeg_dec_seek, mpeg_init). libsndfile 1.2.2's MPEG decoder reports errors by longjmp-ing through a process-global jmpbuf (libmpg123), which is not thread-safe. Concurrent mp3 decodes race on it: one thread's setjmp context is overwritten by another's, the longjmp lands on a stack frame that no longer exists, and the process takes SIGBUS. _load_audio is reached from three concurrency sources at once -- _spec_pool (4 workers), the 16-worker pool at server.py:5867, and ThreadingMixIn on the HTTP server -- so spectrogram rendering fans out enough simultaneous decodes to hit the window regularly. Runs with more accumulated demo mp3s trip it sooner. Serialize the sf.read behind a module-level lock. The resample below it is pure numpy/torch and stays outside, so only decoding is serialized. Verified: 16 threads x 25 rounds (400 concurrent decodes) through the patched _load_audio, three consecutive runs, no crash. Sequential decode of all 127 mp3s passed both before and after, confirming this is not file corruption. Co-Authored-By: Claude Opus 5 --- dashboard/server.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dashboard/server.py b/dashboard/server.py index 106e26d..8c5202b 100644 --- a/dashboard/server.py +++ b/dashboard/server.py @@ -3006,6 +3006,14 @@ def _extract_hyperparams(run): SPEC_W, SPEC_H = 300, 60 _spec_pool = ThreadPoolExecutor(max_workers=4) +# libsndfile 1.2.2's MPEG decoder reports errors by longjmp-ing through a +# process-global jmpbuf (libmpg123). Two threads decoding mp3s at once race on +# it: one clobbers the other's saved context, the longjmp lands on a dead stack +# frame, and the process takes SIGBUS (pc=0x1, EXC_ARM_DA_ALIGN) inside +# mpeg_init. We fan out decodes from _spec_pool, a 16-worker pool, and the +# threaded HTTP server, so serialize every sf.read behind this lock. +_sndfile_lock = threading.Lock() + # Pre-compute band colors as (3, 3) array for vectorized multiply _BAND_COLORS = np.array([c for _, _, c in SPEC_BANDS], dtype=np.float32) @@ -3067,7 +3075,8 @@ def _power_to_db(S, top_db=80.0): def _load_audio(path, target_sr=32000): """Load + resample to target_sr. Returns (channels, samples) like librosa(mono=False).""" - y, sr = sf.read(str(path), dtype='float32', always_2d=False) + with _sndfile_lock: # see _sndfile_lock: concurrent mp3 decode is a SIGBUS + y, sr = sf.read(str(path), dtype='float32', always_2d=False) if y.ndim == 2: y = y.T # soundfile gives (n, c); we want (c, n) if sr != target_sr: