From 0164a1cb3d93c26c0de3e7a399413d80e3ea8c8b Mon Sep 17 00:00:00 2001 From: Takeshi Fujino Date: Thu, 6 Aug 2026 16:51:02 +0900 Subject: [PATCH] Skip allocating unused per-chunk window buffers in CovFastReader CovFastReader_construct passed chunkLen as windowLen to ChunksCreator_constructFromCov for lack of a real window size of its own. That caused every chunk to carry 3 windowLen-sized int arrays (160MB each at the 40e6 chunkLen augment_coverage_by_labels uses) that CovFastReader never reads or writes -- it walks blocks directly via TrackReader/ptBlock, not through Chunk's window arrays. At full-genome scale (507 chunks) this allocated on the order of 240GB that nothing ever touched, hitting RLIMIT_AS/s_vmem on hosts that enforce a virtual-memory ceiling even though actual resident memory stayed under 10GB. Add ChunksCreator_constructFromCovWithOptions with a constructChunksWithAllocatedSeq flag; CovFastReader now passes false to skip building chunksCreator->chunks entirely (left NULL, which ChunksCreator_destruct already handles). hmm_flagger and coverage_format_converter, which genuinely use those buffers with sane window sizes, keep calling the original ChunksCreator_constructFromCov unchanged. Verified: full genome at --threads 8 went from needing ~235GB (matching the ~240GB estimate) to succeeding at 8GB; a 9M-line input that failed even at 64GB now succeeds at 8GB. --- programs/submodules/chunk/chunk.c | 23 +++++++++++++++---- programs/submodules/chunk/chunk.h | 11 +++++++++ .../cov_fast_reader/cov_fast_reader.c | 8 ++++++- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/programs/submodules/chunk/chunk.c b/programs/submodules/chunk/chunk.c index d925521..168b068 100644 --- a/programs/submodules/chunk/chunk.c +++ b/programs/submodules/chunk/chunk.c @@ -140,6 +140,12 @@ ChunksCreator *ChunksCreator_constructEmpty() { ChunksCreator * ChunksCreator_constructFromCov(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads, int windowLen) { + return ChunksCreator_constructFromCovWithOptions(covPath, faiPath, chunkCanonicalLen, nThreads, windowLen, true); +} + +ChunksCreator * +ChunksCreator_constructFromCovWithOptions(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads, + int windowLen, bool constructChunksWithAllocatedSeq) { char *extension = extractFileExtension(covPath); if (strcmp(extension, "cov") != 0 && strcmp(extension, "cov.gz") != 0 && @@ -188,11 +194,18 @@ ChunksCreator_constructFromCov(char *covPath, char *faiPath, int chunkCanonicalL chunksCreator->nextChunkIndexToRead = 0; chunksCreator->nThreads = nThreads; chunksCreator->chunkCanonicalLen = chunkCanonicalLen; - fprintf(stderr, "[%s] Creating empty chunks.\n", get_timestamp()); - // create empty chunks - chunksCreator->chunks = Chunk_constructListWithAllocatedSeq(chunksCreator->templateChunks, - windowLen, - chunksCreator->header->startOnlyMode); + if (constructChunksWithAllocatedSeq) { + fprintf(stderr, "[%s] Creating empty chunks.\n", get_timestamp()); + // create empty chunks + chunksCreator->chunks = Chunk_constructListWithAllocatedSeq(chunksCreator->templateChunks, + windowLen, + chunksCreator->header->startOnlyMode); + } else { + // caller only needs chunksCreator->templateChunks; skip allocating the per-chunk + // windowRegionArray/windowTruthArray/windowPredictionArray/coverageInfoSeq buffers, + // which are sized off windowLen regardless of each chunk's actual span + chunksCreator->chunks = NULL; + } chunksCreator->windowLen = windowLen; chunksCreator->mutex = malloc(sizeof(pthread_mutex_t)); chunksCreator->startOnlyMode = chunksCreator->header->startOnlyMode; diff --git a/programs/submodules/chunk/chunk.h b/programs/submodules/chunk/chunk.h index 7ce2fdd..8c66232 100644 --- a/programs/submodules/chunk/chunk.h +++ b/programs/submodules/chunk/chunk.h @@ -81,6 +81,17 @@ ChunksCreator *ChunksCreator_constructEmpty(); ChunksCreator * ChunksCreator_constructFromCov(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads, int windowLen); +// Like ChunksCreator_constructFromCov, but when constructChunksWithAllocatedSeq is false, skips +// building chunksCreator->chunks (the per-chunk windowRegionArray/windowTruthArray/windowPredictionArray/ +// coverageInfoSeq buffers used by hmm_flagger's windowed processing). Callers that only need +// chunksCreator->templateChunks (e.g. augment_coverage_by_labels via CovFastReader) should pass false: +// those buffers are sized off windowLen regardless of the actual per-chunk span, so reusing a large +// chunkCanonicalLen as windowLen (as CovFastReader does, since it has no real window size of its own) +// would otherwise allocate 3 * windowLen * sizeof(int) bytes per chunk for buffers nothing ever reads. +ChunksCreator * +ChunksCreator_constructFromCovWithOptions(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads, + int windowLen, bool constructChunksWithAllocatedSeq); + int ChunksCreator_getMaximumCoverageValue(ChunksCreator *chunksCreator); void ChunksCreator_subsetChunksToContigs(ChunksCreator *chunksCreator, stList* contigList); diff --git a/programs/submodules/cov_fast_reader/cov_fast_reader.c b/programs/submodules/cov_fast_reader/cov_fast_reader.c index 034f1e8..f13ff0e 100644 --- a/programs/submodules/cov_fast_reader/cov_fast_reader.c +++ b/programs/submodules/cov_fast_reader/cov_fast_reader.c @@ -36,7 +36,13 @@ void CovFastReaderPerThread_destruct(CovFastReaderPerThread *covFastReaderPerThr CovFastReader *CovFastReader_construct(char *covPath, int chunkLen, int threads) { CovFastReader *covFastReader = malloc(sizeof(CovFastReader)); - covFastReader->chunksCreator = ChunksCreator_constructFromCov(covPath, NULL, chunkLen, threads, chunkLen); + // CovFastReader has no window concept of its own (it walks blocks directly via TrackReader/ + // ptBlock, not through ChunksCreator's per-chunk Chunk objects), so pass + // constructChunksWithAllocatedSeq=false to skip allocating the windowRegionArray/ + // windowTruthArray/windowPredictionArray/coverageInfoSeq buffers those chunks would otherwise + // carry sized at chunkLen (here reused as windowLen for lack of a real window size) per chunk. + covFastReader->chunksCreator = ChunksCreator_constructFromCovWithOptions(covPath, NULL, chunkLen, threads, + chunkLen, false); covFastReader->blockTablePerContig = stHash_construct3(stHash_stringKey, stHash_stringEqualKey, NULL, (void (*)(void *)) stList_destruct); covFastReader->threads = threads;