Skip to content

Skip allocating unused per-chunk window buffers in CovFastReader (fixes multi-hundred-GB vmem overrun in augment_coverage_by_labels) #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tafujino wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Skip allocating unused per-chunk window buffers in CovFastReader (fixes multi-hundred-GB vmem overrun in augment_coverage_by_labels) #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions programs/submodules/chunk/chunk.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,12 @@ ChunksCreator *ChunksCreator_constructEmpty() {

ChunksCreator *
ChunksCreator_constructFromCov(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads, int windowLen) {
return ChunksCreator_constructFromCovWithOptions(covPath, faiPath, chunkCanonicalLen, nThreads, windowLen, true);
}

ChunksCreator *
ChunksCreator_constructFromCovWithOptions(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads,
int windowLen, bool constructChunksWithAllocatedSeq) {
char *extension = extractFileExtension(covPath);
if (strcmp(extension, "cov") != 0 &&
strcmp(extension, "cov.gz") != 0 &&
Expand Down Expand Up @@ -188,11 +194,18 @@ ChunksCreator_constructFromCov(char *covPath, char *faiPath, int chunkCanonicalL
chunksCreator->nextChunkIndexToRead = 0;
chunksCreator->nThreads = nThreads;
chunksCreator->chunkCanonicalLen = chunkCanonicalLen;
fprintf(stderr, "[%s] Creating empty chunks.\n", get_timestamp());
// create empty chunks
chunksCreator->chunks = Chunk_constructListWithAllocatedSeq(chunksCreator->templateChunks,
windowLen,
chunksCreator->header->startOnlyMode);
if (constructChunksWithAllocatedSeq) {
fprintf(stderr, "[%s] Creating empty chunks.\n", get_timestamp());
// create empty chunks
chunksCreator->chunks = Chunk_constructListWithAllocatedSeq(chunksCreator->templateChunks,
windowLen,
chunksCreator->header->startOnlyMode);
} else {
// caller only needs chunksCreator->templateChunks; skip allocating the per-chunk
// windowRegionArray/windowTruthArray/windowPredictionArray/coverageInfoSeq buffers,
// which are sized off windowLen regardless of each chunk's actual span
chunksCreator->chunks = NULL;
}
chunksCreator->windowLen = windowLen;
chunksCreator->mutex = malloc(sizeof(pthread_mutex_t));
chunksCreator->startOnlyMode = chunksCreator->header->startOnlyMode;
Expand Down
11 changes: 11 additions & 0 deletions programs/submodules/chunk/chunk.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,17 @@ ChunksCreator *ChunksCreator_constructEmpty();
ChunksCreator *
ChunksCreator_constructFromCov(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads, int windowLen);

// Like ChunksCreator_constructFromCov, but when constructChunksWithAllocatedSeq is false, skips
// building chunksCreator->chunks (the per-chunk windowRegionArray/windowTruthArray/windowPredictionArray/
// coverageInfoSeq buffers used by hmm_flagger's windowed processing). Callers that only need
// chunksCreator->templateChunks (e.g. augment_coverage_by_labels via CovFastReader) should pass false:
// those buffers are sized off windowLen regardless of the actual per-chunk span, so reusing a large
// chunkCanonicalLen as windowLen (as CovFastReader does, since it has no real window size of its own)
// would otherwise allocate 3 * windowLen * sizeof(int) bytes per chunk for buffers nothing ever reads.
ChunksCreator *
ChunksCreator_constructFromCovWithOptions(char *covPath, char *faiPath, int chunkCanonicalLen, int nThreads,
int windowLen, bool constructChunksWithAllocatedSeq);

int ChunksCreator_getMaximumCoverageValue(ChunksCreator *chunksCreator);

void ChunksCreator_subsetChunksToContigs(ChunksCreator *chunksCreator, stList* contigList);
Expand Down
8 changes: 7 additions & 1 deletion programs/submodules/cov_fast_reader/cov_fast_reader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,13 @@ void CovFastReaderPerThread_destruct(CovFastReaderPerThread *covFastReaderPerThr

CovFastReader *CovFastReader_construct(char *covPath, int chunkLen, int threads) {
CovFastReader *covFastReader = malloc(sizeof(CovFastReader));
covFastReader->chunksCreator = ChunksCreator_constructFromCov(covPath, NULL, chunkLen, threads, chunkLen);
// CovFastReader has no window concept of its own (it walks blocks directly via TrackReader/
// ptBlock, not through ChunksCreator's per-chunk Chunk objects), so pass
// constructChunksWithAllocatedSeq=false to skip allocating the windowRegionArray/
// windowTruthArray/windowPredictionArray/coverageInfoSeq buffers those chunks would otherwise
// carry sized at chunkLen (here reused as windowLen for lack of a real window size) per chunk.
covFastReader->chunksCreator = ChunksCreator_constructFromCovWithOptions(covPath, NULL, chunkLen, threads,
chunkLen, false);
covFastReader->blockTablePerContig = stHash_construct3(stHash_stringKey, stHash_stringEqualKey, NULL,
(void (*)(void *)) stList_destruct);
covFastReader->threads = threads;
Expand Down