diff --git a/.gitignore b/.gitignore index 286997f..ecaef8d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ Makefile.in report.coverage TAGS t/test-merge.sh +cscope.out +tags diff --git a/mtbl/block.c b/mtbl/block.c index 131033b..2957117 100644 --- a/mtbl/block.c +++ b/mtbl/block.c @@ -16,11 +16,11 @@ */ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // @@ -32,7 +32,7 @@ // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -202,7 +202,9 @@ static bool parse_next_key(struct block_iter *bi) { bi->current = next_entry_offset(bi); - uint8_t *p = bi->data + bi->current; uint8_t *limit = bi->data + bi->restarts; + uint8_t *p = bi->data + bi->current; + uint8_t *limit = bi->data + bi->restarts; + if (p >= limit) { /* no more entries to return, mark as invalid */ bi->current = bi->restarts; @@ -214,7 +216,7 @@ parse_next_key(struct block_iter *bi) uint32_t shared, non_shared, value_length; p = decode_entry(p, limit, &shared, &non_shared, &value_length); assert(!(p == NULL || ubuf_size(bi->key) < shared)); - + ubuf_clip(bi->key, shared); ubuf_append(bi->key, p, non_shared); bi->next = p + non_shared + value_length; @@ -241,7 +243,7 @@ block_iter_seek_to_first(struct block_iter *bi) parse_next_key(bi); } -void +void block_iter_seek_to_last(struct block_iter *bi) { seek_to_restart_point(bi, bi->num_restarts - 1); @@ -268,7 +270,7 @@ block_iter_seek(struct block_iter *bi, const uint8_t *target, size_t target_len) uint32_t start_ri = bi->restart_index; /* Current key is in this restart-block. */ bool from_start = true; /* Search from start of restart-block? */ - /* + /* * If the restart_index is not zero and not equal to the number of * restarts, then begin with galloping search in the restart array to find * the first restart point with a key >= target, otherwise just do binary @@ -352,7 +354,7 @@ block_iter_next(struct block_iter *bi) return (block_iter_valid(bi)); } -void +void block_iter_prev(struct block_iter *bi) { assert(block_iter_valid(bi)); diff --git a/mtbl/compression.c b/mtbl/compression.c index f4d76c6..5770195 100644 --- a/mtbl/compression.c +++ b/mtbl/compression.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2026 DomainTools LLC * Copyright (c) 2012, 2014-2017, 2021 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -258,7 +259,7 @@ _mtbl_compress_zstd( compression_level = ZSTD_maxCLevel(); zstd_size = ZSTD_compressBound(input_size); - if (zstd_size < INT_MAX/2) { + if (zstd_size < SIZE_MAX/2) { /** * "Compression runs faster if `dstCapacity` >= * `ZSTD_compressBound(srcSize)`." @@ -390,13 +391,14 @@ _mtbl_decompress_zstd( { size_t ret = 0; - if (input_size > INT_MAX) + unsigned long long decompressed_size = ZSTD_getFrameContentSize(input, input_size); + if (decompressed_size == ZSTD_CONTENTSIZE_UNKNOWN || decompressed_size == ZSTD_CONTENTSIZE_ERROR) { return (mtbl_res_failure); - - *output_size = (size_t) ZSTD_getFrameContentSize(input, input_size); - if (*output_size <= 0) + } else if (decompressed_size > SIZE_MAX) { return (mtbl_res_failure); + } + *output_size = (size_t) decompressed_size; *output = my_malloc(*output_size); ret = ZSTD_decompress( @@ -446,6 +448,7 @@ _mtbl_decompress_zlib( size_t *output_size) { int zret; + z_stream zs = { .avail_in = 0, .next_in = Z_NULL, diff --git a/mtbl/reader.c b/mtbl/reader.c index fa2f858..ff0050d 100644 --- a/mtbl/reader.c +++ b/mtbl/reader.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 DomainTools LLC + * Copyright (c) 2022, 2026 DomainTools LLC * Copyright (c) 2012-2018 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -224,11 +224,13 @@ mtbl_reader_init(const char *fname, const struct mtbl_reader_options *opt) int fd; fd = open(fname, O_RDONLY); - if (fd < 0) + if (fd < 0) { return (NULL); + } + r = mtbl_reader_init_fd(fd, opt); - close(fd); + close(fd); return (r); } @@ -455,7 +457,7 @@ reader_iter_seek(void *v, const uint8_t *key, size_t len_key) { struct reader_iter *it = (struct reader_iter *) v; - + const uint8_t *ikey, *ival; size_t len_ikey, len_ival; uint64_t new_offset; @@ -475,7 +477,7 @@ reader_iter_seek(void *v, mtbl_varint_decode64(ival, &new_offset); /* We can skip decoding a new block if our new key is within the - * currently-decoded block. */ + * currently-decoded block. */ if (it->b == NULL || it->block_offset != new_offset) { block_destroy(&it->b); block_iter_destroy(&it->bi); @@ -515,7 +517,10 @@ reader_iter_next(void *v, block_iter_destroy(&it->bi); if (!block_iter_next(it->index_iter)) return (mtbl_res_failure); + it->b = get_block_at_index(it->r, it->index_iter); + assert(it->b != NULL); + it->bi = block_iter_init(it->b); block_iter_seek_to_first(it->bi); it->valid = block_iter_get(it->bi, key, len_key, val, len_val); diff --git a/mtbl/sorter.c b/mtbl/sorter.c index f034bdf..f8ee6ce 100644 --- a/mtbl/sorter.c +++ b/mtbl/sorter.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2026 DomainTools LLC * Copyright (c) 2012-2016 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -139,6 +140,7 @@ mtbl_sorter_init(const struct mtbl_sorter_options *opt) if (s->opt.pool != NULL) { s->pool = s->opt.pool->pool; s->rhandler = result_handler_init(_collect_readers_cb, s); + assert(s->rhandler != NULL); } return (s); @@ -184,7 +186,8 @@ _mtbl_sorter_write_chunk(struct entry_batch *b) char template[64]; /* Temporary file creation: */ - sprintf(template, "/.mtbl.%ld.XXXXXX", (long)getpid()); + (void) snprintf(template, sizeof(template), "/.mtbl.%ld.XXXXXX", (long)getpid()); + ubuf *tmp_fname = ubuf_init(strlen(s->opt.tmp_dname) + strlen(template) + 1); ubuf_append(tmp_fname, (uint8_t *) s->opt.tmp_dname, strlen(s->opt.tmp_dname)); ubuf_append(tmp_fname, (uint8_t *) template, strlen(template)); @@ -198,7 +201,10 @@ _mtbl_sorter_write_chunk(struct entry_batch *b) struct mtbl_writer_options *wopt = mtbl_writer_options_init(); mtbl_writer_options_set_compression(wopt, MTBL_COMPRESSION_SNAPPY); + struct mtbl_writer *w = mtbl_writer_init_fd(fd, wopt); + assert(w != NULL); + mtbl_writer_options_destroy(&wopt); /* Sort and add sorter entries to the temporary file writer. */ @@ -221,6 +227,10 @@ _mtbl_sorter_write_chunk(struct entry_batch *b) entry_val(next_ent), next_ent->len_val, &merge_val, &len_merge_val); if (merge_val == NULL) { + for (size_t j = i; j < entry_vec_size(b->entries); j++) { + free(entry_vec_value(b->entries, j)); + } + entry_vec_destroy(&b->entries); free(b); mtbl_writer_destroy(&w); return (NULL); @@ -241,9 +251,7 @@ _mtbl_sorter_write_chunk(struct entry_batch *b) } } - res = mtbl_writer_add(w, - entry_key(ent), ent->len_key, - entry_val(ent), ent->len_val); + res = mtbl_writer_add(w, entry_key(ent), ent->len_key, entry_val(ent), ent->len_val); free(ent); if (res != mtbl_res_success) break; diff --git a/mtbl/threadpool.c b/mtbl/threadpool.c index c79c10b..72fd0ad 100644 --- a/mtbl/threadpool.c +++ b/mtbl/threadpool.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 DomainTools LLC + * Copyright (c) 2024, 2026 DomainTools LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -165,6 +165,7 @@ static struct thread * threadpool_next(struct threadpool *pool) { struct thread *thr = NULL; + int ret; pthread_mutex_lock(&pool->m); @@ -190,7 +191,9 @@ threadpool_next(struct threadpool *pool) thr->pool = pool; pthread_mutex_init(&thr->m, NULL); pthread_cond_init(&thr->c, NULL); - pthread_create(&thr->t, NULL, thread_worker, thr); + + ret = pthread_create(&thr->t, NULL, thread_worker, thr); + assert(ret == 0); } return thr; @@ -213,6 +216,7 @@ threadpool_dispatch(struct threadpool *pool, struct resultq *rq = rh->rq; struct thread *thr = threadpool_next(pool); + assert(thr != NULL); assert(!thr->running); assert(thr->next == NULL); @@ -378,11 +382,14 @@ struct result_handler * result_handler_init(result_cb cb, void *cbdata) { struct result_handler *rh = calloc(1, sizeof(*rh)); + int ret; rh->rq = resultq_init(); rh->cb = cb; rh->cbdata = cbdata; - pthread_create(&rh->thread, NULL, result_worker, rh); + + ret = pthread_create(&rh->thread, NULL, result_worker, rh); + assert(ret == 0); return rh; } @@ -391,7 +398,9 @@ void result_handler_destroy(struct result_handler **prh) { struct result_handler *rh = *prh; - if (rh == NULL) return; + if (rh == NULL) + return; + resultq_finish(rh->rq); pthread_join(rh->thread, NULL); free(rh); diff --git a/mtbl/writer.c b/mtbl/writer.c index 74a2cac..c8f9208 100644 --- a/mtbl/writer.c +++ b/mtbl/writer.c @@ -162,8 +162,10 @@ mtbl_writer_init_fd(int orig_fd, const struct mtbl_writer_options *opt) * Start writing from the current offset. This allows mtbl's callers * to reserve some initial bytes in the file. */ - w->last_offset = lseek(fd, 0, SEEK_CUR); - w->pending_offset = w->last_offset; + off_t offset = lseek(fd, 0, SEEK_CUR); + assert(offset != (off_t)-1); + + w->pending_offset = (uint64_t)offset; w->last_key = ubuf_init(256); w->m.file_version = MTBL_FORMAT_V2; w->m.compression_algorithm = w->opt.compression_type; @@ -175,6 +177,7 @@ mtbl_writer_init_fd(int orig_fd, const struct mtbl_writer_options *opt) if (w->opt.pool != NULL) { w->pool = w->opt.pool->pool; w->rhandler = result_handler_init(_write_data_block_wrapper, w); + assert(w->rhandler != NULL); } return (w); diff --git a/src/mtbl_dump.c b/src/mtbl_dump.c index a46e005..f878652 100644 --- a/src/mtbl_dump.c +++ b/src/mtbl_dump.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2026 DomainTools LLC * Copyright (c) 2012, 2014-2015, 2021 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +20,7 @@ #include #include #include +#include #include #include "libmy/print_string.h" @@ -36,7 +38,7 @@ static void print_hex_string(const void *data, size_t len, FILE *out) unsigned c = *(str++); fprintf(out, "%02x", c); if (len > 0) - fputc('-', stdout); + fputc('-', out); } } @@ -49,6 +51,7 @@ dump(const char *fname, const bool silent, bool hex, { const uint8_t *key, *val; size_t key_len, val_len; + uint64_t count = 0, expected; struct mtbl_reader *r; struct mtbl_iter *it; @@ -58,8 +61,10 @@ dump(const char *fname, const bool silent, bool hex, return (false); } + expected = mtbl_metadata_count_entries(mtbl_reader_metadata(r)); it = mtbl_source_iter(mtbl_reader_source(r)); while (mtbl_iter_next(it, &key, &key_len, &val, &val_len)) { + count++; if (key_prefix != 0 && (key_len < key_prefix_len || 0 != bcmp(key, key_prefix, key_prefix_len))) @@ -86,6 +91,14 @@ dump(const char *fname, const bool silent, bool hex, } mtbl_iter_destroy(&it); + + if (count != expected) { + fprintf(stderr, "%s: error: read %" PRIu64 " of %" PRIu64 " expected entries;" + " file may be truncated or corrupt\n", fname, count, expected); + mtbl_reader_destroy(&r); + return (false); + } + mtbl_reader_destroy(&r); return (true); @@ -142,31 +155,27 @@ main(int argc, char **argv) } break; case 'K': - if (strlen(optarg) == 0) { - fprintf(stderr, "Need a non-empty argument to -K\n"); - return (EXIT_FAILURE); - } - - key_min_len = atoi(optarg); - - if (key_min_len < 1) { - fprintf(stderr, "Bad value of minimum key length: %s\n", optarg); + { + char *endptr; + long val = strtol(optarg, &endptr, 10); + if (endptr == optarg || *endptr != '\0' || val < 1) { + fprintf(stderr, "Invalid minimum key length: %s\n", optarg); return (EXIT_FAILURE); } + key_min_len = (size_t) val; break; + } case 'V': - if (strlen(optarg) == 0) { - fprintf(stderr, "Need a non-empty argument to -K\n"); - return (EXIT_FAILURE); - } - - val_min_len = atoi(optarg); - - if (val_min_len < 1) { - fprintf(stderr, "Bad value of minimum val length: %s\n", optarg); + { + char *endptr; + long val = strtol(optarg, &endptr, 10); + if (endptr == optarg || *endptr != '\0' || val < 1) { + fprintf(stderr, "Invalid minimum val length: %s\n", optarg); return (EXIT_FAILURE); } + val_min_len = (size_t) val; break; + } default: usage(); } diff --git a/src/mtbl_merge.c b/src/mtbl_merge.c index 9e46587..07df51f 100644 --- a/src/mtbl_merge.c +++ b/src/mtbl_merge.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2026 DomainTools LLC * Copyright (c) 2012, 2014-2016, 2019, 2021 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,6 +62,7 @@ static struct mtbl_writer *writer; static struct timespec start_time; static uint64_t count; static uint64_t count_merged; +static uint64_t total_input_entries; static void usage(void) @@ -135,6 +137,13 @@ print_stats(void) ); } +static void +fprint_hex(FILE *f, const uint8_t *buf, size_t len) +{ + for (size_t i = 0; i < len; i++) + fprintf(f, "%02x", buf[i]); +} + static void merge_func(void *clos, const uint8_t *key, size_t len_key, @@ -147,6 +156,23 @@ merge_func(void *clos, val0, len_val0, val1, len_val1, merged_val, len_merged_val); + + /* + * A NULL *merged_val indicates a merge failure and causes the merger to return mtbl_res_failure. + * This is indistinguishable from end of data at the call site in merge(), so we have to catch it here. + */ + if (*merged_val == NULL) { + fprintf(stderr, "%s: merge function returned NULL\n", program_name); + fprintf(stderr, " key (%zu bytes): ", len_key); + fprint_hex(stderr, key, len_key); + fprintf(stderr, "\n val0 (%zu bytes): ", len_val0); + fprint_hex(stderr, val0, len_val0); + fprintf(stderr, "\n val1 (%zu bytes): ", len_val1); + fprint_hex(stderr, val1, len_val1); + fputc('\n', stderr); + exit(EXIT_FAILURE); + } + count_merged += 1; } @@ -168,6 +194,22 @@ merge(void) mtbl_iter_destroy(&it); mtbl_merger_destroy(&merger); mtbl_writer_destroy(&writer); + + /* + * mtbl_iter_next() returns the same value for end-of-data and error, so we check that the count equals the + * sum of all input entries minus count_merged. Any shortfall means iteration stopped early. + */ + if (count_merged > total_input_entries) { + fprintf(stderr, "%s: error: count_merged (%" PRIu64 ") exceeds total_input_entries (%" PRIu64 ")\n", + program_name, count_merged, total_input_entries); + exit(EXIT_FAILURE); + } + uint64_t expected = total_input_entries - count_merged; + if (count != expected) { + fprintf(stderr, "%s: error: wrote %" PRIu64 " of %" PRIu64 " expected entries; input may be truncated or corrupt\n", + program_name, count, expected); + exit(EXIT_FAILURE); + } } static void @@ -419,10 +461,12 @@ main(int argc, char **argv) fprintf(stderr, "%s: opening input file %s\n", program_name, fname); readers[i] = mtbl_reader_init(fname, NULL); if (readers[i] == NULL) { - fprintf(stderr, "Error: mtbl_reader_init() failed.\n\n"); - usage(); + fprintf(stderr, "%s: ERROR: mtbl_reader_init() failed to open file %s\n\n", + program_name, fname); + exit(EXIT_FAILURE); } mtbl_merger_add_source(merger, mtbl_reader_source(readers[i])); + total_input_entries += mtbl_metadata_count_entries(mtbl_reader_metadata(readers[i])); } /* do merge */ diff --git a/src/mtbl_verify.c b/src/mtbl_verify.c index 6be8dd3..b0a5d47 100644 --- a/src/mtbl_verify.c +++ b/src/mtbl_verify.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 DomainTools LLC + * Copyright (c) 2022, 2026 DomainTools LLC * Copyright (c) 2015, 2017, 2019 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -191,6 +191,14 @@ verify_file(const char *fname) uint64_t bytes_data_blocks = mtbl_metadata_bytes_data_blocks(m); uint64_t index_offset = mtbl_metadata_index_block_offset(m); + if (bytes_data_blocks > index_offset) { + fprintf(stderr, "%s: metadata: bytes_data_blocks (%" PRIu64 ") > index_block_offset (%" PRIu64 ")\n", + fname, bytes_data_blocks, index_offset); + mtbl_reader_destroy(&r); + close(fd); + return false; + } + uint64_t data_offset = index_offset - bytes_data_blocks; if (verify_data_blocks(fd, fname, data_offset, bytes_data_blocks, count_data_blocks, mtbl_metadata_file_version(m))) { diff --git a/t/test-merge.sh.in b/t/test-merge.sh.in index e4020d8..2ca8c1f 100755 --- a/t/test-merge.sh.in +++ b/t/test-merge.sh.in @@ -129,4 +129,4 @@ cmp -s $tmp_data $full_data print_result_of_last_cmd rm -f $tmp_data -return $exitcode +exit $exitcode