diff --git a/.github/workflows/test-tiers.yml b/.github/workflows/test-tiers.yml new file mode 100644 index 000000000..cf8bdc160 --- /dev/null +++ b/.github/workflows/test-tiers.yml @@ -0,0 +1,180 @@ +# Isolation, resource-accounting and lock-mode test tiers. +# +# These three tiers close a structural blind spot: the existing suite is strong +# on crash/durability (test/sim) and on memory safety against malformed input +# (test/fuzz), but was blind to (a) isolation semantics as a checkable property +# and (b) long-running resource accounting. Five external bug reports +# (#136-#140) landed on v5.3.34 through that gap. +# +# B1 test/isolation/ serializability checker -- would have caught #136 +# B2 test/soak/ resource-accounting soak -- would have caught #137/#138 +# B3 test/lockmatrix/ lock-mode matrix under ASan -- would have caught #140 +# +# Gating policy follows the house style: +# - B1 is a HARD GATE on every push/PR. It is fast (~1 min) and its +# expectations are self-correcting: the checker fails both when a new +# serializability violation appears AND when a scenario marked as +# reproducing a known issue stops violating, so a fix cannot land without +# updating the expectation. +# - B3 is advisory (continue-on-error) for now, because on current master it +# legitimately aborts under ASan: that IS the #140 reproduction. Flip it to +# a hard gate in the same PR that fixes #140. +# - B2 (soak) is scheduled/nightly plus manual dispatch: it is a long run and +# the same expectation-flip applies once #137/#138 land. + +name: Test tiers (isolation / soak / lock matrix) + +on: + push: + branches: [master] + pull_request: + paths: + - 'test/isolation/**' + - 'test/soak/**' + - 'test/lockmatrix/**' + - 'src/lock/**' + - 'src/txn/**' + - 'src/mp/**' + - 'dist/**' + - '.github/workflows/test-tiers.yml' + schedule: + # Nightly (04:41 UTC), offset from ci.yml's 03:17 so the runners do not + # contend. This is when the soak tier runs. + - cron: '41 4 * * *' + workflow_dispatch: + inputs: + soak_n: + description: 'Transactions per soak workload' + default: '5000' + +concurrency: + group: test-tiers-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # -------------------------------------------------------------------------- + # Tier B1 -- isolation / anomaly checker. Hard gate: fast and deterministic + # enough to run per push. + # -------------------------------------------------------------------------- + isolation: + name: B1 isolation/anomaly checker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install build deps + run: | + sudo apt-get update + sudo apt-get install -y liburing-dev + + - name: Build libdb (debug, so DB_ASSERTs are live) + working-directory: build_unix + run: | + ../dist/configure --enable-debug + make -j"$(nproc)" + + - name: Run the anomaly scenarios + working-directory: test/isolation + run: ISO_TIMEOUT=600 ./run.sh + + - name: Upload scenario databases on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: isolation-artifacts + path: test/isolation/build/ISODIR.*/** + if-no-files-found: ignore + + # -------------------------------------------------------------------------- + # Tier B3 -- lock-mode matrix under ASan. Advisory until #140 lands, because + # the ASan heap-buffer-overflow it finds in __lock_vec IS the bug report. + # -------------------------------------------------------------------------- + lock-matrix: + name: B3 lock-mode matrix (ASan) + runs-on: ubuntu-latest + continue-on-error: true # advisory until #140 is fixed + steps: + - uses: actions/checkout@v4 + + - name: Install clang + build deps + run: | + sudo apt-get update + sudo apt-get install -y clang llvm liburing-dev + clang --version + + - name: Run the lock-mode matrix (builds an ASan libdb once) + working-directory: test/lockmatrix + run: | + # Pipe-to-tee would mask the driver's exit status behind tee's, and + # an ASan abort is exactly the signal we must not lose. Capture the + # status explicitly and re-raise it after the summary step has run. + set -o pipefail + CC=clang LOCK_TIMEOUT=900 ./run.sh 2>&1 | tee matrix.log + + - name: Summarise + if: always() + working-directory: test/lockmatrix + run: | + if grep -q 'AddressSanitizer' matrix.log; then + echo "::warning::Tier B3 reproduced an ASan fault in the lock list" + echo "Last shape attempted before the fault:" + grep -E '^ (PUT_READ|UPGRADE_WRITE)' matrix.log | tail -1 + grep -m1 'SUMMARY: AddressSanitizer' matrix.log || true + fi + + - name: Upload matrix log + if: always() + uses: actions/upload-artifact@v4 + with: + name: lock-matrix-log + path: test/lockmatrix/matrix.log + if-no-files-found: ignore + + # -------------------------------------------------------------------------- + # Tier B2 -- resource-accounting soak. Scheduled/manual only: it is a long + # run (thousands of sequential transactions per workload). + # -------------------------------------------------------------------------- + soak: + name: B2 resource-accounting soak + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + continue-on-error: true # advisory until #137/#138 are fixed + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Install build deps + run: | + sudo apt-get update + sudo apt-get install -y liburing-dev + + - name: Build libdb (debug) + working-directory: build_unix + run: | + ../dist/configure --enable-debug + make -j"$(nproc)" + + - name: Soak + working-directory: test/soak + run: | + set -o pipefail # do not let tee mask a failing soak + SOAK_N="${{ github.event.inputs.soak_n || 5000 }}" \ + SOAK_TIMEOUT=2400 ./run.sh 2>&1 | tee soak.log + + - name: Summarise growth + if: always() + working-directory: test/soak + run: | + echo "== slopes that exceeded tolerance ==" + grep -E 'GROWING|ENOMEM' soak.log || echo "(none)" + + - name: Upload growth curves + if: always() + uses: actions/upload-artifact@v4 + with: + name: soak-growth-curves + path: test/soak/soak.log + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index d16c3262d..3f37999a5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,17 @@ test/tcl/tclIndex TESTDIR_sim_*/ test/sim/TESTDIR_sim_*/ +# Test-tier build dirs and scratch env dirs (test/isolation, test/soak, +# test/lockmatrix -- each run.sh builds into ./build and each driver creates +# its scratch environments beside the binary) +test/isolation/build/ +test/soak/build/ +test/lockmatrix/build/ +ISODIR.*/ +SOAKDIR.*/ +LOCKDIR/ +build_dst/** + # Meson/Ninja out-of-tree build dirs build-meson/ build/ diff --git a/dist/Makefile.in b/dist/Makefile.in index 70722af76..422f7c6e9 100644 --- a/dist/Makefile.in +++ b/dist/Makefile.in @@ -1916,6 +1916,39 @@ fi_sweep: fi_sweep@o@ $(DEF_LIB) fi_tests: fi_sweep +################################################## +# Isolation / soak / lock-matrix test tiers -- test/isolation/, test/soak/, +# test/lockmatrix/. +# +# These need no library-side hooks at all (they drive the public API only), so +# unlike the DST and faultinject tiers there is no configure option and no +# ADDITIONAL_OBJS. `make tier_tests` builds all three drivers. Each tier also +# has a run.sh that builds standalone against an existing build_unix; see the +# README.md in each directory. +################################################## +test_iso_anomaly@o@: $(testdir)/isolation/test_iso_anomaly.c + $(CC) $(CFLAGS) $(DEPFLAGS) $< +test_iso_anomaly: test_iso_anomaly@o@ $(DEF_LIB) + $(CCLINK) -o $@ \ + $(LDFLAGS) test_iso_anomaly@o@ $(DEF_LIB) $(TEST_LIBS) $(LIBS) + $(POSTLINK) $@ + +test_soak_resources@o@: $(testdir)/soak/test_soak_resources.c + $(CC) $(CFLAGS) $(DEPFLAGS) $< +test_soak_resources: test_soak_resources@o@ $(DEF_LIB) + $(CCLINK) -o $@ \ + $(LDFLAGS) test_soak_resources@o@ $(DEF_LIB) $(TEST_LIBS) $(LIBS) + $(POSTLINK) $@ + +test_lock_matrix@o@: $(testdir)/lockmatrix/test_lock_matrix.c + $(CC) $(CFLAGS) $(DEPFLAGS) $< +test_lock_matrix: test_lock_matrix@o@ $(DEF_LIB) + $(CCLINK) -o $@ \ + $(LDFLAGS) test_lock_matrix@o@ $(DEF_LIB) $(TEST_LIBS) $(LIBS) + $(POSTLINK) $@ + +tier_tests: test_iso_anomaly test_soak_resources test_lock_matrix + ################################################## # Targets for example programs. ################################################## diff --git a/meson.build b/meson.build index 7b540cb81..d434cbfa4 100644 --- a/meson.build +++ b/meson.build @@ -9,6 +9,7 @@ # ninja -C build # parallel build -> libdb # ninja -C build docs # render docs_src/ -> docs-build/ # ninja -C build bench # build the test/bench microbenchmark drivers +# meson test -C build --suite tiers # isolation + lock-matrix tiers project('libdb', 'c', version: '5.3.34', @@ -34,3 +35,8 @@ subdir('dist') if not get_option('hegel').disabled() subdir('test/pbt') endif + +# Isolation / soak / lock-matrix test tiers (B1/B2/B3) -- test/tiers. Always +# entered: the drivers are build_by_default:false so a plain `ninja` is +# unaffected, and they need no library-side hooks (public API only). +subdir('test/tiers') diff --git a/test/isolation/README.md b/test/isolation/README.md new file mode 100644 index 000000000..2efea5c77 --- /dev/null +++ b/test/isolation/README.md @@ -0,0 +1,117 @@ +# Tier B1 — isolation / anomaly checker + +Runs concurrent transaction schedules under `DB_TXN_SNAPSHOT` (which in this +fork means serializable snapshot isolation) and checks the **committed** result +against some serial order of the committed transactions. + +## Why this tier exists + +The existing suite is strong on crash/durability (`test/sim`, 41 DST +scenarios) and on memory safety against malformed input (`test/fuzz`). Neither +can see a write skew that *commits successfully*: nothing crashes, no page is +corrupt, no sanitizer fires — the database just holds a state that no serial +execution could have produced. Issue #136 is exactly that shape, and it +shipped in v5.3.34. + +## The verdict is computed, not hard-coded + +Each scenario declares, per transaction, a `model` function: the transaction's +semantics as a pure function over an abstract state vector. After the schedule +runs, the harness reads the real state back out of the databases (after closing +and reopening the environment, so the verdict is about the **durable** state), +then enumerates every permutation of the transactions that actually committed +and applies their models serially. If no permutation reproduces the observed +state, the history is not serializable and the scenario fails with the schedule +printed. + +The state vector has two kinds of slot: + +- **record slots** — one per database record; the observed value is read back + from the database. +- **observation slots** — what a transaction claims it *read*. These are what + make the read-only anomaly checkable: there the stored state is perfectly + fine and only the read-only transaction's observation has no serial + explanation. Observation slots of a transaction that did not commit are + ignored. + +## Scenarios + +| Scenario | Shape | Expectation on master | +|---|---|---| +| `write_skew_trigger` | two one-page DBs; T2's write lands while T1 is inside `commit` | **XFAIL — reproduces #136** | +| `write_skew_control` | two one-page DBs; T2 writes and commits before T1 commits | PASS (`DB_SNAPSHOT_CONFLICT` to T1) | +| `write_skew_late` | two one-page DBs; T2 writes after T1's commit returned | PASS (`DB_SNAPSHOT_UNSAFE` to T2) | +| `write_skew_samebtree_control` | two records on **different pages of one** B-tree; control timing | PASS | +| `write_skew_samebtree_trigger` | same, trigger timing | **XFAIL — reproduces #136** | +| `g2_antidep` | G2-item: both txns scan for markers, both insert one | PASS | +| `read_only_anomaly` | Fekete's 3-txn pattern; the read-only txn's observation is checked | PASS | +| `lost_update` | both txns read the counter and write read+1 | PASS | +| `read_your_writes` | sanity: a txn must observe its own uncommitted write | PASS | + +`read_your_writes` exists so a *vacuously* passing checker is detectable: if +the harness ever stops driving the engine, that scenario fails. + +### On the "separate defect" reported alongside #136 + +The #136 reporter suspected a second, independent defect: two records on +different pages of one B-tree detecting no conflict at all, *even in the +control*. That does **not** reproduce here. +`write_skew_samebtree_control` builds the shape explicitly (512-byte pages plus +filler keys sorting between `alice` and `bob`, giving 33 leaf pages with +`alice` as the minimum key and `bob` as the maximum, verified via +`DB->stat`→`bt_leaf_pg`) and the control correctly returns +`DB_SNAPSHOT_CONFLICT`. Only the trigger timing commits both. On this +construction the different-pages case has the **same** root cause as #136 +proper (the commit-window race), not an extra page-granularity hole. The +reporter did not publish their same-btree variant, so their shape may differ; +the control is kept as a live PASS expectation precisely so a real +page-granularity regression would surface here. + +## How the #136 interleaving is reached — no engine hook + +Landing T2's write while T1 is **inside** `DB_TXN->commit` is done entirely from +the application side: `pthread_barrier` for the ordered phases, plus an atomic +flag that T1 sets immediately before entering `commit` and T2 spins on. This is +the reporter's own technique. **No engine change, no `HAVE_DST` site, zero +production overhead.** A test-only yield point in the commit path was +considered and not needed. + +That window is genuinely racy — T1's commit can finish before T2's put reaches +the conflict check, degenerating into the benign "late" schedule. So the racy +scenarios run multiple attempts (40 by default) and the rule is asymmetric on +purpose: **one** violation in any attempt is a reproduction, while a pass +requires **every** attempt to be clean. A serializability violation is a real +counterexample; a single clean run of a racy schedule proves nothing. + +In practice both #136 shapes violate on the first attempt. + +## Running it + +```sh +# Build libdb first (once): +cd build_unix && ../dist/configure --enable-debug && make -j"$(nproc)" + +# All scenarios: +cd test/isolation && ./run.sh + +# One scenario, with the btree-shape diagnostics: +ISO_VERBOSE=1 ./run.sh write_skew_samebtree_control + +./run.sh --list # scenario names, with expect-fail marked +./run.sh build # build only +``` + +Environment: `CC`, `LIBDB_BUILD` (default `../../build_unix`), `ISO_TIMEOUT` +(default 300s), `ISO_SAN=1` to add ASan. + +## Exit status + +- `0` — every scenario matched its recorded expectation. +- `1` — a scenario did not. Either a new serializability violation, **or** an + expect-fail scenario that stopped violating, meaning the referenced issue got + fixed and `expect_fail` should be cleared in the table in + `test_iso_anomaly.c`. The message says which. +- `2` — harness error. + +When #136 lands, clear `expect_fail` on `write_skew_trigger` and +`write_skew_samebtree_trigger`; the tier then gates the fix against regression. diff --git a/test/isolation/run.sh b/test/isolation/run.sh new file mode 100755 index 000000000..d72568fa8 --- /dev/null +++ b/test/isolation/run.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# test/isolation/run.sh -- build and run the Tier B1 isolation/anomaly checker. +# +# Builds test_iso_anomaly against an existing libdb build and runs it under a +# timeout. The checker's verdict is computed (enumerate serial orders), so a +# non-zero exit means an outcome disagreed with the recorded expectation -- +# either a new serializability violation, or a known-broken scenario that +# started passing (i.e. the referenced issue got fixed and the table needs +# updating). +# +# Usage: +# ./run.sh # build + run every scenario +# ./run.sh build # build only +# ./run.sh SCENARIO ... # build + run the named scenarios +# ./run.sh --list # list scenario names +# +# Env: +# CC compiler (default: cc) +# LIBDB_BUILD path to a built build_unix (default: ../../build_unix) +# ISO_TIMEOUT seconds for the whole run (default: 300) +# ISO_SAN 1 => also build with ASan/UBSan (default 0) +# +# Run from test/isolation/ inside a `nix develop` shell. + +set -eu + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$HERE" + +CC=${CC:-cc} +LIBDB_BUILD=${LIBDB_BUILD:-"$HERE/../../build_unix"} +ISO_TIMEOUT=${ISO_TIMEOUT:-300} +ISO_SAN=${ISO_SAN:-0} +OUT="$HERE/build" +LIBDBA="$LIBDB_BUILD/libdb.a" + +if [ -f "$LIBDB_BUILD/Makefile" ]; then + LDLIBS=$(sed -n 's/^LIBS=[[:space:]]*//p' "$LIBDB_BUILD/Makefile" | head -1) +fi +LDLIBS="${LDLIBS:--lpthread} -ldl -lpthread" + +CFLAGS="-g -O1 -Wall -Wextra -Wno-unused-parameter -I$LIBDB_BUILD -I$HERE" +[ "$ISO_SAN" = "1" ] && CFLAGS="$CFLAGS -fsanitize=address" + +[ -f "$LIBDBA" ] || { + echo "error: libdb.a not found at $LIBDBA -- build libdb first:" >&2 + echo " (cd $LIBDB_BUILD && ../dist/configure --enable-debug && make -j4)" >&2 + exit 2 +} + +mkdir -p "$OUT" +# shellcheck disable=SC2086 +$CC $CFLAGS "$HERE/test_iso_anomaly.c" "$LIBDBA" $LDLIBS \ + -o "$OUT/test_iso_anomaly" +echo "built $OUT/test_iso_anomaly" + +[ "${1:-}" = "build" ] && exit 0 + +cd "$OUT" +exec timeout "$ISO_TIMEOUT" ./test_iso_anomaly "$@" diff --git a/test/isolation/test_iso_anomaly.c b/test/isolation/test_iso_anomaly.c new file mode 100644 index 000000000..9c953eab0 --- /dev/null +++ b/test/isolation/test_iso_anomaly.c @@ -0,0 +1,1217 @@ +/*- + * test/isolation/test_iso_anomaly.c -- + * Tier B1: isolation / anomaly checker. + * + * Runs concurrent transaction schedules under DB_TXN_SNAPSHOT (which in this + * fork means serializable snapshot isolation) and VALIDATES the committed + * outcome against some serial order of the committed transactions. + * + * The verdict is COMPUTED, never hard-coded: every scenario declares, per + * transaction, a `model' function -- the transaction's semantics as a pure + * function over an abstract state vector. After the schedule runs we read + * the real committed state back out of the databases, then enumerate every + * permutation of the transactions that actually committed and apply their + * models serially. If no permutation reproduces the observed state, the + * history is not serializable and the scenario FAILS with the schedule + * printed. + * + * The abstract state vector holds two kinds of slot: + * [0, ndbkeys) one slot per database record; the "actual" + * value is read back from the database. + * [ndbkeys, nslots) OBSERVATION slots: what a transaction claims + * it read. This is what makes the read-only + * anomaly checkable -- there the final stored + * state is fine and only the read-only + * transaction's observation has no serial + * explanation. Observation slots belonging to a + * transaction that did not commit are ignored. + * + * Scenarios are deterministic. Most are driven single-threaded (snapshot + * isolation cares about txn_begin / commit ORDER, not about wall-clock + * concurrency), so no scheduler is needed. The one schedule that genuinely + * needs two threads is the issue #136 trigger, where T2's write must land + * while T1 is INSIDE DB_TXN->commit; that uses pthread barriers plus atomic + * flags from the application side exactly as the #136 reporter did -- no + * engine hook, no HAVE_DST site, zero production overhead. + * + * Usage: + * ./test_iso_anomaly run every scenario + * ./test_iso_anomaly SCENARIO ... run the named scenarios + * ./test_iso_anomaly --list list scenario names + * + * Exit status: 0 = every scenario matched its expectation, 1 = a scenario + * did not (a serializability violation where none was expected, or an + * expected-fail scenario that unexpectedly PASSED -- i.e. an issue got + * fixed and the expectation needs updating), 2 = harness error. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db.h" + +#define ISO_MAX_SLOT 8 +#define ISO_MAX_TXN 4 +#define ISO_MAX_DB 3 + +typedef struct { + int v[ISO_MAX_SLOT]; +} iso_state; + +typedef struct { + const char *name; + void (*model)(iso_state *); /* Serial semantics. */ + int obs_lo, obs_hi; /* Observation slots owned. */ + int committed; /* Filled in by the run. */ + int end_rc; /* commit/abort-cause rc. */ + char log[256]; /* What the run observed. */ +} iso_txn; + +struct iso_scenario; +typedef int (*iso_run_fn)(struct iso_scenario *, iso_state *, iso_txn *); + +typedef struct iso_scenario { + const char *name; + const char *shape; + int ndbkeys; /* Slots [0,ndbkeys) are DB records. */ + int nslots; /* Total slots (DB + observation). */ + int ntxn; + int expect_fail; /* Known-broken on master. */ + const char *issue; /* Issue it reproduces, if any. */ + /* + * How many times to run the schedule. Single-threaded schedules are + * deterministic and need one attempt. The two-thread schedules whose + * whole point is a narrow window (T2's write must land while T1 is + * inside commit) do NOT hit that window every time -- T1's commit can + * finish before T2's put reaches the conflict check, degenerating into + * the benign "late" schedule. Those get several attempts: the tier's + * job is to FIND a non-serializable history if one exists, so a + * violation in any attempt is a reproduction, and a clean sweep of + * every attempt is the pass. + */ + int attempts; + iso_run_fn run; +} iso_scenario; + +/* + * Shared environment state. One scenario at a time, so globals are fine and + * keep the thread bodies short. + */ +static DB_ENV *env; +static DB *dbs[ISO_MAX_DB]; +static int ndbs; +static char iso_home[512]; +static int verbose; + +static void iso_die(const char *, int) __attribute__((noreturn)); + +static void +iso_die(const char *what, int rc) +{ + fprintf(stderr, "harness error: %s: %s (%d)\n", + what, db_strerror(rc), rc); + exit(2); +} + +static const char * +rc_name(int rc) +{ + if (rc == 0) + return ("success"); + switch (rc) { + case DB_SNAPSHOT_CONFLICT: return ("DB_SNAPSHOT_CONFLICT"); + case DB_SNAPSHOT_UNSAFE: return ("DB_SNAPSHOT_UNSAFE"); + case DB_LOCK_DEADLOCK: return ("DB_LOCK_DEADLOCK"); + case DB_LOCK_NOTGRANTED: return ("DB_LOCK_NOTGRANTED"); + case DB_NOTFOUND: return ("DB_NOTFOUND"); + case DB_KEYEXIST: return ("DB_KEYEXIST"); + default: return (db_strerror(rc)); + } +} + +/* An rc that legitimately means "this transaction had to give up". */ +static int +iso_is_abort_rc(int rc) +{ + return (rc == DB_LOCK_DEADLOCK || rc == DB_LOCK_NOTGRANTED || + rc == DB_SNAPSHOT_CONFLICT || rc == DB_SNAPSHOT_UNSAFE); +} + +/* + * Scratch directory handling. Each scenario gets a fresh home so a scenario + * never inherits another's log files or MVCC state. + */ +static void +iso_rmtree(const char *dir) +{ + char cmd[600]; + + /* Bounded, no recursion into the harness; find(1) is the tool. */ + (void)snprintf(cmd, sizeof(cmd), + "find '%s' -mindepth 1 -delete 2>/dev/null", dir); + (void)system(cmd); +} + +static void +iso_home_init(const char *scenario) +{ + (void)snprintf(iso_home, sizeof(iso_home), + "ISODIR.%s", scenario); + (void)mkdir(iso_home, 0755); + iso_rmtree(iso_home); +} + +/* + * iso_env_open -- + * Open (creating if asked) the environment and `n' databases. pagesize + * != 0 forces a small page size, which the same-btree scenario needs to + * push two records onto different pages. + */ +static void +iso_env_open(u_int32_t create, int n, const char *const *names, + u_int32_t pagesize) +{ + int i, rc; + + if ((rc = db_env_create(&env, 0)) != 0) + iso_die("db_env_create", rc); + if ((rc = env->set_lk_detect(env, DB_LOCK_DEFAULT)) != 0) + iso_die("set_lk_detect", rc); + /* + * A lock timeout keeps the tier self-bounding: a schedule where two + * transactions hold conflicting write locks and neither is waiting on + * the other is not a deadlock, so the detector will not break it and + * a blocked put would hang forever. With a timeout it returns + * DB_LOCK_NOTGRANTED, which the checker treats as "this transaction + * had to give up" -- a legitimate serializable outcome. + */ + if ((rc = env->set_timeout(env, 2000000, DB_SET_LOCK_TIMEOUT)) != 0) + iso_die("set_timeout", rc); + /* + * A modest cache with plenty of MVCC room: too small a cache makes + * snapshot transactions fail with DB_SNAPSHOT_UNSAFE for cache + * reasons, which would mask the isolation property under test. + */ + if ((rc = env->set_cachesize(env, 0, 4 * 1024 * 1024, 1)) != 0) + iso_die("set_cachesize", rc); + if ((rc = env->open(env, iso_home, create | DB_INIT_LOCK | + DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD, 0600)) != 0) + iso_die("DB_ENV->open", rc); + + ndbs = n; + for (i = 0; i < n; i++) { + if ((rc = db_create(&dbs[i], env, 0)) != 0) + iso_die("db_create", rc); + if (pagesize != 0 && (rc = + dbs[i]->set_pagesize(dbs[i], pagesize)) != 0) + iso_die("set_pagesize", rc); + if ((rc = dbs[i]->open(dbs[i], NULL, names[i], NULL, DB_BTREE, + create | DB_MULTIVERSION | DB_AUTO_COMMIT | DB_THREAD, + 0600)) != 0) + iso_die("DB->open", rc); + } +} + +static void +iso_env_close(void) +{ + int i, rc; + + for (i = 0; i < ndbs; i++) + if ((rc = dbs[i]->close(dbs[i], 0)) != 0) + iso_die("DB->close", rc); + ndbs = 0; + if ((rc = env->close(env, 0)) != 0) + iso_die("DB_ENV->close", rc); + env = NULL; +} + +/* + * Record payload. The first sizeof(int) bytes are the value; `pad' extra + * bytes let a scenario make records big enough to force page splits. + */ +#define ISO_PAD_MAX 512 +static u_int32_t iso_pad; + +static int +iso_get(DB *db, DB_TXN *txn, const char *key, int *out) +{ + DBT k, d; + u_int8_t buf[sizeof(int) + ISO_PAD_MAX]; + int rc; + + memset(&k, 0, sizeof(k)); + memset(&d, 0, sizeof(d)); + k.data = (void *)key; + k.size = (u_int32_t)strlen(key); + d.data = buf; + d.ulen = sizeof(buf); + d.flags = DB_DBT_USERMEM; + if ((rc = db->get(db, txn, &k, &d, 0)) != 0) + return (rc); + if (d.size < sizeof(int)) + iso_die("short record", EINVAL); + memcpy(out, buf, sizeof(int)); + return (0); +} + +static int +iso_put(DB *db, DB_TXN *txn, const char *key, int val) +{ + DBT k, d; + u_int8_t buf[sizeof(int) + ISO_PAD_MAX]; + + memset(&k, 0, sizeof(k)); + memset(&d, 0, sizeof(d)); + memset(buf, 0x5a, sizeof(buf)); + memcpy(buf, &val, sizeof(val)); + k.data = (void *)key; + k.size = (u_int32_t)strlen(key); + d.data = buf; + d.size = (u_int32_t)(sizeof(int) + iso_pad); + return (db->put(db, txn, &k, &d, 0)); +} + +static void +iso_note(iso_txn *t, const char *fmt, ...) +{ + va_list ap; + size_t n; + + n = strlen(t->log); + if (n + 2 >= sizeof(t->log)) + return; + if (n != 0) { + t->log[n++] = ';'; + t->log[n++] = ' '; + t->log[n] = '\0'; + } + va_start(ap, fmt); + (void)vsnprintf(t->log + n, sizeof(t->log) - n, fmt, ap); + va_end(ap); +} + +/* + * iso_finish -- + * End a transaction: commit if its operations succeeded, abort if one of + * them already told us to give up. Records the outcome on the txn. + */ +static void +iso_finish(iso_txn *t, DB_TXN *txn, int op_rc) +{ + int rc; + + if (op_rc != 0) { + if ((rc = txn->abort(txn)) != 0) + iso_die("DB_TXN->abort", rc); + t->committed = 0; + t->end_rc = op_rc; + iso_note(t, "abort (%s)", rc_name(op_rc)); + if (!iso_is_abort_rc(op_rc)) + iso_die("unexpected operation failure", op_rc); + return; + } + rc = txn->commit(txn, 0); + t->end_rc = rc; + t->committed = (rc == 0); + iso_note(t, "commit -> %s", rc_name(rc)); + if (rc != 0 && !iso_is_abort_rc(rc)) + iso_die("unexpected commit failure", rc); +} + +/* + * --------------------------------------------------------------------------- + * The serializability verdict. + * --------------------------------------------------------------------------- + */ +static int +iso_state_matches(const iso_scenario *sc, const iso_state *model, + const iso_state *actual, const iso_txn *t) +{ + int i, j; + + for (i = 0; i < sc->ndbkeys; i++) + if (model->v[i] != actual->v[i]) + return (0); + /* Only a committed transaction's observations are binding. */ + for (j = 0; j < sc->ntxn; j++) { + if (!t[j].committed) + continue; + for (i = t[j].obs_lo; i < t[j].obs_hi; i++) + if (model->v[i] != actual->v[i]) + return (0); + } + return (1); +} + +/* + * iso_try_orders -- + * Depth-first enumeration of every permutation of the COMMITTED + * transactions. Returns 1 (and fills `order') as soon as one serial + * order reproduces the observed state. + */ +static int +iso_try_orders(const iso_scenario *sc, const iso_state *cur, + const iso_state *actual, iso_txn *t, int *used, int depth, int ncommitted, + int *order) +{ + iso_state next; + int i; + + if (depth == ncommitted) + return (iso_state_matches(sc, cur, actual, t)); + + for (i = 0; i < sc->ntxn; i++) { + if (used[i] || !t[i].committed) + continue; + used[i] = 1; + next = *cur; + t[i].model(&next); + order[depth] = i; + if (iso_try_orders(sc, &next, actual, t, used, depth + 1, + ncommitted, order)) + return (1); + used[i] = 0; + } + return (0); +} + +/* + * iso_serializable -- + * Is `actual' reachable from `initial' by some serial order of the + * committed transactions? + */ +static int +iso_serializable(const iso_scenario *sc, const iso_state *initial, + const iso_state *actual, iso_txn *t, int *order, int *ncommitted_out) +{ + int i, ncommitted, used[ISO_MAX_TXN]; + + for (i = ncommitted = 0; i < sc->ntxn; i++) { + used[i] = 0; + if (t[i].committed) + ncommitted++; + } + *ncommitted_out = ncommitted; + return (iso_try_orders(sc, initial, actual, t, used, 0, ncommitted, + order)); +} + +/* + * --------------------------------------------------------------------------- + * Scenario 1-4: write skew (the two-doctors shape). + * + * alice and bob are both on call and at least one must stay on call. T1 + * reads bob and takes alice off call; T2 reads alice and takes bob off call. + * Both reads precede both writes, so under serializable isolation one of the + * two must fail. + * + * Slots: 0 = alice.on_call, 1 = bob.on_call. + * --------------------------------------------------------------------------- + */ +#define SK_ALICE 0 +#define SK_BOB 1 + +static void +sk_t1_model(iso_state *s) /* read bob, clear alice */ +{ + if (s->v[SK_BOB]) + s->v[SK_ALICE] = 0; +} + +static void +sk_t2_model(iso_state *s) /* read alice, clear bob */ +{ + if (s->v[SK_ALICE]) + s->v[SK_BOB] = 0; +} + +/* Timing of T2's write relative to T1's commit. */ +enum sk_mode { SK_TRIGGER, SK_CONTROL, SK_LATE }; + +static pthread_barrier_t sk_barrier; +static atomic_int sk_t1_in_commit, sk_t1_commit_done; +static enum sk_mode sk_mode; +static iso_txn *sk_txns; +static DB *sk_alice_db, *sk_bob_db; +static const char *sk_alice_key = "on_call", *sk_bob_key = "on_call"; + +static void +sk_sync(void) +{ + int rc = pthread_barrier_wait(&sk_barrier); + + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) + iso_die("pthread_barrier_wait", rc); +} + +static void * +sk_t2_thread(void *arg) +{ + DB_TXN *txn; + iso_txn *t = &sk_txns[1]; + int alice, rc; + + (void)arg; + if ((rc = env->txn_begin(env, NULL, &txn, DB_TXN_SNAPSHOT)) != 0) + iso_die("T2 txn_begin", rc); + if ((rc = iso_get(sk_alice_db, txn, sk_alice_key, &alice)) != 0) + iso_die("T2 read alice", rc); + iso_note(t, "read alice=%d", alice); + + sk_sync(); /* both reads done */ + sk_sync(); /* T1's write done */ + + /* + * The whole point of the tier: land T2's write while T1 is INSIDE + * DB_TXN->commit. Barriers cannot express "inside a call", so T1 + * publishes an atomic flag immediately before entering commit and we + * spin on it -- application-side only, exactly as issue #136 does. + */ + if (sk_mode == SK_TRIGGER) + while (!atomic_load(&sk_t1_in_commit)) + sched_yield(); + else if (sk_mode == SK_LATE) + while (!atomic_load(&sk_t1_commit_done)) + sched_yield(); + + rc = alice ? iso_put(sk_bob_db, txn, sk_bob_key, 0) : 0; + iso_note(t, "put bob=0 -> %s", rc_name(rc)); + iso_finish(t, txn, rc); + if (sk_mode == SK_CONTROL) + sk_sync(); /* T2 done before T1 commits */ + return (NULL); +} + +static int +sk_run_common(iso_scenario *sc, iso_state *initial, iso_txn *t, + int one_btree, u_int32_t pagesize, u_int32_t pad) +{ + static const char *two_dbs[] = { "alice.db", "bob.db" }; + static const char *one_db[] = { "roster.db" }; + pthread_t t2; + DB_TXN *txn; + int bob, rc; + + t[0].name = "T1"; + t[0].model = sk_t1_model; + t[1].name = "T2"; + t[1].model = sk_t2_model; + + iso_pad = pad; + iso_home_init(sc->name); + if (one_btree) { + iso_env_open(DB_CREATE, 1, one_db, pagesize); + sk_alice_db = sk_bob_db = dbs[0]; + sk_alice_key = "alice"; + sk_bob_key = "bob"; + } else { + iso_env_open(DB_CREATE, 2, two_dbs, pagesize); + sk_alice_db = dbs[0]; + sk_bob_db = dbs[1]; + sk_alice_key = sk_bob_key = "on_call"; + } + + if ((rc = iso_put(sk_alice_db, NULL, sk_alice_key, 1)) != 0 || + (rc = iso_put(sk_bob_db, NULL, sk_bob_key, 1)) != 0) + iso_die("initial put", rc); + initial->v[SK_ALICE] = initial->v[SK_BOB] = 1; + + if (one_btree) { + /* + * The interesting shape is "two records on DIFFERENT pages of + * ONE B-tree". Keys sort as alice < b* < bob, so filling the + * middle with records splits the leaf and pushes alice (the + * minimum key) and bob (the maximum key) onto different + * leaves. The payload must stay under the B-tree overflow + * threshold (pagesize/4) or records move off-page and the leaf + * never splits. + */ + DB_BTREE_STAT *bst; + char fill[16]; + int f; + + for (f = 0; f < 64; f++) { + (void)snprintf(fill, sizeof(fill), "b%04d", f); + if ((rc = iso_put(dbs[0], NULL, fill, f)) != 0) + iso_die("filler put", rc); + } + if ((rc = dbs[0]->stat(dbs[0], NULL, &bst, 0)) != 0) + iso_die("DB->stat", rc); + f = (int)bst->bt_leaf_pg; + free(bst); + if (verbose) + printf(" btree leaf pages = %d (need >= 2 so the " + "min and max key are on different pages)\n", f); + if (f < 2) { + fprintf(stderr, " NOTE: tree did not split;" + " different-pages shape not exercised\n"); + return (-1); + } + } + + if ((rc = pthread_barrier_init(&sk_barrier, NULL, 2)) != 0) + iso_die("pthread_barrier_init", rc); + atomic_store(&sk_t1_in_commit, 0); + atomic_store(&sk_t1_commit_done, 0); + sk_txns = t; + if ((rc = pthread_create(&t2, NULL, sk_t2_thread, NULL)) != 0) + iso_die("pthread_create", rc); + + if ((rc = env->txn_begin(env, NULL, &txn, DB_TXN_SNAPSHOT)) != 0) + iso_die("T1 txn_begin", rc); + if ((rc = iso_get(sk_bob_db, txn, sk_bob_key, &bob)) != 0) + iso_die("T1 read bob", rc); + iso_note(&t[0], "read bob=%d", bob); + sk_sync(); /* both reads done */ + rc = bob ? iso_put(sk_alice_db, txn, sk_alice_key, 0) : 0; + iso_note(&t[0], "put alice=0 -> %s", rc_name(rc)); + sk_sync(); /* T1's write done */ + if (sk_mode == SK_CONTROL) + sk_sync(); /* wait for T2 to finish */ + atomic_store(&sk_t1_in_commit, 1); + iso_finish(&t[0], txn, rc); + atomic_store(&sk_t1_commit_done, 1); + + if ((rc = pthread_join(t2, NULL)) != 0) + iso_die("pthread_join", rc); + (void)pthread_barrier_destroy(&sk_barrier); + return (0); +} + +static int +sk_read_back(iso_scenario *sc, iso_state *actual, int one_btree, + u_int32_t pagesize) +{ + static const char *two_dbs[] = { "alice.db", "bob.db" }; + static const char *one_db[] = { "roster.db" }; + int rc; + + (void)sc; + iso_env_close(); + /* Reopen: the verdict is about the DURABLE committed state. */ + if (one_btree) { + iso_env_open(0, 1, one_db, pagesize); + if ((rc = iso_get(dbs[0], NULL, "alice", + &actual->v[SK_ALICE])) != 0 || + (rc = iso_get(dbs[0], NULL, "bob", + &actual->v[SK_BOB])) != 0) + iso_die("read back", rc); + } else { + iso_env_open(0, 2, two_dbs, pagesize); + if ((rc = iso_get(dbs[0], NULL, "on_call", + &actual->v[SK_ALICE])) != 0 || + (rc = iso_get(dbs[1], NULL, "on_call", + &actual->v[SK_BOB])) != 0) + iso_die("read back", rc); + } + iso_env_close(); + return (0); +} + +static int +sk_run(iso_scenario *sc, iso_state *st, iso_txn *t, enum sk_mode mode, + int one_btree, u_int32_t pagesize, u_int32_t pad) +{ + iso_state initial; + int rc; + + memset(&initial, 0, sizeof(initial)); + sk_mode = mode; + if ((rc = sk_run_common(sc, &initial, t, one_btree, pagesize, + pad)) != 0) + return (rc); + st[0] = initial; + return (sk_read_back(sc, &st[1], one_btree, pagesize)); +} + +static int +sk_trigger(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + return (sk_run(sc, st, t, SK_TRIGGER, 0, 0, 0)); +} + +static int +sk_control(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + return (sk_run(sc, st, t, SK_CONTROL, 0, 0, 0)); +} + +static int +sk_late(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + return (sk_run(sc, st, t, SK_LATE, 0, 0, 0)); +} + +/* + * The possibly-SEPARATE defect reported alongside #136: two records on + * DIFFERENT PAGES of ONE B-tree are said to detect no conflict at all, even + * in the control where T2 commits before T1 calls commit. + * + * OBSERVED on master (c4811dc87), with the shape verified by DB->stat: the + * CONTROL timing DOES return DB_SNAPSHOT_CONFLICT here, exactly like the + * two-one-page-databases control. Only the TRIGGER timing commits both. So + * on this construction the different-pages case has the SAME root cause as + * #136 proper (the commit-window race), and is NOT an additional + * page-granularity conflict-detection hole. The reporter did not publish + * their same-btree variant, so their shape may differ; the control is kept as + * a live PASS expectation precisely so a real page-granularity regression + * would surface here. + * + * 512-byte pages plus filler keys sorting between "alice" and "bob" split the + * leaf so the two records land on different pages. We only OBSERVE here; + * fixing anything belongs to the #136 engine work. + */ +static int +sk_samebtree_control(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + return (sk_run(sc, st, t, SK_CONTROL, 1, 512, 100)); +} + +static int +sk_samebtree_trigger(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + return (sk_run(sc, st, t, SK_TRIGGER, 1, 512, 100)); +} + +/* + * --------------------------------------------------------------------------- + * Scenario: G2 / anti-dependency cycle (G2-item on a predicate read). + * + * Both transactions scan the database counting "marker" records; each inserts + * a marker only if it saw none. Under serializable isolation at most one + * insert may commit. The read is a full cursor scan -- a predicate read -- + * so the anti-dependency runs through the range, not a single record. + * + * Slots: 0 = number of markers stored. + * --------------------------------------------------------------------------- + */ +static void +g2_model(iso_state *s) +{ + if (s->v[0] == 0) + s->v[0] = s->v[0] + 1; +} + +static int +g2_count_markers(DB *db, DB_TXN *txn, int *out) +{ + DBC *dbc; + DBT k, d; + int n, rc; + + if ((rc = db->cursor(db, txn, &dbc, 0)) != 0) + return (rc); + memset(&k, 0, sizeof(k)); + memset(&d, 0, sizeof(d)); + for (n = 0; (rc = dbc->get(dbc, &k, &d, DB_NEXT)) == 0; ) + if (k.size >= 6 && memcmp(k.data, "marker", 6) == 0) + n++; + (void)dbc->close(dbc); + if (rc != DB_NOTFOUND) + return (rc); + *out = n; + return (0); +} + +static int +g2_antidep(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + static const char *names[] = { "markers.db" }; + DB_TXN *txn1, *txn2; + int n1, n2, rc, rc1, rc2; + + t[0].name = "T1"; + t[0].model = g2_model; + t[1].name = "T2"; + t[1].model = g2_model; + + iso_pad = 0; + iso_home_init(sc->name); + iso_env_open(DB_CREATE, 1, names, 0); + /* A non-marker record so the scan has something to walk. */ + if ((rc = iso_put(dbs[0], NULL, "anchor", 0)) != 0) + iso_die("initial put", rc); + memset(&st[0], 0, sizeof(st[0])); + st[0].v[0] = 0; /* no markers */ + + if ((rc = env->txn_begin(env, NULL, &txn1, DB_TXN_SNAPSHOT)) != 0 || + (rc = env->txn_begin(env, NULL, &txn2, DB_TXN_SNAPSHOT)) != 0) + iso_die("txn_begin", rc); + + /* Both predicate reads happen before either write. */ + if ((rc = g2_count_markers(dbs[0], txn1, &n1)) != 0) + iso_die("T1 scan", rc); + if ((rc = g2_count_markers(dbs[0], txn2, &n2)) != 0) + iso_die("T2 scan", rc); + iso_note(&t[0], "scan saw %d markers", n1); + iso_note(&t[1], "scan saw %d markers", n2); + + /* + * T1 writes and commits, then T2 writes from its stale snapshot. The + * anti-dependency (T2's predicate read did not see T1's insert) is + * what SSI must catch; T2 must not be allowed to commit its own + * insert. Writes are serialised this way on purpose: two overlapping + * uncommitted writes to the same page would simply block on the page + * lock, which tests the lock manager rather than isolation. + */ + rc1 = n1 == 0 ? iso_put(dbs[0], txn1, "marker.t1", 1) : 0; + iso_note(&t[0], "insert marker.t1 -> %s", rc_name(rc1)); + iso_finish(&t[0], txn1, rc1); + + rc2 = n2 == 0 ? iso_put(dbs[0], txn2, "marker.t2", 1) : 0; + iso_note(&t[1], "insert marker.t2 -> %s", rc_name(rc2)); + iso_finish(&t[1], txn2, rc2); + + iso_env_close(); + iso_env_open(0, 1, names, 0); + memset(&st[1], 0, sizeof(st[1])); + if ((rc = g2_count_markers(dbs[0], NULL, &st[1].v[0])) != 0) + iso_die("read back", rc); + iso_env_close(); + return (0); +} + +/* + * --------------------------------------------------------------------------- + * Scenario: read-only anomaly (Fekete's three-transaction pattern). + * + * x = savings, y = checking, both start at 0. + * Tdep (deposit): y := y + 20 + * Twdw (withdraw): reads x and y; if x + y >= 11 then y := y - 11 + * else y := y - 11 - 1 (overdraft penalty) + * Tro (read-only): reports x and y. + * + * Schedule: Twdw takes its snapshot first, Tdep commits, Tro runs and commits, + * then Twdw commits. The stored state is fine; Tro's OBSERVATION is what has + * no serial explanation, which is why the checker compares observation slots. + * + * Slots: 0 = x, 1 = y, 2/3 = Tro's observed x/y. + * --------------------------------------------------------------------------- + */ +#define RO_X 0 +#define RO_Y 1 +#define RO_OX 2 +#define RO_OY 3 + +static void +ro_dep_model(iso_state *s) +{ + s->v[RO_Y] += 20; +} + +static void +ro_wdw_model(iso_state *s) +{ + if (s->v[RO_X] + s->v[RO_Y] >= 11) + s->v[RO_Y] -= 11; + else + s->v[RO_Y] -= 12; +} + +static void +ro_ro_model(iso_state *s) +{ + s->v[RO_OX] = s->v[RO_X]; + s->v[RO_OY] = s->v[RO_Y]; +} + +static int +read_only_anomaly(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + static const char *names[] = { "savings.db", "checking.db" }; + DB_TXN *tdep, *twdw, *tro; + int rc, rc_dep, rc_wdw, wx, wy, y; + + t[0].name = "Tdep"; + t[0].model = ro_dep_model; + t[1].name = "Twdw"; + t[1].model = ro_wdw_model; + t[2].name = "Tro"; + t[2].model = ro_ro_model; + t[2].obs_lo = RO_OX; + t[2].obs_hi = RO_OY + 1; + + iso_pad = 0; + iso_home_init(sc->name); + iso_env_open(DB_CREATE, 2, names, 0); + if ((rc = iso_put(dbs[0], NULL, "bal", 0)) != 0 || + (rc = iso_put(dbs[1], NULL, "bal", 0)) != 0) + iso_die("initial put", rc); + memset(&st[0], 0, sizeof(st[0])); + + /* Twdw takes its snapshot first and reads both balances. */ + if ((rc = env->txn_begin(env, NULL, &twdw, DB_TXN_SNAPSHOT)) != 0) + iso_die("Twdw txn_begin", rc); + if ((rc = iso_get(dbs[0], twdw, "bal", &wx)) != 0 || + (rc = iso_get(dbs[1], twdw, "bal", &wy)) != 0) + iso_die("Twdw read", rc); + iso_note(&t[1], "read x=%d y=%d", wx, wy); + + /* Tdep deposits and commits. */ + if ((rc = env->txn_begin(env, NULL, &tdep, DB_TXN_SNAPSHOT)) != 0) + iso_die("Tdep txn_begin", rc); + if ((rc = iso_get(dbs[1], tdep, "bal", &y)) != 0) + iso_die("Tdep read", rc); + rc_dep = iso_put(dbs[1], tdep, "bal", y + 20); + iso_note(&t[0], "y %d -> %d (%s)", y, y + 20, rc_name(rc_dep)); + iso_finish(&t[0], tdep, rc_dep); + + /* Tro starts AFTER Tdep committed and BEFORE Twdw commits. */ + if ((rc = env->txn_begin(env, NULL, &tro, DB_TXN_SNAPSHOT)) != 0) + iso_die("Tro txn_begin", rc); + if ((rc = iso_get(dbs[0], tro, "bal", &st[1].v[RO_OX])) != 0 || + (rc = iso_get(dbs[1], tro, "bal", &st[1].v[RO_OY])) != 0) + iso_die("Tro read", rc); + iso_note(&t[2], "observed x=%d y=%d", + st[1].v[RO_OX], st[1].v[RO_OY]); + iso_finish(&t[2], tro, 0); + + /* Now Twdw writes from its stale snapshot and commits. */ + rc_wdw = iso_put(dbs[1], twdw, "bal", + wx + wy >= 11 ? wy - 11 : wy - 12); + iso_note(&t[1], "y -> %d (%s)", + wx + wy >= 11 ? wy - 11 : wy - 12, rc_name(rc_wdw)); + iso_finish(&t[1], twdw, rc_wdw); + + iso_env_close(); + iso_env_open(0, 2, names, 0); + if ((rc = iso_get(dbs[0], NULL, "bal", &st[1].v[RO_X])) != 0 || + (rc = iso_get(dbs[1], NULL, "bal", &st[1].v[RO_Y])) != 0) + iso_die("read back", rc); + iso_env_close(); + return (0); +} + +/* + * --------------------------------------------------------------------------- + * Scenario: lost update. Both transactions read the same counter and write + * read+1. Under any correct isolation level one must fail; if both commit, + * the counter is 1 and no serial order explains it (serial gives 2). + * + * Slots: 0 = counter. + * --------------------------------------------------------------------------- + */ +static void +lu_model(iso_state *s) +{ + s->v[0] += 1; +} + +static int +lost_update(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + static const char *names[] = { "counter.db" }; + DB_TXN *txn1, *txn2; + int n1, n2, rc, rc1, rc2; + + t[0].name = "T1"; + t[0].model = lu_model; + t[1].name = "T2"; + t[1].model = lu_model; + + iso_pad = 0; + iso_home_init(sc->name); + iso_env_open(DB_CREATE, 1, names, 0); + if ((rc = iso_put(dbs[0], NULL, "n", 0)) != 0) + iso_die("initial put", rc); + memset(&st[0], 0, sizeof(st[0])); + + if ((rc = env->txn_begin(env, NULL, &txn1, DB_TXN_SNAPSHOT)) != 0 || + (rc = env->txn_begin(env, NULL, &txn2, DB_TXN_SNAPSHOT)) != 0) + iso_die("txn_begin", rc); + if ((rc = iso_get(dbs[0], txn1, "n", &n1)) != 0 || + (rc = iso_get(dbs[0], txn2, "n", &n2)) != 0) + iso_die("read", rc); + iso_note(&t[0], "read n=%d", n1); + iso_note(&t[1], "read n=%d", n2); + + rc1 = iso_put(dbs[0], txn1, "n", n1 + 1); + iso_note(&t[0], "put n=%d -> %s", n1 + 1, rc_name(rc1)); + iso_finish(&t[0], txn1, rc1); + rc2 = iso_put(dbs[0], txn2, "n", n2 + 1); + iso_note(&t[1], "put n=%d -> %s", n2 + 1, rc_name(rc2)); + iso_finish(&t[1], txn2, rc2); + + iso_env_close(); + iso_env_open(0, 1, names, 0); + memset(&st[1], 0, sizeof(st[1])); + if ((rc = iso_get(dbs[0], NULL, "n", &st[1].v[0])) != 0) + iso_die("read back", rc); + iso_env_close(); + return (0); +} + +/* + * --------------------------------------------------------------------------- + * Scenario: read-your-writes sanity. A single transaction must see its own + * uncommitted write, and the stored value after commit must match. This + * exists so a vacuously-passing checker is detectable: if the harness ever + * stops driving the engine at all, this scenario fails. + * + * Slots: 0 = value, 1 = the value the transaction read back from itself. + * --------------------------------------------------------------------------- + */ +static void +ryw_model(iso_state *s) +{ + s->v[0] = 7; + s->v[1] = 7; /* it must observe its own write */ +} + +static int +read_your_writes(iso_scenario *sc, iso_state *st, iso_txn *t) +{ + static const char *names[] = { "ryw.db" }; + DB_TXN *txn; + int rc, seen; + + t[0].name = "T1"; + t[0].model = ryw_model; + t[0].obs_lo = 1; + t[0].obs_hi = 2; + + iso_pad = 0; + iso_home_init(sc->name); + iso_env_open(DB_CREATE, 1, names, 0); + if ((rc = iso_put(dbs[0], NULL, "k", 0)) != 0) + iso_die("initial put", rc); + memset(&st[0], 0, sizeof(st[0])); + + if ((rc = env->txn_begin(env, NULL, &txn, DB_TXN_SNAPSHOT)) != 0) + iso_die("txn_begin", rc); + if ((rc = iso_put(dbs[0], txn, "k", 7)) != 0) + iso_die("put", rc); + if ((rc = iso_get(dbs[0], txn, "k", &seen)) != 0) + iso_die("get own write", rc); + iso_note(&t[0], "wrote 7, read back %d", seen); + memset(&st[1], 0, sizeof(st[1])); + st[1].v[1] = seen; + iso_finish(&t[0], txn, 0); + + iso_env_close(); + iso_env_open(0, 1, names, 0); + if ((rc = iso_get(dbs[0], NULL, "k", &st[1].v[0])) != 0) + iso_die("read back", rc); + iso_env_close(); + return (0); +} + +/* + * --------------------------------------------------------------------------- + * The scenario table. + * + * expect_fail marks a scenario that is KNOWN to admit a non-serializable + * history on current master. It is not a licence to be wrong: the runner + * still reports it loudly, and if it starts producing a serializable history + * (i.e. the engine bug is fixed) the run FAILS so the expectation gets + * updated. Flip expect_fail to 0 when the referenced issue lands. + * --------------------------------------------------------------------------- + */ +static iso_scenario scenarios[] = { + { "write_skew_trigger", + "two one-page DBs; T2's write lands while T1 is inside commit", + 2, 2, 2, 1, "#136", 40, sk_trigger }, + { "write_skew_control", + "two one-page DBs; T2 writes and commits before T1 commits", + 2, 2, 2, 0, NULL, 5, sk_control }, + { "write_skew_late", + "two one-page DBs; T2 writes after T1's commit returned", + 2, 2, 2, 0, NULL, 5, sk_late }, + { "write_skew_samebtree_control", + "two records on DIFFERENT pages of ONE btree; control timing", + 2, 2, 2, 0, NULL, 5, sk_samebtree_control }, + { "write_skew_samebtree_trigger", + "two records on DIFFERENT pages of ONE btree; trigger timing", + 2, 2, 2, 1, "#136", 40, sk_samebtree_trigger }, + { "g2_antidep", + "G2-item: both txns scan for markers, both insert one", + 1, 1, 2, 0, NULL, 1, g2_antidep }, + { "read_only_anomaly", + "Fekete 3-txn: read-only txn observes a non-serializable state", + 2, 4, 3, 0, NULL, 1, read_only_anomaly }, + { "lost_update", + "both txns read the counter and write read+1", + 1, 1, 2, 0, NULL, 1, lost_update }, + { "read_your_writes", + "sanity: a txn must observe its own uncommitted write", + 1, 2, 1, 0, NULL, 1, read_your_writes }, +}; +#define NSCENARIOS ((int)(sizeof(scenarios) / sizeof(scenarios[0]))) + +static void +iso_print_state(const iso_scenario *sc, const char *tag, const iso_state *s) +{ + int i; + + printf(" %s: db[", tag); + for (i = 0; i < sc->ndbkeys; i++) + printf("%s%d", i ? "," : "", s->v[i]); + printf("]"); + if (sc->nslots > sc->ndbkeys) { + printf(" obs["); + for (i = sc->ndbkeys; i < sc->nslots; i++) + printf("%s%d", i > sc->ndbkeys ? "," : "", s->v[i]); + printf("]"); + } + printf("\n"); +} + +/* + * run_one_attempt -- + * Run the schedule once. Returns 1 if the resulting history IS + * serializable, 0 if it is not, -1 if the shape could not be set up. + * Prints the schedule when `report' is set (or always, when the history + * is not serializable -- that is the evidence). + */ +static int +run_one_attempt(iso_scenario *sc, int attempt, int report) +{ + iso_state st[2]; /* [0]=initial, [1]=actual */ + iso_txn txns[ISO_MAX_TXN]; + int i, ncommitted, order[ISO_MAX_TXN], serializable, skipped; + + memset(txns, 0, sizeof(txns)); + memset(st, 0, sizeof(st)); + + if ((skipped = sc->run(sc, st, txns)) != 0) + return (-1); + + serializable = iso_serializable(sc, &st[0], &st[1], txns, order, + &ncommitted); + + if (!report && serializable) + return (1); + + if (sc->attempts > 1) + printf(" attempt %d/%d:\n", attempt + 1, sc->attempts); + for (i = 0; i < sc->ntxn; i++) + printf(" %-5s %s\n", txns[i].name, txns[i].log); + iso_print_state(sc, "initial ", &st[0]); + iso_print_state(sc, "observed", &st[1]); + printf(" committed txns = %d/%d, serial order found = %s", + ncommitted, sc->ntxn, serializable ? "yes (" : "NO"); + if (serializable) { + for (i = 0; i < ncommitted; i++) + printf("%s%s", i ? " < " : "", txns[order[i]].name); + printf(")"); + } + printf("\n"); + return (serializable); +} + +/* + * run_scenario -- + * Returns 0 if the outcome matched the expectation, 1 if not. + * + * A scenario PASSES only if EVERY attempt produced a serializable + * history; one violation in any attempt is a reproduction. That + * asymmetry is deliberate: a serializability violation is a real + * counterexample, whereas a single clean run of a racy schedule proves + * nothing. + */ +static int +run_scenario(iso_scenario *sc) +{ + int a, ok, r, violations; + + printf("== %s ==\n shape: %s\n", sc->name, sc->shape); + if (sc->attempts > 1) + printf(" %d attempts (the interleaving is racy; any single " + "violation is a reproduction)\n", sc->attempts); + + for (a = violations = 0; a < sc->attempts; a++) { + /* Report the first attempt, plus every violation. */ + r = run_one_attempt(sc, a, a == 0); + if (r < 0) { + printf(" SKIP: scenario shape could not be " + "established\n\n"); + return (0); + } + if (r == 0) { + violations++; + /* + * One counterexample is enough for an expect_fail + * scenario; keep going otherwise so the report shows + * how reproducible a surprise violation is. + */ + if (sc->expect_fail) { + a++; + break; + } + } + } + if (sc->attempts > 1) + printf(" %d/%d attempt(s) produced a non-serializable " + "history\n", violations, a); + + ok = (violations == 0); + if (ok == sc->expect_fail) { + /* Outcome disagrees with the recorded expectation. */ + if (sc->expect_fail) + printf(" UNEXPECTED PASS: %s is marked as " + "reproducing %s but no attempt produced a " + "non-serializable history -- the issue looks " + "FIXED; clear expect_fail for this scenario.\n\n", + sc->name, sc->issue); + else + printf(" FAIL: no serial order of the committed " + "transactions produces the observed state -- " + "this history is NOT serializable.\n\n"); + return (1); + } + if (sc->expect_fail) + printf(" XFAIL (reproduces %s): the committed history is " + "not serializable, as the issue reports.\n\n", sc->issue); + else + printf(" PASS\n\n"); + return (0); +} + +int +main(int argc, char **argv) +{ + int failures, i, j, ran; + + if (getenv("ISO_VERBOSE") != NULL) + verbose = 1; + if (argc == 2 && strcmp(argv[1], "--list") == 0) { + for (i = 0; i < NSCENARIOS; i++) + printf("%s%s\n", scenarios[i].name, + scenarios[i].expect_fail ? "\t(expect-fail)" : ""); + return (0); + } + + setvbuf(stdout, NULL, _IOLBF, 0); + printf("%s\n\n", db_version(NULL, NULL, NULL)); + + failures = ran = 0; + if (argc == 1) { + for (i = 0; i < NSCENARIOS; i++, ran++) + failures += run_scenario(&scenarios[i]); + } else { + for (j = 1; j < argc; j++) { + for (i = 0; i < NSCENARIOS; i++) + if (strcmp(argv[j], scenarios[i].name) == 0) + break; + if (i == NSCENARIOS) { + fprintf(stderr, "unknown scenario: %s\n", + argv[j]); + return (2); + } + failures += run_scenario(&scenarios[i]); + ran++; + } + } + + printf("%d scenario(s) run, %d unexpected outcome(s)\n", ran, failures); + return (failures != 0); +} diff --git a/test/lockmatrix/README.md b/test/lockmatrix/README.md new file mode 100644 index 000000000..e6a6c5b2b --- /dev/null +++ b/test/lockmatrix/README.md @@ -0,0 +1,135 @@ +# Tier B3 — lock-mode matrix + +Exercises **every** `DB_LOCK_*` mode through the lock-list paths, under ASan, so +that adding a new lock mode cannot silently break list sizing. + +## Why this tier exists + +Issue #140 is a heap out-of-bounds **write** in `__lock_vec`. The root-cause +class is what matters: the SSI work added a new lock mode (`DB_LOCK_SIREAD` = 9) +without auditing pre-existing loops that enumerate lock modes exhaustively. +`__lock_vec`'s `DB_LOCK_PUT_READ` / `DB_LOCK_UPGRADE_WRITE` path sizes an +objlist from `sh_locker->nwrites`, but its release-loop skip condition tests +only `DB_LOCK_READ` and `DB_LOCK_READ_UNCOMMITTED`. `DB_LOCK_SIREAD` matches +neither those read tests nor `IS_WRITELOCK`, so an SIREAD lock is never counted +in `nwrites` yet still consumes an objlist slot — a write past the end of the +allocation. The `DB_ASSERT` that would catch it is diagnostic-only and compiled +out of release builds. + +This tier is the **general matrix**, not a targeted regression test: it asserts +the invariant, so the *next* mode added is covered too. + +## The invariant + +> The set of locks the release loop **skips** must equal the set of locks +> counted in `nwrites`, because `nwrites` sizes the objlist allocation. + +A mode that is neither recognised as a read (and released) nor counted as a +write (and allocated for) breaks it. + +## What it checks + +Entirely through the public `DB_ENV` lock API — `lock_id`, `lock_get`, +`lock_put`, `lock_vec`, `lock_stat`. No internal headers. + +1. **`modes`** — every value in `db_lockmode_t` is acquired and released via + both `lock_get`/`lock_put` and `lock_vec` `DB_LOCK_GET`/`DB_LOCK_PUT_ALL`. A + mode missing from a conflict table or a mode-name switch shows up here. + `DB_LOCK_NG` and `DB_LOCK_WAIT` are listed but not requestable; listing them + keeps the table exhaustive, which is the point. +2. **`conflicts`** — every (held, wanted) cell. The expected verdict is *not* + hard-coded: the engine's conflict table is the specification. What is + asserted are the two properties a table must have regardless of policy: + *totality* (every requestable mode gets a definite granted/`NOTGRANTED` + answer, never an internal error and never a hang) and *symmetry of conflict* + (if held H blocks wanted W then held W blocks wanted H — an asymmetric cell + is how a hand-edited table acquires a hole when a mode is appended). It also + checks `lock_stat`→`st_nmodes` against the number of `db_lockmode_t` values, + which catches "a mode was added without widening the table" directly. The + matrix is printed. +3. **`list`** — the lock-LIST operations that size an objlist from `nwrites`: + `DB_LOCK_PUT_READ` and `DB_LOCK_UPGRADE_WRITE`, driven by a locker holding a + **mix** of write locks and SIREAD locks across a sweep of `(nwrite, nsiread)` + shapes, plus pure-write and write+read shapes as controls. The overflow is + `(nsiread - nwrite)` DBTs, so the sweep walks `nsiread` well past `nwrite`. + +Lock objects are genuine `DB_LOCK_ILOCK`s rather than opaque blobs, because +`__lock_fix_list` treats an object of exactly `sizeof(DB_LOCK_ILOCK)` as a page +lock to be coalesced by fileid — using real ILOCKs is what drives that code. + +## ASan is required + +The interesting failure is an out-of-bounds write inside **libdb's own** +allocation. A harness-only ASan build cannot see it; libdb itself must be +instrumented. So `run.sh` builds (once) an ASan libdb under `build_asan_gate/`, +reusing the same mechanism and directory as `test/fuzz/check-crashes.sh`, and +links against it. + +## Result on current master + +**Reproduces #140.** The very first mixed shape faults: + +``` + PUT_READ nwrite= 0 nsiread= 1 nread= 0 ... +==...==ERROR: AddressSanitizer: heap-buffer-overflow on address ... +WRITE of size 8 at ... thread T0 + #0 ... in __lock_vec .../src/lock/lock.c:457:15 + #1 ... in __lock_vec_api .../src/lock/lock.c:94:9 + #2 ... in __lock_vec_pp .../src/lock/lock.c:76:2 +SUMMARY: AddressSanitizer: heap-buffer-overflow .../src/lock/lock.c:457:15 in __lock_vec +``` + +`nwrite=0, nsiread=1` means the allocation was zero-sized (`nwrites == 0`) and +the loop wrote one DBT into it — the minimal form of the bug. All the +pure-write and write+read control shapes pass first, which is what confirms the +SIREAD mode specifically is the trigger. + +Because an ASan abort kills the process, the harness prints each shape +**before** attempting it, so the last line of output always names the shape that +faulted. + +**Expectations are written for the FIXED engine**: when #140 lands, the whole +matrix must run to completion and exit 0. No `expect_fail` flag is used here — +the tier asserts correct behaviour and currently the engine aborts, which is the +honest signal. + +### It is detectable without ASan too + +The corruption is severe enough that even an **uninstrumented** build trips +glibc's own heap consistency checks. Running the meson-built driver against a +plain `libdb.so` aborts with `double free or corruption (out)` at +`nwrite=1, nsiread=2`. That is a useful second signal — it means the bug is +reachable in a stock build, not only under a sanitizer — but ASan is still what +gives the faulting line (`src/lock/lock.c:457`), so `run.sh` defaults to it. + +## Running it + +```sh +cd test/lockmatrix +./run.sh # build ASan libdb (once) + run every section +./run.sh list # just the #140 sweep +./run.sh modes conflicts # the mode and conflict-matrix sections +./run.sh build # build only + +# Against a non-ASan libdb (the OOB write then goes unnoticed, but the +# mode/conflict checks still run): +LIBDB_ASAN=0 ./run.sh +``` + +Under meson (plain, non-ASan libdb — still aborts, via glibc): + +```sh +meson setup build && ninja -C build test/tiers/test_lock_matrix +meson test -C build --suite tiers-xfail +``` + +Environment: `CC` (default `clang`, needed for the ASan libdb), `LIBDB_ASAN` +(default 1), `LIBDB_BUILD` (explicit build dir, skips the ASan auto-build), +`LOCK_TIMEOUT` (default 600s). + +## Exit status + +- `0` — the matrix completed and every check held. +- `1` — a check failed, **or** ASan aborted (the sanitizer's own exit). The + heap-buffer-overflow report plus the last progress line identify the shape. +- `2` — harness error. diff --git a/test/lockmatrix/run.sh b/test/lockmatrix/run.sh new file mode 100755 index 000000000..b7391b7fe --- /dev/null +++ b/test/lockmatrix/run.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# test/lockmatrix/run.sh -- build and run the Tier B3 lock-mode matrix. +# +# Exercises every db_lockmode_t through lock_get / lock_put / lock_vec, the +# whole conflict matrix, and the lock-LIST operations (DB_LOCK_PUT_READ, +# DB_LOCK_UPGRADE_WRITE) with a locker holding a MIX of write and SIREAD +# locks -- the shape that overflows the objlist allocation in issue #140. +# +# The interesting failure is an out-of-bounds WRITE inside libdb's own +# allocation, which is only observable when LIBDB ITSELF is ASan-instrumented. +# A harness-only ASan build cannot see it. So by default this builds (once, +# reusing the fuzz tier's mechanism and directory) an ASan libdb under +# build_asan_gate/ and links against that. +# +# On current master (#140 unfixed) an ASan run is EXPECTED to abort with a +# heap-buffer-overflow inside __lock_vec. The harness prints each shape +# before attempting it, so the last line of output names the shape that +# faulted. When #140 lands, the whole matrix must complete and exit 0. +# +# Usage: +# ./run.sh # build (ASan libdb) + run every section +# ./run.sh build # build only +# ./run.sh modes|conflicts|list ... +# +# Env: +# CC compiler (default: clang; needed for the ASan libdb) +# LIBDB_ASAN 1 => build/use an ASan-instrumented libdb (default 1) +# LIBDB_BUILD explicit libdb build dir (skips the ASan auto-build) +# LOCK_TIMEOUT seconds for the run (default 600) +# +# Run from test/lockmatrix/ inside a `nix develop` shell. + +set -eu + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$HERE" + +CC=${CC:-clang} +LIBDB_ASAN=${LIBDB_ASAN:-1} +LOCK_TIMEOUT=${LOCK_TIMEOUT:-600} +OUT="$HERE/build" + +# Same mechanism as test/fuzz/check-crashes.sh: one shared ASan libdb under +# build_asan_gate/, built on demand and reused by both tiers. +if [ "$LIBDB_ASAN" = "1" ] && [ -z "${LIBDB_BUILD:-}" ]; then + GATE_BUILD="$HERE/../../build_asan_gate" + if [ ! -f "$GATE_BUILD/libdb.a" ]; then + echo "building ASan libdb in $GATE_BUILD (once) ..." + mkdir -p "$GATE_BUILD" + ( cd "$GATE_BUILD" && + ../dist/configure --enable-debug \ + CC="$CC" CFLAGS="-fsanitize=address -g -O1" \ + >configure.log 2>&1 && + make -j"$(nproc 2>/dev/null || echo 4)" >build.log 2>&1 ) || + echo "warning: ASan libdb build failed; falling back" >&2 + fi + [ -f "$GATE_BUILD/libdb.a" ] && LIBDB_BUILD="$GATE_BUILD" +fi +LIBDB_BUILD=${LIBDB_BUILD:-"$HERE/../../build_unix"} +LIBDBA="$LIBDB_BUILD/libdb.a" + +if [ -f "$LIBDB_BUILD/Makefile" ]; then + LDLIBS=$(sed -n 's/^LIBS=[[:space:]]*//p' "$LIBDB_BUILD/Makefile" | head -1) +fi +LDLIBS="${LDLIBS:--lpthread} -ldl -lpthread" + +# ASan on the harness too, so the harness's own DBTs are red-zoned. +CFLAGS="-g -O1 -Wall -Wextra -Wno-unused-parameter -fsanitize=address" +CFLAGS="$CFLAGS -I$LIBDB_BUILD -I$HERE" + +[ -f "$LIBDBA" ] || { + echo "error: libdb.a not found at $LIBDBA -- build libdb first:" >&2 + echo " (cd $LIBDB_BUILD && ../dist/configure --enable-debug && make -j4)" >&2 + exit 2 +} +echo "linking against $LIBDBA" + +mkdir -p "$OUT" +# shellcheck disable=SC2086 +$CC $CFLAGS "$HERE/test_lock_matrix.c" "$LIBDBA" $LDLIBS \ + -o "$OUT/test_lock_matrix" +echo "built $OUT/test_lock_matrix" + +[ "${1:-}" = "build" ] && exit 0 + +cd "$OUT" +exec timeout "$LOCK_TIMEOUT" ./test_lock_matrix "$@" diff --git a/test/lockmatrix/test_lock_matrix.c b/test/lockmatrix/test_lock_matrix.c new file mode 100644 index 000000000..a5e14d35a --- /dev/null +++ b/test/lockmatrix/test_lock_matrix.c @@ -0,0 +1,523 @@ +/*- + * test/lockmatrix/test_lock_matrix.c -- + * Tier B3: exhaustive lock-mode matrix through the lock-list paths. + * + * The blind spot this closes: adding a new DB_LOCK_* mode silently breaking a + * pre-existing loop that enumerates modes exhaustively. Issue #140 is exactly + * that -- DB_LOCK_SIREAD (=9, the SSI addition) is neither counted as a write + * by nwrites nor recognised by the read tests in __lock_vec's DB_LOCK_PUT_READ + * / DB_LOCK_UPGRADE_WRITE path, so an SIREAD lock consumes an objlist DBT slot + * that was never allocated: a heap out-of-bounds WRITE. The DB_ASSERT that + * would catch it is diagnostic-only and compiled out of release builds. + * + * What this exercises, entirely through the public DB_ENV lock API: + * 1. Every mode in db_lockmode_t is acquired and released (lock_get / + * lock_put and lock_vec DB_LOCK_GET / DB_LOCK_PUT), so a mode missing + * from a conflict table or a mode-name switch shows up. + * 2. Every conflict-matrix cell: for each (held, wanted) pair, a second + * locker requests the mode with DB_LOCK_NOWAIT and the outcome is + * compared against the engine's own conflict verdict for consistency. + * 3. The lock-LIST operations that size an objlist from nwrites -- + * DB_LOCK_PUT_READ and DB_LOCK_UPGRADE_WRITE -- driven by a locker + * holding a MIX of write locks and SIREAD locks, across a sweep of + * (nwrite, nsiread) shapes. This is the #140 shape: the more SIREAD + * locks beyond the number of write locks, the further past the end of + * the allocation the loop writes. + * + * Run it under ASan (SAN=1, or point LIBDB_BUILD at build_asan_gate) so the + * out-of-bounds write is observed rather than silently tolerated. + * + * EXPECTATIONS ARE WRITTEN FOR THE FIXED ENGINE. On current master the + * mixed-mode PUT_READ cases are expected to fault under an ASan-instrumented + * libdb; that is the tier reproducing #140. Because an ASan fault aborts the + * process, the harness prints its progress line BEFORE each risky call, so the + * last line in the log names the exact shape that faulted. When #140 lands, + * the whole matrix must run to completion and exit 0. + * + * Usage: + * ./test_lock_matrix every section + * ./test_lock_matrix modes|conflicts|list one section + * + * Exit status: 0 = the matrix completed and every check held, 1 = a check + * failed, 2 = harness error. An ASan abort (exit 1 from the sanitizer with + * a heap-buffer-overflow report) is the #140 reproduction. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db.h" + +/* + * Every mode in db_lockmode_t (src/dbinc/db.in). DB_LOCK_NG (not granted) + * and DB_LOCK_WAIT (an event, not a lock) are not requestable modes, so they + * are listed but skipped for acquisition -- listing them keeps the table + * exhaustive, which is the point of the tier. + */ +static const struct { + db_lockmode_t mode; + const char *name; + int requestable; + int is_write; /* Per IS_WRITELOCK, src/dbinc/lock.h */ +} modes[] = { + { DB_LOCK_NG, "NG", 0, 0 }, + { DB_LOCK_READ, "READ", 1, 0 }, + { DB_LOCK_WRITE, "WRITE", 1, 1 }, + { DB_LOCK_WAIT, "WAIT", 0, 0 }, + { DB_LOCK_IWRITE, "IWRITE", 1, 1 }, + { DB_LOCK_IREAD, "IREAD", 1, 0 }, + { DB_LOCK_IWR, "IWR", 1, 1 }, + { DB_LOCK_READ_UNCOMMITTED, "READ_UNCOMMITTED", 1, 0 }, + { DB_LOCK_WWRITE, "WWRITE", 1, 1 }, + { DB_LOCK_SIREAD, "SIREAD", 1, 0 }, +}; +#define NMODES ((int)(sizeof(modes) / sizeof(modes[0]))) + +static DB_ENV *env; +static int failures; + +static void lm_die(const char *, int) __attribute__((noreturn)); + +static void +lm_die(const char *what, int rc) +{ + fprintf(stderr, "harness error: %s: %s (%d)\n", + what, db_strerror(rc), rc); + exit(2); +} + +static void +lm_fail(const char *fmt, ...) +{ + va_list ap; + + printf(" FAIL: "); + va_start(ap, fmt); + (void)vprintf(fmt, ap); + va_end(ap); + printf("\n"); + failures++; +} + +static const char * +rc_name(int rc) +{ + if (rc == 0) + return ("granted"); + switch (rc) { + case DB_LOCK_NOTGRANTED: return ("NOTGRANTED"); + case DB_LOCK_DEADLOCK: return ("DEADLOCK"); + default: return (db_strerror(rc)); + } +} + +static void +lm_env_open(void) +{ + static const char *home = "LOCKDIR"; + char cmd[256]; + int rc; + + (void)mkdir(home, 0755); + (void)snprintf(cmd, sizeof(cmd), + "find '%s' -mindepth 1 -delete 2>/dev/null", home); + (void)system(cmd); + + if ((rc = db_env_create(&env, 0)) != 0) + lm_die("db_env_create", rc); + /* + * Room for the widest shape the list section builds, and a lock + * timeout so a NOWAIT-less request can never hang the tier. + */ + if ((rc = env->set_lk_max_locks(env, 5000)) != 0 || + (rc = env->set_lk_max_lockers(env, 500)) != 0 || + (rc = env->set_lk_max_objects(env, 5000)) != 0) + lm_die("set_lk_max_*", rc); + if ((rc = env->set_timeout(env, 1000000, DB_SET_LOCK_TIMEOUT)) != 0) + lm_die("set_timeout", rc); + if ((rc = env->open(env, home, + DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_THREAD, 0600)) != 0) + lm_die("DB_ENV->open", rc); +} + +/* + * lm_obj -- + * Build a lock object. Real page locks are DB_LOCK_ILOCKs, and + * __lock_fix_list treats an object of exactly sizeof(DB_LOCK_ILOCK) as a + * page lock to be coalesced by fileid. Using genuine ILOCKs is what + * drives that coalescing code, so the tier uses them rather than opaque + * blobs. + */ +static void +lm_obj(DBT *dbt, DB_LOCK_ILOCK *ilock, u_int8_t fileid, db_pgno_t pgno, + u_int32_t type) +{ + memset(ilock, 0, sizeof(*ilock)); + memset(ilock->fileid, fileid, DB_FILE_ID_LEN); + ilock->pgno = pgno; + ilock->type = type; + memset(dbt, 0, sizeof(*dbt)); + dbt->data = ilock; + dbt->size = sizeof(*ilock); +} + +/* + * --------------------------------------------------------------------------- + * Section 1: every mode acquired and released. + * --------------------------------------------------------------------------- + */ +static void +section_modes(void) +{ + DB_LOCK lock; + DB_LOCKREQ req; + DB_LOCK_ILOCK ilock; + DBT obj; + u_int32_t locker; + int i, rc; + + printf("== modes: acquire and release every db_lockmode_t ==\n"); + for (i = 0; i < NMODES; i++) { + if (!modes[i].requestable) { + printf(" %-18s skipped (not a requestable mode)\n", + modes[i].name); + continue; + } + if ((rc = env->lock_id(env, &locker)) != 0) + lm_die("lock_id", rc); + lm_obj(&obj, &ilock, 1, (db_pgno_t)i, 0); + + /* lock_get / lock_put. */ + rc = env->lock_get(env, locker, DB_LOCK_NOWAIT, &obj, + modes[i].mode, &lock); + if (rc != 0) + lm_fail("%s: lock_get on an unheld object -> %s", + modes[i].name, rc_name(rc)); + else if ((rc = env->lock_put(env, &lock)) != 0) + lm_fail("%s: lock_put -> %s", + modes[i].name, rc_name(rc)); + + /* The same through lock_vec, which is a different path. */ + memset(&req, 0, sizeof(req)); + req.op = DB_LOCK_GET; + req.mode = modes[i].mode; + req.obj = &obj; + if ((rc = env->lock_vec(env, locker, DB_LOCK_NOWAIT, &req, 1, + NULL)) != 0) + lm_fail("%s: lock_vec DB_LOCK_GET -> %s", + modes[i].name, rc_name(rc)); + else { + memset(&req, 0, sizeof(req)); + req.op = DB_LOCK_PUT_ALL; + if ((rc = env->lock_vec(env, locker, 0, &req, 1, + NULL)) != 0) + lm_fail("%s: lock_vec DB_LOCK_PUT_ALL -> %s", + modes[i].name, rc_name(rc)); + } + if (rc == 0) + printf(" %-18s get/put and vec get/put_all OK\n", + modes[i].name); + if ((rc = env->lock_id_free(env, locker)) != 0) + lm_die("lock_id_free", rc); + } + printf("\n"); +} + +/* + * --------------------------------------------------------------------------- + * Section 2: the whole conflict matrix. + * + * For every (held, wanted) pair, locker A takes `held' and locker B requests + * `wanted' with DB_LOCK_NOWAIT. We do not hard-code the expected verdict: + * the engine's conflict table is the specification, and lock_stat's st_nmodes + * tells us how wide it is. What we ASSERT is the two properties a table must + * have regardless of policy: + * + * - it is total: every requestable mode gets a definite granted / + * NOTGRANTED answer, never an internal error, and never a hang. + * - it is symmetric in conflict: if held H blocks wanted W, then held W + * blocks wanted H. An asymmetric cell is how a hand-edited table + * acquires a hole when a mode is appended. + * --------------------------------------------------------------------------- + */ +static int +conflict_probe(int held, int wanted, db_pgno_t pgno, int *rc_out) +{ + DB_LOCK hl, wl; + DB_LOCK_ILOCK ilock; + DBT obj; + u_int32_t la, lb; + int rc; + + if ((rc = env->lock_id(env, &la)) != 0 || + (rc = env->lock_id(env, &lb)) != 0) + lm_die("lock_id", rc); + lm_obj(&obj, &ilock, 2, pgno, 0); + + if ((rc = env->lock_get(env, la, DB_LOCK_NOWAIT, &obj, + modes[held].mode, &hl)) != 0) { + /* Could not establish the precondition; not a matrix result. */ + (void)env->lock_id_free(env, la); + (void)env->lock_id_free(env, lb); + *rc_out = rc; + return (-1); + } + rc = env->lock_get(env, lb, DB_LOCK_NOWAIT, &obj, + modes[wanted].mode, &wl); + *rc_out = rc; + if (rc == 0) + (void)env->lock_put(env, &wl); + (void)env->lock_put(env, &hl); + (void)env->lock_id_free(env, la); + (void)env->lock_id_free(env, lb); + return (rc == 0 ? 0 : 1); /* 0 = compatible, 1 = blocks */ +} + +static void +section_conflicts(void) +{ + DB_LOCK_STAT *lst; + int blocks[NMODES][NMODES]; + int h, rc, w; + db_pgno_t pgno; + + printf("== conflicts: every (held, wanted) cell ==\n"); + if ((rc = env->lock_stat(env, &lst, 0)) != 0) + lm_die("lock_stat", rc); + printf(" conflict table is %d modes wide; db_lockmode_t has %d " + "values\n", lst->st_nmodes, NMODES); + if (lst->st_nmodes < NMODES) + lm_fail("the conflict table (%d modes) is NARROWER than " + "db_lockmode_t (%d values) -- a mode was added without " + "widening the table", lst->st_nmodes, NMODES); + free(lst); + + pgno = 100; + for (h = 0; h < NMODES; h++) + for (w = 0; w < NMODES; w++) { + blocks[h][w] = -1; + if (!modes[h].requestable || !modes[w].requestable) + continue; + blocks[h][w] = conflict_probe(h, w, pgno++, &rc); + if (blocks[h][w] < 0) + lm_fail("could not hold %s to probe %s: %s", + modes[h].name, modes[w].name, rc_name(rc)); + else if (rc != 0 && rc != DB_LOCK_NOTGRANTED && + rc != DB_LOCK_DEADLOCK) + lm_fail("held %s, wanted %s: unexpected %s", + modes[h].name, modes[w].name, rc_name(rc)); + } + + /* Print the matrix; "." = compatible, "X" = blocks, "-" = n/a. */ + printf(" held \\ wanted"); + for (w = 0; w < NMODES; w++) + printf(" %2d", (int)modes[w].mode); + printf("\n"); + for (h = 0; h < NMODES; h++) { + printf(" %-18s", modes[h].name); + for (w = 0; w < NMODES; w++) + printf(" %2s", blocks[h][w] < 0 ? "-" : + blocks[h][w] ? "X" : "."); + printf("\n"); + } + + for (h = 0; h < NMODES; h++) + for (w = h + 1; w < NMODES; w++) + if (blocks[h][w] >= 0 && blocks[w][h] >= 0 && + blocks[h][w] != blocks[w][h]) + lm_fail("conflict table is asymmetric: " + "held %s / wanted %s says %s, but " + "held %s / wanted %s says %s", + modes[h].name, modes[w].name, + blocks[h][w] ? "conflict" : "compatible", + modes[w].name, modes[h].name, + blocks[w][h] ? "conflict" : "compatible"); + printf("\n"); +} + +/* + * --------------------------------------------------------------------------- + * Section 3: the lock-LIST operations -- the #140 shape. + * + * DB_LOCK_PUT_READ and DB_LOCK_UPGRADE_WRITE in __lock_vec walk a locker's + * held-lock list and, when the caller passes an obj DBT, fill it with one DBT + * per lock the loop did NOT release. The allocation is sized from + * sh_locker->nwrites. So the invariant is: + * + * { locks the release-loop skips } == { locks counted in nwrites } + * + * A mode that is neither recognised as a read (and so released) nor counted as + * a write (and so allocated for) breaks it and the loop writes past the end of + * the allocation. DB_LOCK_SIREAD is such a mode on current master (#140). + * + * The sweep below builds lockers holding nwrite WRITE locks and nsiread SIREAD + * locks and then issues PUT_READ / UPGRADE_WRITE with an objlist. The + * overflow is (nsiread - nwrite) DBTs when nsiread > nwrite, so the sweep goes + * well past nsiread == nwrite. Each shape is announced BEFORE the call, so if + * ASan aborts, the last printed line is the shape that overflowed. + * --------------------------------------------------------------------------- + */ +#define LIST_MAXLOCK 32 + +static int +list_shape(db_lockop_t op, const char *opname, int nwrite, int nsiread, + int nread, db_pgno_t base) +{ + DB_LOCK locks[LIST_MAXLOCK]; + DB_LOCKREQ req; + DB_LOCK_ILOCK ilock[LIST_MAXLOCK]; + DBT objs[LIST_MAXLOCK], objlist; + u_int32_t locker; + int i, n, rc; + + if ((rc = env->lock_id(env, &locker)) != 0) + lm_die("lock_id", rc); + + n = 0; + for (i = 0; i < nwrite; i++, n++) { + lm_obj(&objs[n], &ilock[n], 3, base + (db_pgno_t)n, 0); + if ((rc = env->lock_get(env, locker, DB_LOCK_NOWAIT, &objs[n], + DB_LOCK_WRITE, &locks[n])) != 0) + lm_die("lock_get WRITE", rc); + } + for (i = 0; i < nsiread; i++, n++) { + lm_obj(&objs[n], &ilock[n], 3, base + (db_pgno_t)n, 0); + if ((rc = env->lock_get(env, locker, DB_LOCK_NOWAIT, &objs[n], + DB_LOCK_SIREAD, &locks[n])) != 0) + lm_die("lock_get SIREAD", rc); + } + for (i = 0; i < nread; i++, n++) { + lm_obj(&objs[n], &ilock[n], 3, base + (db_pgno_t)n, 0); + if ((rc = env->lock_get(env, locker, DB_LOCK_NOWAIT, &objs[n], + DB_LOCK_READ, &locks[n])) != 0) + lm_die("lock_get READ", rc); + } + + /* + * Announce BEFORE the call: an ASan heap-buffer-overflow aborts the + * process inside lock_vec, so this line is the evidence of which + * shape did it. + */ + printf(" %-14s nwrite=%2d nsiread=%2d nread=%2d ...", + opname, nwrite, nsiread, nread); + fflush(stdout); + + memset(&objlist, 0, sizeof(objlist)); + memset(&req, 0, sizeof(req)); + req.op = op; + req.obj = &objlist; + rc = env->lock_vec(env, locker, 0, &req, 1, NULL); + printf(" %s", rc == 0 ? "ok" : rc_name(rc)); + if (rc == 0) + printf(", objlist %u bytes", objlist.size); + printf("\n"); + if (rc != 0) + lm_fail("%s with nwrite=%d nsiread=%d nread=%d -> %s", + opname, nwrite, nsiread, nread, rc_name(rc)); + if (objlist.data != NULL) + free(objlist.data); + + memset(&req, 0, sizeof(req)); + req.op = DB_LOCK_PUT_ALL; + if ((rc = env->lock_vec(env, locker, 0, &req, 1, NULL)) != 0 && + rc != DB_LOCK_NOTGRANTED) + lm_fail("PUT_ALL after %s -> %s", opname, rc_name(rc)); + if ((rc = env->lock_id_free(env, locker)) != 0) + lm_die("lock_id_free", rc); + return (0); +} + +static void +section_list(void) +{ + db_pgno_t base; + int nsiread, nwrite; + + printf("== list: PUT_READ / UPGRADE_WRITE with an objlist ==\n"); + printf(" invariant: the locks the release loop SKIPS must all be " + "counted in nwrites,\n because nwrites sizes the objlist " + "allocation. A mode that is neither\n released as a read nor " + "counted as a write overflows it (#140: SIREAD).\n"); + + base = 1000; + /* Pure write shapes: the case the original code was written for. */ + for (nwrite = 0; nwrite <= 4; nwrite++) { + list_shape(DB_LOCK_PUT_READ, "PUT_READ", + nwrite, 0, 0, base); + base += LIST_MAXLOCK; + } + /* Writes plus plain reads: reads are released, so still balanced. */ + for (nwrite = 1; nwrite <= 3; nwrite++) { + list_shape(DB_LOCK_PUT_READ, "PUT_READ", + nwrite, 0, 3, base); + base += LIST_MAXLOCK; + } + /* + * The #140 sweep: a MIX of write locks and SIREAD locks. The + * overflow is (nsiread - nwrite) DBTs, so walk nsiread past nwrite. + */ + for (nwrite = 0; nwrite <= 3; nwrite++) + for (nsiread = 0; nsiread <= 6; nsiread++) { + list_shape(DB_LOCK_PUT_READ, "PUT_READ", + nwrite, nsiread, 0, base); + base += LIST_MAXLOCK; + } + /* Same mix through the UPGRADE_WRITE arm of the same loop. */ + for (nwrite = 0; nwrite <= 2; nwrite++) + for (nsiread = 0; nsiread <= 4; nsiread++) { + list_shape(DB_LOCK_UPGRADE_WRITE, "UPGRADE_WRITE", + nwrite, nsiread, 0, base); + base += LIST_MAXLOCK; + } + /* All three kinds at once. */ + for (nsiread = 1; nsiread <= 4; nsiread++) { + list_shape(DB_LOCK_PUT_READ, "PUT_READ", + 2, nsiread, 2, base); + base += LIST_MAXLOCK; + } + printf("\n"); +} + +int +main(int argc, char **argv) +{ + int rc, want_conflicts, want_list, want_modes; + + setvbuf(stdout, NULL, _IOLBF, 0); + want_conflicts = want_list = want_modes = (argc == 1); + if (argc > 1) { + int i; + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "modes") == 0) + want_modes = 1; + else if (strcmp(argv[i], "conflicts") == 0) + want_conflicts = 1; + else if (strcmp(argv[i], "list") == 0) + want_list = 1; + else { + fprintf(stderr, "usage: %s " + "[modes|conflicts|list]...\n", argv[0]); + return (2); + } + } + } + + printf("%s\n\n", db_version(NULL, NULL, NULL)); + lm_env_open(); + if (want_modes) + section_modes(); + if (want_conflicts) + section_conflicts(); + if (want_list) + section_list(); + if ((rc = env->close(env, 0)) != 0) + lm_die("DB_ENV->close", rc); + + printf("%d check(s) failed\n", failures); + return (failures != 0); +} diff --git a/test/soak/README.md b/test/soak/README.md new file mode 100644 index 000000000..82bcab229 --- /dev/null +++ b/test/soak/README.md @@ -0,0 +1,127 @@ +# Tier B2 — resource-accounting soak + +Runs thousands of **sequential** transactions in **one long-lived** +environment and asserts that region resources return to a steady state instead +of growing with the transaction count. + +## Why this tier exists + +A slot/mutex/locker leak produces no crash, no corrupt page and no sanitizer +report. It only manifests after thousands of sequential transactions, when some +later API call returns `ENOMEM` because the region filled up. Neither the +crash/durability tier (`test/sim`) nor the memory-safety tier (`test/fuzz`) can +see that shape. Issues #137 and #138 are both of it. + +This is a general, reusable soak harness parameterised by workload — not a +targeted regression test for one bug. Adding a workload is one function plus one +table row. + +## Method + +One environment, opened once with **default region sizes** (the point is that a +correct engine does not need a bigger region to run 2000 sequential +transactions, and a leak surfaces as `ENOMEM` precisely because the region is +finite). N transactions of the workload run sequentially, sampling the **public** +stat APIs at intervals: + +| Counter | Source | +|---|---| +| `mutex_inuse` | `DB_ENV->mutex_stat` → `st_mutex_inuse` | +| `lock_lockers` | `DB_ENV->lock_stat` → `st_nlockers` | +| `lock_locks` | `DB_ENV->lock_stat` → `st_nlocks` | +| `lock_objects` | `DB_ENV->lock_stat` → `st_nobjects` | +| `txn_active` | `DB_ENV->txn_stat` → `st_nactive` | +| `txn_snapshot` | `DB_ENV->txn_stat` → `st_nsnapshot` | +| `mpool_dirty` | `DB_ENV->memp_stat` → `st_page_dirty` | + +The verdict is the **least-squares slope** over the samples taken after warmup, +in counter units per 1000 transactions, against a per-counter tolerance +(documented in the `counters[]` table). Least squares rather than +last-minus-first so a single noisy endpoint cannot decide the verdict. + +The first quarter of samples is warmup: lazy region allocation and cache fill +legitimately grow counters there, and the tier should not fight normal +behaviour. Only the steady state is asserted. + +Tolerances are 20 units per 1000 transactions for the region counters — a +genuine per-transaction leak grows at ~1000 units per 1000 transactions, three +orders of magnitude above the tolerance, so the check is not marginal. + +An `ENOMEM` / `DB_RUNRECOVERY` from any API call is recorded (with the call name +and transaction number) and fails the workload, but does **not** abort the run: +"ENOMEM at transaction 1187" is a much better report than a stack trace. + +The **full growth curve is always printed**, so a CI log alone is enough to +diagnose a regression without re-running locally. + +## Workloads + +| Workload | Shape | Expectation on master | +|---|---|---| +| `ro_snapshot` | read-only `DB_TXN_SNAPSHOT` txns, no write | **XFAIL — reproduces #137** | +| `mvcc_retained` | snapshot txns that read *and* write, so their details are MVCC-retained then reaped | **XFAIL — reproduces #138** (see caveat) | +| `rw_plain` | ordinary read-write txns, no snapshot | PASS (flat) | +| `aborted` | snapshot txns that all abort | PASS (flat) | +| `cursor_churn` | plain txns that open, walk and close a cursor | PASS (flat) | + +Observed on master (c4811dc87), 2000 transactions: `ro_snapshot` leaks +**+1000.00 mutex slots and +1000.00 lockers per 1000 transactions** — exactly +one of each per transaction, never returned. The three controls are flat +(`±0.00` to `-1.23`), which is what makes the leak signal credible rather than a +measurement artefact. + +`cursor_churn` is deliberately a **plain** transaction, not a snapshot one: as a +snapshot reader it tripped the #137 locker leak and was simply a second copy of +`ro_snapshot`, telling us nothing about cursors. With a plain txn, growth there +is genuinely a cursor/lock-list accounting problem. + +### Caveat on `mvcc_retained` / #138 + +#138 is a leak in `__txn_reap_si_details` (`src/txn/txn_region.c`), which frees +a parked transaction detail without freeing its `mvcc_mtx`. Reaching it needs +the detail to be parked on the `mvcc_txn` list *and* its SIREAD markers +garbage-collected *and* its MVCC pages evicted. The workload reads several keys +(leaving markers) and writes over a 512-key space (creating MVCC versions and +driving cache turnover) to get there. Whether a given N reaches the reap path is +timing- and cache-dependent; if this workload reports UNEXPECTED PASS, that may +mean the reap path was not reached rather than that #138 is fixed. Check the +`mutex_inuse` column of the printed curve before concluding anything, and see +the targeted test from the #137/#138 fix for a deterministic reproducer. + +## Running it + +```sh +# Build libdb first (once): +cd build_unix && ../dist/configure --enable-debug && make -j"$(nproc)" + +cd test/soak +./run.sh # every workload, 2000 txns each +SOAK_N=10000 ./run.sh # longer soak +./run.sh ro_snapshot # one workload +./run.sh --list # workload names, with expect-leak marked +./run.sh build # build only +``` + +Environment: `CC`, `LIBDB_BUILD` (default `../../build_unix`), `SOAK_N` (default +2000), `SOAK_TIMEOUT` (default 900s), `SOAK_SAN=1` to add ASan. + +## Exit status + +- `0` — every workload matched its recorded expectation. +- `1` — a workload did not. Either a control leaked, **or** an expect-leak + workload stayed flat, meaning the referenced issue got fixed and + `expect_leak` should be cleared in the table in `test_soak_resources.c`. The + message says which. +- `2` — harness error. + +When #137/#138 land, clear `expect_leak` on the corresponding workloads; the +tier then gates the fixes against regression. + +## Relation to the targeted leak tests + +The #137/#138 fix adds its own targeted regression tests. This tier is the +general soak: it is parameterised by workload and lives under `test/soak/` (a +distinct directory, no file-name collisions with `test/c/`). The targeted tests +prove a specific code path frees a specific resource; this tier proves the +*aggregate* accounting is stable over a long run, which is the property that +would have caught both issues before release. diff --git a/test/soak/run.sh b/test/soak/run.sh new file mode 100755 index 000000000..7ff9beda9 --- /dev/null +++ b/test/soak/run.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# test/soak/run.sh -- build and run the Tier B2 resource-accounting soak. +# +# Runs thousands of SEQUENTIAL transactions in ONE long-lived environment and +# asserts that region resources (mutex slots, lockers, locks, objects, txn +# details, dirty pages) return to a steady state instead of growing with the +# transaction count. Counts come from the public stat APIs only +# (DB_ENV->mutex_stat / lock_stat / txn_stat / memp_stat). +# +# A non-zero exit means a workload disagreed with its recorded expectation: +# either a control workload leaked, or a known-leaking workload stayed flat +# (the issue got fixed -- clear expect_leak in the table). +# +# The growth curve is printed for every workload, so a CI log is enough to +# diagnose a regression without re-running locally. +# +# Usage: +# ./run.sh # build + run every workload, default N +# ./run.sh build # build only +# ./run.sh WORKLOAD ... # build + run the named workloads +# ./run.sh --list # list workload names +# +# Env: +# CC compiler (default: cc) +# LIBDB_BUILD path to a built build_unix (default: ../../build_unix) +# SOAK_N transactions per workload (default 2000) +# SOAK_TIMEOUT seconds for the whole run (default 900) +# SOAK_SAN 1 => build with ASan (default 0) +# +# Run from test/soak/ inside a `nix develop` shell. + +set -eu + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$HERE" + +CC=${CC:-cc} +LIBDB_BUILD=${LIBDB_BUILD:-"$HERE/../../build_unix"} +SOAK_N=${SOAK_N:-2000} +SOAK_TIMEOUT=${SOAK_TIMEOUT:-900} +SOAK_SAN=${SOAK_SAN:-0} +OUT="$HERE/build" +LIBDBA="$LIBDB_BUILD/libdb.a" + +if [ -f "$LIBDB_BUILD/Makefile" ]; then + LDLIBS=$(sed -n 's/^LIBS=[[:space:]]*//p' "$LIBDB_BUILD/Makefile" | head -1) +fi +LDLIBS="${LDLIBS:--lpthread} -ldl -lpthread" + +CFLAGS="-g -O1 -Wall -Wextra -Wno-unused-parameter -I$LIBDB_BUILD -I$HERE" +[ "$SOAK_SAN" = "1" ] && CFLAGS="$CFLAGS -fsanitize=address" + +[ -f "$LIBDBA" ] || { + echo "error: libdb.a not found at $LIBDBA -- build libdb first:" >&2 + echo " (cd $LIBDB_BUILD && ../dist/configure --enable-debug && make -j4)" >&2 + exit 2 +} + +mkdir -p "$OUT" +# shellcheck disable=SC2086 +$CC $CFLAGS "$HERE/test_soak_resources.c" "$LIBDBA" $LDLIBS \ + -o "$OUT/test_soak_resources" +echo "built $OUT/test_soak_resources" + +[ "${1:-}" = "build" ] && exit 0 + +cd "$OUT" +if [ "${1:-}" = "--list" ]; then + exec ./test_soak_resources --list +fi +exec timeout "$SOAK_TIMEOUT" ./test_soak_resources -n "$SOAK_N" "$@" diff --git a/test/soak/test_soak_resources.c b/test/soak/test_soak_resources.c new file mode 100644 index 000000000..4e931913a --- /dev/null +++ b/test/soak/test_soak_resources.c @@ -0,0 +1,690 @@ +/*- + * test/soak/test_soak_resources.c -- + * Tier B2: long-running resource-accounting soak. + * + * The blind spot this closes: a slot/mutex/locker leak that only manifests + * after thousands of SEQUENTIAL transactions. Nothing crashes, no page is + * corrupt, no sanitizer fires -- the region just fills up until some later + * API call returns ENOMEM. Crash/durability and memory-safety tiers cannot + * see that shape. + * + * Method: open ONE long-lived environment, run N (>= 2000) sequential + * transactions of a given workload, sampling the PUBLIC stat APIs + * (DB_ENV->mutex_stat, lock_stat, txn_stat, memp_stat) at intervals. A + * healthy workload returns each counter to (or near) its post-warmup + * baseline; a leaking workload grows it monotonically. The verdict is a + * least-squares slope over the samples taken AFTER warmup, expressed in + * units per 1000 transactions, compared against a per-counter tolerance. + * The growth curve is always printed so a regression is diagnosable from + * CI logs alone. + * + * Warmup matters: the first transactions legitimately grow the region + * (lockers get allocated, cache pages get faulted in, the free lists fill). + * Only the steady state is asserted, so the tier does not fight normal + * lazy allocation. + * + * Usage: + * ./test_soak_resources every workload, default N + * ./test_soak_resources -n 5000 N transactions per workload + * ./test_soak_resources WORKLOAD ... only the named workloads + * ./test_soak_resources --list list workload names + * + * Exit status: 0 = every workload matched its expectation, 1 = a workload + * did not (unexpected growth, or an expected-leak workload that stayed flat + * -- i.e. the leak got fixed and the expectation needs updating), 2 = + * harness error. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db.h" + +#define SOAK_MAX_SAMPLE 64 + +/* + * The counters we track. All come from the public stat APIs. Each is a + * region resource that a leak would consume without returning. + */ +enum soak_counter { + C_MUTEX_INUSE, /* mutex_stat: st_mutex_inuse */ + C_LOCK_LOCKERS, /* lock_stat: st_nlockers */ + C_LOCK_LOCKS, /* lock_stat: st_nlocks */ + C_LOCK_OBJECTS, /* lock_stat: st_nobjects */ + C_TXN_ACTIVE, /* txn_stat: st_nactive */ + C_TXN_SNAPSHOT, /* txn_stat: st_nsnapshot */ + C_MPOOL_DIRTY, /* memp_stat: st_page_dirty */ + C_NCOUNTER +}; + +static const struct { + const char *name; + /* + * Tolerated steady-state growth, in counter units per 1000 + * transactions. Zero would be ideal but is too brittle: the mutex + * and lock regions legitimately wobble by a slot or two as free + * lists are recycled, and MVCC page retention is asynchronous. A + * genuine per-transaction leak grows by ~1000 units per 1000 txns, + * three orders of magnitude above these tolerances. + */ + double tolerance; +} counters[C_NCOUNTER] = { + { "mutex_inuse", 20.0 }, + { "lock_lockers", 20.0 }, + { "lock_locks", 20.0 }, + { "lock_objects", 20.0 }, + { "txn_active", 2.0 }, + { "txn_snapshot", 20.0 }, + { "mpool_dirty", 50.0 }, +}; + +typedef struct { + long txns; + double v[C_NCOUNTER]; +} soak_sample; + +struct soak_workload; +/* + * A workload runs exactly one transaction and returns 0, or an rc the + * workload considers a legitimate give-up (deadlock etc.). Returning + * anything else is a harness error. + */ +typedef int (*soak_txn_fn)(struct soak_workload *, long); + +typedef struct soak_workload { + const char *name; + const char *shape; + soak_txn_fn one; + int expect_leak; /* Known-broken on master. */ + const char *issue; + /* Which counters this workload asserts on; 0 => all of them. */ + unsigned mask; +} soak_workload; + +static DB_ENV *env; +static DB *db; +static char soak_home[512]; +static long soak_n = 2000; +static long soak_enomem_at = -1; /* First txn that saw ENOMEM. */ +static const char *soak_enomem_call; + +static void soak_die(const char *, int) __attribute__((noreturn)); + +static void +soak_die(const char *what, int rc) +{ + fprintf(stderr, "harness error: %s: %s (%d)\n", + what, db_strerror(rc), rc); + exit(2); +} + +static int +soak_is_giveup(int rc) +{ + return (rc == DB_LOCK_DEADLOCK || rc == DB_LOCK_NOTGRANTED || + rc == DB_SNAPSHOT_CONFLICT || rc == DB_SNAPSHOT_UNSAFE); +} + +/* + * soak_resource_rc -- + * Is this rc the resource exhaustion we are hunting? ENOMEM from a + * libdb region allocation is the #137/#138 failure mode. It is NOT + * treated as a harness error: recording it and continuing gives a much + * better report ("ENOMEM at txn 1187") than dying does. + */ +static int +soak_resource_rc(int rc) +{ + return (rc == ENOMEM || rc == DB_RUNRECOVERY); +} + +static void +soak_note_enomem(const char *call, long txn) +{ + if (soak_enomem_at < 0) { + soak_enomem_at = txn; + soak_enomem_call = call; + } +} + +static void +soak_rmtree(const char *dir) +{ + char cmd[600]; + + (void)snprintf(cmd, sizeof(cmd), + "find '%s' -mindepth 1 -delete 2>/dev/null", dir); + (void)system(cmd); +} + +/* + * soak_env_open -- + * One long-lived environment. The region sizes are deliberately left + * at their defaults: the point of the tier is that a correct engine + * does not need a bigger region to run 2000 sequential transactions, + * and a leak shows up as ENOMEM exactly because the default region is + * finite. + */ +static void +soak_env_open(const char *workload) +{ + int rc; + + (void)snprintf(soak_home, sizeof(soak_home), "SOAKDIR.%s", workload); + (void)mkdir(soak_home, 0755); + soak_rmtree(soak_home); + + if ((rc = db_env_create(&env, 0)) != 0) + soak_die("db_env_create", rc); + if ((rc = env->set_lk_detect(env, DB_LOCK_DEFAULT)) != 0) + soak_die("set_lk_detect", rc); + if ((rc = env->set_cachesize(env, 0, 8 * 1024 * 1024, 1)) != 0) + soak_die("set_cachesize", rc); + if ((rc = env->set_timeout(env, 2000000, DB_SET_LOCK_TIMEOUT)) != 0) + soak_die("set_timeout", rc); + if ((rc = env->open(env, soak_home, DB_CREATE | DB_INIT_LOCK | + DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD, 0600)) != 0) + soak_die("DB_ENV->open", rc); + + if ((rc = db_create(&db, env, 0)) != 0) + soak_die("db_create", rc); + if ((rc = db->open(db, NULL, "soak.db", NULL, DB_BTREE, DB_CREATE | + DB_MULTIVERSION | DB_AUTO_COMMIT | DB_THREAD, 0600)) != 0) + soak_die("DB->open", rc); +} + +static void +soak_env_close(void) +{ + int rc; + + if ((rc = db->close(db, 0)) != 0) + soak_die("DB->close", rc); + if ((rc = env->close(env, 0)) != 0) + soak_die("DB_ENV->close", rc); + env = NULL; + db = NULL; +} + +/* + * soak_sample_now -- + * Read every counter through the public stat APIs. DB_STAT_SUBSYSTEM + * is not used; each call is the plain documented one so the tier stays + * a legitimate API consumer. + */ +static void +soak_sample_now(soak_sample *s, long txns) +{ + DB_MUTEX_STAT *mst; + DB_LOCK_STAT *lst; + DB_TXN_STAT *tst; + DB_MPOOL_STAT *gst; + int rc; + + memset(s, 0, sizeof(*s)); + s->txns = txns; + + if ((rc = env->mutex_stat(env, &mst, 0)) != 0) + soak_die("DB_ENV->mutex_stat", rc); + s->v[C_MUTEX_INUSE] = mst->st_mutex_inuse; + free(mst); + + if ((rc = env->lock_stat(env, &lst, 0)) != 0) + soak_die("DB_ENV->lock_stat", rc); + s->v[C_LOCK_LOCKERS] = lst->st_nlockers; + s->v[C_LOCK_LOCKS] = lst->st_nlocks; + s->v[C_LOCK_OBJECTS] = lst->st_nobjects; + free(lst); + + if ((rc = env->txn_stat(env, &tst, 0)) != 0) + soak_die("DB_ENV->txn_stat", rc); + s->v[C_TXN_ACTIVE] = tst->st_nactive; + s->v[C_TXN_SNAPSHOT] = tst->st_nsnapshot; + free(tst); + + if ((rc = env->memp_stat(env, &gst, NULL, 0)) != 0) + soak_die("DB_ENV->memp_stat", rc); + s->v[C_MPOOL_DIRTY] = gst->st_page_dirty; + free(gst); +} + +/* + * --------------------------------------------------------------------------- + * Workloads. Each runs ONE transaction; the driver repeats it. + * --------------------------------------------------------------------------- + */ +/* + * Keyspace width. Wide enough that writes spread over many pages (so MVCC + * versions accumulate and the cache turns over, which is what drives the + * detail-reaping path), narrow enough that the working set stays in the + * 8MB cache and no workload starts failing for cache reasons. + */ +#define SOAK_NKEY 512 + +static int +soak_get(DB_TXN *txn, long i, int *out) +{ + DBT k, d; + char key[32]; + int v, rc; + + (void)snprintf(key, sizeof(key), "k%06ld", i % SOAK_NKEY); + memset(&k, 0, sizeof(k)); + memset(&d, 0, sizeof(d)); + k.data = key; + k.size = (u_int32_t)strlen(key); + d.data = &v; + d.ulen = sizeof(v); + d.flags = DB_DBT_USERMEM; + if ((rc = db->get(db, txn, &k, &d, 0)) != 0) + return (rc); + if (out != NULL) + *out = v; + return (0); +} + +static int +soak_put(DB_TXN *txn, long i, int v) +{ + DBT k, d; + char key[32]; + + (void)snprintf(key, sizeof(key), "k%06ld", i % SOAK_NKEY); + memset(&k, 0, sizeof(k)); + memset(&d, 0, sizeof(d)); + k.data = key; + k.size = (u_int32_t)strlen(key); + d.data = &v; + d.size = sizeof(v); + return (db->put(db, txn, &k, &d, 0)); +} + +/* + * ro_snapshot -- the #137 shape. + * A read-only DB_TXN_SNAPSHOT transaction, begun and committed with no + * write at all. #137 reports that the SIREAD cleanup does not reclaim + * the committed reader's locker, so each such transaction consumes + * region resources permanently and txn_begin eventually returns ENOMEM. + */ +static int +wl_ro_snapshot(soak_workload *w, long i) +{ + DB_TXN *txn; + int rc, v; + + (void)w; + if ((rc = env->txn_begin(env, NULL, &txn, DB_TXN_SNAPSHOT)) != 0) { + if (soak_resource_rc(rc)) { + soak_note_enomem("txn_begin", i); + return (0); + } + return (rc); + } + if ((rc = soak_get(txn, i, &v)) != 0 && rc != DB_NOTFOUND) { + (void)txn->abort(txn); + return (soak_is_giveup(rc) ? 0 : rc); + } + if ((rc = txn->commit(txn, 0)) != 0) + return (soak_is_giveup(rc) ? 0 : rc); + return (0); +} + +/* + * mvcc_retained -- the #138 shape. + * A snapshot transaction that both READS (creating SIREAD markers, so + * __txn_end parks its detail on the mvcc_txn list with TXN_DTL_SNAPSHOT + * rather than freeing it) and WRITES (creating MVCC buffer versions, so + * mvcc_ref is nonzero too). Once the markers are garbage-collected and + * the MVCC pages evicted, the detail is reclaimed by + * __txn_reap_si_details -- which per #138 frees the detail without + * freeing its mvcc_mtx, leaking one mutex slot per reaped detail. + * + * Driving that path needs cache turnover, so the workload spreads its + * writes over a keyspace far wider than the ro_snapshot one and reads a + * range rather than a single key. + */ +static int +wl_mvcc_retained(soak_workload *w, long i) +{ + DB_TXN *txn; + int j, rc, v; + + (void)w; + if ((rc = env->txn_begin(env, NULL, &txn, DB_TXN_SNAPSHOT)) != 0) { + if (soak_resource_rc(rc)) { + soak_note_enomem("txn_begin", i); + return (0); + } + return (rc); + } + /* Read a few keys: this is what leaves SIREAD markers behind. */ + for (j = 0; j < 4; j++) + if ((rc = soak_get(txn, i + j, &v)) != 0 && + rc != DB_NOTFOUND) { + (void)txn->abort(txn); + return (soak_is_giveup(rc) ? 0 : rc); + } + if ((rc = soak_put(txn, i, (int)i)) != 0) { + (void)txn->abort(txn); + if (soak_resource_rc(rc)) { + soak_note_enomem("DB->put", i); + return (0); + } + return (soak_is_giveup(rc) ? 0 : rc); + } + if ((rc = txn->commit(txn, 0)) != 0) + return (soak_is_giveup(rc) ? 0 : rc); + return (0); +} + +/* rw_plain -- ordinary read-write transaction, no snapshot. Control. */ +static int +wl_rw_plain(soak_workload *w, long i) +{ + DB_TXN *txn; + int rc, v; + + (void)w; + if ((rc = env->txn_begin(env, NULL, &txn, 0)) != 0) { + if (soak_resource_rc(rc)) { + soak_note_enomem("txn_begin", i); + return (0); + } + return (rc); + } + if ((rc = soak_get(txn, i, &v)) != 0 && rc != DB_NOTFOUND) { + (void)txn->abort(txn); + return (soak_is_giveup(rc) ? 0 : rc); + } + if ((rc = soak_put(txn, i, (int)i)) != 0) { + (void)txn->abort(txn); + return (soak_is_giveup(rc) ? 0 : rc); + } + if ((rc = txn->commit(txn, 0)) != 0) + return (soak_is_giveup(rc) ? 0 : rc); + return (0); +} + +/* aborted -- every transaction aborts; the undo path must free too. */ +static int +wl_aborted(soak_workload *w, long i) +{ + DB_TXN *txn; + int rc; + + (void)w; + if ((rc = env->txn_begin(env, NULL, &txn, DB_TXN_SNAPSHOT)) != 0) { + if (soak_resource_rc(rc)) { + soak_note_enomem("txn_begin", i); + return (0); + } + return (rc); + } + if ((rc = soak_put(txn, i, (int)i)) != 0 && !soak_is_giveup(rc)) { + (void)txn->abort(txn); + if (soak_resource_rc(rc)) { + soak_note_enomem("DB->put", i); + return (0); + } + return (rc); + } + if ((rc = txn->abort(txn)) != 0) + return (rc); + return (0); +} + +/* + * cursor_churn -- cursor open/close accounting. + * Deliberately a PLAIN (non-snapshot) transaction: a snapshot reader + * would trip the #137 locker leak and the workload would then be a + * second copy of ro_snapshot instead of telling us anything about + * cursors. With a plain txn, any growth here is genuinely a cursor or + * lock-list accounting problem. + */ +static int +wl_cursor_churn(soak_workload *w, long i) +{ + DB_TXN *txn; + DBC *dbc; + DBT k, d; + int rc; + + (void)w; + if ((rc = env->txn_begin(env, NULL, &txn, 0)) != 0) { + if (soak_resource_rc(rc)) { + soak_note_enomem("txn_begin", i); + return (0); + } + return (rc); + } + if ((rc = db->cursor(db, txn, &dbc, 0)) != 0) { + (void)txn->abort(txn); + if (soak_resource_rc(rc)) { + soak_note_enomem("DB->cursor", i); + return (0); + } + return (rc); + } + memset(&k, 0, sizeof(k)); + memset(&d, 0, sizeof(d)); + /* Walk a few records so the cursor actually acquires locks. */ + for (rc = dbc->get(dbc, &k, &d, DB_FIRST); + rc == 0; rc = dbc->get(dbc, &k, &d, DB_NEXT)) + continue; + if (rc != DB_NOTFOUND && !soak_is_giveup(rc)) { + (void)dbc->close(dbc); + (void)txn->abort(txn); + return (rc); + } + if ((rc = dbc->close(dbc)) != 0) { + (void)txn->abort(txn); + return (rc); + } + if ((rc = txn->commit(txn, 0)) != 0) + return (soak_is_giveup(rc) ? 0 : rc); + return (0); +} + +static soak_workload workloads[] = { + { "ro_snapshot", + "read-only DB_TXN_SNAPSHOT txns, no write (the #137 shape)", + wl_ro_snapshot, 1, "#137", 0 }, + { "mvcc_retained", + "snapshot txns that read and write, details MVCC-retained (#138)", + wl_mvcc_retained, 1, "#138", 0 }, + { "rw_plain", + "ordinary read-write txns, no snapshot (control)", + wl_rw_plain, 0, NULL, 0 }, + { "aborted", + "snapshot txns that all abort (control)", + wl_aborted, 0, NULL, 0 }, + { "cursor_churn", + "plain txns that open, walk and close a cursor (control)", + wl_cursor_churn, 0, NULL, 0 }, +}; +#define NWORKLOADS ((int)(sizeof(workloads) / sizeof(workloads[0]))) + +/* + * soak_slope -- + * Least-squares slope of counter `c' over samples [lo, hi), in units + * per 1000 transactions. Least squares rather than (last - first) + * because a single noisy endpoint should not decide the verdict. + */ +static double +soak_slope(const soak_sample *s, int lo, int hi, int c) +{ + double den, mx, my, num; + int i, n; + + if ((n = hi - lo) < 2) + return (0.0); + for (i = lo, mx = my = 0.0; i < hi; i++) { + mx += (double)s[i].txns; + my += s[i].v[c]; + } + mx /= n; + my /= n; + for (i = lo, num = den = 0.0; i < hi; i++) { + double dx = (double)s[i].txns - mx; + num += dx * (s[i].v[c] - my); + den += dx * dx; + } + if (den == 0.0) + return (0.0); + return (num / den * 1000.0); +} + +/* + * run_workload -- + * Returns 0 if the outcome matched the expectation, 1 if not. + */ +static int +run_workload(soak_workload *w) +{ + soak_sample s[SOAK_MAX_SAMPLE]; + double slope[C_NCOUNTER]; + long every, i; + int c, leaked, nsample, ok, warm; + + printf("== %s ==\n shape: %s\n %ld sequential transactions\n", + w->name, w->shape, soak_n); + soak_enomem_at = -1; + soak_enomem_call = NULL; + + soak_env_open(w->name); + /* Seed the keyspace so read-only workloads find records. */ + for (i = 0; i < SOAK_NKEY; i++) + if (soak_put(NULL, i, 0) != 0) + soak_die("seed put", EINVAL); + + every = soak_n / (SOAK_MAX_SAMPLE - 1); + if (every < 1) + every = 1; + nsample = 0; + soak_sample_now(&s[nsample++], 0); + for (i = 1; i <= soak_n; i++) { + int rc = w->one(w, i); + + if (rc != 0) + soak_die("workload transaction", rc); + if (i % every == 0 && nsample < SOAK_MAX_SAMPLE) + soak_sample_now(&s[nsample++], i); + } + if (nsample < SOAK_MAX_SAMPLE && s[nsample - 1].txns != soak_n) + soak_sample_now(&s[nsample++], soak_n); + + /* + * Warmup: ignore the first quarter of the samples. Lazy region + * allocation and cache fill legitimately grow counters there. + */ + warm = nsample / 4; + if (warm < 1) + warm = 1; + if (nsample - warm < 2) + warm = 0; + + printf(" growth curve (steady-state samples marked *):\n"); + printf(" %8s", "txns"); + for (c = 0; c < C_NCOUNTER; c++) + printf(" %14s", counters[c].name); + printf("\n"); + for (i = 0; i < nsample; i++) { + printf(" %s %8ld", i >= warm ? "*" : " ", s[i].txns); + for (c = 0; c < C_NCOUNTER; c++) + printf(" %14.0f", s[i].v[c]); + printf("\n"); + } + + leaked = 0; + printf(" steady-state slope (units per 1000 txns, tolerance):\n"); + for (c = 0; c < C_NCOUNTER; c++) { + slope[c] = soak_slope(s, warm, nsample, c); + printf(" %-14s %+10.2f (tol %6.2f)%s\n", + counters[c].name, slope[c], counters[c].tolerance, + slope[c] > counters[c].tolerance ? " <== GROWING" : ""); + if (slope[c] > counters[c].tolerance) + leaked = 1; + } + if (soak_enomem_at >= 0) { + printf(" ENOMEM/RUNRECOVERY from %s at transaction " + "%ld -- the region ran out\n", + soak_enomem_call, soak_enomem_at); + leaked = 1; + } + + soak_env_close(); + + ok = !leaked; + if (ok == w->expect_leak) { + if (w->expect_leak) + printf(" UNEXPECTED PASS: %s is marked as " + "reproducing %s but resources stayed flat -- the " + "issue looks FIXED; clear expect_leak for this " + "workload.\n\n", w->name, w->issue); + else + printf(" FAIL: a region resource grew " + "monotonically beyond tolerance over %ld " + "sequential transactions.\n\n", soak_n); + return (1); + } + if (w->expect_leak) + printf(" XFAIL (reproduces %s): resources grow with " + "transaction count, as the issue reports.\n\n", w->issue); + else + printf(" PASS: resources stayed within tolerance.\n\n"); + return (0); +} + +int +main(int argc, char **argv) +{ + int failures, i, j, ran; + + setvbuf(stdout, NULL, _IOLBF, 0); + + /* -n N may precede the workload names. */ + i = 1; + if (argc >= 3 && strcmp(argv[1], "-n") == 0) { + soak_n = atol(argv[2]); + if (soak_n < 10) { + fprintf(stderr, "-n must be >= 10\n"); + return (2); + } + i = 3; + } + if (i < argc && strcmp(argv[i], "--list") == 0) { + for (j = 0; j < NWORKLOADS; j++) + printf("%s%s\n", workloads[j].name, + workloads[j].expect_leak ? "\t(expect-leak)" : ""); + return (0); + } + + printf("%s\n\n", db_version(NULL, NULL, NULL)); + failures = ran = 0; + if (i >= argc) { + for (j = 0; j < NWORKLOADS; j++, ran++) + failures += run_workload(&workloads[j]); + } else { + for (; i < argc; i++) { + for (j = 0; j < NWORKLOADS; j++) + if (strcmp(argv[i], workloads[j].name) == 0) + break; + if (j == NWORKLOADS) { + fprintf(stderr, "unknown workload: %s\n", + argv[i]); + return (2); + } + failures += run_workload(&workloads[j]); + ran++; + } + } + + printf("%d workload(s) run, %d unexpected outcome(s)\n", + ran, failures); + return (failures != 0); +} diff --git a/test/tiers/meson.build b/test/tiers/meson.build new file mode 100644 index 000000000..b93d7e55f --- /dev/null +++ b/test/tiers/meson.build @@ -0,0 +1,58 @@ +# test/tiers/meson.build -- Meson wiring for the isolation / soak / lock-matrix +# test tiers (B1/B2/B3). +# +# Entered from the ROOT meson.build (not from dist/, because subdir() cannot +# climb back out of dist/ with '..'). The variables it relies on -- `inc`, +# `libdb`, `thread_dep` -- are set by dist/meson.build and stay in scope +# because subdir() shares scope. +# +# These tiers need no library-side hooks: they drive the public DB_ENV / DB API +# only, so there is no build option to enable and nothing is added to the +# library. The executables are build_by_default: false, so a plain +# `ninja -C build` is unaffected; build them explicitly: +# +# ninja -C build test/tiers/test_iso_anomaly \ +# test/tiers/test_soak_resources \ +# test/tiers/test_lock_matrix +# +# and `meson test -C build --suite tiers` runs the gating tier (B1). The other +# two are in separate suites because on current master they legitimately fail: +# +# meson test -C build --suite tiers # B1, gating +# meson test -C build --suite tiers-xfail # B3, reproduces #140 +# meson test -C build --suite tiers-slow # B2, minutes-long soak +# +# For the ASan build that tier B3 is designed for (the out-of-bounds write is +# inside libdb's own allocation, so libdb itself must be instrumented for a +# precise report), use test/lockmatrix/run.sh, which reuses the +# build_asan_gate/ mechanism. Note that the corruption is severe enough that +# even an UNinstrumented build trips glibc's own heap checks ("double free or +# corruption"), so this plain-meson run also detects it -- just without the +# faulting line number. + +tiers_tests = [ + # name source dir source file suite args + ['iso_anomaly', 'isolation', 'test_iso_anomaly.c', 'tiers', []], + # B3 aborts on current master: that abort IS the #140 reproduction, so it + # lives in its own suite rather than failing the gating one. Move it to + # 'tiers' in the same PR that fixes #140. + ['lock_matrix', 'lockmatrix', 'test_lock_matrix.c', 'tiers-xfail', []], + # The soak is minutes, not seconds: keep it out of the default suite and run + # a reduced transaction count when it is asked for by name. + ['soak_resources', 'soak', 'test_soak_resources.c', 'tiers-slow', ['-n', '2000']], +] + +foreach t : tiers_tests + exe = executable('test_' + t[0], + files(meson.project_source_root() / 'test' / t[1] / t[2]), + include_directories: inc, + link_with: libdb, + dependencies: thread_dep, + build_by_default: false, + install: false) + test(t[0], exe, + args: t[4], + suite: t[3], + workdir: meson.current_build_dir(), + timeout: t[3] == 'tiers-slow' ? 1800 : 600) +endforeach