Skip to content

perf(multitude): optimize hot allocation paths and harden chunk lifecycle #498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
geeknoid wants to merge 1 commit into
base: main
Choose a base branch
from
Open

perf(multitude): optimize hot allocation paths and harden chunk lifecycle #498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .spelling
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -593,3 +593,8 @@ deterministically
dereferences
honour
MPSC
precomputed
u32
2^30
~2^30
POV
179 changes: 94 additions & 85 deletions crates/multitude/README.md
View file
Open in desktop

Large diffs are not rendered by default.

104 changes: 72 additions & 32 deletions crates/multitude/benches/criterion_alloc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ const SLICE_LEN: usize = 8;
/// `warm_bump` (which similarly primes its cursor with a no-op
/// alloc) and isolates the comparison to "in-chunk bump cost
/// only" with no cold refill amortized into the per-op number.
///
/// Only used by benches that exercise **both** flavors in the timed
/// region: the `vec_builder` benches build a local `Vec` and then
/// `into_arc()` it. Benches that touch a single flavor must use
/// [`warm_arena_local`] / [`warm_arena_shared`] instead: priming the
/// unused flavor allocates a dead-weight 64 KiB chunk that doubles the
/// arena's memory footprint and inflates the measured per-op time
/// through extra cache/TLB pressure at the batch working-set sizes
/// criterion picks (verified: priming the unused shared chunk made
/// `alloc_slice_copy` measure ~9x slower than its true cost).
fn warm_arena() -> Arena {
let arena = Arena::builder()
.with_capacity_local(64 * 1024)
Expand All @@ -52,6 +62,29 @@ fn warm_arena() -> Arena {
arena
}

/// Like [`warm_arena`] but primes **only** the local (ref/value)
/// flavor. Used by benches whose timed region allocates exclusively
/// from the local chunk (`alloc`, `alloc_str`, `alloc_slice_*` ref,
/// `alloc_string*`). Priming the shared flavor here would add a
/// dead-weight 64 KiB chunk — see [`warm_arena`].
fn warm_arena_local() -> Arena {
let arena = Arena::builder().with_capacity_local(64 * 1024).build();
let _: &mut u64 = arena.alloc(0_u64);
arena
}

/// Like [`warm_arena`] but primes **only** the shared (`Box`/`Arc`)
/// flavor. Used by benches whose timed region allocates exclusively
/// from the shared chunk (every `*_box` / `*_arc` bench — both smart
/// pointers are backed by refcounted shared chunks). Priming the local
/// flavor here would add a dead-weight 64 KiB chunk — see
/// [`warm_arena`].
fn warm_arena_shared() -> Arena {
let arena = Arena::builder().with_capacity_shared(64 * 1024).build();
let _ = arena.alloc_arc(0_u64);
arena
}

/// Build a [`bumpalo::Bump`] pre-sized to fit the timed region.
///
/// `Bump::with_capacity(N)` reserves a chunk of at least N bytes from
Expand Down Expand Up @@ -84,7 +117,7 @@ fn bench_alloc_u64(c: &mut Criterion) {

g.bench_function("alloc", |b| {
b.iter_batched(
warm_arena,
warm_arena_local,
|arena| {
for i in 0..N {
let _: &mut u64 = black_box(arena.alloc(black_box(i as u64)));
Expand All @@ -96,7 +129,7 @@ fn bench_alloc_u64(c: &mut Criterion) {
});
g.bench_function("alloc_with", |b| {
b.iter_batched(
warm_arena,
warm_arena_local,
|arena| {
for i in 0..N {
let _: &mut u64 = black_box(arena.alloc_with(|| black_box(i as u64)));
Expand All @@ -109,7 +142,7 @@ fn bench_alloc_u64(c: &mut Criterion) {

g.bench_function("alloc_box", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Box<u64>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Box<u64>>::with_capacity(N)),
|(arena, mut h)| {
for i in 0..N {
h.push(arena.alloc_box(black_box(i as u64)));
Expand All @@ -121,7 +154,7 @@ fn bench_alloc_u64(c: &mut Criterion) {
});
g.bench_function("alloc_box_with", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Box<u64>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Box<u64>>::with_capacity(N)),
|(arena, mut h)| {
for i in 0..N {
h.push(arena.alloc_box_with(|| black_box(i as u64)));
Expand All @@ -133,7 +166,7 @@ fn bench_alloc_u64(c: &mut Criterion) {
});
g.bench_function("alloc_uninit_box", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Box<MaybeUninit<u64>>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Box<MaybeUninit<u64>>>::with_capacity(N)),
|(arena, mut h)| {
for _ in 0..N {
h.push(arena.alloc_uninit_box::<u64>());
Expand All @@ -145,7 +178,7 @@ fn bench_alloc_u64(c: &mut Criterion) {
});
g.bench_function("alloc_zeroed_box", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Box<MaybeUninit<u64>>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Box<MaybeUninit<u64>>>::with_capacity(N)),
|(arena, mut h)| {
for _ in 0..N {
h.push(arena.alloc_zeroed_box::<u64>());
Expand All @@ -158,7 +191,7 @@ fn bench_alloc_u64(c: &mut Criterion) {

g.bench_function("alloc_arc", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Arc<u64>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Arc<u64>>::with_capacity(N)),
|(arena, mut h)| {
for i in 0..N {
h.push(arena.alloc_arc(black_box(i as u64)));
Expand All @@ -170,7 +203,7 @@ fn bench_alloc_u64(c: &mut Criterion) {
});
g.bench_function("alloc_arc_with", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Arc<u64>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Arc<u64>>::with_capacity(N)),
|(arena, mut h)| {
for i in 0..N {
h.push(arena.alloc_arc_with(|| black_box(i as u64)));
Expand All @@ -182,7 +215,7 @@ fn bench_alloc_u64(c: &mut Criterion) {
});
g.bench_function("alloc_uninit_arc", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Arc<MaybeUninit<u64>>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Arc<MaybeUninit<u64>>>::with_capacity(N)),
|(arena, mut h)| {
for _ in 0..N {
h.push(arena.alloc_uninit_arc::<u64>());
Expand All @@ -194,7 +227,7 @@ fn bench_alloc_u64(c: &mut Criterion) {
});
g.bench_function("alloc_zeroed_arc", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Arc<MaybeUninit<u64>>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Arc<MaybeUninit<u64>>>::with_capacity(N)),
|(arena, mut h)| {
for _ in 0..N {
h.push(arena.alloc_zeroed_arc::<u64>());
Expand Down Expand Up @@ -242,10 +275,10 @@ fn bench_alloc_str(c: &mut Criterion) {

g.bench_function("alloc_str", |b| {
b.iter_batched(
warm_arena,
warm_arena_local,
|arena| {
for w in &words {
let _: &mut str = black_box(arena.alloc_str(black_box(w)));
let _: &mut str = black_box(arena.alloc_str(black_box(w.as_str())));
}
arena
},
Expand All @@ -254,10 +287,10 @@ fn bench_alloc_str(c: &mut Criterion) {
});
g.bench_function("alloc_str_box", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Box<str>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Box<str>>::with_capacity(N)),
|(arena, mut o)| {
for w in &words {
o.push(arena.alloc_str_box(black_box(w)));
o.push(arena.alloc_str_box(black_box(w.as_str())));
}
(o, arena)
},
Expand All @@ -266,10 +299,10 @@ fn bench_alloc_str(c: &mut Criterion) {
});
g.bench_function("alloc_str_arc", |b| {
b.iter_batched(
|| (warm_arena(), Vec::<Arc<str>>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<Arc<str>>::with_capacity(N)),
|(arena, mut o)| {
for w in &words {
o.push(arena.alloc_str_arc(black_box(w)));
o.push(arena.alloc_str_arc(black_box(w.as_str())));
}
(o, arena)
},
Expand All @@ -281,7 +314,7 @@ fn bench_alloc_str(c: &mut Criterion) {
warm_bump,
|bump| {
for w in &words {
let _: &mut str = black_box(bump.alloc_str(black_box(w)));
let _: &mut str = black_box(bump.alloc_str(black_box(w.as_str())));
}
bump
},
Expand All @@ -303,7 +336,7 @@ fn bench_alloc_slice(c: &mut Criterion) {
($name:literal, $body:expr) => {
g.bench_function($name, |b| {
b.iter_batched(
warm_arena,
warm_arena_local,
|arena| {
let r = $body(&arena);
// Return owned values so their `Drop` runs outside the timed region.
Expand All @@ -318,7 +351,7 @@ fn bench_alloc_slice(c: &mut Criterion) {
($name:literal, $T:ty, $body:expr) => {
g.bench_function($name, |b| {
b.iter_batched(
|| (warm_arena(), Vec::<$T>::with_capacity(N)),
|| (warm_arena_shared(), Vec::<$T>::with_capacity(N)),
|(arena, mut o)| {
$body(&arena, &mut o);
(o, arena)
Expand Down Expand Up @@ -347,7 +380,7 @@ fn bench_alloc_slice(c: &mut Criterion) {
// ref
bench_arena!("alloc_slice_copy", |arena: &Arena| {
for s in &slices {
let _: &mut [u64] = black_box(arena.alloc_slice_copy(black_box(s)));
let _: &mut [u64] = black_box(arena.alloc_slice_copy(black_box(s.as_slice())));
}
});
bench_arena!("alloc_slice_clone", |arena: &Arena| {
Expand All @@ -369,7 +402,7 @@ fn bench_alloc_slice(c: &mut Criterion) {
// box
bench_arena_collect!("alloc_slice_copy_box", Box<[u64]>, |arena: &Arena, o: &mut Vec<Box<[u64]>>| {
for s in &slices {
o.push(arena.alloc_slice_copy_box(black_box(s)));
o.push(arena.alloc_slice_copy_box(black_box(s.as_slice())));
}
});
bench_arena_collect!("alloc_slice_clone_box", Box<[u64]>, |arena: &Arena, o: &mut Vec<Box<[u64]>>| {
Expand Down Expand Up @@ -407,7 +440,7 @@ fn bench_alloc_slice(c: &mut Criterion) {
// arc
bench_arena_collect!("alloc_slice_copy_arc", Arc<[u64]>, |arena: &Arena, o: &mut Vec<Arc<[u64]>>| {
for s in &slices {
o.push(arena.alloc_slice_copy_arc(black_box(s)));
o.push(arena.alloc_slice_copy_arc(black_box(s.as_slice())));
}
});
bench_arena_collect!("alloc_slice_clone_arc", Arc<[u64]>, |arena: &Arena, o: &mut Vec<Arc<[u64]>>| {
Expand Down Expand Up @@ -476,28 +509,36 @@ fn bench_string_builder(c: &mut Criterion) {

g.bench_function("alloc_string", |b| {
b.iter_batched(
warm_arena,
warm_arena_local,
|arena| {
let mut s = arena.alloc_string();
for w in &words {
s.push_str(black_box(w.as_str()));
}
let frozen = s.into_arena_box_str();
(frozen, arena)
// Mirror bumpalo's `into_bump_str`: take a `&str` view of
// the in-place chunk storage with no copy into a `Box<str>`.
// Dropping `s` is ~free (`u8` has no drop; storage is
// reclaimed at arena teardown) and must precede returning
// `arena`, which `s` borrows.
black_box(s.as_str());
drop(s);
arena
},
BatchSize::SmallInput,
);
});
g.bench_function("alloc_string_with_capacity", |b| {
b.iter_batched(
warm_arena,
warm_arena_local,
|arena| {
let mut s = arena.alloc_string_with_capacity(N * 6);
for w in &words {
s.push_str(black_box(w.as_str()));
}
let frozen = s.into_arena_box_str();
(frozen, arena)
// See `alloc_string` above.
black_box(s.as_str());
drop(s);
arena
},
BatchSize::SmallInput,
);
Expand Down Expand Up @@ -551,7 +592,7 @@ fn bench_vec_builder(c: &mut Criterion) {
for &i in &ints {
v.push(black_box(i));
}
let frozen = v.into_arena_arc();
let frozen = v.into_arc();
(frozen, arena)
},
BatchSize::SmallInput,
Expand All @@ -565,7 +606,7 @@ fn bench_vec_builder(c: &mut Criterion) {
for &i in &ints {
v.push(black_box(i));
}
let frozen = v.into_arena_arc();
let frozen = v.into_arc();
(frozen, arena)
},
BatchSize::SmallInput,
Expand Down Expand Up @@ -611,11 +652,10 @@ fn bench_vec_builder(c: &mut Criterion) {
fn bench_arena_creation(c: &mut Criterion) {
let mut g = c.benchmark_group("arena_creation");

g.bench_function("multitude", |b| {
g.bench_function("multitude_new", |b| {
b.iter(|| {
let arena = Arena::new();
black_box(&arena);
// Drop is part of the lifecycle — included in the timed region.
drop(arena);
});
});
Expand Down
Loading
Loading