From 73642d8e8900b9db5ddcf5c4a7cb4ddd4771acaa Mon Sep 17 00:00:00 2001 From: Martin Taillefer Date: Fri, 12 Jun 2026 13:34:18 +0000 Subject: [PATCH] perf(multitude): optimize hot allocation paths and harden chunk lifecycle Follow-up to the multitude rewrite focused on allocation throughput, memory-safety hardening, and richer runtime accounting. --- .spelling | 6 + crates/multitude/README.md | 184 ++-- crates/multitude/benches/criterion_alloc.rs | 104 +- .../multitude/benches/gungraun_alloc/linux.rs | 91 +- .../multitude/benches/gungraun_drop/linux.rs | 44 +- crates/multitude/docs/PERF.md | 142 +-- crates/multitude/docs/TODO.md | 247 ++--- crates/multitude/examples/multitude_basic.rs | 2 +- crates/multitude/examples/strings.rs | 2 +- crates/multitude/scripts/perf_report.rs | 60 +- crates/multitude/src/allocator_impl.rs | 2 +- crates/multitude/src/arc.rs | 4 +- crates/multitude/src/arena/alloc_growable.rs | 186 ++++ crates/multitude/src/arena/alloc_prefixed.rs | 11 +- crates/multitude/src/arena/alloc_slice_arc.rs | 28 +- crates/multitude/src/arena/alloc_slice_box.rs | 31 +- crates/multitude/src/arena/alloc_slice_ref.rs | 215 ++-- crates/multitude/src/arena/alloc_str.rs | 4 - crates/multitude/src/arena/alloc_unsized.rs | 8 +- crates/multitude/src/arena/alloc_utf16.rs | 49 +- crates/multitude/src/arena/alloc_value.rs | 185 ++-- crates/multitude/src/arena/mod.rs | 248 ++++- crates/multitude/src/arena/reserve.rs | 46 + crates/multitude/src/arena/retired_local.rs | 160 +++ crates/multitude/src/arena_stats.rs | 41 +- crates/multitude/src/box.rs | 9 + crates/multitude/src/bytemuck.rs | 8 +- crates/multitude/src/from_in.rs | 65 ++ crates/multitude/src/internal/arena_buf.rs | 47 +- crates/multitude/src/internal/chunk_alloc.rs | 102 +- .../multitude/src/internal/chunk_mutator.rs | 190 +++- crates/multitude/src/internal/chunk_ops.rs | 35 + .../multitude/src/internal/chunk_provider.rs | 106 +- crates/multitude/src/internal/drop_entry.rs | 113 +- crates/multitude/src/internal/local_chunk.rs | 244 +++-- crates/multitude/src/internal/shared_chunk.rs | 325 +++++- crates/multitude/src/internal/uninit.rs | 1 - crates/multitude/src/lib.rs | 31 +- crates/multitude/src/strings/format_macro.rs | 2 +- .../src/strings/format_utf16_macro.rs | 2 +- .../multitude/src/strings/from_utf16_error.rs | 31 + crates/multitude/src/strings/mod.rs | 15 +- crates/multitude/src/strings/string.rs | 426 +++++++- crates/multitude/src/strings/string_common.rs | 44 +- crates/multitude/src/strings/utf16_string.rs | 387 ++++++- crates/multitude/src/vec/basic.rs | 24 +- crates/multitude/src/vec/freeze.rs | 110 +- crates/multitude/src/vec/from_in.rs | 73 ++ crates/multitude/src/vec/mod.rs | 19 +- crates/multitude/src/vec/mutate.rs | 90 +- crates/multitude/src/vec/splice.rs | 132 +++ crates/multitude/src/vec/traits.rs | 51 + crates/multitude/src/vec/vec_macro.rs | 4 +- crates/multitude/src/zerocopy.rs | 8 +- crates/multitude/tests/alloc_ref.rs | 322 +++++- crates/multitude/tests/arena.rs | 28 +- crates/multitude/tests/arena_string.rs | 167 +-- crates/multitude/tests/arena_vec.rs | 107 +- crates/multitude/tests/audit_repro.rs | 44 + crates/multitude/tests/bolero.rs | 28 +- crates/multitude/tests/chunk_footprint.rs | 148 +++ crates/multitude/tests/coverage_extras.rs | 66 +- crates/multitude/tests/coverage_gaps.rs | 86 +- crates/multitude/tests/dst.rs | 18 +- .../tests/mutant_kills_boundaries.rs | 2 +- crates/multitude/tests/mutant_kills_more.rs | 92 +- .../multitude/tests/mutant_kills_post_fix.rs | 48 +- .../multitude/tests/mutant_kills_strings.rs | 4 +- crates/multitude/tests/mutants_extras.rs | 56 +- crates/multitude/tests/std_alignment.rs | 970 ++++++++++++++++++ crates/multitude/tests/utf16.rs | 186 ++-- 71 files changed, 5752 insertions(+), 1414 deletions(-) create mode 100644 crates/multitude/src/arena/retired_local.rs create mode 100644 crates/multitude/src/from_in.rs create mode 100644 crates/multitude/src/strings/from_utf16_error.rs create mode 100644 crates/multitude/src/vec/from_in.rs create mode 100644 crates/multitude/src/vec/splice.rs create mode 100644 crates/multitude/tests/chunk_footprint.rs create mode 100644 crates/multitude/tests/std_alignment.rs diff --git a/.spelling b/.spelling index 3527af725..fcae43ee0 100644 --- a/.spelling +++ b/.spelling @@ -593,3 +593,9 @@ deterministically dereferences honour MPSC +precomputed +u32 +2^30 +~2^30 +POV +lossy diff --git a/crates/multitude/README.md b/crates/multitude/README.md index 450d0a2cf..97a90dab5 100644 --- a/crates/multitude/README.md +++ b/crates/multitude/README.md @@ -189,12 +189,23 @@ builders**. They carry a data pointer + length + capacity + arena reference. Once you’re done building, you can **freeze them** into immutable smart pointers: -* [`String::into_arena_box_str`][__link31] → [`Box`][__link32] (**8 bytes**, thin). +* [`String::into_boxed_str`][__link31] → + [`Box`][__link32] (**8 bytes**, thin), or `Box::from(string)`. The freeze is **O(n)** — it copies the bytes into a compact, length-prefixed allocation so the resulting single pointer is `Send`-safe and can outlive the arena. -* [`Vec::into_arena_box`][__link33] → [`Box<[T]>`][__link34] (**8 bytes**, thin). - For `T: !Drop`, the freeze is **O(1)**. +* [`Vec::into_boxed_slice`][__link33] → + [`Box<[T]>`][__link34] (**8 bytes**, thin), or `Box::from(vec)`. + The freeze is **O(n)** — it moves the elements into a fresh compact, + length-prefixed allocation so the resulting single pointer is + `Send`-safe and can outlive the arena. +* `Arc::from(vec)` / `Arc::from(string)` → [`Arc<[T]>`][__link35] / + [`Arc`][__link36], the shared, reference-counted freeze + (mirroring `std`’s `From> for Arc<[T]>`). +* [`Vec::leak`][__link37] → `&mut [T]` (or `&*v.leak()` for `&[T]`) + borrowed for the arena’s lifetime. For `T: !Drop`, this freeze is + **O(1) and allocation-free** — the existing buffer is reinterpreted in + place. Unlike the `Box`/`Arc` freezes, the slice does not outlive the arena. The `Vec` freeze also reclaims any unused capacity left in the buffer when the conditions allow it, so those bytes become available @@ -211,7 +222,7 @@ builder.push_str("hello, "); builder.push_str("world"); // Freeze for storage: 8-byte single-pointer smart pointer. O(n) — copies the bytes. -let stored: Box = builder.into_arena_box_str(); +let stored: Box = builder.into_boxed_str(); assert_eq!(&*stored, "hello, world"); ``` @@ -219,13 +230,13 @@ Use this pattern whenever you’d be storing many strings or slices long-term — the per-pointer savings (8 bytes for both strings and slices) add up quickly across millions of items. -See [`BUMPALO.md`][__link35] -for a feature-by-feature comparison with [`bumpalo`][__link36]. +See [`BUMPALO.md`][__link38] +for a feature-by-feature comparison with [`bumpalo`][__link39]. ## Strings `multitude` provides a family of arena-resident string types in the -[`strings`][__link37] module. The model is the same one used for arbitrary +[`strings`][__link40] module. The model is the same one used for arbitrary values elsewhere in the crate — bump-allocation backed by a per-chunk refcount — but specialized for UTF-8 / UTF-16 text and a compact single-pointer representation. @@ -240,30 +251,30 @@ There are two roles a string type can play: |UTF-8|UTF-16|Sharing|Mutable|Notes| |-----|------|-------|-------|-----| - |[`Arc`][__link38]|[`ArcUtf16Str`][__link39]|atomic refcount; `Clone`, `Send + Sync`|no|cross-thread sharing| - |[`Box`][__link40]|[`BoxUtf16Str`][__link41]|unique owner; `Send + Sync` (not `Clone`)|yes|drops eagerly| + |[`Arc`][__link41]|[`ArcUtf16Str`][__link42]|atomic refcount; `Clone`, `Send + Sync`|no|cross-thread sharing| + |[`Box`][__link43]|[`BoxUtf16Str`][__link44]|unique owner; `Send + Sync` (not `Clone`)|yes|drops eagerly| Like the other arena smart pointers, they keep their owning chunk - alive via a refcount, so they can outlive the [`Arena`][__link42] they came + alive via a refcount, so they can outlive the [`Arena`][__link45] they came from. -1. **Builders (mutable, growable).** [`String`][__link43] and - [`Utf16String`][__link44] are transient growable +1. **Builders (mutable, growable).** [`String`][__link46] and + [`Utf16String`][__link47] are transient growable buffers — small structs (32 bytes) carrying a data pointer + length + capacity + arena reference. You build them up with - `push_str` / `push_char` / [`format!`][__link45] / - [`format_utf16!`][__link46], then **freeze** them + `push_str` / `push` / [`format!`][__link48] / + [`format_utf16!`][__link49], then **freeze** them into one of the smart pointers above: |Builder|Freeze method|Result| |-------|-------------|------| - |[`String`][__link47]|[`into_arena_box_str`][__link48]|[`Box`][__link49]| - |[`Utf16String`][__link50]|[`into_arena_box_utf16_str`][__link51]|[`BoxUtf16Str`][__link52]| + |[`String`][__link50]|[`into_boxed_str`][__link51]|[`Box`][__link52]| + |[`Utf16String`][__link53]|[`into_boxed_utf16_str`][__link54]|[`BoxUtf16Str`][__link55]| The UTF-16 freeze reuses the buffer in place (O(1)) and returns any unused tail capacity to the chunk’s bump cursor when it can. The UTF-8 freeze copies the bytes (O(n)) into a compact, - length-prefixed allocation so [`Box`][__link53] stays a + length-prefixed allocation so [`Box`][__link56] stays a single, `Send`-safe pointer. UTF-16 support requires the `utf16` Cargo feature. Strict (validated) @@ -288,7 +299,7 @@ assert_eq!(&*s, "hello, world"); let mut b = arena.alloc_string(); b.push_str("abc"); b.push_str("123"); -let frozen: Box = b.into_arena_box_str(); +let frozen: Box = b.into_boxed_str(); assert_eq!(&*frozen, "abc123"); // format!-style: @@ -317,7 +328,7 @@ assert_eq!(&*s2, utf16str!("hello")); let mut b = arena.alloc_utf16_string(); b.push_str(utf16str!("abc")); b.push_from_str("123"); -let frozen = b.into_arena_box_utf16_str(); +let frozen = b.into_boxed_utf16_str(); assert_eq!(&*frozen, utf16str!("abc123")); // format!-style: @@ -328,16 +339,16 @@ assert_eq!(greeting.as_utf16_str(), utf16str!("Hello, Alice!")); ## Building DSTs -With the `dst` Cargo feature enabled, [`Arena`][__link54] exposes -[`Arena::alloc_dst_arc`][__link55] and -[`Arena::alloc_dst_box`][__link56] (and their `try_*` siblings) for +With the `dst` Cargo feature enabled, [`Arena`][__link57] exposes +[`Arena::alloc_dst_arc`][__link58] and +[`Arena::alloc_dst_box`][__link59] (and their `try_*` siblings) for constructing values whose layout is only known at runtime (custom DSTs, fat pointers, trait objects). -Each of these takes a [`Layout`][__link57], a +Each of these takes a [`Layout`][__link60], a pointer-metadata value (e.g. a slice length, a `DynMetadata`), and a closure that initializes the buffer through a typed fat pointer. -For most users, the [`dst-factory`][__link58] companion crate is the +For most users, the [`dst-factory`][__link61] companion crate is the recommended high-level driver; the low-level interface looks like: ```rust @@ -368,15 +379,15 @@ existing `_arc` slice methods). |Feature|Description| |-------|-----------| -|`std` *(default)*|Enables [`std::io::Write`][__link59] on [`Vec`][__link60] for use with `write!`, `std::io::copy`, `serde_json::to_writer`, and similar. Disable for `#![no_std]` environments (the crate still requires `alloc`).| +|`std` *(default)*|Enables [`std::io::Write`][__link62] on [`Vec`][__link63] for use with `write!`, `std::io::copy`, `serde_json::to_writer`, and similar. Disable for `#![no_std]` environments (the crate still requires `alloc`).| |`stats`|Enables runtime instrumentation counters returned by `Arena::stats`. Disable for the tightest allocation throughput when you don’t need observability.| -|`serde`|Adds `Serialize` impls for [`Arc`][__link61], [`Box`][__link62], [`String`][__link63], and [`Vec`][__link64]. With `serde + utf16`, also adds impls for the UTF-16 types (transcoded to UTF-8 on the wire).| -|`dst`|Enables the `dst` module for constructing true dynamically-sized types and trait objects in the arena via [`Arena::alloc_dst_arc`][__link65] / [`Arena::alloc_dst_box`][__link66], plus eight `Arena::alloc_slice_*_box` methods.| -|`utf16`|Adds a parallel UTF-16 string surface ([`ArcUtf16Str`][__link67], [`BoxUtf16Str`][__link68], [`Utf16String`][__link69], and [`format_utf16!`][__link70]) backed by the [`widestring`][__link71] crate. Lengths are counted in `u16` elements.| -|`zerocopy`|Provides [`ZerocopyView`][__link72] for safe zero-initialized allocation of types implementing [`zerocopy::FromZeros`][__link73]. Access via [`Arena::zerocopy()`][__link74].| -|`bytemuck`|Provides [`BytemuckView`][__link75] for safe zero-initialized allocation of types implementing [`bytemuck::Zeroable`][__link76]. Access via [`Arena::bytemuck()`][__link77].| -|`bytes`|Adds [`From`][__link78] conversions from [`Arc<[u8]>`][__link79] and [`Arc`][__link80] into [`bytes::Bytes`][__link81], enabling zero-copy integration with the Tokio / Hyper async ecosystem.| -|`bytesbuf`|Implements [`bytesbuf::mem::Memory`][__link82] directly on [`Arena`][__link83], so that [`BytesBuf`][__link84] buffers can be backed by arena chunks. Implies `std`.| +|`serde`|Adds `Serialize` impls for [`Arc`][__link64], [`Box`][__link65], [`String`][__link66], and [`Vec`][__link67]. With `serde + utf16`, also adds impls for the UTF-16 types (transcoded to UTF-8 on the wire).| +|`dst`|Enables the `dst` module for constructing true dynamically-sized types and trait objects in the arena via [`Arena::alloc_dst_arc`][__link68] / [`Arena::alloc_dst_box`][__link69], plus eight `Arena::alloc_slice_*_box` methods.| +|`utf16`|Adds a parallel UTF-16 string surface ([`ArcUtf16Str`][__link70], [`BoxUtf16Str`][__link71], [`Utf16String`][__link72], and [`format_utf16!`][__link73]) backed by the [`widestring`][__link74] crate. Lengths are counted in `u16` elements.| +|`zerocopy`|Provides [`ZerocopyView`][__link75] for safe zero-initialized allocation of types implementing [`zerocopy::FromZeros`][__link76]. Access via [`Arena::zerocopy()`][__link77].| +|`bytemuck`|Provides [`BytemuckView`][__link78] for safe zero-initialized allocation of types implementing [`bytemuck::Zeroable`][__link79]. Access via [`Arena::bytemuck()`][__link80].| +|`bytes`|Adds [`From`][__link81] conversions from [`Arc<[u8]>`][__link82] and [`Arc`][__link83] into [`bytes::Bytes`][__link84], enabling zero-copy integration with the Tokio / Hyper async ecosystem.| +|`bytesbuf`|Implements [`bytesbuf::mem::Memory`][__link85] directly on [`Arena`][__link86], so that [`BytesBuf`][__link87] buffers can be backed by arena chunks. Implies `std`.|
@@ -384,7 +395,7 @@ existing `_arc` slice methods). This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjJhdIQbLiTyV0MU86EbZU15e0PmecoboQ9jo59bnAEbyDXw04U13GlhYvRhcoQbL7eM_c-g_FIbcvUKD8Z1WTMbdKENvBr-UV8bFzF5ShZNBethZIWCaGJ5dGVtdWNrZjEuMjUuMIJlYnl0ZXNmMS4xMS4xgmhieXRlc2J1ZmUwLjUuM4JpbXVsdGl0dWRlZTAuMS4zgmh6ZXJvY29weWYwLjguNTA + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjJhdIQbLiTyV0MU86EbZU15e0PmecoboQ9jo59bnAEbyDXw04U13GlhYvRhcoQbNi47luKnc6Ubx7xxyyKNjFQb1fXTasooMaAbK7exZHPLL-RhZIWCaGJ5dGVtdWNrZjEuMjUuMIJlYnl0ZXNmMS4xMS4xgmhieXRlc2J1ZmUwLjUuM4JpbXVsdGl0dWRlZTAuMS4zgmh6ZXJvY29weWYwLjguNTA [__link0]: https://crates.io/crates/bumpalo [__link1]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc [__link10]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec @@ -410,63 +421,66 @@ This crate was developed as part of The Oxidizer Project. Br [__link29]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String [__link3]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc [__link30]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec - [__link31]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String::into_arena_box_str + [__link31]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String::into_boxed_str [__link32]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link33]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec::into_arena_box + [__link33]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec::into_boxed_slice [__link34]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link35]: https://github.com/microsoft/oxidizer/blob/main/crates/multitude/BUMPALO.md - [__link36]: https://crates.io/crates/bumpalo - [__link37]: https://docs.rs/multitude/0.1.3/multitude/strings/index.html - [__link38]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc - [__link39]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::ArcUtf16Str + [__link35]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc + [__link36]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc + [__link37]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec::leak + [__link38]: https://github.com/microsoft/oxidizer/blob/main/crates/multitude/BUMPALO.md + [__link39]: https://crates.io/crates/bumpalo [__link4]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link40]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link41]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::BoxUtf16Str - [__link42]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena - [__link43]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String - [__link44]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String - [__link45]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::format - [__link46]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::format_utf16 - [__link47]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String - [__link48]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String::into_arena_box_str - [__link49]: https://docs.rs/multitude/0.1.3/multitude/?search=Box + [__link40]: https://docs.rs/multitude/0.1.3/multitude/strings/index.html + [__link41]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc + [__link42]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::ArcUtf16Str + [__link43]: https://docs.rs/multitude/0.1.3/multitude/?search=Box + [__link44]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::BoxUtf16Str + [__link45]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena + [__link46]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String + [__link47]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String + [__link48]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::format + [__link49]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::format_utf16 [__link5]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link50]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String - [__link51]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String::into_arena_box_utf16_str - [__link52]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::BoxUtf16Str - [__link53]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link54]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena - [__link55]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_arc - [__link56]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_box - [__link57]: https://doc.rust-lang.org/stable/core/?search=alloc::Layout - [__link58]: https://crates.io/crates/dst-factory - [__link59]: https://doc.rust-lang.org/stable/std/?search=io::Write + [__link50]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String + [__link51]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String::into_boxed_str + [__link52]: https://docs.rs/multitude/0.1.3/multitude/?search=Box + [__link53]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String + [__link54]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String::into_boxed_utf16_str + [__link55]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::BoxUtf16Str + [__link56]: https://docs.rs/multitude/0.1.3/multitude/?search=Box + [__link57]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena + [__link58]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_arc + [__link59]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_box [__link6]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link60]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec - [__link61]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc - [__link62]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link63]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String - [__link64]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec - [__link65]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_arc - [__link66]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_box - [__link67]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::ArcUtf16Str - [__link68]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::BoxUtf16Str - [__link69]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String + [__link60]: https://doc.rust-lang.org/stable/core/?search=alloc::Layout + [__link61]: https://crates.io/crates/dst-factory + [__link62]: https://doc.rust-lang.org/stable/std/?search=io::Write + [__link63]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec + [__link64]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc + [__link65]: https://docs.rs/multitude/0.1.3/multitude/?search=Box + [__link66]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String + [__link67]: https://docs.rs/multitude/0.1.3/multitude/?search=vec::Vec + [__link68]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_arc + [__link69]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::alloc_dst_box [__link7]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc - [__link70]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::format_utf16 - [__link71]: https://crates.io/crates/widestring - [__link72]: https://docs.rs/multitude/0.1.3/multitude/?search=zerocopy::ZerocopyView - [__link73]: https://docs.rs/zerocopy/0.8.50/zerocopy/?search=FromZeros - [__link74]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::zerocopy - [__link75]: https://docs.rs/multitude/0.1.3/multitude/?search=bytemuck::BytemuckView - [__link76]: https://docs.rs/bytemuck/1.25.0/bytemuck/?search=Zeroable - [__link77]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::bytemuck - [__link78]: https://doc.rust-lang.org/stable/std/convert/trait.From.html - [__link79]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc + [__link70]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::ArcUtf16Str + [__link71]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::BoxUtf16Str + [__link72]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::Utf16String + [__link73]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::format_utf16 + [__link74]: https://crates.io/crates/widestring + [__link75]: https://docs.rs/multitude/0.1.3/multitude/?search=zerocopy::ZerocopyView + [__link76]: https://docs.rs/zerocopy/0.8.50/zerocopy/?search=FromZeros + [__link77]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::zerocopy + [__link78]: https://docs.rs/multitude/0.1.3/multitude/?search=bytemuck::BytemuckView + [__link79]: https://docs.rs/bytemuck/1.25.0/bytemuck/?search=Zeroable [__link8]: https://docs.rs/multitude/0.1.3/multitude/?search=Box - [__link80]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc - [__link81]: https://docs.rs/bytes/1.11.1/bytes/?search=Bytes - [__link82]: https://docs.rs/bytesbuf/0.5.3/bytesbuf/?search=mem::Memory - [__link83]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena - [__link84]: https://docs.rs/bytesbuf/0.5.3/bytesbuf/?search=BytesBuf + [__link80]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena::bytemuck + [__link81]: https://doc.rust-lang.org/stable/std/convert/trait.From.html + [__link82]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc + [__link83]: https://docs.rs/multitude/0.1.3/multitude/?search=Arc + [__link84]: https://docs.rs/bytes/1.11.1/bytes/?search=Bytes + [__link85]: https://docs.rs/bytesbuf/0.5.3/bytesbuf/?search=mem::Memory + [__link86]: https://docs.rs/multitude/0.1.3/multitude/?search=Arena + [__link87]: https://docs.rs/bytesbuf/0.5.3/bytesbuf/?search=BytesBuf [__link9]: https://docs.rs/multitude/0.1.3/multitude/?search=strings::String diff --git a/crates/multitude/benches/criterion_alloc.rs b/crates/multitude/benches/criterion_alloc.rs index 87ca719d9..6d4554c89 100644 --- a/crates/multitude/benches/criterion_alloc.rs +++ b/crates/multitude/benches/criterion_alloc.rs @@ -42,6 +42,16 @@ const SLICE_LEN: usize = 8; /// `warm_bump` (which similarly primes its cursor with a no-op /// alloc) and isolates the comparison to "in-chunk bump cost /// only" with no cold refill amortized into the per-op number. +/// +/// Only used by benches that exercise **both** flavors in the timed +/// region: the `vec_builder` benches build a local `Vec` and then +/// `into_arc()` it. Benches that touch a single flavor must use +/// [`warm_arena_local`] / [`warm_arena_shared`] instead: priming the +/// unused flavor allocates a dead-weight 64 KiB chunk that doubles the +/// arena's memory footprint and inflates the measured per-op time +/// through extra cache/TLB pressure at the batch working-set sizes +/// criterion picks (verified: priming the unused shared chunk made +/// `alloc_slice_copy` measure ~9x slower than its true cost). fn warm_arena() -> Arena { let arena = Arena::builder() .with_capacity_local(64 * 1024) @@ -52,6 +62,29 @@ fn warm_arena() -> Arena { arena } +/// Like [`warm_arena`] but primes **only** the local (ref/value) +/// flavor. Used by benches whose timed region allocates exclusively +/// from the local chunk (`alloc`, `alloc_str`, `alloc_slice_*` ref, +/// `alloc_string*`). Priming the shared flavor here would add a +/// dead-weight 64 KiB chunk — see [`warm_arena`]. +fn warm_arena_local() -> Arena { + let arena = Arena::builder().with_capacity_local(64 * 1024).build(); + let _: &mut u64 = arena.alloc(0_u64); + arena +} + +/// Like [`warm_arena`] but primes **only** the shared (`Box`/`Arc`) +/// flavor. Used by benches whose timed region allocates exclusively +/// from the shared chunk (every `*_box` / `*_arc` bench — both smart +/// pointers are backed by refcounted shared chunks). Priming the local +/// flavor here would add a dead-weight 64 KiB chunk — see +/// [`warm_arena`]. +fn warm_arena_shared() -> Arena { + let arena = Arena::builder().with_capacity_shared(64 * 1024).build(); + let _ = arena.alloc_arc(0_u64); + arena +} + /// Build a [`bumpalo::Bump`] pre-sized to fit the timed region. /// /// `Bump::with_capacity(N)` reserves a chunk of at least N bytes from @@ -84,7 +117,7 @@ fn bench_alloc_u64(c: &mut Criterion) { g.bench_function("alloc", |b| { b.iter_batched( - warm_arena, + warm_arena_local, |arena| { for i in 0..N { let _: &mut u64 = black_box(arena.alloc(black_box(i as u64))); @@ -96,7 +129,7 @@ fn bench_alloc_u64(c: &mut Criterion) { }); g.bench_function("alloc_with", |b| { b.iter_batched( - warm_arena, + warm_arena_local, |arena| { for i in 0..N { let _: &mut u64 = black_box(arena.alloc_with(|| black_box(i as u64))); @@ -109,7 +142,7 @@ fn bench_alloc_u64(c: &mut Criterion) { g.bench_function("alloc_box", |b| { b.iter_batched( - || (warm_arena(), Vec::>::with_capacity(N)), + || (warm_arena_shared(), Vec::>::with_capacity(N)), |(arena, mut h)| { for i in 0..N { h.push(arena.alloc_box(black_box(i as u64))); @@ -121,7 +154,7 @@ fn bench_alloc_u64(c: &mut Criterion) { }); g.bench_function("alloc_box_with", |b| { b.iter_batched( - || (warm_arena(), Vec::>::with_capacity(N)), + || (warm_arena_shared(), Vec::>::with_capacity(N)), |(arena, mut h)| { for i in 0..N { h.push(arena.alloc_box_with(|| black_box(i as u64))); @@ -133,7 +166,7 @@ fn bench_alloc_u64(c: &mut Criterion) { }); g.bench_function("alloc_uninit_box", |b| { b.iter_batched( - || (warm_arena(), Vec::>>::with_capacity(N)), + || (warm_arena_shared(), Vec::>>::with_capacity(N)), |(arena, mut h)| { for _ in 0..N { h.push(arena.alloc_uninit_box::()); @@ -145,7 +178,7 @@ fn bench_alloc_u64(c: &mut Criterion) { }); g.bench_function("alloc_zeroed_box", |b| { b.iter_batched( - || (warm_arena(), Vec::>>::with_capacity(N)), + || (warm_arena_shared(), Vec::>>::with_capacity(N)), |(arena, mut h)| { for _ in 0..N { h.push(arena.alloc_zeroed_box::()); @@ -158,7 +191,7 @@ fn bench_alloc_u64(c: &mut Criterion) { g.bench_function("alloc_arc", |b| { b.iter_batched( - || (warm_arena(), Vec::>::with_capacity(N)), + || (warm_arena_shared(), Vec::>::with_capacity(N)), |(arena, mut h)| { for i in 0..N { h.push(arena.alloc_arc(black_box(i as u64))); @@ -170,7 +203,7 @@ fn bench_alloc_u64(c: &mut Criterion) { }); g.bench_function("alloc_arc_with", |b| { b.iter_batched( - || (warm_arena(), Vec::>::with_capacity(N)), + || (warm_arena_shared(), Vec::>::with_capacity(N)), |(arena, mut h)| { for i in 0..N { h.push(arena.alloc_arc_with(|| black_box(i as u64))); @@ -182,7 +215,7 @@ fn bench_alloc_u64(c: &mut Criterion) { }); g.bench_function("alloc_uninit_arc", |b| { b.iter_batched( - || (warm_arena(), Vec::>>::with_capacity(N)), + || (warm_arena_shared(), Vec::>>::with_capacity(N)), |(arena, mut h)| { for _ in 0..N { h.push(arena.alloc_uninit_arc::()); @@ -194,7 +227,7 @@ fn bench_alloc_u64(c: &mut Criterion) { }); g.bench_function("alloc_zeroed_arc", |b| { b.iter_batched( - || (warm_arena(), Vec::>>::with_capacity(N)), + || (warm_arena_shared(), Vec::>>::with_capacity(N)), |(arena, mut h)| { for _ in 0..N { h.push(arena.alloc_zeroed_arc::()); @@ -242,10 +275,10 @@ fn bench_alloc_str(c: &mut Criterion) { g.bench_function("alloc_str", |b| { b.iter_batched( - warm_arena, + warm_arena_local, |arena| { for w in &words { - let _: &mut str = black_box(arena.alloc_str(black_box(w))); + let _: &mut str = black_box(arena.alloc_str(black_box(w.as_str()))); } arena }, @@ -254,10 +287,10 @@ fn bench_alloc_str(c: &mut Criterion) { }); g.bench_function("alloc_str_box", |b| { b.iter_batched( - || (warm_arena(), Vec::>::with_capacity(N)), + || (warm_arena_shared(), Vec::>::with_capacity(N)), |(arena, mut o)| { for w in &words { - o.push(arena.alloc_str_box(black_box(w))); + o.push(arena.alloc_str_box(black_box(w.as_str()))); } (o, arena) }, @@ -266,10 +299,10 @@ fn bench_alloc_str(c: &mut Criterion) { }); g.bench_function("alloc_str_arc", |b| { b.iter_batched( - || (warm_arena(), Vec::>::with_capacity(N)), + || (warm_arena_shared(), Vec::>::with_capacity(N)), |(arena, mut o)| { for w in &words { - o.push(arena.alloc_str_arc(black_box(w))); + o.push(arena.alloc_str_arc(black_box(w.as_str()))); } (o, arena) }, @@ -281,7 +314,7 @@ fn bench_alloc_str(c: &mut Criterion) { warm_bump, |bump| { for w in &words { - let _: &mut str = black_box(bump.alloc_str(black_box(w))); + let _: &mut str = black_box(bump.alloc_str(black_box(w.as_str()))); } bump }, @@ -303,7 +336,7 @@ fn bench_alloc_slice(c: &mut Criterion) { ($name:literal, $body:expr) => { g.bench_function($name, |b| { b.iter_batched( - warm_arena, + warm_arena_local, |arena| { let r = $body(&arena); // Return owned values so their `Drop` runs outside the timed region. @@ -318,7 +351,7 @@ fn bench_alloc_slice(c: &mut Criterion) { ($name:literal, $T:ty, $body:expr) => { g.bench_function($name, |b| { b.iter_batched( - || (warm_arena(), Vec::<$T>::with_capacity(N)), + || (warm_arena_shared(), Vec::<$T>::with_capacity(N)), |(arena, mut o)| { $body(&arena, &mut o); (o, arena) @@ -347,7 +380,7 @@ fn bench_alloc_slice(c: &mut Criterion) { // ref bench_arena!("alloc_slice_copy", |arena: &Arena| { for s in &slices { - let _: &mut [u64] = black_box(arena.alloc_slice_copy(black_box(s))); + let _: &mut [u64] = black_box(arena.alloc_slice_copy(black_box(s.as_slice()))); } }); bench_arena!("alloc_slice_clone", |arena: &Arena| { @@ -369,7 +402,7 @@ fn bench_alloc_slice(c: &mut Criterion) { // box bench_arena_collect!("alloc_slice_copy_box", Box<[u64]>, |arena: &Arena, o: &mut Vec>| { for s in &slices { - o.push(arena.alloc_slice_copy_box(black_box(s))); + o.push(arena.alloc_slice_copy_box(black_box(s.as_slice()))); } }); bench_arena_collect!("alloc_slice_clone_box", Box<[u64]>, |arena: &Arena, o: &mut Vec>| { @@ -407,7 +440,7 @@ fn bench_alloc_slice(c: &mut Criterion) { // arc bench_arena_collect!("alloc_slice_copy_arc", Arc<[u64]>, |arena: &Arena, o: &mut Vec>| { for s in &slices { - o.push(arena.alloc_slice_copy_arc(black_box(s))); + o.push(arena.alloc_slice_copy_arc(black_box(s.as_slice()))); } }); bench_arena_collect!("alloc_slice_clone_arc", Arc<[u64]>, |arena: &Arena, o: &mut Vec>| { @@ -476,28 +509,36 @@ fn bench_string_builder(c: &mut Criterion) { g.bench_function("alloc_string", |b| { b.iter_batched( - warm_arena, + warm_arena_local, |arena| { let mut s = arena.alloc_string(); for w in &words { s.push_str(black_box(w.as_str())); } - let frozen = s.into_arena_box_str(); - (frozen, arena) + // Mirror bumpalo's `into_bump_str`: take a `&str` view of + // the in-place chunk storage with no copy into a `Box`. + // Dropping `s` is ~free (`u8` has no drop; storage is + // reclaimed at arena teardown) and must precede returning + // `arena`, which `s` borrows. + black_box(s.as_str()); + drop(s); + arena }, BatchSize::SmallInput, ); }); g.bench_function("alloc_string_with_capacity", |b| { b.iter_batched( - warm_arena, + warm_arena_local, |arena| { let mut s = arena.alloc_string_with_capacity(N * 6); for w in &words { s.push_str(black_box(w.as_str())); } - let frozen = s.into_arena_box_str(); - (frozen, arena) + // See `alloc_string` above. + black_box(s.as_str()); + drop(s); + arena }, BatchSize::SmallInput, ); @@ -551,7 +592,7 @@ fn bench_vec_builder(c: &mut Criterion) { for &i in &ints { v.push(black_box(i)); } - let frozen = v.into_arena_arc(); + let frozen = multitude::Arc::from(v); (frozen, arena) }, BatchSize::SmallInput, @@ -565,7 +606,7 @@ fn bench_vec_builder(c: &mut Criterion) { for &i in &ints { v.push(black_box(i)); } - let frozen = v.into_arena_arc(); + let frozen = multitude::Arc::from(v); (frozen, arena) }, BatchSize::SmallInput, @@ -611,11 +652,10 @@ fn bench_vec_builder(c: &mut Criterion) { fn bench_arena_creation(c: &mut Criterion) { let mut g = c.benchmark_group("arena_creation"); - g.bench_function("multitude", |b| { + g.bench_function("multitude_new", |b| { b.iter(|| { let arena = Arena::new(); black_box(&arena); - // Drop is part of the lifecycle — included in the timed region. drop(arena); }); }); diff --git a/crates/multitude/benches/gungraun_alloc/linux.rs b/crates/multitude/benches/gungraun_alloc/linux.rs index 6b9aaced5..b410a0897 100644 --- a/crates/multitude/benches/gungraun_alloc/linux.rs +++ b/crates/multitude/benches/gungraun_alloc/linux.rs @@ -152,7 +152,7 @@ fn bump_slices() -> (bumpalo::Bump, Vec<[u64; SLICE_LEN]>) { #[bench::run(warm_arena())] fn alloc(arena: Arena) -> Arena { for i in 0..N { - let _: &mut u64 = black_box(black_box(&arena).alloc(black_box(i as u64))); + let _: &mut u64 = black_box(arena.alloc(black_box(i as u64))); } arena } @@ -161,7 +161,7 @@ fn alloc(arena: Arena) -> Arena { #[bench::run(warm_arena())] fn alloc_with(arena: Arena) -> Arena { for i in 0..N { - let _: &mut u64 = black_box(black_box(&arena).alloc_with(|| black_box(i as u64))); + let _: &mut u64 = black_box(arena.alloc_with(|| black_box(i as u64))); } arena } @@ -248,18 +248,18 @@ fn alloc_zeroed_arc(state: (Arena, Vec>>)) -> (Arena, Vec bumpalo::Bump { +fn bumpalo_alloc(bump: bumpalo::Bump) -> bumpalo::Bump { for i in 0..N { - let _: &mut u64 = black_box(black_box(&bump).alloc(black_box(i as u64))); + let _: &mut u64 = black_box(bump.alloc(black_box(i as u64))); } bump } #[library_benchmark] #[bench::run(warm_bump())] -fn alloc_u64_bumpalo_alloc_with(bump: bumpalo::Bump) -> bumpalo::Bump { +fn bumpalo_alloc_with(bump: bumpalo::Bump) -> bumpalo::Bump { for i in 0..N { - let _: &mut u64 = black_box(black_box(&bump).alloc_with(|| black_box(i as u64))); + let _: &mut u64 = black_box(bump.alloc_with(|| black_box(i as u64))); } bump } @@ -271,7 +271,7 @@ fn alloc_u64_bumpalo_alloc_with(bump: bumpalo::Bump) -> bumpalo::Bump { fn alloc_str(state: (Arena, Vec, Vec<*mut str>)) -> (Arena, Vec, Vec<*mut str>) { let (arena, words, mut out) = state; for w in &words { - let s: &mut str = black_box(arena.alloc_str(black_box(w))); + let s: &mut str = black_box(arena.alloc_str(black_box(w.as_str()))); out.push(s as *mut str); } (arena, words, out) @@ -282,7 +282,7 @@ fn alloc_str(state: (Arena, Vec, Vec<*mut str>)) -> (Arena, Vec, fn alloc_str_box(state: (Arena, Vec, Vec>)) -> (Arena, Vec, Vec>) { let (arena, words, mut out) = state; for w in &words { - out.push(black_box(arena.alloc_str_box(black_box(w)))); + out.push(black_box(arena.alloc_str_box(black_box(w.as_str())))); } (arena, words, out) } @@ -292,17 +292,17 @@ fn alloc_str_box(state: (Arena, Vec, Vec>)) -> (Arena, Vec, Vec>)) -> (Arena, Vec, Vec>) { let (arena, words, mut out) = state; for w in &words { - out.push(black_box(arena.alloc_str_arc(black_box(w)))); + out.push(black_box(arena.alloc_str_arc(black_box(w.as_str())))); } (arena, words, out) } #[library_benchmark] #[bench::run(bump_words_out())] -fn alloc_str_bumpalo_alloc_str(state: (bumpalo::Bump, Vec, Vec<*mut str>)) -> (bumpalo::Bump, Vec, Vec<*mut str>) { +fn bumpalo_alloc_str(state: (bumpalo::Bump, Vec, Vec<*mut str>)) -> (bumpalo::Bump, Vec, Vec<*mut str>) { let (bump, words, mut out) = state; for w in &words { - let s: &mut str = black_box(black_box(&bump).alloc_str(black_box(w))); + let s: &mut str = black_box(bump.alloc_str(black_box(w.as_str()))); out.push(s as *mut str); } (bump, words, out) @@ -315,7 +315,7 @@ fn alloc_str_bumpalo_alloc_str(state: (bumpalo::Bump, Vec, Vec<*mut str> fn alloc_slice_copy(state: (Arena, Vec<[u64; SLICE_LEN]>)) -> (Arena, Vec<[u64; SLICE_LEN]>) { let (arena, slices) = state; for s in &slices { - let _: &mut [u64] = black_box(arena.alloc_slice_copy(black_box(s))); + let _: &mut [u64] = black_box(arena.alloc_slice_copy(black_box(s.as_slice()))); } (arena, slices) } @@ -355,7 +355,7 @@ fn alloc_slice_fill_iter(arena: Arena) -> Arena { fn alloc_slice_copy_box(state: (Arena, Vec<[u64; SLICE_LEN]>, Vec>)) -> (Arena, Vec<[u64; SLICE_LEN]>, Vec>) { let (arena, slices, mut out) = state; for s in &slices { - out.push(black_box(arena.alloc_slice_copy_box(black_box(s)))); + out.push(black_box(arena.alloc_slice_copy_box(black_box(s.as_slice())))); } (arena, slices, out) } @@ -421,7 +421,7 @@ fn alloc_zeroed_slice_box(state: (Arena, Vec]>>)) -> (Aren fn alloc_slice_copy_arc(state: (Arena, Vec<[u64; SLICE_LEN]>, Vec>)) -> (Arena, Vec<[u64; SLICE_LEN]>, Vec>) { let (arena, slices, mut out) = state; for s in &slices { - out.push(black_box(arena.alloc_slice_copy_arc(black_box(s)))); + out.push(black_box(arena.alloc_slice_copy_arc(black_box(s.as_slice())))); } (arena, slices, out) } @@ -484,38 +484,38 @@ fn alloc_zeroed_slice_arc(state: (Arena, Vec]>>)) -> (Aren #[library_benchmark] #[bench::run(bump_slices())] -fn alloc_slice_bumpalo_alloc_slice_copy(state: (bumpalo::Bump, Vec<[u64; SLICE_LEN]>)) -> (bumpalo::Bump, Vec<[u64; SLICE_LEN]>) { +fn bumpalo_alloc_slice_copy(state: (bumpalo::Bump, Vec<[u64; SLICE_LEN]>)) -> (bumpalo::Bump, Vec<[u64; SLICE_LEN]>) { let (bump, slices) = state; for s in &slices { - let _: &mut [u64] = black_box(black_box(&bump).alloc_slice_copy(black_box(s.as_slice()))); + let _: &mut [u64] = black_box(bump.alloc_slice_copy(black_box(s.as_slice()))); } (bump, slices) } #[library_benchmark] #[bench::run(bump_slices())] -fn alloc_slice_bumpalo_alloc_slice_clone(state: (bumpalo::Bump, Vec<[u64; SLICE_LEN]>)) -> (bumpalo::Bump, Vec<[u64; SLICE_LEN]>) { +fn bumpalo_alloc_slice_clone(state: (bumpalo::Bump, Vec<[u64; SLICE_LEN]>)) -> (bumpalo::Bump, Vec<[u64; SLICE_LEN]>) { let (bump, slices) = state; for s in &slices { - let _: &mut [u64] = black_box(black_box(&bump).alloc_slice_clone(black_box(s.as_slice()))); + let _: &mut [u64] = black_box(bump.alloc_slice_clone(black_box(s.as_slice()))); } (bump, slices) } #[library_benchmark] #[bench::run(warm_bump())] -fn alloc_slice_bumpalo_alloc_slice_fill_with(bump: bumpalo::Bump) -> bumpalo::Bump { +fn bumpalo_alloc_slice_fill_with(bump: bumpalo::Bump) -> bumpalo::Bump { for _ in 0..N { - let _: &mut [u64] = black_box(black_box(&bump).alloc_slice_fill_with::(SLICE_LEN, |j| black_box(j as u64))); + let _: &mut [u64] = black_box(bump.alloc_slice_fill_with::(SLICE_LEN, |j| black_box(j as u64))); } bump } #[library_benchmark] #[bench::run(warm_bump())] -fn alloc_slice_bumpalo_alloc_slice_fill_iter(bump: bumpalo::Bump) -> bumpalo::Bump { +fn bumpalo_alloc_slice_fill_iter(bump: bumpalo::Bump) -> bumpalo::Bump { for _ in 0..N { - let _: &mut [u64] = black_box(black_box(&bump).alloc_slice_fill_iter((0..SLICE_LEN).map(|j| black_box(j as u64)))); + let _: &mut [u64] = black_box(bump.alloc_slice_fill_iter((0..SLICE_LEN).map(|j| black_box(j as u64)))); } bump } @@ -524,31 +524,36 @@ fn alloc_slice_bumpalo_alloc_slice_fill_iter(bump: bumpalo::Bump) -> bumpalo::Bu #[library_benchmark] #[bench::run(arena_words())] -fn alloc_string(state: (Arena, Vec)) -> (Box, Arena, Vec) { +fn alloc_string(state: (Arena, Vec)) -> (*const str, Arena, Vec) { let (arena, words) = state; let mut s = arena.alloc_string(); for w in &words { s.push_str(black_box(w.as_str())); } - let frozen = black_box(s.into_arena_box_str()); + // Mirror bumpalo's `into_bump_str`: take a `&str` view of the in-place + // chunk storage with no copy into a `Box`. The bytes stay valid + // until arena teardown, so the returned pointer remains usable. + let frozen: *const str = black_box(s.as_str() as *const str); + drop(s); (frozen, arena, words) } #[library_benchmark] #[bench::run(arena_words())] -fn alloc_string_with_capacity(state: (Arena, Vec)) -> (Box, Arena, Vec) { +fn alloc_string_with_capacity(state: (Arena, Vec)) -> (*const str, Arena, Vec) { let (arena, words) = state; let mut s = arena.alloc_string_with_capacity(N * 6); for w in &words { s.push_str(black_box(w.as_str())); } - let frozen = black_box(s.into_arena_box_str()); + let frozen: *const str = black_box(s.as_str() as *const str); + drop(s); (frozen, arena, words) } #[library_benchmark] #[bench::run(bump_words())] -fn string_builder_bumpalo_string_new_in(state: (bumpalo::Bump, Vec)) -> (*const str, bumpalo::Bump, Vec) { +fn bumpalo_string_new_in(state: (bumpalo::Bump, Vec)) -> (*const str, bumpalo::Bump, Vec) { let (bump, words) = state; let mut s = bumpalo::collections::String::new_in(&bump); for w in &words { @@ -560,7 +565,7 @@ fn string_builder_bumpalo_string_new_in(state: (bumpalo::Bump, Vec)) -> #[library_benchmark] #[bench::run(bump_words())] -fn string_builder_bumpalo_string_with_capacity_in(state: (bumpalo::Bump, Vec)) -> (*const str, bumpalo::Bump, Vec) { +fn bumpalo_string_with_capacity_in(state: (bumpalo::Bump, Vec)) -> (*const str, bumpalo::Bump, Vec) { let (bump, words) = state; let mut s = bumpalo::collections::String::with_capacity_in(N * 6, &bump); for w in &words { @@ -574,31 +579,31 @@ fn string_builder_bumpalo_string_with_capacity_in(state: (bumpalo::Bump, Vec)) -> (Arc<[i32]>, Arena, Vec) { +fn alloc_vec(state: (Arena, Vec)) -> (*const [i32], Arena, Vec) { let (arena, ints) = state; let mut v = arena.alloc_vec::(); for &i in &ints { v.push(black_box(i)); } - let frozen = black_box(v.into_arena_arc()); + let frozen: *const [i32] = black_box(v.leak() as *const [i32]); (frozen, arena, ints) } #[library_benchmark] #[bench::run(arena_ints())] -fn alloc_vec_with_capacity(state: (Arena, Vec)) -> (Arc<[i32]>, Arena, Vec) { +fn alloc_vec_with_capacity(state: (Arena, Vec)) -> (*const [i32], Arena, Vec) { let (arena, ints) = state; let mut v = arena.alloc_vec_with_capacity::(N); for &i in &ints { v.push(black_box(i)); } - let frozen = black_box(v.into_arena_arc()); + let frozen: *const [i32] = black_box(v.leak() as *const [i32]); (frozen, arena, ints) } #[library_benchmark] #[bench::run(bump_ints())] -fn vec_builder_bumpalo_vec_new_in(state: (bumpalo::Bump, Vec)) -> (*const [i32], bumpalo::Bump, Vec) { +fn bumpalo_vec_new_in(state: (bumpalo::Bump, Vec)) -> (*const [i32], bumpalo::Bump, Vec) { let (bump, ints) = state; let mut v: bumpalo::collections::Vec<'_, i32> = bumpalo::collections::Vec::new_in(&bump); for &i in &ints { @@ -610,7 +615,7 @@ fn vec_builder_bumpalo_vec_new_in(state: (bumpalo::Bump, Vec)) -> (*const [ #[library_benchmark] #[bench::run(bump_ints())] -fn vec_builder_bumpalo_vec_with_capacity_in(state: (bumpalo::Bump, Vec)) -> (*const [i32], bumpalo::Bump, Vec) { +fn bumpalo_vec_with_capacity_in(state: (bumpalo::Bump, Vec)) -> (*const [i32], bumpalo::Bump, Vec) { let (bump, ints) = state; let mut v: bumpalo::collections::Vec<'_, i32> = bumpalo::collections::Vec::with_capacity_in(N, &bump); for &i in &ints { @@ -627,13 +632,13 @@ fn vec_builder_bumpalo_vec_with_capacity_in(state: (bumpalo::Bump, Vec)) -> // involves a system allocation. #[library_benchmark] -fn arena_creation_multitude() { +fn multitude_new() { let arena = black_box(Arena::new()); drop(arena); } #[library_benchmark] -fn arena_creation_bumpalo_new() { +fn bumpalo_new() { let bump = black_box(bumpalo::Bump::new()); drop(bump); } @@ -641,15 +646,15 @@ fn arena_creation_bumpalo_new() { library_benchmark_group!( name = alloc_group; benchmarks = - arena_creation_multitude, arena_creation_bumpalo_new, + multitude_new, bumpalo_new, alloc, alloc_with, alloc_box, alloc_box_with, alloc_uninit_box, alloc_zeroed_box, alloc_arc, alloc_arc_with, alloc_uninit_arc, alloc_zeroed_arc, - alloc_u64_bumpalo_alloc, alloc_u64_bumpalo_alloc_with, + bumpalo_alloc, bumpalo_alloc_with, alloc_str, alloc_str_box, - alloc_str_arc, alloc_str_bumpalo_alloc_str, + alloc_str_arc, bumpalo_alloc_str, alloc_slice_copy, alloc_slice_clone, alloc_slice_fill_with, alloc_slice_fill_iter, alloc_slice_copy_box, alloc_slice_clone_box, @@ -658,12 +663,12 @@ library_benchmark_group!( alloc_slice_copy_arc, alloc_slice_clone_arc, alloc_slice_fill_with_arc, alloc_slice_fill_iter_arc, alloc_uninit_slice_arc, alloc_zeroed_slice_arc, - alloc_slice_bumpalo_alloc_slice_copy, alloc_slice_bumpalo_alloc_slice_clone, - alloc_slice_bumpalo_alloc_slice_fill_with, alloc_slice_bumpalo_alloc_slice_fill_iter, + bumpalo_alloc_slice_copy, bumpalo_alloc_slice_clone, + bumpalo_alloc_slice_fill_with, bumpalo_alloc_slice_fill_iter, alloc_string, alloc_string_with_capacity, - string_builder_bumpalo_string_new_in, string_builder_bumpalo_string_with_capacity_in, + bumpalo_string_new_in, bumpalo_string_with_capacity_in, alloc_vec, alloc_vec_with_capacity, - vec_builder_bumpalo_vec_new_in, vec_builder_bumpalo_vec_with_capacity_in + bumpalo_vec_new_in, bumpalo_vec_with_capacity_in ); main!( diff --git a/crates/multitude/benches/gungraun_drop/linux.rs b/crates/multitude/benches/gungraun_drop/linux.rs index 9c0b84643..9e0b3b523 100644 --- a/crates/multitude/benches/gungraun_drop/linux.rs +++ b/crates/multitude/benches/gungraun_drop/linux.rs @@ -185,109 +185,109 @@ fn setup_alloc() -> Arena { #[library_benchmark] #[bench::run(setup_box_u64())] -fn drop_box_u64(state: (Vec>, Arena)) { +fn box_u64(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_rc_u64())] -fn drop_rc_u64(state: (Vec>, Arena)) { +fn rc_u64(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_arc_u64())] -fn drop_arc_u64(state: (Vec>, Arena)) { +fn arc_u64(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_box_droppy())] -fn drop_box_droppy(state: (Vec>, Arena)) { +fn box_droppy(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_rc_droppy())] -fn drop_rc_droppy(state: (Vec>, Arena)) { +fn rc_droppy(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_arc_droppy())] -fn drop_arc_droppy(state: (Vec>, Arena)) { +fn arc_droppy(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_str_box())] -fn drop_str_box(state: (Vec>, Arena)) { +fn str_box(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_str_rc())] -fn drop_str_rc(state: (Vec>, Arena)) { +fn str_rc(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_str_arc())] -fn drop_str_arc(state: (Vec>, Arena)) { +fn str_arc(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_slice_box_u64())] -fn drop_slice_box_u64(state: (Vec>, Arena)) { +fn slice_box_u64(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_slice_rc_u64())] -fn drop_slice_rc_u64(state: (Vec>, Arena)) { +fn slice_rc_u64(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_slice_arc_u64())] -fn drop_slice_arc_u64(state: (Vec>, Arena)) { +fn slice_arc_u64(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_slice_box_droppy())] -fn drop_slice_box_droppy(state: (Vec>, Arena)) { +fn slice_box_droppy(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_slice_rc_droppy())] -fn drop_slice_rc_droppy(state: (Vec>, Arena)) { +fn slice_rc_droppy(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_slice_arc_droppy())] -fn drop_slice_arc_droppy(state: (Vec>, Arena)) { +fn slice_arc_droppy(state: (Vec>, Arena)) { black_box(state); } #[library_benchmark] #[bench::run(setup_alloc())] -fn drop_alloc(state: Arena) { +fn alloc(state: Arena) { black_box(state); } library_benchmark_group!( name = drop_group; benchmarks = - drop_box_u64, drop_rc_u64, drop_arc_u64, - drop_box_droppy, drop_rc_droppy, drop_arc_droppy, - drop_str_box, drop_str_rc, drop_str_arc, - drop_slice_box_u64, drop_slice_rc_u64, drop_slice_arc_u64, - drop_slice_box_droppy, drop_slice_rc_droppy, drop_slice_arc_droppy, - drop_alloc + box_u64, rc_u64, arc_u64, + box_droppy, rc_droppy, arc_droppy, + str_box, str_rc, str_arc, + slice_box_u64, slice_rc_u64, slice_arc_u64, + slice_box_droppy, slice_rc_droppy, slice_arc_droppy, + alloc ); main!( diff --git a/crates/multitude/docs/PERF.md b/crates/multitude/docs/PERF.md index 5524d36d6..173e127e6 100644 --- a/crates/multitude/docs/PERF.md +++ b/crates/multitude/docs/PERF.md @@ -13,98 +13,98 @@ Bench names are aligned between criterion and gungraun via the `GROUPS` table in | Variant | Time (criterion) | Instructions | Branch misses | Mem accesses | |---|---:|---:|---:|---:| -| `multitude` | 37 ns | 335 | 9 | 495 | +| `multitude_new` | 38 ns | 316 | 9 | 457 | | `bumpalo_new` | 1 ns | 16 | 1 | 26 | ## `alloc_u64` | Variant | Time (criterion) | Instructions | Branch misses | Mem accesses | |---|---:|---:|---:|---:| -| `alloc` | 4.02 µs | 16,032 | 7 | 25,055 | -| `alloc_with` | 4.07 µs | 16,031 | 11 | 25,052 | -| `alloc_box` | 8.62 µs | 25,047 | 6 | 41,086 | -| `alloc_box_with` | 8.60 µs | 26,047 | 6 | 42,086 | -| `alloc_uninit_box` | 6.93 µs | 22,047 | 6 | 35,086 | -| `alloc_zeroed_box` | 9.16 µs | 23,047 | 6 | 37,086 | -| `alloc_arc` | 8.49 µs | 25,047 | 8 | 41,086 | -| `alloc_arc_with` | 8.67 µs | 26,047 | 6 | 42,086 | -| `alloc_uninit_arc` | 6.82 µs | 22,047 | 6 | 35,086 | -| `alloc_zeroed_arc` | 9.10 µs | 23,047 | 6 | 37,086 | -| `bumpalo_alloc` | 4.02 µs | 21,025 | 5 | 31,042 | -| `bumpalo_alloc_with` | 4.00 µs | 21,023 | 3 | 31,038 | +| `alloc` | 6.45 µs | 14,026 | 6 | 21,043 | +| `alloc_with` | 6.53 µs | 14,024 | 11 | 21,040 | +| `alloc_box` | 5.24 µs | 23,043 | 9 | 37,078 | +| `alloc_box_with` | 5.18 µs | 24,043 | 9 | 38,078 | +| `alloc_uninit_box` | 2.33 µs | 20,043 | 9 | 31,078 | +| `alloc_zeroed_box` | 4.85 µs | 21,043 | 9 | 33,078 | +| `alloc_arc` | 5.36 µs | 23,043 | 7 | 37,078 | +| `alloc_arc_with` | 5.19 µs | 24,043 | 9 | 38,078 | +| `alloc_uninit_arc` | 2.33 µs | 20,043 | 9 | 31,078 | +| `alloc_zeroed_arc` | 4.96 µs | 21,043 | 9 | 33,078 | +| `bumpalo_alloc` | 5.97 µs | 19,022 | 4 | 27,037 | +| `bumpalo_alloc_with` | 6.08 µs | 19,020 | 4 | 27,034 | ## `alloc_str` | Variant | Time (criterion) | Instructions | Branch misses | Mem accesses | |---|---:|---:|---:|---:| -| `alloc_str` | 7.20 µs | 52,056 | 15 | 78,103 | -| `alloc_str_box` | 15.24 µs | 62,057 | 13 | 90,106 | -| `alloc_str_arc` | 15.33 µs | 62,057 | 13 | 90,106 | -| `bumpalo_alloc_str` | 6.98 µs | 54,048 | 13 | 81,088 | +| `alloc_str` | 8.24 µs | 51,053 | 10 | 76,098 | +| `alloc_str_box` | 11.83 µs | 59,053 | 11 | 85,098 | +| `alloc_str_arc` | 11.89 µs | 59,053 | 11 | 85,098 | +| `bumpalo_alloc_str` | 9.13 µs | 50,048 | 13 | 75,088 | ## `alloc_slice` | Variant | Time (criterion) | Instructions | Branch misses | Mem accesses | |---|---:|---:|---:|---:| -| `alloc_slice_copy` | 41.44 µs | 20,057 | 10 | 32,102 | -| `alloc_slice_clone` | 42.13 µs | 54,054 | 12 | 69,100 | -| `alloc_slice_fill_with` | 44.35 µs | 36,026 | 9 | 65,043 | -| `alloc_slice_fill_iter` | 44.34 µs | 38,029 | 11 | 68,048 | -| `alloc_slice_copy_box` | 51.55 µs | 33,624 | 29 | 55,883 | -| `alloc_slice_clone_box` | 52.56 µs | 78,624 | 36 | 104,883 | -| `alloc_slice_fill_with_box` | 52.95 µs | 51,564 | 33 | 91,778 | -| `alloc_slice_fill_iter_box` | 53.24 µs | 53,564 | 31 | 95,778 | -| `alloc_uninit_slice_box` | 50.50 µs | 25,564 | 31 | 40,778 | -| `alloc_zeroed_slice_box` | 50.73 µs | 30,564 | 32 | 48,778 | -| `alloc_slice_copy_arc` | 51.13 µs | 32,624 | 30 | 53,883 | -| `alloc_slice_clone_arc` | 51.97 µs | 67,624 | 32 | 90,883 | -| `alloc_slice_fill_with_arc` | 53.57 µs | 49,564 | 31 | 87,778 | -| `alloc_slice_fill_iter_arc` | 53.28 µs | 50,564 | 30 | 89,778 | -| `alloc_uninit_slice_arc` | 51.79 µs | 24,564 | 31 | 38,778 | -| `alloc_zeroed_slice_arc` | 50.73 µs | 28,564 | 30 | 44,778 | -| `bumpalo_alloc_slice_copy` | 41.49 µs | 42,042 | 4 | 61,076 | -| `bumpalo_alloc_slice_clone` | 41.31 µs | 63,046 | 9 | 79,083 | -| `bumpalo_alloc_slice_fill_with` | 43.33 µs | 42,023 | 5 | 74,038 | -| `bumpalo_alloc_slice_fill_iter` | 43.38 µs | 42,023 | 5 | 74,038 | +| `alloc_slice_copy` | 22.82 µs | 41,049 | 4 | 57,090 | +| `alloc_slice_clone` | 22.50 µs | 45,050 | 10 | 58,091 | +| `alloc_slice_fill_with` | 24.07 µs | 38,026 | 11 | 68,043 | +| `alloc_slice_fill_iter` | 24.18 µs | 38,027 | 11 | 68,044 | +| `alloc_slice_copy_box` | 41.99 µs | 55,646 | 33 | 83,916 | +| `alloc_slice_clone_box` | 42.18 µs | 68,646 | 40 | 92,915 | +| `alloc_slice_fill_with_box` | 43.59 µs | 48,585 | 40 | 86,809 | +| `alloc_slice_fill_iter_box` | 43.84 µs | 50,585 | 39 | 90,809 | +| `alloc_uninit_slice_box` | 39.83 µs | 23,585 | 40 | 36,809 | +| `alloc_zeroed_slice_box` | 40.73 µs | 27,585 | 40 | 43,809 | +| `alloc_slice_copy_arc` | 42.49 µs | 53,647 | 34 | 80,917 | +| `alloc_slice_clone_arc` | 42.37 µs | 59,645 | 39 | 80,914 | +| `alloc_slice_fill_with_arc` | 44.47 µs | 46,585 | 41 | 82,809 | +| `alloc_slice_fill_iter_arc` | 43.77 µs | 47,585 | 40 | 84,809 | +| `alloc_uninit_slice_arc` | 40.01 µs | 22,585 | 40 | 34,809 | +| `alloc_zeroed_slice_arc` | 41.27 µs | 25,585 | 40 | 39,809 | +| `bumpalo_alloc_slice_copy` | 23.49 µs | 38,042 | 4 | 55,076 | +| `bumpalo_alloc_slice_clone` | 24.38 µs | 60,046 | 9 | 74,083 | +| `bumpalo_alloc_slice_fill_with` | 25.44 µs | 40,020 | 5 | 70,033 | +| `bumpalo_alloc_slice_fill_iter` | 25.43 µs | 40,020 | 5 | 70,033 | ## `string_builder` | Variant | Time (criterion) | Instructions | Branch misses | Mem accesses | |---|---:|---:|---:|---:| -| `alloc_string` | 9.58 µs | 42,499 | 45 | 59,285 | -| `alloc_string_with_capacity` | 9.68 µs | 41,915 | 26 | 58,487 | -| `bumpalo_string_new_in` | 9.28 µs | 35,843 | 76 | 50,867 | -| `bumpalo_string_with_capacity_in` | 8.90 µs | 34,708 | 27 | 49,159 | +| `alloc_string` | 8.05 µs | 36,836 | 32 | 51,184 | +| `alloc_string_with_capacity` | 7.64 µs | 37,194 | 21 | 52,304 | +| `bumpalo_string_new_in` | 9.20 µs | 35,843 | 76 | 50,867 | +| `bumpalo_string_with_capacity_in` | 10.62 µs | 34,708 | 28 | 49,159 | ## `vec_builder` | Variant | Time (criterion) | Instructions | Branch misses | Mem accesses | |---|---:|---:|---:|---:| -| `alloc_vec` | 1.23 µs | 12,209 | 45 | 17,775 | -| `alloc_vec_with_capacity` | 1.22 µs | 12,577 | 26 | 18,939 | -| `bumpalo_vec_new_in` | 3.98 µs | 12,281 | 59 | 18,888 | -| `bumpalo_vec_with_capacity_in` | 1.03 µs | 11,069 | 7 | 17,116 | +| `alloc_vec` | 1.25 µs | 11,765 | 31 | 17,053 | +| `alloc_vec_with_capacity` | 1.23 µs | 12,132 | 8 | 18,215 | +| `bumpalo_vec_new_in` | 3.72 µs | 12,281 | 61 | 18,888 | +| `bumpalo_vec_with_capacity_in` | 3.48 µs | 11,069 | 2 | 17,116 | ## `drop` | Variant | Time (criterion) | Instructions | Branch misses | Mem accesses | |---|---:|---:|---:|---:| -| `box_u64` | 7.76 µs | 10,746 | 70 | 14,599 | -| `rc_u64` | 7.78 µs | 10,746 | 70 | 14,599 | -| `arc_u64` | 7.75 µs | 10,746 | 70 | 14,599 | -| `box_droppy` | 15.01 µs | 186,587 | 86 | 273,293 | -| `rc_droppy` | 27.14 µs | 218,987 | 92 | 320,869 | -| `arc_droppy` | 27.05 µs | 218,987 | 92 | 320,869 | -| `str_box` | 7.45 µs | 10,746 | 71 | 14,599 | -| `str_rc` | 7.42 µs | 10,746 | 71 | 14,599 | -| `str_arc` | 7.54 µs | 10,746 | 71 | 14,599 | -| `slice_box_u64` | 14.36 µs | 11,520 | 74 | 15,742 | -| `slice_rc_u64` | 14.55 µs | 11,520 | 73 | 15,742 | -| `slice_arc_u64` | 14.42 µs | 11,520 | 73 | 15,742 | -| `slice_box_droppy` | 125.90 µs | 1,480,329 | 1,120 | 2,162,947 | -| `slice_rc_droppy` | 131.59 µs | 1,505,378 | 1,122 | 2,201,008 | -| `slice_arc_droppy` | 129.98 µs | 1,505,378 | 1,122 | 2,201,008 | -| `alloc` | 976 ns | 410 | 16 | 617 | +| `box_u64` | 8.42 µs | 10,309 | 55 | 13,904 | +| `rc_u64` | 8.18 µs | 10,309 | 55 | 13,904 | +| `arc_u64` | 8.36 µs | 10,309 | 55 | 13,904 | +| `box_droppy` | 22.06 µs | 186,161 | 77 | 272,621 | +| `rc_droppy` | 27.37 µs | 219,386 | 80 | 320,930 | +| `arc_droppy` | 27.25 µs | 219,386 | 80 | 320,930 | +| `str_box` | 7.59 µs | 10,309 | 55 | 13,904 | +| `str_rc` | 7.68 µs | 10,309 | 55 | 13,904 | +| `str_arc` | 7.71 µs | 10,309 | 55 | 13,904 | +| `slice_box_u64` | 13.96 µs | 10,819 | 58 | 14,639 | +| `slice_rc_u64` | 12.30 µs | 10,819 | 58 | 14,639 | +| `slice_arc_u64` | 12.59 µs | 10,819 | 58 | 14,639 | +| `slice_box_droppy` | 115.72 µs | 1,520,210 | 1,848 | 2,214,775 | +| `slice_rc_droppy` | 122.93 µs | 1,546,283 | 1,110 | 2,253,860 | +| `slice_arc_droppy` | 122.05 µs | 1,546,283 | 1,110 | 2,253,860 | +| `alloc` | 686 ns | 337 | 15 | 504 | ## Multitude vs Bumpalo Head-to-Head @@ -112,14 +112,14 @@ Direct comparisons of multitude versus bumpalo on identical workloads (the multi | Workload | Multitude time | Bumpalo time | Δ time | Multitude instr | Bumpalo instr | Δ instr | |---|---:|---:|---:|---:|---:|---:| -| `alloc_u64/alloc` vs `bumpalo_alloc` | 4.02 µs | 4.02 µs | +0.1% | 16,032 | 21,025 | -23.7% | -| `alloc_str/alloc_str` vs `bumpalo_alloc_str` | 7.20 µs | 6.98 µs | +3.1% | 52,056 | 54,048 | -3.7% | -| `alloc_slice/alloc_slice_copy` vs `bumpalo_alloc_slice_copy` | 41.44 µs | 41.49 µs | -0.1% | 20,057 | 42,042 | -52.3% | -| `alloc_slice/alloc_slice_clone` vs `bumpalo_alloc_slice_clone` | 42.13 µs | 41.31 µs | +2.0% | 54,054 | 63,046 | -14.3% | -| `alloc_slice/alloc_slice_fill_with` vs `bumpalo_alloc_slice_fill_with` | 44.35 µs | 43.33 µs | +2.4% | 36,026 | 42,023 | -14.3% | -| `alloc_slice/alloc_slice_fill_iter` vs `bumpalo_alloc_slice_fill_iter` | 44.34 µs | 43.38 µs | +2.2% | 38,029 | 42,023 | -9.5% | -| `string_builder/alloc_string` vs `bumpalo_string_new_in` | 9.58 µs | 9.28 µs | +3.3% | 42,499 | 35,843 | +18.6% | -| `string_builder/alloc_string_with_capacity` vs `bumpalo_string_with_capacity_in` | 9.68 µs | 8.90 µs | +8.8% | 41,915 | 34,708 | +20.8% | -| `vec_builder/alloc_vec` vs `bumpalo_vec_new_in` | 1.23 µs | 3.98 µs | -69.2% | 12,209 | 12,281 | -0.6% | -| `vec_builder/alloc_vec_with_capacity` vs `bumpalo_vec_with_capacity_in` | 1.22 µs | 1.03 µs | +17.6% | 12,577 | 11,069 | +13.6% | +| `alloc` vs `bumpalo_alloc` | 6.45 µs | 5.97 µs | +8.1% | 14,026 | 19,022 | -26.3% | +| `alloc_str` vs `bumpalo_alloc_str` | 8.24 µs | 9.13 µs | -9.7% | 51,053 | 50,048 | +2.0% | +| `alloc_slice_copy` vs `bumpalo_alloc_slice_copy` | 22.82 µs | 23.49 µs | -2.9% | 41,049 | 38,042 | +7.9% | +| `alloc_slice_clone` vs `bumpalo_alloc_slice_clone` | 22.50 µs | 24.38 µs | -7.7% | 45,050 | 60,046 | -25.0% | +| `alloc_slice_fill_with` vs `bumpalo_alloc_slice_fill_with` | 24.07 µs | 25.44 µs | -5.4% | 38,026 | 40,020 | -5.0% | +| `alloc_slice_fill_iter` vs `bumpalo_alloc_slice_fill_iter` | 24.18 µs | 25.43 µs | -4.9% | 38,027 | 40,020 | -5.0% | +| `alloc_string` vs `bumpalo_string_new_in` | 8.05 µs | 9.20 µs | -12.5% | 36,836 | 35,843 | +2.8% | +| `alloc_string_with_capacity` vs `bumpalo_string_with_capacity_in` | 7.64 µs | 10.62 µs | -28.0% | 37,194 | 34,708 | +7.2% | +| `alloc_vec` vs `bumpalo_vec_new_in` | 1.25 µs | 3.72 µs | -66.3% | 11,765 | 12,281 | -4.2% | +| `alloc_vec_with_capacity` vs `bumpalo_vec_with_capacity_in` | 1.23 µs | 3.48 µs | -64.6% | 12,132 | 11,069 | +9.6% | diff --git a/crates/multitude/docs/TODO.md b/crates/multitude/docs/TODO.md index 39edc84c0..6c5a16c70 100644 --- a/crates/multitude/docs/TODO.md +++ b/crates/multitude/docs/TODO.md @@ -1,170 +1,81 @@ # TODO -- Eliminate vec of chunk mutators. This should be a linked list - -- When expanding a vec that's in an oversized chunk, we should return the -previous chunk back to the system allocator instead of holding onto it in the -retired chunk list. - -- **Remove the `capacity` field from `LocalChunk` / `SharedChunk` headers** - (saves 8 bytes per chunk header). The field is redundant with the - slice-tail length metadata of the DST `data: [UnsafeCell]`, except on - the cache-pop path: chunks sit on the provider freelist linked via a - thin `*mut u8` and `header_to_fat` (`src/internal/local_chunk.rs:185`, - matching code in `src/internal/shared_chunk.rs`) currently reads the - stored `capacity` to reconstruct the slice length. The provider already - pins each freelist to a single class (`local_cache_class` / - `shared_cache_class` in `src/internal/chunk_provider.rs`), so the size is - known by context. Thread `SizeClass` through the pop path, reconstruct - `cap = class.bytes() - header_size()` in `header_to_fat`, do the same - for `SharedChunk`, and update `destroy` callers (which need the layout) - to take the class too. Verify with the gungraun benchmarks that the - header shrink doesn't move alignment / payload-offset and that the - size-class plumbing is not on a hot path. - -- **Batched shared-chunk refcount increments on Arc/Box allocation.** Each - `Arc`/`Box` allocation currently performs an atomic `fetch_add(1, Relaxed)` - on the owning `SharedChunk`'s `ref_count` (see `acquire_shared_chunk_ref` in - `src/arena/alloc_value.rs` calling `SharedChunk::inc_ref` in - `src/internal/shared_chunk.rs`). The previous generation of the crate - avoided this by accumulating refcount increments locally in the active - shared mutator and flushing them to the atomic counter only at - chunk-transition / mutator-drop time. Bring this back: add a - `pending_refs: usize` counter to the shared `ChunkMutator`; have - `acquire_shared_chunk_ref` bump it instead of calling `inc_ref` per - allocation; flush via a single `fetch_add(pending_refs, Relaxed)` when the - mutator is uninstalled or dropped. Teardown / `dec_ref` must observe that - the chunk's effective refcount is `atomic_ref_count + pending_refs` while - the mutator is installed, and the overflow guard must still trip when the - combined value would saturate. - -## Unsafe-block-reduction opportunities (analysis 2026-06-06) - -### Medium confidence (sound, but verify ordering/lifetimes) - -- **Hoist `Arc::from_raw` / `Box::from_raw` out of the alloc retry loops.** The - shared-slice and uninit retry loops construct the smart pointer separately in - each exit arm (current-chunk fast path, oversized closure, post-refill): - `src/arena/alloc_slice_arc.rs:170,183,217,228,253`, - `src/arena/alloc_value.rs:603,625,662,675` (uninit arc/slice-arc), and - `src/arena/alloc_unsized.rs:237,251` (dst box). Refactor each loop to `break` - with the raw `NonNull` payload pointer and perform a single - `Arc::from_raw`/`Box::from_raw` after the loop. CAVEAT: the `ChunkRef::forget()` - that retains the fresh `+1` refcount, and any `publish_drop_count()` / - stats recording, must still happen *before* the `break` in every arm so the - adopted reference survives to the single construction point. - Estimated: **net -6 blocks** across the three files. - -- **Centralize the "initialized `NonNull` -> `&'a mut`" reborrows in - `uninit.rs`.** `src/internal/uninit.rs:89,146,200,249,339,450` repeat - `unsafe { ptr.as_mut() }` / `unsafe { &mut *ptr.as_ptr() }` at the end of the - init paths. Add one private helper `fn initialized_mut<'a, T: ?Sized>(ptr: - NonNull) -> &'a mut T` that holds the single `unsafe`. CAVEAT: the helper - forges the `'a` lifetime, so it must stay private to `uninit.rs` and only be - called after the ticket has been consumed and the value fully initialized - (same precondition the call sites already satisfy). - Estimated: **net -5 blocks**. - -- **Encapsulate the `DropEntry::placeholder` raw writes in `chunk_mutator.rs`.** - `src/internal/chunk_mutator.rs:352-354,376-378,424-426,517-519` all do - `unsafe { core::ptr::write(drop_slot.as_ptr(), DropEntry::placeholder(...)) }`. - Add a private `fn write_drop_placeholder(drop_slot, value_offset, len)` - holding the single `unsafe`. CAVEAT: this is a *safe* fn wrapping an unchecked - write, sound only because every caller passes a freshly reserved, aligned, - exclusively-owned slot from `try_reserve_drop_entry`; keep it private and - document that invariant on the helper. - Estimated: **net -3 blocks**. - -- **Drop `chunk_ptr_unchecked` (`unwrap_unchecked`) in favor of an early - `self.chunk?`.** `src/internal/chunk_mutator.rs:155-157` plus call sites `229`, - `252` rely on the sentinel "empty mutator" proof to justify - `unsafe { self.chunk.unwrap_unchecked() }`. Take `let chunk = self.chunk?;` - up front; the subsequent `try_alloc*` already returns `None` for the empty - mutator, so behavior is preserved without the unchecked unwrap. CAVEAT: confirm - the `None`-propagation matches the sentinel behavior exactly for the empty - mutator before removing the helper. - Estimated: **net -3 blocks**. - -### Lower value - -- **`PrefixedUtf16Ptr` newtype** wrapping the length-prefixed `NonNull` in - `src/strings/arc_utf16_str.rs:66,76` and `src/strings/box_utf16_str.rs:61,70,98`, - with `len()` / `as_utf16_str()` / `as_mut_utf16_str()` methods holding the - `read_prefix_len` + `from_raw_parts` + `from_slice_unchecked` unsafe. Sound only - if constructed exclusively via the existing unsafe `from_raw` paths. - Estimated: **net -2 blocks**. - -### Deferred (perf-risk) - -- **Consolidate try-current/oversized/refill loops.** `impl_alloc_local_with`, - `impl_alloc_smart_with`, the slice-Arc copy/fill loops, prefixed shared - loop, UTF-16 transcoding loop, and DST Box/Arc smart loops repeat the - same "try current reservation; route oversized; refill and retry" shape. - Each is `#[inline(always)]` on the allocation fast path; structural - differences (local vs shared, with-drop vs without, ZST/uninit/zeroed - branches, stats recording, slot-init helpers, different smart-pointer - constructions) make a single macro/closure abstraction either fragile - (closure-state capture risks codegen drift) or unwieldy (a macro with - many positional knobs hurts readability without saving meaningful - unsafe). Deferred to keep gungraun instruction counts stable. See - simplification-report item 1.2. - -## Simplification opportunities (analysis 2026-06-08) - -### High-confidence wins (mechanical, no risk) - -- **Dedup the "prefixed slice" arithmetic in `chunk_mutator.rs`.** - `try_alloc_uninit_slice_prefixed` (`src/internal/chunk_mutator.rs:290-320`) - and `try_alloc_uninit_slice_with_drop_prefixed` - (`src/internal/chunk_mutator.rs:381-417`) compute the same - `prefix_size / payload_offset / payload_bytes / total` and run the same - unsafe block writing the prefix word and projecting the payload - `NonNull`. Extract `fn try_alloc_prefixed_payload(&self, len: usize) - -> Option<(InChunk<…>, /*payload_addr*/ usize)>` that owns the layout - math + prefix write; the two callers add only the drop-entry plumbing. - Touches one file, ~25 lines. - -- **Make `ChunkMutator::from_owned` reuse the payload-range math.** - `from_owned` (`src/internal/chunk_mutator.rs:~65-85`) re-derives - `start_addr / aligned_end_addr / aligned_end_offset` from `payload_ptr` - / `capacity`; `payload_range` - (`src/internal/chunk_mutator.rs:122-133`) already encapsulates exactly - that calculation. Add a `payload_range_for(chunk)` taking a - `NonNull` (or set `self.chunk` first and call `payload_range()`). - Touches one file, ~10 lines. - -### Medium-confidence (worth doing, slight refactor) - -- **Unify `release_local` / `release_shared` cache-bypass branches.** - `src/internal/chunk_provider.rs:466-487` vs `494-505`. Same - structure: read total → if uncacheable or below floor, destroy + - release_bytes → else push to cache. Differences: which floor atomic - (`Acquire` vs `Relaxed`), which `destroy`, single-threaded - `local_cache.with` vs `push_shared`. Extract - `fn should_bypass_cache(&self, total: usize, floor: &AtomicU8, ord: - Ordering) -> bool` for at least the decision. The `Ordering` - difference is real and intentional; pass it as a parameter. - -- **Unify `acquire_normal_local` / `acquire_normal_shared`.** - `src/internal/chunk_provider.rs:267-292` vs `377-393`. Same - "advance floor if needed, pop cache, else allocate fresh" shape. - Extract the floor-bump/pop control flow; pass flavor-specific pop / - reinit / allocate-fresh closures. Slight closure overhead — verify - codegen still inlines on the hot path. - -- **Dedup `allocate_fresh_local` / `allocate_fresh_shared`.** - `src/internal/chunk_provider.rs:336-351` vs `441-456`. Identical - reserve-bytes / allocate / release-on-error scaffolding. Introduce - `fn allocate_with_budget Result>(&self, - total: usize, build: F) -> Result` that handles the - budget rollback. ~20 lines net reduction. - -### Speculative (needs perf validation) - -- **Collapse the uninit-slice allocation family.** `try_alloc_bytes`, - `try_alloc_uninit_slice`, `try_alloc_uninit_slice_prefixed` in - `src/internal/chunk_mutator.rs:245-320` share "compute size, reserve, - convert ticket" with different ticket shapes. Could be split into a - low-level `reserve_bytes_for_slice` + thin wrappers. Risk: hottest - alloc paths; even minor codegen drift could move the gungraun - benchmark numbers. Do not land without before/after callgrind data. +## API cleanup + +- Review the public API surface of `Vec`, `String`, `Arc`, and `Box` against the + standard library and ensure parity with the corresponding `std` / + `alloc` types (method names, signatures, trait impls) wherever it makes sense + for an arena-backed allocator, noting and documenting any intentional + divergences. + +- Add the missing fallible freeze variants `try_into_box` and `try_into_arc` for + Vec/String/Utf16String + +- Assess whether it would be possible to support serde-based deserialization into an arena + +- Consider introducing arena-friendly hash map and hash set + +## Zero-copy `Vec`/`String` → `Arc<[T]>` / `Arc` + +### Problem + +`Vec` and `String` are backed by a `LocalChunk` (non-atomic refcount — +`LocalChunk::inc_ref` is `unreachable!`). Freezing them into `Arc<[T]>` / +`Arc` therefore copies the data into a `SharedChunk` and returns an `Arc` +pointing at the copy. The copy is mandatory today because: + +- `Arc` holds an **atomic** refcount on a `SharedChunk`, and +- a thin `Arc<[T]>` recovers its length from a `usize` prefix word and its chunk + header via the 64 KiB `CHUNK_BASE_MASK`. + +Bytes living in a local chunk satisfy neither, so the only way to avoid the copy +is to **build the buffer in a shared chunk from the start**. Customers building +data specifically to hand out as `Arc<[T]>` / `Arc` want to skip that copy. + +### Implementation crux (shared by all API options) + +A *shared* growable buffer needs two things a local one doesn't: + +1. **Hold a refcount on its backing shared chunk during growth.** Otherwise an + interleaved `alloc_arc` that triggers `refill_shared` rotates the chunk out; + with nothing holding it, the chunk can be torn down and the builder dangles. + So a shared builder carries a `ChunkRef` (a `+1`), re-acquired on each + relocation. (Note the interaction with the pre-credited surplus / + `local_shared_count` accounting in `arena/mod.rs`.) +2. **Reserve the `Arc` length-prefix slot** up front so freeze is a pointer + fix-up rather than a copy. + +With those in place, `into_arc` becomes **O(1)**: write the final length into the +prefix slot, then `mem::forget` the builder's `ChunkRef` to transfer its `+1` to +the new `Arc`. Once this machinery exists, the API-shape choice below is +secondary and cheap to swap. + +### API options + +- **A — separate `SharedVec` / `SharedString` types.** Clearest intent, + type-safe freeze, no runtime branch. Source duplication can be contained with + a `impl_arena_vec_common!` macro emitting both flavors from one body (mirrors + the existing `impl_arena_string_common!`). +- **B — one type, flavor chosen at construction via a runtime flag** + (`alloc_vec` vs `alloc_shared_vec`). Smallest API, but a branch in the + growth/freeze paths and no compile-time signal that `into_arc` is free vs. a + copy. +- **C — one type with a zero-cost marker type parameter (recommended).** + `Vec<'a, T, A = Global, F = Local>` with a sealed `ChunkFlavor` trait + abstracting reserve / refill / oversized / ref-holding. `Local` = today's + behavior (default, so existing code is unchanged); `Shared` holds the + `ChunkRef` + prefix slot. Expose `SharedVec` / `SharedString` as type aliases. + - common ops (`push`, `extend`, `len`, …) in `impl` — one body; + - `into_arc` is O(1) only on `F = Shared`, still available (O(n) copy) on `Local`; + - `into_slice` (arena-lifetime, no copy) stays `Local`-only. + Zero runtime branch (monomorphized), type-safe freeze, single impl. Cost: + generic noise in signatures, mitigated by the `F = Local` default and aliases. + +`String` wraps `Vec`, so flavor support added to `Vec` extends to `String` +(and `Utf16String`) for free. + +### Recommendation + +Option C for zero-cost + compile-time freeze guarantees; fall back to A-via-macro +if the generic parameter is too noisy. All options require the same underlying +"shared growable buffer holds a `ChunkRef` + prefix slot" work — start there. diff --git a/crates/multitude/examples/multitude_basic.rs b/crates/multitude/examples/multitude_basic.rs index df0ab3b76..0c078c8ba 100644 --- a/crates/multitude/examples/multitude_basic.rs +++ b/crates/multitude/examples/multitude_basic.rs @@ -44,7 +44,7 @@ fn main() { v.push(i * 100); } println!("v = {v:?}"); - let frozen: Arc<[u64], _> = v.into_arena_arc(); + let frozen: Arc<[u64], _> = Arc::from(v); println!("frozen = {frozen:?}"); // -- Collect from an iterator ------------------------------------- diff --git a/crates/multitude/examples/strings.rs b/crates/multitude/examples/strings.rs index 356bc9d7f..f1419b84d 100644 --- a/crates/multitude/examples/strings.rs +++ b/crates/multitude/examples/strings.rs @@ -20,7 +20,7 @@ fn main() { builder.push_str("up "); builder.push_str("incrementally"); println!("ArenaString (mutable): {}", builder.as_str()); - let s2: Box = builder.into_arena_box_str(); + let s2: Box = builder.into_boxed_str(); println!("frozen: {}", &*s2); let name = "Alice"; diff --git a/crates/multitude/scripts/perf_report.rs b/crates/multitude/scripts/perf_report.rs index 99e3fbd1c..fab6425a8 100644 --- a/crates/multitude/scripts/perf_report.rs +++ b/crates/multitude/scripts/perf_report.rs @@ -85,8 +85,8 @@ const GROUPS: &[Group] = &[ ( "arena_creation", &[ - ("multitude", Some("arena_creation_multitude")), - ("bumpalo_new", Some("arena_creation_bumpalo_new")), + ("multitude_new", Some("multitude_new")), + ("bumpalo_new", Some("bumpalo_new")), ], ), ( @@ -102,8 +102,8 @@ const GROUPS: &[Group] = &[ ("alloc_arc_with", Some("alloc_arc_with")), ("alloc_uninit_arc", Some("alloc_uninit_arc")), ("alloc_zeroed_arc", Some("alloc_zeroed_arc")), - ("bumpalo_alloc", Some("alloc_u64_bumpalo_alloc")), - ("bumpalo_alloc_with", Some("alloc_u64_bumpalo_alloc_with")), + ("bumpalo_alloc", Some("bumpalo_alloc")), + ("bumpalo_alloc_with", Some("bumpalo_alloc_with")), ], ), ( @@ -112,7 +112,7 @@ const GROUPS: &[Group] = &[ ("alloc_str", Some("alloc_str")), ("alloc_str_box", Some("alloc_str_box")), ("alloc_str_arc", Some("alloc_str_arc")), - ("bumpalo_alloc_str", Some("alloc_str_bumpalo_alloc_str")), + ("bumpalo_alloc_str", Some("bumpalo_alloc_str")), ], ), ( @@ -134,10 +134,10 @@ const GROUPS: &[Group] = &[ ("alloc_slice_fill_iter_arc", Some("alloc_slice_fill_iter_arc")), ("alloc_uninit_slice_arc", Some("alloc_uninit_slice_arc")), ("alloc_zeroed_slice_arc", Some("alloc_zeroed_slice_arc")), - ("bumpalo_alloc_slice_copy", Some("alloc_slice_bumpalo_alloc_slice_copy")), - ("bumpalo_alloc_slice_clone", Some("alloc_slice_bumpalo_alloc_slice_clone")), - ("bumpalo_alloc_slice_fill_with", Some("alloc_slice_bumpalo_alloc_slice_fill_with")), - ("bumpalo_alloc_slice_fill_iter", Some("alloc_slice_bumpalo_alloc_slice_fill_iter")), + ("bumpalo_alloc_slice_copy", Some("bumpalo_alloc_slice_copy")), + ("bumpalo_alloc_slice_clone", Some("bumpalo_alloc_slice_clone")), + ("bumpalo_alloc_slice_fill_with", Some("bumpalo_alloc_slice_fill_with")), + ("bumpalo_alloc_slice_fill_iter", Some("bumpalo_alloc_slice_fill_iter")), ], ), ( @@ -145,8 +145,8 @@ const GROUPS: &[Group] = &[ &[ ("alloc_string", Some("alloc_string")), ("alloc_string_with_capacity", Some("alloc_string_with_capacity")), - ("bumpalo_string_new_in", Some("string_builder_bumpalo_string_new_in")), - ("bumpalo_string_with_capacity_in", Some("string_builder_bumpalo_string_with_capacity_in")), + ("bumpalo_string_new_in", Some("bumpalo_string_new_in")), + ("bumpalo_string_with_capacity_in", Some("bumpalo_string_with_capacity_in")), ], ), ( @@ -154,29 +154,29 @@ const GROUPS: &[Group] = &[ &[ ("alloc_vec", Some("alloc_vec")), ("alloc_vec_with_capacity", Some("alloc_vec_with_capacity")), - ("bumpalo_vec_new_in", Some("vec_builder_bumpalo_vec_new_in")), - ("bumpalo_vec_with_capacity_in", Some("vec_builder_bumpalo_vec_with_capacity_in")), + ("bumpalo_vec_new_in", Some("bumpalo_vec_new_in")), + ("bumpalo_vec_with_capacity_in", Some("bumpalo_vec_with_capacity_in")), ], ), ( "drop", &[ - ("box_u64", Some("drop_box_u64")), - ("rc_u64", Some("drop_rc_u64")), - ("arc_u64", Some("drop_arc_u64")), - ("box_droppy", Some("drop_box_droppy")), - ("rc_droppy", Some("drop_rc_droppy")), - ("arc_droppy", Some("drop_arc_droppy")), - ("str_box", Some("drop_str_box")), - ("str_rc", Some("drop_str_rc")), - ("str_arc", Some("drop_str_arc")), - ("slice_box_u64", Some("drop_slice_box_u64")), - ("slice_rc_u64", Some("drop_slice_rc_u64")), - ("slice_arc_u64", Some("drop_slice_arc_u64")), - ("slice_box_droppy", Some("drop_slice_box_droppy")), - ("slice_rc_droppy", Some("drop_slice_rc_droppy")), - ("slice_arc_droppy", Some("drop_slice_arc_droppy")), - ("alloc", Some("drop_alloc")), + ("box_u64", Some("box_u64")), + ("rc_u64", Some("rc_u64")), + ("arc_u64", Some("arc_u64")), + ("box_droppy", Some("box_droppy")), + ("rc_droppy", Some("rc_droppy")), + ("arc_droppy", Some("arc_droppy")), + ("str_box", Some("str_box")), + ("str_rc", Some("str_rc")), + ("str_arc", Some("str_arc")), + ("slice_box_u64", Some("slice_box_u64")), + ("slice_rc_u64", Some("slice_rc_u64")), + ("slice_arc_u64", Some("slice_arc_u64")), + ("slice_box_droppy", Some("slice_box_droppy")), + ("slice_rc_droppy", Some("slice_rc_droppy")), + ("slice_arc_droppy", Some("slice_arc_droppy")), + ("alloc", Some("alloc")), ], ), ]; @@ -494,7 +494,7 @@ fn build_report(crit: &[(String, f64)], g_alloc: &[GungEntry], g_drop: &[GungEnt }; let _ = writeln!( out, - "| `{group}/{mvar}` vs `{bvar}` | {} | {} | {} | {} | {} | {} |", + "| `{mvar}` vs `{bvar}` | {} | {} | {} | {} | {} | {} |", fmt_ns(mt), fmt_ns(bt), dt, diff --git a/crates/multitude/src/allocator_impl.rs b/crates/multitude/src/allocator_impl.rs index ac57a43b2..b2ef39ddd 100644 --- a/crates/multitude/src/allocator_impl.rs +++ b/crates/multitude/src/allocator_impl.rs @@ -58,7 +58,7 @@ unsafe impl Allocator for &Arena { let refill_hint = layout.size().saturating_add(layout.align()); loop { if let Some((slot, chunk_ptr)) = self.current_shared().try_alloc_with_chunk(layout.size(), layout.align()) { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let ptr = slot.as_non_null(); let _ = chunk_ref.forget(); return Ok(NonNull::slice_from_raw_parts(ptr, layout.size())); diff --git a/crates/multitude/src/arc.rs b/crates/multitude/src/arc.rs index 2e3d0f08e..087d459f1 100644 --- a/crates/multitude/src/arc.rs +++ b/crates/multitude/src/arc.rs @@ -344,9 +344,9 @@ where A: Send + Sync, { /// Freeze a [`Vec`](crate::vec::Vec) into an immutable - /// [`Arc<[T], A>`](crate::Arc). See [`Vec::into_arena_arc`](crate::vec::Vec::into_arena_arc). + /// [`Arc<[T], A>`](crate::Arc). Mirrors `std`'s `From> for Arc<[T]>`. #[inline] fn from(v: Vec<'a, T, A>) -> Self { - v.into_arena_arc() + v.freeze_into_arc() } } diff --git a/crates/multitude/src/arena/alloc_growable.rs b/crates/multitude/src/arena/alloc_growable.rs index b59429f9e..405605348 100644 --- a/crates/multitude/src/arena/alloc_growable.rs +++ b/crates/multitude/src/arena/alloc_growable.rs @@ -63,6 +63,192 @@ impl Arena { String::try_with_capacity_in(cap, self) } + /// Validate `bytes` as UTF-8 and copy them into a fresh arena + /// [`String`]. The arena-bound analog of + /// [`std::string::String::from_utf8`] (taking a borrowed slice rather + /// than an owning `Vec`). + /// + /// # Errors + /// + /// Returns a [`Utf8Error`](core::str::Utf8Error) if `bytes` is not valid + /// UTF-8. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. Allocation failure is reported + /// via a panic, not the returned `Result`. + pub fn alloc_string_from_utf8(&self, bytes: &[u8]) -> Result, core::str::Utf8Error> { + Ok(String::from_str_in(core::str::from_utf8(bytes)?, self)) + } + + /// Copy `bytes` into a fresh arena [`String`], replacing any invalid + /// UTF-8 sequences with `U+FFFD`. The arena-bound analog of + /// [`std::string::String::from_utf8_lossy`]. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. + #[must_use] + pub fn alloc_string_from_utf8_lossy(&self, bytes: &[u8]) -> String<'_, A> { + String::from_str_in(&alloc::string::String::from_utf8_lossy(bytes), self) + } + + /// Copy `bytes` into a fresh arena [`String`] without validating that + /// they are UTF-8. The arena-bound analog of + /// [`std::string::String::from_utf8_unchecked`]. + /// + /// # Safety + /// + /// `bytes` must be valid UTF-8. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. + #[must_use] + pub unsafe fn alloc_string_from_utf8_unchecked(&self, bytes: &[u8]) -> String<'_, A> { + // SAFETY: the caller guarantees `bytes` is valid UTF-8. + String::from_str_in(unsafe { core::str::from_utf8_unchecked(bytes) }, self) + } + + /// Decode native-endian UTF-16 `units` into a fresh arena [`String`]. + /// The arena-bound analog of [`std::string::String::from_utf16`]. + /// + /// # Errors + /// + /// Returns a [`DecodeUtf16Error`](core::char::DecodeUtf16Error) on the + /// first unpaired surrogate. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. Allocation failure is reported + /// via a panic, not the returned `Result`. + pub fn alloc_string_from_utf16(&self, units: &[u16]) -> Result, core::char::DecodeUtf16Error> { + let mut out = self.alloc_string_with_capacity(units.len()); + for unit in char::decode_utf16(units.iter().copied()) { + out.push(unit?); + } + Ok(out) + } + + /// Decode native-endian UTF-16 `units` into a fresh arena [`String`], + /// replacing unpaired surrogates with `U+FFFD`. The arena-bound analog + /// of [`std::string::String::from_utf16_lossy`]. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. + #[must_use] + pub fn alloc_string_from_utf16_lossy(&self, units: &[u16]) -> String<'_, A> { + let mut out = self.alloc_string_with_capacity(units.len()); + for unit in char::decode_utf16(units.iter().copied()) { + out.push(unit.unwrap_or(char::REPLACEMENT_CHARACTER)); + } + out + } + + /// Decode little-endian UTF-16 `bytes` into a fresh arena [`String`]. The + /// arena-bound analog of [`std::string::String::from_utf16le`]. + /// + /// # Errors + /// + /// Returns a [`FromUtf16Error`](crate::strings::FromUtf16Error) if `bytes` + /// has an odd length or contains an unpaired surrogate. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. Allocation failure is reported + /// via a panic, not the returned `Result`. + pub fn alloc_string_from_utf16le(&self, bytes: &[u8]) -> Result, crate::strings::FromUtf16Error> { + self.alloc_string_from_utf16_bytes(bytes, false) + } + + /// Decode big-endian UTF-16 `bytes` into a fresh arena [`String`]. The + /// arena-bound analog of [`std::string::String::from_utf16be`]. + /// + /// # Errors + /// + /// Returns a [`FromUtf16Error`](crate::strings::FromUtf16Error) if `bytes` + /// has an odd length or contains an unpaired surrogate. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. Allocation failure is reported + /// via a panic, not the returned `Result`. + pub fn alloc_string_from_utf16be(&self, bytes: &[u8]) -> Result, crate::strings::FromUtf16Error> { + self.alloc_string_from_utf16_bytes(bytes, true) + } + + /// Decode little-endian UTF-16 `bytes` into a fresh arena [`String`], + /// replacing odd trailing bytes and unpaired surrogates with `U+FFFD`. The + /// arena-bound analog of [`std::string::String::from_utf16le_lossy`]. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. + #[must_use] + pub fn alloc_string_from_utf16le_lossy(&self, bytes: &[u8]) -> String<'_, A> { + self.alloc_string_from_utf16_bytes_lossy(bytes, false) + } + + /// Decode big-endian UTF-16 `bytes` into a fresh arena [`String`], + /// replacing odd trailing bytes and unpaired surrogates with `U+FFFD`. The + /// arena-bound analog of [`std::string::String::from_utf16be_lossy`]. + /// + /// # Panics + /// + /// Panics if the backing allocator fails. + #[must_use] + pub fn alloc_string_from_utf16be_lossy(&self, bytes: &[u8]) -> String<'_, A> { + self.alloc_string_from_utf16_bytes_lossy(bytes, true) + } + + /// Shared body for the byte-oriented UTF-16 constructors. `big_endian` + /// selects the byte order used to assemble each `u16` code unit. + #[allow( + clippy::map_err_ignore, + reason = "FromUtf16Error is intentionally opaque; the DecodeUtf16Error carries no extra recoverable detail" + )] + fn alloc_string_from_utf16_bytes(&self, bytes: &[u8], big_endian: bool) -> Result, crate::strings::FromUtf16Error> { + if !bytes.len().is_multiple_of(2) { + return Err(crate::strings::FromUtf16Error::new()); + } + let mut out = self.alloc_string_with_capacity(bytes.len() / 2); + let units = bytes.chunks_exact(2).map(|pair| { + let raw = [pair[0], pair[1]]; + if big_endian { + u16::from_be_bytes(raw) + } else { + u16::from_le_bytes(raw) + } + }); + for unit in char::decode_utf16(units) { + out.push(unit.map_err(|_| crate::strings::FromUtf16Error::new())?); + } + Ok(out) + } + + /// Shared body for the lossy byte-oriented UTF-16 constructors. + fn alloc_string_from_utf16_bytes_lossy(&self, bytes: &[u8], big_endian: bool) -> String<'_, A> { + let mut out = self.alloc_string_with_capacity(bytes.len() / 2 + 1); + let units = bytes.chunks_exact(2).map(|pair| { + let raw = [pair[0], pair[1]]; + if big_endian { + u16::from_be_bytes(raw) + } else { + u16::from_le_bytes(raw) + } + }); + for unit in char::decode_utf16(units) { + out.push(unit.unwrap_or(char::REPLACEMENT_CHARACTER)); + } + if !bytes.len().is_multiple_of(2) { + // An odd trailing byte is invalid; mirror std by appending one + // replacement character. + out.push(char::REPLACEMENT_CHARACTER); + } + out + } + /// Create a new, empty growable [`Vec`](crate::vec::Vec) backed by this arena. /// No allocation is performed until the first push. /// diff --git a/crates/multitude/src/arena/alloc_prefixed.rs b/crates/multitude/src/arena/alloc_prefixed.rs index 18d738761..ee29c271f 100644 --- a/crates/multitude/src/arena/alloc_prefixed.rs +++ b/crates/multitude/src/arena/alloc_prefixed.rs @@ -24,6 +24,7 @@ use allocator_api2::alloc::{AllocError, Allocator}; use super::Arena; use super::alloc_value::acquire_shared_chunk_ref; use crate::internal::chunk_ref::ChunkRef; +use crate::internal::drop_entry::DropEntry; /// Byte size of the inline element-count prefix written immediately /// before every prefixed-shared payload. @@ -49,8 +50,8 @@ pub(crate) fn worst_case_thin_slice_payload(len: usize) -> usize { // worst-case align-up at the front of the reservation). .saturating_add(elem_align); if mem::needs_drop::() { - base.saturating_add(mem::size_of::()) - .saturating_add(mem::align_of::()) + base.saturating_add(mem::size_of::()) + .saturating_add(mem::align_of::()) } else { base } @@ -87,12 +88,10 @@ impl Arena { // payload (at offset PREFIX_BYTES, a multiple of any align // ≤ usize's) ends up naturally aligned for `T` reads/writes. if let Some((uninit, chunk_ptr)) = self.current_shared().try_alloc_with_chunk(total, elem_align) { - let chunk_ref: ChunkRef = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref: ChunkRef = self.acquire_current_shared_chunk_ref(chunk_ptr); let payload = write_prefixed_payload::(uninit.as_non_null(), src); // Hand the +1 over to the caller's smart pointer. let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(len.saturating_mul(elem_size)); return Ok(payload); } if self.is_oversized_shared(total) { @@ -103,8 +102,6 @@ impl Arena { let chunk_ref: ChunkRef = acquire_shared_chunk_ref::(chunk_ptr); let payload = write_prefixed_payload::(base.as_non_null(), src); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(len.saturating_mul(elem_size)); payload }); } diff --git a/crates/multitude/src/arena/alloc_slice_arc.rs b/crates/multitude/src/arena/alloc_slice_arc.rs index 5e4e10752..c0a2b6b0c 100644 --- a/crates/multitude/src/arena/alloc_slice_arc.rs +++ b/crates/multitude/src/arena/alloc_slice_arc.rs @@ -155,16 +155,18 @@ impl Arena { check_slice_arc_layout::(src.len())?; let len = src.len(); // Copy is never `Drop`, so use the no-drop reservation. - #[cfg(feature = "stats")] - let payload_bytes = mem::size_of::().saturating_mul(len); let bytes_needed = worst_case_thin_slice_payload::(len); + // `src` is a live `&[T]`, so `size_of_val(src)` is a valid + // `usize`. Hoisting the precomputed byte size lets the inner + // reservation helper skip the `checked_mul` overflow guard. + let payload_bytes = mem::size_of_val(src); loop { - if let Some((uninit, chunk_ptr)) = self.try_reserve_shared_slice::(len) { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + // SAFETY: `payload_bytes == size_of_val(src) == size_of::() * len`. + let reserved = unsafe { self.try_reserve_shared_slice_with_size::(len, payload_bytes) }; + if let Some((uninit, chunk_ptr)) = reserved { + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let slice_ptr = uninit.init_copy_from_slice_ptr(src); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(payload_bytes); // SAFETY: `slice_ptr` points to `len` initialized `T`s in a // shared chunk with a fresh +1; `Arc::from_raw` adopts that // +1. Chunk-wide provenance preserved via `init_copy_from_slice_ptr`. @@ -178,8 +180,6 @@ impl Arena { let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); let slice_ptr = ticket.init_copy_from_slice_ptr(src); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(payload_bytes); // SAFETY: see the non-oversized branch. unsafe { Arc::from_raw(slice_ptr.cast::()) } }); @@ -194,8 +194,6 @@ impl Arena { #[inline] fn impl_alloc_slice_arc_with T>(&self, len: usize, f: F) -> Result, AllocError> { check_slice_arc_layout::(len)?; - #[cfg(feature = "stats")] - let payload_bytes = mem::size_of::().saturating_mul(len); // Refill hint accounts for the length prefix, payload alignment // slack, payload bytes, and (for `T: Drop`) a drop-entry slot. let bytes_needed = worst_case_thin_slice_payload::(len); @@ -205,12 +203,10 @@ impl Arena { // pick the right reservation helper. if const { mem::needs_drop::() } { if let Some((uninit, chunk_ptr)) = self.try_reserve_shared_slice_with_drop::(len) { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let f = f.take().expect("with closure taken twice"); let slice_ptr = uninit.init_with_ptr(f); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(payload_bytes); // SAFETY: see `impl_alloc_slice_arc_copy`; the drop entry // was committed by `init_with_ptr` for the chunk-teardown // path. `slice_ptr` carries chunk-wide provenance so the @@ -218,12 +214,10 @@ impl Arena { return Ok(unsafe { Arc::from_raw(slice_ptr.cast::()) }); } } else if let Some((uninit, chunk_ptr)) = self.try_reserve_shared_slice::(len) { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let f = f.take().expect("with closure taken twice"); let slice_ptr = uninit.init_with_ptr(f); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(payload_bytes); // SAFETY: see `impl_alloc_slice_arc_copy`; chunk-wide // provenance preserved via `init_with_ptr`. return Ok(unsafe { Arc::from_raw(slice_ptr.cast::()) }); @@ -248,8 +242,6 @@ impl Arena { let _ = chunk_ref.forget(); p }; - #[cfg(feature = "stats")] - self.record_alloc(payload_bytes); // SAFETY: see the non-oversized branches above. unsafe { Arc::from_raw(slice_ptr.cast::()) } }); diff --git a/crates/multitude/src/arena/alloc_slice_box.rs b/crates/multitude/src/arena/alloc_slice_box.rs index 70d19eea7..dcfa0553f 100644 --- a/crates/multitude/src/arena/alloc_slice_box.rs +++ b/crates/multitude/src/arena/alloc_slice_box.rs @@ -163,7 +163,11 @@ impl Arena { fn impl_alloc_slice_box_copy(&self, src: &[T]) -> Result, AllocError> { check_slice_box_layout::(src.len())?; let len = src.len(); - let ptr = self.reserve_slice_box::(len, |slot_ptr| { + // `src` is a live `&[T]`, so `size_of_val(src)` is a valid + // `usize`. Hoisting it past the refill loop spares the inner + // reservation a `checked_mul` overflow guard. + let payload_bytes = mem::size_of_val(src); + let ptr = self.reserve_slice_box::(len, payload_bytes, |slot_ptr| { // SAFETY: `slot_ptr` is the reservation start; `len` elements // of `T` fit by construction. unsafe { ptr::copy_nonoverlapping(src.as_ptr(), slot_ptr, len) }; @@ -180,7 +184,12 @@ impl Arena { #[inline] fn impl_alloc_slice_box_with T>(&self, len: usize, mut f: F) -> Result, AllocError> { check_slice_box_layout::(len)?; - let ptr = self.reserve_slice_box::(len, |slot_ptr| { + // Caller-provided `len`: must overflow-check the payload size + // up front so the hot loop can skip the `checked_mul`. On + // overflow we report `AllocError` immediately rather than spin + // refilling. + let payload_bytes = mem::size_of::().checked_mul(len).ok_or(AllocError)?; + let ptr = self.reserve_slice_box::(len, payload_bytes, |slot_ptr| { // SAFETY: `slot_ptr` is the reservation start; we init `len` slots // with panic-safe rollback via `InitGuard`. unsafe { @@ -208,30 +217,30 @@ impl Arena { }) } - /// Reserve `len` `T` slots in the current shared chunk, bump the + /// Reserve `len` `T` slots (with precomputed `payload_bytes == + /// size_of::() * len`) in the current shared chunk, bump the /// chunk's strong refcount, call `init(slot_ptr)`, and return the /// base pointer on success. On allocator failure, refills and /// retries; on `init` panic, the refcount bump is released via /// `ChunkRef::Drop` (reservation is leaked in-chunk). #[inline] - fn reserve_slice_box(&self, len: usize, init: impl FnOnce(*mut T)) -> Result, AllocError> { + fn reserve_slice_box(&self, len: usize, payload_bytes: usize, init: impl FnOnce(*mut T)) -> Result, AllocError> { + debug_assert_eq!(payload_bytes, mem::size_of::().wrapping_mul(len)); // Width budget includes prefix + payload alignment slack + // payload bytes. - #[cfg(feature = "stats")] - let payload_bytes = mem::size_of::().saturating_mul(len); let bytes_needed = worst_case_thin_slice_payload::(len); let mut init = Some(init); loop { - if let Some((uninit, chunk_ptr)) = self.try_reserve_shared_slice::(len) { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + // SAFETY: `payload_bytes == size_of::() * len` per caller contract. + let reserved = unsafe { self.try_reserve_shared_slice_with_size::(len, payload_bytes) }; + if let Some((uninit, chunk_ptr)) = reserved { + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let (base, _len) = uninit.into_raw_buffer(); // Run the init under the chunk_ref's Drop guard: a panic // releases the +1 so the chunk is not leaked. let init = init.take().expect("reserve_slice_box init taken twice"); init(base.as_ptr()); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(payload_bytes); return Ok(base); } if self.is_oversized_shared(bytes_needed) { @@ -244,8 +253,6 @@ impl Arena { let (base, _len) = ticket.into_raw_buffer(); init_owned(base.as_ptr()); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(payload_bytes); base }); } diff --git a/crates/multitude/src/arena/alloc_slice_ref.rs b/crates/multitude/src/arena/alloc_slice_ref.rs index db1a58ee7..ea96a67e3 100644 --- a/crates/multitude/src/arena/alloc_slice_ref.rs +++ b/crates/multitude/src/arena/alloc_slice_ref.rs @@ -15,7 +15,9 @@ //! `const { mem::needs_drop::() }` to specialize away the drop-entry //! reservation for trivial-drop element types. +use core::hint::assert_unchecked; use core::mem; +use core::ptr::NonNull; use allocator_api2::alloc::{AllocError, Allocator}; @@ -69,6 +71,38 @@ fn worst_case_slice_payload(len: usize) -> usize { } } +/// Empty `&mut [T]` backed by a well-aligned dangling pointer. +/// +/// Used by the `impl_alloc_slice_*` fast paths to bypass the reservation +/// machinery on `len == 0`: a length-0 reservation would otherwise trip +/// the zero-size probe-byte guard in `try_alloc` (which exists to keep +/// smart-pointer value pointers strictly inside the chunk for header +/// recovery). For a plain `&mut [T]` there is no header recovery, and +/// Rust permits a zero-length slice to alias a well-aligned dangling +/// pointer. +#[inline(always)] +#[allow(clippy::mut_from_ref, reason = "the returned empty slice borrows nothing")] +fn empty_slice<'a, T>() -> &'a mut [T] { + // SAFETY: `NonNull::::dangling()` is well-aligned and non-null; + // an empty `&mut [T]` is well-defined regardless of the pointer + // value as long as alignment is correct. + unsafe { core::slice::from_raw_parts_mut(NonNull::::dangling().as_ptr(), 0) } +} + +/// Optimizer hint: tells codegen that `len` is non-zero. Every caller +/// guards `len == 0` with an early return first, so this lets LLVM fold +/// the `size.max(1)` clamp in the reservation probe down to `size`. +#[inline(always)] +// Mutation testing is suppressed: this is purely an optimization hint +// with no runtime effect. `assert_unchecked(len >= 0)` is vacuously +// true for `usize` and behaves identically to `len > 0`, so the +// `>`→`>=` mutant is an equivalent mutant no behavioral test can detect. +#[cfg_attr(test, mutants::skip)] +fn assume_nonzero_len(len: usize) { + // SAFETY: callers handle `len == 0` via an early return before this. + unsafe { assert_unchecked(len > 0) }; +} + impl Arena { /// Bump-allocate a copy of `slice` (element-by-element `Copy`) into the arena. /// @@ -112,7 +146,7 @@ impl Arena { /// panic propagates. #[must_use] #[inline] - pub fn alloc_slice_fill_with T>(&self, len: usize, f: F) -> &mut [T] { + pub fn alloc_slice_fill_with T>(&self, len: usize, f: F) -> &mut [T] { (self.impl_alloc_slice_fill_with::(len, f)).expect_alloc() } @@ -128,7 +162,7 @@ impl Arena { /// If `f` panics, already-initialized elements are dropped. #[allow(clippy::mut_from_ref, reason = "simple references: see Self::try_alloc_with")] #[inline] - pub fn try_alloc_slice_fill_with T>(&self, len: usize, f: F) -> Result<&mut [T], AllocError> { + pub fn try_alloc_slice_fill_with T>(&self, len: usize, f: F) -> Result<&mut [T], AllocError> { self.impl_alloc_slice_fill_with::(len, f) } @@ -146,7 +180,7 @@ impl Arena { #[must_use] #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] #[inline] - pub fn alloc_slice_clone(&self, slice: impl AsRef<[T]>) -> &mut [T] { + pub fn alloc_slice_clone(&self, slice: impl AsRef<[T]>) -> &mut [T] { (self.impl_alloc_slice_clone::(slice.as_ref())).expect_alloc() } @@ -163,7 +197,7 @@ impl Arena { /// are dropped before the panic propagates. #[allow(clippy::mut_from_ref, reason = "simple references: see Self::try_alloc_with")] #[inline] - pub fn try_alloc_slice_clone(&self, slice: impl AsRef<[T]>) -> Result<&mut [T], AllocError> { + pub fn try_alloc_slice_clone(&self, slice: impl AsRef<[T]>) -> Result<&mut [T], AllocError> { self.impl_alloc_slice_clone::(slice.as_ref()) } @@ -181,7 +215,7 @@ impl Arena { /// `ExactSizeIterator::len()` reported. #[must_use] #[inline] - pub fn alloc_slice_fill_iter(&self, iter: I) -> &mut [T] + pub fn alloc_slice_fill_iter(&self, iter: I) -> &mut [T] where I: IntoIterator, I::IntoIter: ExactSizeIterator, @@ -202,7 +236,7 @@ impl Arena { /// `ExactSizeIterator::len()` reported. #[inline] #[allow(clippy::mut_from_ref, reason = "see `try_alloc_with`")] - pub fn try_alloc_slice_fill_iter(&self, iter: I) -> Result<&mut [T], AllocError> + pub fn try_alloc_slice_fill_iter(&self, iter: I) -> Result<&mut [T], AllocError> where I: IntoIterator, I::IntoIter: ExactSizeIterator, @@ -219,29 +253,68 @@ impl Arena { fn impl_alloc_slice_copy(&self, src: &[T]) -> Result<&mut [T], AllocError> { reject_over_aligned::()?; let len = src.len(); - let refill_hint = worst_case_slice_payload::(len); + if len == 0 { + return Ok(empty_slice::()); + } + // `len == 0` was handled above; hint the optimizer so the + // `size.max(1)` clamp in `try_alloc`'s probe folds to `size`. + assume_nonzero_len(len); + // `src` is a live `&[T]`, so `size_of_val(src)` is a valid + // `usize`. Hoisting the precomputed byte size lets the inner + // reservation helper skip the `checked_mul` overflow guard, + // removing a `shr/jne` pair from the bump-copy loop. + let size = mem::size_of_val(src); loop { - if let Some(u) = self.try_reserve_local_slice::(len) { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of_val(src)); + if let Some(u) = self.try_reserve_local_slice_with_size::(len, size) { return Ok(u.init_copy_from_slice(src)); } - if self.is_oversized_local(refill_hint) { - let mut ptr = self.alloc_oversized_local_with(refill_hint, |mutator| { - let ticket = mutator - .try_alloc_uninit_slice::(len) - .expect("dedicated oversized chunk sized to fit slice"); - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of_val(src)); - ticket.init_copy_from_slice_ptr(src) - })?; - // SAFETY: chunk retained in `retired_local` for `&self`. - return Ok(unsafe { ptr.as_mut() }); + if let Some(slice) = self.refill_or_alloc_oversized_slice_copy::(src)? { + return Ok(slice); } - self.refill_local(refill_hint)?; } } + /// Cold fall-back for [`Self::impl_alloc_slice_copy`]: either refills + /// the current local chunk (return `Ok(None)` so the caller retries) + /// or returns the slice from a dedicated oversized chunk + /// (`Ok(Some(_))`). `refill_hint` is computed here so the hot loop + /// in the caller doesn't keep it live across iterations. + #[cold] + #[inline(never)] + #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] + // Mutation testing is suppressed: the whole-body `-> Ok(None)` + // replacement drops the refill side effect, so the caller's + // reserve→refill retry loop spins forever (the suite hangs rather + // than fails). The `Ok(None)` return is the correct "refilled, + // please retry" signal. + #[cfg_attr(test, mutants::skip)] + fn refill_or_alloc_oversized_slice_copy(&self, src: &[T]) -> Result, AllocError> { + let refill_hint = worst_case_slice_payload::(src.len()); + if self.is_oversized_local(refill_hint) { + return Ok(Some(self.alloc_oversized_slice_copy::(refill_hint, src)?)); + } + self.refill_local(refill_hint)?; + Ok(None) + } + + /// Cold oversized-fallback for [`Self::impl_alloc_slice_copy`]. + /// Kept out-of-line so the hot loop doesn't keep `src`/`len` + /// addressable for a captured-by-move closure environment (see the + /// rationale on [`Self::alloc_oversized_value_with`]). + #[cold] + #[inline(never)] + #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] + fn alloc_oversized_slice_copy(&self, refill_hint: usize, src: &[T]) -> Result<&mut [T], AllocError> { + let mutator = self.acquire_oversized_local_mutator(refill_hint)?; + let ticket = mutator + .try_alloc_uninit_slice::(src.len()) + .expect("dedicated oversized chunk sized to fit slice"); + let mut ptr = ticket.init_copy_from_slice_ptr(src); + self.retain_oversized_local_mutator(mutator); + // SAFETY: chunk retained in `retired_local` for `&self`. + Ok(unsafe { ptr.as_mut() }) + } + /// Closure-free fast path for `alloc_slice_clone` / /// `try_alloc_slice_clone`. Mirrors `impl_alloc_value`: a /// `const { mem::needs_drop::() }` branch picks the @@ -253,42 +326,64 @@ impl Arena { reject_over_aligned::()?; reject_drop_slice_too_long::(src.len())?; let len = src.len(); - let refill_hint = worst_case_slice_payload::(len); + if len == 0 { + return Ok(empty_slice::()); + } + // See `impl_alloc_slice_copy`. + assume_nonzero_len(len); + // See `impl_alloc_slice_copy`. Hoisted byte size lets the + // `!needs_drop` reservation arm skip the `checked_mul` overflow + // guard. The drop-tracked arm still uses the byte-size-unaware + // helper because it additionally reserves a drop entry slot. + let size = mem::size_of_val(src); loop { if const { mem::needs_drop::() } { if let Some(u) = self.try_reserve_local_slice_with_drop::(len) { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of_val(src)); return Ok(u.init_clone_from_slice(src)); } - } else if let Some(u) = self.try_reserve_local_slice::(len) { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of_val(src)); + } else if let Some(u) = self.try_reserve_local_slice_with_size::(len, size) { return Ok(u.init_clone_from_slice(src)); } - if self.is_oversized_local(refill_hint) { - let mut ptr = self.alloc_oversized_local_with(refill_hint, |mutator| { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of_val(src)); - if const { mem::needs_drop::() } { - let ticket = mutator - .try_alloc_uninit_slice_with_drop::(len) - .expect("dedicated oversized chunk sized to fit slice + drop entry"); - ticket.init_with_ptr(|i| src[i].clone()) - } else { - let ticket = mutator - .try_alloc_uninit_slice::(len) - .expect("dedicated oversized chunk sized to fit slice"); - ticket.init_with_ptr(|i| src[i].clone()) - } - })?; - // SAFETY: chunk retained in `retired_local` for `&self`. - return Ok(unsafe { ptr.as_mut() }); + if let Some(slice) = self.refill_or_alloc_oversized_slice_clone::(src)? { + return Ok(slice); } - self.refill_local(refill_hint)?; } } + /// Cold fall-back for [`Self::impl_alloc_slice_clone`]. See + /// [`Self::refill_or_alloc_oversized_slice_copy`] for the rationale + /// behind the split. + #[cold] + #[inline(never)] + #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] + // Mutation testing is suppressed: see + // `refill_or_alloc_oversized_slice_copy` — `-> Ok(None)` spins the + // caller's retry loop forever. + #[cfg_attr(test, mutants::skip)] + fn refill_or_alloc_oversized_slice_clone(&self, src: &[T]) -> Result, AllocError> { + let len = src.len(); + let refill_hint = worst_case_slice_payload::(len); + if self.is_oversized_local(refill_hint) { + let mut ptr = self.alloc_oversized_local_with(refill_hint, |mutator| { + if const { mem::needs_drop::() } { + let ticket = mutator + .try_alloc_uninit_slice_with_drop::(len) + .expect("dedicated oversized chunk sized to fit slice + drop entry"); + ticket.init_with_ptr(|i| src[i].clone()) + } else { + let ticket = mutator + .try_alloc_uninit_slice::(len) + .expect("dedicated oversized chunk sized to fit slice"); + ticket.init_with_ptr(|i| src[i].clone()) + } + })?; + // SAFETY: chunk retained in `retired_local` for `&self`. + return Ok(Some(unsafe { ptr.as_mut() })); + } + self.refill_local(refill_hint)?; + Ok(None) + } + /// Closure-bearing fast path for `alloc_slice_fill_with` / /// `try_alloc_slice_fill_with`. Mirrors `impl_alloc_slice_clone`: a /// `const { mem::needs_drop::() }` branch picks the @@ -301,27 +396,26 @@ impl Arena { fn impl_alloc_slice_fill_with T>(&self, len: usize, f: F) -> Result<&mut [T], AllocError> { reject_over_aligned::()?; reject_drop_slice_too_long::(len)?; + if len == 0 { + return Ok(empty_slice::()); + } + // See `impl_alloc_slice_copy`. + assume_nonzero_len(len); let refill_hint = worst_case_slice_payload::(len); let mut f = Some(f); loop { if const { mem::needs_drop::() } { if let Some(u) = self.try_reserve_local_slice_with_drop::(len) { let f = f.take().expect("with closure taken twice"); - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::() * len); return Ok(u.init_with(f)); } } else if let Some(u) = self.try_reserve_local_slice::(len) { let f = f.take().expect("with closure taken twice"); - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::() * len); return Ok(u.init_with(f)); } if self.is_oversized_local(refill_hint) { let f = f.take().expect("with closure taken twice"); let mut ptr = self.alloc_oversized_local_with(refill_hint, |mutator| { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::() * len); if const { mem::needs_drop::() } { let ticket = mutator .try_alloc_uninit_slice_with_drop::(len) @@ -353,27 +447,30 @@ impl Arena { reject_over_aligned::()?; let len = iter.len(); reject_drop_slice_too_long::(len)?; + if len == 0 { + // Drop the iterator without consuming it: the contract is + // "fill `len` slots from the iterator", so a zero-length + // fill consumes nothing. + drop(iter); + return Ok(empty_slice::()); + } + // See `impl_alloc_slice_copy`. + assume_nonzero_len(len); let refill_hint = worst_case_slice_payload::(len); let mut iter = Some(iter); loop { if const { mem::needs_drop::() } { if let Some(u) = self.try_reserve_local_slice_with_drop::(len) { let it = iter.take().expect("iterator taken twice"); - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::() * len); return Ok(u.init_from_iter(it)); } } else if let Some(u) = self.try_reserve_local_slice::(len) { let it = iter.take().expect("iterator taken twice"); - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::() * len); return Ok(u.init_from_iter(it)); } if self.is_oversized_local(refill_hint) { let mut it = iter.take().expect("iterator taken twice"); let mut ptr = self.alloc_oversized_local_with(refill_hint, |mutator| { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::() * len); if const { mem::needs_drop::() } { let ticket = mutator .try_alloc_uninit_slice_with_drop::(len) diff --git a/crates/multitude/src/arena/alloc_str.rs b/crates/multitude/src/arena/alloc_str.rs index 867388bb5..efe2ab9b3 100644 --- a/crates/multitude/src/arena/alloc_str.rs +++ b/crates/multitude/src/arena/alloc_str.rs @@ -148,15 +148,11 @@ impl Arena { let len = s.len(); loop { if let Some(u) = self.try_reserve_local_bytes(len) { - #[cfg(feature = "stats")] - self.record_alloc(len); return Ok(u.init_copy_from_str(s)); } if self.is_oversized_local(len) { let ptr = self.alloc_oversized_local_with(len, |mutator| { let ticket = mutator.try_alloc_bytes(len).expect("dedicated oversized chunk sized to fit string"); - #[cfg(feature = "stats")] - self.record_alloc(len); // `init_copy_from_str` returns `&mut str` bound to the // mutator's borrow; we hand back a raw pointer + len so // the lifetime can be re-attached to `&Arena` once the diff --git a/crates/multitude/src/arena/alloc_unsized.rs b/crates/multitude/src/arena/alloc_unsized.rs index 309166a7e..1d30ce9e0 100644 --- a/crates/multitude/src/arena/alloc_unsized.rs +++ b/crates/multitude/src/arena/alloc_unsized.rs @@ -224,7 +224,7 @@ impl Arena { loop { if let Some((reservation, chunk_ptr)) = self.current_shared().try_alloc_with_chunk(total, layout.align().max(1)) { let init = init.take().expect("init taken twice"); - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); // SAFETY: see `write_dst_prefix_and_init` — `reservation` // is the freshly reserved exclusive storage; we write // metadata at `payload - meta_bytes` and hand `init` a @@ -305,7 +305,7 @@ impl Arena { if let Some((base_in_chunk, drop_slot_opt, chunk_ptr)) = reservation { let init = init.take().expect("init taken twice"); - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); // SAFETY: see `write_dst_prefix_and_init`. let payload_nn = unsafe { write_dst_prefix_and_init::(base_in_chunk.as_non_null(), payload_offset, meta_bytes, metadata, init) }; @@ -318,8 +318,6 @@ impl Arena { } } let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(layout.size()); return Ok(payload_nn); } @@ -343,8 +341,6 @@ impl Arena { } } let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(layout.size()); payload_nn }); } diff --git a/crates/multitude/src/arena/alloc_utf16.rs b/crates/multitude/src/arena/alloc_utf16.rs index 442a9171f..fcc0255f2 100644 --- a/crates/multitude/src/arena/alloc_utf16.rs +++ b/crates/multitude/src/arena/alloc_utf16.rs @@ -16,13 +16,14 @@ use allocator_api2::alloc::{AllocError, Allocator}; use super::alloc_prefixed::PREFIX_BYTES; use super::alloc_value::acquire_shared_chunk_ref; use super::{Arena, ExpectAlloc}; +use crate::strings::{ArcUtf16Str, BoxUtf16Str, Utf16String}; #[cfg_attr(docsrs, doc(cfg(feature = "utf16")))] impl Arena { /// Copy `s` into a `Shared`-flavor chunk and return an [`ArcUtf16Str`](crate::strings::ArcUtf16Str). #[must_use] #[inline] - pub fn alloc_utf16_str_arc(&self, s: impl AsRef) -> crate::strings::ArcUtf16Str + pub fn alloc_utf16_str_arc(&self, s: impl AsRef) -> ArcUtf16Str where A: Send + Sync, { @@ -37,19 +38,19 @@ impl Arena { /// exhaustion or oversize-cutover budget) or if the source string /// length would overflow the inline length-prefix accounting. #[inline] - pub fn try_alloc_utf16_str_arc(&self, s: impl AsRef) -> Result, AllocError> + pub fn try_alloc_utf16_str_arc(&self, s: impl AsRef) -> Result, AllocError> where A: Send + Sync, { self.impl_alloc_prefixed_shared::(s.as_ref().as_slice()).map(|ptr| // SAFETY: see `Self::alloc_utf16_str_arc`. - unsafe { crate::strings::ArcUtf16Str::from_raw(ptr) }) + unsafe { ArcUtf16Str::from_raw(ptr) }) } /// Copy `s` into the arena and return a [`BoxUtf16Str`](crate::strings::BoxUtf16Str). #[must_use] #[inline] - pub fn alloc_utf16_str_box(&self, s: impl AsRef) -> crate::strings::BoxUtf16Str { + pub fn alloc_utf16_str_box(&self, s: impl AsRef) -> BoxUtf16Str { self.try_alloc_utf16_str_box(s).expect_alloc() } @@ -61,16 +62,16 @@ impl Arena { /// exhaustion or oversize-cutover budget) or if the source string /// length would overflow the inline length-prefix accounting. #[inline] - pub fn try_alloc_utf16_str_box(&self, s: impl AsRef) -> Result, AllocError> { + pub fn try_alloc_utf16_str_box(&self, s: impl AsRef) -> Result, AllocError> { self.impl_alloc_prefixed_shared::(s.as_ref().as_slice()).map(|ptr| // SAFETY: see `Self::alloc_utf16_str_arc`. - unsafe { crate::strings::BoxUtf16Str::from_raw(ptr) }) + unsafe { BoxUtf16Str::from_raw(ptr) }) } /// Transcode `s` from UTF-8 to UTF-16 and return an [`ArcUtf16Str`](crate::strings::ArcUtf16Str). #[must_use] #[inline] - pub fn alloc_utf16_str_arc_from_str(&self, s: impl AsRef) -> crate::strings::ArcUtf16Str + pub fn alloc_utf16_str_arc_from_str(&self, s: impl AsRef) -> ArcUtf16Str where A: Send + Sync, { @@ -85,19 +86,19 @@ impl Arena { /// exhaustion or oversize-cutover budget) or if the source string /// length would overflow the inline length-prefix accounting. #[inline] - pub fn try_alloc_utf16_str_arc_from_str(&self, s: impl AsRef) -> Result, AllocError> + pub fn try_alloc_utf16_str_arc_from_str(&self, s: impl AsRef) -> Result, AllocError> where A: Send + Sync, { self.impl_alloc_utf16_prefixed_from_str(s.as_ref()).map(|ptr| // SAFETY: see `Self::alloc_utf16_str_arc`. - unsafe { crate::strings::ArcUtf16Str::from_raw(ptr) }) + unsafe { ArcUtf16Str::from_raw(ptr) }) } /// Transcode `s` from UTF-8 to UTF-16 and return a [`BoxUtf16Str`](crate::strings::BoxUtf16Str). #[must_use] #[inline] - pub fn alloc_utf16_str_box_from_str(&self, s: impl AsRef) -> crate::strings::BoxUtf16Str { + pub fn alloc_utf16_str_box_from_str(&self, s: impl AsRef) -> BoxUtf16Str { self.try_alloc_utf16_str_box_from_str(s).expect_alloc() } @@ -109,17 +110,17 @@ impl Arena { /// exhaustion or oversize-cutover budget) or if the source string /// length would overflow the inline length-prefix accounting. #[inline] - pub fn try_alloc_utf16_str_box_from_str(&self, s: impl AsRef) -> Result, AllocError> { + pub fn try_alloc_utf16_str_box_from_str(&self, s: impl AsRef) -> Result, AllocError> { self.impl_alloc_utf16_prefixed_from_str(s.as_ref()).map(|ptr| // SAFETY: see `Self::alloc_utf16_str_arc`. - unsafe { crate::strings::BoxUtf16Str::from_raw(ptr) }) + unsafe { BoxUtf16Str::from_raw(ptr) }) } /// Create a new, empty growable [`Utf16String`](crate::strings::Utf16String) backed by this arena. #[must_use] #[inline] - pub const fn alloc_utf16_string(&self) -> crate::strings::Utf16String<'_, A> { - crate::strings::Utf16String::new_in(self) + pub const fn alloc_utf16_string(&self) -> Utf16String<'_, A> { + Utf16String::new_in(self) } /// Create a new growable arena-backed [`Utf16String`](crate::strings::Utf16String) with capacity. @@ -132,8 +133,8 @@ impl Arena { /// [`Self::try_alloc_utf16_string_with_capacity`] for a fallible variant. #[must_use] #[inline] - pub fn alloc_utf16_string_with_capacity(&self, cap: usize) -> crate::strings::Utf16String<'_, A> { - crate::strings::Utf16String::with_capacity_in(cap, self) + pub fn alloc_utf16_string_with_capacity(&self, cap: usize) -> Utf16String<'_, A> { + Utf16String::with_capacity_in(cap, self) } /// Fallible variant of [`Self::alloc_utf16_string_with_capacity`]. @@ -144,8 +145,8 @@ impl Arena { /// exhaustion or oversize-cutover budget) or if the source string /// length would overflow the inline length-prefix accounting. #[inline] - pub fn try_alloc_utf16_string_with_capacity(&self, cap: usize) -> Result, AllocError> { - crate::strings::Utf16String::try_with_capacity_in(cap, self) + pub fn try_alloc_utf16_string_with_capacity(&self, cap: usize) -> Result, AllocError> { + Utf16String::try_with_capacity_in(cap, self) } /// Shared body for `alloc_utf16_str_arc_from_str` / @@ -176,11 +177,9 @@ impl Arena { let total = PREFIX_BYTES.checked_add(payload_bytes).ok_or(AllocError)?; loop { if let Some((uninit, chunk_ptr)) = self.current_shared().try_alloc_with_chunk(total, elem_align) { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let payload = transcode_utf16_into(uninit.as_non_null(), s, exact); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(exact * elem_size); return Ok(payload); } if self.is_oversized_shared(total) { @@ -191,8 +190,6 @@ impl Arena { let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); let payload = transcode_utf16_into(base.as_non_null(), s, exact); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(exact * elem_size); payload }); } @@ -206,6 +203,12 @@ impl Arena { /// returns a thin pointer to the first payload element. `exact` must /// match `s.chars().map(char::len_utf16).sum()` from a pre-walk. #[inline(always)] +// Exercised by the `from_str` UTF-16 tests, but as a concrete (non-generic) +// `#[inline(always)]` free fn it has no standalone instantiation, so +// `llvm-cov` cannot attribute coverage to its body even though every call +// site runs it. Excluded from coverage to avoid a false gap (cf. +// `chunk_mutator::offset_or_unwind`). +#[cfg_attr(coverage_nightly, coverage(off))] #[allow( clippy::cast_ptr_alignment, reason = "see callers: `base + PREFIX_BYTES` is u16-aligned by construction" diff --git a/crates/multitude/src/arena/alloc_value.rs b/crates/multitude/src/arena/alloc_value.rs index a112dc02d..ffc4a4f6b 100644 --- a/crates/multitude/src/arena/alloc_value.rs +++ b/crates/multitude/src/arena/alloc_value.rs @@ -15,9 +15,11 @@ use super::{Arena, ExpectAlloc}; use crate::arc::Arc; use crate::r#box::Box; use crate::internal::Chunk; +use crate::internal::chunk_ref::ChunkRef; use crate::internal::constants::max_smart_ptr_align; use crate::internal::drop_entry::DropEntry; use crate::internal::shared_chunk::SharedChunk; +use crate::internal::uninit::{Uninit, UninitDrop}; /// Worst-case bytes consumed by a single value allocation of type `T` in /// a chunk: value bytes + alignment padding, plus one [`DropEntry`] slot @@ -343,7 +345,7 @@ impl Arena { /// ``` #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] #[inline] - pub fn alloc(&self, value: T) -> &mut T { + pub fn alloc(&self, value: T) -> &mut T { (self.impl_alloc_value_with::(move || value)).expect_alloc() } @@ -359,7 +361,7 @@ impl Arena { /// the request. #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] #[inline] - pub fn try_alloc(&self, value: T) -> Result<&mut T, AllocError> { + pub fn try_alloc(&self, value: T) -> Result<&mut T, AllocError> { self.impl_alloc_value_with::(move || value) } @@ -377,7 +379,7 @@ impl Arena { /// refcount bumped) but the chunk itself reclaims normally. #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] #[inline] - pub fn alloc_with T>(&self, f: F) -> &mut T { + pub fn alloc_with T>(&self, f: F) -> &mut T { // See `alloc` for why the `Err` arm uses `panic_alloc!()` rather than // `unsafe { unreachable_unchecked() }`. (self.impl_alloc_value_with::(f)).expect_alloc() @@ -397,7 +399,7 @@ impl Arena { reason = "Simple references: each call returns a fresh, disjoint &mut T; the borrow checker treats the returned reference as exclusive of its own region but harmlessly aliasing-with-shared with the &Arena borrow" )] #[inline] - pub fn try_alloc_with T>(&self, f: F) -> Result<&mut T, AllocError> { + pub fn try_alloc_with T>(&self, f: F) -> Result<&mut T, AllocError> { self.impl_alloc_value_with::(f) } @@ -415,41 +417,56 @@ impl Arena { loop { if const { mem::needs_drop::() } { if let Some(u) = self.try_reserve_local_with_drop::() { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); return Ok(u.init(f())); } } else if let Some(u) = self.try_reserve_local::() { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); return Ok(u.init(f())); } let wcp = worst_case_payload::(); if self.is_oversized_local(wcp) { - let ptr = self.alloc_oversized_local_with(wcp, |mutator| { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); - if const { mem::needs_drop::() } { - let ticket = mutator - .try_alloc_uninit_with_drop::() - .expect("dedicated oversized chunk sized to fit one value + drop entry"); - ticket.init_raw(f()) - } else { - let ticket = mutator - .try_alloc_uninit::() - .expect("dedicated oversized chunk sized to fit one value"); - ticket.init_raw(f()) - } - })?; - // SAFETY: the chunk is retained in `retired_local` for the - // `&self` borrow, so `ptr` stays valid; the value is - // freshly initialized and uniquely held. - return Ok(unsafe { &mut *ptr.as_ptr() }); + return self.alloc_oversized_value_with::(wcp, f); } self.refill_local(wcp)?; } } + /// Cold oversized-value fallback for [`Self::impl_alloc_value_with`]. + /// + /// Kept `#[inline(never)]` so the fast-path body stays small + /// enough for the public scalar entry points to inline into their + /// callers; the bench shows that re-inlining this branch into + /// `impl_alloc_value_with` blows `alloc`'s instruction budget past + /// the inlining heuristic and turns every call site into a real + /// function call. + /// + /// Closure-free in the user-`f` argument: capturing `f` inside an + /// `impl FnOnce` passed to `alloc_oversized_local_with` would force + /// `f`'s environment (e.g. `&loop_counter` for a default-by-ref + /// capture) into an addressable stack slot, adding a per-iteration + /// spill on the hot path even when this cold branch is never taken. + #[cold] + #[inline(never)] + #[allow(clippy::mut_from_ref, reason = "Simple references: see Self::try_alloc_with")] + fn alloc_oversized_value_with T>(&self, wcp: usize, f: F) -> Result<&mut T, AllocError> { + let mutator = self.acquire_oversized_local_mutator(wcp)?; + let value_ptr = if const { mem::needs_drop::() } { + let ticket = mutator + .try_alloc_uninit_with_drop::() + .expect("dedicated oversized chunk sized to fit one value + drop entry"); + ticket.init_raw(f()) + } else { + let ticket = mutator + .try_alloc_uninit::() + .expect("dedicated oversized chunk sized to fit one value"); + ticket.init_raw(f()) + }; + self.retain_oversized_local_mutator(mutator); + // SAFETY: the chunk is retained in `retired_local` for the + // `&self` borrow, so `value_ptr` stays valid; the value is + // freshly initialized and uniquely held. + Ok(unsafe { &mut *value_ptr.as_ptr() }) + } + /// Shared fast-path body for the `alloc_box` family. /// /// Delegates to [`Self::impl_alloc_smart_with`] and wraps the @@ -509,42 +526,82 @@ impl Arena { return Err(AllocError); } loop { + // A ZST whose allocation reserves no drop entry does not + // advance the bump cursor (`try_alloc(0, _)` is a no-op on + // the cursor), so back-to-back handouts would never refill + // the chunk. The per-allocation handout count is tracked in + // the non-atomic `local_shared_count` and draws down the + // pre-credited ref surplus; an unbounded run from a single + // chunk could exhaust that surplus, driving the chunk's + // atomic refcount to zero while it is still installed + // (use-after-free) or underflowing the surplus reconciliation + // at retire (double-free). Pre-reserve a 1-byte tag so each + // such handout advances the cursor, bounding per-chunk + // handouts to the chunk capacity (well below the surplus). + // The drop-entry path below already advances `drop_top`, so + // drop-registering reservations need no tag. Mirrors the + // guard in `impl_alloc_uninit_smart`. + if const { mem::size_of::() == 0 && !(REGISTER_DROP && mem::needs_drop::()) } + && self.current_shared().try_alloc(1, 1).is_none() + { + self.refill_shared(worst_case_payload::())?; + continue; + } if const { REGISTER_DROP && mem::needs_drop::() } { if let Some((uninit, chunk_ptr)) = self.try_reserve_shared_with_drop::() { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); - return Ok(init_smart_slot_with_drop::(uninit, chunk_ptr, f)); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); + return Ok(init_smart_slot_with_drop::(uninit, chunk_ref, f)); } } else if let Some((uninit, chunk_ptr)) = self.try_reserve_shared::() { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); - return Ok(init_smart_slot::(uninit, chunk_ptr, f)); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); + return Ok(init_smart_slot::(uninit, chunk_ref, f)); } // Worst-case payload includes a drop entry for `T: Drop` // so refill always sizes the chunk for the with-drop // reservation above. let wcp = worst_case_payload::(); if self.is_oversized_shared(wcp) { - return self.alloc_oversized_shared_with(wcp, |mutator, chunk_ptr| { - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); - if const { REGISTER_DROP && mem::needs_drop::() } { - let ticket = mutator - .try_alloc_uninit_with_drop::() - .expect("dedicated oversized chunk sized to fit one value + drop entry"); - init_smart_slot_with_drop::(ticket, chunk_ptr, f) - } else { - let ticket = mutator - .try_alloc_uninit::() - .expect("dedicated oversized chunk sized to fit one value"); - init_smart_slot::(ticket, chunk_ptr, f) - } - }); + return self.alloc_oversized_smart_with::(wcp, f); } self.refill_shared(wcp)?; } } + /// Cold oversized-smart-pointer fallback for + /// [`Self::impl_alloc_smart_with`]. + /// + /// Kept `#[inline(never)]` for the same reason as + /// [`Self::alloc_oversized_value_with`]: the fast-path body must + /// stay small enough for the public smart-pointer entry points to + /// inline; closure-free in `f` to avoid spilling the user closure's + /// environment to memory on the hot path. + #[cold] + #[inline(never)] + fn alloc_oversized_smart_with T, const REGISTER_DROP: bool>( + &self, + wcp: usize, + f: F, + ) -> Result, AllocError> { + let (mutator, chunk_ptr) = self.acquire_oversized_shared_mutator(wcp)?; + let ptr = if const { REGISTER_DROP && mem::needs_drop::() } { + let ticket = mutator + .try_alloc_uninit_with_drop::() + .expect("dedicated oversized chunk sized to fit one value + drop entry"); + let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + init_smart_slot_with_drop::(ticket, chunk_ref, f) + } else { + let ticket = mutator + .try_alloc_uninit::() + .expect("dedicated oversized chunk sized to fit one value"); + let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + init_smart_slot::(ticket, chunk_ref, f) + }; + // `mutator` drops here, releasing its `+1`. The smart-pointer + // `chunk_ref` taken above owns the surviving `+1`. + drop(mutator); + Ok(ptr) + } + /// Shared body for the uninit/zeroed `Arc>` family, /// **for `T: Drop` only** (callers route `T: !Drop` to the ordinary /// no-entry value-Arc path). @@ -585,7 +642,7 @@ impl Arena { continue; } if let Some((uninit, chunk_ptr)) = self.try_reserve_shared_with_drop::() { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let value = if zeroed { mem::MaybeUninit::::zeroed() } else { @@ -595,8 +652,6 @@ impl Arena { let _ = chunk_ref.forget(); // Publish the just-written placeholder so `assume_init` sees it. self.current_shared().publish_drop_count(); - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); // SAFETY: the chunk was bumped +1 for this `Arc` and a // placeholder drop entry is reserved and published; // `assume_init` commits the real shim once the value is set. @@ -616,8 +671,6 @@ impl Arena { }; let ptr = ticket.into_uninit_placeholder(value); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(mem::size_of::()); // SAFETY: see the non-oversized branch above. The // temporary mutator's `Drop` publishes the drop-entry // count before this function returns, so `assume_init` @@ -644,19 +697,15 @@ impl Arena { return Err(AllocError); } reject_uninit_slice_arc_too_long(len)?; - #[cfg(feature = "stats")] - let bytes = mem::size_of::().saturating_mul(len); // Refill hint accounts for prefix + payload alignment slack + // payload bytes + drop entry. let min_payload = super::alloc_prefixed::worst_case_thin_slice_payload::(len); loop { if let Some((uninit, chunk_ptr)) = self.try_reserve_shared_slice_with_drop::(len) { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); + let chunk_ref = self.acquire_current_shared_chunk_ref(chunk_ptr); let ptr = uninit.into_uninit_slice_placeholder(zeroed); let _ = chunk_ref.forget(); self.current_shared().publish_drop_count(); - #[cfg(feature = "stats")] - self.record_alloc(bytes); // SAFETY: as in `impl_alloc_uninit_arc`; the placeholder slice // drop entry is reserved and published for `assume_init`. return Ok(unsafe { Arc::from_raw(ptr.cast::()) }); @@ -669,8 +718,6 @@ impl Arena { let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); let ptr = ticket.into_uninit_slice_placeholder(zeroed); let _ = chunk_ref.forget(); - #[cfg(feature = "stats")] - self.record_alloc(bytes); // SAFETY: see the non-oversized branch above. unsafe { Arc::from_raw(ptr.cast::()) } }); @@ -696,12 +743,7 @@ fn reject_uninit_slice_arc_too_long(len: usize) -> Result<(), AllocError> { /// of [`Arena::impl_alloc_smart_with`] so the closure-panic path runs /// the refcount-release guard. #[inline(always)] -fn init_smart_slot T>( - uninit: crate::internal::uninit::Uninit<'_, T>, - chunk_ptr: NonNull>, - f: F, -) -> NonNull { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); +fn init_smart_slot T>(uninit: Uninit<'_, T>, chunk_ref: ChunkRef, f: F) -> NonNull { let value = f(); let _ = chunk_ref.forget(); uninit.init_raw(value) @@ -712,11 +754,10 @@ fn init_smart_slot T>( /// value's `Drop` runs from the chunk's drop-list at teardown. #[inline(always)] fn init_smart_slot_with_drop T>( - uninit: crate::internal::uninit::UninitDrop<'_, T>, - chunk_ptr: NonNull>, + uninit: UninitDrop<'_, T>, + chunk_ref: ChunkRef, f: F, ) -> NonNull { - let chunk_ref = acquire_shared_chunk_ref::(chunk_ptr); let value = f(); let _ = chunk_ref.forget(); uninit.init_raw(value) @@ -728,9 +769,7 @@ fn init_smart_slot_with_drop T>( /// [`Arena::init_arc_slot`] so the unsafe `inc_ref`/`adopt` pair lives /// in one place. #[inline(always)] -pub(crate) fn acquire_shared_chunk_ref( - chunk_ptr: NonNull>, -) -> crate::internal::chunk_ref::ChunkRef { +pub(crate) fn acquire_shared_chunk_ref(chunk_ptr: NonNull>) -> ChunkRef { // SAFETY: `chunk_ptr` belongs to a currently-installed shared // mutator and the arena holds a +1 on it for the duration of // `&self`; we bump for the soon-to-be smart pointer and adopt @@ -740,6 +779,6 @@ pub(crate) fn acquire_shared_chunk_ref( // semantics of the `alloc_box_with` / `alloc_arc_with` family). unsafe { chunk_ptr.as_ref().inc_ref(); - crate::internal::chunk_ref::ChunkRef::::adopt(chunk_ptr) + ChunkRef::::adopt(chunk_ptr) } } diff --git a/crates/multitude/src/arena/mod.rs b/crates/multitude/src/arena/mod.rs index 1a3bb80da..4762b41da 100644 --- a/crates/multitude/src/arena/mod.rs +++ b/crates/multitude/src/arena/mod.rs @@ -4,8 +4,7 @@ #![allow(clippy::inline_always, reason = "hot bump-allocator helpers must inline into their callers")] use alloc::sync::Arc as StdArc; -use alloc::vec::Vec; -use core::cell::{Cell, RefCell}; +use core::cell::Cell; use core::fmt; use core::ptr::NonNull; @@ -16,11 +15,31 @@ use crate::arena_builder::ArenaBuilder; use crate::arena_stats::ArenaStats; use crate::internal::chunk_mutator::ChunkMutator; use crate::internal::chunk_provider::{ChunkProvider, ChunkProviderConfig}; -use crate::internal::constants::SizeClass; +use crate::internal::chunk_ref::ChunkRef; +use crate::internal::constants::{MAX_NORMAL_ALLOC, SizeClass}; use crate::internal::current_chunk::CurrentChunk; use crate::internal::local_chunk::LocalChunk; use crate::internal::shared_chunk::SharedChunk; +/// Surplus of shared-chunk strong refs the arena pre-credits to the +/// chunk's atomic `ref_count` at install time. The arena tracks +/// per-allocation handouts in a non-atomic local counter +/// (`local_shared_count`) and reconciles the surplus with a single +/// `fetch_sub(LARGE_SHARED_REF_SURPLUS - local_shared_count)` when +/// the chunk is retired (refill / reset / arena drop). +/// +/// While the chunk is current, the atomic refcount stays +/// `>= 1 + LARGE_SHARED_REF_SURPLUS - drops_observed`. Concurrent +/// `Arc::drop` on other threads can only ever subtract; with this +/// surplus picked at 2^30 the refcount cannot underflow below the +/// per-chunk u16-capped allocation count, and concurrent +/// `Arc::clone` cannot overflow either (u32 leaves ~2^30 headroom). +/// +/// Net cost of the per-allocation atomic disappears entirely; in +/// exchange we pay one extra atomic op per chunk install and one per +/// chunk retire. +const LARGE_SHARED_REF_SURPLUS: u32 = 1 << 30; + mod alloc_growable; pub(crate) mod alloc_prefixed; mod alloc_slice_arc; @@ -34,6 +53,9 @@ mod alloc_unsized; mod alloc_utf16; pub(crate) mod alloc_value; mod reserve; +mod retired_local; + +use retired_local::RetiredLocalChunks; /// A flexible bump allocator. /// @@ -74,11 +96,24 @@ pub struct Arena { /// `Arc`s independently hold the chunk's refcount. current_shared: CurrentChunk>, + /// Non-atomic count of strong references the arena has handed + /// out from `current_shared`'s chunk since installing it. The + /// chunk's atomic `ref_count` is pre-credited with + /// `LARGE_SHARED_REF_SURPLUS` at install so each handout can just + /// bump this counter — no atomic op in the per-allocation hot + /// path. At retire (refill / reset / arena drop) the surplus is + /// reconciled with a single + /// `fetch_sub(LARGE_SHARED_REF_SURPLUS - local_shared_count)`, + /// leaving the chunk's atomic count equal to the number of + /// escaped handles. Stays at `0` whenever `current_shared` holds + /// an empty mutator. + local_shared_count: Cell, + /// Local-chunk mutators whose chunk was rotated out while it might still /// have outstanding simple-ref `&mut T` borrows. Each retained mutator /// holds a `+1` on its chunk, keeping it alive (and preventing teardown /// / drop-replay) until the arena is reset or dropped. - retired_local: RefCell>>>, + retired_local: RetiredLocalChunks, /// Geometric-growth chunk-class hint for the next local refill: each /// successful refill bumps this toward the largest cacheable class so @@ -92,13 +127,6 @@ pub struct Arena { provider: StdArc>, - /// Running counter of user-requested bytes. Updated on every - /// successful allocation with the `Layout::size()` the caller - /// asked for; excludes alignment padding, drop-entry overhead, - /// and chunk headers. - #[cfg(feature = "stats")] - total_bytes_allocated: Cell, - /// Running count of buffer relocations (growable collections moved to /// a fresh, larger buffer because they could not grow in place). #[cfg(feature = "stats")] @@ -184,7 +212,7 @@ impl Arena { where A: 'static, { - expect_alloc(Self::try_from_config(allocator, crate::internal::constants::MAX_NORMAL_ALLOC, None)) + expect_alloc(Self::try_from_config(allocator, MAX_NORMAL_ALLOC, None)) } /// Fallible variant of [`Self::new_in`]. @@ -198,7 +226,7 @@ impl Arena { where A: 'static, { - Self::try_from_config(allocator, crate::internal::constants::MAX_NORMAL_ALLOC, None) + Self::try_from_config(allocator, MAX_NORMAL_ALLOC, None) } /// Internal builder entry point: assemble an `Arena` from a fully @@ -215,13 +243,12 @@ impl Arena { Ok(Self { current_local: CurrentChunk::new(ChunkMutator::>::empty()), current_shared: CurrentChunk::new(ChunkMutator::>::empty()), - retired_local: RefCell::new(Vec::new()), + local_shared_count: Cell::new(0), + retired_local: RetiredLocalChunks::new(), next_local_class: Cell::new(SizeClass::ZERO), next_shared_class: Cell::new(SizeClass::ZERO), provider, #[cfg(feature = "stats")] - total_bytes_allocated: Cell::new(0), - #[cfg(feature = "stats")] relocations: Cell::new(0), }) } @@ -264,8 +291,17 @@ impl Arena { #[inline] pub fn stats(&self) -> ArenaStats { let chunks = self.provider.chunk_alloc_stats(); + // `wasted_tail_bytes` is a live gauge over chunks that are + // currently *not* accepting allocations (retired, or held by + // outstanding handles). Fold in the currently-active local and + // shared chunks' free tails so the reported value also reflects + // the slack that would become wasted if the next alloc forced + // a refill right now. + let current_local_free = u64::from(self.current_local.borrow().wasted_tail_for_stats()); + let current_shared_free = u64::from(self.current_shared.borrow().wasted_tail_for_stats()); ArenaStats { - total_bytes_allocated: self.total_bytes_allocated.get(), + total_bytes_allocated: self.provider.bytes_outstanding(), + wasted_tail_bytes: self.provider.wasted_tail_bytes() + current_local_free + current_shared_free, normal_local_chunks_allocated: chunks.normal_local(), oversized_local_chunks_allocated: chunks.oversized_local(), normal_shared_chunks_allocated: chunks.normal_shared(), @@ -275,14 +311,6 @@ impl Arena { } } - /// Record a successful user allocation of `bytes` bytes. - #[cfg(feature = "stats")] - #[inline(always)] - pub(crate) fn record_alloc(&self, bytes: usize) { - let prev = self.total_bytes_allocated.get(); - self.total_bytes_allocated.set(prev + bytes as u64); - } - /// Record a buffer relocation (a growable collection moved to a fresh /// allocation because it could not grow in place). #[cfg(feature = "stats")] @@ -303,7 +331,12 @@ impl Arena { /// allocation, mirroring the lazy semantics of [`Self::new`]. #[cold] pub fn reset(&mut self) { - self.retired_local.borrow_mut().clear(); + // Reconcile the surplus on the current shared chunk before + // the mutator's Drop fires its own dec_ref — keeps the + // chunk's atomic refcount in sync with the number of escaped + // handles. + self.reconcile_shared_surplus(); + self.retired_local.clear(); *self.current_local.get_mut() = ChunkMutator::>::empty(); *self.current_shared.get_mut() = ChunkMutator::>::empty(); } @@ -404,7 +437,7 @@ impl Arena { // moved into the new ChunkMutator. let new_mutator = unsafe { ChunkMutator::>::from_owned(new_chunk) }; let old = self.current_local.replace(new_mutator); - self.retired_local.borrow_mut().push(old); + self.retired_local.push(old); self.next_local_class.set(self.next_local_class.get().saturating_inc()); Ok(()) } @@ -421,11 +454,38 @@ impl Arena { #[inline(never)] #[cfg_attr(test, mutants::skip)] // see `refill_local` pub(crate) fn refill_shared(&self, min_payload: usize) -> Result<(), AllocError> { + // Reconcile the surplus on the old chunk before its mutator's + // Drop fires its own dec_ref — keeps the chunk's atomic + // refcount equal to the number of escaped handles regardless + // of how many we pre-credited. + self.reconcile_shared_surplus(); // Release the exhausted current chunk's refcount *before* reserving // the replacement so a now-unreferenced chunk frees its bytes and // lets the new reservation reuse the budget. self.current_shared.drop_replace(ChunkMutator::>::empty()); + // The previous `drop_replace` may have run user-supplied drop + // shims (chunk teardown). Those can re-enter the arena via + // `alloc_arc`/`alloc_box` which call `refill_shared` + // recursively and install a fresh chunk into `current_shared`. + // Honor that installation as-is: returning `Ok` lets the + // caller's retry loop re-attempt the allocation against the + // reentry-installed chunk. If it doesn't fit `min_payload`, + // the caller will simply call us again and we'll reconcile + + // replace that chunk in turn (its own `local_shared_count` + // already tracks any nested handouts). + if self.current_shared.borrow().chunk_ptr().is_some() { + return Ok(()); + } let new_chunk = self.provider.acquire_shared(min_payload, self.next_shared_class.get())?; + // Pre-credit a large surplus of refs on the new chunk so the + // per-allocation hot path can just bump a non-atomic local + // counter — the surplus absorbs any concurrent Arc::drop on + // other threads (capacity-bounded handouts can't underflow + // it). + // SAFETY: we hold the +1 from `acquire_shared`; the chunk is + // live for the duration of this borrow. + unsafe { new_chunk.as_ref().pre_credit_refs(LARGE_SHARED_REF_SURPLUS as usize) }; + debug_assert_eq!(self.local_shared_count.get(), 0, "local_shared_count must be 0 after reconcile"); // SAFETY: `acquire_shared` returns a refcount-1 chunk. let new_mutator = unsafe { ChunkMutator::>::from_owned(new_chunk) }; self.current_shared.drop_replace(new_mutator); @@ -460,6 +520,34 @@ impl Arena { Ok(do_alloc(&mutator, chunk)) } + /// Closure-free variant of [`Self::alloc_oversized_shared_with`] for + /// hot callers whose `do_alloc` would otherwise capture a + /// user-provided `FnOnce`. See + /// [`Self::acquire_oversized_local_mutator`] for the per-iteration + /// spill rationale this avoids. + /// + /// Caller contract: the returned `mutator` owns the chunk's `+1` + /// strong reference. Perform the bump reservation, take any + /// smart-pointer `+1` via `acquire_shared_chunk_ref`, then drop the + /// mutator (it releases its `+1` automatically). If the smart- + /// pointer ref was taken, the chunk stays alive via that ref; + /// otherwise the chunk is torn down here. + #[cold] + #[inline(never)] + #[allow( + clippy::type_complexity, + reason = "Returning both the mutator and the chunk pointer keeps the cold helper closure-free" + )] + pub(crate) fn acquire_oversized_shared_mutator( + &self, + min_payload: usize, + ) -> Result<(ChunkMutator>, NonNull>), AllocError> { + let chunk = self.provider.acquire_oversized_shared(min_payload)?; + // SAFETY: `acquire_oversized_shared` returns a refcount-1 chunk. + let mutator = unsafe { ChunkMutator::>::from_owned(chunk) }; + Ok((mutator, chunk)) + } + /// Local mirror of [`Self::alloc_oversized_shared_with`]. The /// temporary mutator is pushed into `retired_local` on success so /// the chunk's `+1` strong reference is retained for the duration @@ -481,9 +569,106 @@ impl Arena { let mutator = unsafe { ChunkMutator::>::from_owned(chunk) }; let result = do_alloc(&mutator); // Retain the mutator (and its `+1`) for the `&Arena` lifetime. - self.retired_local.borrow_mut().push(mutator); + self.retired_local.push(mutator); Ok(result) } + + /// Closure-free variant of [`Self::alloc_oversized_local_with`] for + /// hot callers whose `do_alloc` would otherwise capture a + /// user-provided `FnOnce`. Capturing such a closure into the + /// `do_alloc` callback forces the user closure's environment + /// (e.g. `&loop_counter` for a default-by-ref capture) to live in + /// an addressable stack slot, which materializes as a per-iteration + /// spill on the hot path even when the oversized branch is never + /// taken. + /// + /// Caller contract: invoke the bump allocator on the returned + /// mutator, perform any value init, then call + /// [`Self::retain_oversized_local_mutator`] to transfer the + /// mutator's `+1` into `retired_local`. If anything between this + /// call and the `retain_*` call unwinds, the mutator is dropped + /// normally and the oversized chunk is torn down — same panic + /// semantics as the closure form. + #[cold] + #[inline(never)] + pub(crate) fn acquire_oversized_local_mutator(&self, min_payload: usize) -> Result>, AllocError> { + let chunk = self.provider.acquire_oversized_local(min_payload)?; + // SAFETY: `acquire_oversized_local` returns a refcount-1 chunk; + // the `+1` moves into the mutator. + Ok(unsafe { ChunkMutator::>::from_owned(chunk) }) + } + + /// Retains an oversized-local mutator obtained from + /// [`Self::acquire_oversized_local_mutator`] by pushing it into + /// `retired_local`. See that method for the full contract. + #[cold] + #[inline(never)] + pub(crate) fn retain_oversized_local_mutator(&self, mutator: ChunkMutator>) { + self.retired_local.push(mutator); + } + + /// Per-allocation hot path: acquires one strong reference on + /// `current_shared`'s chunk by bumping the non-atomic + /// `local_shared_count`. No atomic op: the surplus pre-credited + /// at chunk install absorbs the handout. + /// + /// `chunk_ptr` must be the chunk currently installed in + /// `current_shared`; callers obtain it from `try_reserve_shared*` + /// or `try_alloc_with_chunk` which return the chunk pointer + /// alongside the reservation. The arena's `+1` plus the + /// pre-credited surplus on that chunk make the adopt sound — the + /// chunk cannot be torn down while we hold any of the surplus. + #[expect(clippy::inline_always, reason = "hot-path entry; must inline fully for arena performance")] + #[inline(always)] + pub(crate) fn acquire_current_shared_chunk_ref(&self, chunk_ptr: NonNull>) -> ChunkRef { + // Wrap-protected: per-chunk handouts are bounded by the + // chunk's u16-capped allocation count, well below `u32::MAX`. + self.local_shared_count.set(self.local_shared_count.get().wrapping_add(1)); + // SAFETY: we hold (and are handing out) one of the + // pre-credited surplus refs on `chunk_ptr`. + unsafe { ChunkRef::::adopt(chunk_ptr) } + } + + /// Reconcile the pre-credited surplus on `current_shared`'s + /// chunk with the locally-tracked handout count. After this call + /// the chunk's atomic `ref_count` equals `1 + escaped_handles` + /// (the `+1` is the mutator's own reference, released when the + /// mutator drops). No-op when no chunk is currently installed. + #[inline] + fn reconcile_shared_surplus(&self) { + let local = self.local_shared_count.replace(0); + // `local_shared_count` is only ever non-zero while + // `current_shared` holds a real (non-empty) mutator whose + // chunk is the same one we pre-credited against. + let Some(chunk) = self.current_shared.borrow().chunk_ptr() else { + debug_assert_eq!(local, 0, "local_shared_count must be 0 when no shared chunk installed"); + return; + }; + // Subtract the unused portion of the surplus: we pre-credited + // `LARGE_SHARED_REF_SURPLUS` at install and handed out + // `local` refs, so `LARGE_SHARED_REF_SURPLUS - local` + // surplus refs remain unhanded-out and must be released. + let refund = LARGE_SHARED_REF_SURPLUS - local; + // SAFETY: arena holds the mutator's +1 plus the + // pre-credited surplus on this chunk; we own exactly + // `refund` previously-credited refs that were never handed + // out. + unsafe { chunk.as_ref().refund_refs(refund as usize) }; + } +} + +impl Drop for Arena { + fn drop(&mut self) { + // Reconcile any pre-credited surplus before the current + // shared mutator's Drop releases its own +1. Without this, + // the chunk's atomic refcount would still carry the surplus, + // preventing the chunk from reaching zero and being torn + // down even when no handles remain. + self.reconcile_shared_surplus(); + // Field drops (current_local, current_shared, retired_local, + // provider Arc) run after this returns and release all + // remaining +1s. + } } impl fmt::Debug for Arena { @@ -492,9 +677,10 @@ impl fmt::Debug for Arena { } } -// No explicit `Drop` impl: field drops (Cells/RefCells of mutators) release -// chunk refcounts, and the `Arc` releases the cache, which -// returns retained chunks to the backing allocator. +// `Drop` impl above refunds unused pre-credited shared refs; field +// drops (Cells/RefCells of mutators) then release chunk refcounts, +// and the `Arc` releases the cache, which returns +// retained chunks to the backing allocator. /// Convert a fallible alloc result to its `Ok` value, panicking on /// `Err` with the canonical multitude allocator-failure message. diff --git a/crates/multitude/src/arena/reserve.rs b/crates/multitude/src/arena/reserve.rs index e2d59f197..4a93d6f2f 100644 --- a/crates/multitude/src/arena/reserve.rs +++ b/crates/multitude/src/arena/reserve.rs @@ -68,6 +68,24 @@ impl Arena { Some(unsafe { ticket.rebind() }) } + /// Like [`Self::try_reserve_local_slice`] but takes the precomputed + /// byte size; the slice-copy/clone fast paths hold an existing + /// `&[T]` and compute `size_of_val(src)` once outside the refill + /// loop, sparing the inner reservation a `checked_mul` overflow + /// guard. + /// + /// # Safety + /// + /// `size` must equal `size_of::() * len` (without overflow). + #[inline(always)] + #[cfg_attr(test, mutants::skip)] // see `try_reserve_local` + pub(crate) fn try_reserve_local_slice_with_size(&self, len: usize, size: usize) -> Option> { + // SAFETY: forwarded to the caller. + let ticket = unsafe { self.current_local().try_alloc_uninit_slice_with_size::(len, size) }?; + // SAFETY: see module-level rationale. + Some(unsafe { ticket.rebind() }) + } + /// Try to reserve `len` consecutive bytes in the current local chunk. /// Byte-slice fast path that skips alignment math and overflow checks. #[inline(always)] @@ -134,6 +152,34 @@ impl Arena { Some(unsafe { (ticket.rebind(), mutator.chunk_ptr_unchecked()) }) } + /// Like [`Self::try_reserve_shared_slice`] but takes the precomputed + /// payload byte size; the slice-copy fast paths hold an existing + /// `&[T]` and compute `size_of_val(src)` once outside the refill + /// loop, sparing the inner reservation a `checked_mul` overflow + /// guard. + /// + /// # Safety + /// + /// `payload_bytes` must equal `size_of::() * len` (without + /// overflow). + #[inline(always)] + #[cfg_attr(test, mutants::skip)] // see `try_reserve_shared` + #[allow( + clippy::type_complexity, + reason = "ticket + chunk-ptr tuple is the natural shape; type alias would obscure rather than clarify" + )] + pub(crate) unsafe fn try_reserve_shared_slice_with_size( + &self, + len: usize, + payload_bytes: usize, + ) -> Option<(Uninit<'_, [T]>, NonNull>)> { + let mutator = self.current_shared(); + // SAFETY: forwarded to the caller. + let ticket = unsafe { mutator.try_alloc_uninit_slice_prefixed_with_size::(len, payload_bytes) }?; + // SAFETY: see `try_reserve_shared`. + Some(unsafe { (ticket.rebind(), mutator.chunk_ptr_unchecked()) }) + } + /// Try to reserve uninitialized storage for `len` consecutive `T`s /// plus a drop entry slot in the current shared chunk. Includes a /// thin-pointer DST length prefix immediately before the payload. diff --git a/crates/multitude/src/arena/retired_local.rs b/crates/multitude/src/arena/retired_local.rs new file mode 100644 index 000000000..fffcb5ed3 --- /dev/null +++ b/crates/multitude/src/arena/retired_local.rs @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Intrusive singly-linked list of retired local chunks. +//! +//! The arena keeps a LIFO list of [`LocalChunk`]s it has rotated out of +//! `current_local` but cannot yet recycle, because some outstanding +//! `&mut T` borrow (or [`crate::vec::Vec`] / [`crate::strings::String`] +//! handle) still references payload bytes in them. Each chunk on the +//! list logically owns one strong reference (refcount = 1) — the same +//! `+1` that the originally-retiring [`ChunkMutator`] held. +//! +//! Linkage is **intrusive**: each `LocalChunk` carries a +//! [`next`](LocalChunk) field used to thread chunks together +//! without per-retirement heap allocation. The list head holds a +//! thin `*mut u8` to the topmost chunk's header (`null` for empty); +//! the fat DST pointer is reconstructed via +//! [`LocalChunk::header_to_fat`] when the list is drained. +//! +//! Only two writers ever touch this structure: +//! * [`Arena::refill_local`](crate::Arena) pushes the rotated-out +//! mutator's chunk. +//! * [`Arena::reset`](crate::Arena), `Arena::drop`, and the +//! oversized-Vec-grow path drain (or splice from) the list. +//! +//! The drain is iterative *and* re-checks `head` on each pass so that +//! reentrant pushes performed by user-supplied chunk-teardown +//! destructors (drop shims invoked from `LocalChunk::teardown_and_release`) +//! never leak. + +use core::cell::Cell; +use core::marker::PhantomData; +use core::ptr::{self, NonNull}; + +use allocator_api2::alloc::Allocator; + +use crate::internal::chunk::Chunk; +use crate::internal::chunk_mutator::ChunkMutator; +use crate::internal::local_chunk::LocalChunk; + +/// LIFO list of retired local chunks linked through each chunk's +/// own [`next`](LocalChunk) field. +/// +/// `head` is a thin `*mut u8` header pointer (or `null`). Carrying +/// the metadata-less form keeps the head field 8 bytes and matches +/// the encoding the cache freelist uses elsewhere in the chunk +/// provider; the fat pointer is rebuilt via `header_to_fat` when +/// chunks are popped. +pub(crate) struct RetiredLocalChunks { + head: Cell<*mut u8>, + /// `LocalChunk` is only referenced via raw pointers from this + /// list; this marker propagates `A`'s `Send` / auto-traits to the + /// list type. + _marker: PhantomData>>, +} + +// SAFETY: when `ChunkMutator>` is `Send`, so is this +// list — every node logically holds the same `+1` a mutator does and +// is single-owner; moving the arena between threads moves every node +// with it. `!Sync` is inherited from `Cell`, matching the bound +// `Arena` needs. +unsafe impl Send for RetiredLocalChunks {} + +impl RetiredLocalChunks { + #[inline] + pub(crate) const fn new() -> Self { + Self { + head: Cell::new(ptr::null_mut()), + _marker: PhantomData, + } + } + + /// Retire `mutator`'s chunk by linking its header onto the list + /// head. Empty mutators are a no-op (nothing to retire). + /// + /// The mutator's `+1` becomes the list's; its `Drop` is bypassed + /// (via [`ChunkMutator::forget_into_chunk`]) so the chunk is *not* + /// torn down here. Teardown happens when the list is drained. + #[inline] + pub(crate) fn push(&self, mutator: ChunkMutator>) { + let Some(chunk) = mutator.forget_into_chunk() else { + return; + }; + // SAFETY: we just took ownership of the +1 on `chunk`, so it + // is live and uniquely owned. `set_next` is a single + // `Cell::replace` on the chunk's own header field. + unsafe { + let prev_head = self.head.replace(chunk.cast::().as_ptr()); + LocalChunk::set_next(chunk, prev_head); + } + } + + /// Drop every retained chunk. Iterative on two levels: + /// * The inner `while let` walks the chain one node at a time + /// (re-using the chunk's own `next` link to advance), so a + /// long list never overflows the stack via recursive `Drop`. + /// * The outer `loop` re-checks `self.head` after each chain + /// drain so reentrant pushes performed by chunk-teardown drop + /// shims are captured rather than leaked. + pub(crate) fn clear(&self) { + loop { + let mut cur = self.head.replace(ptr::null_mut()); + if cur.is_null() { + return; + } + while !cur.is_null() { + // SAFETY: while a node sits in this list we own its + // `+1`, so the chunk allocation is live. `header_to_fat` + // reads the header's `capacity` field through the + // live allocation. + unsafe { + let fat = LocalChunk::::header_to_fat(cur); + let chunk = NonNull::new_unchecked(fat); + // Detach the node *before* releasing its + // refcount: a panicking drop shim invoked from + // `teardown_and_release` must not see this chunk + // re-entrantly through the retired list. We also + // clear the field so the chunk, if recycled into + // the provider cache, starts in a clean state. + let next = LocalChunk::set_next(chunk, ptr::null_mut()); + Self::release_retired_chunk(chunk); + cur = next; + } + } + } + } + + /// Release one strong reference on a chunk that has been + /// unlinked from the list. The list logically holds exactly one + /// `+1` per node, so `dec_ref` always returns `true` here; we + /// fall through to `teardown_and_release` to run any drop shims + /// and route the chunk back to the cache (or the system + /// allocator). + /// + /// # Safety + /// + /// Caller must have just removed `chunk` from this list and must + /// not retain any further references to it. + #[inline] + unsafe fn release_retired_chunk(chunk: NonNull>) { + use crate::internal::chunk_ops::ChunkOps; + // SAFETY: caller hands over the list's `+1`; LocalChunk + // refcounts are bounded to 0/1, so this decrement always + // hits zero. + unsafe { + let chunk_ref = chunk.as_ref(); + let last = chunk_ref.dec_ref(); + debug_assert!(last, "retired LocalChunk refcount must be 1; dec_ref must hit zero"); + if last { + as ChunkOps>::teardown_and_release(chunk); + } + } + } +} + +impl Drop for RetiredLocalChunks { + fn drop(&mut self) { + self.clear(); + } +} diff --git a/crates/multitude/src/arena_stats.rs b/crates/multitude/src/arena_stats.rs index 08ab83aea..4960f40f7 100644 --- a/crates/multitude/src/arena_stats.rs +++ b/crates/multitude/src/arena_stats.rs @@ -3,8 +3,10 @@ /// Runtime statistics for an [`Arena`](crate::Arena). /// -/// All fields are lifetime counters: they accumulate over the life of -/// the arena and never decrease. A zero-cost snapshot is returned by +/// Most fields are lifetime counters that accumulate over the life of +/// the arena. The exceptions are [`total_bytes_allocated`](Self::total_bytes_allocated) +/// and [`wasted_tail_bytes`](Self::wasted_tail_bytes), which are *live* +/// gauges reflecting current state. A zero-cost snapshot is returned by /// [`Arena::stats`](crate::Arena::stats). /// /// The fields are `pub` because this is a value-semantic data type; the @@ -38,20 +40,35 @@ pub struct ArenaStats { /// definition of "oversized". pub oversized_shared_chunks_allocated: u64, - /// Sum of bytes requested by user allocations (i.e., the `size` - /// field of each successful allocation's `Layout`). + /// Total bytes currently held from the underlying allocator: the sum + /// of every chunk (header + payload) the arena owns right now — + /// active `current_*` chunks, retired chunks still kept alive (e.g. + /// by outstanding `Arc`/`Box` handles), and chunks parked in the + /// size-class cache. /// - /// Excludes internal chunk overhead such as headers, alignment padding, and - /// per-allocation drop-tracking metadata. + /// This is a **live gauge**, not a lifetime counter: it rises when a + /// chunk is allocated from the underlying allocator and falls when a + /// chunk is freed back to it. It includes internal chunk overhead + /// (headers and alignment padding), so it reflects real allocator + /// footprint rather than the sum of user-requested `Layout::size()` + /// bytes. pub total_bytes_allocated: u64, - /// Bytes "wasted" as unused tail space when a chunk was rotated out - /// — either by a follow-up allocation (refill) or by [`Arena::reset`](crate::Arena::reset) - /// retiring its currently-active chunks. + /// Bytes "wasted" as unused tail space (the free region between + /// the bump cursor and the drop-entry top), summed across every + /// chunk the arena currently holds — both the active + /// `current_local` / `current_shared` chunks and any chunks that + /// have been retired but not yet returned to the cache or freed + /// back to the underlying allocator (e.g., chunks held alive by + /// outstanding `Arc`/`Box` handles). /// - /// Does **not** include slack still in current chunks, slack at - /// chunk teardown (when an `Arc`/`Box` releases the chunk's - /// last refcount), or fragmentation inside a chunk (multiple + /// Bumped up when a chunk is retired from a current slot, bumped + /// back down when the same chunk is later released to the size- + /// class cache or returned to the underlying allocator. The + /// active-chunks contribution is computed on demand at + /// [`Arena::stats`](crate::Arena::stats) time. + /// + /// Does **not** include fragmentation inside a chunk (multiple /// allocations leaving gaps between them). pub wasted_tail_bytes: u64, diff --git a/crates/multitude/src/box.rs b/crates/multitude/src/box.rs index db25fc564..c8e1b099a 100644 --- a/crates/multitude/src/box.rs +++ b/crates/multitude/src/box.rs @@ -314,3 +314,12 @@ impl ExactSizeIte } impl FusedIterator for Box {} + +impl<'a, T, A: Allocator + Clone> From> for Box<[T], A> { + /// Freeze a [`Vec`](crate::vec::Vec) into an immutable + /// [`Box<[T], A>`](crate::Box). Mirrors `std`'s `From> for Box<[T]>`. + #[inline] + fn from(v: crate::vec::Vec<'a, T, A>) -> Self { + v.into_boxed_slice() + } +} diff --git a/crates/multitude/src/bytemuck.rs b/crates/multitude/src/bytemuck.rs index 0ee98c95f..46e913a9d 100644 --- a/crates/multitude/src/bytemuck.rs +++ b/crates/multitude/src/bytemuck.rs @@ -56,7 +56,7 @@ impl<'a, A: Allocator + Clone> BytemuckView<'a, A> { /// Panics if the backing allocator fails or if `T` requires alignment of 64 KiB or greater (which exceeds the arena chunk alignment). #[must_use] #[inline] - pub fn alloc(&self) -> &'a mut T { + pub fn alloc(&self) -> &'a mut T { self.arena .try_alloc_with::(T::zeroed) .expect("bytemuck: arena allocation failed") @@ -69,7 +69,7 @@ impl<'a, A: Allocator + Clone> BytemuckView<'a, A> { /// Returns [`AllocError`] if the backing allocator fails or if `T` requires alignment /// >= 64 KiB. #[inline] - pub fn try_alloc(&self) -> Result<&'a mut T, AllocError> { + pub fn try_alloc(&self) -> Result<&'a mut T, AllocError> { self.arena.try_alloc_with::(T::zeroed) } @@ -80,7 +80,7 @@ impl<'a, A: Allocator + Clone> BytemuckView<'a, A> { /// Panics if the backing allocator fails or if `T` requires alignment of 64 KiB or greater. #[must_use] #[inline] - pub fn alloc_slice(&self, len: usize) -> &'a mut [T] { + pub fn alloc_slice(&self, len: usize) -> &'a mut [T] { self.arena .try_alloc_slice_fill_with(len, |_| T::zeroed()) .expect("bytemuck: arena allocation failed") @@ -93,7 +93,7 @@ impl<'a, A: Allocator + Clone> BytemuckView<'a, A> { /// Returns [`AllocError`] if the backing allocator fails or if `T` requires alignment /// >= 64 KiB. #[inline] - pub fn try_alloc_slice(&self, len: usize) -> Result<&'a mut [T], AllocError> { + pub fn try_alloc_slice(&self, len: usize) -> Result<&'a mut [T], AllocError> { self.arena.try_alloc_slice_fill_with(len, |_| T::zeroed()) } diff --git a/crates/multitude/src/from_in.rs b/crates/multitude/src/from_in.rs new file mode 100644 index 000000000..3adcb150d --- /dev/null +++ b/crates/multitude/src/from_in.rs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Arena-aware analogs of [`From`] / [`Into`]. +//! +//! Several `std` `From` impls (e.g. `From<&[T]> for Vec`, +//! `From<&str> for String`) cannot be mirrored directly because they allocate +//! "from nothing" and thus require an allocator. [`FromIn`] is the arena-aware +//! counterpart: it threads the allocator (for our types, `&'a Arena`) +//! through, exactly as [`FromIteratorIn`](crate::vec::FromIteratorIn) does for +//! [`FromIterator`](core::iter::FromIterator). +//! +//! [`IntoIn`] is the [`Into`]-style companion, blanket-implemented for every +//! type via [`FromIn`]. +//! +//! ``` +//! use multitude::vec::Vec; +//! use multitude::{Arena, FromIn, IntoIn}; +//! +//! let arena = Arena::new(); +//! +//! // Via `FromIn`: +//! let v: Vec = Vec::from_in([1_u32, 2, 3], &arena); +//! assert_eq!(&*v, &[1, 2, 3]); +//! +//! // Via `IntoIn` (the target type drives inference): +//! let w: Vec = [4_u32, 5].into_in(&arena); +//! assert_eq!(&*w, &[4, 5]); +//! ``` + +/// Arena-aware counterpart to [`From`]: build `Self` from `value`, allocating +/// into `allocator`. +/// +/// The allocator is passed as a smart-pointer-shaped value (for our types, +/// `&'a Arena`). Use this for conversions that `std` exposes as `From` but +/// that need an allocator to materialize the result. +pub trait FromIn: Sized { + /// The allocator smart pointer this type needs in order to be built. + type Allocator; + + /// Build `Self` from `value`, allocating into `allocator`. + fn from_in(value: T, allocator: Self::Allocator) -> Self; +} + +/// Extension trait providing [`Into`]-style `.into_in(allocator)`, the +/// counterpart to [`FromIn`]. Blanket-implemented for every type; the target +/// collection `C` is usually pinned by a type annotation. +/// +/// ``` +/// use multitude::strings::String; +/// use multitude::{Arena, IntoIn}; +/// +/// let arena = Arena::new(); +/// let s: String = "hello".into_in(&arena); +/// assert_eq!(s.as_str(), "hello"); +/// ``` +pub trait IntoIn: Sized { + /// Convert `self` into `C`, allocating into `allocator`. + #[inline] + fn into_in>(self, allocator: C::Allocator) -> C { + C::from_in(self, allocator) + } +} + +impl IntoIn for T {} diff --git a/crates/multitude/src/internal/arena_buf.rs b/crates/multitude/src/internal/arena_buf.rs index fac2a6ef3..810854b28 100644 --- a/crates/multitude/src/internal/arena_buf.rs +++ b/crates/multitude/src/internal/arena_buf.rs @@ -38,6 +38,26 @@ pub(crate) struct ArenaBuf<'a, T> { _phantom: PhantomData<&'a mut [T]>, } +impl ArenaBuf<'_, T> { + /// Reconstruct a buffer from raw parts. + /// + /// # Safety + /// + /// The `(ptr, len, cap)` triple must satisfy the type's invariants for + /// some live arena chunk that outlives `'a` — e.g. parts taken from + /// another `ArenaBuf` (possibly reinterpreted, as in + /// [`Vec::into_flattened`](crate::vec::Vec::into_flattened)). + #[inline] + pub(crate) const unsafe fn from_raw_parts(ptr: NonNull, len: usize, cap: usize) -> Self { + Self { + ptr, + len, + cap, + _phantom: PhantomData, + } + } +} + impl<'a, T> ArenaBuf<'a, T> { /// Creates an empty buffer with no backing storage. ZSTs are /// initialized with `cap = usize::MAX` since no real storage is @@ -187,13 +207,17 @@ impl<'a, T> ArenaBuf<'a, T> { T: Copy, { debug_assert!(self.remaining_cap() >= src.len(), "extend_copy: insufficient capacity"); - if src.is_empty() { - return; - } // SAFETY: the tail `[len .. len + src.len()]` is in-bounds uninit // storage (invariant + caller-checked capacity); `src` is a // caller-supplied slice and cannot alias the freshly-reserved - // chunk storage. `T: Copy` permits bitwise duplication. + // chunk storage. `T: Copy` permits bitwise duplication. When + // `src.len() > 0`, the caller-checked `remaining_cap() >= src.len()` + // forces `cap > 0`, so the destination is a live, `T`-aligned + // allocation. When `src.len() == 0` the call is a no-op for which + // `copy_nonoverlapping` only requires both pointers to be non-null + // and `T`-aligned — satisfied because `self.ptr` is a `NonNull` + // that `ArenaBuf` keeps `T`-aligned even while dangling (`cap == 0` + // / ZST), and `src.as_ptr()` is likewise non-null and aligned. unsafe { ptr::copy_nonoverlapping(src.as_ptr(), self.ptr.as_ptr().add(self.len), src.len()); } @@ -385,6 +409,21 @@ impl<'a, T> ArenaBuf<'a, T> { self.cap = new_cap; } + /// Returns the spare capacity `[len, cap)` as a mutable slice of + /// `MaybeUninit`. + #[inline] + pub(crate) fn spare_capacity_mut(&mut self) -> &mut [mem::MaybeUninit] { + let spare = self.cap - self.len; + // SAFETY: by the invariants, `ptr + len` addresses `cap - len` + // uninitialized `T` slots; `MaybeUninit` has the same layout as + // `T`, and the `&mut self` borrow grants exclusive access. For ZSTs + // the dangling-but-aligned pointer is valid for any length. + unsafe { + let ptr = self.ptr.as_ptr().add(self.len).cast::>(); + slice::from_raw_parts_mut(ptr, spare) + } + } + /// Returns an owning, double-ended iterator that yields the live /// elements in order, leaving the buffer empty. The iterator's /// `Drop` drops any elements that were not yielded. The iterator diff --git a/crates/multitude/src/internal/chunk_alloc.rs b/crates/multitude/src/internal/chunk_alloc.rs index aa6a11299..404e639a1 100644 --- a/crates/multitude/src/internal/chunk_alloc.rs +++ b/crates/multitude/src/internal/chunk_alloc.rs @@ -11,35 +11,70 @@ use core::ptr::NonNull; use allocator_api2::alloc::{AllocError, Allocator}; -/// Allocate a `header + payload_size` byte allocation aligned to -/// `header_align`. +/// Computes the canonical `Layout` for a chunk allocation, the single +/// source of truth shared by every `allocate`/`destroy` pair so the two +/// can never disagree (a mismatched `deallocate` layout is UB). +/// +/// Two *distinct* alignments are at play and must not be conflated: +/// +/// * `value_align` — the chunk type's own alignment (`align_of::()`, +/// ignoring the `[UnsafeCell]` tail which is align-1). Rust rounds +/// the size of any value up to a multiple of its alignment, so a +/// reference built from the fat pointer covers `round_up(total, +/// value_align)` bytes. The allocation's **size** is rounded up to this +/// so the reference's footprint matches the allocation exactly (a +/// shortfall is UB, caught by Miri; an excess silently over-allocates). +/// +/// * `base_align` — the alignment of the allocation's **base address**, +/// which may be far larger than `value_align` (e.g. `CHUNK_ALIGN` = +/// 64 KiB for shared chunks, so the chunk header is recoverable by +/// masking the low bits of any interior pointer). This governs only the +/// `Layout` alignment; the **size is never rounded up to it**, otherwise +/// every shared chunk would inflate to a full `CHUNK_ALIGN`. +/// +/// `base_align >= value_align` and both must be powers of two. +#[allow( + clippy::map_err_ignore, + reason = "LayoutError carries no payload; only the AllocError variant matters" +)] +#[inline] +pub(crate) fn chunk_layout(header_size: usize, payload_size: usize, value_align: usize, base_align: usize) -> Result { + debug_assert!(value_align.is_power_of_two(), "value_align must be a power of two"); + debug_assert!(base_align.is_power_of_two(), "base_align must be a power of two"); + debug_assert!(base_align >= value_align, "base_align must be >= value_align"); + let total = header_size.checked_add(payload_size).ok_or(AllocError)?; + // Round the *size* up to the value alignment (not the base alignment). + let mask = value_align - 1; + let rounded = total.checked_add(mask).ok_or(AllocError)? & !mask; + Layout::from_size_align(rounded, base_align).map_err(|_| AllocError) +} + +/// Allocate a `header + payload_size` byte allocation whose base address +/// is `base_align`-aligned and whose size is rounded up to `value_align` +/// (see [`chunk_layout`]). /// /// Returns `(raw_u8_ptr, layout)` on success. The pointer carries /// provenance over the full allocation and is suitable as the data field /// of a slice-DST fat pointer with metadata `payload_size`. The layout is /// the exact one passed to `allocator.allocate`, suitable for a matching -/// `deallocate` call. +/// `deallocate` call (reproduced by [`chunk_layout`] at destroy time). /// /// On size-overflow or end-of-address-space overflow, the allocation is /// freed and `AllocError` is returned. -#[allow( - clippy::map_err_ignore, - reason = "LayoutError carries no payload; only the AllocError variant matters" -)] #[cfg_attr(test, mutants::skip)] #[inline] pub(crate) fn alloc_chunk_raw( allocator: &A, header_size: usize, - header_align: usize, payload_size: usize, + value_align: usize, + base_align: usize, ) -> Result<(*mut u8, Layout), AllocError> { - let total = header_size.checked_add(payload_size).ok_or(AllocError)?; - let layout = Layout::from_size_align(total, header_align).map_err(|_| AllocError)?; + let layout = chunk_layout(header_size, payload_size, value_align, base_align)?; let raw = allocator.allocate(layout)?; let raw_u8_ptr: *mut u8 = raw.cast::().as_ptr(); let start_addr = raw_u8_ptr as usize; - let end_addr = start_addr.checked_add(total).ok_or(AllocError)?; + let end_addr = start_addr.checked_add(layout.size()).ok_or(AllocError)?; if end_addr > isize::MAX as usize { // SAFETY: matches the `allocator.allocate` pair; nothing has // been stored in the allocation yet. @@ -50,3 +85,48 @@ pub(crate) fn alloc_chunk_raw( } Ok((raw_u8_ptr, layout)) } + +#[cfg(test)] +mod tests { + use super::chunk_layout; + + /// `chunk_layout` must round the allocation *size* up to `value_align`. + /// Pins the exact round-up so the `value_align - 1` mask can't be + /// mutated to `value_align + 1` or `value_align / 1` — both corrupt the + /// rounding for totals that aren't already `value_align`-aligned (the + /// size-class tests use pre-aligned totals, so they can't catch this). + #[test] + fn rounds_size_up_to_value_align() { + // A large power-of-two base (mirrors shared chunks); it governs the + // layout *alignment* only and must not affect the size rounding. + const BASE: usize = 65_536; + // (header, payload, value_align, expected_size). Totals are chosen + // to be NON-multiples of `value_align` so the mask actually rounds. + let cases = [ + (10_usize, 7_usize, 8_usize, 24_usize), // total 17 -> 24 + (34, 16, 8, 56), // total 50 -> 56 + (1, 0, 8, 8), // total 1 -> 8 + (10, 7, 16, 32), // total 17 -> 32 + (5, 0, 4, 8), // total 5 -> 8 + ]; + for (header, payload, value_align, expected) in cases { + let layout = chunk_layout(header, payload, value_align, BASE).expect("layout fits"); + assert_eq!( + layout.size(), + expected, + "round_up({header}+{payload}, {value_align}) must be {expected}" + ); + assert_eq!(layout.size() % value_align, 0, "size must be a multiple of value_align"); + assert_eq!(layout.align(), BASE, "alignment must be the base alignment"); + } + } + + /// An already-`value_align`-aligned total is returned unchanged (the + /// round-up is a no-op). + #[test] + fn aligned_total_is_unchanged() { + const BASE: usize = 65_536; + let layout = chunk_layout(8, 8, 8, BASE).expect("layout fits"); // total 16 + assert_eq!(layout.size(), 16); + } +} diff --git a/crates/multitude/src/internal/chunk_mutator.rs b/crates/multitude/src/internal/chunk_mutator.rs index f828a97e2..1e8233b03 100644 --- a/crates/multitude/src/internal/chunk_mutator.rs +++ b/crates/multitude/src/internal/chunk_mutator.rs @@ -66,14 +66,12 @@ impl ChunkMutator { /// incremented in this mutator's name. The mutator becomes the unique /// owner of that +1 and will release it on drop. pub(crate) unsafe fn from_owned(chunk: NonNull) -> Self { - // SAFETY: caller asserts `chunk` is live; payload_ptr and capacity - // are accessible while the chunk is live. - let (start, cap) = unsafe { (C::payload_ptr(chunk), chunk.as_ref().capacity()) }; - let start_addr = start.as_ptr() as usize; - // Align the drop-top down to `align_of::()`. - let entry_align = mem::align_of::(); - let aligned_end_addr = (start_addr + cap) & !(entry_align - 1); - let aligned_end_offset = aligned_end_addr - start_addr; + // SAFETY: caller asserts `chunk` is live; `payload_range_for` only + // dereferences `chunk` to read `payload_ptr()` and `capacity()`. + let (start_addr, end_addr) = unsafe { Self::payload_range_for(chunk) }; + // SAFETY: caller asserts `chunk` is live. + let start = unsafe { C::payload_ptr(chunk) }; + let aligned_end_offset = end_addr - start_addr; // SAFETY: `aligned_end_offset <= cap`; `start.byte_add` lands // within (or at one-past-end of) the chunk payload. let drop_top = unsafe { start.byte_add(aligned_end_offset) }; @@ -111,6 +109,21 @@ impl ChunkMutator { top.saturating_sub(cur) } + /// Free byte count between the bump cursor and the drop-entry top. + /// Used by stats accounting at retire (`ChunkMutator::Drop` and + /// `ChunkMutator::forget_into_chunk`) and by `Arena::stats` to fold + /// the currently-active chunks' unused tails into + /// `ArenaStats::wasted_tail_bytes`. The empty-mutator sentinel + /// returns 0 (saturating). The value is reported as `u32` since + /// chunk capacity is bounded well below `u32::MAX`. + #[cfg(feature = "stats")] + #[inline] + pub(crate) fn wasted_tail_for_stats(&self) -> u32 { + let top = self.drop_top.get().as_ptr() as usize; + let cur = self.bump.get().as_ptr() as usize; + u32::try_from(top.saturating_sub(cur)).unwrap_or(u32::MAX) + } + /// Reads the chunk's payload start and end addresses. /// /// # Panics @@ -123,10 +136,23 @@ impl ChunkMutator { let chunk = self.chunk.expect("payload_range: chunk must be set"); // SAFETY: mutator owns one strong reference to `chunk` for as // long as `&self` is borrowed. + unsafe { Self::payload_range_for(chunk) } + } + + /// Reads `chunk`'s payload start address and drop-region-aligned end + /// address. + /// + /// # Safety + /// + /// `chunk` must reference a live chunk. + #[inline] + unsafe fn payload_range_for(chunk: NonNull) -> (usize, usize) { + // SAFETY: caller asserts `chunk` is live. let (start, cap) = unsafe { (C::payload_ptr(chunk), chunk.as_ref().capacity()) }; let start_addr = start.as_ptr() as usize; - // Align the reported end down to match `from_owned`'s drop_top - // alignment, keeping drop-entry math consistent. + // Align the reported end down to `align_of::()` so the + // drop-entry region (entries grow down from this point) stays + // naturally aligned regardless of payload-start alignment. let entry_align = mem::align_of::(); let end_addr = (start_addr + cap) & !(entry_align - 1); (start_addr, end_addr) @@ -147,6 +173,13 @@ impl ChunkMutator { unsafe { self.chunk.unwrap_unchecked() } } + /// Returns the chunk this mutator owns, or `None` for the empty + /// (sentinel) mutator that has no chunk installed. + #[inline] + pub(crate) fn chunk_ptr(&self) -> Option> { + self.chunk + } + /// Reserves `size` bytes aligned to `align`. Returns an `InChunk` /// pointing at the start, or `None` if the chunk has insufficient /// room or the request would overflow. @@ -269,6 +302,23 @@ impl ChunkMutator { Some(Uninit::new(bytes.into_slice::(len))) } + /// Like [`Self::try_alloc_uninit_slice`] but takes the precomputed + /// byte size, skipping the `size_of::().checked_mul(len)` + /// overflow guard. + /// + /// # Safety + /// + /// `size` must equal `size_of::() * len` (without overflow). + /// Callers holding an existing `&[T]` satisfy this trivially via + /// [`core::mem::size_of_val`] (which is an unchecked intrinsic + /// guaranteed not to overflow for any live slice). + #[cfg_attr(test, mutants::skip)] // see `try_alloc`: body→None ⇒ refill spin + pub(crate) unsafe fn try_alloc_uninit_slice_with_size(&self, len: usize, size: usize) -> Option> { + debug_assert_eq!(size, mem::size_of::().wrapping_mul(len)); + let bytes = self.try_alloc(size, mem::align_of::())?; + Some(Uninit::new(bytes.into_slice::(len))) + } + /// Like [`Self::try_alloc_uninit_slice`] but reserves /// `size_of::()` extra bytes immediately before the payload /// for a thin-pointer DST length prefix, and writes `len` into that @@ -288,7 +338,56 @@ impl ChunkMutator { reason = "prefix slot may be unaligned for T's whose align < align_of::(); paired with write_unaligned/read_unaligned" )] pub(crate) fn try_alloc_uninit_slice_prefixed(&self, len: usize) -> Option> { - let elem_size = mem::size_of::(); + let (payload, _) = self.try_alloc_prefixed_slice_payload::(len)?; + Some(Uninit::new(payload)) + } + + /// Like [`Self::try_alloc_uninit_slice_prefixed`] but takes the + /// precomputed payload byte size, skipping the + /// `size_of::().checked_mul(len)` overflow guard. + /// + /// # Safety + /// + /// `payload_bytes` must equal `size_of::() * len` (without + /// overflow). Callers holding an existing `&[T]` satisfy this via + /// [`core::mem::size_of_val`]. + #[cfg_attr(test, mutants::skip)] // see `try_alloc` + #[allow( + clippy::cast_ptr_alignment, + reason = "prefix slot may be unaligned for T's whose align < align_of::(); paired with write_unaligned/read_unaligned" + )] + pub(crate) unsafe fn try_alloc_uninit_slice_prefixed_with_size(&self, len: usize, payload_bytes: usize) -> Option> { + // SAFETY: forwarded to the caller. + let (payload, _) = unsafe { self.try_alloc_prefixed_slice_payload_unchecked::(len, payload_bytes) }?; + Some(Uninit::new(payload)) + } + + /// Layout + alloc + prefix-write for "thin DST slice" reservations. + /// On success returns the payload ticket and the absolute payload + /// address (used by drop-tracked callers to encode `value_offset`). + #[inline] + #[cfg_attr(test, mutants::skip)] // see `try_alloc` + fn try_alloc_prefixed_slice_payload(&self, len: usize) -> Option<(InChunk<[T]>, usize)> { + let payload_bytes = mem::size_of::().checked_mul(len)?; + // SAFETY: just verified by `checked_mul`. + unsafe { self.try_alloc_prefixed_slice_payload_unchecked::(len, payload_bytes) } + } + + /// Inner helper for the prefixed slice path with a caller-provided + /// `payload_bytes`. + /// + /// # Safety + /// + /// `payload_bytes` must equal `size_of::() * len` (without + /// overflow). + #[inline] + #[cfg_attr(test, mutants::skip)] // see `try_alloc` + #[allow( + clippy::cast_ptr_alignment, + reason = "prefix slot may be unaligned for T's whose align < align_of::(); paired with write_unaligned/read_unaligned" + )] + unsafe fn try_alloc_prefixed_slice_payload_unchecked(&self, len: usize, payload_bytes: usize) -> Option<(InChunk<[T]>, usize)> { + debug_assert_eq!(payload_bytes, mem::size_of::().wrapping_mul(len)); let elem_align = mem::align_of::(); let prefix_size = mem::size_of::(); // Payload starts at the lowest elem-align-aligned offset >= @@ -300,7 +399,7 @@ impl ChunkMutator { // this, an empty slice (`len == 0` or ZST element) at the chunk // tail could return a payload pointer at `chunk_base + // CHUNK_ALIGN`, masking to the wrong tile on smart-pointer Drop. - let payload_bytes = elem_size.checked_mul(len)?.max(1); + let payload_bytes = payload_bytes.max(1); let total = payload_offset.checked_add(payload_bytes)?; let base_in_chunk = self.try_alloc(total, elem_align.max(1))?; // SAFETY: `base + payload_offset` is elem-align-aligned (both @@ -315,7 +414,7 @@ impl ChunkMutator { ptr::write_unaligned(prefix_ptr, len); let payload_nn = NonNull::new_unchecked(payload_ptr); let payload_in_chunk = InChunk::from_raw(payload_nn).into_slice::(len); - Some(Uninit::new(payload_in_chunk)) + Some((payload_in_chunk, payload_ptr as usize)) } } @@ -379,35 +478,16 @@ impl ChunkMutator { reason = "prefix slot may be unaligned for T's whose align < align_of::(); paired with write_unaligned/read_unaligned" )] pub(crate) fn try_alloc_uninit_slice_with_drop_prefixed(&self, len: usize) -> Option> { - let elem_size = mem::size_of::(); - let elem_align = mem::align_of::(); - let prefix_size = mem::size_of::(); - let payload_offset = prefix_size.max(elem_align); - // See `try_alloc_uninit_slice_prefixed`: floor the payload byte - // count to 1 so the returned payload pointer is strictly inside - // the reservation, never landing at `chunk_base + CHUNK_ALIGN`. - let payload_bytes = elem_size.checked_mul(len)?.max(1); - let total = payload_offset.checked_add(payload_bytes)?; // `len` must fit in the drop entry's `u16` element-count field. let len_u16 = u16::try_from(len).ok()?; let drop_slot = self.try_reserve_drop_entry()?; - let Some(base_in_chunk) = self.try_alloc(total, elem_align.max(1)) else { + let Some((value, payload_addr)) = self.try_alloc_prefixed_slice_payload::(len) else { self.unwind_drop_entry(); return None; }; - // SAFETY: see `try_alloc_uninit_slice_prefixed`. The drop - // entry's `value_offset` encodes the *payload* address + // The drop entry's `value_offset` encodes the *payload* address // (post-prefix) so `replay_drops` runs `drop_in_place::<[T]>` // on the real elements. - let value = unsafe { - let base_ptr = base_in_chunk.as_ptr(); - let payload_ptr = base_ptr.add(payload_offset); - let prefix_ptr = payload_ptr.sub(prefix_size).cast::(); - ptr::write_unaligned(prefix_ptr, len); - let payload_nn = NonNull::new_unchecked(payload_ptr); - InChunk::from_raw(payload_nn).into_slice::(len) - }; - let payload_addr = (base_in_chunk.as_ptr() as usize) + payload_offset; let value_offset = self.offset_or_unwind(payload_addr)?; // SAFETY: `drop_slot` is freshly reserved, aligned, exclusively // owned slot in the chunk's drop region. @@ -554,7 +634,7 @@ impl ChunkMutator { } /// Cold helper: roll back the most recently reserved drop entry and - /// return `None`. Extracted from compound-reservation paths so the + /// return `None`. Out-of-line from compound-reservation paths so the /// genuinely-unreachable `u16::try_from(...) == Err` arm is a single /// line at the call site. #[cold] @@ -609,6 +689,36 @@ impl ChunkMutator { chunk.as_ref().set_drop_entry_count(self.local_drop_entry_count()); } } + + /// Consumes the mutator, publishing the locally-tracked drop-entry + /// count to the chunk header and returning the chunk pointer with + /// the mutator's `+1` retained ownership transferred to the caller. + /// The mutator's `Drop` (which would otherwise release the `+1`) is + /// bypassed. + /// + /// Under the `stats` feature, this is also a "retire" event for + /// wasted-tail accounting: the chunk's free tail is recorded and + /// added to the provider's wasted-tail counter (the matching subtract + /// happens in `release_*` when the chunk is eventually cached or + /// destroyed). This matters for the `retired_local` push path, where + /// the chunk is removed from `current_local` (so its tail is wasted + /// from the user's POV) but the mutator's `Drop` is bypassed. + /// + /// Returns `None` for the empty (sentinel) mutator that has no + /// chunk installed. + #[inline] + pub(crate) fn forget_into_chunk(self) -> Option> { + self.publish_drop_count(); + let chunk = self.chunk; + #[cfg(feature = "stats")] + if let Some(chunk) = chunk { + // SAFETY: chunk is live; the mutator still holds its +1 + // (ownership transfers to the caller via `mem::forget` below). + unsafe { C::record_retire(chunk, self.wasted_tail_for_stats()) }; + } + mem::forget(self); + chunk + } } impl Drop for ChunkMutator { @@ -622,6 +732,16 @@ impl Drop for ChunkMutator { // unique remaining reference, and `teardown_and_release` will read // the count to walk the drop list. unsafe { + #[cfg(feature = "stats")] + { + // Record the wasted-tail at retire BEFORE dec_ref so that + // (a) the chunk header carries the value for the eventual + // `release_*` subtract (handles may outlive us), and (b) the + // provider counter goes up before any potential immediate + // release-driven subtract. + let wasted = self.wasted_tail_for_stats(); + C::record_retire(chunk, wasted); + } let chunk_ref = chunk.as_ref(); chunk_ref.set_drop_entry_count(self.local_drop_entry_count()); if chunk_ref.dec_ref() { diff --git a/crates/multitude/src/internal/chunk_ops.rs b/crates/multitude/src/internal/chunk_ops.rs index 7ab4fce5e..61e8c88be 100644 --- a/crates/multitude/src/internal/chunk_ops.rs +++ b/crates/multitude/src/internal/chunk_ops.rs @@ -47,6 +47,20 @@ pub(crate) trait ChunkOps: Chunk { /// /// Caller must hold the unique remaining reference to `chunk`. unsafe fn teardown_and_release(chunk: NonNull); + + /// Stashes `wasted` on the chunk header and adds it to the provider's + /// wasted-tail counter. Called from `ChunkMutator::Drop` at retire-time + /// (i.e., as the mutator's `+1` is about to be released). The matching + /// subtract happens in [`ChunkProvider::release_local`] / + /// [`ChunkProvider::release_shared`] when the chunk is later cached + /// or destroyed. + /// + /// # Safety + /// + /// `chunk` must reference a live chunk. Caller (the mutator) holds at + /// least one reference. + #[cfg(feature = "stats")] + unsafe fn record_retire(chunk: NonNull, wasted: u32); } #[allow( @@ -90,6 +104,15 @@ impl ChunkOps for LocalChunk { debug_assert!(!provider.is_null(), "local-chunk provider back-pointer is null in teardown"); (*provider).release_local(chunk); } + + #[cfg(feature = "stats")] + unsafe fn record_retire(chunk: NonNull, wasted: u32) { + let chunk_ref = &*chunk.as_ptr(); + chunk_ref.set_wasted_at_retire(wasted); + let provider = chunk_ref.provider(); + debug_assert!(!provider.is_null(), "local-chunk provider back-pointer is null at retire"); + (*provider).record_wasted_tail(u64::from(wasted)); + } } #[allow( @@ -128,6 +151,18 @@ impl ChunkOps for SharedChunk { SharedChunk::destroy(chunk); } } + + #[cfg(feature = "stats")] + unsafe fn record_retire(chunk: NonNull, wasted: u32) { + let chunk_ref = &*chunk.as_ptr(); + chunk_ref.set_wasted_at_retire(wasted); + // If the provider has already been dropped (shared chunks can + // outlive their arena), there is no counter left to update; + // the stashed `wasted_at_retire` will simply never be read. + if let Some(provider) = chunk_ref.provider().upgrade() { + provider.record_wasted_tail(u64::from(wasted)); + } + } } // Note: the prior `orphan_local_chunk_is_destroyed_on_mutator_drop` test diff --git a/crates/multitude/src/internal/chunk_provider.rs b/crates/multitude/src/internal/chunk_provider.rs index 3ba8be79b..e57c7de1a 100644 --- a/crates/multitude/src/internal/chunk_provider.rs +++ b/crates/multitude/src/internal/chunk_provider.rs @@ -135,8 +135,8 @@ pub(crate) struct ChunkProvider { /// `AcqRel` speculative-add. bytes_outstanding: AtomicUsize, /// Single-thread local-chunk cache: thin `*mut u8` header pointer to - /// the freelist head (chunks linked via [`LocalChunk::write_cached_next`] - /// / [`LocalChunk::read_cached_next`]). Holds at most one freelist for + /// the freelist head (chunks linked via [`LocalChunk::set_next`] + /// / [`LocalChunk::next`]). Holds at most one freelist for /// the **current class floor** ([`Self::local_cache_class`]); chunks /// below the floor are destroyed instead of cached. local_cache: OwnerThreadCell<*mut u8>, @@ -166,6 +166,13 @@ pub(crate) struct ChunkProvider { /// Lifetime count of oversized one-shot shared chunks allocated. #[cfg(feature = "stats")] oversized_shared_chunks_allocated: AtomicU64, + /// Bytes currently "wasted" in the unused free region of chunks that have + /// been retired from an arena's `current_*` slot but have not yet been + /// returned to the cache or freed back to the underlying allocator. Bumped + /// up when a chunk is retired, bumped back down when the same chunk is + /// later cached or destroyed. + #[cfg(feature = "stats")] + wasted_tail_bytes: AtomicU64, } // SAFETY: `local_cache` is `OwnerThreadCell`, which exposes only `unsafe` @@ -201,6 +208,8 @@ impl ChunkProvider { normal_shared_chunks_allocated: AtomicU64::new(0), #[cfg(feature = "stats")] oversized_shared_chunks_allocated: AtomicU64::new(0), + #[cfg(feature = "stats")] + wasted_tail_bytes: AtomicU64::new(0), }) } @@ -215,6 +224,39 @@ impl ChunkProvider { } } + /// Total bytes currently outstanding from the underlying allocator: the + /// sum of every chunk (header + payload) that has been allocated and not + /// yet freed. Chunks released back to the size-class cache stay counted; + /// only chunks returned to the underlying allocator (cache evictions, + /// oversized one-shots dropped, drain-on-provider-drop) decrement. + #[cfg(feature = "stats")] + pub(crate) fn bytes_outstanding(&self) -> u64 { + self.bytes_outstanding.load(Ordering::Relaxed) as u64 + } + + /// Currently "wasted" tail bytes (free region between bump cursor and + /// drop-entry top) across chunks that have been retired from a current + /// `ChunkMutator` slot but have not yet been returned to the cache or + /// freed back to the underlying allocator. + #[cfg(feature = "stats")] + pub(crate) fn wasted_tail_bytes(&self) -> u64 { + self.wasted_tail_bytes.load(Ordering::Relaxed) + } + + /// Adds `n` to the wasted-tail-bytes counter. Called when a chunk is + /// retired from a current `ChunkMutator` slot. + #[cfg(feature = "stats")] + pub(crate) fn record_wasted_tail(&self, n: u64) { + self.wasted_tail_bytes.fetch_add(n, Ordering::Relaxed); + } + + /// Subtracts `n` from the wasted-tail-bytes counter. Called when a + /// retired chunk is later cached or destroyed. + #[cfg(feature = "stats")] + pub(crate) fn release_wasted_tail(&self, n: u64) { + self.wasted_tail_bytes.fetch_sub(n, Ordering::Relaxed); + } + /// Returns the provider's configuration. #[inline] #[cfg_attr(test, mutants::skip)] // Default::default mutation observably equivalent for reachable inputs @@ -278,7 +320,7 @@ impl ChunkProvider { } else { let fat = LocalChunk::::header_to_fat(cur); let head_nn = NonNull::new_unchecked(fat); - *head = LocalChunk::read_cached_next(head_nn); + *head = LocalChunk::next(head_nn); LocalChunk::reinit_for_acquire(head_nn); Some(head_nn) } @@ -312,10 +354,10 @@ impl ChunkProvider { while !cur.is_null() { let fat = LocalChunk::::header_to_fat(cur); let chunk_nn = NonNull::new_unchecked(fat); - let next = LocalChunk::read_cached_next(chunk_nn); + let next = LocalChunk::next(chunk_nn); let total = LocalChunk::::header_size() + (*chunk_nn.as_ptr()).capacity(); if total >= new_min_total { - LocalChunk::write_cached_next(chunk_nn, new_head); + LocalChunk::set_next(chunk_nn, new_head); new_head = cur; } else { LocalChunk::destroy(chunk_nn, &self.allocator); @@ -470,6 +512,16 @@ impl ChunkProvider { // writing its cache-link slot. let capacity = (*chunk.as_ptr()).capacity(); let total = LocalChunk::::header_size() + capacity; + #[cfg(feature = "stats")] + { + // Decrement the wasted-tail counter by the value stashed on + // the chunk header at retire time (0 for chunks that never + // went through a mutator, e.g. preallocated cache fills). + let wasted = u64::from((*chunk.as_ptr()).wasted_at_retire()); + if wasted != 0 { + self.release_wasted_tail(wasted); + } + } // Bypass the cache for non-class-size totals (oversized one-shots // whose total isn't a power of two) and for chunks below the // current cache class floor. The floor ratchets monotonically as @@ -481,7 +533,7 @@ impl ChunkProvider { return; } self.local_cache.with(|head| { - LocalChunk::write_cached_next(chunk, *head); + LocalChunk::set_next(chunk, *head); *head = chunk.cast::().as_ptr(); }); } @@ -495,6 +547,16 @@ impl ChunkProvider { // SAFETY: chunk is live and uniquely owned by caller. let capacity = (*chunk.as_ptr()).capacity(); let total = SharedChunk::::header_size() + capacity; + #[cfg(feature = "stats")] + { + // See `release_local` for the symmetric subtract semantics. + // Acquire load on the shared chunk's atomic — the store may + // have happened on a different thread (last `Arc::drop`). + let wasted = u64::from((*chunk.as_ptr()).wasted_at_retire()); + if wasted != 0 { + self.release_wasted_tail(wasted); + } + } // See `release_local` for the cache-bypass conditions. if !is_cacheable_size(total) || total < SizeClass::new(self.shared_cache_class.load(Ordering::Acquire)).bytes() { SharedChunk::destroy(chunk); @@ -565,7 +627,13 @@ impl ChunkProvider { /// single allocation, and the current chunk is left untouched so /// subsequent small allocations continue to use it. pub(crate) fn acquire_oversized_local(&self, min_payload: usize) -> Result>, AllocError> { - let payload = round_up_to_drop_align(min_payload)?; + // Add `oversized_payload_align_slack()` to absorb the worst-case + // alignment skew the bump cursor pays at the start of an unaligned + // payload (chunk headers do not pad the payload to be 8-aligned). + // Callers requesting an `elem_align > align_of::()` must + // pre-size `min_payload` accordingly; in practice oversized callers + // pass alignments ≤ 8. + let payload = round_up_to_drop_align(min_payload.checked_add(oversized_payload_align_slack()).ok_or(AllocError)?)?; let total = LocalChunk::::header_size().checked_add(payload).ok_or(AllocError)?; self.reserve_bytes(total)?; match LocalChunk::::allocate(&self.allocator, ptr::from_ref(self), payload) { @@ -583,7 +651,8 @@ impl ChunkProvider { /// Shared-chunk mirror of [`Self::acquire_oversized_local`]. pub(crate) fn acquire_oversized_shared(&self, min_payload: usize) -> Result>, AllocError> { - let payload = round_up_to_drop_align(min_payload)?; + // See `acquire_oversized_local` for the alignment-slack rationale. + let payload = round_up_to_drop_align(min_payload.checked_add(oversized_payload_align_slack()).ok_or(AllocError)?)?; let total = SharedChunk::::header_size().checked_add(payload).ok_or(AllocError)?; self.reserve_bytes(total)?; match SharedChunk::::allocate(self.allocator.clone(), Weak::clone(&self.weak_self), payload) { @@ -684,7 +753,7 @@ impl ChunkProvider { while !cur.is_null() { let fat = LocalChunk::::header_to_fat(cur); let chunk_nn = NonNull::new_unchecked(fat); - let next = LocalChunk::read_cached_next(chunk_nn); + let next = LocalChunk::next(chunk_nn); LocalChunk::destroy(chunk_nn, &self.allocator); cur = next; } @@ -711,7 +780,7 @@ impl Drop for ChunkProvider { /// Convenience: cache lookup by total allocation size. #[inline] -fn is_cacheable_size(total: usize) -> bool { +pub(crate) fn is_cacheable_size(total: usize) -> bool { (MIN_CHUNK_BYTES..=MAX_CHUNK_BYTES).contains(&total) && total.is_power_of_two() } @@ -730,6 +799,19 @@ fn round_up_to_drop_align(min_payload: usize) -> Result { min_payload.checked_add(mask).map(|v| v & !mask).ok_or(AllocError) } +/// Worst-case alignment skew the bump cursor pays at the start of an +/// oversized chunk's (possibly unaligned) payload. Added to oversized +/// requests so the first allocation always fits after alignment. +#[inline] +// Mutation testing is suppressed: `align - 1` is the exact maximum skew. +// The `-`→`+` / `-`→`/` mutants only ever *over*-reserve by a few bytes +// (never under-allocate), so they are equivalent for correctness and +// invisible through any public API contract. +#[cfg_attr(test, mutants::skip)] +fn oversized_payload_align_slack() -> usize { + mem::align_of::() - 1 +} + /// Wraps the `needed_total > MAX_CHUNK_BYTES` check used by the /// `acquire_*` routing gates. #[cfg_attr(test, mutants::skip)] // boundary unreachable: max_normal_alloc capped well below @@ -917,7 +999,3 @@ mod tests { ); } } - -// Legacy CAS-retry helper removed: all the local CAS loops now use -// `AtomicX::fetch_update`, whose contention-path retry lives in the -// std library and is hidden from local coverage instrumentation. diff --git a/crates/multitude/src/internal/drop_entry.rs b/crates/multitude/src/internal/drop_entry.rs index ac4e9e9e6..91bf4aa57 100644 --- a/crates/multitude/src/internal/drop_entry.rs +++ b/crates/multitude/src/internal/drop_entry.rs @@ -180,7 +180,14 @@ pub(crate) unsafe fn commit_placeholder_drop_fn( ) -> bool { let entry_size = mem::size_of::(); let entry_align = mem::align_of::(); - let aligned_len = payload_len & !(entry_align - 1); + // Align the *absolute* payload-end address down to `entry_align`, + // matching `ChunkMutator::from_owned`'s `aligned_end_addr` formula. + // Doing the alignment on absolute addresses (rather than on + // `payload_len` alone) keeps the entry positions valid even when + // `payload` itself is not `entry_align`-aligned — the chunk + // headers don't pad their payload start anymore. + let payload_addr = payload as usize; + let aligned_end_offset = ((payload_addr.wrapping_add(payload_len)) & !(entry_align - 1)).wrapping_sub(payload_addr); // Find the placeholder by (value_offset, len) and unconditionally // store the real shim. Concurrent `assume_init` calls on cloned // handles for the same allocation race here; both calls compute @@ -194,9 +201,9 @@ pub(crate) unsafe fn commit_placeholder_drop_fn( // addresses across invocations of the same function. The single- // pass unconditional store sidesteps the comparison entirely. for i in 0..drop_entry_count { - let entry_off = aligned_len - (i + 1) * entry_size; - // SAFETY: `entry_off + entry_size <= aligned_len <= payload_len`, so - // the entry lies inside the payload; the caller guarantees an + let entry_off = aligned_end_offset - (i + 1) * entry_size; + // SAFETY: `entry_off + entry_size <= aligned_end_offset <= payload_len`, + // so the entry lies inside the payload; the caller guarantees an // initialized `DropEntry` was written there. We hold a chunk // reference, so the slot stays live for this read/write. let entry = &*(payload.add(entry_off).cast::()); @@ -258,17 +265,23 @@ pub(crate) unsafe fn replay_drops(payload: *mut u8, payload_len: usize, drop_ent } let entry_size = mem::size_of::(); let entry_align = mem::align_of::(); - // Align the effective payload end down to `entry_align`. The - // allocator (see `ChunkMutator::from_owned`) reserves drop entries - // starting from this aligned end, so the trailing bytes between - // `aligned_len` and `payload_len` were never populated. - let aligned_len = payload_len & !(entry_align - 1); - // Iterate newest-first (LIFO): the last-written entry sits closest to - // the aligned payload end. Index `i` runs from 0 (newest) up to - // `drop_entry_count - 1` (oldest). - for i in 0..drop_entry_count { - let entry_off = aligned_len - (i + 1) * entry_size; - // SAFETY: `entry_off + entry_size <= aligned_len <= payload_len`, + // Align the *absolute* payload-end address down to `entry_align`, + // matching `ChunkMutator::from_owned`'s `aligned_end_addr` formula + // (which the allocator uses when reserving drop entries). Computing + // the alignment on absolute addresses keeps drop-entry positions + // valid even when `payload` itself is not `entry_align`-aligned — + // chunk headers do not pad the payload start. + let payload_addr = payload as usize; + let aligned_end_offset = ((payload_addr.wrapping_add(payload_len)) & !(entry_align - 1)).wrapping_sub(payload_addr); + // Iterate newest-first (LIFO) so child values drop before their + // parents, matching Rust's drop semantics. Entries grow downward + // from the aligned payload end, so the newest (last-written) entry + // sits at the lowest address (`aligned_end - count * entry_size`) + // and the oldest at the highest (`aligned_end - entry_size`). + // Visiting `i` from `count - 1` down to `0` walks newest -> oldest. + for i in (0..drop_entry_count).rev() { + let entry_off = aligned_end_offset - (i + 1) * entry_size; + // SAFETY: `entry_off + entry_size <= aligned_end_offset <= payload_len`, // so the entry lies inside the payload allocation; the caller // guarantees that an initialized `DropEntry` was previously // written there. If committed, the entry's @@ -350,6 +363,76 @@ mod tests { assert!(installed.is_some()); } + /// Regression test for the unaligned-payload formula: when + /// `payload_ptr` is **not** `align_of::()`-aligned (as + /// happens for the post-padding-removal chunk layouts), both + /// `commit_placeholder_drop_fn` and `replay_drops` must still place + /// drop entries at absolutely-aligned addresses near the payload + /// tail. The buffer below intentionally offsets the payload start + /// by `entry_align - 1` bytes from an aligned base, so the + /// payload start address is 1-aligned but the *end* of the + /// reserved payload still lands on an `entry_align` multiple. + #[test] + fn replay_and_commit_tolerate_unaligned_payload_start() { + use std::sync::atomic::{AtomicUsize, Ordering}; + static CALLS: AtomicUsize = AtomicUsize::new(0); + fn counting_shim(_p: *mut u8, _n: usize) { + CALLS.fetch_add(1, Ordering::Relaxed); + } + CALLS.store(0, Ordering::Relaxed); + + let entry_size = mem::size_of::(); + let entry_align = mem::align_of::(); + // Buffer big enough to host two entries plus the misalignment slack. + let extra = entry_align - 1; + let payload_len = entry_size * 2 + extra; + // Over-allocate by `entry_align` so we can choose an unaligned start. + let mut buf = std::vec![0u8; payload_len + entry_align]; + let base_addr = buf.as_mut_ptr() as usize; + // Pick a payload_start address that's odd-aligned. Anchor 1 byte + // past an aligned base so payload_addr mod entry_align == 1. + let aligned_base = (base_addr + entry_align - 1) & !(entry_align - 1); + let payload_start_addr = aligned_base + 1; + let payload_offset = payload_start_addr - base_addr; + // SAFETY: `payload_offset + payload_len` ≤ `buf.len()` by construction. + let payload_ptr = unsafe { buf.as_mut_ptr().add(payload_offset) }; + assert_ne!((payload_ptr as usize) % entry_align, 0, "payload must be unaligned for this test"); + + // Where the entries *must* land: at the absolute-aligned end. + let aligned_end_addr = (payload_start_addr + payload_len) & !(entry_align - 1); + let aligned_end_offset = aligned_end_addr - payload_start_addr; + + let value_offset: u16 = 0; + let len: u16 = 1; + let shim_fn = counting_shim as DropFn; + + // Write two placeholders at the correctly-aligned offsets. + // SAFETY: both offsets are within the payload buffer and produce + // entry_align-aligned addresses by construction. + unsafe { + let top_off = aligned_end_offset - entry_size; + let next_off = aligned_end_offset - 2 * entry_size; + // Top: non-matching placeholder. + ptr::write(payload_ptr.add(top_off).cast::(), DropEntry::placeholder(99, 1)); + // Below: matching placeholder. + ptr::write( + payload_ptr.add(next_off).cast::(), + DropEntry::placeholder(value_offset, len), + ); + } + + // Commit phase must locate the matching entry and install the shim. + // SAFETY: both entries are initialized; payload_len includes them. + let committed = unsafe { commit_placeholder_drop_fn(payload_ptr, payload_len, 2, value_offset as usize, len as usize, shim_fn) }; + assert!(committed); + + // Replay phase must invoke the installed shim exactly once + // (the non-matching placeholder still has no shim). + // SAFETY: payload_ptr + payload_len bounds the live buffer. + unsafe { replay_drops(payload_ptr, payload_len, 2) }; + assert_eq!(CALLS.load(Ordering::Relaxed), 1); + } + /// `raw_used` returns the byte sum of the un-padded `DropEntry` /// fields: a `DropFn` (function pointer, `usize`-sized) + two /// `u16`s. Pin the exact value so additive/multiplicative mutations diff --git a/crates/multitude/src/internal/local_chunk.rs b/crates/multitude/src/internal/local_chunk.rs index 57b46618e..491200874 100644 --- a/crates/multitude/src/internal/local_chunk.rs +++ b/crates/multitude/src/internal/local_chunk.rs @@ -10,7 +10,6 @@ #![allow(unsafe_op_in_unsafe_fn, reason = "see module doc: inner unsafe blocks in unsafe fn add noise here")] #![allow(clippy::unnecessary_safety_comment, reason = "safety rationale documented at function level")] -use core::alloc::Layout; use core::cell::{Cell, UnsafeCell}; use core::mem; use core::ptr::{self, NonNull}; @@ -40,8 +39,7 @@ use super::drop_entry::replay_drops; /// live; chunks in the cache are destroyed directly from the provider's own /// `Drop` (`drain_all`) without going through the back-pointer. The provider /// therefore strictly outlives every local-chunk teardown that dereferences -/// this pointer, removing the need for a Weak refcount and the dead "orphan" -/// branch the upgrade used to require. +/// this pointer, so no Weak refcount or orphan-handling branch is needed. #[repr(C)] pub(crate) struct LocalChunk { /// Non-owning back-pointer to the chunk's provider. See the type-level @@ -52,20 +50,31 @@ pub(crate) struct LocalChunk { /// to route the chunk back to the cache. provider: *const ChunkProvider, capacity: usize, + /// Intrusive next-link, used in two disjoint phases of the chunk's + /// life. Stored as a thin `*mut u8` header pointer (`null` for end- + /// of-list); the fat DST pointer is recovered via + /// [`Self::header_to_fat`] when consumers walk the list. + /// + /// * While the chunk is **retired** (refcount = 1, sitting on + /// [`RetiredLocalChunks`](crate::arena::retired_local::RetiredLocalChunks)) + /// the field links the next retired chunk. + /// * While the chunk is **cached** (refcount = 0, sitting on the + /// provider's local freelist) it links the next cached chunk. + /// + /// Those two phases are mutually exclusive in time, so a single + /// field serves both purposes. Placed after `capacity` (both + /// `usize`-aligned) so the smaller `ref_count` / `drop_entry_count` + /// fields can pack into the tail without trailing padding. + next: Cell<*mut u8>, ref_count: Cell, drop_entry_count: Cell, - /// Explicit padding so the header size stays a multiple of 8, keeping - /// the payload start 8-aligned. The payload start must be 8-aligned both - /// for the cache link stored there while the chunk is free (see - /// [`read_cached_next`](Self::read_cached_next)) and for the `DropEntry`s - /// the bump allocator packs against the payload tail (which are positioned - /// relative to the payload start; see [`replay_drops`](super::drop_entry::replay_drops)). - /// Without it, shrinking `ref_count`/`drop_entry_count` below `usize` would - /// land the payload at a non-8-aligned offset, which is UB. This is - /// temporary: once those payload-relative accesses are made tolerant of an - /// unaligned payload base, this padding can be removed and the header shrunk - /// from 24 to 20 bytes. - _padding: [u8; 4], + /// Free bytes between the bump cursor and the drop-entry top at the + /// time this chunk was retired from a `ChunkMutator`. Set in the + /// mutator's `Drop` and read by [`ChunkProvider::release_local`] to + /// decrement the wasted-tail counter. Stays at 0 for chunks that + /// never went through a mutator (e.g. preallocated cache fills). + #[cfg(feature = "stats")] + wasted_at_retire: Cell, /// Bump-payload tail. `data.len() == capacity`. Declared as /// `[UnsafeCell]` (same layout as `[u8]`) so that shared /// borrows of the chunk allow interior-mutable writes into the @@ -74,6 +83,11 @@ pub(crate) struct LocalChunk { /// (essential for Miri's Stacked / Tree Borrows: a sized-struct /// header pointer would have provenance for only the header bytes, /// making any payload-derivation undefined behavior). + /// + /// The payload start is **not** required to be `DropEntry`-aligned: + /// [`replay_drops`](super::drop_entry::replay_drops) computes drop- + /// entry positions via the absolute payload-end address, so they + /// remain aligned regardless of where the payload begins. data: [UnsafeCell], } @@ -89,20 +103,49 @@ unsafe impl Send for LocalChunk {} impl LocalChunk { /// Size in bytes of the chunk header (everything before the payload). #[inline] + // Mutation testing is suppressed: `header_size` is a pure const + // layout computation, exhaustively pinned by `header_size_for_global_matches_layout` + // (exact-value assertion under both feature configs), so any arithmetic + // mutation is already caught by that test. cargo-mutants runs with + // `all_features = true`, under which the `#[cfg(not(feature = "stats"))]` + // branch is dead code, so its mutants are structurally unkillable by that + // run regardless. + #[cfg_attr(test, mutants::skip)] pub(crate) const fn header_size() -> usize { - // The slice tail has align 1 so sits flush against - // `drop_entry_count`; computing via the last fixed-size field's - // offset + size avoids relying on `offset_of!` for DST tails. - mem::offset_of!(Self, _padding) + mem::size_of::<[u8; 4]>() + // The last fixed-size field's offset + its size; the + // `[UnsafeCell]` tail has align 1 so sits flush against it. + // Under `stats`, `wasted_at_retire` is the last fixed field; + // otherwise it's `drop_entry_count`. + #[cfg(feature = "stats")] + { + mem::offset_of!(Self, wasted_at_retire) + mem::size_of::>() + } + #[cfg(not(feature = "stats"))] + { + mem::offset_of!(Self, drop_entry_count) + mem::size_of::>() + } } /// Alignment to use when allocating/deallocating a chunk's backing memory. - /// `A` is no longer stored in the chunk header, so we only need to honour - /// the alignment of the header fields (max is `usize`, 8 bytes on - /// 64-bit). The chunk pointer therefore doesn't need to be over-aligned - /// for `A`. + /// `A` is not stored in the chunk header, so only the header fields' + /// alignment matters (max is `usize`, 8 bytes on 64-bit). The chunk + /// pointer therefore doesn't need to be over-aligned for `A`. + /// + /// Unlike [`SharedChunk`](super::shared_chunk::SharedChunk), local + /// chunks need no `CHUNK_ALIGN` base alignment (they hand out no + /// header-recovering smart pointers), so the base and value alignments + /// coincide. #[inline] pub(crate) const fn struct_align() -> usize { + Self::value_align() + } + + /// The chunk type's own alignment (`align_of::()`, ignoring the + /// align-1 `[UnsafeCell]` tail), used to round the allocation size. + /// Equal to [`Self::struct_align`] for local chunks. Pinned against the + /// real `align_of_val` by `value_align_matches_real_alignment`. + #[inline] + pub(crate) const fn value_align() -> usize { mem::align_of::() } @@ -123,8 +166,13 @@ impl LocalChunk { // unreachable exact-`isize::MAX` boundary. #[cfg_attr(test, mutants::skip)] pub(crate) fn allocate(allocator: &A, provider: *const ChunkProvider, payload_size: usize) -> Result, AllocError> { - let (raw_u8_ptr, _layout) = - crate::internal::chunk_alloc::alloc_chunk_raw(allocator, Self::header_size(), Self::struct_align(), payload_size)?; + let (raw_u8_ptr, _layout) = crate::internal::chunk_alloc::alloc_chunk_raw( + allocator, + Self::header_size(), + payload_size, + Self::value_align(), + Self::struct_align(), + )?; // Construct the fat DST pointer with slice metadata = payload_size. // Its data field is `raw_u8_ptr` (carrying full allocation // provenance), so the resulting `*mut Self` has provenance over @@ -139,6 +187,9 @@ impl LocalChunk { ptr::write(&raw mut (*fat).capacity, payload_size); ptr::write(&raw mut (*fat).ref_count, Cell::new(1)); ptr::write(&raw mut (*fat).drop_entry_count, Cell::new(0)); + ptr::write(&raw mut (*fat).next, Cell::new(ptr::null_mut())); + #[cfg(feature = "stats")] + ptr::write(&raw mut (*fat).wasted_at_retire, Cell::new(0)); Ok(NonNull::new_unchecked(fat)) } } @@ -153,6 +204,24 @@ impl LocalChunk { self.provider } + /// Reads the free byte count stashed by the owning `ChunkMutator`'s + /// `Drop` (the gap between bump cursor and drop-entry top at retire). + /// `0` for chunks that never went through a mutator. + #[cfg(feature = "stats")] + #[inline] + pub(crate) fn wasted_at_retire(&self) -> u32 { + self.wasted_at_retire.get() + } + + /// Stashes the chunk's wasted-tail bytes at retire time, to be + /// subtracted from the provider's wasted-tail counter when the chunk + /// is eventually released to the cache or destroyed. + #[cfg(feature = "stats")] + #[inline] + pub(crate) fn set_wasted_at_retire(&self, n: u32) { + self.wasted_at_retire.set(n); + } + /// Pointer to the first byte of the chunk's payload. /// /// # Safety @@ -199,54 +268,51 @@ impl LocalChunk { /// passed to [`Self::allocate`] when this chunk was created. pub(crate) unsafe fn destroy(chunk: NonNull, allocator: &A) { let header = Self::header_size(); - let align = Self::struct_align(); // SAFETY: caller owns the only reference; we read trivial fields, // replay drops in the payload, then deallocate using the caller- - // supplied allocator. The pair `(raw_ptr, layout)` exactly matches - // the one returned by `allocator.allocate` in `allocate`. The - // header carries no Drop-implementing field (the provider - // back-pointer is a plain raw pointer), so nothing else needs to - // be dropped in place before deallocation. + // supplied allocator. The layout exactly matches the one returned + // by `allocator.allocate` in `allocate` (both go through + // `chunk_layout`). The header carries no Drop-implementing field + // (the provider back-pointer is a plain raw pointer), so nothing + // else needs to be dropped in place before deallocation. let header_ref = &*chunk.as_ptr(); let capacity = header_ref.capacity; let drop_count = header_ref.drop_entry_count.get() as usize; replay_drops(Self::payload_ptr(chunk).as_ptr(), capacity, drop_count); - let total = header + capacity; - let layout = Layout::from_size_align(total, align).expect("matches allocate(); header+capacity stayed within isize::MAX"); + let layout = crate::internal::chunk_alloc::chunk_layout(header, capacity, Self::value_align(), Self::struct_align()) + .expect("matches allocate(); header+capacity stayed within isize::MAX"); let raw_ptr = chunk.as_ptr().cast::(); allocator.deallocate(NonNull::new_unchecked(raw_ptr), layout); } - /// Reads the next-pointer of a cached chunk (stored in the first - /// bytes of the payload while the chunk lives on a free list). - /// Returns a thin `*mut u8` header pointer; cache stores thin - /// pointers since `*mut Self` is fat for the DST. + /// Reads the intrusive next-link without modifying it. The chunk + /// participates in two singly-linked lists at different points in + /// its lifecycle — the arena's retired list and the provider's + /// cache freelist — and this field encodes both. Returns a thin + /// `*mut u8` header pointer (`null` for end-of-list). /// /// # Safety /// - /// Chunk must be in the "cached" state (refcount zero, exclusively - /// owned by the cache). - #[allow( - clippy::cast_ptr_alignment, - reason = "payload base is usize-aligned (header padded to keep payload 8-aligned); cache link fits within that alignment" - )] + /// Chunk must be live (allocated) — either retained (refcount ≥ 1) + /// for the retired-list reader or exclusively cache-owned + /// (refcount = 0) for the cache reader. #[inline] - pub(crate) unsafe fn read_cached_next(chunk: NonNull) -> *mut u8 { - ptr::read(Self::payload_ptr(chunk).as_ptr().cast::<*mut u8>()) + pub(crate) unsafe fn next(chunk: NonNull) -> *mut u8 { + chunk.as_ref().next.get() } - /// Writes the next-pointer of a cached chunk. + /// Replaces the intrusive next-link, returning the previous value. + /// Used by both the retired-list and cache-freelist push paths. /// /// # Safety /// - /// Same as [`read_cached_next`](Self::read_cached_next). - #[allow( - clippy::cast_ptr_alignment, - reason = "payload base is usize-aligned (header padded to keep payload 8-aligned); cache link fits within that alignment" - )] + /// Same as [`Self::next`]; in addition, the caller must hold + /// exclusive access to the field for the duration of the call. + /// `LocalChunk` is `!Sync`, so this is satisfied by the + /// owning-thread invariant. #[inline] - pub(crate) unsafe fn write_cached_next(chunk: NonNull, next: *mut u8) { - ptr::write(Self::payload_ptr(chunk).as_ptr().cast::<*mut u8>(), next); + pub(crate) unsafe fn set_next(chunk: NonNull, next: *mut u8) -> *mut u8 { + chunk.as_ref().next.replace(next) } /// Re-initializes a chunk popped from the cache: refcount → 1, @@ -283,6 +349,10 @@ pub(crate) const fn max_bump_extent() -> usize { impl Chunk for LocalChunk { #[inline] + // Mutation testing is suppressed: returning 0/1 makes every chunk + // report no usable payload, sending the allocator's bump-fit loop + // into an unbounded refill spin (the suite hangs rather than fails). + #[cfg_attr(test, mutants::skip)] fn capacity(&self) -> usize { self.capacity } @@ -330,13 +400,54 @@ mod tests { use super::*; /// `struct_align` returns `align_of::()` (the largest alignment - /// of any header field) regardless of `A` — the chunk no longer stores - /// an allocator copy. + /// of any header field) regardless of `A` — the chunk stores no + /// allocator copy. #[test] fn struct_align_matches_usize() { assert_eq!(LocalChunk::::struct_align(), mem::align_of::()); } + /// For local chunks the value and base alignments coincide (no + /// `CHUNK_ALIGN` over-alignment), and both equal the real + /// `align_of_val` of a constructed chunk — so the allocation size is + /// rounded to exactly the chunk's own alignment (never inflated). + #[test] + fn value_align_matches_struct_align_and_real_alignment() { + assert_eq!(LocalChunk::::value_align(), LocalChunk::::struct_align()); + // SAFETY: single-threaded test owning the only reference. + unsafe { + let chunk = LocalChunk::::allocate(&Global, core::ptr::null(), 64).expect("allocate chunk"); + let real = mem::align_of_val(chunk.as_ref()); + assert_eq!( + LocalChunk::::value_align(), + real, + "value_align must equal align_of_val of the real chunk DST" + ); + LocalChunk::destroy(chunk, &Global); + } + } + + /// Local-chunk `chunk_layout` rounds the size up to `value_align` (8) + /// and keeps the base alignment at `value_align` too (no + /// over-alignment); the resulting size matches each size class exactly. + #[test] + fn chunk_layout_sizes_match_classes() { + use super::super::chunk_alloc::chunk_layout; + use super::super::constants::{NUM_CHUNK_CLASSES, SizeClass}; + + let header = LocalChunk::::header_size(); + let value_align = LocalChunk::::value_align(); + let base_align = LocalChunk::::struct_align(); + for i in 0..NUM_CHUNK_CLASSES { + let class = SizeClass::new(i); + let total = class.bytes(); + let payload = total - header; + let layout = chunk_layout(header, payload, value_align, base_align).expect("layout fits"); + assert_eq!(layout.size(), total, "class {i} size must equal class bytes"); + assert_eq!(layout.align(), value_align); + } + } + /// `max_bump_extent` subtracts the header from `MAX_CHUNK_BYTES`; /// pin the relation so `- → +` mutation (which would balloon the /// result past the real allocation) is caught. @@ -371,15 +482,22 @@ mod tests { } } - /// `header_size` is `offset_of!(_padding) + size_of::<[u8; 4]>()`. For - /// `LocalChunk`, the header layout is fixed: 8 (provider) + - /// 8 (capacity) + 1 (`ref_count`) + 1 pad + 2 (`drop_entry_count`) + - /// 4 (`_padding`) = 24 bytes. Pinning the exact value catches an - /// arithmetic-operator mutation (`+ → *`) that would silently shift the - /// payload base. + /// `header_size` is `offset_of!() + size_of::<>()`. + /// For `LocalChunk`, the header layout is fixed: + /// 8 (provider) + 8 (capacity) + 8 (`next`) + 1 (`ref_count`) + + /// 1 pad + 2 (`drop_entry_count`) = 28 bytes. Under the `stats` + /// feature an additional `wasted_at_retire: Cell` field is + /// appended (after 0 pad bytes since the prior offset is already + /// 4-aligned at 28), for 32 bytes total. Reordering moved `next` + /// ahead of the small fields so the trailing + /// `ref_count` / `drop_entry_count` pair packs into 4 bytes + /// without end-of-struct padding. #[test] - fn header_size_for_global_is_24() { - assert_eq!(LocalChunk::::header_size(), 24); + fn header_size_for_global_matches_layout() { + #[cfg(not(feature = "stats"))] + assert_eq!(LocalChunk::::header_size(), 28); + #[cfg(feature = "stats")] + assert_eq!(LocalChunk::::header_size(), 32); } /// `Chunk::inc_ref` on a local chunk is unreachable in production — local diff --git a/crates/multitude/src/internal/shared_chunk.rs b/crates/multitude/src/internal/shared_chunk.rs index 9aa82b6bf..63f13d318 100644 --- a/crates/multitude/src/internal/shared_chunk.rs +++ b/crates/multitude/src/internal/shared_chunk.rs @@ -10,10 +10,11 @@ #![allow(clippy::unnecessary_safety_comment, reason = "safety rationale documented at function level")] use alloc::sync::Weak; -use core::alloc::Layout; use core::cell::UnsafeCell; use core::mem; use core::ptr::{self, NonNull}; +#[cfg(feature = "stats")] +use core::sync::atomic::AtomicU32; use core::sync::atomic::{AtomicPtr, AtomicU16, AtomicUsize, Ordering, fence}; use allocator_api2::alloc::{AllocError, Allocator}; @@ -35,18 +36,36 @@ pub(crate) struct SharedChunk { provider: Weak>, capacity: usize, ref_count: AtomicUsize, + /// Intrusive cache-freelist link, used while the chunk sits on + /// the provider's shared cache (refcount = 0). CAS-pushed and + /// CAS-popped from any thread, so the storage is atomic. `null` + /// when not on the list. Placed after `ref_count` (both 8-aligned) + /// so the trailing `drop_entry_count` (`u16`) packs against + /// `data` without end-of-struct padding. + /// + /// Unlike `LocalChunk::next`, this slot is *only* used for the + /// cache freelist: shared chunks don't have a retired-list phase + /// since handouts outlive the arena and chunks transition + /// directly from refcount = 1 → 0 → cached (or destroyed). + next: AtomicPtr, drop_entry_count: AtomicU16, - /// Explicit padding so the header size stays a multiple of 8, keeping - /// the payload start 8-aligned. The payload start must be 8-aligned both - /// for the `AtomicPtr` cache link stored there while the chunk is free - /// (see [`cache_link`](Self::cache_link)) and for the `DropEntry`s the bump - /// allocator packs against the payload tail (which are positioned relative - /// to the payload start; see [`replay_drops`](super::drop_entry::replay_drops)). - /// Without it, shrinking `drop_entry_count` from `usize` to `u16` would land - /// the payload at a non-8-aligned offset, which is UB. This is temporary: - /// once those payload-relative accesses are made tolerant of an unaligned - /// payload base, this padding can be removed and the header shrunk. - _padding: [u8; 6], + /// Free bytes between the bump cursor and the drop-entry top at the + /// time this chunk was retired from a `ChunkMutator`. Set in the + /// mutator's `Drop` and read by [`ChunkProvider::release_shared`] + /// to decrement the wasted-tail counter. Stays at 0 for chunks that + /// never went through a mutator (e.g. preallocated cache fills). + /// + /// Read in `release_shared` after the chunk's atomic refcount has + /// dropped to zero (with an acquire fence); the mutator's `Drop` + /// performs the `set` before its own `dec_ref`, so the store is + /// visible. + #[cfg(feature = "stats")] + wasted_at_retire: AtomicU32, + /// Bump-payload tail. See `LocalChunk` for the + /// [`UnsafeCell]` provenance rationale. The payload start is + /// **not** required to be `DropEntry`-aligned: + /// [`replay_drops`](super::drop_entry::replay_drops) aligns drop- + /// entry positions via the absolute payload-end address. data: [UnsafeCell], } @@ -59,18 +78,72 @@ impl SharedChunk { &self.provider } + /// Reads the free byte count stashed by the owning `ChunkMutator`'s + /// `Drop` (the gap between bump cursor and drop-entry top at retire). + /// `0` for chunks that never went through a mutator. + #[cfg(feature = "stats")] + #[inline] + pub(crate) fn wasted_at_retire(&self) -> u32 { + // Acquire pairs with the `Release` store in `set_wasted_at_retire`; + // shared chunks may be inspected on a different thread than the + // one that performed the retire (the last `Arc::drop`). + self.wasted_at_retire.load(Ordering::Acquire) + } + + /// Stashes the chunk's wasted-tail bytes at retire time, to be + /// subtracted from the provider's wasted-tail counter when the chunk + /// is eventually released to the cache or destroyed. + /// + /// `Release` so cross-thread `release_shared` callers observe the + /// stored value after their acquire fence on refcount = 0. + #[cfg(feature = "stats")] #[inline] + pub(crate) fn set_wasted_at_retire(&self, n: u32) { + self.wasted_at_retire.store(n, Ordering::Release); + } + + #[inline] + // Mutation testing is suppressed: see `LocalChunk::header_size`. This + // is a pure const layout computation pinned exactly by the + // `header_size_for_global_matches_layout` test under both feature configs, and + // the `#[cfg(not(feature = "stats"))]` branch is dead code under the + // all-features mutants run. + #[cfg_attr(test, mutants::skip)] pub(crate) const fn header_size() -> usize { - mem::offset_of!(Self, _padding) + mem::size_of::<[u8; 6]>() + // Under `stats`, `wasted_at_retire` is the last fixed-size field; + // otherwise it's `drop_entry_count`. The `[UnsafeCell]` tail + // has align 1 and sits flush against whichever it is. + #[cfg(feature = "stats")] + { + mem::offset_of!(Self, wasted_at_retire) + mem::size_of::() + } + #[cfg(not(feature = "stats"))] + { + mem::offset_of!(Self, drop_entry_count) + mem::size_of::() + } } #[inline] #[cfg_attr(test, mutants::skip)] // both branches saturate at CHUNK_ALIGN pub(crate) const fn struct_align() -> usize { + let base = Self::value_align(); + if base >= CHUNK_ALIGN { base } else { CHUNK_ALIGN } + } + + /// The chunk type's own alignment (`align_of::()`, ignoring the + /// align-1 `[UnsafeCell]` tail): the max of `align_of::()` and + /// `align_of::()` (every other header field — the atomics and + /// the `Weak` pointer — has alignment `<= align_of::()`). + /// + /// Used to round the allocation *size* (vs. [`Self::struct_align`], + /// the larger *base*-address alignment). Pinned against the real + /// `align_of_val` by `value_align_matches_real_alignment`. + #[inline] + #[cfg_attr(test, mutants::skip)] // pure layout constant pinned by a dedicated test + pub(crate) const fn value_align() -> usize { let a = mem::align_of::(); let b = mem::align_of::(); - let base = if a >= b { a } else { b }; - if base >= CHUNK_ALIGN { base } else { CHUNK_ALIGN } + if a >= b { a } else { b } } /// Recovers the chunk header (as a thin `*mut u8` carrying the @@ -113,8 +186,13 @@ impl SharedChunk { // unreachable exact-`isize::MAX` boundary. #[cfg_attr(test, mutants::skip)] pub(crate) fn allocate(allocator: A, provider: Weak>, payload_size: usize) -> Result, AllocError> { - let (raw_u8_ptr, _layout) = - crate::internal::chunk_alloc::alloc_chunk_raw(&allocator, Self::header_size(), Self::struct_align(), payload_size)?; + let (raw_u8_ptr, _layout) = crate::internal::chunk_alloc::alloc_chunk_raw( + &allocator, + Self::header_size(), + payload_size, + Self::value_align(), + Self::struct_align(), + )?; let fat: *mut Self = ptr::slice_from_raw_parts_mut(raw_u8_ptr, payload_size) as *mut Self; // SAFETY: see `LocalChunk::allocate`. unsafe { @@ -123,6 +201,9 @@ impl SharedChunk { ptr::write(&raw mut (*fat).capacity, payload_size); ptr::write(&raw mut (*fat).ref_count, AtomicUsize::new(1)); ptr::write(&raw mut (*fat).drop_entry_count, AtomicU16::new(0)); + ptr::write(&raw mut (*fat).next, AtomicPtr::new(ptr::null_mut())); + #[cfg(feature = "stats")] + ptr::write(&raw mut (*fat).wasted_at_retire, AtomicU32::new(0)); Ok(NonNull::new_unchecked(fat)) } } @@ -140,34 +221,31 @@ impl SharedChunk { /// zero with an acquire fence covering all prior releases). pub(crate) unsafe fn destroy(chunk: NonNull) { let header = Self::header_size(); - let align = Self::struct_align(); let header_ref = &*chunk.as_ptr(); let capacity = header_ref.capacity; let drop_count = header_ref.drop_entry_count.load(Ordering::Acquire) as usize; replay_drops(Self::payload_ptr(chunk).as_ptr(), capacity, drop_count); let allocator: A = ptr::read(&raw const (*chunk.as_ptr()).allocator); ptr::drop_in_place(&raw mut (*chunk.as_ptr()).provider); - let total = header + capacity; - let layout = Layout::from_size_align(total, align).expect("matches allocate(); header+capacity stayed within isize::MAX"); + let layout = crate::internal::chunk_alloc::chunk_layout(header, capacity, Self::value_align(), Self::struct_align()) + .expect("matches allocate(); header+capacity stayed within isize::MAX"); let raw_ptr = chunk.as_ptr().cast::(); allocator.deallocate(NonNull::new_unchecked(raw_ptr), layout); drop(allocator); } - /// Pointer to the `AtomicPtr` cache link stored in the first - /// bytes of the chunk's payload. Cache stores thin pointers since - /// `*mut Self` is fat for the DST. + /// Pointer to the chunk's intrusive cache-freelist link + /// (`AtomicPtr` storing a thin header pointer; cache stores + /// thin pointers since `*mut Self` is fat for the DST). The field + /// lives in the chunk header. /// /// # Safety /// - /// Chunk must be in the cached state. - #[allow( - clippy::cast_ptr_alignment, - reason = "SharedChunk payload is CHUNK_ALIGN-aligned; AtomicPtr fits within that alignment" - )] + /// Chunk must be allocated (header live); ordering of accesses + /// through the returned pointer is the caller's responsibility. #[inline] pub(crate) unsafe fn cache_link(chunk: NonNull) -> *const AtomicPtr { - Self::payload_ptr(chunk).as_ptr().cast::>() + &raw const (*chunk.as_ptr()).next } /// Re-initializes a chunk popped from the cache: refcount → 1, @@ -234,10 +312,61 @@ impl SharedChunk { Self::teardown_and_release(chunk); } } + + /// Atomically reserves `n` additional strong references on this + /// chunk in a single `fetch_add`, in addition to whatever the + /// caller already holds. Aborts the process on overflow. + /// + /// Used by the arena's per-chunk surplus pre-credit: at chunk + /// install time the arena reserves a large surplus of refs so + /// per-allocation handouts can be tracked in a non-atomic local + /// counter; the unused portion is returned to the chunk via + /// [`Self::refund_refs`] when the chunk is retired. + #[inline] + pub(crate) fn pre_credit_refs(&self, n: usize) { + #[cfg_attr(coverage_nightly, coverage(off))] + #[inline(never)] + #[cold] + fn overflow() -> ! { + refcount_overflow_abort() + } + if n == 0 { + return; + } + let prev = self.ref_count.fetch_add(n, Ordering::Relaxed); + if prev.checked_add(n).is_none() { + overflow(); + } + } + + /// Atomically returns `n` previously pre-credited but unused + /// refs to the chunk's counter via `fetch_sub` with `Release` + /// ordering. `Release` matches the existing per-ref `dec_ref` + /// ordering so any writes the arena thread performed into the + /// chunk are visible to other-thread holders that may observe + /// the lower count. + /// + /// # Safety + /// + /// Caller must own exactly `n` previously-credited (unhanded-out) + /// strong references on this chunk. After this call those `n` + /// references no longer exist. + #[inline] + pub(crate) unsafe fn refund_refs(&self, n: usize) { + if n == 0 { + return; + } + let prev = self.ref_count.fetch_sub(n, Ordering::Release); + debug_assert!(prev >= n, "refund_refs underflow: prev={prev} n={n}"); + } } impl Chunk for SharedChunk { #[inline] + // Mutation testing is suppressed: see `LocalChunk::capacity` — a + // 0/1 capacity drives the allocator's refill loop into an unbounded + // spin, hanging the suite instead of failing it. + #[cfg_attr(test, mutants::skip)] fn capacity(&self) -> usize { self.capacity } @@ -298,6 +427,23 @@ mod tests { use super::*; + /// `header_size` is `offset_of!() + size_of::<>()`. + /// For `SharedChunk`, the header layout is fixed: + /// 0 (allocator ZST) + 8 (provider `Weak`) + 8 (capacity) + + /// 8 (`ref_count`) + 8 (`next`) + 2 (`drop_entry_count`) = 34 bytes. + /// Under the `stats` feature an additional `wasted_at_retire: + /// AtomicU32` is appended after 2 pad bytes (offset 36) for 40 bytes + /// total. `next` is placed between `ref_count` and + /// `drop_entry_count` so the trailing `u16` packs against `data` + /// without padding when stats are off. + #[test] + fn header_size_for_global_matches_layout() { + #[cfg(not(feature = "stats"))] + assert_eq!(SharedChunk::::header_size(), 34); + #[cfg(feature = "stats")] + assert_eq!(SharedChunk::::header_size(), 40); + } + /// `struct_align` returns the max of `align_of::()`, /// `align_of::()`, and `CHUNK_ALIGN`. Pin the exact value so /// the `>= → <` mutation flips it. @@ -314,6 +460,71 @@ mod tests { assert_eq!(got, super::super::constants::CHUNK_ALIGN); } + /// Regression guard for the "every shared chunk is allocated at 64 KiB" + /// bug: `chunk_layout` must round the allocation *size* up to + /// `value_align` (8) and set the *base* alignment to `struct_align` + /// (`CHUNK_ALIGN`), but must NOT round the size up to `CHUNK_ALIGN`. + /// Each cacheable size class must therefore produce an allocation + /// whose size equals the class bytes, not 64 KiB. + #[test] + fn chunk_layout_does_not_inflate_size_to_base_align() { + use super::super::chunk_alloc::chunk_layout; + use super::super::constants::{CHUNK_ALIGN, NUM_CHUNK_CLASSES, SizeClass}; + + let header = SharedChunk::::header_size(); + let value_align = SharedChunk::::value_align(); + let base_align = SharedChunk::::struct_align(); + assert_eq!(base_align, CHUNK_ALIGN, "shared chunks need CHUNK_ALIGN base alignment"); + assert!(value_align <= base_align); + + for i in 0..NUM_CHUNK_CLASSES { + let class = SizeClass::new(i); + let total = class.bytes(); + let payload = total - header; + let layout = chunk_layout(header, payload, value_align, base_align).expect("layout fits"); + // Size classes are powers-of-two multiples of 512, hence + // already `value_align`-aligned, so the rounded size is exactly + // the class bytes — crucially NOT inflated to `CHUNK_ALIGN`. + assert_eq!( + layout.size(), + total, + "class {i} size must equal class bytes, not be padded to base align" + ); + assert_eq!( + layout.align(), + CHUNK_ALIGN, + "base must stay CHUNK_ALIGN-aligned for header recovery" + ); + if total < CHUNK_ALIGN { + assert!( + layout.size() < CHUNK_ALIGN, + "class {i} ({total} B) must not be inflated to {CHUNK_ALIGN} B" + ); + } + } + } + + /// Pins `value_align()` (a hand-computed layout constant) against the + /// real alignment of a constructed chunk, so a future field with a + /// larger alignment can't silently make the size-rounding too small + /// (which would be UB). `align_of_val` is valid on the DST reference. + #[test] + fn value_align_matches_real_alignment() { + // SAFETY: single-threaded test; refcount forced to 0 before destroy. + unsafe { + let chunk = SharedChunk::::allocate(Global, Weak::new(), 64).expect("allocate chunk"); + let real = mem::align_of_val(chunk.as_ref()); + assert_eq!( + SharedChunk::::value_align(), + real, + "value_align must equal align_of_val of the real chunk DST" + ); + assert!(SharedChunk::::struct_align() >= real); + chunk.as_ref().set_ref_count_for_test(0); + SharedChunk::destroy(chunk); + } + } + /// `max_bump_extent` subtracts the header from `MAX_CHUNK_BYTES`; /// pin the relation so `- → +` mutation is caught. #[test] @@ -347,4 +558,60 @@ mod tests { std::panic::resume_unwind(result.expect_err("inc_ref must panic")); } } + + // Covers the `n == 0` early-return guards in `pre_credit_refs` / + // `refund_refs` (no-op, refcount untouched) plus a non-zero + // credit/refund round-trip that returns the count to its start. + #[test] + fn pre_credit_and_refund_zero_are_noops() { + // SAFETY: single-threaded test owning the only references; we + // restore the refcount to 0 before destroying the chunk. + unsafe { + let chunk = SharedChunk::::allocate(Global, Weak::new(), 64).expect("allocate chunk"); + let header = chunk.as_ref(); + let before = header.ref_count.load(Ordering::Relaxed); + + header.pre_credit_refs(0); + header.refund_refs(0); + assert_eq!( + header.ref_count.load(Ordering::Relaxed), + before, + "zero credit/refund must not move the count" + ); + + header.pre_credit_refs(5); + assert_eq!(header.ref_count.load(Ordering::Relaxed), before + 5); + header.refund_refs(5); + assert_eq!( + header.ref_count.load(Ordering::Relaxed), + before, + "credit/refund round-trip must restore the count" + ); + + header.set_ref_count_for_test(0); + SharedChunk::destroy(chunk); + } + } + + // Covers `pre_credit_refs`' overflow guard call site: forcing the + // refcount to its saturation point and pre-crediting one more routes + // through `refcount_overflow_abort`, which panics under `cfg(test)`. + #[test] + #[should_panic(expected = "refcount overflow")] + fn pre_credit_refs_overflow_triggers_abort_guard() { + // SAFETY: single-threaded test. Mirrors + // `inc_ref_overflow_triggers_abort_guard`: drive the guard, catch + // the panic, restore the refcount so the chunk destroys cleanly, + // then resume unwinding so `should_panic` observes it. + unsafe { + let chunk = SharedChunk::::allocate(Global, Weak::new(), 64).expect("allocate chunk"); + chunk.as_ref().set_ref_count_for_test(usize::MAX); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + chunk.as_ref().pre_credit_refs(1); + })); + chunk.as_ref().set_ref_count_for_test(0); + SharedChunk::destroy(chunk); + std::panic::resume_unwind(result.expect_err("pre_credit_refs must panic")); + } + } } diff --git a/crates/multitude/src/internal/uninit.rs b/crates/multitude/src/internal/uninit.rs index 7a9fe9755..2a1fb3c51 100644 --- a/crates/multitude/src/internal/uninit.rs +++ b/crates/multitude/src/internal/uninit.rs @@ -339,7 +339,6 @@ impl<'a, T> UninitDrop<'a, T> { // storage exclusively. unsafe { &mut *ptr.as_ptr() } } - /// Same as [`init`](Self::init) but returns a raw pointer with no /// lifetime. Used by the arena layer when the resulting reference's /// lifetime must be tied to `&Arena` rather than to the consumed diff --git a/crates/multitude/src/lib.rs b/crates/multitude/src/lib.rs index 6fc5af65c..e5de982af 100644 --- a/crates/multitude/src/lib.rs +++ b/crates/multitude/src/lib.rs @@ -185,12 +185,23 @@ //! //! Once you're done building, you can **freeze them** into immutable smart pointers: //! -//! - [`String::into_arena_box_str`](strings::String::into_arena_box_str) → [`Box`](crate::Box) (**8 bytes**, thin). +//! - [`String::into_boxed_str`](strings::String::into_boxed_str) → +//! [`Box`](crate::Box) (**8 bytes**, thin), or `Box::from(string)`. //! The freeze is **O(n)** — it copies the bytes into a compact, //! length-prefixed allocation so the resulting single pointer is //! `Send`-safe and can outlive the arena. -//! - [`Vec::into_arena_box`](vec::Vec::into_arena_box) → [`Box<[T]>`](crate::Box) (**8 bytes**, thin). -//! For `T: !Drop`, the freeze is **O(1)**. +//! - [`Vec::into_boxed_slice`](vec::Vec::into_boxed_slice) → +//! [`Box<[T]>`](crate::Box) (**8 bytes**, thin), or `Box::from(vec)`. +//! The freeze is **O(n)** — it moves the elements into a fresh compact, +//! length-prefixed allocation so the resulting single pointer is +//! `Send`-safe and can outlive the arena. +//! - `Arc::from(vec)` / `Arc::from(string)` → [`Arc<[T]>`](crate::Arc) / +//! [`Arc`](crate::Arc), the shared, reference-counted freeze +//! (mirroring `std`'s `From> for Arc<[T]>`). +//! - [`Vec::leak`](vec::Vec::leak) → `&mut [T]` (or `&*v.leak()` for `&[T]`) +//! borrowed for the arena's lifetime. For `T: !Drop`, this freeze is +//! **O(1) and allocation-free** — the existing buffer is reinterpreted in +//! place. Unlike the `Box`/`Arc` freezes, the slice does not outlive the arena. //! //! The `Vec` freeze also reclaims any unused capacity left in the //! buffer when the conditions allow it, so those bytes become available @@ -207,7 +218,7 @@ //! builder.push_str("world"); //! //! // Freeze for storage: 8-byte single-pointer smart pointer. O(n) — copies the bytes. -//! let stored: Box = builder.into_arena_box_str(); +//! let stored: Box = builder.into_boxed_str(); //! assert_eq!(&*stored, "hello, world"); //! ``` //! @@ -247,14 +258,14 @@ //! [`Utf16String`](strings::Utf16String) are transient growable //! buffers — small structs (32 bytes) carrying a data pointer + //! length + capacity + arena reference. You build them up with -//! `push_str` / `push_char` / [`format!`](strings::format!) / +//! `push_str` / `push` / [`format!`](strings::format!) / //! [`format_utf16!`](strings::format_utf16!), then **freeze** them //! into one of the smart pointers above: //! //! | Builder | Freeze method | Result | //! |---|---|---| -//! | [`String`](strings::String) | [`into_arena_box_str`](strings::String::into_arena_box_str) | [`Box`](crate::Box) | -//! | [`Utf16String`](strings::Utf16String) | [`into_arena_box_utf16_str`](strings::Utf16String::into_arena_box_utf16_str) | [`BoxUtf16Str`](strings::BoxUtf16Str) | +//! | [`String`](strings::String) | [`into_boxed_str`](strings::String::into_boxed_str) | [`Box`](crate::Box) | +//! | [`Utf16String`](strings::Utf16String) | [`into_boxed_utf16_str`](strings::Utf16String::into_boxed_utf16_str) | [`BoxUtf16Str`](strings::BoxUtf16Str) | //! //! The UTF-16 freeze reuses the buffer in place (O(1)) and returns //! any unused tail capacity to the chunk's bump cursor when it can. @@ -284,7 +295,7 @@ //! let mut b = arena.alloc_string(); //! b.push_str("abc"); //! b.push_str("123"); -//! let frozen: Box = b.into_arena_box_str(); +//! let frozen: Box = b.into_boxed_str(); //! assert_eq!(&*frozen, "abc123"); //! //! // format!-style: @@ -314,7 +325,7 @@ //! let mut b = arena.alloc_utf16_string(); //! b.push_str(utf16str!("abc")); //! b.push_from_str("123"); -//! let frozen = b.into_arena_box_utf16_str(); +//! let frozen = b.into_boxed_utf16_str(); //! assert_eq!(&*frozen, utf16str!("abc123")); //! //! // format!-style: @@ -396,6 +407,7 @@ mod r#box; #[cfg(feature = "dst")] #[cfg_attr(docsrs, doc(cfg(feature = "dst")))] pub mod dst; +mod from_in; mod internal; pub mod strings; mod thin_smart_ptr_common; @@ -432,3 +444,4 @@ pub use self::arena_builder::ArenaBuilder; #[cfg_attr(docsrs, doc(cfg(feature = "stats")))] pub use self::arena_stats::ArenaStats; pub use self::r#box::Box; +pub use self::from_in::{FromIn, IntoIn}; diff --git a/crates/multitude/src/strings/format_macro.rs b/crates/multitude/src/strings/format_macro.rs index 74df24c4b..10de1cf09 100644 --- a/crates/multitude/src/strings/format_macro.rs +++ b/crates/multitude/src/strings/format_macro.rs @@ -4,7 +4,7 @@ /// `format!`-style macro that writes into a fresh arena-backed /// [`String`](crate::strings::String). /// -/// Freeze the result with [`String::into_arena_box_str`](crate::strings::String::into_arena_box_str) +/// Freeze the result with [`String::into_boxed_str`](crate::strings::String::into_boxed_str) /// if you want an immutable [`Box`](crate::Box). /// /// # Panics diff --git a/crates/multitude/src/strings/format_utf16_macro.rs b/crates/multitude/src/strings/format_utf16_macro.rs index 36ce8f09c..c91d372bb 100644 --- a/crates/multitude/src/strings/format_utf16_macro.rs +++ b/crates/multitude/src/strings/format_utf16_macro.rs @@ -4,7 +4,7 @@ /// `format!`-style macro that writes into a fresh arena-backed /// [`Utf16String`](crate::strings::Utf16String). /// -/// Freeze the result with [`Utf16String::into_arena_box_utf16_str`](crate::strings::Utf16String::into_arena_box_utf16_str) +/// Freeze the result with [`Utf16String::into_boxed_utf16_str`](crate::strings::Utf16String::into_boxed_utf16_str) /// if you want an immutable [`BoxUtf16Str`](crate::strings::BoxUtf16Str). /// /// # Panics diff --git a/crates/multitude/src/strings/from_utf16_error.rs b/crates/multitude/src/strings/from_utf16_error.rs new file mode 100644 index 000000000..03d1403c4 --- /dev/null +++ b/crates/multitude/src/strings/from_utf16_error.rs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Error type for the byte-oriented UTF-16 string constructors. + +use core::fmt; + +/// Error from the byte-oriented UTF-16 string constructors. +/// +/// Returned by +/// [`Arena::alloc_string_from_utf16le`](crate::Arena::alloc_string_from_utf16le) +/// and friends when the input byte slice is not valid UTF-16 — either it had +/// an odd length, or it contained an unpaired surrogate. The arena-bound +/// analog of [`std::string::FromUtf16Error`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FromUtf16Error(()); + +impl FromUtf16Error { + #[inline] + pub(crate) const fn new() -> Self { + Self(()) + } +} + +impl fmt::Display for FromUtf16Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("invalid UTF-16: odd byte length or unpaired surrogate") + } +} + +impl core::error::Error for FromUtf16Error {} diff --git a/crates/multitude/src/strings/mod.rs b/crates/multitude/src/strings/mod.rs index 778abe0ab..77ceefdcb 100644 --- a/crates/multitude/src/strings/mod.rs +++ b/crates/multitude/src/strings/mod.rs @@ -3,15 +3,11 @@ //! Arena-backed string builders and frozen string handles. //! -//! [`String`] builds UTF-8 strings that can be frozen into -//! [`Arc`](crate::Arc) or [`Box`](crate::Box) — both 8-byte -//! thin smart pointers with `Deref` and string-flavored -//! impls (`PartialEq`, `Serialize`, `From> for Arc<[u8]>`, -//! etc.). With `utf16`, the crate also exposes the parallel UTF-16 -//! types ([`ArcUtf16Str`], [`BoxUtf16Str`], [`Utf16String`]) and -//! `format_utf16!`. +//! [`String`] builds UTF-8 strings in an arena, while the optional +//! [`Utf16String`] type supports UTF-16. mod format_macro; +mod from_utf16_error; mod str_impls; mod string; mod string_common; @@ -34,10 +30,11 @@ pub use arc_utf16_str::ArcUtf16Str; #[cfg(feature = "utf16")] #[cfg_attr(docsrs, doc(cfg(feature = "utf16")))] pub use box_utf16_str::BoxUtf16Str; -pub use string::String; +pub use from_utf16_error::FromUtf16Error; +pub use string::{Drain, String}; #[cfg(feature = "utf16")] #[cfg_attr(docsrs, doc(cfg(feature = "utf16")))] -pub use utf16_string::Utf16String; +pub use utf16_string::{Utf16Drain, Utf16String}; #[doc(inline)] pub use crate::__multitude_format as format; diff --git a/crates/multitude/src/strings/string.rs b/crates/multitude/src/strings/string.rs index 45db7431e..7525c9862 100644 --- a/crates/multitude/src/strings/string.rs +++ b/crates/multitude/src/strings/string.rs @@ -16,16 +16,12 @@ use core::str; use allocator_api2::alloc::{AllocError, Allocator, Global}; -use crate::Arena; use crate::strings::string_common::impl_arena_string_common; +use crate::vec::{FromIteratorIn, Vec}; +use crate::{Arc, Arena, Box, FromIn}; /// A growable, mutable UTF-8 string that lives in an [`Arena`]. /// -/// `String` is a **transient builder**: 32 bytes on 64-bit (data pointer + -/// length + capacity + arena reference). Its purpose is to be filled and -/// then frozen via [`Self::into_arena_box_str`] into a compact, immutable -/// [`Box`](crate::Box) (8 bytes). -/// /// # Example /// /// ``` @@ -36,11 +32,11 @@ use crate::strings::string_common::impl_arena_string_common; /// s.push_str("hello, "); /// s.push_str("world!"); /// assert_eq!(s.as_str(), "hello, world!"); -/// let frozen = s.into_arena_box_str(); +/// let frozen = s.into_boxed_str(); /// assert_eq!(&*frozen, "hello, world!"); /// ``` pub struct String<'a, A: Allocator + Clone = Global> { - pub(super) inner: crate::vec::Vec<'a, u8, A>, + pub(super) inner: Vec<'a, u8, A>, } impl<'a, A: Allocator + Clone> String<'a, A> { @@ -79,7 +75,7 @@ impl<'a, A: Allocator + Clone> String<'a, A> { /// /// Panics if the backing allocator fails. #[must_use] - pub fn from_str_in(s: &str, arena: &'a Arena) -> Self { + pub(crate) fn from_str_in(s: &str, arena: &'a Arena) -> Self { let mut out = Self::with_capacity_in(s.len(), arena); out.push_str(s); out @@ -179,7 +175,7 @@ impl<'a, A: Allocator + Clone> String<'a, A> { // processed so far (retained chars shifted into place) and drops // the unprocessed tail, leaving the string valid UTF-8. struct Guard<'g, 'a, A: Allocator + Clone> { - inner: &'g mut crate::vec::Vec<'a, u8, A>, + inner: &'g mut Vec<'a, u8, A>, idx: usize, del_bytes: usize, } @@ -308,18 +304,212 @@ impl<'a, A: Allocator + Clone> String<'a, A> { self.inner.try_extend_from_slice(s.as_ref().as_bytes()) } - /// Freeze into an owned, mutable [`Box`](crate::Box). + /// Consume the `String`, returning the underlying byte vector. Mirrors + /// [`std::string::String::into_bytes`]. + #[must_use] + pub fn into_bytes(self) -> Vec<'a, u8, A> { + self.inner + } + + /// Returns a mutable reference to the underlying byte vector. Mirrors + /// [`std::string::String::as_mut_vec`]. + /// + /// # Safety + /// + /// The caller must ensure the bytes remain valid UTF-8 before the + /// borrow ends; the `String` invariant is otherwise violated. + #[must_use] + pub unsafe fn as_mut_vec(&mut self) -> &mut Vec<'a, u8, A> { + &mut self.inner + } + + /// Split the string into two at the given byte index, returning the + /// tail `[at, len)` as a new `String` in the same arena and leaving + /// `[0, at)` in `self`. Mirrors [`std::string::String::split_off`]. + /// + /// # Panics + /// + /// Panics if `at` is not on a `char` boundary, or is past the end. + #[must_use] + pub fn split_off(&mut self, at: usize) -> Self { + assert!(self.as_str().is_char_boundary(at), "String::split_off: `at` is not a char boundary"); + Self { + inner: self.inner.split_off(at), + } + } + + /// Clone the bytes in `src` (a byte-index range into `self`) and append + /// them to the end. Mirrors [`std::string::String::extend_from_within`]. + /// + /// # Panics + /// + /// Panics if the range is out of bounds or its bounds are not on `char` + /// boundaries. + pub fn extend_from_within>(&mut self, src: R) { + let len = self.len(); + let start = match src.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i.checked_add(1).expect("extend_from_within: start bound overflows usize"), + Bound::Unbounded => 0, + }; + let end = match src.end_bound() { + Bound::Included(&i) => i.checked_add(1).expect("extend_from_within: end bound overflows usize"), + Bound::Excluded(&i) => i, + Bound::Unbounded => len, + }; + assert!(start <= end, "extend_from_within: start > end"); + assert!(end <= len, "extend_from_within: end > len"); + let s_ref = self.as_str(); + assert!(s_ref.is_char_boundary(start), "extend_from_within: start is not on a char boundary"); + assert!(s_ref.is_char_boundary(end), "extend_from_within: end is not on a char boundary"); + self.inner.extend_from_within(start..end); + } + + /// Remove the `char`s in the byte range `range` from the string, returning + /// a draining iterator over them. Mirrors [`std::string::String::drain`]. + /// + /// The drained range is removed immediately; the returned iterator yields + /// the removed characters (it is also double-ended). + /// + /// # Panics + /// + /// Panics if `range`'s bounds are out of range or not on `char` + /// boundaries. + pub fn drain>(&mut self, range: R) -> Drain<'_, 'a, A> { + let len = self.len(); + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i.checked_add(1).expect("drain: start bound overflows usize"), + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(&i) => i.checked_add(1).expect("drain: end bound overflows usize"), + Bound::Excluded(&i) => i, + Bound::Unbounded => len, + }; + assert!(start <= end, "drain: start > end"); + assert!(end <= len, "drain: end > len"); + let s_ref = self.as_str(); + assert!(s_ref.is_char_boundary(start), "drain: start is not on a char boundary"); + assert!(s_ref.is_char_boundary(end), "drain: end is not on a char boundary"); + Drain { + inner: self.inner.drain(start..end), + } + } + + /// Freeze into an owned, mutable [`Box`](crate::Box). Mirrors + /// [`std::string::String::into_boxed_str`]; [`Box::from`] is the trait + /// form. + /// + /// **O(n)** — copies the contents. + /// + /// # Panics /// - /// **O(n)** — copies the bytes into a compact, length-prefixed - /// allocation in the arena's shared chunks and produces an owned - /// [`Box`](crate::Box) (8 bytes) whose `Drop` releases - /// the chunk hold. The copy is the deliberate trade-off for - /// `Box` being a `Send`-safe, atomically-refcounted single - /// pointer that can outlive the arena. + /// Panics if the underlying allocator fails. #[must_use] - pub fn into_arena_box_str(self) -> crate::Box { + pub fn into_boxed_str(self) -> Box { self.inner.arena().alloc_str_box(self.as_str()) } + + /// Consume the `String`, returning an arena-lifetime mutable string + /// reference `&'a mut str`. Mirrors [`std::string::String::leak`]. + /// + /// **O(1) and allocation-free**: reinterprets the existing UTF-8 buffer + /// in place. + #[must_use] + pub fn leak(self) -> &'a mut str { + let bytes = self.inner.leak(); + // SAFETY: `String` maintains the UTF-8 invariant over its bytes. + unsafe { str::from_utf8_unchecked_mut(bytes) } + } +} + +/// Number of bytes in the UTF-8 sequence whose leading byte is `b0`. +const fn utf8_seq_len(b0: u8) -> usize { + if b0 < 0x80 { + 1 + } else if b0 < 0xE0 { + 2 + } else if b0 < 0xF0 { + 3 + } else { + 4 + } +} + +/// Draining iterator over a byte range of a [`String`], returned by +/// [`String::drain`]. Yields the removed [`char`]s (double-ended). The +/// arena-bound analog of [`std::string::Drain`]. +pub struct Drain<'d, 'a, A: Allocator + Clone> { + inner: crate::vec::Drain<'d, 'a, u8, A>, +} + +impl fmt::Debug for Drain<'_, '_, A> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("Drain").finish_non_exhaustive() + } +} + +impl Iterator for Drain<'_, '_, A> { + type Item = char; + + fn next(&mut self) -> Option { + let b0 = self.inner.next()?; + let len = utf8_seq_len(b0); + let mut buf = [b0, 0, 0, 0]; + for slot in buf.iter_mut().take(len).skip(1) { + *slot = self.inner.next().expect("Drain holds valid UTF-8"); + } + // SAFETY: `String::drain` validated the range on `char` boundaries, so + // the drained bytes form well-formed UTF-8. + unsafe { core::str::from_utf8_unchecked(&buf[..len]) }.chars().next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + // Each remaining `char` is 1–4 bytes. + let bytes = self.inner.len(); + (bytes.div_ceil(4), Some(bytes)) + } +} + +impl DoubleEndedIterator for Drain<'_, '_, A> { + fn next_back(&mut self) -> Option { + let last = self.inner.next_back()?; + let mut buf = [0_u8; 4]; + buf[3] = last; + let mut n = 1; + let mut b = last; + // Pull continuation bytes (`0b10xxxxxx`) from the back until the + // leading byte; the drained range is valid UTF-8 so this terminates. + while b & 0xC0 == 0x80 { + b = self.inner.next_back().expect("Drain holds valid UTF-8"); + n += 1; + buf[4 - n] = b; + } + // SAFETY: see `next`. + unsafe { core::str::from_utf8_unchecked(&buf[4 - n..]) }.chars().next() + } +} + +impl core::iter::FusedIterator for Drain<'_, '_, A> {} + +impl<'a, A: Allocator + Clone> From> for Box { + /// Freeze a [`String`] into an immutable [`Box`](crate::Box). + /// Mirrors `std`'s `From for Box`. + #[inline] + fn from(s: String<'a, A>) -> Self { + s.into_boxed_str() + } +} + +impl<'a, A: Allocator + Clone + Send + Sync> From> for Arc { + /// Freeze a [`String`] into a shared [`Arc`](crate::Arc). + /// Mirrors `std`'s `From for Arc`. + #[inline] + fn from(s: String<'a, A>) -> Self { + s.inner.arena().alloc_str_arc(s.as_str()) + } } impl Deref for String<'_, A> { @@ -440,7 +630,7 @@ impl serde::ser::Serialize for String<'_, A> { serializer.serialize_str(self.as_str()) } } -impl<'a, A: Allocator + Clone> crate::vec::FromIteratorIn for String<'a, A> { +impl<'a, A: Allocator + Clone> FromIteratorIn for String<'a, A> { type Allocator = &'a Arena; fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { @@ -450,7 +640,7 @@ impl<'a, A: Allocator + Clone> crate::vec::FromIteratorIn for String<'a, A } } -impl<'a, 'b, A: Allocator + Clone> crate::vec::FromIteratorIn<&'b str> for String<'a, A> { +impl<'a, 'b, A: Allocator + Clone> FromIteratorIn<&'b str> for String<'a, A> { type Allocator = &'a Arena; fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { @@ -459,3 +649,199 @@ impl<'a, 'b, A: Allocator + Clone> crate::vec::FromIteratorIn<&'b str> for Strin s } } + +impl<'a, 'b, A: Allocator + Clone> FromIteratorIn<&'b char> for String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl<'a, A: Allocator + Clone> FromIteratorIn for String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl<'a, A: Allocator + Clone> FromIteratorIn> for String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIteratorIn> for String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl, A: Allocator + Clone> core::ops::Index for String<'_, A> { + type Output = I::Output; + #[inline] + fn index(&self, index: I) -> &Self::Output { + core::ops::Index::index(self.as_str(), index) + } +} + +impl, A: Allocator + Clone> core::ops::IndexMut for String<'_, A> { + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + core::ops::IndexMut::index_mut(self.as_mut_str(), index) + } +} + +impl AsRef<[u8]> for String<'_, A> { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +#[cfg(feature = "std")] +impl AsRef for String<'_, A> { + #[inline] + fn as_ref(&self) -> &std::ffi::OsStr { + self.as_str().as_ref() + } +} + +#[cfg(feature = "std")] +impl AsRef for String<'_, A> { + #[inline] + fn as_ref(&self) -> &std::path::Path { + self.as_str().as_ref() + } +} + +impl core::ops::Add<&str> for String<'_, A> { + type Output = Self; + /// Concatenates a `&str` onto the end of this `String`. Mirrors + /// `std`'s `Add<&str> for String`. + #[inline] + fn add(mut self, rhs: &str) -> Self { + self.push_str(rhs); + self + } +} + +impl core::ops::AddAssign<&str> for String<'_, A> { + #[inline] + fn add_assign(&mut self, rhs: &str) { + self.push_str(rhs); + } +} + +impl<'b, A: Allocator + Clone> Extend<&'b char> for String<'_, A> { + fn extend>(&mut self, iter: I) { + for c in iter { + self.push(*c); + } + } +} + +impl<'b, A: Allocator + Clone> Extend> for String<'_, A> { + fn extend>>(&mut self, iter: I) { + for s in iter { + self.push_str(&s); + } + } +} + +impl Extend for String<'_, A> { + fn extend>(&mut self, iter: I) { + for s in iter { + self.push_str(&s); + } + } +} + +impl Extend> for String<'_, A> { + fn extend>>(&mut self, iter: I) { + for s in iter { + self.push_str(&s); + } + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIn<&'b str> for String<'a, A> { + type Allocator = &'a Arena; + /// Copy `value` into a fresh arena string. Mirrors `std`'s `From<&str>`. + #[inline] + fn from_in(value: &'b str, allocator: &'a Arena) -> Self { + Self::from_str_in(value, allocator) + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIn<&'b mut str> for String<'a, A> { + type Allocator = &'a Arena; + /// Copy `value` into a fresh arena string. Mirrors `std`'s `From<&mut str>`. + #[inline] + fn from_in(value: &'b mut str, allocator: &'a Arena) -> Self { + Self::from_str_in(value, allocator) + } +} + +impl<'a, A: Allocator + Clone> FromIn for String<'a, A> { + type Allocator = &'a Arena; + /// Build a one-character arena string. Mirrors `std`'s `From`. + #[inline] + fn from_in(value: char, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.push(value); + s + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIn> for String<'a, A> { + type Allocator = &'a Arena; + /// Copy a clone-on-write string into the arena. Mirrors `std`'s `From>`. + #[inline] + fn from_in(value: alloc::borrow::Cow<'b, str>, allocator: &'a Arena) -> Self { + Self::from_str_in(&value, allocator) + } +} + +impl<'a, A: Allocator + Clone> FromIn> for String<'a, A> { + type Allocator = &'a Arena; + /// Copy a boxed string into the arena. Mirrors `std`'s `From>`. + #[inline] + fn from_in(value: alloc::boxed::Box, allocator: &'a Arena) -> Self { + Self::from_str_in(&value, allocator) + } +} + +#[cfg(test)] +mod tests { + use super::utf8_seq_len; + + /// Pins [`utf8_seq_len`] at every UTF-8 lead-byte class boundary. The + /// drain decoder relies on these exact lengths, and the boundary + /// comparisons (`<`) must not drift to `<=`: e.g. `0xE0` leads a 3-byte + /// sequence, never a 2-byte one. + #[test] + fn utf8_seq_len_matches_every_class_boundary() { + assert_eq!(utf8_seq_len(0x00), 1); + assert_eq!(utf8_seq_len(0x7F), 1); + assert_eq!(utf8_seq_len(0x80), 2); + assert_eq!(utf8_seq_len(0xDF), 2); + assert_eq!(utf8_seq_len(0xE0), 3); + assert_eq!(utf8_seq_len(0xEF), 3); + assert_eq!(utf8_seq_len(0xF0), 4); + assert_eq!(utf8_seq_len(0xFF), 4); + } +} diff --git a/crates/multitude/src/strings/string_common.rs b/crates/multitude/src/strings/string_common.rs index 537d93ad9..f88ca28ab 100644 --- a/crates/multitude/src/strings/string_common.rs +++ b/crates/multitude/src/strings/string_common.rs @@ -18,7 +18,7 @@ macro_rules! impl_arena_string_common { /// /// No allocation is performed until the first push. #[must_use] - pub const fn new_in(arena: &'a $crate::Arena) -> Self { + pub(crate) const fn new_in(arena: &'a $crate::Arena) -> Self { Self { inner: $crate::vec::Vec::new_in(arena), } @@ -32,7 +32,7 @@ macro_rules! impl_arena_string_common { /// Panics if the backing allocator fails. Use /// [`Self::try_with_capacity_in`] for a fallible variant. #[must_use] - pub fn with_capacity_in(cap: usize, arena: &'a $crate::Arena) -> Self { + pub(crate) fn with_capacity_in(cap: usize, arena: &'a $crate::Arena) -> Self { Self { inner: $crate::vec::Vec::with_capacity_in(cap, arena), } @@ -44,7 +44,7 @@ macro_rules! impl_arena_string_common { /// /// Returns [`allocator_api2::alloc::AllocError`] if the backing /// allocator fails. - pub fn try_with_capacity_in(cap: usize, arena: &'a $crate::Arena) -> Result { + pub(crate) fn try_with_capacity_in(cap: usize, arena: &'a $crate::Arena) -> Result { Ok(Self { inner: $crate::vec::Vec::try_with_capacity_in(cap, arena)?, }) @@ -70,20 +70,6 @@ macro_rules! impl_arena_string_common { self.inner.capacity() } - /// Return a raw const pointer to the string's elements. - #[must_use] - #[inline] - pub const fn as_ptr(&self) -> *const $Elem { - self.inner.as_ptr() - } - - /// Return a raw mutable pointer to the string's elements. - #[allow(clippy::needless_pass_by_ref_mut, reason = "API shape mirrors std::String::as_mut_ptr")] - #[inline] - pub const fn as_mut_ptr(&mut self) -> *mut $Elem { - self.inner.as_mut_ptr() - } - /// Reserve capacity for at least `additional` more elements. /// /// # Panics @@ -106,6 +92,24 @@ macro_rules! impl_arena_string_common { self.inner.try_reserve(additional) } + /// Reserve capacity for at least `additional` more elements, + /// without the amortized-growth slack of [`Self::reserve`]. + #[inline] + pub fn reserve_exact(&mut self, additional: usize) { + self.inner.reserve_exact(additional); + } + + /// Fallible variant of [`Self::reserve_exact`]. + /// + /// # Errors + /// + /// Returns [`allocator_api2::alloc::AllocError`] if the backing + /// allocator fails. + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), allocator_api2::alloc::AllocError> { + self.inner.try_reserve_exact(additional) + } + /// Release any unused capacity back to the chunk's bump cursor. /// /// O(1) when the backing buffer is at the chunk's bump cursor; @@ -114,6 +118,12 @@ macro_rules! impl_arena_string_common { self.inner.shrink_to_fit(); } + /// Shrink the capacity with a lower bound (in elements). See + /// [`crate::vec::Vec::shrink_to`]. + pub fn shrink_to(&mut self, min_capacity: usize) { + self.inner.shrink_to(min_capacity); + } + /// Truncates this string, removing all contents. /// /// The capacity is preserved. diff --git a/crates/multitude/src/strings/utf16_string.rs b/crates/multitude/src/strings/utf16_string.rs index 390276cb9..8279d7fd6 100644 --- a/crates/multitude/src/strings/utf16_string.rs +++ b/crates/multitude/src/strings/utf16_string.rs @@ -17,15 +17,13 @@ use core::ops::{Bound, Deref, DerefMut, RangeBounds}; use allocator_api2::alloc::{AllocError, Allocator, Global}; use widestring::Utf16Str; -use crate::Arena; use crate::strings::string_common::impl_arena_string_common; +use crate::strings::{ArcUtf16Str, BoxUtf16Str}; +use crate::vec::{FromIteratorIn, Vec}; +use crate::{Arena, FromIn}; /// A growable, mutable UTF-16 string that lives in an [`Arena`]. /// -/// `Utf16String` is a **transient builder**: 32 bytes on 64-bit (data -/// pointer + length + capacity + arena reference). Lengths and -/// capacities are counted in `u16` elements. -/// /// # Example /// /// ``` @@ -38,12 +36,12 @@ use crate::strings::string_common::impl_arena_string_common; /// s.push_str(utf16str!("hello, ")); /// s.push_str(utf16str!("world!")); /// assert_eq!(s.as_utf16_str(), utf16str!("hello, world!")); -/// let frozen = s.into_arena_box_utf16_str(); +/// let frozen = s.into_boxed_utf16_str(); /// assert_eq!(&*frozen, utf16str!("hello, world!")); /// # } /// ``` pub struct Utf16String<'a, A: Allocator + Clone = Global> { - pub(super) inner: crate::vec::Vec<'a, u16, A>, + pub(super) inner: Vec<'a, u16, A>, } impl<'a, A: Allocator + Clone> Utf16String<'a, A> { @@ -81,7 +79,7 @@ impl<'a, A: Allocator + Clone> Utf16String<'a, A> { /// Construct an `Utf16String` by transcoding a `&str` into UTF-16, /// copied into `arena`. #[must_use] - pub fn from_str_in(s: &str, arena: &'a Arena) -> Self { + pub(crate) fn from_str_in(s: &str, arena: &'a Arena) -> Self { let mut out = Self::with_capacity_in(s.len(), arena); out.push_from_str(s); out @@ -307,19 +305,197 @@ impl<'a, A: Allocator + Clone> Utf16String<'a, A> { self.inner.extend_from_slice(staging.as_slice()); } + /// Consume the string, returning the underlying `u16` vector. The + /// `into_bytes` analog for UTF-16. + #[must_use] + pub fn into_vec(self) -> Vec<'a, u16, A> { + self.inner + } + + /// Returns a mutable reference to the underlying `u16` vector. + /// + /// # Safety + /// + /// The caller must keep the units well-formed UTF-16 before the borrow + /// ends; the `Utf16String` invariant is otherwise violated. + #[must_use] + pub unsafe fn as_mut_vec(&mut self) -> &mut Vec<'a, u16, A> { + &mut self.inner + } + + /// Split the string into two at the given `u16` index, returning the + /// tail `[at, len)` as a new `Utf16String` in the same arena and + /// leaving `[0, at)` in `self`. The UTF-8 [`String::split_off`](crate::strings::String::split_off) analog. + /// + /// # Panics + /// + /// Panics if `at` is not on a `char` boundary (i.e. would split a + /// surrogate pair), or is past the end. + #[must_use] + pub fn split_off(&mut self, at: usize) -> Self { + assert!( + self.as_utf16_str().is_char_boundary(at), + "Utf16String::split_off: `at` is not a char boundary" + ); + Self { + inner: self.inner.split_off(at), + } + } + + /// Clone the `u16` units in `src` (an index range into `self`) and append + /// them to the end. The UTF-16 analog of + /// [`String::extend_from_within`](crate::strings::String::extend_from_within). + /// + /// # Panics + /// + /// Panics if the range is out of bounds or its bounds are not on `char` + /// boundaries (i.e. would split a surrogate pair). + pub fn extend_from_within>(&mut self, src: R) { + let len = self.inner.len(); + let start = match src.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i.checked_add(1).expect("extend_from_within: start bound overflows usize"), + Bound::Unbounded => 0, + }; + let end = match src.end_bound() { + Bound::Included(&i) => i.checked_add(1).expect("extend_from_within: end bound overflows usize"), + Bound::Excluded(&i) => i, + Bound::Unbounded => len, + }; + assert!(start <= end, "extend_from_within: start > end"); + assert!(end <= len, "extend_from_within: end > len"); + let s_ref = self.as_utf16_str(); + assert!(s_ref.is_char_boundary(start), "extend_from_within: start is not on a char boundary"); + assert!(s_ref.is_char_boundary(end), "extend_from_within: end is not on a char boundary"); + self.inner.extend_from_within(start..end); + } + /// Freeze into an owned, mutable - /// [`BoxUtf16Str`](crate::strings::BoxUtf16Str). + /// [`BoxUtf16Str`](crate::strings::BoxUtf16Str). [`BoxUtf16Str::from`] + /// is the trait form. + /// + /// **O(n)** — copies the contents. /// - /// **O(n)** — copies the `u16` units into a compact, length-prefixed - /// allocation in the arena's shared chunks and produces an owned - /// [`BoxUtf16Str`](crate::strings::BoxUtf16Str) (8 bytes) whose - /// `Drop` releases the chunk hold. The copy is the deliberate - /// trade-off for `BoxUtf16Str` being a `Send`-safe, - /// atomically-refcounted single pointer that can outlive the arena. + /// # Panics + /// + /// Panics if the underlying allocator fails. #[must_use] - pub fn into_arena_box_utf16_str(self) -> crate::strings::BoxUtf16Str { + pub fn into_boxed_utf16_str(self) -> BoxUtf16Str { self.inner.arena().alloc_utf16_str_box(self.as_utf16_str()) } + + /// Remove the `char`s in the `u16` index range `range`, returning a + /// draining iterator over them. The UTF-16 analog of + /// [`String::drain`](crate::strings::String::drain). + /// + /// The drained range is removed immediately; the returned iterator yields + /// the removed characters (it is also double-ended). + /// + /// # Panics + /// + /// Panics if `range`'s bounds are out of range or not on `char` + /// boundaries (i.e. would split a surrogate pair). + pub fn drain>(&mut self, range: R) -> Utf16Drain<'_, 'a, A> { + let len = self.inner.len(); + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i.checked_add(1).expect("drain: start bound overflows usize"), + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(&i) => i.checked_add(1).expect("drain: end bound overflows usize"), + Bound::Excluded(&i) => i, + Bound::Unbounded => len, + }; + assert!(start <= end, "drain: start > end"); + assert!(end <= len, "drain: end > len"); + let s_ref = self.as_utf16_str(); + assert!(s_ref.is_char_boundary(start), "drain: start is not on a char boundary"); + assert!(s_ref.is_char_boundary(end), "drain: end is not on a char boundary"); + Utf16Drain { + inner: self.inner.drain(start..end), + } + } + + /// Consume the string, returning an arena-lifetime mutable reference + /// `&'a mut Utf16Str`. Mirrors [`String::leak`](crate::strings::String::leak). + /// + /// **O(1) and allocation-free**: reinterprets the existing buffer in place. + #[must_use] + pub fn leak(self) -> &'a mut Utf16Str { + let units = self.inner.leak(); + // SAFETY: `Utf16String` maintains the well-formed-UTF-16 invariant. + unsafe { Utf16Str::from_slice_unchecked_mut(units) } + } +} + +/// Draining iterator over a `u16` index range of a [`Utf16String`], returned +/// by [`Utf16String::drain`]. Yields the removed [`char`]s (double-ended). +pub struct Utf16Drain<'d, 'a, A: Allocator + Clone> { + inner: crate::vec::Drain<'d, 'a, u16, A>, +} + +impl fmt::Debug for Utf16Drain<'_, '_, A> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("Utf16Drain").finish_non_exhaustive() + } +} + +impl Iterator for Utf16Drain<'_, '_, A> { + type Item = char; + + fn next(&mut self) -> Option { + let u0 = self.inner.next()?; + let decoded = if (0xD800..=0xDBFF).contains(&u0) { + let u1 = self.inner.next().expect("Utf16Drain holds valid UTF-16"); + char::decode_utf16([u0, u1]).next() + } else { + char::decode_utf16([u0]).next() + }; + Some(decoded.expect("non-empty").expect("Utf16Drain holds valid UTF-16")) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + // Each remaining `char` is 1–2 `u16` units. + let units = self.inner.len(); + (units.div_ceil(2), Some(units)) + } +} + +impl DoubleEndedIterator for Utf16Drain<'_, '_, A> { + fn next_back(&mut self) -> Option { + let last = self.inner.next_back()?; + // A trailing unit at a char boundary is either a BMP scalar or the + // low surrogate completing a pair; never a lone high surrogate. + let decoded = if (0xDC00..=0xDFFF).contains(&last) { + let high = self.inner.next_back().expect("Utf16Drain holds valid UTF-16"); + char::decode_utf16([high, last]).next() + } else { + char::decode_utf16([last]).next() + }; + Some(decoded.expect("non-empty").expect("Utf16Drain holds valid UTF-16")) + } +} + +impl core::iter::FusedIterator for Utf16Drain<'_, '_, A> {} + +impl<'a, A: Allocator + Clone> From> for BoxUtf16Str { + /// Freeze a [`Utf16String`] into an immutable + /// [`BoxUtf16Str`](crate::strings::BoxUtf16Str). + #[inline] + fn from(s: Utf16String<'a, A>) -> Self { + s.into_boxed_utf16_str() + } +} + +impl<'a, A: Allocator + Clone + Send + Sync> From> for ArcUtf16Str { + /// Freeze a [`Utf16String`] into a shared + /// [`ArcUtf16Str`](crate::strings::ArcUtf16Str). + #[inline] + fn from(s: Utf16String<'a, A>) -> Self { + s.inner.arena().alloc_utf16_str_arc(s.as_utf16_str()) + } } impl Deref for Utf16String<'_, A> { @@ -455,7 +631,7 @@ impl fmt::Write for Utf16String<'_, A> { } } -impl<'a, A: Allocator + Clone> crate::vec::FromIteratorIn for Utf16String<'a, A> { +impl<'a, A: Allocator + Clone> FromIteratorIn for Utf16String<'a, A> { type Allocator = &'a Arena; fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { @@ -465,7 +641,7 @@ impl<'a, A: Allocator + Clone> crate::vec::FromIteratorIn for Utf16String< } } -impl<'a, 'b, A: Allocator + Clone> crate::vec::FromIteratorIn<&'b Utf16Str> for Utf16String<'a, A> { +impl<'a, 'b, A: Allocator + Clone> FromIteratorIn<&'b Utf16Str> for Utf16String<'a, A> { type Allocator = &'a Arena; fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { @@ -475,7 +651,7 @@ impl<'a, 'b, A: Allocator + Clone> crate::vec::FromIteratorIn<&'b Utf16Str> for } } -impl<'a, 'b, A: Allocator + Clone> crate::vec::FromIteratorIn<&'b str> for Utf16String<'a, A> { +impl<'a, 'b, A: Allocator + Clone> FromIteratorIn<&'b str> for Utf16String<'a, A> { type Allocator = &'a Arena; fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { @@ -484,3 +660,176 @@ impl<'a, 'b, A: Allocator + Clone> crate::vec::FromIteratorIn<&'b str> for Utf16 s } } + +impl<'a, 'b, A: Allocator + Clone> FromIteratorIn<&'b char> for Utf16String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl<'a, A: Allocator + Clone> FromIteratorIn for Utf16String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl<'a, A: Allocator + Clone> FromIteratorIn> for Utf16String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIteratorIn> for Utf16String<'a, A> { + type Allocator = &'a Arena; + + fn from_iter_in>>(iter: I, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.extend(iter); + s + } +} + +impl AsRef<[u16]> for Utf16String<'_, A> { + #[inline] + fn as_ref(&self) -> &[u16] { + self.as_slice() + } +} + +impl core::ops::Add<&Utf16Str> for Utf16String<'_, A> { + type Output = Self; + #[inline] + fn add(mut self, rhs: &Utf16Str) -> Self { + self.push_str(rhs); + self + } +} + +impl core::ops::AddAssign<&Utf16Str> for Utf16String<'_, A> { + #[inline] + fn add_assign(&mut self, rhs: &Utf16Str) { + self.push_str(rhs); + } +} + +impl core::ops::Index for Utf16String<'_, A> +where + I: core::ops::RangeBounds + core::slice::SliceIndex<[u16], Output = [u16]>, +{ + type Output = Utf16Str; + #[inline] + fn index(&self, index: I) -> &Utf16Str { + core::ops::Index::index(self.as_utf16_str(), index) + } +} + +impl core::ops::IndexMut for Utf16String<'_, A> +where + I: core::ops::RangeBounds + core::slice::SliceIndex<[u16], Output = [u16]>, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut Utf16Str { + core::ops::IndexMut::index_mut(self.as_mut_utf16_str(), index) + } +} + +impl<'b, A: Allocator + Clone> Extend<&'b char> for Utf16String<'_, A> { + fn extend>(&mut self, iter: I) { + for c in iter { + self.push(*c); + } + } +} + +impl<'b, A: Allocator + Clone> Extend> for Utf16String<'_, A> { + fn extend>>(&mut self, iter: I) { + for s in iter { + self.push_from_str(&s); + } + } +} + +impl Extend for Utf16String<'_, A> { + fn extend>(&mut self, iter: I) { + for s in iter { + self.push_from_str(&s); + } + } +} + +impl Extend> for Utf16String<'_, A> { + fn extend>>(&mut self, iter: I) { + for s in iter { + self.push_from_str(&s); + } + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIn<&'b Utf16Str> for Utf16String<'a, A> { + type Allocator = &'a Arena; + /// Copy `value` into a fresh arena string. The `&Utf16Str` analog of + /// `std`'s `From<&str> for String`. + #[inline] + fn from_in(value: &'b Utf16Str, allocator: &'a Arena) -> Self { + Self::from_utf16_str_in(value, allocator) + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIn<&'b str> for Utf16String<'a, A> { + type Allocator = &'a Arena; + /// Transcode a UTF-8 `&str` into a fresh arena UTF-16 string. + #[inline] + fn from_in(value: &'b str, allocator: &'a Arena) -> Self { + Self::from_str_in(value, allocator) + } +} + +impl<'a, A: Allocator + Clone> FromIn for Utf16String<'a, A> { + type Allocator = &'a Arena; + /// Build a one-character arena string. Mirrors `std`'s `From`. + #[inline] + fn from_in(value: char, allocator: &'a Arena) -> Self { + let mut s = Self::new_in(allocator); + s.push(value); + s + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIn> for Utf16String<'a, A> { + type Allocator = &'a Arena; + /// Copy a clone-on-write UTF-16 string into the arena. + #[inline] + fn from_in(value: alloc::borrow::Cow<'b, Utf16Str>, allocator: &'a Arena) -> Self { + Self::from_utf16_str_in(&value, allocator) + } +} + +impl<'a, 'b, A: Allocator + Clone> FromIn> for Utf16String<'a, A> { + type Allocator = &'a Arena; + /// Transcode a clone-on-write UTF-8 string into the arena. + #[inline] + fn from_in(value: alloc::borrow::Cow<'b, str>, allocator: &'a Arena) -> Self { + Self::from_str_in(&value, allocator) + } +} + +impl<'a, A: Allocator + Clone> FromIn> for Utf16String<'a, A> { + type Allocator = &'a Arena; + /// Transcode a boxed UTF-8 string into the arena. + #[inline] + fn from_in(value: alloc::boxed::Box, allocator: &'a Arena) -> Self { + Self::from_str_in(&value, allocator) + } +} diff --git a/crates/multitude/src/vec/basic.rs b/crates/multitude/src/vec/basic.rs index ab2619396..4a85df266 100644 --- a/crates/multitude/src/vec/basic.rs +++ b/crates/multitude/src/vec/basic.rs @@ -14,7 +14,7 @@ impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { /// Create an empty vector backed by `arena`. No allocation until the first push. #[inline] #[must_use] - pub const fn new_in(arena: &'a Arena) -> Self { + pub(crate) const fn new_in(arena: &'a Arena) -> Self { Self::from_buf(ArenaBuf::new(), arena) } @@ -25,7 +25,7 @@ impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { /// Panics if the backing allocator fails or if the data alignment is at least 32 KiB. /// Use [`Self::try_with_capacity_in`] for a fallible variant. #[must_use] - pub fn with_capacity_in(cap: usize, arena: &'a Arena) -> Self { + pub(crate) fn with_capacity_in(cap: usize, arena: &'a Arena) -> Self { (Self::try_with_capacity_in(cap, arena)).expect_alloc() } @@ -35,7 +35,7 @@ impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { /// /// Returns [`AllocError`] if the backing allocator fails or if the data /// alignment is at least 32 KiB. - pub fn try_with_capacity_in(cap: usize, arena: &'a Arena) -> Result { + pub(crate) fn try_with_capacity_in(cap: usize, arena: &'a Arena) -> Result { let mut v = Self::new_in(arena); if cap > 0 { v.try_grow_to(cap)?; @@ -178,9 +178,13 @@ impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { T: Copy, { let src = other.as_ref(); - let needed = self.buf.len().checked_add(src.len()).ok_or(AllocError)?; - if needed > self.buf.cap() { - self.try_grow_to(grow_target(self.buf.cap(), needed))?; + let cap = self.buf.cap(); + let len = self.buf.len(); + // `len <= cap` is a Vec invariant, so this subtraction never underflows + // and avoids the `checked_add` overflow guard on the hot fast path. + if cap - len < src.len() { + let needed = len.checked_add(src.len()).ok_or(AllocError)?; + self.try_grow_to(grow_target(cap, needed))?; } self.buf.extend_copy(src); Ok(()) @@ -215,6 +219,14 @@ impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { pub const fn as_mut_ptr(&mut self) -> *mut T { self.buf.as_mut_ptr() } + + /// Returns the remaining spare capacity of the vector as a slice of + /// `MaybeUninit`. Mirrors [`std::vec::Vec::spare_capacity_mut`]. + #[must_use] + #[inline] + pub fn spare_capacity_mut(&mut self) -> &mut [core::mem::MaybeUninit] { + self.buf.spare_capacity_mut() + } } impl Vec<'_, T, A> { diff --git a/crates/multitude/src/vec/freeze.rs b/crates/multitude/src/vec/freeze.rs index 850737871..d3ba9410d 100644 --- a/crates/multitude/src/vec/freeze.rs +++ b/crates/multitude/src/vec/freeze.rs @@ -1,8 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. //! Freeze a transient builder into arena-owned `Arc` or `Box` slices. +//! +//! The infallible freezes are exposed as `From>` impls on +//! [`Arc`](crate::Arc) / [`Box`](crate::Box) (mirroring `std`'s +//! `From> for Box<[T]>` / `Arc<[T]>`) plus the `std`-named +//! [`Vec::into_boxed_slice`] / [`Vec::leak`] methods. Fallible variants +//! ([`Vec::try_into_arc`] / [`Vec::try_into_boxed_slice`]) have no `std` +//! counterpart and stay as inherent methods. -use core::mem::ManuallyDrop; +use core::mem::{self, ManuallyDrop}; +use core::slice; use allocator_api2::alloc::{AllocError, Allocator}; @@ -10,39 +18,55 @@ use super::Vec; use crate::arc::Arc; use crate::r#box::Box; -impl Vec<'_, T, A> { - /// Freeze into an [`Arc<[T], A>`](crate::Arc). +impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { + /// Freeze into a [`Box<[T], A>`](crate::Box). /// - /// The contents are moved into a fresh shared-flavor arena - /// allocation. + /// **O(n)** — moves the elements into a fresh shared allocation + /// (no `Copy`/`Clone` required). Mirrors + /// [`std::vec::Vec::into_boxed_slice`]; [`Box::from`] is the trait form. /// /// # Panics /// - /// Panics if the underlying allocator fails. + /// Panics if the underlying allocator fails, or — for `T: Drop` — if + /// `len` exceeds `u16::MAX`. #[must_use] - pub fn into_arena_arc(self) -> Arc<[T], A> - where - T: Send + Sync, - A: Send + Sync, - { + pub fn into_boxed_slice(self) -> Box<[T], A> { let arena = self.arena; let mut me = ManuallyDrop::new(self); let iter = me.buf.drain_all(); - let arc = arena.alloc_slice_fill_iter_arc::(iter); + let bx = arena.alloc_slice_fill_iter_box::(iter); // `drain_all` set `buf.len = 0`, so `into_inner`'s normal `Drop` // only releases the (unused) backing buffer. drop(ManuallyDrop::into_inner(me)); - arc + bx } - /// Fallible variant of [`Self::into_arena_arc`]. + /// Fallible variant of [`Self::into_boxed_slice`]. /// /// # Errors /// /// Returns [`AllocError`] if the backing shared-flavor allocation /// fails. On error, `self` is consumed and any elements remaining /// after a partial move are dropped before this function returns. - pub fn try_into_arena_arc(self) -> Result, AllocError> + pub fn try_into_boxed_slice(self) -> Result, AllocError> { + let arena = self.arena; + let mut me = ManuallyDrop::new(self); + let iter = me.buf.drain_all(); + let result = arena.try_alloc_slice_fill_iter_box::(iter); + // See `into_boxed_slice`. + drop(ManuallyDrop::into_inner(me)); + result + } + + /// Fallible variant of the [`Arc<[T], A>`](crate::Arc) freeze + /// ([`Arc::from`]). + /// + /// # Errors + /// + /// Returns [`AllocError`] if the backing shared-flavor allocation + /// fails. On error, `self` is consumed and any elements remaining + /// after a partial move are dropped before this function returns. + pub fn try_into_arc(self) -> Result, AllocError> where T: Send + Sync, A: Send + Sync, @@ -51,33 +75,55 @@ impl Vec<'_, T, A> { let mut me = ManuallyDrop::new(self); let iter = me.buf.drain_all(); let result = arena.try_alloc_slice_fill_iter_arc::(iter); - // See `into_arena_arc`. + // See `into_boxed_slice`. drop(ManuallyDrop::into_inner(me)); result } - /// Freeze into a [`Box<[T], A>`](crate::Box). - /// - /// **O(1)**: the existing buffer is handed to the `Box` directly. - /// `Box<[T]>`'s `Drop` runs `drop_in_place::<[T]>` over the slice - /// (using its fat-pointer length) when the box is dropped, so no - /// trailing chunk drop entry is needed regardless of `T: Drop`. + /// Consume the `Vec`, returning an arena-lifetime mutable slice + /// reference `&'a mut [T]`. Mirrors [`std::vec::Vec::leak`]. /// - /// If the buffer's tail still sits at the chunk's bump cursor, the - /// unused tail (`cap - len`) is returned to the cursor. + /// **O(1) and allocation-free**: the existing buffer is reinterpreted + /// as a slice reference in place. No copy, no new allocation. The + /// unused tail (`cap - len`) is left in the chunk and reclaimed when + /// the arena is dropped. /// - /// # Panics - /// - /// Panics if the underlying allocator fails on the copy fallback - /// for the ZST / empty-builder cases. + /// Available only when `T` does not need `Drop` (compile-time + /// asserted). For drop types, freeze via [`Box::from`] / [`Arc::from`]. #[must_use] - pub fn into_arena_box(self) -> Box<[T], A> { + pub fn leak(self) -> &'a mut [T] { + const { + assert!( + !mem::needs_drop::(), + "Vec::leak requires T not to need Drop; freeze via Box::from / Arc::from instead", + ); + } + let mut me = ManuallyDrop::new(self); + let ptr = me.buf.as_mut_ptr(); + let len = me.buf.len(); + // SAFETY: by `ArenaBuf`'s invariants, `ptr` addresses `len` + // initialized `T`s in an arena chunk that outlives `'a`. We + // `ManuallyDrop` the `Vec` so neither the `ArenaBuf` nor its + // contained elements are dropped here. Since `T` does not need + // `Drop` (const-asserted above), abandoning the buffer without + // registering a chunk drop entry is sound — the chunk storage + // itself is reclaimed at arena teardown. + unsafe { slice::from_raw_parts_mut(ptr, len) } + } + + /// Internal: shared body for the infallible `Arc<[T]>` freeze, used by + /// `From> for Arc<[T], A>`. + pub(crate) fn freeze_into_arc(self) -> Arc<[T], A> + where + T: Send + Sync, + A: Send + Sync, + { let arena = self.arena; let mut me = ManuallyDrop::new(self); let iter = me.buf.drain_all(); - let bx = arena.alloc_slice_fill_iter_box::(iter); - // See `into_arena_arc`. + let arc = arena.alloc_slice_fill_iter_arc::(iter); + // See `into_boxed_slice`. drop(ManuallyDrop::into_inner(me)); - bx + arc } } diff --git a/crates/multitude/src/vec/from_in.rs b/crates/multitude/src/vec/from_in.rs new file mode 100644 index 000000000..23bd2e407 --- /dev/null +++ b/crates/multitude/src/vec/from_in.rs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Arena-aware [`FromIn`] conversions for [`Vec`], mirroring `std`'s +//! `From<&[T]>` / `From<&mut [T]>` / `From<[T; N]>` / `From>` / +//! `From>` impls. + +use alloc::borrow::Cow; +use alloc::boxed::Box as StdBox; + +use allocator_api2::alloc::Allocator; + +use super::Vec; +use crate::{Arena, FromIn}; + +impl<'a, 'b, T: Clone, A: Allocator + Clone> FromIn<&'b [T]> for Vec<'a, T, A> { + type Allocator = &'a Arena; + + /// Clone each element of `value` into a fresh arena vector. Mirrors + /// `std`'s `From<&[T]> for Vec`. + #[inline] + fn from_in(value: &'b [T], allocator: &'a Arena) -> Self { + Self::from_iter_in(value.iter().cloned(), allocator) + } +} + +impl<'a, 'b, T: Clone, A: Allocator + Clone> FromIn<&'b mut [T]> for Vec<'a, T, A> { + type Allocator = &'a Arena; + + /// Clone each element of `value` into a fresh arena vector. Mirrors + /// `std`'s `From<&mut [T]> for Vec`. + #[inline] + fn from_in(value: &'b mut [T], allocator: &'a Arena) -> Self { + Self::from_iter_in(value.iter().cloned(), allocator) + } +} + +impl<'a, T, A: Allocator + Clone, const N: usize> FromIn<[T; N]> for Vec<'a, T, A> { + type Allocator = &'a Arena; + + /// Move the array's elements into a fresh arena vector. Mirrors `std`'s + /// `From<[T; N]> for Vec`. + #[inline] + fn from_in(value: [T; N], allocator: &'a Arena) -> Self { + Self::from_iter_in(value, allocator) + } +} + +impl<'a, T, A: Allocator + Clone> FromIn> for Vec<'a, T, A> { + type Allocator = &'a Arena; + + /// Move the boxed slice's elements into a fresh arena vector. Mirrors + /// `std`'s `From> for Vec`. + #[inline] + fn from_in(value: StdBox<[T]>, allocator: &'a Arena) -> Self { + Self::from_iter_in(value, allocator) + } +} + +impl<'a, 'b, T: Clone, A: Allocator + Clone> FromIn> for Vec<'a, T, A> { + type Allocator = &'a Arena; + + /// Build a fresh arena vector from a clone-on-write slice — cloning a + /// borrowed slice or moving an owned one. Mirrors `std`'s + /// `From> for Vec`. + #[inline] + fn from_in(value: Cow<'b, [T]>, allocator: &'a Arena) -> Self { + match value { + Cow::Borrowed(s) => Self::from_iter_in(s.iter().cloned(), allocator), + Cow::Owned(v) => Self::from_iter_in(v, allocator), + } + } +} diff --git a/crates/multitude/src/vec/mod.rs b/crates/multitude/src/vec/mod.rs index c6d37f9ce..7d3c81906 100644 --- a/crates/multitude/src/vec/mod.rs +++ b/crates/multitude/src/vec/mod.rs @@ -15,11 +15,6 @@ )] //! Arena-backed growable vectors and the `vec!` macro. -//! -//! [`Vec`] is a transient builder that can be frozen into compact arena -//! handles such as [`Vec::into_arena_arc`] or [`Vec::into_arena_box`]. -//! -//! For the string equivalents, see [`crate::strings`]. use core::marker::PhantomData; use core::mem; @@ -33,9 +28,11 @@ mod basic; mod collect_in; mod drain; mod freeze; +mod from_in; mod from_iterator_in; mod into_iter; mod mutate; +mod splice; mod traits; mod vec_macro; @@ -43,16 +40,13 @@ pub use collect_in::CollectIn; pub use drain::Drain; pub use from_iterator_in::FromIteratorIn; pub use into_iter::IntoIter; +pub use splice::Splice; #[doc(inline)] pub use crate::__multitude_vec as vec; /// A growable, mutable vector that lives in an [`Arena`]. /// -/// `Vec` is a **transient builder**: 32 bytes on 64-bit (data pointer + -/// length + capacity + arena reference). Its purpose is to be filled and -/// then frozen via [`Self::into_arena_arc`] or [`Self::into_arena_box`]. -/// /// `push`, `pop`, `extend`, `iter`, and other standard vector methods /// behave the same as on `std::vec::Vec`. /// @@ -66,7 +60,7 @@ pub use crate::__multitude_vec as vec; /// v.push(1); /// v.push(2); /// v.push(3); -/// let frozen = v.into_arena_box(); +/// let frozen = v.into_boxed_slice(); /// assert_eq!(&*frozen, &[1, 2, 3]); /// ``` pub struct Vec<'a, T, A: Allocator + Clone = Global> { @@ -154,6 +148,11 @@ impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { // hence for `'a`); the reservation is fresh and // non-overlapping with the old buffer. unsafe { self.buf.replace_buffer_raw(new_ptr, new_cap_actual) }; + // The previous oversized chunk is left in `retired_local` + // until arena reset/drop. It is NOT released here: a + // zero-copy `split_off` can leave a sibling buffer + // pointing into the same chunk, and freeing it on one + // half's growth would dangle the other (use-after-free). return Ok(()); } self.arena.refill_local(refill_hint)?; diff --git a/crates/multitude/src/vec/mutate.rs b/crates/multitude/src/vec/mutate.rs index 905842cab..5a8459c20 100644 --- a/crates/multitude/src/vec/mutate.rs +++ b/crates/multitude/src/vec/mutate.rs @@ -9,7 +9,7 @@ use allocator_api2::alloc::{AllocError, Allocator}; use super::Vec; use crate::internal::arena_buf::ArenaBuf; -/// Reclaim-byte arithmetic extracted from [`Vec::shrink_to_fit`]. +/// Reclaim-byte arithmetic for [`Vec::shrink_to_fit`]. #[inline] #[cfg_attr(test, mutants::skip)] // arithmetic not observable via public API fn shrink_reclaim_bytes(cap: usize, len: usize, elem: usize) -> usize { @@ -111,12 +111,24 @@ impl Vec<'_, T, A> { // correctness gate. #[cfg_attr(test, mutants::skip)] pub fn shrink_to_fit(&mut self) { + self.shrink_to(0); + } + + /// Shrink the capacity with a lower bound. + /// + /// The capacity will remain at least as large as both `self.len()` and + /// `min_capacity`. Reclamation only succeeds while the buffer still sits + /// at the chunk's bump cursor; otherwise this is a no-op (matching + /// [`std::vec::Vec::shrink_to`]'s "best-effort" contract). + #[cfg_attr(test, mutants::skip)] + pub fn shrink_to(&mut self, min_capacity: usize) { if const { mem::size_of::() == 0 } { return; } let len = self.buf.len(); + let target = len.max(min_capacity); let cap = self.buf.cap(); - if cap == len { + if cap <= target { return; } let elem = mem::size_of::(); @@ -133,12 +145,46 @@ impl Vec<'_, T, A> { return; } let end_addr = data_addr + total_bytes; - let reclaim_bytes = shrink_reclaim_bytes(cap, len, elem); + let reclaim_bytes = shrink_reclaim_bytes(cap, target, elem); if self.arena.current_local().try_reclaim_tail(end_addr, reclaim_bytes) { - // SAFETY: the chunk reclaimed `[len*elem, cap*elem)`, so this + // SAFETY: the chunk reclaimed `[target*elem, cap*elem)`, so this // buffer no longer owns that span; the live prefix `[0, len)` - // is untouched and still initialized, and `len <= len`. - unsafe { self.buf.set_cap(len) } + // is untouched and still initialized, and `len <= target`. + unsafe { self.buf.set_cap(target) } + } + } + + /// Clone the elements in `src` (an index range into `self`) and append + /// them to the end. Mirrors [`std::vec::Vec::extend_from_within`]. + /// + /// # Panics + /// + /// Panics if the range is out of bounds, or if the backing allocator + /// fails while reserving. + pub fn extend_from_within>(&mut self, src: R) + where + T: Clone, + { + let len = self.buf.len(); + let start = match src.start_bound() { + core::ops::Bound::Included(&n) => n, + core::ops::Bound::Excluded(&n) => n.checked_add(1).expect("extend_from_within: start bound overflows usize"), + core::ops::Bound::Unbounded => 0, + }; + let end = match src.end_bound() { + core::ops::Bound::Included(&n) => n.checked_add(1).expect("extend_from_within: end bound overflows usize"), + core::ops::Bound::Excluded(&n) => n, + core::ops::Bound::Unbounded => len, + }; + assert!(start <= end, "extend_from_within: start > end"); + assert!(end <= len, "extend_from_within: range end out of bounds"); + let count = end - start; + // Reserve up front so the subsequent pushes cannot relocate the + // buffer (which would invalidate the source indices we read from). + self.reserve(count); + for i in start..end { + let cloned = self.buf.as_slice()[i].clone(); + self.push(cloned); } } @@ -357,3 +403,35 @@ impl Vec<'_, T, A> { if predicate(last) { self.buf.pop() } else { None } } } + +impl<'a, T, A: Allocator + Clone, const N: usize> Vec<'a, [T; N], A> { + /// Flatten a `Vec<[T; N]>` into a `Vec` in place (no copy). Mirrors + /// [`std::vec::Vec::into_flattened`]. + /// + /// # Panics + /// + /// Panics on the (practically unreachable) `len * N` / `cap * N` overflow. + #[must_use] + pub fn into_flattened(self) -> Vec<'a, T, A> { + let arena = self.arena; + let mut me = core::mem::ManuallyDrop::new(self); + let len = me.buf.len(); + let cap = me.buf.cap(); + let ptr = me.buf.as_mut_ptr().cast::(); + let new_len = len.checked_mul(N).expect("Vec::into_flattened: length overflow"); + let new_cap = if mem::size_of::() == 0 { + usize::MAX + } else { + cap.checked_mul(N).expect("Vec::into_flattened: capacity overflow") + }; + // SAFETY: `[T; N]` and a run of `N` `T`s share layout and alignment, + // so the buffer holds `len * N` initialized `T`s within `cap * N` + // slots of the same arena chunk that outlives `'a`. `ptr` is non-null + // (it came from a `NonNull` buffer base) and `T`-aligned (alignment of + // `[T; N]` equals that of `T`). `ManuallyDrop` keeps the source buffer + // and its elements from being dropped here; ownership of the elements + // transfers wholesale to the returned `Vec`. + let buf = unsafe { ArenaBuf::from_raw_parts(core::ptr::NonNull::new_unchecked(ptr), new_len, new_cap) }; + Vec::from_buf(buf, arena) + } +} diff --git a/crates/multitude/src/vec/splice.rs b/crates/multitude/src/vec/splice.rs new file mode 100644 index 000000000..25bc21577 --- /dev/null +++ b/crates/multitude/src/vec/splice.rs @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! [`Vec::splice`] and its [`Splice`] iterator. + +use core::fmt; +use core::iter::FusedIterator; +use core::marker::PhantomData; +use core::ops::{Bound, RangeBounds}; + +use allocator_api2::alloc::Allocator; + +use super::Vec; + +impl<'a, T, A: Allocator + Clone> Vec<'a, T, A> { + /// Replace the elements in `range` with the contents of `replace_with`, + /// returning an iterator over the removed elements. Mirrors + /// [`std::vec::Vec::splice`]. + /// + /// # Divergence from `std` + /// + /// `std::vec::Vec::splice` is lazy: it consumes `replace_with` and inserts + /// the replacement only when the returned iterator is dropped. This + /// implementation is **eager** — `replace_with` is fully consumed and both + /// the removal and the insertion happen before `splice` returns; the + /// returned [`Splice`] then only yields the already-removed elements. + /// + /// The final contents of the vector are the same, but the observable side + /// effects differ: `replace_with`'s iterator (and any of its side effects + /// or panics) runs eagerly rather than on drop, the vector is mutated + /// immediately rather than on drop, and dropping the returned [`Splice`] + /// without consuming it still leaves the replacement in place. + /// + /// # Panics + /// + /// Panics if the start of the range is greater than the end, or if the end + /// is greater than `len`. + pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, 'a, T, A> + where + R: RangeBounds, + I: IntoIterator, + { + let len = self.buf.len(); + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i.checked_add(1).expect("splice: start bound overflows usize"), + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(&i) => i.checked_add(1).expect("splice: end bound overflows usize"), + Bound::Excluded(&i) => i, + Bound::Unbounded => len, + }; + assert!(start <= end, "splice: start > end"); + assert!(end <= len, "splice: end > len"); + + // Collect the replacement first: this is the only step that can panic + // (allocation aborts aside), and at this point `self` is untouched. + let replacement: allocator_api2::vec::Vec = replace_with.into_iter().collect(); + + // Pop the whole tail `[start, len)` off (reverse order, so reverse it + // back), then split it into the removed prefix `[start, end)` and the + // kept suffix `[end, len)`. + let tail_count = len - start; + let mut tail: allocator_api2::vec::Vec = allocator_api2::vec::Vec::with_capacity(tail_count); + for _ in 0..tail_count { + tail.push(self.buf.pop().expect("tail length matches len - start")); + } + tail.reverse(); + let removed_count = end - start; + let mut tail_iter = tail.into_iter(); + let removed: allocator_api2::vec::Vec = tail_iter.by_ref().take(removed_count).collect(); + let kept: allocator_api2::vec::Vec = tail_iter.collect(); + + // `self.buf` is now the prefix `[0, start)`. Append the replacement, + // then the kept suffix. + self.reserve( + replacement + .len() + .checked_add(kept.len()) + .expect("splice: replacement + kept length overflows usize"), + ); + for elem in replacement { + self.push(elem); + } + for elem in kept { + self.push(elem); + } + + Splice { + removed: removed.into_iter(), + _marker: PhantomData, + } + } +} + +/// Splicing iterator returned from [`Vec::splice`]; yields the removed +/// elements (double-ended). The replacement has already been inserted. +pub struct Splice<'d, 'a, T, A: Allocator + Clone> { + removed: allocator_api2::vec::IntoIter, + _marker: PhantomData<&'d mut Vec<'a, T, A>>, +} + +impl fmt::Debug for Splice<'_, '_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Splice").field("remaining", &self.removed.len()).finish() + } +} + +impl Iterator for Splice<'_, '_, T, A> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.removed.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.removed.size_hint() + } +} + +impl DoubleEndedIterator for Splice<'_, '_, T, A> { + #[inline] + fn next_back(&mut self) -> Option { + self.removed.next_back() + } +} + +impl ExactSizeIterator for Splice<'_, '_, T, A> {} +impl FusedIterator for Splice<'_, '_, T, A> {} diff --git a/crates/multitude/src/vec/traits.rs b/crates/multitude/src/vec/traits.rs index 086f3bbfb..918b88c61 100644 --- a/crates/multitude/src/vec/traits.rs +++ b/crates/multitude/src/vec/traits.rs @@ -216,3 +216,54 @@ impl io::Write for Vec<'_, u8, A> { Ok(()) } } + +impl, T, A: Allocator + Clone> core::ops::Index for Vec<'_, T, A> { + type Output = I::Output; + #[inline] + fn index(&self, index: I) -> &Self::Output { + core::ops::Index::index(&**self, index) + } +} + +impl, T, A: Allocator + Clone> core::ops::IndexMut for Vec<'_, T, A> { + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + core::ops::IndexMut::index_mut(&mut **self, index) + } +} + +impl AsRef for Vec<'_, T, A> { + #[inline] + fn as_ref(&self) -> &Self { + self + } +} + +impl AsMut for Vec<'_, T, A> { + #[inline] + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl<'a, T, A: Allocator + Clone, const N: usize> TryFrom> for [T; N] { + type Error = Vec<'a, T, A>; + + /// Consume the `Vec` into an array `[T; N]` when `len == N`; on a length + /// mismatch the original `Vec` is returned unchanged. Mirrors `std`'s + /// `TryFrom> for [T; N]`. + fn try_from(mut v: Vec<'a, T, A>) -> Result { + if v.len() != N { + return Err(v); + } + // SAFETY: `v` has exactly `N` initialized elements. Read them out as + // an array, then set the length to 0 so the `Vec`'s `Drop` does not + // re-drop the moved-out elements (the backing buffer is released + // without touching them). + unsafe { + let arr = core::ptr::read(v.as_ptr().cast::<[T; N]>()); + v.set_len(0); + Ok(arr) + } + } +} diff --git a/crates/multitude/src/vec/vec_macro.rs b/crates/multitude/src/vec/vec_macro.rs index 739f54fb9..79680e693 100644 --- a/crates/multitude/src/vec/vec_macro.rs +++ b/crates/multitude/src/vec/vec_macro.rs @@ -25,11 +25,11 @@ #[macro_export] macro_rules! __multitude_vec { (in $arena:expr) => { - $crate::vec::Vec::new_in($arena) + $crate::Arena::alloc_vec($arena) }; (in $arena:expr; $elem:expr; $n:expr) => {{ let __multitude_n: ::core::primitive::usize = $n; - let mut __multitude_buf = $crate::vec::Vec::with_capacity_in(__multitude_n, $arena); + let mut __multitude_buf = $crate::Arena::alloc_vec_with_capacity($arena, __multitude_n); __multitude_buf.resize(__multitude_n, $elem); __multitude_buf }}; diff --git a/crates/multitude/src/zerocopy.rs b/crates/multitude/src/zerocopy.rs index e061a60d6..7c1eda8ac 100644 --- a/crates/multitude/src/zerocopy.rs +++ b/crates/multitude/src/zerocopy.rs @@ -53,7 +53,7 @@ impl<'a, A: Allocator + Clone> ZerocopyView<'a, A> { /// Panics if the backing allocator fails or if `T` requires alignment of 64 KiB or greater (which exceeds the arena chunk alignment). #[must_use] #[inline] - pub fn alloc(&self) -> &'a mut T { + pub fn alloc(&self) -> &'a mut T { self.arena .try_alloc_with::(T::new_zeroed) .expect("zerocopy: arena allocation failed") @@ -66,7 +66,7 @@ impl<'a, A: Allocator + Clone> ZerocopyView<'a, A> { /// Returns [`AllocError`] if the backing allocator fails or if `T` requires alignment /// >= 64 KiB. #[inline] - pub fn try_alloc(&self) -> Result<&'a mut T, AllocError> { + pub fn try_alloc(&self) -> Result<&'a mut T, AllocError> { self.arena.try_alloc_with::(T::new_zeroed) } @@ -77,7 +77,7 @@ impl<'a, A: Allocator + Clone> ZerocopyView<'a, A> { /// Panics if the backing allocator fails or if `T` requires alignment of 64 KiB or greater. #[must_use] #[inline] - pub fn alloc_slice(&self, len: usize) -> &'a mut [T] { + pub fn alloc_slice(&self, len: usize) -> &'a mut [T] { self.arena .try_alloc_slice_fill_with(len, |_| T::new_zeroed()) .expect("zerocopy: arena allocation failed") @@ -90,7 +90,7 @@ impl<'a, A: Allocator + Clone> ZerocopyView<'a, A> { /// Returns [`AllocError`] if the backing allocator fails or if `T` requires alignment /// >= 64 KiB. #[inline] - pub fn try_alloc_slice(&self, len: usize) -> Result<&'a mut [T], AllocError> { + pub fn try_alloc_slice(&self, len: usize) -> Result<&'a mut [T], AllocError> { self.arena.try_alloc_slice_fill_with(len, |_| T::new_zeroed()) } diff --git a/crates/multitude/tests/alloc_ref.rs b/crates/multitude/tests/alloc_ref.rs index 5ae6f442d..f935aa120 100644 --- a/crates/multitude/tests/alloc_ref.rs +++ b/crates/multitude/tests/alloc_ref.rs @@ -233,10 +233,326 @@ fn alloc_lifetime_bound_by_arena_borrow() { #[test] fn alloc_charges_stats() { let arena = Arena::new(); - let before = arena.stats().total_bytes_allocated; let _r: &mut u64 = arena.alloc(42); - let after = arena.stats().total_bytes_allocated; - assert!(after >= before + 8); + // After any successful allocation, the provider must have obtained + // at least one chunk from the underlying allocator; that chunk is + // strictly larger than the 8-byte payload. + assert!(arena.stats().total_bytes_allocated >= 8); +} + +/// `wasted_tail_bytes` is a *live* gauge of the unused tail space +/// across the active `current_*` chunks plus any chunks that have +/// been retired from a `current_*` slot but not yet cached or +/// destroyed. It must be zero on a fresh arena (no chunks held at +/// all), and return to zero after `reset` releases every chunk back +/// to the cache / underlying allocator (which leaves the arena with +/// the empty-mutator sentinels, contributing 0 slack each). +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_bytes_is_live_and_returns_to_zero_after_reset() { + let mut arena = Arena::new(); + assert_eq!(arena.stats().wasted_tail_bytes, 0, "fresh arena has no chunks"); + // Force at least one allocation so the arena obtains a chunk. + for _ in 0..32 { + let _r: &mut u64 = arena.alloc(0); + } + // The active chunk now contributes its free tail to the gauge + // (the chunk has plenty of room left even after 32 u64 allocs). + assert!( + arena.stats().wasted_tail_bytes > 0, + "active chunk's free tail must contribute to wasted_tail_bytes", + ); + arena.reset(); + // After reset every chunk has been released (either cached or + // destroyed). `current_*` are reset to empty-mutator sentinels + // which contribute 0 slack. + assert_eq!( + arena.stats().wasted_tail_bytes, + 0, + "reset returned every chunk and reinstalled the empty sentinel; \ + wasted-tail must be zero again", + ); +} + +/// Specifically exercises the "active chunk contributes its tail" +/// path: with zero retired chunks, `wasted_tail_bytes` must still +/// reflect the free region of the current local/shared chunks, and +/// it must shrink as further allocations consume that region. +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_includes_active_chunks_and_shrinks_with_allocs() { + let arena = Arena::new(); + // Trigger a single small local alloc to pin a local chunk. + let _: &mut u8 = arena.alloc(0); + let chunks_before = arena.stats().normal_local_chunks_allocated; + let after_one = arena.stats().wasted_tail_bytes; + assert!(after_one > 0, "active local chunk's free tail must be included even with 0 retires"); + // A handful of small allocs that fit in the current chunk's + // remaining capacity. The free tail must strictly decrease + // because no refill occurred. + for _ in 0..16 { + let _: &mut u64 = arena.alloc(0); + } + assert_eq!( + arena.stats().normal_local_chunks_allocated, + chunks_before, + "test relies on no refill happening; tighten the loop if this fires", + ); + let after_many = arena.stats().wasted_tail_bytes; + assert!( + after_many < after_one, + "subsequent allocs consumed bump space; the active chunk's \ + contribution to wasted_tail_bytes must shrink (before={after_one}, \ + after={after_many})", + ); +} + +/// Retiring a chunk while a smart-pointer handle still holds it alive +/// keeps the wasted tail counted until the handle drops (the chunk +/// reaches `release_shared` only when its refcount finally hits zero). +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_bytes_is_held_by_outstanding_arc() { + let arena = Arena::new(); + // Allocate one `Arc` in the initial (small) shared chunk and keep + // it alive across a refill that retires the chunk. + let pinned = arena.alloc_arc::(7); + // Force the current shared chunk to refill until at least one + // chunk gets retired without being released (i.e., a handle is + // still keeping it alive). Detect retire via the shared chunk + // count rather than the wasted-tail gauge, because the latter is + // never zero now that the active chunk's free tail contributes. + let initial_shared = arena.stats().normal_shared_chunks_allocated; + let mut tries = 0; + while arena.stats().normal_shared_chunks_allocated == initial_shared { + // 2 KiB slice allocations fill the small initial shared chunk + // quickly; refill is triggered well before we hit the cap. + drop(arena.alloc_slice_copy_arc::(&[0_u8; 2048])); + tries += 1; + assert!(tries < 1_000, "shared chunk never refilled — retire path appears broken"); + } + let held_wasted = arena.stats().wasted_tail_bytes; + drop(pinned); + // With the handle gone the original chunk's refcount hits zero, + // it routes through `release_shared`, and the counter decrements. + let after_drop = arena.stats().wasted_tail_bytes; + assert!( + after_drop < held_wasted, + "dropping the last handle must release the retired chunk's wasted tail \ + (before={held_wasted}, after={after_drop})", + ); +} + +/// `refill_local` is the other major retire path: when the +/// `current_local` slot's chunk is full, the old mutator is pushed +/// into `retired_local` (which keeps a `+1` for the duration of the +/// `&Arena` borrow). The wasted tail of every retired chunk must be +/// counted; `reset` releases every retired chunk back to the cache +/// and the counter must return cleanly to zero. +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_grows_on_local_refill_and_clears_on_reset() { + let mut arena = Arena::new(); + // Force the first chunk to be acquired so subsequent allocs trigger + // refills rather than the initial empty-mutator → first-chunk path. + let _: &mut u8 = arena.alloc(0); + let baseline = arena.stats().wasted_tail_bytes; + let chunks_before = arena.stats().normal_local_chunks_allocated; + let mut refills_observed = 0u64; + let mut saw_growth_over_baseline = false; + // Use a prime allocation size so the chunk cannot be exactly + // exhausted (which would leave a true wasted-tail of zero); this + // guarantees at least one refill leaves visible slack. + // + // `allocs` is a safety valve: if the chunk-allocation counter never + // advances (e.g. a broken `stats()` / `normal_local()` / a + // zero-length `alloc_slice_fill_with`), the refill condition can + // never be met and this loop would otherwise spin forever. Bound it + // so such a regression fails loudly instead of hanging. + let mut allocs = 0u64; + while refills_observed < 8 { + let _: &mut [u8] = arena.alloc_slice_fill_with(509, |_| 0_u8); + allocs += 1; + assert!( + allocs < 100_000, + "after {allocs} allocations only {refills_observed}/8 refills were observed — \ + chunk-allocation accounting (stats/normal_local) or slice fill appears broken", + ); + let now_chunks = arena.stats().normal_local_chunks_allocated; + if now_chunks > chunks_before + refills_observed { + refills_observed += 1; + // After a refill the gauge must include both the retired + // chunk's tail AND the new active chunk's tail, so it must + // exceed the baseline (which was only the very first + // active chunk's tail just after one tiny alloc). + if arena.stats().wasted_tail_bytes > baseline { + saw_growth_over_baseline = true; + } + } + } + assert!( + saw_growth_over_baseline, + "across {refills_observed} refills with a prime allocation size, \ + the wasted-tail counter never exceeded its single-active-chunk \ + baseline — retire-side accounting is broken", + ); + arena.reset(); + assert_eq!( + arena.stats().wasted_tail_bytes, + 0, + "reset must release every chunk and reinstall the empty sentinels, \ + taking the gauge back to zero", + ); +} + +/// **Conservation invariant**: across a full retire-and-release cycle, +/// the wasted-tail counter must return to exactly its starting value. +/// Catches off-by-one or asymmetric-arithmetic bugs (e.g., add 4 KiB, +/// subtract 4096) that observation-of-non-zero tests would miss. +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_returns_to_exactly_baseline_across_full_cycle() { + let mut arena = Arena::new(); + for cycle in 0..10 { + let before = arena.stats().wasted_tail_bytes; + assert_eq!(before, 0, "cycle {cycle}: baseline must be 0 before allocations begin"); + // Mix of all major allocation paths to exercise every retire- + // generating code path within a single cycle: + for _ in 0..4 { + let _: &mut u64 = arena.alloc(42); + let _: &mut [u8] = arena.alloc_slice_fill_with(256, |_| 0); + drop(arena.alloc_arc::(1)); + drop(arena.alloc_box::(2)); + drop(arena.alloc_slice_copy_arc::(&[0_u8; 1024])); + } + arena.reset(); + let after = arena.stats().wasted_tail_bytes; + assert_eq!( + after, 0, + "cycle {cycle}: after reset, the counter must return to exactly 0 \ + (got {after}) — asymmetric add/subtract would leave a residue", + ); + } +} + +/// Cache-reuse must not leak: a chunk that's cached on reset, then +/// re-acquired in the next epoch, then re-retired must contribute its +/// new wasted tail (not the stale stashed value from the previous +/// epoch, and not double-counted). +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_correct_after_cache_reuse_cycles() { + let mut arena = Arena::new(); + let mut acquired_chunks_total = 0u64; + for _ in 0..20 { + // Force at least one full chunk's worth of allocs so we cycle + // through `current_local` AND populate the cache on reset. + for _ in 0..64 { + let _: &mut u64 = arena.alloc(0); + } + let stats = arena.stats(); + acquired_chunks_total = stats.normal_local_chunks_allocated; + arena.reset(); + // After every reset the counter must be 0 — even though the + // chunk's `wasted_at_retire` field still holds the previous + // value, the subtract at cache-push consumed it exactly once, + // and the next epoch's retire will re-set + re-add. + assert_eq!(arena.stats().wasted_tail_bytes, 0, "cache reuse leaked into wasted-tail counter"); + } + // Sanity: we actually exercised real allocations, not a no-op. + assert!(acquired_chunks_total >= 1, "test did not allocate any chunks"); +} + +/// Multiple shared chunks pinned by outstanding Arcs each contribute +/// their wasted tail; dropping the handles one at a time must +/// monotonically shrink the counter without underflow. +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_decreases_monotonically_as_pinned_arcs_drop() { + let arena = Arena::new(); + let mut pins = std::vec::Vec::new(); + // Build up several pinned chunks by interleaving a pin with allocs + // that force a shared refill. + for _ in 0..5 { + pins.push(arena.alloc_arc::(99)); + for _ in 0..10 { + drop(arena.alloc_slice_copy_arc::(&[0_u8; 2048])); + } + } + let peak = arena.stats().wasted_tail_bytes; + // We may not get a contribution from every iteration (some Arcs + // may share a chunk with later ones), but at least some chunks + // were retired while pinned. + assert!(peak > 0, "expected outstanding pins to keep retired chunks counted"); + // Drop the pins. The counter must never grow, never underflow, + // and end at most equal to whatever the currently-active shared + // chunk would contribute (which is 0 since it's not yet retired). + let mut prev = peak; + while let Some(p) = pins.pop() { + drop(p); + let cur = arena.stats().wasted_tail_bytes; + assert!(cur <= prev, "dropping a pin must never grow the counter (prev={prev}, cur={cur})"); + // Underflow on a u64 atomic would show up as a value near + // `u64::MAX`. Guard against that explicitly. + assert!( + cur < u64::MAX / 2, + "counter underflowed (cur={cur}); subtract was unbalanced from add", + ); + prev = cur; + } +} + +/// Oversized local allocations route through `alloc_oversized_local_*`, +/// which pushes a temporary mutator into `retired_local` so the +/// caller's simple reference can outlive the call. That mutator's +/// chunk participates in wasted-tail accounting just like a refill- +/// retired chunk: it must contribute on retire and release exactly on +/// reset. +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_handles_oversized_local_retire() { + let mut arena = Arena::new(); + // Three oversized allocations create three retired oversized chunks. + let _: &mut [u8] = arena.alloc_slice_fill_with(20 * 1024, |_| 0_u8); + let mid = arena.stats().wasted_tail_bytes; + let _: &mut [u8] = arena.alloc_slice_fill_with(20 * 1024, |_| 0_u8); + let _: &mut [u8] = arena.alloc_slice_fill_with(20 * 1024, |_| 0_u8); + let after = arena.stats().wasted_tail_bytes; + // Each oversized chunk is sized to its request plus alignment and + // drop-entry slack; the wasted tail per chunk may be 0 or small + // depending on alignment. Either way, accumulating retires must + // never *decrease* the counter (no spurious subtracts). + assert!( + after >= mid, + "more oversized retires must not shrink the counter (mid={mid}, after={after})", + ); + arena.reset(); + assert_eq!( + arena.stats().wasted_tail_bytes, + 0, + "reset must release every oversized-retired chunk", + ); +} + +/// Smoke-test against u64 wrap-around: stress every retire+release path +/// many times. If the subtract ever exceeded the matching add even by +/// one byte, the running counter would underflow to a value near +/// `u64::MAX`. +#[cfg(feature = "stats")] +#[test] +fn wasted_tail_never_underflows_under_stress() { + let mut arena = Arena::new(); + for _ in 0..256 { + let _: &mut u64 = arena.alloc(0); + let _: &mut [u8] = arena.alloc_slice_fill_with(64, |_| 0); + drop(arena.alloc_arc::(0)); + drop(arena.alloc_box::(0)); + drop(arena.alloc_slice_copy_arc::(&[0_u8; 4096])); + // Always-positive: counter never observed as huge. + assert!(arena.stats().wasted_tail_bytes < u64::MAX / 2); + } + arena.reset(); + assert_eq!(arena.stats().wasted_tail_bytes, 0); } use crate::common::FailingAllocator; diff --git a/crates/multitude/tests/arena.rs b/crates/multitude/tests/arena.rs index 9dc80cbe6..0374dd5fe 100644 --- a/crates/multitude/tests/arena.rs +++ b/crates/multitude/tests/arena.rs @@ -2540,21 +2540,27 @@ mod coverage_arena_gaps { #[cfg(feature = "std")] #[test] fn alloc_with_closure_induced_eviction_commits_drop_entry() { - use std::cell::Cell; + use std::sync::Arc as StdArc; + use std::sync::atomic::{AtomicU32, Ordering}; - struct D<'a>(&'a Cell); - impl Drop for D<'_> { + // `Send` drop type (the deferred-drop `alloc_with` path requires + // `T: Send`; a `!Send` value here would be a soundness hazard at + // cross-thread arena teardown). + struct D(StdArc); + impl Drop for D { fn drop(&mut self) { - self.0.set(self.0.get() + 1); + self.0.fetch_add(1, Ordering::Relaxed); } } - let drops = Cell::new(0_u32); + let drops = StdArc::new(AtomicU32::new(0)); let arena = ArenaBuilder::::new().max_normal_alloc(4096).build(); // Warm up so the outer `alloc_with` below takes the fast path // (the cold slow path bypasses the eviction-commit branch). let _ = arena.alloc::(0); - let _outer: &mut D<'_> = arena.alloc_with(|| { + let counter = drops.clone(); + let arena_ref = &arena; + let _outer: &mut D = arena.alloc_with(move || { // Fill the current_local chunk so the OUTER allocation's // reserved slot ends up in a chunk that gets evicted before // the closure returns. The outer must then take the @@ -2562,12 +2568,12 @@ mod coverage_arena_gaps { // 2048 u64 allocs still force multiple refills with this 4 KiB // normal-allocation cap, so the reserved chunk is evicted. for _ in 0..2048_u32 { - let _ = arena.alloc::(0); + let _ = arena_ref.alloc::(0); } - D(&drops) + D(counter) }); drop(arena); - assert_eq!(drops.get(), 1, "outer D's drop must run via eviction commit path"); + assert_eq!(drops.load(Ordering::Relaxed), 1, "outer D's drop must run via eviction commit path"); } #[test] @@ -3169,14 +3175,14 @@ mod from_mutants_extras_stats { } #[test] - fn vec_into_arena_box_nonempty_nonzst_takes_inplace_path() { + fn vec_into_box_allocates_no_additional_local_chunk() { let arena = Arena::new(); let mut v: ArenaVec<'_, u32> = arena.alloc_vec_with_capacity(8); for i in 0..4_u32 { v.push(i); } let chunks_before = arena.stats().normal_local_chunks_allocated; - let _b: ArenaBox<[u32]> = v.into_arena_box(); + let _b: ArenaBox<[u32]> = v.into_boxed_slice(); assert_eq!(arena.stats().normal_local_chunks_allocated, chunks_before); } diff --git a/crates/multitude/tests/arena_string.rs b/crates/multitude/tests/arena_string.rs index 5b5526e54..a8c610b90 100644 --- a/crates/multitude/tests/arena_string.rs +++ b/crates/multitude/tests/arena_string.rs @@ -19,9 +19,9 @@ mod common; use core::cmp::Ordering; -use multitude::Arena; use multitude::strings::String; use multitude::vec::CollectIn; +use multitude::{Arena, FromIn}; #[test] fn clear_and_reuse() { @@ -183,7 +183,7 @@ fn try_reserve_succeeds() { #[test] fn try_with_capacity_in_succeeds() { let arena = Arena::new(); - let s = String::try_with_capacity_in(32, &arena).unwrap(); + let s = arena.try_alloc_string_with_capacity(32).unwrap(); assert!(s.capacity() >= 32); assert!(s.is_empty()); } @@ -191,7 +191,7 @@ fn try_with_capacity_in_succeeds() { #[test] fn try_with_capacity_in_zero_does_not_allocate() { let arena = Arena::new(); - let s = String::try_with_capacity_in(0, &arena).unwrap(); + let s = arena.try_alloc_string_with_capacity(0).unwrap(); assert_eq!(s.capacity(), 0); } @@ -226,7 +226,7 @@ fn try_reserve_returns_err_on_alloc_failure() { fn try_with_capacity_in_returns_err_on_alloc_failure() { let alloc = common::FailingAllocator::new(0); let arena = Arena::new_in(alloc); - let result = String::try_with_capacity_in(16, &arena); + let result = arena.try_alloc_string_with_capacity(16); let _ = result.unwrap_err(); } @@ -234,7 +234,7 @@ fn try_with_capacity_in_returns_err_on_alloc_failure() { fn try_grow_path_via_push_str_after_initial() { // Drives try_grow_to_at_least's slow path (cap > 0 branch). let arena = Arena::new(); - let mut s = String::try_with_capacity_in(4, &arena).unwrap(); + let mut s = arena.try_alloc_string_with_capacity(4).unwrap(); s.try_push_str("abcd").unwrap(); // fills initial cap exactly s.try_push_str("e").unwrap(); // forces grow assert_eq!(&*s, "abcde"); @@ -242,17 +242,17 @@ fn try_grow_path_via_push_str_after_initial() { } #[test] -fn from_str_in_copies_content() { +fn from_in_str_copies_content() { let arena = Arena::new(); - let s = String::from_str_in("hello, world", &arena); + let s = String::from_in("hello, world", &arena); assert_eq!(s.as_str(), "hello, world"); assert!(s.capacity() >= "hello, world".len()); } #[test] -fn from_str_in_empty() { +fn from_in_str_empty() { let arena = Arena::new(); - let s = String::from_str_in("", &arena); + let s = String::from_in("", &arena); assert!(s.is_empty()); assert_eq!(s.capacity(), 0); assert_eq!(s.as_str(), ""); @@ -261,7 +261,7 @@ fn from_str_in_empty() { #[test] fn as_bytes_returns_correct_bytes() { let arena = Arena::new(); - let s = String::from_str_in("héllo", &arena); + let s = String::from_in("héllo", &arena); assert_eq!(s.as_bytes(), "héllo".as_bytes()); } @@ -275,7 +275,7 @@ fn as_bytes_empty() { #[test] fn as_mut_str_allows_mutation() { let arena = Arena::new(); - let mut s = String::from_str_in("hello", &arena); + let mut s = String::from_in("hello", &arena); s.as_mut_str().make_ascii_uppercase(); assert_eq!(s.as_str(), "HELLO"); } @@ -290,26 +290,34 @@ fn as_mut_str_empty() { #[test] fn as_ptr_and_as_mut_ptr() { let arena = Arena::new(); - let mut s = String::from_str_in("hi", &arena); + let mut s = String::from_in("hi", &arena); + // Read the bytes through a shared pointer *before* taking any mutable + // pointer: `as_mut_ptr` reborrows `&mut str`/`&mut [u8]`, which would + // invalidate an earlier shared pointer under Stacked Borrows — exactly as + // it does in `std`, where both pointers come via `Deref`/`DerefMut` to + // `str`. let p = s.as_ptr(); - let q = s.as_mut_ptr(); - assert_eq!(p, q.cast_const()); - // SAFETY: valid pointer to len bytes. + // SAFETY: `p` addresses the first of the two initialized bytes "hi". unsafe { assert_eq!(*p, b'h'); } - // SAFETY: valid pointer to len bytes; offset 1 is in bounds. + // SAFETY: offset 1 is within the two initialized bytes. let p1 = unsafe { p.add(1) }; - // SAFETY: valid pointer to a byte. + // SAFETY: `p1` addresses the second initialized byte. unsafe { assert_eq!(*p1, b'i'); } + let const_addr = p.addr(); + // `as_ptr` and `as_mut_ptr` address the same buffer (compare addresses + // only; the pointers' borrow tags differ). + let mut_addr = s.as_mut_ptr().addr(); + assert_eq!(const_addr, mut_addr); } #[test] fn pop_returns_chars_in_reverse() { let arena = Arena::new(); - let mut s = String::from_str_in("a💖é", &arena); + let mut s = String::from_in("a💖é", &arena); assert_eq!(s.pop(), Some('é')); assert_eq!(s.pop(), Some('💖')); assert_eq!(s.pop(), Some('a')); @@ -320,7 +328,7 @@ fn pop_returns_chars_in_reverse() { #[test] fn truncate_shortens() { let arena = Arena::new(); - let mut s = String::from_str_in("hello", &arena); + let mut s = String::from_in("hello", &arena); let cap = s.capacity(); s.truncate(3); assert_eq!(s.as_str(), "hel"); @@ -330,7 +338,7 @@ fn truncate_shortens() { #[test] fn truncate_noop_when_longer() { let arena = Arena::new(); - let mut s = String::from_str_in("hi", &arena); + let mut s = String::from_in("hi", &arena); s.truncate(50); assert_eq!(s.as_str(), "hi"); } @@ -339,14 +347,14 @@ fn truncate_noop_when_longer() { #[should_panic(expected = "char boundary")] fn truncate_panics_on_non_boundary() { let arena = Arena::new(); - let mut s = String::from_str_in("é", &arena); // 2 bytes + let mut s = String::from_in("é", &arena); // 2 bytes s.truncate(1); } #[test] fn shrink_to_fit_reclaims_when_at_cursor() { let arena = Arena::new(); - let mut s = String::with_capacity_in(1024, &arena); + let mut s = arena.alloc_string_with_capacity(1024); s.push_str("short"); let _len = s.len(); s.shrink_to_fit(); @@ -363,7 +371,7 @@ fn shrink_to_fit_empty_or_full_noop() { s.shrink_to_fit(); assert_eq!(s.capacity(), 0); - let mut s2 = String::with_capacity_in(4, &arena); + let mut s2 = arena.alloc_string_with_capacity(4); s2.push_str("abcd"); let cap = s2.capacity(); // Folded mutant-kill: the same guard must also no-op when len == cap. @@ -374,7 +382,7 @@ fn shrink_to_fit_empty_or_full_noop() { #[test] fn insert_at_various_positions() { let arena = Arena::new(); - let mut s = String::from_str_in("ac", &arena); + let mut s = String::from_in("ac", &arena); s.insert(1, 'b'); assert_eq!(s.as_str(), "abc"); s.insert(0, 'Z'); @@ -386,7 +394,7 @@ fn insert_at_various_positions() { #[test] fn insert_multibyte_char() { let arena = Arena::new(); - let mut s = String::from_str_in("ab", &arena); + let mut s = String::from_in("ab", &arena); s.insert(1, '💖'); assert_eq!(s.as_str(), "a💖b"); } @@ -394,7 +402,7 @@ fn insert_multibyte_char() { #[test] fn insert_str_grows() { let arena = Arena::new(); - let mut s = String::from_str_in("ad", &arena); + let mut s = String::from_in("ad", &arena); s.insert_str(1, "bc"); assert_eq!(s.as_str(), "abcd"); } @@ -402,7 +410,7 @@ fn insert_str_grows() { #[test] fn insert_str_empty_is_noop() { let arena = Arena::new(); - let mut s = String::from_str_in("hi", &arena); + let mut s = String::from_in("hi", &arena); s.insert_str(1, ""); assert_eq!(s.as_str(), "hi"); } @@ -411,7 +419,7 @@ fn insert_str_empty_is_noop() { #[should_panic(expected = "char boundary")] fn insert_panics_on_bad_index() { let arena = Arena::new(); - let mut s = String::from_str_in("é", &arena); + let mut s = String::from_in("é", &arena); s.insert(1, 'x'); } @@ -419,14 +427,14 @@ fn insert_panics_on_bad_index() { #[should_panic(expected = "insertion index out of bounds")] fn insert_panics_when_idx_past_end() { let arena = Arena::new(); - let mut s = String::from_str_in("hi", &arena); + let mut s = String::from_in("hi", &arena); s.insert(99, 'x'); } #[test] fn remove_returns_char() { let arena = Arena::new(); - let mut s = String::from_str_in("a💖c", &arena); + let mut s = String::from_in("a💖c", &arena); let ch = s.remove(1); assert_eq!(ch, '💖'); assert_eq!(s.as_str(), "ac"); @@ -435,7 +443,7 @@ fn remove_returns_char() { #[test] fn remove_first_and_last() { let arena = Arena::new(); - let mut s = String::from_str_in("abcd", &arena); + let mut s = String::from_in("abcd", &arena); assert_eq!(s.remove(0), 'a'); assert_eq!(s.as_str(), "bcd"); assert_eq!(s.remove(s.len() - 1), 'd'); @@ -453,7 +461,7 @@ fn remove_panics_when_empty() { #[test] fn retain_filters_chars() { let arena = Arena::new(); - let mut s = String::from_str_in("a1b2c3", &arena); + let mut s = String::from_in("a1b2c3", &arena); s.retain(|c| c.is_ascii_alphabetic()); assert_eq!(s.as_str(), "abc"); } @@ -461,7 +469,7 @@ fn retain_filters_chars() { #[test] fn retain_removes_all() { let arena = Arena::new(); - let mut s = String::from_str_in("hello", &arena); + let mut s = String::from_in("hello", &arena); s.retain(|_| false); assert!(s.is_empty()); } @@ -469,7 +477,7 @@ fn retain_removes_all() { #[test] fn retain_keeps_all() { let arena = Arena::new(); - let mut s = String::from_str_in("hello", &arena); + let mut s = String::from_in("hello", &arena); s.retain(|_| true); assert_eq!(s.as_str(), "hello"); } @@ -477,7 +485,7 @@ fn retain_keeps_all() { #[test] fn retain_with_multibyte() { let arena = Arena::new(); - let mut s = String::from_str_in("a💖b💖c", &arena); + let mut s = String::from_in("a💖b💖c", &arena); s.retain(|c| c != '💖'); assert_eq!(s.as_str(), "abc"); } @@ -485,7 +493,7 @@ fn retain_with_multibyte() { #[test] fn replace_range_same_length() { let arena = Arena::new(); - let mut s = String::from_str_in("hello world", &arena); + let mut s = String::from_in("hello world", &arena); s.replace_range(6..11, "earth"); assert_eq!(s.as_str(), "hello earth"); } @@ -493,7 +501,7 @@ fn replace_range_same_length() { #[test] fn replace_range_grow() { let arena = Arena::new(); - let mut s = String::from_str_in("hi world", &arena); + let mut s = String::from_in("hi world", &arena); s.replace_range(0..2, "hello"); assert_eq!(s.as_str(), "hello world"); } @@ -501,7 +509,7 @@ fn replace_range_grow() { #[test] fn replace_range_shrink() { let arena = Arena::new(); - let mut s = String::from_str_in("hello world", &arena); + let mut s = String::from_in("hello world", &arena); s.replace_range(0..5, "hi"); assert_eq!(s.as_str(), "hi world"); } @@ -509,7 +517,7 @@ fn replace_range_shrink() { #[test] fn replace_range_unbounded() { let arena = Arena::new(); - let mut s = String::from_str_in("hello", &arena); + let mut s = String::from_in("hello", &arena); s.replace_range(.., "goodbye"); assert_eq!(s.as_str(), "goodbye"); } @@ -517,7 +525,7 @@ fn replace_range_unbounded() { #[test] fn replace_range_empty_replacement() { let arena = Arena::new(); - let mut s = String::from_str_in("hello world", &arena); + let mut s = String::from_in("hello world", &arena); s.replace_range(5..11, ""); assert_eq!(s.as_str(), "hello"); } @@ -525,7 +533,7 @@ fn replace_range_empty_replacement() { #[test] fn replace_range_inclusive() { let arena = Arena::new(); - let mut s = String::from_str_in("abcdef", &arena); + let mut s = String::from_in("abcdef", &arena); s.replace_range(1..=3, "XYZW"); assert_eq!(s.as_str(), "aXYZWef"); } @@ -534,14 +542,14 @@ fn replace_range_inclusive() { #[should_panic(expected = "char boundary")] fn replace_range_panics_on_non_boundary() { let arena = Arena::new(); - let mut s = String::from_str_in("é", &arena); + let mut s = String::from_in("é", &arena); s.replace_range(0..1, "x"); } #[test] fn clone_produces_equal_independent_string() { let arena = Arena::new(); - let original = String::from_str_in("hello", &arena); + let original = String::from_in("hello", &arena); let mut cloned = original.clone(); assert_eq!(original.as_str(), cloned.as_str()); // Independent buffers @@ -563,7 +571,7 @@ fn clone_empty() { #[test] fn deref_mut_allows_mutation() { let arena = Arena::new(); - let mut s = String::from_str_in("hello", &arena); + let mut s = String::from_in("hello", &arena); let r: &mut str = &mut s; r.make_ascii_uppercase(); assert_eq!(s.as_str(), "HELLO"); @@ -572,7 +580,7 @@ fn deref_mut_allows_mutation() { #[test] fn as_mut_trait_allows_mutation() { let arena = Arena::new(); - let mut s = String::from_str_in("abc", &arena); + let mut s = String::from_in("abc", &arena); let r: &mut str = AsMut::as_mut(&mut s); r.make_ascii_uppercase(); assert_eq!(s.as_str(), "ABC"); @@ -581,7 +589,7 @@ fn as_mut_trait_allows_mutation() { #[test] fn borrow_mut_trait_allows_mutation() { let arena = Arena::new(); - let mut s = String::from_str_in("xyz", &arena); + let mut s = String::from_in("xyz", &arena); let r: &mut str = core::borrow::BorrowMut::borrow_mut(&mut s); r.make_ascii_uppercase(); assert_eq!(s.as_str(), "XYZ"); @@ -646,7 +654,7 @@ fn write_macro_formats_into_string() { fn write_macro_appends() { use core::fmt::Write; let arena = Arena::new(); - let mut s = String::from_str_in("prefix:", &arena); + let mut s = String::from_in("prefix:", &arena); write!(s, " {}", 100).unwrap(); assert_eq!(s.as_str(), "prefix: 100"); } @@ -956,10 +964,10 @@ mod mutants_for_string { #[test] fn with_capacity_zero_does_not_allocate() { let arena = Arena::new(); - let s0 = MString::with_capacity_in(0, &arena); + let s0 = arena.alloc_string_with_capacity(0); assert_eq!(s0.capacity(), 0); // Compare against `new_in` (the documented no-alloc constructor). - let s_new = MString::new_in(&arena); + let s_new = arena.alloc_string(); assert_eq!(s_new.capacity(), 0); // Both have the same dangling data pointer (== 1 by NonNull::dangling()). // ptr identity is the strongest observable signal here. @@ -972,7 +980,7 @@ mod mutants_for_string { #[test] fn string_reserve_at_exact_fit_does_not_regrow() { let arena = Arena::new(); - let mut s = MString::with_capacity_in(16, &arena); + let mut s = arena.alloc_string_with_capacity(16); let cap = s.capacity(); let ptr_before = s.as_ptr(); s.reserve(cap); // additional == cap → needed == cap (len was 0) @@ -980,26 +988,69 @@ mod mutants_for_string { assert_eq!(s.as_ptr(), ptr_before); } - /// Kills `string.rs:510:21 == → !=` in `into_arena_box_str`. With + /// Kills `string.rs:510:21 == → !=` in `into_box`. With /// `!=`, the empty-fast-path triggers on non-empty inputs (wrong /// output, possibly UB) and the data-path triggers on empty (UB on /// dangling). #[test] - fn into_arena_box_str_handles_empty_and_non_empty() { + fn into_box_handles_empty_and_non_empty() { let arena = Arena::new(); - let s_empty = MString::new_in(&arena); - let b_empty = s_empty.into_arena_box_str(); + let s_empty = arena.alloc_string(); + let b_empty = s_empty.into_boxed_str(); assert_eq!(&*b_empty, ""); assert_eq!(b_empty.len(), 0); - let mut s = MString::with_capacity_in(16, &arena); + let mut s = arena.alloc_string_with_capacity(16); s.push_str("hello"); - let b = s.into_arena_box_str(); + let b = s.into_boxed_str(); assert_eq!(&*b, "hello"); assert_eq!(b.len(), 5); } + /// `String::into_arc` freezes into a shared, reference-counted + /// `Arc` whose contents match the builder, for both empty and + /// non-empty inputs, and which can be cloned and outlive the arena. + #[test] + fn into_arc_handles_empty_and_non_empty() { + use multitude::Arc; + + let arena = Arena::new(); + + let s_empty = arena.alloc_string(); + let a_empty: Arc = Arc::from(s_empty); + assert_eq!(&*a_empty, ""); + assert_eq!(a_empty.len(), 0); + + let mut s = arena.alloc_string_with_capacity(16); + s.push_str("hello"); + let a: Arc = Arc::from(s); + assert_eq!(&*a, "hello"); + assert_eq!(a.len(), 5); + + // Cloning shares the same backing allocation. + let a2 = a.clone(); + assert_eq!(&*a2, "hello"); + assert_eq!(a.as_ptr(), a2.as_ptr()); + } + + /// An `Arc` produced by `into_arc` outlives the arena it was + /// built from (the backing shared chunk is held by the refcount). + #[test] + fn into_arc_outlives_arena() { + use multitude::Arc; + + let escaped: Arc = { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push_str("survives"); + let a = Arc::from(s); + drop(arena); + a + }; + assert_eq!(&*escaped, "survives"); + } + /// Kills `string.rs:528:9 try_reclaim_tail → ()` (body becomes a /// no-op), `528:21 >= → <` (early-return inverted), and `534:29 - → /` /// (`total - used` arithmetic). @@ -1013,9 +1064,9 @@ mod mutants_for_string { #[test] fn reclaim_tail_does_not_corrupt_frozen_string() { let arena = Arena::new(); - let mut s = MString::with_capacity_in(256, &arena); + let mut s = arena.alloc_string_with_capacity(256); s.push_str("frozen!"); - let frozen = s.into_arena_box_str(); + let frozen = s.into_boxed_str(); let _filler: multitude::vec::Vec<'_, u64> = { let mut v = arena.alloc_vec_with_capacity::(64); for i in 0..64 { diff --git a/crates/multitude/tests/arena_vec.rs b/crates/multitude/tests/arena_vec.rs index a42d112d5..2168af867 100644 --- a/crates/multitude/tests/arena_vec.rs +++ b/crates/multitude/tests/arena_vec.rs @@ -110,7 +110,7 @@ fn traits_compile() { #[test] fn try_push_succeeds() { let arena = Arena::new(); - let mut v = Vec::new_in(&arena); + let mut v = arena.alloc_vec(); v.try_push(1_u32).unwrap(); v.try_push(2_u32).unwrap(); assert_eq!(&*v, &[1, 2]); @@ -119,7 +119,7 @@ fn try_push_succeeds() { #[test] fn try_reserve_succeeds() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.try_reserve(64).unwrap(); assert!(v.capacity() >= 64); } @@ -127,7 +127,7 @@ fn try_reserve_succeeds() { #[test] fn try_with_capacity_in_succeeds() { let arena = Arena::new(); - let v: Vec = Vec::try_with_capacity_in(32, &arena).unwrap(); + let v: Vec = arena.try_alloc_vec_with_capacity(32).unwrap(); assert!(v.capacity() >= 32); assert!(v.is_empty()); } @@ -135,7 +135,7 @@ fn try_with_capacity_in_succeeds() { #[test] fn try_with_capacity_in_zero_does_not_allocate() { let arena = Arena::new(); - let v: Vec = Vec::try_with_capacity_in(0, &arena).unwrap(); + let v: Vec = arena.try_alloc_vec_with_capacity(0).unwrap(); assert_eq!(v.capacity(), 0); } @@ -143,7 +143,7 @@ fn try_with_capacity_in_zero_does_not_allocate() { fn try_push_returns_err_on_alloc_failure() { let alloc = common::FailingAllocator::new(0); let arena = Arena::new_in(alloc); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); let _ = v.try_push(1).unwrap_err(); } @@ -151,7 +151,7 @@ fn try_push_returns_err_on_alloc_failure() { fn try_reserve_returns_err_on_alloc_failure() { let alloc = common::FailingAllocator::new(0); let arena = Arena::new_in(alloc); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); let _ = v.try_reserve(16).unwrap_err(); } @@ -159,21 +159,21 @@ fn try_reserve_returns_err_on_alloc_failure() { fn try_with_capacity_in_returns_err_on_alloc_failure() { let alloc = common::FailingAllocator::new(0); let arena = Arena::new_in(alloc); - let result: Result, _> = Vec::try_with_capacity_in(16, &arena); + let result: Result, _> = arena.try_alloc_vec_with_capacity(16); let _ = result.unwrap_err(); } #[test] fn with_capacity_in_pub_succeeds() { let arena = Arena::new(); - let v: Vec = Vec::with_capacity_in(8, &arena); + let v: Vec = arena.alloc_vec_with_capacity(8); assert!(v.capacity() >= 8); } #[test] fn new_in_pub_succeeds() { let arena = Arena::new(); - let v: Vec = Vec::new_in(&arena); + let v: Vec = arena.alloc_vec(); assert_eq!(v.len(), 0); assert_eq!(v.capacity(), 0); } @@ -219,7 +219,7 @@ fn insert_remove_swap_remove() { #[should_panic(expected = "insertion index")] fn insert_out_of_bounds_panics() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.insert(99, 1); } @@ -227,7 +227,7 @@ fn insert_out_of_bounds_panics() { #[should_panic(expected = "removal index")] fn remove_out_of_bounds_panics() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); let _ = v.remove(0); } @@ -235,7 +235,7 @@ fn remove_out_of_bounds_panics() { #[should_panic(expected = "swap_remove index")] fn swap_remove_out_of_bounds_panics() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); let _ = v.swap_remove(0); } @@ -347,7 +347,7 @@ fn append_moves_elements() { #[test] fn reserve_exact_grows() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.reserve_exact(50); assert!(v.capacity() >= 50); } @@ -355,7 +355,7 @@ fn reserve_exact_grows() { #[test] fn try_reserve_exact_succeeds() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.try_reserve_exact(40).unwrap(); assert!(v.capacity() >= 40); } @@ -364,7 +364,7 @@ fn try_reserve_exact_succeeds() { fn try_reserve_exact_returns_err_on_alloc_failure() { let alloc = common::FailingAllocator::new(0); let arena = Arena::new_in(alloc); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); let _ = v.try_reserve_exact(16).unwrap_err(); } @@ -778,7 +778,7 @@ fn pop_if_keeps_when_false() { #[test] fn pop_if_empty_returns_none() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); let r = v.pop_if(|_| true); assert_eq!(r, None); } @@ -841,7 +841,7 @@ fn into_iter_mut_borrowed() { #[test] fn extend_ref_for_copy_types() { let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); let src = [1_u8, 2, 3]; v.extend(src.iter()); assert_eq!(v.as_slice(), &[1, 2, 3]); @@ -1484,29 +1484,90 @@ mod mutants_for_vec { } /// Kills `vec/vec.rs:762:17 += → -=` and `762:17 += → *=` in - /// `into_arena_box_copy`. + /// `into_box`. /// /// `idx += 1` after each read. With `-=` idx underflows → next read /// is wildly out of bounds → segfault/UB. With `*=` idx stays 0 → /// the same element is read N times → wrong output. #[test] - fn into_arena_box_copy_yields_distinct_elements() { + fn into_box_yields_distinct_elements() { use multitude::vec::Vec as MVec; let arena = Arena::new(); - // Use a vec of types where `into_arena_box` takes the copy path - // (any type works for into_arena_box, but copy path is the cold + // Use a vec of types where `into_box` takes the copy path + // (any type works for into_box, but copy path is the cold // tail; we exercise it indirectly via empty-builder edge case in - // tests/arena_vec.rs). Here we exercise the public into_arena_box + // tests/arena_vec.rs). Here we exercise the public into_box // path with non-Copy types so the slow-path is taken on some // configurations. let mut v: MVec<'_, String> = arena.alloc_vec(); for i in 0..8_u32 { v.push(format!("item-{i}")); } - let b = v.into_arena_box(); + let b = v.into_boxed_slice(); for (i, s) in b.iter().enumerate() { assert_eq!(s, &format!("item-{i}")); } } } + +/// Regression test for a use-after-free between `Vec::split_off`'s +/// zero-copy sharing and oversized-chunk reclamation on growth. +/// +/// `split_off` of an oversized-backed `Vec` leaves the head and tail +/// sharing one chunk's payload (no copy). Growing one half past the +/// oversized threshold relocates it to a fresh chunk. The old chunk +/// must NOT be freed at that point, because the sibling half still +/// points into it — freeing it would dangle the sibling. We grow the +/// head, then read and drop the tail (whose elements run real +/// destructors); under the bug this touches freed memory. +#[test] +fn split_off_sibling_survives_oversized_growth_of_other_half() { + use std::sync::Arc as StdArc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Dropper(StdArc); + impl Drop for Dropper { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + let counter = StdArc::new(AtomicUsize::new(0)); + let arena = Arena::new(); + + // Build an oversized-backed Vec (> 16 KiB normal-class cutover): + // 4096 * size_of::() (8 bytes) = 32 KiB. + let mut head = arena.alloc_vec::(); + for _ in 0..4096 { + head.push(Dropper(counter.clone())); + } + // Zero-copy split: `tail` shares the same oversized chunk as `head`. + let tail = head.split_off(2048); + assert_eq!(head.len(), 2048); + assert_eq!(tail.len(), 2048); + + // Grow `head` past the oversized threshold again, forcing it to + // relocate to a fresh oversized chunk. The chunk shared with `tail` + // must remain alive. + for _ in 0..6144 { + head.push(Dropper(counter.clone())); + } + assert_eq!(head.len(), 8192); + + // Touch every element of the sibling: a UAF would read freed memory. + let mut sum = 0_usize; + for d in tail.as_slice() { + sum += StdArc::strong_count(&d.0); + } + assert!(sum > 0); + + // Drop the sibling explicitly: runs 2048 real destructors over the + // shared chunk's storage — must not be a use-after-free. + drop(tail); + drop(head); + drop(arena); + // 4096 (initial) + 6144 (regrowth) Droppers were created and all + // must have been dropped exactly once. + assert_eq!(counter.load(Ordering::Relaxed), 4096 + 6144); +} diff --git a/crates/multitude/tests/audit_repro.rs b/crates/multitude/tests/audit_repro.rs index f5abe803d..7d3a76665 100644 --- a/crates/multitude/tests/audit_repro.rs +++ b/crates/multitude/tests/audit_repro.rs @@ -141,3 +141,47 @@ fn alloc_slice_ref_accepts_half_chunk_alignment_for_non_drop() { let c = arena.alloc_slice_clone::(src); assert_eq!(c.len(), 1); } + +/// Audit finding: non-`Drop` ZST `alloc_arc` / `alloc_box` handouts did +/// not advance the bump cursor (`try_alloc(0, _)` is a cursor no-op), so +/// a single chunk could hand out unbounded refcounted handles. Each +/// handout draws down the pre-credited shared-ref surplus via the +/// non-atomic `local_shared_count`; an unbounded run exhausts it, +/// driving the chunk's atomic refcount to zero while the chunk is still +/// installed (use-after-free) or underflowing the surplus reconciliation +/// at retire (double-free). The fix reserves a 1-byte tag per such +/// handout so the cursor advances and per-chunk handouts stay bounded by +/// the chunk capacity (far below the surplus). +/// +/// Deterministic regression proxy: consecutive ZST shared handouts must +/// occupy distinct addresses (pre-fix they shared one address and the +/// cursor never moved). The create-and-drop loop exercises the refills +/// the tag now forces and confirms the arena stays consistent afterward. +#[test] +fn zst_shared_handouts_advance_cursor() { + let arena = Arena::new(); + + let a = arena.alloc_arc(()); + let b = arena.alloc_arc(()); + let c = arena.alloc_arc(()); + assert_ne!(a.as_ptr(), b.as_ptr(), "ZST Arc handouts must get distinct addresses"); + assert_ne!(b.as_ptr(), c.as_ptr(), "ZST Arc handouts must get distinct addresses"); + + let bx1 = arena.alloc_box(()); + let bx2 = arena.alloc_box(()); + assert_ne!(bx1.as_ptr(), bx2.as_ptr(), "ZST Box handouts must get distinct addresses"); + + // Many create-and-drop cycles force the chunk to fill (1 byte each) + // and refill. Pre-fix the cursor never advanced, so this pattern + // could drive the live chunk's atomic refcount to zero. + for _ in 0..2_000 { + drop(arena.alloc_arc(())); + drop(arena.alloc_box(())); + } + + // Arena remains usable for both flavors after the churn. + let z = arena.alloc_arc(()); + let _ = arena.alloc_arc(7_u64); + assert!(!z.as_ptr().is_null()); + drop(arena); +} diff --git a/crates/multitude/tests/bolero.rs b/crates/multitude/tests/bolero.rs index a61dcc8a5..f18386400 100644 --- a/crates/multitude/tests/bolero.rs +++ b/crates/multitude/tests/bolero.rs @@ -413,7 +413,7 @@ mod bolero_lifecycle { for i in 0..n { v.push(Tracker::new(&created, &dropped, payload.wrapping_add(i as u64))); } - vec_arcs.push(v.into_arena_arc()); + vec_arcs.push(multitude::Arc::from(v)); } Op::DropVecArc { idx } => { if !vec_arcs.is_empty() { @@ -428,7 +428,7 @@ mod bolero_lifecycle { for i in 0..n { v.push(Tracker::new(&created, &dropped, payload.wrapping_add(i as u64))); } - vec_boxes.push(v.into_arena_box()); + vec_boxes.push(v.into_boxed_slice()); } Op::DropVecBox { idx } => { if !vec_boxes.is_empty() { @@ -448,7 +448,7 @@ mod bolero_lifecycle { s.push('!'); s.shrink_to_fit(); } - built_str_boxes.push(s.into_arena_box_str()); + built_str_boxes.push(s.into_boxed_str()); } Op::DropBuiltStringBox { idx } => { if !built_str_boxes.is_empty() { @@ -493,7 +493,7 @@ mod bolero_lifecycle { s.push('!'); s.shrink_to_fit(); } - built_utf16_str_boxes.push(s.into_arena_box_utf16_str()); + built_utf16_str_boxes.push(s.into_boxed_utf16_str()); } #[cfg(feature = "utf16")] Op::DropBuiltUtf16StringBox { idx } => { @@ -551,10 +551,6 @@ mod bolero_lifecycle { "at least one chunk must have been allocated (created={created_n}, stats={s:?})", ); } - assert!( - s.total_bytes_allocated >= (created_n as u64).saturating_mul(8), - "total_bytes_allocated under-reported (created={created_n}, stats={s:?})", - ); } } @@ -646,11 +642,9 @@ mod bolero_panic_safety { #![allow(clippy::clone_on_ref_ptr, reason = "tests prefer concise method-call form")] #![allow(clippy::panic, reason = "test deliberately injects panics to verify recovery")] #![allow(clippy::manual_assert, reason = "panic-injection sites are clearer with explicit panic!")] - use std::cell::Cell; use std::panic::{AssertUnwindSafe, catch_unwind}; - use std::rc::Rc as StdRc; use std::sync::Arc as StdArc; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicI32, AtomicUsize, Ordering}; use bolero::TypeGenerator; use multitude::Arena; @@ -684,21 +678,21 @@ mod bolero_panic_safety { struct ClonePanicTracker { created: Counter, dropped: Counter, - remaining_before_panic: StdRc>, + remaining_before_panic: StdArc, payload: u64, armed: bool, } impl Clone for ClonePanicTracker { fn clone(&self) -> Self { - let remaining = self.remaining_before_panic.get(); + let remaining = self.remaining_before_panic.load(Ordering::Relaxed); assert!(remaining > 0, "clone-panic-injection"); - self.remaining_before_panic.set(remaining - 1); + self.remaining_before_panic.store(remaining - 1, Ordering::Relaxed); let _ = self.created.fetch_add(1, Ordering::Relaxed); Self { created: StdArc::clone(&self.created), dropped: StdArc::clone(&self.dropped), - remaining_before_panic: StdRc::clone(&self.remaining_before_panic), + remaining_before_panic: StdArc::clone(&self.remaining_before_panic), payload: self.payload, armed: true, } @@ -718,7 +712,7 @@ mod bolero_panic_safety { Self { created: StdArc::clone(&self.created), dropped: StdArc::clone(&self.dropped), - remaining_before_panic: StdRc::clone(&self.remaining_before_panic), + remaining_before_panic: StdArc::clone(&self.remaining_before_panic), payload: self.payload, armed: false, } @@ -775,7 +769,7 @@ mod bolero_panic_safety { let seed = ClonePanicTracker { created: StdArc::clone(&created), dropped: StdArc::clone(&dropped), - remaining_before_panic: StdRc::new(Cell::new(panic_idx_i32)), + remaining_before_panic: StdArc::new(AtomicI32::new(panic_idx_i32)), payload, armed: false, }; diff --git a/crates/multitude/tests/chunk_footprint.rs b/crates/multitude/tests/chunk_footprint.rs new file mode 100644 index 000000000..ae4476082 --- /dev/null +++ b/crates/multitude/tests/chunk_footprint.rs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! End-to-end regression guards for the chunk allocation *footprint*: +//! the number of bytes actually requested from the underlying allocator +//! must match the chunk's size class, not be inflated to `CHUNK_ALIGN` +//! (64 KiB). This is the gap that previously let every shared chunk be +//! silently allocated at 64 KiB while the byte budget / stats tracked the +//! much smaller unpadded size. + +#![cfg(feature = "std")] +#![allow(clippy::unwrap_used, reason = "test code")] +#![allow(clippy::std_instead_of_core, reason = "test code uses std")] +#![allow(clippy::collection_is_never_read, reason = "tests retain handles to keep chunks alive")] + +use std::alloc::Layout; +use std::ptr::NonNull; +use std::sync::{Arc, Mutex}; + +use allocator_api2::alloc::{AllocError, Allocator, Global}; +use multitude::Arena; + +/// `CHUNK_ALIGN` / `MAX_CHUNK_BYTES`: the design-fixed shared-chunk base +/// alignment and maximum cacheable chunk size. Mirrored here because the +/// constant is crate-internal. +const CHUNK_ALIGN: usize = 65_536; + +/// Shared log of `(size, align)` allocation requests. +type RequestLog = Arc>>; + +/// Records the `(size, align)` of every allocation request so a test can +/// assert what was actually asked of the underlying allocator. The log is +/// an `Arc` (not a leaked `'static`) so nothing leaks under Miri. +#[derive(Clone)] +struct Recorder { + requests: RequestLog, +} + +// SAFETY: forwards faithfully to `Global`; only observes the layout. +unsafe impl Allocator for Recorder { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.requests.lock().unwrap().push((layout.size(), layout.align())); + Global.allocate(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // SAFETY: forwarded per the `Allocator` contract. + unsafe { Global.deallocate(ptr, layout) }; + } +} + +fn recorder() -> (Recorder, RequestLog) { + let requests = Arc::new(Mutex::new(Vec::new())); + ( + Recorder { + requests: Arc::clone(&requests), + }, + requests, + ) +} + +/// Sizes of the allocations that landed on a `CHUNK_ALIGN` boundary — i.e. +/// the shared chunks. +fn shared_chunk_sizes(log: &RequestLog) -> Vec { + log.lock() + .unwrap() + .iter() + .filter(|(_, align)| *align >= CHUNK_ALIGN) + .map(|(size, _)| *size) + .collect() +} + +/// A single small `Arc` needs only a class-0 shared chunk (512 B). +/// The allocator must therefore be asked for far less than `CHUNK_ALIGN` +/// bytes — before the fix it was asked for exactly 64 KiB. +#[test] +fn small_shared_chunk_is_not_inflated_to_chunk_align() { + let (rec, log) = recorder(); + { + let arena = Arena::new_in(rec); + let a = arena.alloc_arc(7_u32); + assert_eq!(*a, 7); + + let shared = shared_chunk_sizes(&log); + assert!( + !shared.is_empty(), + "expected at least one shared-chunk allocation, got {:?}", + log.lock().unwrap() + ); + for size in &shared { + assert!( + *size < CHUNK_ALIGN, + "a shared chunk for a single small Arc was allocated at {size} B (>= {CHUNK_ALIGN}); the size was inflated to the base alignment" + ); + } + } +} + +/// The byte-accounting gauge must match the real allocator footprint: with +/// the size inflated to 64 KiB but the budget tracking the unpadded size, +/// `total_bytes_allocated` severely under-reported. After the fix the real +/// shared-chunk bytes equal the tracked bytes. +#[cfg(feature = "stats")] +#[test] +fn total_bytes_allocated_matches_real_shared_footprint() { + let (rec, log) = recorder(); + let arena = Arena::new_in(rec); + let _a = arena.alloc_arc(123_u64); + + let real_shared_bytes: usize = shared_chunk_sizes(&log).iter().sum(); + assert!( + real_shared_bytes > 0, + "expected a shared-chunk allocation, got {:?}", + log.lock().unwrap() + ); + + let tracked = arena.stats().total_bytes_allocated; + assert_eq!( + tracked, real_shared_bytes as u64, + "total_bytes_allocated ({tracked}) must equal the real shared-chunk bytes requested from the allocator ({real_shared_bytes})" + ); +} + +/// Allocating many small Arcs creates several shared chunks as the bump +/// cursor exhausts each one. Their sizes must be power-of-two class sizes +/// capped at `CHUNK_ALIGN`, with at least one strictly below it — before +/// the fix *every* shared chunk was exactly 64 KiB. +#[test] +fn shared_chunk_sizes_track_size_classes() { + let (rec, log) = recorder(); + let arena = Arena::new_in(rec); + + let mut handles = Vec::new(); + for i in 0..2_000_u32 { + handles.push(arena.alloc_arc(i)); + } + + let shared = shared_chunk_sizes(&log); + assert!(shared.len() >= 2, "expected multiple shared chunks, got sizes {shared:?}"); + assert!( + shared.iter().any(|&s| s < CHUNK_ALIGN), + "every shared chunk was allocated at {CHUNK_ALIGN} B: sizes {shared:?}" + ); + for &s in &shared { + assert!(s <= CHUNK_ALIGN, "chunk size {s} exceeds max chunk bytes"); + assert!(s.is_power_of_two(), "chunk size {s} is not a power-of-two class size"); + } +} diff --git a/crates/multitude/tests/coverage_extras.rs b/crates/multitude/tests/coverage_extras.rs index 92dfc173a..55eba0e9c 100644 --- a/crates/multitude/tests/coverage_extras.rs +++ b/crates/multitude/tests/coverage_extras.rs @@ -672,7 +672,7 @@ mod coverage { #[test] fn arena_string_try_push_str_initial_alloc_err() { let a = fail_arena(); - let mut s = multitude::strings::String::new_in(&a); + let mut s = a.alloc_string(); assert!(s.try_push_str("hello").is_err()); } @@ -681,7 +681,7 @@ mod coverage { // Allow the initial chunk alloc, fail the grow's new-chunk alloc by // requesting a capacity that exceeds the chunk_size. let a = Arena::builder().allocator_in(FailingAllocator::new(1)).build(); - let mut s = multitude::strings::String::try_with_capacity_in(4, &a).unwrap(); + let mut s = a.try_alloc_string_with_capacity(4).unwrap(); s.try_push_str("abcd").unwrap(); // Forces grow_for_string → needs new (oversized) chunk → allocator fails. assert!(s.try_reserve(64 * 1024).is_err()); @@ -691,7 +691,7 @@ mod coverage { fn panic_arena_string_grow_to_at_least() { expect_panic(|| { let a = Arena::builder().allocator_in(FailingAllocator::new(1)).build(); - let mut s = multitude::strings::String::try_with_capacity_in(4, &a).unwrap(); + let mut s = a.try_alloc_string_with_capacity(4).unwrap(); s.try_push_str("abcd").unwrap(); // grow_to_at_least asks for a new chunk; allocator is exhausted. s.push_str("x".repeat(64 * 1024)); @@ -735,7 +735,7 @@ mod coverage { fn vec_try_reserve_no_growth_needed() { // Line 182: try_reserve when capacity already sufficient → Ok(()) without growing. let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.push(1); v.push(2); // capacity should be >= 4 after the initial growth; reserve 1 more (already have room). @@ -747,7 +747,7 @@ mod coverage { fn vec_try_reserve_exact_realloc_and_overflow() { // Lines 432-436: try_reserve_exact that needs realloc. let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.push(1); // Force exact reserve beyond current capacity. assert!(v.try_reserve_exact(100).is_ok()); @@ -765,7 +765,7 @@ mod coverage { fn vec_resize_with_shrink() { // Lines 473-475: resize_with to a smaller size calls truncate. let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); for i in 0..10 { v.push(i); } @@ -780,7 +780,7 @@ mod coverage { // Lines 512-513: Excluded start bound and Unbounded start. // Lines 516-518: Included end bound and Unbounded end. let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); for i in 0..10 { v.push(i); } @@ -792,7 +792,7 @@ mod coverage { // drain(..) → Unbounded start, Unbounded end → start=0, end=len let arena2 = Arena::new(); - let mut v2: Vec = Vec::new_in(&arena2); + let mut v2: Vec = arena2.alloc_vec(); for i in 0..5 { v2.push(i); } @@ -805,7 +805,7 @@ mod coverage { fn vec_zst_operations() { // Lines 360, 586, 594-596: ZST Vec realloc and shrink_to_fit. let arena = Arena::new(); - let mut v: Vec<()> = Vec::new_in(&arena); + let mut v: Vec<()> = arena.alloc_vec(); for _ in 0..100 { v.push(()); } @@ -820,7 +820,7 @@ mod coverage { // Lines 833-835: Drain Debug format. // Lines 865-875: Drain::next_back (DoubleEndedIterator). let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); for i in 0..5 { v.push(i); } @@ -840,7 +840,7 @@ mod coverage { fn vec_insert_triggers_growth() { // Line 284: insert when len == cap forces grow_one. let arena = Arena::new(); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); // Fill to capacity (initial growth is 4). for i in 0..4 { v.push(i); @@ -857,7 +857,7 @@ mod coverage { // Line 126: grow_one → panic_alloc. expect_panic(|| { let arena = Arena::new_in(FailingAllocator::new(1)); // 1 alloc for initial chunk - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); // First pushes may succeed using the chunk, but growth will fail. // FailingAllocator(1) gives exactly one chunk; the second // realloc-on-grow trips the panic well before 100 pushes — @@ -873,7 +873,7 @@ mod coverage { // Line 168: reserve → panic_alloc. expect_panic(|| { let arena = Arena::new_in(FailingAllocator::new(0)); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.reserve(1); }); } @@ -883,7 +883,7 @@ mod coverage { // Line 422: reserve_exact → panic_alloc. expect_panic(|| { let arena = Arena::new_in(FailingAllocator::new(0)); - let mut v: Vec = Vec::new_in(&arena); + let mut v: Vec = arena.alloc_vec(); v.reserve_exact(1); }); } @@ -1268,7 +1268,7 @@ mod coverage_more { use allocator_api2::alloc::Allocator; use multitude::strings::String as ArenaString; use multitude::vec::Vec as ArenaVec; - use multitude::{Arc, Arena, ArenaBuilder}; + use multitude::{Arc, Arena, ArenaBuilder, FromIn as _}; #[expect(unused_imports, reason = "merged test module re-exports common helpers")] use crate::common; @@ -1295,7 +1295,7 @@ mod coverage_more { // ---- src/arc.rs / src/rc.rs gaps ---- #[test] - fn arc_from_arena_vec_uses_into_arena_arc() { + fn arc_from_arena_vec_uses_into_arc() { let arena = Arena::new(); let mut v: ArenaVec<'_, i32> = arena.alloc_vec(); v.push(1); @@ -1346,7 +1346,7 @@ mod coverage_more { #[test] fn string_retain_panic_restores_guard_len() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcd", &arena); + let mut s = ArenaString::from_in("abcd", &arena); let result = panic::catch_unwind(AssertUnwindSafe(|| { s.retain(|ch| { @@ -1434,7 +1434,7 @@ mod coverage_more { #[should_panic(expected = "allocator returned AllocError")] fn string_replace_range_panics_from_grow_to_at_least() { let arena = ArenaBuilder::new_in(FailingAllocator::new(1)).build(); - let mut s = ArenaString::from_str_in("a", &arena); + let mut s = ArenaString::from_in("a", &arena); // `FailingAllocator` denies every allocation after the first // regardless of size; a moderate replacement (well past the // initial small chunk's residual capacity) is sufficient. @@ -1445,7 +1445,7 @@ mod coverage_more { #[test] fn string_reserve_zero_on_nonempty_string_is_noop() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("already allocated", &arena); + let mut s = ArenaString::from_in("already allocated", &arena); let cap = s.capacity(); s.reserve(0); assert_eq!(s.capacity(), cap); @@ -1460,46 +1460,46 @@ mod coverage_more { #[should_panic(expected = "allocator returned AllocError")] fn vec_with_capacity_panics_on_allocator_error() { let arena = ArenaBuilder::new_in(FailingAllocator::new(0)).build(); - let _v: ArenaVec<'_, u8, _> = ArenaVec::with_capacity_in(8, &arena); + let _v: ArenaVec<'_, u8, _> = arena.alloc_vec_with_capacity(8); } #[test] #[should_panic(expected = "allocator returned AllocError")] - fn vec_into_arena_arc_panics_on_shared_allocator_error() { + fn vec_into_arc_panics_on_shared_allocator_error() { let arena = ArenaBuilder::new_in(SendFailingAllocator::new(1)).build(); - let mut v: ArenaVec<'_, u8, _> = ArenaVec::with_capacity_in(4, &arena); + let mut v: ArenaVec<'_, u8, _> = arena.alloc_vec_with_capacity(4); v.extend([1, 2, 3, 4]); - let _arc = v.into_arena_arc(); + let _arc = multitude::Arc::from(v); } #[test] - fn vec_into_arena_box_copy_handles_zst_fallback() { + fn vec_into_box_handles_zst_fallback() { let arena = Arena::new(); let mut v = arena.alloc_vec::<()>(); for _ in 0..16 { v.push(()); } // Folded mutant-kill: vec.rs:834 `==`/`!=` must keep ZST Vecs on the copy fallback. - let b = v.into_arena_box(); + let b = v.into_boxed_slice(); assert_eq!(b.len(), 16); } #[test] #[should_panic(expected = "allocator returned AllocError")] - fn vec_into_arena_box_copy_panics_on_zst_drop_alloc_error() { + fn vec_into_box_panics_on_zst_drop_alloc_error() { let arena = ArenaBuilder::new_in(FailingAllocator::new(0)).build(); let mut v = arena.alloc_vec::(); v.extend([DropZst, DropZst, DropZst]); - let _ = v.into_arena_box(); + let _ = v.into_boxed_slice(); } #[test] - fn vec_into_arena_box_falls_back_when_drop_entry_install_misses() { + fn vec_into_box_falls_back_when_drop_entry_install_misses() { let arena = Arena::new(); let mut v = arena.alloc_vec::(); v.extend([Droppy("a"), Droppy("b")]); let _decoy = arena.alloc_slice_fill_with(70_000, |i| i as u8); - let b = v.into_arena_box(); + let b = v.into_boxed_slice(); assert_eq!(b.len(), 2); } @@ -1511,11 +1511,11 @@ mod coverage_more { // runtime-checked assertion, not a memory-safety property, so Miri // adds no value beyond what `cargo test` already verifies. #[cfg_attr(miri, ignore)] - fn vec_into_arena_box_panics_when_drop_slice_is_too_long_for_entry() { + fn vec_into_box_panics_when_drop_slice_is_too_long_for_entry() { let arena = Arena::new(); let mut v = arena.alloc_vec::(); v.extend((0..=u16::MAX).map(|_| Droppy("many"))); - let _ = v.into_arena_box(); + let _ = v.into_boxed_slice(); } #[test] @@ -1529,7 +1529,7 @@ mod coverage_more { #[test] fn vec_realloc_edge_cases_are_observable_through_public_api() { let arena = Arena::new(); - let mut v = ArenaVec::with_capacity_in(8, &arena); + let mut v = arena.alloc_vec_with_capacity(8); v.extend([1_u32, 2, 3, 4]); v.reserve_exact(0); @@ -1553,7 +1553,7 @@ mod coverage_more { // refuse any subsequent allocation, demonstrating that no // allocator call is made. let arena = ArenaBuilder::new_in(FailingAllocator::new(1)).max_normal_alloc(4096).build(); - let mut v = ArenaVec::with_capacity_in(70_000, &arena); + let mut v = arena.alloc_vec_with_capacity(70_000); let cap_before = v.capacity(); v.extend([1_u32, 2, 3, 4]); v.shrink_to_fit(); diff --git a/crates/multitude/tests/coverage_gaps.rs b/crates/multitude/tests/coverage_gaps.rs index e7ea48115..5c8f95078 100644 --- a/crates/multitude/tests/coverage_gaps.rs +++ b/crates/multitude/tests/coverage_gaps.rs @@ -772,34 +772,34 @@ mod drop_slice_over_u16_max_returns_err { } // ============================================================================ -// vec/freeze.rs — `try_into_arena_arc` (lines 46–60). The infallible -// `into_arena_arc` is exercised elsewhere; this targets the fallible +// vec/freeze.rs — `try_into_arc` (lines 46–60). The infallible +// `into_arc` is exercised elsewhere; this targets the fallible // variant and its allocator-error path via `FailingAllocator(1)` (the // initial vec buffer consumes the one allowed chunk, the freeze refill // then fails). // ============================================================================ -mod vec_freeze_try_into_arena_arc { +mod vec_freeze_try_into_arc { use multitude::Arena; use crate::sync_failing::SyncFailingAllocator; #[test] - fn try_into_arena_arc_ok() { + fn try_into_arc_ok() { let a = Arena::new(); let mut v = a.alloc_vec::(); v.push(1); v.push(2); v.push(3); - let arc = v.try_into_arena_arc().unwrap(); + let arc = v.try_into_arc().unwrap(); assert_eq!(&*arc, &[1, 2, 3][..]); } #[test] - fn try_into_arena_arc_err_on_failing_allocator() { + fn try_into_arc_err_on_failing_allocator() { let a = Arena::new_in(SyncFailingAllocator::new(1)); let mut v = a.alloc_vec::(); v.push(1); - let r = v.try_into_arena_arc(); + let r = v.try_into_arc(); assert!(r.is_err()); } } @@ -1686,3 +1686,75 @@ mod oversized_paths { assert_eq!(b.as_str(), "HELLO"); } } + +// ============================================================================ +// vec/freeze.rs — `Vec::leak` (the O(1), allocation-free freeze for +// `T: !Drop`, plus `&*v.leak()` for the shared variant), plus the empty-slice +// early returns and the oversized `!Drop` arm of the slice-ref allocators +// (alloc_slice_ref.rs). +// ============================================================================ +mod freeze_and_slice_edges { + use multitude::Arena; + + #[test] + fn vec_leak_shared_reborrow_returns_arena_lifetime_slice() { + let arena = Arena::new(); + let mut v = arena.alloc_vec::(); + for i in 0..6_u32 { + v.push(i * 10); + } + // `u32: !Drop`, so `leak` is the in-place reinterpret (no copy, no + // drop entry); reborrow as shared for `&[T]`. + let s: &[u32] = &*v.leak(); + assert_eq!(s, &[0, 10, 20, 30, 40, 50]); + } + + #[test] + fn vec_leak_allows_in_place_mutation() { + let arena = Arena::new(); + let mut v = arena.alloc_vec::(); + v.push(1); + v.push(2); + v.push(3); + let s: &mut [u32] = v.leak(); + for x in s.iter_mut() { + *x *= 2; + } + assert_eq!(s, &[2, 4, 6]); + } + + #[test] + fn vec_leak_empty() { + let arena = Arena::new(); + let v = arena.alloc_vec::(); + let s: &[u32] = &*v.leak(); + assert!(s.is_empty()); + } + + #[test] + fn alloc_slice_copy_empty_returns_empty_slice() { + let arena = Arena::new(); + let s: &mut [u32] = arena.alloc_slice_copy::(&[]); + assert!(s.is_empty()); + } + + #[test] + fn alloc_slice_clone_empty_returns_empty_slice() { + let arena = Arena::new(); + let src: [String; 0] = []; + let s: &mut [String] = arena.alloc_slice_clone(&src); + assert!(s.is_empty()); + } + + #[test] + fn alloc_slice_fill_iter_oversized_non_drop() { + let arena = Arena::new(); + // 5000 × u32 = 20 KiB > MAX_NORMAL_ALLOC (16 KiB) ⇒ oversized path; + // `u32: !Drop` ⇒ the non-drop oversized arm of + // `impl_alloc_slice_fill_iter`. + let s: &mut [u32] = arena.alloc_slice_fill_iter((0_u32..5000).map(|i| i.wrapping_mul(3))); + assert_eq!(s.len(), 5000); + assert_eq!(s[0], 0); + assert_eq!(s[4999], 4999_u32.wrapping_mul(3)); + } +} diff --git a/crates/multitude/tests/dst.rs b/crates/multitude/tests/dst.rs index b0e96a196..e8a94be98 100644 --- a/crates/multitude/tests/dst.rs +++ b/crates/multitude/tests/dst.rs @@ -793,7 +793,7 @@ mod from_mutants_extras_dst { } #[test] - fn into_arena_box_empty_drop_type_takes_copy_path() { + fn into_box_empty_drop_type_takes_copy_path() { struct D; impl Drop for D { fn drop(&mut self) {} @@ -801,36 +801,36 @@ mod from_mutants_extras_dst { let arena = Arena::new(); let v: ArenaVec<'_, D> = arena.alloc_vec_with_capacity(4); assert_eq!(v.len(), 0); - let b: ArenaBox<[D], _> = v.into_arena_box(); + let b: ArenaBox<[D], _> = v.into_boxed_slice(); assert_eq!(b.len(), 0); } #[test] - fn into_arena_box_with_full_capacity_for_drop_type_no_reclaim() { + fn into_box_with_full_capacity_for_drop_type_no_reclaim() { let arena = Arena::new(); let mut v: ArenaVec<'_, OneByteDrop> = arena.alloc_vec_with_capacity(4); for i in 0..4 { v.push(OneByteDrop(i)); } assert_eq!(v.len(), v.capacity()); - let b: ArenaBox<[OneByteDrop], _> = v.into_arena_box(); + let b: ArenaBox<[OneByteDrop], _> = v.into_boxed_slice(); assert_eq!(b.len(), 4); } #[test] - fn into_arena_box_with_full_capacity_for_non_drop_type_no_reclaim() { + fn into_box_with_full_capacity_for_non_drop_type_no_reclaim() { let arena = Arena::new(); let mut v: ArenaVec<'_, u32> = arena.alloc_vec_with_capacity(4); for i in 0..4 { v.push(i); } assert_eq!(v.len(), v.capacity()); - let b: ArenaBox<[u32], _> = v.into_arena_box(); + let b: ArenaBox<[u32], _> = v.into_boxed_slice(); assert_eq!(b.len(), 4); } #[test] - fn into_arena_box_copy_advances_consumed_index() { + fn into_box_advances_consumed_index() { use std::sync::Arc as StdArc; use std::sync::atomic::{AtomicU32, Ordering}; @@ -849,7 +849,7 @@ mod from_mutants_extras_dst { let arena = Arena::new(); // 16-byte D; default max_normal_alloc = 16 KiB. with_capacity(1100) // requests 17.6 KiB > max_normal_alloc → buffer goes to oversized - // chunk, install fails, into_arena_box falls back to the copy path. + // chunk, install fails, into_box falls back to the copy path. let mut v: ArenaVec<'_, D> = arena.alloc_vec_with_capacity(1100); for i in 0..1100_u32 { v.push(D { @@ -857,7 +857,7 @@ mod from_mutants_extras_dst { seen: seen.clone(), }); } - let b: ArenaBox<[D], _> = v.into_arena_box(); + let b: ArenaBox<[D], _> = v.into_boxed_slice(); for (i, d) in b.iter().enumerate() { assert_eq!(d.idx, i as u32, "element {i} should have idx {i}"); } diff --git a/crates/multitude/tests/mutant_kills_boundaries.rs b/crates/multitude/tests/mutant_kills_boundaries.rs index 28f091227..4658faa76 100644 --- a/crates/multitude/tests/mutant_kills_boundaries.rs +++ b/crates/multitude/tests/mutant_kills_boundaries.rs @@ -139,7 +139,7 @@ fn vec_shrink_to_fit_with_room_to_shrink_reduces_capacity() { // already cover `cap == len` no-op via existing tests; here pin // the actual shrink behavior for `cap > len`. let arena = Arena::new(); - let mut v: ArenaVec<'_, u32> = ArenaVec::with_capacity_in(64, &arena); + let mut v: ArenaVec<'_, u32> = arena.alloc_vec_with_capacity(64); v.extend([1_u32, 2, 3, 4]); let cap_before = v.capacity(); assert!(cap_before >= 64); diff --git a/crates/multitude/tests/mutant_kills_more.rs b/crates/multitude/tests/mutant_kills_more.rs index e9999ecd6..d69603c5d 100644 --- a/crates/multitude/tests/mutant_kills_more.rs +++ b/crates/multitude/tests/mutant_kills_more.rs @@ -1,10 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Mutation-test kills for stats-recorded allocation byte counts, -//! `Arena::max_normal_alloc` routing boundaries, and `Vec::shrink_to_fit`'s -//! oversized-route bypass. Each test targets a specific `cargo mutants` -//! finding flagged as MISSED. +//! Mutation-test kills for `Arena::max_normal_alloc` routing boundaries and +//! `Vec::shrink_to_fit`'s oversized-route bypass. Each test targets a specific +//! `cargo mutants` finding flagged as MISSED. #![allow(clippy::std_instead_of_core, reason = "tests use std")] #![allow(clippy::unwrap_used, reason = "test code")] @@ -12,89 +11,6 @@ use multitude::vec::Vec as ArenaVec; use multitude::{Arena, ArenaBuilder}; -// --- record_alloc multiplication mutations (stats feature) ------------------- -// -// `Arena::record_alloc(bytes)` is the only observation of the -// `size_of::() * len` expression inside `impl_alloc_slice_fill_with` / -// `impl_alloc_slice_fill_iter` / `impl_alloc_utf16_smart_from_str`. -// Replacing `*` with `+` or `/` records a wildly different byte count; -// `Arena::stats().total_bytes_allocated` exposes it. - -#[cfg(feature = "stats")] -#[test] -fn stats_alloc_slice_fill_with_records_bytes_equals_size_times_len() { - // T = u32 (size 4), len = 7 → size * len = 28, size + len = 11, size / len = 0 - let arena = Arena::new(); - let baseline = arena.stats().total_bytes_allocated; - let _: &mut [u32] = arena.alloc_slice_fill_with::(7, |i| u32::try_from(i).expect("test bench length fits in u32")); - let delta = arena.stats().total_bytes_allocated - baseline; - assert_eq!(delta, 4 * 7, "fill_with must record size * len bytes"); -} - -#[cfg(feature = "stats")] -#[test] -fn stats_alloc_slice_fill_with_drop_records_bytes_equals_size_times_len() { - // Exercise the `needs_drop` branch (the inner `record_alloc` call - // sits next to a different `*`-bearing reservation site than the - // !Drop branch above). - #[derive(Clone)] - struct D(#[expect(dead_code, reason = "field gives the type a non-zero size")] u32); - // An empty `Drop` impl is exactly what this test needs: the body - // is irrelevant, only the `needs_drop::() == true` constant - // matters for routing through the with-drop reservation path. - #[expect(clippy::empty_drop, reason = "intentional: forces needs_drop::() = true")] - impl Drop for D { - fn drop(&mut self) {} - } - let arena = Arena::new(); - let baseline = arena.stats().total_bytes_allocated; - let _: &mut [D] = arena.alloc_slice_fill_with::(5, |i| D(u32::try_from(i).expect("test bench length fits in u32"))); - let delta = arena.stats().total_bytes_allocated - baseline; - assert_eq!(delta, core::mem::size_of::() as u64 * 5); -} - -#[cfg(feature = "stats")] -#[test] -fn stats_alloc_slice_fill_iter_records_bytes_equals_size_times_len() { - let arena = Arena::new(); - let baseline = arena.stats().total_bytes_allocated; - let _: &mut [u32] = arena.alloc_slice_fill_iter::(0..9_u32); - let delta = arena.stats().total_bytes_allocated - baseline; - assert_eq!(delta, 4 * 9, "fill_iter must record size * len bytes"); -} - -#[cfg(feature = "stats")] -#[test] -fn stats_alloc_slice_fill_iter_drop_records_bytes_equals_size_times_len() { - #[derive(Clone)] - struct D(#[expect(dead_code, reason = "field gives the type a non-zero size")] u32); - #[expect(clippy::empty_drop, reason = "intentional: forces needs_drop::() = true")] - impl Drop for D { - fn drop(&mut self) {} - } - let arena = Arena::new(); - let baseline = arena.stats().total_bytes_allocated; - let v: std::vec::Vec = (0..6_u32).map(D).collect(); - let _: &mut [D] = arena.alloc_slice_fill_iter::(v); - let delta = arena.stats().total_bytes_allocated - baseline; - assert_eq!(delta, core::mem::size_of::() as u64 * 6); -} - -#[cfg(feature = "stats")] -#[test] -fn stats_alloc_utf16_records_bytes_equals_units_times_two() { - // ASCII: 1 utf-16 code unit per char; mixed BMP / supplementary - // would also work but pure ASCII gives a deterministic count. - let arena = Arena::new(); - let baseline = arena.stats().total_bytes_allocated; - let s = "hello!"; // 6 utf-16 code units - let _ = arena.alloc_utf16_str_arc_from_str(s); - let delta = arena.stats().total_bytes_allocated - baseline; - // `exact * size_of::()` => 6 * 2 = 12. Mutants `+` → 8, - // `/` → 3. - assert_eq!(delta, 12, "alloc_utf16 must record code_units * 2 bytes"); -} - // --- ChunkProvider::config (kills `config -> Default`) ----------------------- // // `config().max_normal_alloc` decides whether an allocation routes to @@ -216,7 +132,7 @@ fn vec_shrink_to_fit_at_max_normal_alloc_boundary_reclaims() { // chunk (`refill_hint <= mna`), so the Vec lives in `current_local` // and `try_reclaim_tail` has a chance to fire. let cap = mna - 1; - let mut v: ArenaVec<'_, u8> = ArenaVec::with_capacity_in(cap, &arena); + let mut v: ArenaVec<'_, u8> = arena.alloc_vec_with_capacity(cap); v.extend_from_slice([7_u8; 16]); assert_eq!(v.capacity(), cap); v.shrink_to_fit(); diff --git a/crates/multitude/tests/mutant_kills_post_fix.rs b/crates/multitude/tests/mutant_kills_post_fix.rs index 2364a23c0..00c7d35e5 100644 --- a/crates/multitude/tests/mutant_kills_post_fix.rs +++ b/crates/multitude/tests/mutant_kills_post_fix.rs @@ -62,52 +62,6 @@ fn is_oversized_local_routes_above_threshold_via_oversized() { assert!(after_oversized > before_oversized); } -// record_alloc(size * len) in alloc_slice_fill_with/fill_iter -#[test] -fn slice_fill_with_records_size_times_len() { - let arena = Arena::new(); - let before = arena.stats().total_bytes_allocated; - let _s: &mut [u32] = arena.alloc_slice_fill_with(10, |i| u32::try_from(i).unwrap()); - let after = arena.stats().total_bytes_allocated; - assert_eq!(after - before, (10 * core::mem::size_of::()) as u64); -} - -#[test] -fn slice_fill_iter_records_size_times_len() { - let arena = Arena::new(); - let before = arena.stats().total_bytes_allocated; - let _s: &mut [u32] = arena.alloc_slice_fill_iter(0_u32..10); - let after = arena.stats().total_bytes_allocated; - assert_eq!(after - before, (10 * core::mem::size_of::()) as u64); -} - -// Same `record_alloc(size * len)` arithmetic on the oversized branches -// of fill_with / fill_iter (lines 322 / 372). The previous tests stay -// on the in-arena fast path (sub-`max_normal_alloc` allocations) and -// don't reach the oversized stats line. Trigger oversized routing by -// lowering `max_normal_alloc` below the slice payload size. -#[test] -fn slice_fill_with_oversized_records_size_times_len() { - let mna = 4096; - let arena = ArenaBuilder::new().max_normal_alloc(mna).build(); - let len = mna * 2; // exceeds threshold → oversized local chunk - let before = arena.stats().total_bytes_allocated; - let _s: &mut [u8] = arena.alloc_slice_fill_with(len, |_| 0); - let after = arena.stats().total_bytes_allocated; - assert_eq!(after - before, len as u64); -} - -#[test] -fn slice_fill_iter_oversized_records_size_times_len() { - let mna = 4096; - let arena = ArenaBuilder::new().max_normal_alloc(mna).build(); - let len = mna * 2; - let before = arena.stats().total_bytes_allocated; - let _s: &mut [u8] = arena.alloc_slice_fill_iter((0..len).map(|_| 0_u8)); - let after = arena.stats().total_bytes_allocated; - assert_eq!(after - before, len as u64); -} - // Vec::shrink_to_fit boundary: total < mna must reclaim (catches `==`/`>=` // mutants that would early-return at total == mna and below). #[test] @@ -119,7 +73,7 @@ fn shrink_to_fit_reclaims_strictly_below_max_normal_alloc() { // its end IS at the bump cursor. `total_bytes = cap = mna - 1`, // strictly below the threshold. let cap = mna - 1; - let mut v: multitude::vec::Vec<'_, u8> = multitude::vec::Vec::with_capacity_in(cap, &arena); + let mut v: multitude::vec::Vec<'_, u8> = arena.alloc_vec_with_capacity(cap); v.extend_from_slice([7_u8; 16]); assert_eq!(v.capacity(), cap); v.shrink_to_fit(); diff --git a/crates/multitude/tests/mutant_kills_strings.rs b/crates/multitude/tests/mutant_kills_strings.rs index 30ca9ab07..caed8341a 100644 --- a/crates/multitude/tests/mutant_kills_strings.rs +++ b/crates/multitude/tests/mutant_kills_strings.rs @@ -153,8 +153,8 @@ fn string_partial_eq_ref_str_true_and_false() { #[cfg(feature = "utf16")] mod utf16_kills { - use multitude::Arena; use multitude::strings::{ArcUtf16Str, BoxUtf16Str, Utf16String}; + use multitude::{Arena, FromIn as _}; use widestring::{Utf16Str, utf16str}; use super::hash_of; @@ -257,7 +257,7 @@ mod utf16_kills { #[test] fn utf16_string_display_renders_contents() { let arena = Arena::new(); - let mut s = Utf16String::from_str_in("hello", &arena); + let mut s = Utf16String::from_in("hello", &arena); assert_eq!(format!("{s}"), "hello"); s.push_from_str(" world"); assert_eq!(format!("{s}"), "hello world"); diff --git a/crates/multitude/tests/mutants_extras.rs b/crates/multitude/tests/mutants_extras.rs index a7a996c55..dec209449 100644 --- a/crates/multitude/tests/mutants_extras.rs +++ b/crates/multitude/tests/mutants_extras.rs @@ -775,41 +775,37 @@ mod mutants_for_kill2 { } } - /// Kills `vec.rs:760 + → -, *` in `Vec::try_into_arena_arc` and the - /// mirror at `vec.rs:859` in `Vec::into_arena_box_copy`. The closure - /// passed to `try_alloc_slice_fill_with_*` advances its read index - /// via `consumed_cell.set(idx + 1)`. The `* 1` mutation freezes the + /// Kills `vec.rs:760 + → -, *` in `Vec::try_into_arc` and the + /// mirror in `Vec::into_box`. The closure passed to + /// `alloc_slice_fill_iter_*` advances its read index via + /// `consumed_cell.set(idx + 1)`. The `* 1` mutation freezes the /// index at 0, so every element of the new slice is a bitwise copy of /// the original Vec's element 0. The `- 1` mutation underflows on the /// second iteration (UB / crash). A distinct-value assertion catches /// both. #[test] - fn vec_into_arena_arc_advances_read_index() { + fn vec_into_arc_advances_read_index() { let arena = multitude::Arena::new(); - let mut v: multitude::vec::Vec = multitude::vec::Vec::new_in(&arena); + let mut v: multitude::vec::Vec = arena.alloc_vec(); v.push(10); v.push(20); v.push(30); - let arc: multitude::Arc<[u32]> = v.into_arena_arc(); + let arc: multitude::Arc<[u32]> = multitude::Arc::from(v); assert_eq!(&*arc, &[10, 20, 30]); } #[test] - fn vec_into_arena_box_advances_read_index() { - // Force the copy fallback path (`into_arena_box_copy`) by using a - // builder-detached Vec (Vec::new), then into_arena_box, which - // routes to into_arena_box_copy when the buffer doesn't sit at the - // bump cursor. + fn vec_into_box_advances_read_index() { + // `Vec::into_box` moves the elements into a fresh shared + // allocation via `alloc_slice_fill_iter_box`, whose fill closure + // advances its read index per element. This exercises that + // advance and confirms the elements land in order. let arena = multitude::Arena::new(); - // Allocate another value to push the bump cursor past where this - // Vec's buffer lives, forcing the copy fallback. - let mut v: multitude::vec::Vec = multitude::vec::Vec::with_capacity_in(3, &arena); + let mut v: multitude::vec::Vec = arena.alloc_vec_with_capacity(3); v.push(11); v.push(22); v.push(33); - // Allocate something to detach the buffer from the bump cursor. - let _detach: &mut u64 = arena.alloc(0xdead_beef_u64); - let b: multitude::Box<[u32]> = v.into_arena_box(); + let b: multitude::Box<[u32]> = v.into_boxed_slice(); assert_eq!(&*b, &[11, 22, 33]); } } @@ -1381,17 +1377,17 @@ mod mutants_for_kill3 { assert_eq!(v[1], 99); // unchanged } - /// Kills: vec.rs:762:17 `+= -> -=` / `+= -> *=` in into_arena_box_copy + /// Kills: vec.rs:762:17 `+= -> -=` / `+= -> *=` in into_box /// `idx += 1` — if -= or *=, idx goes wrong and elements are /// read from wrong positions or the loop never terminates. #[test] - fn vec_762_into_arena_box_copy() { + fn vec_762_into_box() { let arena = Arena::new(); let mut v = arena.alloc_vec_with_capacity::(10); for i in 0..5 { v.push(i * 10); } - let boxed = v.into_arena_box(); + let boxed = v.into_boxed_slice(); assert_eq!(boxed.len(), 5); assert_eq!(boxed[0], 0); assert_eq!(boxed[1], 10); @@ -1821,7 +1817,7 @@ mod mutants_for_kill3 { assert_eq!(v.len(), 2); } - // vec:762 into_arena_box_copy: ZST/empty path only. ZSTs have no + // vec:762 into_box: ZST/empty path only. ZSTs have no // distinguishable element identity, empty vecs don't call the closure. // EQUIVALENT. @@ -2180,7 +2176,7 @@ mod mutants_for_audit { // ============================================================================ // vec.rs:634 — into_arena_rc's `if needs_drop && len > 0` - // vec.rs:837 — into_arena_box's `if needs_drop && self.len > 0` + // vec.rs:837 — into_box's `if needs_drop && self.len > 0` // // Mutant: `>` → `>=`. With `>= 0` (always true for usize) the empty Drop // vec would attempt to install a slice DropEntry of len=0, which would @@ -2199,7 +2195,7 @@ mod mutants_for_audit { // ============================================================================ // ============================================================================ - // vec.rs:911 — into_arena_box_copy's `consumed_cell.set(idx + 1)` + // vec.rs:911 — into_box's `consumed_cell.set(idx + 1)` // Mutant `+ → *`: at idx==0 both yield 0 → infinite loop / wrong index. // Kill: copy at least 2 elements and verify all are present. // ============================================================================ @@ -2711,7 +2707,7 @@ mod mutants_for_audit { } // ============================================================================ - // vec.rs:911:35 — `consumed_cell.set(idx + 1)` in `into_arena_box_copy`. + // vec.rs:911:35 — `consumed_cell.set(idx + 1)` in `into_box`. // Mutant `+` -> `*`: with idx==0, `0 * 1 == 0`; consumed_cell never // advances, every closure invocation reads `data[0]`. Kill: route the // buffer to an oversized chunk so install fails, then verify boxed @@ -3392,14 +3388,14 @@ mod mutants_for_final { } // ============================================================================ - // `cap == len` short-circuit: into_arena_box at exact cap=len skips reclaim. + // `cap == len` short-circuit: into_box at exact cap=len skips reclaim. // ---------------------------------------------------------------------------- // At `cap == len`, original skips reclaim; mutant `>=` tries to reclaim 0 // bytes (no-op). Behavior observable through chunk count not changing. // ============================================================================ // ============================================================================ - // `into_arena_box`'s ZST/empty routing (`== with !=` at line 834) + // `into_box`'s ZST/empty routing (`== with !=` at line 834) // ---------------------------------------------------------------------------- // Mutant inverts the early-return condition. Non-ZST non-empty vec must // take the in-place path (no new chunk). With mutant, it takes the copy @@ -3407,15 +3403,15 @@ mod mutants_for_final { // ============================================================================ #[test] - fn vec_into_arena_box_empty_routes_through_copy_path() { + fn vec_into_box_empty_routes_through_copy_path() { let arena = Arena::new(); let v: ArenaVec<'_, u32> = arena.alloc_vec(); - let b: ArenaBox<[u32]> = v.into_arena_box(); + let b: ArenaBox<[u32]> = v.into_boxed_slice(); assert_eq!(b.len(), 0); } // ============================================================================ - // `into_arena_box_copy`'s `consumed_cell.set(idx + 1)` (line 922) + // `into_box`'s `consumed_cell.set(idx + 1)` (line 922) // ---------------------------------------------------------------------------- // Mutant `+ with *`: `set(idx * 1) = idx`. Loop never advances and resulting // slice holds N copies of element 0. Detection: copy path with distinct diff --git a/crates/multitude/tests/std_alignment.rs b/crates/multitude/tests/std_alignment.rs new file mode 100644 index 000000000..bf0f42a0b --- /dev/null +++ b/crates/multitude/tests/std_alignment.rs @@ -0,0 +1,970 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for the `std`-aligned API surface added to `Vec` / `String` / +//! `Utf16String`: `From`-based freezing, `leak`, `shrink_to`, +//! `extend_from_within`, `spare_capacity_mut`, `Index`/`IndexMut`, +//! `Add`/`AddAssign`, `AsRef`, `TryFrom`, `into_bytes`/`into_vec`, +//! `split_off`, and `reserve_exact`. + +#![allow(clippy::unwrap_used, reason = "test code")] +#![allow(clippy::std_instead_of_core, reason = "test code uses std")] +#![allow(clippy::many_single_char_names, reason = "test code uses terse local bindings")] +#![allow(clippy::items_after_statements, reason = "test-local type aliases")] +#![allow(clippy::iter_on_single_items, reason = "tests exercise single-element iterators on purpose")] +#![allow(clippy::assertions_on_result_states, reason = "test code asserts Result states directly")] +#![allow(clippy::cast_possible_truncation, reason = "test code casts small, known-bounded counts")] + +use multitude::vec::Vec as MVec; +use multitude::{Arc, Arena, Box as MBox}; + +#[test] +fn vec_freeze_via_from_traits() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([1, 2, 3]); + let b: MBox<[u32]> = MBox::from(v); + assert_eq!(&*b, &[1, 2, 3]); + + let mut v2: MVec<'_, u32> = arena.alloc_vec(); + v2.extend([4, 5]); + let a: Arc<[u32]> = Arc::from(v2); + assert_eq!(&*a, &[4, 5]); +} + +#[test] +fn vec_into_boxed_slice_and_leak() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([7, 8, 9]); + let b = v.into_boxed_slice(); + assert_eq!(&*b, &[7, 8, 9]); + + let mut v2: MVec<'_, u32> = arena.alloc_vec(); + v2.extend([10, 20]); + let leaked: &mut [u32] = v2.leak(); + leaked[0] = 11; + assert_eq!(leaked, &[11, 20]); +} + +#[test] +fn vec_try_into_arc_and_boxed_slice() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([1, 2]); + let a = v.try_into_arc().unwrap(); + assert_eq!(&*a, &[1, 2]); + + let mut v2: MVec<'_, u32> = arena.alloc_vec(); + v2.extend([3, 4]); + let b = v2.try_into_boxed_slice().unwrap(); + assert_eq!(&*b, &[3, 4]); +} + +#[test] +fn vec_shrink_to_and_spare_capacity_mut() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec_with_capacity(16); + v.extend([1, 2, 3]); + v.shrink_to(8); + assert!(v.capacity() >= 8); + assert!(v.capacity() >= v.len()); + + let spare = v.spare_capacity_mut(); + assert_eq!(spare.len(), v.capacity() - 3); +} + +#[test] +fn vec_extend_from_within() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([1, 2, 3, 4]); + v.extend_from_within(1..3); + assert_eq!(&*v, &[1, 2, 3, 4, 2, 3]); +} + +#[test] +fn vec_index_and_as_ref() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([5, 6, 7]); + assert_eq!(v[1], 6); + assert_eq!(&v[1..], &[6, 7]); + v[0] = 50; + assert_eq!(v[0], 50); + let r: &MVec<'_, u32> = v.as_ref(); + assert_eq!(r.len(), 3); +} + +#[test] +fn vec_try_from_array() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([1, 2, 3]); + let arr: [u32; 3] = <[u32; 3]>::try_from(v).unwrap(); + assert_eq!(arr, [1, 2, 3]); + + let mut v2: MVec<'_, u32> = arena.alloc_vec(); + v2.extend([1, 2]); + let err = <[u32; 3]>::try_from(v2); + assert!(err.is_err()); + assert_eq!(err.unwrap_err().len(), 2); +} + +#[test] +fn string_freeze_via_from_and_into_boxed_str() { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push_str("hello"); + let b = MBox::::from(s); + assert_eq!(&*b, "hello"); + + let mut s2 = arena.alloc_string(); + s2.push_str("world"); + let a = Arc::::from(s2); + assert_eq!(&*a, "world"); + + let mut s3 = arena.alloc_string(); + s3.push_str("boxed"); + assert_eq!(&*s3.into_boxed_str(), "boxed"); +} + +#[test] +fn string_add_index_asref_leak() { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push_str("foo"); + s += "bar"; + assert_eq!(s.as_str(), "foobar"); + assert_eq!(&s[0..3], "foo"); + let bytes: &[u8] = s.as_ref(); + assert_eq!(bytes, b"foobar"); + + let s2 = arena.alloc_string() + "leaked"; + let leaked: &mut str = s2.leak(); + assert_eq!(&*leaked, "leaked"); +} + +#[test] +fn string_into_bytes_split_off_reserve_exact() { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push_str("hello world"); + let tail = s.split_off(5); + assert_eq!(s.as_str(), "hello"); + assert_eq!(tail.as_str(), " world"); + + let mut s2 = arena.alloc_string(); + s2.reserve_exact(32); + assert!(s2.capacity() >= 32); + s2.push_str("abc"); + let bytes = s2.into_bytes(); + assert_eq!(&*bytes, b"abc"); +} + +#[cfg(feature = "std")] +#[test] +fn string_as_ref_osstr_path() { + use std::ffi::OsStr; + use std::path::Path; + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push_str("/tmp/x"); + let os: &OsStr = s.as_ref(); + assert_eq!(os, OsStr::new("/tmp/x")); + let p: &Path = s.as_ref(); + assert_eq!(p, Path::new("/tmp/x")); +} + +#[test] +fn string_extend_variants() { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.extend([&'a', &'b']); + s.extend([std::string::String::from("cd")]); + s.extend([std::borrow::Cow::Borrowed("ef")]); + assert_eq!(s.as_str(), "abcdef"); +} + +#[cfg(feature = "utf16")] +#[test] +fn utf16_freeze_add_into_vec() { + use multitude::strings::{ArcUtf16Str, BoxUtf16Str}; + use widestring::utf16str; + let arena = Arena::new(); + + let mut s = arena.alloc_utf16_string(); + s.push_str(utf16str!("hi")); + let b = BoxUtf16Str::from(s); + assert_eq!(&*b, utf16str!("hi")); + + let mut s2 = arena.alloc_utf16_string(); + s2.push_str(utf16str!("yo")); + let a = ArcUtf16Str::from(s2); + assert_eq!(&*a, utf16str!("yo")); + + let mut s3 = arena.alloc_utf16_string(); + s3.push_str(utf16str!("ab")); + s3 += utf16str!("cd"); + assert_eq!(s3.as_utf16_str(), utf16str!("abcd")); + let units = s3.into_vec(); + assert_eq!(&*units, utf16str!("abcd").as_slice()); +} + +#[cfg(feature = "utf16")] +#[test] +fn utf16_split_off_index_extend() { + use widestring::utf16str; + let arena = Arena::new(); + + let mut s = arena.alloc_utf16_string(); + s.push_str(utf16str!("hello world")); + let tail = s.split_off(5); + assert_eq!(s.as_utf16_str(), utf16str!("hello")); + assert_eq!(tail.as_utf16_str(), utf16str!(" world")); + + // Range indexing (Output = Utf16Str). + assert_eq!(&s[0..3], utf16str!("hel")); + + let mut e = arena.alloc_utf16_string(); + e.extend([&'a', &'b']); + e.extend([std::string::String::from("cd")]); + e.extend([std::borrow::Cow::Borrowed("ef")]); + e.extend([std::boxed::Box::::from("gh")]); + assert_eq!(e.as_utf16_str(), utf16str!("abcdefgh")); + + // reserve_exact / shrink_to parity (shared shell). + let mut r = arena.alloc_utf16_string(); + r.reserve_exact(32); + assert!(r.capacity() >= 32); +} + +// ---- Drop-correctness edge cases (Miri-validated) ---- + +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct CountDrop<'c>(&'c AtomicUsize); +impl Drop for CountDrop<'_> { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +#[test] +fn try_from_vec_array_drop_no_double_free() { + let drops = AtomicUsize::new(0); + { + let arena = Arena::new(); + let mut v: MVec<'_, CountDrop<'_>> = arena.alloc_vec(); + v.push(CountDrop(&drops)); + v.push(CountDrop(&drops)); + let Ok(arr): Result<[CountDrop<'_>; 2], _> = <[CountDrop<'_>; 2]>::try_from(v) else { + panic!("try_from should succeed for matching length"); + }; + assert_eq!(drops.load(Ordering::SeqCst), 0, "no drops while owned by the array"); + drop(arr); + assert_eq!(drops.load(Ordering::SeqCst), 2, "exactly two drops from the array"); + } + // Arena teardown must not re-drop the moved-out elements. + assert_eq!(drops.load(Ordering::SeqCst), 2, "no double free at arena teardown"); +} + +#[test] +fn try_from_vec_array_length_mismatch_preserves_elements() { + let drops = AtomicUsize::new(0); + { + let arena = Arena::new(); + let mut v: MVec<'_, CountDrop<'_>> = arena.alloc_vec(); + v.push(CountDrop(&drops)); + let Err(v) = <[CountDrop<'_>; 3]>::try_from(v) else { + panic!("try_from should fail for length mismatch"); + }; + // The original element is still owned by the returned Vec. + assert_eq!(drops.load(Ordering::SeqCst), 0); + assert_eq!(v.len(), 1); + } + assert_eq!(drops.load(Ordering::SeqCst), 1, "single element dropped exactly once"); +} + +#[test] +fn extend_from_within_clone_type_no_leak() { + let arena = Arena::new(); + let mut v: MVec<'_, std::string::String> = arena.alloc_vec(); + v.push("a".to_string()); + v.push("b".to_string()); + v.extend_from_within(..); + assert_eq!(v.len(), 4); + assert_eq!(&v[2], "a"); + assert_eq!(&v[3], "b"); +} + +#[test] +#[should_panic(expected = "char boundary")] +fn string_split_off_non_char_boundary_panics() { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push('é'); // 2 bytes + let _ = s.split_off(1); +} + +#[test] +fn vec_spare_capacity_mut_write_then_set_len() { + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec_with_capacity(4); + { + let spare = v.spare_capacity_mut(); + assert!(spare.len() >= 3); + spare[0].write(10); + spare[1].write(20); + spare[2].write(30); + } + // SAFETY: 3 elements initialized above. + unsafe { v.set_len(3) }; + assert_eq!(&*v, &[10, 20, 30]); +} + +// ---- Coverage of the remaining new surface ---- + +#[test] +fn vec_as_mut_and_extend_from_within_bound_variants() { + use core::ops::Bound; + let arena = Arena::new(); + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([1, 2, 3, 4]); + + // AsMut. + let m: &mut MVec<'_, u32> = v.as_mut(); + m.push(5); + assert_eq!(v.len(), 5); + + // Inclusive-end bound (hits the `Included` end arm). + let mut a: MVec<'_, u32> = arena.alloc_vec(); + a.extend([10, 20, 30]); + a.extend_from_within(0..=1); + assert_eq!(&*a, &[10, 20, 30, 10, 20]); + + // Excluded-start bound (hits the `Excluded` start arm). + let mut b: MVec<'_, u32> = arena.alloc_vec(); + b.extend([7, 8, 9]); + b.extend_from_within((Bound::Excluded(0usize), Bound::Unbounded)); + assert_eq!(&*b, &[7, 8, 9, 8, 9]); +} + +#[test] +fn vec_extend_from_within_empty_range_does_not_reserve() { + // An empty source range clones nothing, so `extend_from_within` must not + // grow the backing buffer. This pins down the reserved `count = end - start` + // (an inflated count would reallocate). The range and assertion are derived + // from the *observed* capacity, so this is independent of growth policy. + let arena = Arena::new(); + let mut b: MVec<'_, u32> = arena.alloc_vec(); + b.extend(0..16u32); + let len = b.len(); + let cap = b.capacity(); + // `start == end` (empty), chosen so a count of `end + start` would exceed + // the current capacity while the correct count of `0` leaves it untouched. + let start = (cap - len) / 2 + 1; + assert!(start <= len, "test precondition: spare capacity is small"); + let before: std::vec::Vec = b.as_slice().to_vec(); + b.extend_from_within(start..start); + assert_eq!(b.capacity(), cap, "empty extend_from_within must not reserve"); + assert_eq!(b.as_slice(), &*before); +} + +#[test] +fn string_index_mut_as_mut_vec_extend_box_reserve_shrink() { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push_str("hello"); + + // IndexMut → &mut str. + let sub: &mut str = &mut s[0..2]; + sub.make_ascii_uppercase(); + assert_eq!(s.as_str(), "HEllo"); + + // as_mut_vec (unsafe): append a valid ASCII byte. + // SAFETY: pushing an ASCII byte keeps the buffer valid UTF-8. + unsafe { s.as_mut_vec() }.push(b'!'); + assert_eq!(s.as_str(), "HEllo!"); + + // Extend>. + s.extend([std::boxed::Box::::from("XY")]); + assert_eq!(s.as_str(), "HEllo!XY"); + + // try_reserve_exact + shrink_to (shared shell). + s.try_reserve_exact(64).unwrap(); + assert!(s.capacity() >= s.len() + 64); + s.shrink_to(0); + assert!(s.capacity() >= s.len()); +} + +#[cfg(feature = "utf16")] +#[test] +fn utf16_leak_as_mut_vec_asref_add_indexmut() { + use widestring::utf16str; + let arena = Arena::new(); + + // Add (`+`, not `+=`). + let s = arena.alloc_utf16_string() + utf16str!("ab"); + let added = s + utf16str!("cd"); + assert_eq!(added.as_utf16_str(), utf16str!("abcd")); + + // AsRef<[u16]>. + let r: &[u16] = added.as_ref(); + assert_eq!(r, utf16str!("abcd").as_slice()); + + // IndexMut → &mut Utf16Str. + let mut m = arena.alloc_utf16_string(); + m.push_str(utf16str!("hello")); + { + let _sub: &mut widestring::Utf16Str = &mut m[0..2]; + } + + // as_mut_vec (unsafe): append a BMP unit. + // SAFETY: U+0021 ('!') is a standalone BMP code unit, keeping UTF-16 valid. + unsafe { m.as_mut_vec() }.push(0x0021); + assert_eq!(m.len(), 6); + + // leak. + let leaked: &mut widestring::Utf16Str = m.leak(); + assert_eq!(leaked.len(), 6); +} + +// ---- FromIn / IntoIn ---- + +use multitude::{FromIn, IntoIn}; + +#[test] +fn vec_from_in_all_variants() { + let arena = Arena::new(); + + // &[T] + let src = [1_u32, 2, 3]; + let a: MVec<'_, u32> = MVec::from_in(&src[..], &arena); + assert_eq!(&*a, &[1, 2, 3]); + + // &mut [T] + let mut srcm = [4_u32, 5]; + let b: MVec<'_, u32> = MVec::from_in(&mut srcm[..], &arena); + assert_eq!(&*b, &[4, 5]); + + // [T; N] + let c: MVec<'_, u32> = MVec::from_in([6_u32, 7, 8], &arena); + assert_eq!(&*c, &[6, 7, 8]); + + // Box<[T]> + let boxed: std::boxed::Box<[u32]> = std::boxed::Box::from([9_u32, 10]); + let d: MVec<'_, u32> = MVec::from_in(boxed, &arena); + assert_eq!(&*d, &[9, 10]); + + // Cow<[T]> borrowed + owned + let e: MVec<'_, u32> = MVec::from_in(std::borrow::Cow::Borrowed(&src[..]), &arena); + assert_eq!(&*e, &[1, 2, 3]); + let owned: std::vec::Vec = vec![11, 12]; + let f: MVec<'_, u32> = MVec::from_in(std::borrow::Cow::Owned(owned), &arena); + assert_eq!(&*f, &[11, 12]); + + // IntoIn companion (target type drives inference). + let g: MVec<'_, u32> = [13_u32, 14].into_in(&arena); + assert_eq!(&*g, &[13, 14]); +} + +#[test] +fn string_from_in_all_variants() { + let arena = Arena::new(); + + let a: multitude::strings::String<'_> = FromIn::from_in("abc", &arena); + assert_eq!(a.as_str(), "abc"); + + let c: multitude::strings::String<'_> = FromIn::from_in('Z', &arena); + assert_eq!(c.as_str(), "Z"); + + let mut owned = std::string::String::from("mut"); + let m: multitude::strings::String<'_> = FromIn::from_in(owned.as_mut_str(), &arena); + assert_eq!(m.as_str(), "mut"); + + let d: multitude::strings::String<'_> = FromIn::from_in(std::borrow::Cow::Borrowed("cow"), &arena); + assert_eq!(d.as_str(), "cow"); + + let e: multitude::strings::String<'_> = FromIn::from_in(std::boxed::Box::::from("boxed"), &arena); + assert_eq!(e.as_str(), "boxed"); + + // IntoIn + let f: multitude::strings::String<'_> = "into".into_in(&arena); + assert_eq!(f.as_str(), "into"); +} + +#[cfg(feature = "utf16")] +#[test] +fn utf16_from_in_all_variants() { + use widestring::utf16str; + let arena = Arena::new(); + + let a: multitude::strings::Utf16String<'_> = FromIn::from_in(utf16str!("hi"), &arena); + assert_eq!(a.as_utf16_str(), utf16str!("hi")); + + let b: multitude::strings::Utf16String<'_> = FromIn::from_in("u8", &arena); + assert_eq!(b.as_utf16_str(), utf16str!("u8")); + + let c: multitude::strings::Utf16String<'_> = FromIn::from_in('Q', &arena); + assert_eq!(c.as_utf16_str(), utf16str!("Q")); + + let d: multitude::strings::Utf16String<'_> = FromIn::from_in(std::borrow::Cow::Borrowed("cow"), &arena); + assert_eq!(d.as_utf16_str(), utf16str!("cow")); + + let cow16: std::borrow::Cow<'_, widestring::Utf16Str> = std::borrow::Cow::Borrowed(utf16str!("w16")); + let g: multitude::strings::Utf16String<'_> = FromIn::from_in(cow16, &arena); + assert_eq!(g.as_utf16_str(), utf16str!("w16")); + + let e: multitude::strings::Utf16String<'_> = FromIn::from_in(std::boxed::Box::::from("bx"), &arena); + assert_eq!(e.as_utf16_str(), utf16str!("bx")); + + // IntoIn + let f: multitude::strings::Utf16String<'_> = utf16str!("yo").into_in(&arena); + assert_eq!(f.as_utf16_str(), utf16str!("yo")); +} + +// ---- FromIteratorIn breadth (matching std's FromIterator set) ---- + +use multitude::vec::CollectIn; + +#[test] +fn string_collect_in_all_item_types() { + let arena = Arena::new(); + type S<'a> = multitude::strings::String<'a>; + + let a: S<'_> = ['a', 'b', 'c'].into_iter().collect_in(&arena); + assert_eq!(a.as_str(), "abc"); + + let chars = ['x', 'y']; + let b: S<'_> = chars.iter().collect_in(&arena); // &char + assert_eq!(b.as_str(), "xy"); + + let c: S<'_> = ["fo", "ob", "ar"].into_iter().collect_in(&arena); // &str + assert_eq!(c.as_str(), "foobar"); + + let d: S<'_> = [std::string::String::from("hi"), std::string::String::from("!")] + .into_iter() + .collect_in(&arena); + assert_eq!(d.as_str(), "hi!"); + + let e: S<'_> = [std::boxed::Box::::from("bo"), std::boxed::Box::::from("x")] + .into_iter() + .collect_in(&arena); + assert_eq!(e.as_str(), "box"); + + let f: S<'_> = [ + std::borrow::Cow::Borrowed("co"), + std::borrow::Cow::Owned(std::string::String::from("w")), + ] + .into_iter() + .collect_in(&arena); + assert_eq!(f.as_str(), "cow"); +} + +#[cfg(feature = "utf16")] +#[test] +fn utf16_collect_in_all_item_types() { + use widestring::utf16str; + let arena = Arena::new(); + type U<'a> = multitude::strings::Utf16String<'a>; + + let a: U<'_> = ['a', 'b'].into_iter().collect_in(&arena); + assert_eq!(a.as_utf16_str(), utf16str!("ab")); + + let chars = ['x', 'y']; + let b: U<'_> = chars.iter().collect_in(&arena); // &char + assert_eq!(b.as_utf16_str(), utf16str!("xy")); + + let c: U<'_> = ["fo", "ob"].into_iter().collect_in(&arena); // &str (transcode) + assert_eq!(c.as_utf16_str(), utf16str!("foob")); + + let d: U<'_> = [utf16str!("u1"), utf16str!("6s")].into_iter().collect_in(&arena); // &Utf16Str + assert_eq!(d.as_utf16_str(), utf16str!("u16s")); + + let e: U<'_> = [std::string::String::from("hi")].into_iter().collect_in(&arena); + assert_eq!(e.as_utf16_str(), utf16str!("hi")); + + let f: U<'_> = [std::boxed::Box::::from("bx")].into_iter().collect_in(&arena); + assert_eq!(f.as_utf16_str(), utf16str!("bx")); + + let g: U<'_> = [std::borrow::Cow::Borrowed("cw")].into_iter().collect_in(&arena); + assert_eq!(g.as_utf16_str(), utf16str!("cw")); +} + +// ---- Arena string byte/unit constructors ---- + +#[test] +fn arena_string_from_utf8_variants() { + let arena = Arena::new(); + + let ok = arena.alloc_string_from_utf8(b"h\xC3\xA9llo").unwrap(); + assert_eq!(ok.as_str(), "héllo"); + + let err = arena.alloc_string_from_utf8(&[0x66, 0xFF, 0x6F]); + assert!(err.is_err()); + + let lossy = arena.alloc_string_from_utf8_lossy(&[0x66, 0xFF, 0x6F]); + assert_eq!(lossy.as_str(), "f\u{FFFD}o"); + + // SAFETY: the literal is valid UTF-8. + let unchecked = unsafe { arena.alloc_string_from_utf8_unchecked(b"abc") }; + assert_eq!(unchecked.as_str(), "abc"); +} + +#[test] +fn arena_string_from_utf16_variants() { + let arena = Arena::new(); + + // "ab" + U+10000 (surrogate pair D800 DC00). + let units = [0x0061_u16, 0x0062, 0xD800, 0xDC00]; + let ok = arena.alloc_string_from_utf16(&units).unwrap(); + assert_eq!(ok.as_str(), "ab\u{10000}"); + + // Unpaired high surrogate. + let bad = [0x0061_u16, 0xD800, 0x0062]; + assert!(arena.alloc_string_from_utf16(&bad).is_err()); + + let lossy = arena.alloc_string_from_utf16_lossy(&bad); + assert_eq!(lossy.as_str(), "a\u{FFFD}b"); +} + +#[test] +fn arena_string_from_utf16_endian_variants() { + let arena = Arena::new(); + + // "ab" + U+10000 (D800 DC00) in little-endian bytes. + let le = [0x61, 0x00, 0x62, 0x00, 0x00, 0xD8, 0x00, 0xDC]; + let s = arena.alloc_string_from_utf16le(&le).unwrap(); + assert_eq!(s.as_str(), "ab\u{10000}"); + + // Same in big-endian. + let be = [0x00, 0x61, 0x00, 0x62, 0xD8, 0x00, 0xDC, 0x00]; + let s = arena.alloc_string_from_utf16be(&be).unwrap(); + assert_eq!(s.as_str(), "ab\u{10000}"); + + // Odd length → error. + assert!(arena.alloc_string_from_utf16le(&[0x61, 0x00, 0x62]).is_err()); + // Unpaired surrogate (lone high surrogate 0xD800 LE) → error. + let err = arena.alloc_string_from_utf16le(&[0x00, 0xD8]).unwrap_err(); + assert!(!err.to_string().is_empty()); + + // Lossy: odd trailing byte and unpaired surrogate both → U+FFFD. + let lossy = arena.alloc_string_from_utf16le_lossy(&[0x61, 0x00, 0x00, 0xD8, 0x62]); + assert_eq!(lossy.as_str(), "a\u{FFFD}\u{FFFD}"); + let lossy_be = arena.alloc_string_from_utf16be_lossy(&[0x00, 0x61]); + assert_eq!(lossy_be.as_str(), "a"); +} + +// ---- extend_from_within / into_flattened / drain / splice ---- + +#[test] +fn string_extend_from_within() { + let arena = Arena::new(); + let mut s = arena.alloc_string(); + s.push_str("abcd"); + s.extend_from_within(1..3); + assert_eq!(s.as_str(), "abcdbc"); + s.extend_from_within(..); + assert_eq!(s.as_str(), "abcdbcabcdbc"); + + // Explicit Excluded-start / Included-end bounds. + let mut s2 = arena.alloc_string(); + s2.push_str("abcd"); + s2.extend_from_within((core::ops::Bound::Excluded(0), core::ops::Bound::Included(2))); // 1..3 => "bc" + assert_eq!(s2.as_str(), "abcdbc"); +} + +#[cfg(feature = "utf16")] +#[test] +fn utf16_extend_from_within() { + use widestring::utf16str; + let arena = Arena::new(); + let mut s = arena.alloc_utf16_string(); + s.push_str(utf16str!("abcd")); + s.extend_from_within(1..3); + assert_eq!(s.as_utf16_str(), utf16str!("abcdbc")); + + // Unbounded start + Included end. + let mut s2 = arena.alloc_utf16_string(); + s2.push_str(utf16str!("abcd")); + s2.extend_from_within(..=1); // 0..2 => "ab" + assert_eq!(s2.as_utf16_str(), utf16str!("abcdab")); + + // Excluded start + Unbounded end. + let mut s3 = arena.alloc_utf16_string(); + s3.push_str(utf16str!("abcd")); + s3.extend_from_within((core::ops::Bound::Excluded(1), core::ops::Bound::Unbounded)); // 2..4 => "cd" + assert_eq!(s3.as_utf16_str(), utf16str!("abcdcd")); +} + +#[test] +fn vec_into_flattened() { + let arena = Arena::new(); + let mut v: MVec<'_, [u32; 3]> = arena.alloc_vec(); + v.push([1, 2, 3]); + v.push([4, 5, 6]); + let flat: MVec<'_, u32> = v.into_flattened(); + assert_eq!(&*flat, &[1, 2, 3, 4, 5, 6]); + + // ZST element type exercises the `usize::MAX` capacity branch. + let mut zv: MVec<'_, [(); 2]> = arena.alloc_vec(); + zv.push([(), ()]); + zv.push([(), ()]); + let flatz: MVec<'_, ()> = zv.into_flattened(); + assert_eq!(flatz.len(), 4); + + // Drop type: every element dropped exactly once. + let drops = AtomicUsize::new(0); + { + let arena = Arena::new(); + let mut v: MVec<'_, [CountDrop<'_>; 2]> = arena.alloc_vec(); + v.push([CountDrop(&drops), CountDrop(&drops)]); + let flat = v.into_flattened(); + assert_eq!(flat.len(), 2); + assert_eq!(drops.load(Ordering::SeqCst), 0); + } + assert_eq!(drops.load(Ordering::SeqCst), 2); +} + +/// Collects a `char`-yielding drain iterator, panicking if it fails to +/// terminate within `max` items. Bounding the iteration means a mutated +/// `next`/`next_back` that never returns `None` (e.g. replaced with +/// `Some(Default::default())`) is caught as a fast failure instead of +/// hanging the test process. +fn bounded_chars>(it: I, max: usize) -> std::string::String { + let mut out = std::string::String::new(); + let mut n = 0usize; + for c in it { + n += 1; + assert!(n <= max, "drain iterator failed to terminate within {max} items"); + out.push(c); + } + out +} + +#[test] +fn string_drain_forward_back_and_removal() { + let arena = Arena::new(); + + let mut s = arena.alloc_string(); + s.push_str("héllo wörld"); + let drained = bounded_chars(s.drain(0..7), 16); // "héllo " (é is 2 bytes) + assert_eq!(drained, "héllo "); + assert_eq!(s.as_str(), "wörld"); + + // Double-ended (reverse) consumption. + let mut s2 = arena.alloc_string(); + s2.push_str("abçd"); + assert_eq!(bounded_chars(s2.drain(..).rev(), 16), "dçba"); + + // Explicit forward `next` values (kills a `next -> Some(Default)` mutant + // without iterating to completion); partial consumption still removes the + // whole range. + let mut s3 = arena.alloc_string(); + s3.push_str("0123456789"); + { + let mut d = s3.drain(2..8); + assert_eq!(d.next(), Some('2')); + assert_eq!(d.next(), Some('3')); + } + assert_eq!(s3.as_str(), "0189"); + + // Explicit backward `next_back` over a 4-byte char: exercises the + // continuation-buffer fill (`buf[4 - n]`) at `n == 4`. + let mut s4 = arena.alloc_string(); + s4.push_str("x\u{1F600}"); // 'x' + a 4-byte char + { + let mut d = s4.drain(..); + assert_eq!(d.next_back(), Some('\u{1F600}')); + assert_eq!(d.next_back(), Some('x')); + assert_eq!(d.next_back(), None); + } + + // 3-byte and 4-byte chars, full forward drain, plus `Debug`. + let mut s5 = arena.alloc_string(); + s5.push_str("a€b\u{1F600}c"); // € is 3 bytes, U+1F600 is 4 bytes + let d5 = s5.drain(..); + let _ = format!("{d5:?}"); + assert_eq!(bounded_chars(d5, 16), "a€b\u{1F600}c"); + assert!(s5.as_str().is_empty()); + + // `size_hint`: 8 remaining bytes => lower bound ceil(8/4)=2, upper 8. + let mut s6 = arena.alloc_string(); + s6.push_str("abcdefgh"); + let d6 = s6.drain(..); + assert_eq!(d6.size_hint(), (2, Some(8))); + + // Included-end (`..=`) and Excluded-start bounds (removal is eager). + let mut s7 = arena.alloc_string(); + s7.push_str("0123456789"); + assert_eq!(bounded_chars(s7.drain(1..=3), 8), "123"); // bytes 1..4 + let mut s8 = arena.alloc_string(); + s8.push_str("0123456789"); + assert_eq!( + bounded_chars(s8.drain((core::ops::Bound::Excluded(0), core::ops::Bound::Unbounded)), 16), + "123456789" + ); +} + +#[cfg(feature = "utf16")] +#[test] +fn utf16_drain_forward_back() { + use widestring::utf16str; + let arena = Arena::new(); + + let mut s = arena.alloc_utf16_string(); + s.push_str(utf16str!("ab")); // BMP + s.push('\u{10000}'); // surrogate pair (2 units) + s.push_str(utf16str!("cd")); + // units: a b [hi lo] c d => indices 0..6 + assert_eq!(bounded_chars(s.drain(2..4), 8), "\u{10000}"); // the U+10000 char + assert_eq!(s.as_utf16_str(), utf16str!("abcd")); + + // Reverse over a string containing a surrogate pair. + let mut s2 = arena.alloc_utf16_string(); + s2.push('x'); + s2.push('\u{1F600}'); // pair + s2.push('y'); + assert_eq!(bounded_chars(s2.drain(..).rev(), 8), "y\u{1F600}x"); + + // Explicit forward `next` over BMP chars (kills a `next -> Some(Default)` + // mutant) plus `Debug`. + let mut s3 = arena.alloc_utf16_string(); + s3.push_str(utf16str!("abc")); + let mut d3 = s3.drain(..); + let _ = format!("{d3:?}"); + assert_eq!(d3.next(), Some('a')); + assert_eq!(d3.next(), Some('b')); + assert_eq!(d3.next(), Some('c')); + assert_eq!(d3.next(), None); + + // Explicit backward `next_back`. + let mut s3b = arena.alloc_utf16_string(); + s3b.push_str(utf16str!("pq")); + { + let mut d = s3b.drain(..); + assert_eq!(d.next_back(), Some('q')); + assert_eq!(d.next_back(), Some('p')); + assert_eq!(d.next_back(), None); + } + + // `size_hint`: 6 remaining units => lower ceil(6/2)=3, upper 6. + let mut s4 = arena.alloc_utf16_string(); + s4.push_str(utf16str!("abcdef")); + let d4 = s4.drain(..); + assert_eq!(d4.size_hint(), (3, Some(6))); + + // Included-end / Excluded-start bounds. + let mut s5 = arena.alloc_utf16_string(); + s5.push_str(utf16str!("0123456789")); + assert_eq!(bounded_chars(s5.drain(1..=3), 8), "123"); + let mut s6 = arena.alloc_utf16_string(); + s6.push_str(utf16str!("0123456789")); + assert_eq!( + bounded_chars(s6.drain((core::ops::Bound::Excluded(0), core::ops::Bound::Unbounded)), 16), + "123456789" + ); +} + +#[test] +fn vec_splice() { + let arena = Arena::new(); + + // Replace a middle range with more elements. + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([1, 2, 3, 4, 5]); + let removed: std::vec::Vec = v.splice(1..4, [20, 30, 40, 50]).collect(); + assert_eq!(removed, vec![2, 3, 4]); + assert_eq!(&*v, &[1, 20, 30, 40, 50, 5]); + + // Empty range = pure insert; fewer replacements = net removal. + let mut v2: MVec<'_, u32> = arena.alloc_vec(); + v2.extend([1, 2, 3]); + let r2: std::vec::Vec = v2.splice(1..1, [9]).collect(); + assert!(r2.is_empty()); + assert_eq!(&*v2, &[1, 9, 2, 3]); + + let mut v3: MVec<'_, u32> = arena.alloc_vec(); + v3.extend([1, 2, 3, 4]); + let r3: std::vec::Vec = v3.splice(1..3, core::iter::empty()).collect(); + assert_eq!(r3, vec![2, 3]); + assert_eq!(&*v3, &[1, 4]); + + // Bound variants: Unbounded start + Included end. + let mut v4: MVec<'_, u32> = arena.alloc_vec(); + v4.extend([1, 2, 3, 4, 5]); + let r4: std::vec::Vec = v4.splice(..=1, [9]).collect(); // 0..2 + assert_eq!(r4, vec![1, 2]); + assert_eq!(&*v4, &[9, 3, 4, 5]); + + // Excluded start + Unbounded end. + let mut v5: MVec<'_, u32> = arena.alloc_vec(); + v5.extend([1, 2, 3, 4]); + let r5: std::vec::Vec = v5 + .splice((core::ops::Bound::Excluded(0), core::ops::Bound::Unbounded), [7, 8]) + .collect(); // 1..4 + assert_eq!(r5, vec![2, 3, 4]); + assert_eq!(&*v5, &[1, 7, 8]); + + // `Debug` + double-ended consumption. + let mut v6: MVec<'_, u32> = arena.alloc_vec(); + v6.extend([1, 2, 3, 4]); + let mut sp = v6.splice(0..3, core::iter::empty()); + let _ = format!("{sp:?}"); + assert_eq!(sp.next_back(), Some(3)); + assert_eq!(sp.next(), Some(1)); + drop(sp); + assert_eq!(&*v6, &[4]); +} + +#[test] +fn vec_splice_size_hint_and_reserve() { + let arena = Arena::new(); + + // `size_hint` reflects the number of removed elements. + let mut v: MVec<'_, u32> = arena.alloc_vec(); + v.extend([1, 2, 3, 4, 5]); + let sp = v.splice(1..4, core::iter::empty()); // removes 3 + assert_eq!(sp.size_hint(), (3, Some(3))); + drop(sp); + assert_eq!(&*v, &[1, 5]); + + // The up-front `reserve(replacement.len() + kept.len())` must use the + // *sum*, not the product. With `replacement == capacity - 2` and + // `kept == 2`, the correct sum exactly fills the existing capacity (no + // realloc), whereas the product would force a larger allocation. Both + // counts are derived from the observed capacity, so this holds for any + // growth policy. + let mut w: MVec<'_, u32> = arena.alloc_vec(); + w.extend(0..8u32); + let cap = w.capacity(); + assert!(cap > 4); + let r = cap - 2; // replacement count + let end = w.len() - 2; // keep the last 2 elements (kept == 2) + let removed: std::vec::Vec = w.splice(0..end, 0..r as u32).collect(); + assert_eq!(removed, (0..end as u32).collect::>()); + let mut expected: std::vec::Vec = (0..r as u32).collect(); + expected.extend([6u32, 7]); + assert_eq!(&*w, &*expected); + assert_eq!(w.len(), cap); // start(0) + r + kept(2) == cap + assert_eq!(w.capacity(), cap, "splice reserved the sum, not the product (+ -> *)"); +} + +#[test] +fn arena_string_from_utf16_bytes_capacity_hints() { + let arena = Arena::new(); + // 10 ASCII chars => 20 little-endian bytes, 10 code units, 10 UTF-8 bytes. + let bytes: std::vec::Vec = "ABCDEFGHIJ".bytes().flat_map(|b| [b, 0]).collect(); + + // Non-lossy: capacity hint is `bytes.len() / 2` == 10; the decoded ASCII + // fills it exactly, so the (exact) preallocation leaves capacity == 10. + let s = arena.alloc_string_from_utf16le(&bytes).unwrap(); + assert_eq!(s.as_str(), "ABCDEFGHIJ"); + assert_eq!(s.capacity(), 10); + + // Lossy: capacity hint is `bytes.len() / 2 + 1` == 11. + let s_lossy = arena.alloc_string_from_utf16le_lossy(&bytes); + assert_eq!(s_lossy.as_str(), "ABCDEFGHIJ"); + assert_eq!(s_lossy.capacity(), 11); +} diff --git a/crates/multitude/tests/utf16.rs b/crates/multitude/tests/utf16.rs index 35fd7465b..7d9276f28 100644 --- a/crates/multitude/tests/utf16.rs +++ b/crates/multitude/tests/utf16.rs @@ -56,7 +56,7 @@ mod utf16_smoke { } #[test] - fn arc_into_arena_arc_slice() { + fn arc_into_arc_slice() { let arena = Arena::new(); let a = arena.alloc_utf16_str_arc(utf16str!("abc")); let bytes: multitude::Arc<[u16]> = a.into(); @@ -67,7 +67,7 @@ mod utf16_smoke { // === merged from tests/utf16_string_builder.rs === mod utf16_string_builder { - use multitude::Arena; + use multitude::{Arena, FromIn as _}; use widestring::utf16str; #[expect(unused_imports, reason = "merged test module re-exports common helpers")] @@ -187,9 +187,9 @@ mod utf16_string_builder { } #[test] - fn from_str_in_and_from_utf16_str_in() { + fn from_in_str_and_from_utf16_str_in() { let arena = Arena::new(); - let a = multitude::strings::Utf16String::from_str_in("hello", &arena); + let a = multitude::strings::Utf16String::from_in("hello", &arena); assert_eq!(a.as_utf16_str(), utf16str!("hello")); let b = multitude::strings::Utf16String::from_utf16_str_in(utf16str!("world"), &arena); assert_eq!(b.as_utf16_str(), utf16str!("world")); @@ -282,7 +282,7 @@ mod utf16_string_builder { #[test] fn try_with_capacity_in_zero_does_not_allocate_but_is_usable() { let arena = Arena::new(); - let s = multitude::strings::Utf16String::try_with_capacity_in(0, &arena).unwrap(); + let s = arena.try_alloc_utf16_string_with_capacity(0).unwrap(); assert_eq!(s.len(), 0); assert_eq!(s.capacity(), 0); assert!(s.is_empty()); @@ -291,7 +291,7 @@ mod utf16_string_builder { #[test] fn try_with_capacity_in_respects_requested_capacity() { let arena = Arena::new(); - let s = multitude::strings::Utf16String::try_with_capacity_in(8, &arena).unwrap(); + let s = arena.try_alloc_utf16_string_with_capacity(8).unwrap(); assert!(s.capacity() >= 8); assert_eq!(s.len(), 0); } @@ -299,7 +299,7 @@ mod utf16_string_builder { #[test] fn try_with_capacity_in_one_allocates_at_least_one_unit() { let arena = Arena::new(); - let s = multitude::strings::Utf16String::try_with_capacity_in(1, &arena).unwrap(); + let s = arena.try_alloc_utf16_string_with_capacity(1).unwrap(); assert!(s.capacity() >= 1); assert_eq!(s.len(), 0); } @@ -524,7 +524,7 @@ mod utf16_coverage { use std::panic::AssertUnwindSafe; use multitude::strings::{String, Utf16String}; - use multitude::{Arc, Arena, Box}; + use multitude::{Arc, Arena, Box, FromIn as _}; use widestring::{Utf16Str, utf16str}; #[expect(unused_imports, reason = "merged test module re-exports common helpers")] @@ -715,21 +715,21 @@ mod utf16_coverage { #[test] fn try_push_err() { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); assert!(s.try_push('a').is_err()); } #[test] fn try_push_str_err() { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); assert!(s.try_push_str(utf16str!("abc")).is_err()); } #[test] fn try_push_from_str_err() { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); assert!(s.try_push_from_str("abc").is_err()); } @@ -745,7 +745,7 @@ mod utf16_coverage { fn reserve_panics_when_alloc_fails() { expect_panic(|| { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); s.reserve(64); }); } @@ -753,7 +753,7 @@ mod utf16_coverage { #[test] fn try_reserve_err() { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); assert!(s.try_reserve(64).is_err()); } @@ -853,7 +853,7 @@ mod utf16_coverage { #[test] fn from_utf16_str_in_and_from_str_in() { let arena = Arena::new(); - let a = Utf16String::from_str_in("hello, 💖", &arena); + let a = Utf16String::from_in("hello, 💖", &arena); assert_eq!(a.as_utf16_str(), utf16str!("hello, 💖")); let b = Utf16String::from_utf16_str_in(utf16str!("world"), &arena); assert_eq!(b.as_utf16_str(), utf16str!("world")); @@ -941,7 +941,7 @@ mod utf16_coverage { #[test] fn grow_doubling_path() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(4, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(4); s.push_str(utf16str!("abcd")); s.push('e'); // forces grow; doubling 4*2 = 8 covers needed 5 assert!(s.capacity() >= 8); @@ -951,7 +951,7 @@ mod utf16_coverage { #[test] fn grow_uses_min_cap_when_doubling_too_small() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(4, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(4); s.push_str(utf16str!("abcd")); let big = "x".repeat(100); s.push_from_str(&big); // forces grow; min_cap > 2*old_cap @@ -1016,7 +1016,7 @@ mod utf16_coverage { #[test] fn insert_grows_capacity() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(4, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(4); s.push_str(utf16str!("abcd")); s.insert_utf16_str(2, utf16str!("XYZW")); assert_eq!(s.as_utf16_str(), utf16str!("abXYZWcd")); @@ -1026,7 +1026,7 @@ mod utf16_coverage { #[test] fn replace_range_grows_capacity() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(4, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(4); s.push_str(utf16str!("abcd")); s.replace_range(0..1, utf16str!("XXXXX")); // adds 4 → needs cap 8 assert_eq!(s.as_utf16_str(), utf16str!("XXXXXbcd")); @@ -1118,7 +1118,7 @@ mod utf16_coverage { // FailingAllocator(1): builder gets initial chunk (cap=4), then a huge // try_reserve would need a fresh oversized chunk → second alloc fails. let arena = Arena::builder().allocator_in(FailingAllocator::new(1)).build(); - let mut s = Utf16String::try_with_capacity_in(4, &arena).unwrap(); + let mut s = arena.try_alloc_utf16_string_with_capacity(4).unwrap(); s.try_push_str(utf16str!("abcd")).unwrap(); assert!(s.try_reserve(64 * 1024).is_err()); } @@ -1127,7 +1127,7 @@ mod utf16_coverage { fn panic_grow_to_at_least() { expect_panic(|| { let arena = Arena::builder().allocator_in(FailingAllocator::new(1)).build(); - let mut s = Utf16String::try_with_capacity_in(4, &arena).unwrap(); + let mut s = arena.try_alloc_utf16_string_with_capacity(4).unwrap(); s.try_push_str(utf16str!("abcd")).unwrap(); // grow_to_at_least's panic_alloc lambda fires here. s.push_from_str("x".repeat(64 * 1024)); @@ -1142,7 +1142,7 @@ mod utf16_coverage { fn panic_push_when_alloc_fails() { expect_panic(|| { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); s.push('a'); }); } @@ -1151,7 +1151,7 @@ mod utf16_coverage { fn panic_push_str_when_alloc_fails() { expect_panic(|| { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); s.push_str(utf16str!("abc")); }); } @@ -1160,7 +1160,7 @@ mod utf16_coverage { fn panic_push_from_str_when_alloc_fails() { expect_panic(|| { let a = fail_arena(); - let mut s = Utf16String::new_in(&a); + let mut s = a.alloc_utf16_string(); s.push_from_str("abc"); }); } @@ -1168,7 +1168,7 @@ mod utf16_coverage { #[test] fn reserve_no_growth_path() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(16, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(16); s.reserve(4); // already have cap >= len + 4; no-op branch assert_eq!(s.capacity(), 16); assert_eq!(s.len(), 0); @@ -1184,14 +1184,14 @@ mod utf16_coverage { // cap*2 > isize::MAX but ≤ usize::MAX on 64-bit. checked_mul succeeds, // checked_add succeeds, isize::try_from fails → AllocError. let cap = (isize::MAX.unsigned_abs() / 2) + 1000; - let r = Utf16String::try_with_capacity_in(cap, &arena); + let r = arena.try_alloc_utf16_string_with_capacity(cap); r.unwrap_err(); } #[test] fn try_grow_isize_overflow_guard() { let arena = Arena::new(); - let mut s = Utf16String::try_with_capacity_in(4, &arena).unwrap(); + let mut s = arena.try_alloc_utf16_string_with_capacity(4).unwrap(); // Force try_grow_to_at_least → new_cap such that new_cap*2 > isize::MAX // but ≤ usize::MAX. The isize::try_from check on new_total returns Err. let huge = (isize::MAX.unsigned_abs() / 2) + 1000; @@ -1250,7 +1250,7 @@ mod mutants_for_utf16_strings { #[test] fn shrink_to_fit_reduces_capacity_to_len() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(64, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(64); s.push_str(utf16str!("abc")); assert_eq!(s.len(), 3); let cap_before = s.capacity(); @@ -1275,7 +1275,7 @@ mod mutants_for_utf16_strings { #[test] fn reserve_exact_capacity_does_not_regrow() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(8, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(8); assert!(s.capacity() >= 8); let cap = s.capacity(); let ptr_before = s.as_ptr(); @@ -1294,7 +1294,7 @@ mod mutants_for_utf16_strings { #[test] fn push_slice_at_exact_fit_does_not_regrow() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(8, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(8); let cap = s.capacity(); let ptr_before = s.as_ptr(); // Push exactly `cap` u16s. @@ -1342,7 +1342,7 @@ mod mutants_for_utf16_strings { // Insert exactly at the grow boundary: build a String at known // cap, then insert enough to hit needed == cap. - let mut t = Utf16String::with_capacity_in(8, &arena); + let mut t = arena.alloc_utf16_string_with_capacity(8); t.push_str(utf16str!("abcd")); let ptr_before = t.as_ptr(); let cap = t.capacity(); @@ -1382,7 +1382,7 @@ mod mutants_for_utf16_strings { assert_eq!(s.as_utf16_str(), utf16str!("ll")); // Folded from_mutants_extras_utf16_scattered::utf16_remove_shifts_correct_byte_count. - let mut s = Utf16String::new_in(&arena); + let mut s = arena.alloc_utf16_string(); s.push_from_str("abcdefghij"); assert_eq!(s.remove(4), 'e'); assert_eq!(s.as_slice().to_vec(), widestring::Utf16String::from_str("abcdfghij").into_vec()); @@ -1454,12 +1454,12 @@ mod mutants_for_utf16_strings { assert_eq!(s.as_utf16_str(), utf16str!("abcxyz")); // Replace into an empty string. - let mut s = Utf16String::new_in(&arena); + let mut s = arena.alloc_utf16_string(); s.replace_range(0..0, utf16str!("abc")); assert_eq!(s.as_utf16_str(), utf16str!("abc")); } - /// Kills `utf16_string.rs:466:21 == → !=` in `into_arena_box_utf16_str`. + /// Kills `utf16_string.rs:466:21 == → !=` in `into_box`. /// /// The branch is `if self.cap == 0 { … return empty … }`. With `!=`, /// the empty fast-path runs for non-empty strings (UB / wrong data) @@ -1467,21 +1467,63 @@ mod mutants_for_utf16_strings { /// We test both branches by freezing an empty and a non-empty /// string into a `BoxUtf16Str`. #[test] - fn into_arena_box_utf16_str_handles_empty_and_non_empty() { + fn into_box_handles_empty_and_non_empty() { let arena = Arena::new(); // Empty case. - let s = Utf16String::new_in(&arena); - let b = s.into_arena_box_utf16_str(); + let s = arena.alloc_utf16_string(); + let b = s.into_boxed_utf16_str(); assert_eq!(&*b, utf16str!("")); assert_eq!(b.len(), 0); // Non-empty case. let s = Utf16String::from_utf16_str_in(utf16str!("hello, world"), &arena); - let b = s.into_arena_box_utf16_str(); + let b = s.into_boxed_utf16_str(); assert_eq!(&*b, utf16str!("hello, world")); assert_eq!(b.len(), 12); } + /// `Utf16String::into_arc` freezes into a shared, reference-counted + /// `ArcUtf16Str` whose contents match the builder, for both empty + /// and non-empty inputs, and which can be cloned and outlive the + /// arena. + #[test] + fn into_arc_handles_empty_and_non_empty() { + use multitude::strings::ArcUtf16Str; + + let arena = Arena::new(); + + let s_empty = arena.alloc_utf16_string(); + let a_empty: ArcUtf16Str = ArcUtf16Str::from(s_empty); + assert_eq!(&*a_empty, utf16str!("")); + assert_eq!(a_empty.len(), 0); + + let s = Utf16String::from_utf16_str_in(utf16str!("hello, world"), &arena); + let a: ArcUtf16Str = ArcUtf16Str::from(s); + assert_eq!(&*a, utf16str!("hello, world")); + assert_eq!(a.len(), 12); + + // Cloning shares the same backing allocation. + let a2 = a.clone(); + assert_eq!(&*a2, utf16str!("hello, world")); + assert_eq!(a.as_ptr(), a2.as_ptr()); + } + + /// An `ArcUtf16Str` produced by `into_arc` outlives the arena it was + /// built from (the backing shared chunk is held by the refcount). + #[test] + fn into_arc_outlives_arena() { + use multitude::strings::ArcUtf16Str; + + let escaped: ArcUtf16Str = { + let arena = Arena::new(); + let s = Utf16String::from_utf16_str_in(utf16str!("survives"), &arena); + let a = ArcUtf16Str::from(s); + drop(arena); + a + }; + assert_eq!(&*escaped, utf16str!("survives")); + } + /// Kills `utf16_string.rs:491:29 - → /` in `try_reclaim_tail`. /// /// `total - used` (= unused trailing bytes). With `/` produces a tiny @@ -1499,9 +1541,9 @@ mod mutants_for_utf16_strings { #[test] fn reclaim_tail_does_not_corrupt_frozen_string() { let arena = Arena::new(); - let mut s = Utf16String::with_capacity_in(256, &arena); + let mut s = arena.alloc_utf16_string_with_capacity(256); s.push_str(utf16str!("frozen")); - let frozen = s.into_arena_box_utf16_str(); + let frozen = s.into_boxed_utf16_str(); // Allocate something in the same arena to potentially overlap with // the reclaimed tail bytes. let _filler: multitude::vec::Vec<'_, u64> = { @@ -1531,7 +1573,7 @@ mod mutation_coverage { use multitude::strings::{BoxUtf16Str, String as ArenaString, Utf16String}; use multitude::vec::Vec as ArenaVec; - use multitude::{Arena, ArenaBuilder, Box}; + use multitude::{Arena, ArenaBuilder, Box, FromIn as _}; use widestring::{Utf16Str, utf16str}; use crate::common; @@ -1553,15 +1595,15 @@ mod mutation_coverage { #[test] fn test_string_is_empty_false_when_nonempty() { let arena = Arena::new(); - let s = ArenaString::from_str_in("alpha", &arena); + let s = ArenaString::from_in("alpha", &arena); assert!(!s.is_empty()); } #[test] fn test_string_partial_eq_distinguishes_different_values() { let arena = Arena::new(); - let a = ArenaString::from_str_in("alpha", &arena); - let b = ArenaString::from_str_in("beta", &arena); + let a = ArenaString::from_in("alpha", &arena); + let b = ArenaString::from_in("beta", &arena); let beta = "beta"; assert_ne!(a, b); @@ -1572,8 +1614,8 @@ mod mutation_coverage { #[test] fn test_string_hash_depends_on_contents() { let arena = Arena::new(); - let a = ArenaString::from_str_in("alpha", &arena); - let b = ArenaString::from_str_in("beta", &arena); + let a = ArenaString::from_in("alpha", &arena); + let b = ArenaString::from_in("beta", &arena); assert_ne!(hash_value(&a), hash_value(&b)); } @@ -1581,7 +1623,7 @@ mod mutation_coverage { #[test] fn test_string_as_ref_returns_expected_contents() { let arena = Arena::new(); - let s = ArenaString::from_str_in("expected", &arena); + let s = ArenaString::from_in("expected", &arena); let r: &str = s.as_ref(); assert_eq!(r, "expected"); @@ -1614,7 +1656,7 @@ mod mutation_coverage { #[test] fn test_string_remove_from_middle_preserves_tail() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcde", &arena); + let mut s = ArenaString::from_in("abcde", &arena); assert_eq!(s.remove(2), 'c'); assert_eq!(s.as_str(), "abde"); @@ -1623,7 +1665,7 @@ mod mutation_coverage { #[test] fn test_string_retain_keeps_only_requested_chars() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abracadabra", &arena); + let mut s = ArenaString::from_in("abracadabra", &arena); s.retain(|ch| ch == 'a'); @@ -1644,7 +1686,7 @@ mod mutation_coverage { #[test] fn test_string_replace_range_shorter_shifts_tail_left() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcXYZdef", &arena); + let mut s = ArenaString::from_in("abcXYZdef", &arena); s.replace_range(3..6, "Q"); @@ -1804,7 +1846,7 @@ mod mutation_coverage { #[test] fn test_vec_is_empty_false_when_nonempty() { let arena = Arena::new(); - let mut v: ArenaVec = ArenaVec::new_in(&arena); + let mut v: ArenaVec = arena.alloc_vec(); v.push(1); assert!(!v.is_empty()); } @@ -2002,7 +2044,7 @@ mod mutation_coverage { #[test] fn test_string_partial_eq_str_returns_false_for_mismatch() { let arena = Arena::new(); - let s = ArenaString::from_str_in("alpha", &arena); + let s = ArenaString::from_in("alpha", &arena); assert!(!>::eq(&s, "beta")); } @@ -2016,7 +2058,7 @@ mod mutation_coverage { #[test] fn test_vec_try_with_capacity_in_zero_does_not_allocate() { let arena = Arena::new(); - let v = ArenaVec::::try_with_capacity_in(0, &arena).unwrap(); + let v = arena.try_alloc_vec_with_capacity::(0).unwrap(); assert_eq!(v.capacity(), 0); assert_eq!(v.len(), 0); } @@ -2024,7 +2066,7 @@ mod mutation_coverage { #[test] fn test_vec_try_reserve_at_exact_boundary_does_not_grow() { let arena = Arena::new(); - let mut v = ArenaVec::::try_with_capacity_in(8, &arena).unwrap(); + let mut v = arena.try_alloc_vec_with_capacity::(8).unwrap(); v.push(1); v.push(2); // cap=8, len=2, spare = cap - len = 6 @@ -2119,7 +2161,7 @@ mod mutation_coverage { #[test] fn test_vec_try_reserve_exact_at_boundary_does_not_grow() { let arena = Arena::new(); - let mut v = ArenaVec::::try_with_capacity_in(8, &arena).unwrap(); + let mut v = arena.try_alloc_vec_with_capacity::(8).unwrap(); v.push(1); v.push(2); let cap_before = v.capacity(); @@ -2189,7 +2231,7 @@ mod mutation_coverage { #[test] fn test_string_shrink_to_fit_cap_equals_len_is_noop() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("hello", &arena); + let mut s = ArenaString::from_in("hello", &arena); // Force a shrink_to_fit so cap == len s.shrink_to_fit(); let cap_after_first = s.capacity(); @@ -2206,7 +2248,7 @@ mod mutation_coverage { assert_eq!(s.capacity(), 0); // Can't mutably borrow to call shrink_to_fit on a zero-capacity string // through the arena API, but we can create one fresh - let mut s2: ArenaString = ArenaString::new_in(&arena); + let mut s2: ArenaString = arena.alloc_string(); s2.shrink_to_fit(); // cap == 0 path assert_eq!(s2.len(), 0); } @@ -2214,7 +2256,7 @@ mod mutation_coverage { #[test] fn test_string_insert_str_at_start_shifts_all() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("world", &arena); + let mut s = ArenaString::from_in("world", &arena); s.insert_str(0, "hello "); assert_eq!(s.as_str(), "hello world"); } @@ -2233,7 +2275,7 @@ mod mutation_coverage { #[test] fn test_string_remove_first_char_shifts_all() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcde", &arena); + let mut s = ArenaString::from_in("abcde", &arena); assert_eq!(s.remove(0), 'a'); assert_eq!(s.as_str(), "bcde"); } @@ -2241,7 +2283,7 @@ mod mutation_coverage { #[test] fn test_string_remove_multibyte_char() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("aéc", &arena); + let mut s = ArenaString::from_in("aéc", &arena); assert_eq!(s.remove(1), 'é'); assert_eq!(s.as_str(), "ac"); } @@ -2249,7 +2291,7 @@ mod mutation_coverage { #[test] fn test_string_retain_removes_alternating() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcdef", &arena); + let mut s = ArenaString::from_in("abcdef", &arena); let mut toggle = false; s.retain(|_| { toggle = !toggle; @@ -2261,7 +2303,7 @@ mod mutation_coverage { #[test] fn test_string_retain_removes_none() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abc", &arena); + let mut s = ArenaString::from_in("abc", &arena); s.retain(|_| true); assert_eq!(s.as_str(), "abc"); } @@ -2269,7 +2311,7 @@ mod mutation_coverage { #[test] fn test_string_retain_removes_all() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abc", &arena); + let mut s = ArenaString::from_in("abc", &arena); s.retain(|_| false); assert_eq!(s.as_str(), ""); } @@ -2287,7 +2329,7 @@ mod mutation_coverage { #[test] fn test_string_replace_range_shrink_by_exact_amount() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcdef", &arena); + let mut s = ArenaString::from_in("abcdef", &arena); // Replace "bcde" (4 bytes) with "X" (1 byte), shrink of 3 s.replace_range(1..5, "X"); assert_eq!(s.as_str(), "aXf"); @@ -2296,7 +2338,7 @@ mod mutation_coverage { #[test] fn test_string_replace_range_same_size() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcdef", &arena); + let mut s = ArenaString::from_in("abcdef", &arena); s.replace_range(2..4, "CD"); assert_eq!(s.as_str(), "abCDef"); } @@ -2304,7 +2346,7 @@ mod mutation_coverage { #[test] fn test_string_replace_range_shrink_at_end() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("abcdef", &arena); + let mut s = ArenaString::from_in("abcdef", &arena); // Replace "ef" with "" — shrink == removed, no tail shift needed s.replace_range(4..6, ""); assert_eq!(s.as_str(), "abcd"); @@ -2356,7 +2398,7 @@ mod mutation_coverage { #[test] fn test_string_extend_empty_chars() { let arena = Arena::new(); - let mut s = ArenaString::from_str_in("hello", &arena); + let mut s = ArenaString::from_in("hello", &arena); let cap_before = s.capacity(); s.extend(std::iter::empty::()); assert_eq!(s.as_str(), "hello"); @@ -2377,7 +2419,7 @@ mod mutation_coverage { #[test] fn test_utf16_string_shrink_to_fit_cap_zero_is_noop() { let arena = Arena::new(); - let mut s = Utf16String::new_in(&arena); + let mut s = arena.alloc_utf16_string(); s.shrink_to_fit(); assert_eq!(s.len(), 0); } @@ -3057,7 +3099,7 @@ mod from_coverage_extras_utf16 { #![allow(clippy::empty_line_after_doc_comments, reason = "relocated test doc-comments")] use multitude::strings::Utf16String; use multitude::vec::FromIteratorIn; - use multitude::{Arena, ArenaBuilder}; + use multitude::{Arena, ArenaBuilder, FromIn as _}; use widestring::utf16str; #[expect(unused_imports, reason = "relocated tests may reference common helpers")] @@ -3075,7 +3117,7 @@ mod from_coverage_extras_utf16 { #[should_panic(expected = "allocator returned AllocError")] fn utf16_string_insert_panics_from_grow_to_at_least() { let arena = ArenaBuilder::new_in(FailingAllocator::new(1)).build(); - let mut s = Utf16String::from_str_in("a", &arena); + let mut s = Utf16String::from_in("a", &arena); // Any replacement that forces a grow request triggers the panic; // the `FailingAllocator` denies every allocation after the first // regardless of size, so a moderate replacement (well past the @@ -3087,7 +3129,7 @@ mod from_coverage_extras_utf16 { #[test] fn utf16_string_reserve_zero_on_nonempty_string_is_noop() { let arena = Arena::new(); - let mut s = Utf16String::from_str_in("already allocated", &arena); + let mut s = Utf16String::from_in("already allocated", &arena); let cap = s.capacity(); s.reserve(0); assert_eq!(s.capacity(), cap); @@ -3277,7 +3319,7 @@ mod from_mutants_extras_utf16_scattered { fn utf16_shrink_to_fit_uses_multiplication() { use multitude::strings::Utf16String; let arena = Arena::new(); - let mut s: Utf16String<'_> = Utf16String::new_in(&arena); + let mut s: Utf16String<'_> = arena.alloc_utf16_string(); s.reserve(32); for _ in 0..8_u16 { s.push('a'); @@ -3333,7 +3375,7 @@ mod from_mutants_extras_utf16_scattered { fn utf16_exact_fit_push_is_observably_identical() { use multitude::strings::Utf16String; let arena = Arena::new(); - let mut s: Utf16String<'_> = Utf16String::new_in(&arena); + let mut s: Utf16String<'_> = arena.alloc_utf16_string(); s.reserve(4); let cap = s.capacity(); // Push exactly `cap` ASCII chars (each 1 u16). After the last push,