From d898c8f590819fb54cdfa374c9181ade59eb7533 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Fri, 21 Aug 2026 19:53:32 -0700 Subject: [PATCH] introduce an options struct for constructing backend storage (#97284) Improve legibility a bit, requested by @bgw here https://github.com/vercel/next.js/pull/95976#discussion_r3762093713 --- .../src/next_api/project.rs | 13 ++++--- .../src/next_api/turbopack_ctx.rs | 18 +++++----- .../src/database/turbo/mod.rs | 36 +++++++++---------- .../src/kv_backing_storage.rs | 30 ++++++++++++---- .../crates/turbo-tasks-backend/src/lib.rs | 26 ++++++++++---- .../turbo-tasks-backend/tests/eviction.rs | 12 ++++--- .../turbo-tasks-backend/tests/test_config.trs | 7 ++-- .../turbo-tasks-fetch/tests/test_config.trs | 7 ++-- .../turbopack-analyze/tests/test_config.trs | 7 ++-- .../crates/turbopack-cli/src/build/mod.rs | 15 +++++--- turbopack/crates/turbopack-cli/src/dev/mod.rs | 15 +++++--- .../turbopack-node/tests/test_config.trs | 7 ++-- .../tests/node-file-trace.rs | 7 ++-- 13 files changed, 125 insertions(+), 75 deletions(-) diff --git a/crates/next-napi-bindings/src/next_api/project.rs b/crates/next-napi-bindings/src/next_api/project.rs index fdefb38e1a2d..3a19cc9628df 100644 --- a/crates/next-napi-bindings/src/next_api/project.rs +++ b/crates/next-napi-bindings/src/next_api/project.rs @@ -57,7 +57,7 @@ use turbo_tasks::{ trace::TraceRawVcs, unmark_top_level_task_may_leak_eventually_consistent_state, }; -use turbo_tasks_backend::db_invalidation::invalidation_reasons; +use turbo_tasks_backend::{BackingStorageOptions, db_invalidation::invalidation_reasons}; #[cfg(windows)] use turbo_tasks_fs::windows::to_verbatim_with_case_folded_disk; use turbo_tasks_fs::{ @@ -576,18 +576,17 @@ pub fn project_new<'env>( env.spawn_future( async move { let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true); - let is_ci = turbo_engine_options.is_ci.unwrap_or(false); - let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false); - let skip_compaction = turbo_engine_options.skip_compaction.unwrap_or(false); let turbopack_memory_eviction = turbo_engine_options.turbopack_memory_eviction; let turbo_tasks = create_turbo_tasks( PathBuf::from(&options.dist_dir), &options.next_version, options.is_persistent_caching_enabled, dependency_tracking, - is_ci, - is_short_session, - skip_compaction, + BackingStorageOptions { + is_ci: turbo_engine_options.is_ci.unwrap_or(false), + is_short_session: turbo_engine_options.is_short_session.unwrap_or(false), + skip_compaction: turbo_engine_options.skip_compaction.unwrap_or(false), + }, turbopack_memory_eviction, )?; let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks); diff --git a/crates/next-napi-bindings/src/next_api/turbopack_ctx.rs b/crates/next-napi-bindings/src/next_api/turbopack_ctx.rs index 0f4276c649f5..5b7c790ca17d 100644 --- a/crates/next-napi-bindings/src/next_api/turbopack_ctx.rs +++ b/crates/next-napi-bindings/src/next_api/turbopack_ctx.rs @@ -25,8 +25,9 @@ use turbo_tasks::{ message_queue::{CompilationEvent, Severity}, }; use turbo_tasks_backend::{ - BackendOptions, EvictionMode, GitVersionInfo, StartupCacheState, TurboTasksBackend, - db_invalidation::invalidation_reasons, noop_backing_storage, turbo_backing_storage, + BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, StartupCacheState, + TurboTasksBackend, db_invalidation::invalidation_reasons, noop_backing_storage, + turbo_backing_storage, }; pub type NextTurboTasks = Arc>; @@ -289,20 +290,21 @@ pub fn create_turbo_tasks( next_version: &str, persistent_caching: bool, dependency_tracking: bool, - is_ci: bool, - is_short_session: bool, - skip_compaction: bool, + storage_options: BackingStorageOptions, turbopack_memory_eviction: MemoryEvictionMode, ) -> Result { + let BackingStorageOptions { + is_ci, + is_short_session, + .. + } = storage_options; Ok(if persistent_caching { let describe = cache_describe(next_version); let version_info = git_version_info(&describe); let (backing_storage, cache_state) = turbo_backing_storage( &output_path.join("cache").join("turbopack"), &version_info, - is_ci, - is_short_session, - skip_compaction, + storage_options, )?; let tt = TurboTasks::new(TurboTasksBackend::new( BackendOptions { diff --git a/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs b/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs index 7f7603ec7731..33ea4ef40330 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs @@ -17,7 +17,10 @@ use turbo_tasks::{ turbo_tasks, }; -use crate::database::{key_value_database::KeySpace, write_batch::WriteBuffer}; +use crate::{ + BackingStorageOptions, + database::{key_value_database::KeySpace, write_batch::WriteBuffer}, +}; mod parallel_scheduler; pub(crate) use parallel_scheduler::TurboTasksParallelScheduler; @@ -49,31 +52,22 @@ pub const COMPACT_CONFIG: CompactConfig = CompactConfig { pub struct TurboKeyValueDatabase { db: TurboPersistence, - is_ci: bool, - is_short_session: bool, + options: BackingStorageOptions, is_fresh: bool, - skip_compaction: bool, } impl TurboKeyValueDatabase { - pub fn new( - versioned_path: PathBuf, - is_ci: bool, - is_short_session: bool, - skip_compaction: bool, - ) -> Result { + pub fn new(versioned_path: PathBuf, options: BackingStorageOptions) -> Result { assert!( - !skip_compaction || is_short_session, + !options.skip_compaction || options.is_short_session, "skip_compaction=true requires is_short_session=true" ); let db = TurboPersistence::open_with_config(versioned_path, db_config())?; let is_fresh = db.is_empty(); Ok(Self { db, - is_ci, - is_short_session, + options, is_fresh, - skip_compaction, }) } @@ -83,10 +77,12 @@ impl TurboKeyValueDatabase { pub fn empty_in_memory() -> Self { Self { db: TurboPersistence::empty_in_memory_with_config(db_config()), - is_ci: false, - is_short_session: true, + options: BackingStorageOptions { + is_ci: false, + is_short_session: true, + skip_compaction: true, + }, is_fresh: true, - skip_compaction: true, } } @@ -129,7 +125,7 @@ impl TurboKeyValueDatabase { /// Returns `Ok(Some(stats))` with the bytes written/deleted if compaction actually merged /// files, `Ok(None)` if there was nothing to compact. pub fn compact(&self) -> Result> { - if self.is_short_session || self.db.is_empty() { + if self.options.is_short_session || self.db.is_empty() { return Ok(None); } do_compact( @@ -148,8 +144,8 @@ impl TurboKeyValueDatabase { pub fn shutdown(&self) -> Result<()> { // Compact the database on shutdown // (Avoid compacting a fresh database since we don't have any usage info yet) - if !self.is_fresh && !self.skip_compaction { - if self.is_ci { + if !self.is_fresh && !self.options.skip_compaction { + if self.options.is_ci { // Fully compact in CI to reduce cache size do_compact(&self.db, COMPACTION_MESSAGE, usize::MAX)?; } else { diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index d08e6759fb09..d6d5a9c52fbb 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -491,7 +491,18 @@ mod tests { use turbo_tasks::TaskId; use super::*; - use crate::database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer}; + use crate::{ + BackingStorageOptions, + database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer}, + }; + + /// Options used by these tests. `is_short_session` disables background compaction, which + /// requires a turbo-tasks context that these tests don't set up. + const TEST_STORAGE_OPTIONS: BackingStorageOptions = BackingStorageOptions { + is_ci: false, + is_short_session: true, + skip_compaction: false, + }; /// Helper to write to the database using the concurrent batch API. fn write_task_cache_entry( @@ -533,9 +544,7 @@ mod tests { let tempdir = tempfile::tempdir()?; let path = tempdir.path(); - // Use is_short_session=true to disable background compaction (which requires turbo-tasks - // context) - let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?; + let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?; // Simulate a hash collision by writing multiple TaskIds with the same hash key let collision_hash: u64 = 0xDEADBEEF; @@ -575,7 +584,7 @@ mod tests { // Write all entries in a single batch with flush (like save_snapshot does) { - let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?; + let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?; let batch = db.write_batch()?; for (hash, task_id) in hashes.iter().zip(task_ids.iter()) { @@ -594,7 +603,7 @@ mod tests { // Reopen and verify all entries are readable { - let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?; + let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?; let mut found = 0; let mut missing = 0; for (hash, expected_id) in hashes.iter().zip(task_ids.iter()) { @@ -632,7 +641,14 @@ mod tests { let survivor_id = TaskId::try_from(222u32).unwrap(); let deleted_key = (*deleted_id).to_le_bytes(); - let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?; + let db = TurboKeyValueDatabase::new( + path.to_path_buf(), + BackingStorageOptions { + is_ci: false, + is_short_session: true, + skip_compaction: false, + }, + )?; // Both ids collide in one TaskCache bucket, purely on disk; the deleted task also has // meta and data entries. diff --git a/turbopack/crates/turbo-tasks-backend/src/lib.rs b/turbopack/crates/turbo-tasks-backend/src/lib.rs index ced95af8cd44..e7eececc584b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/lib.rs +++ b/turbopack/crates/turbo-tasks-backend/src/lib.rs @@ -25,19 +25,33 @@ pub use crate::{ kv_backing_storage::TurboBackingStorage, }; +/// Options controlling how the on-disk persistent cache database is opened and compacted. +#[derive(Clone, Copy, Debug, Default)] +pub struct BackingStorageOptions { + /// Whether the process is running in a CI environment. Enables more aggressive (full) + /// compaction on shutdown to reduce the size of the cache that gets uploaded. + pub is_ci: bool, + /// Whether this is a short-lived session (e.g. a single build). Disables background + /// persistence during the session + pub is_short_session: bool, + /// Whether to skip database compaction on shutdown entirely + pub skip_compaction: bool, +} + /// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`]. /// /// Information about the state of the on-disk cache is returned using [`StartupCacheState`]. pub fn turbo_backing_storage( base_path: &Path, version_info: &GitVersionInfo, - is_ci: bool, - is_short_session: bool, - skip_compaction: bool, + options: BackingStorageOptions, ) -> Result<(TurboBackingStorage, StartupCacheState)> { - TurboBackingStorage::open_versioned_on_disk(base_path.to_owned(), version_info, is_ci, |path| { - TurboKeyValueDatabase::new(path, is_ci, is_short_session, skip_compaction) - }) + TurboBackingStorage::open_versioned_on_disk( + base_path.to_owned(), + version_info, + options.is_ci, + |path| TurboKeyValueDatabase::new(path, options), + ) } /// Creates an in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. Backed by diff --git a/turbopack/crates/turbo-tasks-backend/tests/eviction.rs b/turbopack/crates/turbo-tasks-backend/tests/eviction.rs index 5cb01a81b78b..0dec4c995701 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/eviction.rs +++ b/turbopack/crates/turbo-tasks-backend/tests/eviction.rs @@ -11,7 +11,9 @@ use anyhow::Result; use turbo_tasks::{ ResolvedVc, State, TurboTasks, Vc, unmark_top_level_task_may_leak_eventually_consistent_state, }; -use turbo_tasks_backend::{BackendOptions, EvictionMode, GitVersionInfo, TurboTasksBackend}; +use turbo_tasks_backend::{ + BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, TurboTasksBackend, +}; /// Creates a fresh per-call persistence directory in the OS temp dir, with the test `name` as a /// prefix so a leaked directory from a failed run is identifiable. The unique suffix from @@ -48,9 +50,11 @@ fn create_tt_with_workers( describe: "test-unversioned", dirty: false, }, - false, - true, - true, + BackingStorageOptions { + is_short_session: true, + skip_compaction: true, + ..Default::default() + }, ) .unwrap() .0, diff --git a/turbopack/crates/turbo-tasks-backend/tests/test_config.trs b/turbopack/crates/turbo-tasks-backend/tests/test_config.trs index 6ba3540fa199..152b1b254260 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/test_config.trs +++ b/turbopack/crates/turbo-tasks-backend/tests/test_config.trs @@ -20,9 +20,10 @@ describe: "test-unversioned", dirty: false, }, - false, - true, - false, + turbo_tasks_backend::BackingStorageOptions { + is_short_session: true, + ..Default::default() + }, ).unwrap().0 ) ) diff --git a/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs b/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs index e181c52b334a..af936dfcbdf0 100644 --- a/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs +++ b/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs @@ -19,9 +19,10 @@ describe: "test-unversioned", dirty: false, }, - false, - true, - false, + turbo_tasks_backend::BackingStorageOptions { + is_short_session: true, + ..Default::default() + }, ).unwrap().0 ) ) diff --git a/turbopack/crates/turbopack-analyze/tests/test_config.trs b/turbopack/crates/turbopack-analyze/tests/test_config.trs index e181c52b334a..af936dfcbdf0 100644 --- a/turbopack/crates/turbopack-analyze/tests/test_config.trs +++ b/turbopack/crates/turbopack-analyze/tests/test_config.trs @@ -19,9 +19,10 @@ describe: "test-unversioned", dirty: false, }, - false, - true, - false, + turbo_tasks_backend::BackingStorageOptions { + is_short_session: true, + ..Default::default() + }, ).unwrap().0 ) ) diff --git a/turbopack/crates/turbopack-cli/src/build/mod.rs b/turbopack/crates/turbopack-cli/src/build/mod.rs index 22723b4a6517..442fa088cdcb 100644 --- a/turbopack/crates/turbopack-cli/src/build/mod.rs +++ b/turbopack/crates/turbopack-cli/src/build/mod.rs @@ -14,8 +14,8 @@ use turbo_tasks::{ read_strongly_consistent_and_apply_effects, take_effects, }; use turbo_tasks_backend::{ - BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend, - noop_backing_storage, turbo_backing_storage, + BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode, + TurboTasksBackend, noop_backing_storage, turbo_backing_storage, }; use turbo_tasks_fs::FileSystem; use turbo_unix_path::join_path; @@ -543,8 +543,15 @@ pub async fn build(args: &BuildArguments) -> Result<()> { .cache_dir .clone() .unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache")); - let (backing_storage, cache_state) = - turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?; + let (backing_storage, cache_state) = turbo_backing_storage( + &cache_dir, + &version_info, + BackingStorageOptions { + is_ci, + is_short_session, + skip_compaction: false, + }, + )?; let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() { StorageMode::ReadOnly } else if is_ci || is_short_session { diff --git a/turbopack/crates/turbopack-cli/src/dev/mod.rs b/turbopack/crates/turbopack-cli/src/dev/mod.rs index 28c7244e5c55..1b0a13eae4ca 100644 --- a/turbopack/crates/turbopack-cli/src/dev/mod.rs +++ b/turbopack/crates/turbopack-cli/src/dev/mod.rs @@ -18,8 +18,8 @@ use turbo_tasks::{ util::{FormatBytes, FormatDuration}, }; use turbo_tasks_backend::{ - BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend, - noop_backing_storage, turbo_backing_storage, + BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode, + TurboTasksBackend, noop_backing_storage, turbo_backing_storage, }; use turbo_tasks_fs::FileSystem; use turbo_tasks_malloc::TurboMalloc; @@ -390,8 +390,15 @@ pub async fn start_server(args: &DevArguments) -> Result<()> { .cache_dir .clone() .unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache")); - let (backing_storage, cache_state) = - turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?; + let (backing_storage, cache_state) = turbo_backing_storage( + &cache_dir, + &version_info, + BackingStorageOptions { + is_ci, + is_short_session, + skip_compaction: false, + }, + )?; let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() { StorageMode::ReadOnly } else if is_ci || is_short_session { diff --git a/turbopack/crates/turbopack-node/tests/test_config.trs b/turbopack/crates/turbopack-node/tests/test_config.trs index 6ba3540fa199..152b1b254260 100644 --- a/turbopack/crates/turbopack-node/tests/test_config.trs +++ b/turbopack/crates/turbopack-node/tests/test_config.trs @@ -20,9 +20,10 @@ describe: "test-unversioned", dirty: false, }, - false, - true, - false, + turbo_tasks_backend::BackingStorageOptions { + is_short_session: true, + ..Default::default() + }, ).unwrap().0 ) ) diff --git a/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs b/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs index a7d08d6cb0b3..ee62d2065312 100644 --- a/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs +++ b/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs @@ -331,9 +331,10 @@ fn node_file_trace_persistent(#[case] input: CaseInput) { describe: "test-unversioned", dirty: false, }, - false, - true, - false, + turbo_tasks_backend::BackingStorageOptions { + is_short_session: true, + ..Default::default() + }, ) .unwrap() .0,