[pull] canary from vercel:canary

crates/next-napi-bindings/src/next_api/project.rs

@@ -57,7 +57,7 @@ use turbo_tasks::{
     trace::TraceRawVcs,
     unmark_top_level_task_may_leak_eventually_consistent_state,
 };
-use turbo_tasks_backend::db_invalidation::invalidation_reasons;
+use turbo_tasks_backend::{BackingStorageOptions, db_invalidation::invalidation_reasons};
 #[cfg(windows)]
 use turbo_tasks_fs::windows::to_verbatim_with_case_folded_disk;
 use turbo_tasks_fs::{
@@ -576,18 +576,17 @@ pub fn project_new<'env>(
     env.spawn_future(
         async move {
             let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
-            let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
-            let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false);
-            let skip_compaction = turbo_engine_options.skip_compaction.unwrap_or(false);
             let turbopack_memory_eviction = turbo_engine_options.turbopack_memory_eviction;
             let turbo_tasks = create_turbo_tasks(
                 PathBuf::from(&options.dist_dir),
                 &options.next_version,
                 options.is_persistent_caching_enabled,
                 dependency_tracking,
-                is_ci,
-                is_short_session,
-                skip_compaction,
+                BackingStorageOptions {
+                    is_ci: turbo_engine_options.is_ci.unwrap_or(false),
+                    is_short_session: turbo_engine_options.is_short_session.unwrap_or(false),
+                    skip_compaction: turbo_engine_options.skip_compaction.unwrap_or(false),
+                },
                 turbopack_memory_eviction,
             )?;
             let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);

crates/next-napi-bindings/src/next_api/turbopack_ctx.rs

@@ -25,8 +25,9 @@ use turbo_tasks::{
     message_queue::{CompilationEvent, Severity},
 };
 use turbo_tasks_backend::{
-    BackendOptions, EvictionMode, GitVersionInfo, StartupCacheState, TurboTasksBackend,
-    db_invalidation::invalidation_reasons, noop_backing_storage, turbo_backing_storage,
+    BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, StartupCacheState,
+    TurboTasksBackend, db_invalidation::invalidation_reasons, noop_backing_storage,
+    turbo_backing_storage,
 };
 
 pub type NextTurboTasks = Arc<TurboTasks<TurboTasksBackend>>;
@@ -289,20 +290,21 @@ pub fn create_turbo_tasks(
     next_version: &str,
     persistent_caching: bool,
     dependency_tracking: bool,
-    is_ci: bool,
-    is_short_session: bool,
-    skip_compaction: bool,
+    storage_options: BackingStorageOptions,
     turbopack_memory_eviction: MemoryEvictionMode,
 ) -> Result<NextTurboTasks> {
+    let BackingStorageOptions {
+        is_ci,
+        is_short_session,
+        ..
+    } = storage_options;
     Ok(if persistent_caching {
         let describe = cache_describe(next_version);
         let version_info = git_version_info(&describe);
         let (backing_storage, cache_state) = turbo_backing_storage(
             &output_path.join("cache").join("turbopack"),
             &version_info,
-            is_ci,
-            is_short_session,
-            skip_compaction,
+            storage_options,
         )?;
         let tt = TurboTasks::new(TurboTasksBackend::new(
             BackendOptions {

turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs

@@ -17,7 +17,10 @@ use turbo_tasks::{
     turbo_tasks,
 };
 
-use crate::database::{key_value_database::KeySpace, write_batch::WriteBuffer};
+use crate::{
+    BackingStorageOptions,
+    database::{key_value_database::KeySpace, write_batch::WriteBuffer},
+};
 
 mod parallel_scheduler;
 pub(crate) use parallel_scheduler::TurboTasksParallelScheduler;
@@ -49,31 +52,22 @@ pub const COMPACT_CONFIG: CompactConfig = CompactConfig {
 
 pub struct TurboKeyValueDatabase {
     db: TurboPersistence<TurboTasksParallelScheduler, FAMILIES>,
-    is_ci: bool,
-    is_short_session: bool,
+    options: BackingStorageOptions,
     is_fresh: bool,
-    skip_compaction: bool,
 }
 
 impl TurboKeyValueDatabase {
-    pub fn new(
-        versioned_path: PathBuf,
-        is_ci: bool,
-        is_short_session: bool,
-        skip_compaction: bool,
-    ) -> Result<Self> {
+    pub fn new(versioned_path: PathBuf, options: BackingStorageOptions) -> Result<Self> {
         assert!(
-            !skip_compaction || is_short_session,
+            !options.skip_compaction || options.is_short_session,
             "skip_compaction=true requires is_short_session=true"
         );
         let db = TurboPersistence::open_with_config(versioned_path, db_config())?;
         let is_fresh = db.is_empty();
         Ok(Self {
             db,
-            is_ci,
-            is_short_session,
+            options,
             is_fresh,
-            skip_compaction,
         })
     }
 
@@ -83,10 +77,12 @@ impl TurboKeyValueDatabase {
     pub fn empty_in_memory() -> Self {
         Self {
             db: TurboPersistence::empty_in_memory_with_config(db_config()),
-            is_ci: false,
-            is_short_session: true,
+            options: BackingStorageOptions {
+                is_ci: false,
+                is_short_session: true,
+                skip_compaction: true,
+            },
             is_fresh: true,
-            skip_compaction: true,
         }
     }
 
@@ -129,7 +125,7 @@ impl TurboKeyValueDatabase {
     /// Returns `Ok(Some(stats))` with the bytes written/deleted if compaction actually merged
     /// files, `Ok(None)` if there was nothing to compact.
     pub fn compact(&self) -> Result<Option<CommitStats>> {
-        if self.is_short_session || self.db.is_empty() {
+        if self.options.is_short_session || self.db.is_empty() {
             return Ok(None);
         }
         do_compact(
@@ -148,8 +144,8 @@ impl TurboKeyValueDatabase {
     pub fn shutdown(&self) -> Result<()> {
         // Compact the database on shutdown
         // (Avoid compacting a fresh database since we don't have any usage info yet)
-        if !self.is_fresh && !self.skip_compaction {
-            if self.is_ci {
+        if !self.is_fresh && !self.options.skip_compaction {
+            if self.options.is_ci {
                 // Fully compact in CI to reduce cache size
                 do_compact(&self.db, COMPACTION_MESSAGE, usize::MAX)?;
             } else {

turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs

@@ -491,7 +491,18 @@ mod tests {
     use turbo_tasks::TaskId;
 
     use super::*;
-    use crate::database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer};
+    use crate::{
+        BackingStorageOptions,
+        database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer},
+    };
+
+    /// Options used by these tests. `is_short_session` disables background compaction, which
+    /// requires a turbo-tasks context that these tests don't set up.
+    const TEST_STORAGE_OPTIONS: BackingStorageOptions = BackingStorageOptions {
+        is_ci: false,
+        is_short_session: true,
+        skip_compaction: false,
+    };
 
     /// Helper to write to the database using the concurrent batch API.
     fn write_task_cache_entry(
@@ -533,9 +544,7 @@ mod tests {
         let tempdir = tempfile::tempdir()?;
         let path = tempdir.path();
 
-        // Use is_short_session=true to disable background compaction (which requires turbo-tasks
-        // context)
-        let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
+        let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
 
         // Simulate a hash collision by writing multiple TaskIds with the same hash key
         let collision_hash: u64 = 0xDEADBEEF;
@@ -575,7 +584,7 @@ mod tests {
 
         // Write all entries in a single batch with flush (like save_snapshot does)
         {
-            let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
+            let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
             let batch = db.write_batch()?;
 
             for (hash, task_id) in hashes.iter().zip(task_ids.iter()) {
@@ -594,7 +603,7 @@ mod tests {
 
         // Reopen and verify all entries are readable
         {
-            let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
+            let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
             let mut found = 0;
             let mut missing = 0;
             for (hash, expected_id) in hashes.iter().zip(task_ids.iter()) {
@@ -632,7 +641,14 @@ mod tests {
         let survivor_id = TaskId::try_from(222u32).unwrap();
         let deleted_key = (*deleted_id).to_le_bytes();
 
-        let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
+        let db = TurboKeyValueDatabase::new(
+            path.to_path_buf(),
+            BackingStorageOptions {
+                is_ci: false,
+                is_short_session: true,
+                skip_compaction: false,
+            },
+        )?;
 
         // Both ids collide in one TaskCache bucket, purely on disk; the deleted task also has
         // meta and data entries.

turbopack/crates/turbo-tasks-backend/src/lib.rs

@@ -25,19 +25,33 @@ pub use crate::{
     kv_backing_storage::TurboBackingStorage,
 };
 
+/// Options controlling how the on-disk persistent cache database is opened and compacted.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct BackingStorageOptions {
+    /// Whether the process is running in a CI environment. Enables more aggressive (full)
+    /// compaction on shutdown to reduce the size of the cache that gets uploaded.
+    pub is_ci: bool,
+    /// Whether this is a short-lived session (e.g. a single build). Disables background
+    /// persistence during the session
+    pub is_short_session: bool,
+    /// Whether to skip database compaction on shutdown entirely
+    pub skip_compaction: bool,
+}
+
 /// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`].
 ///
 /// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
 pub fn turbo_backing_storage(
     base_path: &Path,
     version_info: &GitVersionInfo,
-    is_ci: bool,
-    is_short_session: bool,
-    skip_compaction: bool,
+    options: BackingStorageOptions,
 ) -> Result<(TurboBackingStorage, StartupCacheState)> {
-    TurboBackingStorage::open_versioned_on_disk(base_path.to_owned(), version_info, is_ci, |path| {
-        TurboKeyValueDatabase::new(path, is_ci, is_short_session, skip_compaction)
-    })
+    TurboBackingStorage::open_versioned_on_disk(
+        base_path.to_owned(),
+        version_info,
+        options.is_ci,
+        |path| TurboKeyValueDatabase::new(path, options),
+    )
 }
 
 /// Creates an in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. Backed by

turbopack/crates/turbo-tasks-backend/tests/eviction.rs

@@ -11,7 +11,9 @@ use anyhow::Result;
 use turbo_tasks::{
     ResolvedVc, State, TurboTasks, Vc, unmark_top_level_task_may_leak_eventually_consistent_state,
 };
-use turbo_tasks_backend::{BackendOptions, EvictionMode, GitVersionInfo, TurboTasksBackend};
+use turbo_tasks_backend::{
+    BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, TurboTasksBackend,
+};
 
 /// Creates a fresh per-call persistence directory in the OS temp dir, with the test `name` as a
 /// prefix so a leaked directory from a failed run is identifiable. The unique suffix from
@@ -48,9 +50,11 @@ fn create_tt_with_workers(
                 describe: "test-unversioned",
                 dirty: false,
             },
-            false,
-            true,
-            true,
+            BackingStorageOptions {
+                is_short_session: true,
+                skip_compaction: true,
+                ..Default::default()
+            },
         )
         .unwrap()
         .0,

turbopack/crates/turbo-tasks-backend/tests/test_config.trs

@@ -20,9 +20,10 @@
           describe: "test-unversioned",
           dirty: false,
         },
-        false,
-        true,
-        false,
+        turbo_tasks_backend::BackingStorageOptions {
+          is_short_session: true,
+          ..Default::default()
+        },
       ).unwrap().0
     )
   )

turbopack/crates/turbo-tasks-fetch/tests/test_config.trs

@@ -19,9 +19,10 @@
           describe: "test-unversioned",
           dirty: false,
         },
-        false,
-        true,
-        false,
+        turbo_tasks_backend::BackingStorageOptions {
+          is_short_session: true,
+          ..Default::default()
+        },
       ).unwrap().0
     )
   )

turbopack/crates/turbopack-analyze/tests/test_config.trs

@@ -19,9 +19,10 @@
           describe: "test-unversioned",
           dirty: false,
         },
-        false,
-        true,
-        false,
+        turbo_tasks_backend::BackingStorageOptions {
+          is_short_session: true,
+          ..Default::default()
+        },
       ).unwrap().0
     )
   )

turbopack/crates/turbopack-cli/src/build/mod.rs

@@ -14,8 +14,8 @@ use turbo_tasks::{
     read_strongly_consistent_and_apply_effects, take_effects,
 };
 use turbo_tasks_backend::{
-    BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend,
-    noop_backing_storage, turbo_backing_storage,
+    BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode,
+    TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
 };
 use turbo_tasks_fs::FileSystem;
 use turbo_unix_path::join_path;
@@ -543,8 +543,15 @@ pub async fn build(args: &BuildArguments) -> Result<()> {
             .cache_dir
             .clone()
             .unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache"));
-        let (backing_storage, cache_state) =
-            turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?;
+        let (backing_storage, cache_state) = turbo_backing_storage(
+            &cache_dir,
+            &version_info,
+            BackingStorageOptions {
+                is_ci,
+                is_short_session,
+                skip_compaction: false,
+            },
+        )?;
         let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
             StorageMode::ReadOnly
         } else if is_ci || is_short_session {

turbopack/crates/turbopack-cli/src/dev/mod.rs

@@ -18,8 +18,8 @@ use turbo_tasks::{
     util::{FormatBytes, FormatDuration},
 };
 use turbo_tasks_backend::{
-    BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend,
-    noop_backing_storage, turbo_backing_storage,
+    BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode,
+    TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
 };
 use turbo_tasks_fs::FileSystem;
 use turbo_tasks_malloc::TurboMalloc;
@@ -390,8 +390,15 @@ pub async fn start_server(args: &DevArguments) -> Result<()> {
             .cache_dir
             .clone()
             .unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache"));
-        let (backing_storage, cache_state) =
-            turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?;
+        let (backing_storage, cache_state) = turbo_backing_storage(
+            &cache_dir,
+            &version_info,
+            BackingStorageOptions {
+                is_ci,
+                is_short_session,
+                skip_compaction: false,
+            },
+        )?;
         let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
             StorageMode::ReadOnly
         } else if is_ci || is_short_session {