Load cuQuantum libraries with RTLD_LOCAL and explain fork-after-CUDA-init failures

crates/pecos-cuquantum-sys/src/loader.rs

@@ -50,8 +50,9 @@ macro_rules! load_sym {
 #[allow(non_snake_case)]
 pub struct CuQuantumBackend {
     // Keep libraries alive.
-    // Rust drops fields in declaration order, so dependents must come before
-    // their dependencies. cuda_rt is last because everything depends on it.
+    // Rust drops fields in declaration order, so cuDensityMat and cuTensorNet must
+    // come before their cuTensor dependency. cuda_rt remains last so its direct
+    // CUDA runtime symbols stay loaded until the other library handles are released.
     _cudensitymat: Library,
     _cutensornet: Library,
     _cutensor: Option<Library>,
@@ -178,11 +179,17 @@ unsafe impl Sync for CuQuantumBackend {}
 static BACKEND: OnceLock<Result<CuQuantumBackend, CuQuantumLoadError>> = OnceLock::new();
 
 /// Load cuQuantum libraries. Thread-safe, loads only once.
+///
+/// Availability probing must only open libraries; it must never create CUDA handles or contexts,
+/// because initializing the driver in a parent process poisons every later forked child.
 pub fn try_load() -> Result<&'static CuQuantumBackend, &'static CuQuantumLoadError> {
     BACKEND.get_or_init(load_all).as_ref()
 }
 
 /// Check if cuQuantum is available at runtime.
+///
+/// This probe only opens libraries and must not create CUDA handles, contexts, or otherwise
+/// initialize the driver before a caller may fork worker processes.
 pub fn is_available() -> bool {
     try_load().is_ok()
 }
@@ -269,20 +276,23 @@ fn cutensor_search_paths() -> Vec<PathBuf> {
     paths
 }
 
-/// Load a shared library with RTLD_GLOBAL so its symbols are visible to subsequent loads.
-/// This is necessary because cuQuantum libs have transitive dependencies (e.g.
-/// libcutensornet depends on libcutensor) that need to resolve via the global symbol table.
+/// Load a shared library without adding its symbols to the process-global lookup scope.
+///
+/// cuTensor is preloaded by full path before libraries with a `NEEDED` entry for its SONAME. ELF
+/// loaders reuse that already-loaded object when resolving the entry, so global symbol visibility
+/// is unnecessary. CUDA libraries export unmangled C symbols; keeping them local prevents this
+/// loader from polluting symbol resolution for other CUDA-using libraries in the same process.
 #[cfg(unix)]
-fn load_global<P: AsRef<std::ffi::OsStr>>(path: P) -> Result<Library, libloading::Error> {
+fn load_local<P: AsRef<std::ffi::OsStr>>(path: P) -> Result<Library, libloading::Error> {
     // RTLD_NOW: resolve all symbols immediately
-    // RTLD_GLOBAL: make symbols available for subsequent dlopen calls
-    let flags = libc::RTLD_NOW | libc::RTLD_GLOBAL;
+    // RTLD_LOCAL: keep exported symbols out of the process-global lookup scope
+    let flags = libc::RTLD_NOW | libc::RTLD_LOCAL;
     let lib = unsafe { UnixLibrary::open(Some(path.as_ref()), flags) }?;
     Ok(lib.into())
 }
 
 #[cfg(not(unix))]
-fn load_global<P: AsRef<std::ffi::OsStr>>(path: P) -> Result<Library, libloading::Error> {
+fn load_local<P: AsRef<std::ffi::OsStr>>(path: P) -> Result<Library, libloading::Error> {
     unsafe { Library::new(path.as_ref()) }
 }
 
@@ -296,7 +306,7 @@ fn try_load_lib(names: &[&str], search_dirs: &[PathBuf]) -> LoadResult<Library>
         for dir in search_dirs {
             let path = dir.join(name);
             log::debug!("Trying to load {name} from: {}", path.display());
-            match load_global(&path) {
+            match load_local(&path) {
                 Ok(lib) => {
                     log::info!("Loaded {name} from: {}", path.display());
                     return Ok(lib);
@@ -306,7 +316,7 @@ fn try_load_lib(names: &[&str], search_dirs: &[PathBuf]) -> LoadResult<Library>
         }
         // Fall back to bare name (system linker search)
         log::debug!("Trying system path for {name}");
-        if let Ok(lib) = load_global(*name) {
+        if let Ok(lib) = load_local(*name) {
             log::info!("Loaded {name} from system path");
             return Ok(lib);
         }
@@ -328,16 +338,17 @@ fn load_all() -> Result<CuQuantumBackend, CuQuantumLoadError> {
     let cq_paths = cuquantum_search_paths();
     let ct_paths = cutensor_search_paths();
 
-    // Load CUDA runtime first (transitive dependency for everything else).
+    // Load CUDA runtime for the API symbols used directly by this backend. As dependency
+    // preloading, this is only a best-effort compatibility measure for variants that dynamically
+    // depend on libcudart; the shipped cuQuantum 25.11 libraries link the runtime statically.
     // Try versioned soname first -- unversioned symlink may not exist on runtime-only installs.
     let cuda_rt = try_load_lib(
         &["libcudart.so.13", "libcudart.so.12", "libcudart.so"],
         &cuda_paths,
     )?;
 
-    // Load cuTensor before cuTensorNet (transitive dependency).
-    // The handle must stay alive in CuQuantumBackend so dlclose doesn't
-    // unload the library while cuTensorNet still needs its symbols.
+    // Load cuTensor before cuTensorNet and cuDensityMat, whose dynamic `NEEDED` entries resolve
+    // its SONAME to this object. The handle must stay alive so it is not unloaded first.
     let cutensor = try_load_lib(&["libcutensor.so.2", "libcutensor.so"], &ct_paths).ok();
 
     let custatevec = try_load_lib(&["libcustatevec.so.1", "libcustatevec.so"], &cq_paths)?;

crates/pecos-cuquantum/src/error.rs

@@ -52,7 +52,11 @@ pub enum CuQuantumError {
 /// cuStateVec-specific error
 #[derive(Error, Debug, Clone, Copy, PartialEq, Eq)]
 pub enum StateVecError {
-    #[error("Not initialized")]
+    #[error(
+        "Not initialized. CUDA could not initialize in this process; a common cause is running \
+         in a forked child of a process that already initialized CUDA (CUDA contexts do not \
+         survive fork) — use the multiprocessing \"spawn\" start method."
+    )]
     NotInitialized,
 
     #[error("Allocation failed")]
@@ -142,7 +146,11 @@ impl From<custatevecStatus_t> for CuQuantumError {
 /// cuStabilizer-specific error
 #[derive(Error, Debug, Clone, Copy, PartialEq, Eq)]
 pub enum StabilizerError {
-    #[error("Not initialized")]
+    #[error(
+        "Not initialized. CUDA could not initialize in this process; a common cause is running \
+         in a forked child of a process that already initialized CUDA (CUDA contexts do not \
+         survive fork) — use the multiprocessing \"spawn\" start method."
+    )]
     NotInitialized,
 
     #[error("Allocation failed")]
@@ -201,7 +209,11 @@ impl From<custabilizerStatus_t> for CuQuantumError {
 /// cuTensorNet-specific error
 #[derive(Error, Debug, Clone, Copy, PartialEq, Eq)]
 pub enum TensorNetError {
-    #[error("Not initialized")]
+    #[error(
+        "Not initialized. CUDA could not initialize in this process; a common cause is running \
+         in a forked child of a process that already initialized CUDA (CUDA contexts do not \
+         survive fork) — use the multiprocessing \"spawn\" start method."
+    )]
     NotInitialized,
 
     #[error("Allocation failed")]
@@ -289,7 +301,11 @@ impl From<cutensornetStatus_t> for CuQuantumError {
 /// cuDensityMat-specific error
 #[derive(Error, Debug, Clone, Copy, PartialEq, Eq)]
 pub enum DensityMatError {
-    #[error("Not initialized")]
+    #[error(
+        "Not initialized. CUDA could not initialize in this process; a common cause is running \
+         in a forked child of a process that already initialized CUDA (CUDA contexts do not \
+         survive fork) — use the multiprocessing \"spawn\" start method."
+    )]
     NotInitialized,
 
     #[error("Allocation failed")]
@@ -461,6 +477,36 @@ mod tests {
         assert!(msg.contains("Invalid value"));
     }
 
+    #[test]
+    fn test_cuquantum_not_initialized_display_has_fork_hint_only_for_that_status() {
+        let not_initialized_messages = [
+            CuQuantumError::from(custatevecStatus_t::CUSTATEVEC_STATUS_NOT_INITIALIZED).to_string(),
+            CuQuantumError::from(custabilizerStatus_t::CUSTABILIZER_STATUS_NOT_INITIALIZED)
+                .to_string(),
+            CuQuantumError::from(cutensornetStatus_t::CUTENSORNET_STATUS_NOT_INITIALIZED)
+                .to_string(),
+            CuQuantumError::from(cudensitymatStatus_t::CUDENSITYMAT_STATUS_NOT_INITIALIZED)
+                .to_string(),
+        ];
+        for message in not_initialized_messages {
+            assert!(message.contains("forked child"));
+            assert!(message.contains("\"spawn\" start method"));
+        }
+
+        let other_messages = [
+            CuQuantumError::from(custatevecStatus_t::CUSTATEVEC_STATUS_INVALID_VALUE).to_string(),
+            CuQuantumError::from(custabilizerStatus_t::CUSTABILIZER_STATUS_INVALID_VALUE)
+                .to_string(),
+            CuQuantumError::from(cutensornetStatus_t::CUTENSORNET_STATUS_INVALID_VALUE).to_string(),
+            CuQuantumError::from(cudensitymatStatus_t::CUDENSITYMAT_STATUS_INVALID_VALUE)
+                .to_string(),
+        ];
+        for message in other_messages {
+            assert!(!message.contains("forked child"));
+            assert!(!message.contains("\"spawn\" start method"));
+        }
+    }
+
     #[test]
     fn test_stabilizer_status_conversion() {
         let err = StabilizerError::from(custabilizerStatus_t::CUSTABILIZER_STATUS_INVALID_VALUE);

docs/user-guide/cuda-setup.md

@@ -27,6 +27,29 @@ Both approaches require:
 - CUDA Toolkit (system-level installation)
 - cuQuantum SDK (for Rust bindings) or Python packages (for Python bindings)
 
+## CUDA Simulators and Multiprocessing
+
+When a parent process has initialized CUDA, its children must use the multiprocessing
+`spawn` start method rather than `fork`. A forked child cannot use the parent's CUDA
+context because CUDA contexts do not survive fork. Depending on the backend, this
+fails with `cudaErrorInitializationError`, `CUDA_ERROR_NOT_INITIALIZED`, or a
+cuQuantum `Not initialized` error from a cuStateVec, cuStabilizer, cuTensorNet, or
+cuDensityMat handle.
+
+PECOS's built-in multiprocessing engine already uses `spawn`. Host programs that
+embed PECOS in their own process pools must also select `spawn` if the parent may
+initialize CUDA before creating workers. The Rust cuQuantum bindings load their CUDA
+libraries with local symbol visibility so they do not disturb other CUDA-using
+libraries in the same process. Availability checks only load those libraries; they
+do not create CUDA handles or contexts or initialize the CUDA driver.
+
+PECOS records when its CUDA simulator wrappers begin CUDA initialization. If one of
+those simulators is later constructed in a child forked from that process, PECOS
+fails fast with the spawn guidance above. PECOS cannot detect CUDA initialization by
+another library in the parent; those cases still surface the guided error from the
+CUDA or cuQuantum layer. When a marked parent forks, PECOS also emits a one-time
+`RuntimeWarning` with the same guidance before the fork.
+
 ## System Requirements
 
 ### Hardware Requirements

python/quantum-pecos/src/pecos/simulators/_cuda_fork_guard.py

@@ -0,0 +1,78 @@
+# Copyright 2026 The PECOS Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+
+"""Detect processes forked after PECOS initialized CUDA.
+
+The Python CUDA simulator wrappers mark their first CUDA call and use this guard
+to fail before touching an inherited, unusable CUDA context. Direct users of the
+raw ``pecos_rslib_cuda`` extension deliberately bypass this guard; its guided
+``Not initialized`` errors remain their backstop.
+
+Registering fork hooks does not load CUDA libraries, create handles, or initialize
+the CUDA driver.
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+
+__all__ = ["CUDA_FORK_ERROR_MESSAGE", "check_fork_poison", "mark_cuda_initialized"]
+
+CUDA_FORK_ERROR_MESSAGE = (
+    "CUDA could not initialize in this process; a common cause is running in a forked child of a process that "
+    'already initialized CUDA (CUDA contexts do not survive fork) — use the multiprocessing "spawn" start method.'
+)
+
+_state = {
+    "cuda_initialized": False,
+    "forked_after_cuda_init": False,
+    "fork_warning_emitted": False,
+}
+
+
+def _warn_before_fork() -> None:
+    """Warn once when this process forks after a PECOS CUDA call."""
+    if _state["cuda_initialized"] and not _state["fork_warning_emitted"]:
+        _state["fork_warning_emitted"] = True
+        warnings.warn(
+            "This process is forking after CUDA initialization, so CUDA will be unusable in the forked child. "
+            + CUDA_FORK_ERROR_MESSAGE,
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+
+def _mark_forked_child() -> None:
+    """Record that the child inherited evidence of parent CUDA initialization."""
+    if _state["cuda_initialized"]:
+        _state["forked_after_cuda_init"] = True
+
+
+def mark_cuda_initialized() -> None:
+    """Record that this process is about to make a real CUDA call.
+
+    The mark deliberately latches if that call fails because partial driver
+    initialization still makes later forked children unsafe.
+    """
+    _state["cuda_initialized"] = True
+
+
+def check_fork_poison() -> None:
+    """Fail before CUDA use in a child forked after parent CUDA initialization."""
+    if _state["forked_after_cuda_init"]:
+        raise RuntimeError(CUDA_FORK_ERROR_MESSAGE)
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(before=_warn_before_fork)
+    os.register_at_fork(after_in_child=_mark_forked_child)

python/quantum-pecos/src/pecos/simulators/cuda_stabilizer/state.py

@@ -21,6 +21,7 @@
 
 from pecos_rslib_cuda import CuStabilizer as CuStabilizerRs
 
+from pecos.simulators._cuda_fork_guard import check_fork_poison, mark_cuda_initialized
 from pecos.simulators.cuda_stabilizer import bindings
 from pecos.simulators.sim_class_types import Stabilizer
 
@@ -68,6 +69,7 @@ def __init__(self, num_qubits: int, seed: int | None = None) -> None:
             num_qubits: Number of qubits to simulate.
             seed: Optional random seed for reproducibility.
         """
+        check_fork_poison()
         if not isinstance(num_qubits, int):
             msg = "``num_qubits`` should be of type ``int``."
             raise TypeError(msg)
@@ -78,13 +80,15 @@ def __init__(self, num_qubits: int, seed: int | None = None) -> None:
         self.num_qubits = num_qubits
 
         # Create the Rust backend
+        mark_cuda_initialized()
         if seed is not None:
             self.backend = CuStabilizerRs.with_seed(num_qubits, seed)
         else:
             self.backend = CuStabilizerRs(num_qubits)
 
     def reset(self) -> Self:
         """Reset the quantum state to |0...0>."""
+        check_fork_poison()
         self.backend.reset()
         return self
 

python/quantum-pecos/src/pecos/simulators/cuda_statevec/state.py

@@ -21,6 +21,7 @@
 
 from pecos_rslib_cuda import CuStateVec as CuStateVecRs
 
+from pecos.simulators._cuda_fork_guard import check_fork_poison, mark_cuda_initialized
 from pecos.simulators.cuda_statevec import bindings
 from pecos.simulators.sim_class_types import StateVector
 
@@ -63,6 +64,7 @@ def __init__(self, num_qubits: int, seed: int | None = None) -> None:
             num_qubits: Number of qubits to simulate.
             seed: Optional random seed for reproducibility.
         """
+        check_fork_poison()
         if not isinstance(num_qubits, int):
             msg = "``num_qubits`` should be of type ``int``."
             raise TypeError(msg)
@@ -73,13 +75,15 @@ def __init__(self, num_qubits: int, seed: int | None = None) -> None:
         self.num_qubits = num_qubits
 
         # Create the Rust backend
+        mark_cuda_initialized()
         if seed is not None:
             self.backend = CuStateVecRs.with_seed(num_qubits, seed)
         else:
             self.backend = CuStateVecRs(num_qubits)
 
     def reset(self) -> Self:
         """Reset the quantum state to |0...0>."""
+        check_fork_poison()
         self.backend.reset()
         return self