From e7b754a2535103b7735faede2ed5cc149346d067 Mon Sep 17 00:00:00 2001 From: madkoding Date: Sun, 8 Mar 2026 13:16:29 -0300 Subject: [PATCH 1/3] test: fix all unit tests and clippy warnings - Add tempfile dependency for tests - Fix type annotations in storage and upload tests - Fix test_fractal_strategy_creation to properly initialize SurrealDB client - Fix test_complete_upload_flow to respect chunk size constraints - Resolve all clippy warnings: - Use .div_ceil() instead of manual division - Use is_none_or instead of map_or - Remove unnecessary format! calls - Fix field_reassign_with_default violations - Collapse nested if statements - Remove needless lifetimes - Use as_deref instead of as_ref().map() - Use .ends_with() instead of chars().last() - Apply cargo fmt to all files All 255 tests passing, clippy --lib clean, fmt check clean --- Cargo.toml | 3 +- examples/llm_usage.rs | 16 +- src/api/error.rs | 4 +- src/api/handlers.rs | 504 +++++++++++++-------- src/api/progress.rs | 37 +- src/api/routes.rs | 25 +- src/api/types.rs | 21 +- src/cache/config.rs | 2 +- src/cache/embedding_cache.rs | 18 +- src/cache/lru_cache.rs | 7 +- src/cache/mod.rs | 10 +- src/cache/node_cache.rs | 16 +- src/db/connection.rs | 20 +- src/db/mod.rs | 3 +- src/db/queries.rs | 76 ++-- src/db/schema.rs | 2 +- src/embeddings/config.rs | 12 +- src/embeddings/fastembed_provider.rs | 28 +- src/embeddings/mock_provider.rs | 29 +- src/embeddings/mod.rs | 8 +- src/embeddings/provider.rs | 8 +- src/embeddings/service.rs | 9 +- src/graph/raptor.rs | 42 +- src/graph/similarity.rs | 8 +- src/graph/sssp.rs | 25 +- src/lib.rs | 14 +- src/main.rs | 45 +- src/models/edge.rs | 2 +- src/models/embedding.rs | 30 +- src/models/llm/brain.rs | 126 ++++-- src/models/llm/config.rs | 2 +- src/models/llm/fractal_model.rs | 26 +- src/models/llm/gguf_parser.rs | 51 ++- src/models/llm/mod.rs | 8 +- src/models/llm/providers/anthropic.rs | 4 +- src/models/llm/providers/mod.rs | 6 +- src/models/llm/providers/ollama.rs | 66 ++- src/models/llm/strategy.rs | 215 ++++----- src/models/llm/traits_llm.rs | 2 +- src/models/mod.rs | 10 +- src/models/namespace.rs | 17 +- src/models/node.rs | 8 +- src/models/upload_session.rs | 45 +- src/services/config.rs | 5 +- src/services/fractal_builder.rs | 79 ++-- src/services/ingestion/chunker.rs | 25 +- src/services/ingestion/config.rs | 24 +- src/services/ingestion/extractors/image.rs | 19 +- src/services/ingestion/extractors/pdf.rs | 5 +- src/services/ingestion/extractors/text.rs | 16 +- src/services/ingestion/service.rs | 25 +- src/services/mod.rs | 13 +- src/services/model_conversion.rs | 255 +++++++---- src/services/rem_phase.rs | 75 ++- src/services/rem_scheduler.rs | 83 ++-- src/services/storage/mod.rs | 205 +++++---- src/services/upload/mod.rs | 319 +++++++------ src/services/web_search.rs | 11 +- tests/integration_handlers.rs | 104 +++-- tests/integration_ingest_handler.rs | 40 +- tests/integration_ingestion_service.rs | 36 +- 61 files changed, 1651 insertions(+), 1298 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6c30156..889f272 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ lru = "0.12" lazy_static = "1.5" # HTTP Client (para búsqueda web en fase REM) -reqwest = { version = "0.12", features = ["json", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] } # URL parsing y encoding url = "2.5" @@ -88,6 +88,7 @@ half = "2.7.1" [dev-dependencies] mockall = "0.13" criterion = "0.5" +tempfile = "3.10" [features] default = [] diff --git a/examples/llm_usage.rs b/examples/llm_usage.rs index 7dd6e69..1acbd4c 100644 --- a/examples/llm_usage.rs +++ b/examples/llm_usage.rs @@ -4,7 +4,6 @@ /// 1. Generar embeddings /// 2. Hacer consultas de chat /// 3. Resumir textos (fase REM) - use fractalmind::models::llm::{BrainConfig, ModelBrain}; #[tokio::main] @@ -47,7 +46,10 @@ async fn example_local_config() -> anyhow::Result<()> { let brain = ModelBrain::new(config).await?; let info = brain.get_models_info(); - println!("Embedding Model: {} ({}D)", info.embedding_model, info.embedding_dimension); + println!( + "Embedding Model: {} ({}D)", + info.embedding_model, info.embedding_dimension + ); println!("Chat Model: {}", info.chat_model); println!("Summarizer Model: {}", info.summarizer_model); println!("Fully Local: {}", brain.is_fully_local()); @@ -62,8 +64,7 @@ async fn example_hybrid_config() -> anyhow::Result<()> { println!("2. Configuración Híbrida (Ollama + OpenAI)"); println!("===========================================\n"); - let api_key = std::env::var("OPENAI_API_KEY") - .expect("OPENAI_API_KEY no configurado"); + let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY no configurado"); let config = BrainConfig::hybrid(api_key); let brain = ModelBrain::new(config).await?; @@ -126,12 +127,11 @@ async fn example_chat_with_context() -> anyhow::Result<()> { let config = BrainConfig::default_local(); let brain = ModelBrain::new(config).await?; - let system_prompt = "Eres un experto en sistemas de memoria artificial y grafos de conocimiento."; + let system_prompt = + "Eres un experto en sistemas de memoria artificial y grafos de conocimiento."; let user_question = "Explica cómo funciona RAPTOR en 2 frases."; - let response = brain - .chat_with_system(system_prompt, user_question) - .await?; + let response = brain.chat_with_system(system_prompt, user_question).await?; println!("Sistema: {}", system_prompt); println!("Usuario: {}", user_question); diff --git a/src/api/error.rs b/src/api/error.rs index 9e8398e..c68d67d 100644 --- a/src/api/error.rs +++ b/src/api/error.rs @@ -56,7 +56,9 @@ impl IntoResponse for ApiError { ApiError::EmbeddingError(_) => (StatusCode::INTERNAL_SERVER_ERROR, "EMBEDDING_ERROR"), ApiError::LlmError(_) => (StatusCode::INTERNAL_SERVER_ERROR, "LLM_ERROR"), ApiError::ValidationError(_) => (StatusCode::BAD_REQUEST, "VALIDATION_ERROR"), - ApiError::ServiceUnavailable(_) => (StatusCode::SERVICE_UNAVAILABLE, "SERVICE_UNAVAILABLE"), + ApiError::ServiceUnavailable(_) => { + (StatusCode::SERVICE_UNAVAILABLE, "SERVICE_UNAVAILABLE") + } }; let body = ErrorResponse { diff --git a/src/api/handlers.rs b/src/api/handlers.rs index 1067053..5654a53 100644 --- a/src/api/handlers.rs +++ b/src/api/handlers.rs @@ -5,23 +5,26 @@ use std::sync::Arc; use std::time::Instant; -use axum::{extract::{State, Multipart}, Json}; -use serde::{Deserialize}; +use axum::{ + extract::{Multipart, State}, + Json, +}; +use serde::Deserialize; use serde_json; use tokio::sync::RwLock; use tracing::{debug, error, info, warn}; use uuid::Uuid; -use crate::services::ingestion::extractors::{ExtractorFactory, ContentExtractor}; -use crate::services::ingestion::config::{FileType, IngestionConfig}; +use crate::db::queries::{EdgeRepository, NodeRepository}; use crate::services::ingestion::chunker::TextChunker; +use crate::services::ingestion::config::{FileType, IngestionConfig}; +use crate::services::ingestion::extractors::{ContentExtractor, ExtractorFactory}; use crate::services::FractalBuilder; -use crate::db::queries::{NodeRepository, EdgeRepository}; use crate::cache::{EmbeddingCache, NodeCache}; use crate::db::connection::DatabaseConnection; use crate::models::llm::ModelBrain; -use crate::models::{EmbeddingVector, FractalNode, FractalEdge, NodeMetadata}; +use crate::models::{EmbeddingVector, FractalEdge, FractalNode, NodeMetadata, NodeStatus}; use surrealdb::sql::Thing; use super::error::{ApiError, ApiResult}; @@ -42,10 +45,10 @@ pub struct AppState { /// Embedding cache pub embedding_cache: EmbeddingCache, - + /// Progress tracker for long-running operations pub progress_tracker: ProgressTracker, - + /// Upload session manager for chunked model uploads pub upload_manager: Arc, } @@ -105,11 +108,15 @@ pub async fn ingest( // Validate request if request.content.trim().is_empty() { - return Err(ApiError::ValidationError("Content cannot be empty".to_string())); + return Err(ApiError::ValidationError( + "Content cannot be empty".to_string(), + )); } let state = state.read().await; - let namespace = request.namespace.unwrap_or_else(|| "global_knowledge".to_string()); + let namespace = request + .namespace + .unwrap_or_else(|| "global_knowledge".to_string()); debug!("Ingesting content into namespace: {}", namespace); @@ -144,7 +151,9 @@ pub async fn ingest( embedding_response.embedding.clone(), crate::models::EmbeddingModel::NomicEmbedTextV15, ); - state.embedding_cache.put(&request.content, embedding_vector.clone()); + state + .embedding_cache + .put(&request.content, embedding_vector.clone()); // Create metadata let mut metadata = NodeMetadata::default(); @@ -182,13 +191,19 @@ pub async fn ingest( .with_summaries(false) .with_min_nodes(3); let fractal_builder = FractalBuilder::new(&state.db, config); - let fractal_msg = match fractal_builder.build_for_namespace(&namespace_clone, Some(&state.brain)).await { + let fractal_msg = match fractal_builder + .build_for_namespace(&namespace_clone, Some(&state.brain)) + .await + { Ok(result) if result.parent_nodes_created > 0 => { info!( "Fractal structure updated: {} parent nodes, {} edges", result.parent_nodes_created, result.edges_created ); - format!(" + fractal updated ({} parents)", result.parent_nodes_created) + format!( + " + fractal updated ({} parents)", + result.parent_nodes_created + ) } Ok(_) => String::new(), Err(e) => { @@ -237,15 +252,16 @@ pub async fn ingest_file( .map_err(|e| ApiError::BadRequest(format!("Failed to read file bytes: {}", e)))? .to_vec(); if data.is_empty() { - return Err(ApiError::ValidationError("Uploaded file is empty".to_string())); + return Err(ApiError::ValidationError( + "Uploaded file is empty".to_string(), + )); } file_bytes = Some(data); } Some("namespace") => { - let txt = field - .text() - .await - .map_err(|e| ApiError::BadRequest(format!("Failed to read namespace: {}", e)))?; + let txt = field.text().await.map_err(|e| { + ApiError::BadRequest(format!("Failed to read namespace: {}", e)) + })?; if !txt.trim().is_empty() { namespace = Some(txt); } @@ -278,8 +294,9 @@ pub async fn ingest_file( } } - let file_bytes = file_bytes - .ok_or_else(|| ApiError::ValidationError("Missing 'file' field in multipart".to_string()))?; + let file_bytes = file_bytes.ok_or_else(|| { + ApiError::ValidationError("Missing 'file' field in multipart".to_string()) + })?; // Size check let file_size = file_bytes.len(); @@ -302,7 +319,9 @@ pub async fn ingest_file( }; if !file_type.is_supported() { - return Err(ApiError::ValidationError("Unsupported or unknown file type".to_string())); + return Err(ApiError::ValidationError( + "Unsupported or unknown file type".to_string(), + )); } // Choose extractor @@ -310,14 +329,19 @@ pub async fn ingest_file( FileType::Text => ExtractorFactory::text(), FileType::Pdf => { if !config.enable_pdf { - return Err(ApiError::ValidationError("PDF ingestion is disabled".to_string())); + return Err(ApiError::ValidationError( + "PDF ingestion is disabled".to_string(), + )); } - ExtractorFactory::create(FileType::Pdf) - .ok_or_else(|| ApiError::ValidationError("PDF extractor not available".to_string()))? + ExtractorFactory::create(FileType::Pdf).ok_or_else(|| { + ApiError::ValidationError("PDF extractor not available".to_string()) + })? } FileType::Image => { if !config.enable_ocr { - return Err(ApiError::ValidationError("Image OCR ingestion is disabled".to_string())); + return Err(ApiError::ValidationError( + "Image OCR ingestion is disabled".to_string(), + )); } // Image extractor is feature-gated #[cfg(feature = "ocr")] @@ -331,7 +355,11 @@ pub async fn ingest_file( )); } } - _ => return Err(ApiError::ValidationError("Unsupported file type".to_string())), + _ => { + return Err(ApiError::ValidationError( + "Unsupported file type".to_string(), + )) + } }; // Run extraction @@ -341,7 +369,9 @@ pub async fn ingest_file( .map_err(|e| ApiError::InternalError(format!("Extraction failed: {}", e)))?; if !extraction.is_successful() { - return Err(ApiError::ValidationError("Could not extract text from file".to_string())); + return Err(ApiError::ValidationError( + "Could not extract text from file".to_string(), + )); } // Use provided namespace or default @@ -350,7 +380,7 @@ pub async fn ingest_file( // Chunk the extracted text to avoid exceeding embedding model context limits let chunker = TextChunker::from_config(&config); let chunking_result = chunker.chunk(&extraction.text); - + info!( "Chunked document into {} chunks (original: {} chars, avg chunk: {} chars)", chunking_result.count(), @@ -365,13 +395,17 @@ pub async fn ingest_file( .map(|v| v == "true") .unwrap_or(false); - let skip_db = std::env::var("TEST_SKIP_DB_WRITES").map(|v| v == "true").unwrap_or(false); + let skip_db = std::env::var("TEST_SKIP_DB_WRITES") + .map(|v| v == "true") + .unwrap_or(false); let node_repo = NodeRepository::new(&state.db); // Process each chunk let mut created_node_ids: Vec = Vec::new(); let mut embedding_dimension: Option = None; - let source_filename = filename.clone().unwrap_or_else(|| "uploaded_file".to_string()); + let source_filename = filename + .clone() + .unwrap_or_else(|| "uploaded_file".to_string()); for chunk in chunking_result.chunks { // Generate embedding for this chunk @@ -387,16 +421,14 @@ pub async fn ingest_file( latency_ms: 0, } } else { - state - .brain - .embed(&chunk.content) - .await - .map_err(|e| ApiError::EmbeddingError(format!( + state.brain.embed(&chunk.content).await.map_err(|e| { + ApiError::EmbeddingError(format!( "Embedding failed for chunk {}/{}: {}", chunk.index + 1, chunk.total, e - )))? + )) + })? }; embedding_dimension = Some(embedding_response.dimension); @@ -406,7 +438,9 @@ pub async fn ingest_file( embedding_response.embedding.clone(), crate::models::EmbeddingModel::NomicEmbedTextV15, ); - state.embedding_cache.put(&chunk.content, embedding_vector.clone()); + state + .embedding_cache + .put(&chunk.content, embedding_vector.clone()); // Prepare metadata for this chunk let mut metadata = NodeMetadata::default(); @@ -427,7 +461,9 @@ pub async fn ingest_file( metadata.tags = t.clone(); } // Add chunk info to metadata tags - metadata.tags.push(format!("chunk:{}/{}", chunk.index + 1, chunk.total)); + metadata + .tags + .push(format!("chunk:{}/{}", chunk.index + 1, chunk.total)); // Create node for this chunk let node = FractalNode::new_leaf( @@ -442,15 +478,14 @@ pub async fn ingest_file( let node_id = if skip_db { Uuid::new_v4().to_string() } else { - let created = node_repo - .create(&node) - .await - .map_err(|e| ApiError::DatabaseError(format!( + let created = node_repo.create(&node).await.map_err(|e| { + ApiError::DatabaseError(format!( "Failed to create node for chunk {}/{}: {}", chunk.index + 1, chunk.total, e - )))?; + )) + })?; created.to_string() }; @@ -467,12 +502,18 @@ pub async fn ingest_file( // Auto-build fractal structure after ingestion let fractal_result = if !skip_db && total_chunks > 0 { - info!("Auto-building fractal structure for namespace '{}'", namespace); + info!( + "Auto-building fractal structure for namespace '{}'", + namespace + ); let config = crate::services::FractalBuilderConfig::new() .with_summaries(false) .with_min_nodes(3); let fractal_builder = FractalBuilder::new(&state.db, config); - match fractal_builder.build_for_namespace(&namespace, Some(&state.brain)).await { + match fractal_builder + .build_for_namespace(&namespace, Some(&state.brain)) + .await + { Ok(result) => { info!( "Fractal structure built: {} parent nodes, {} edges", @@ -517,10 +558,12 @@ pub async fn remember( Json(request): Json, ) -> ApiResult> { if request.content.trim().is_empty() { - return Err(ApiError::ValidationError("Content cannot be empty".to_string())); + return Err(ApiError::ValidationError( + "Content cannot be empty".to_string(), + )); } - let start = Instant::now(); + let _start = Instant::now(); let state = state.read().await; // Determine namespace (personal or context-based) @@ -548,22 +591,26 @@ pub async fn remember( embedding_response.embedding.clone(), crate::models::EmbeddingModel::NomicEmbedTextV15, ); - + // Cache the embedding state.embedding_cache.put(&request.content, vector.clone()); vector }; // Create metadata for episodic memory - let mut metadata = NodeMetadata::default(); - metadata.source = "episodic_memory".to_string(); - metadata.source_type = crate::models::SourceType::Text; - metadata.language = request.language.clone().unwrap_or_else(|| "en".to_string()); - metadata.tags = vec!["episodic".to_string(), "memory".to_string()]; - - if let Some(context) = &request.context { - metadata.tags.push(format!("context:{}", context)); - } + let metadata = NodeMetadata { + source: "episodic_memory".to_string(), + source_type: crate::models::SourceType::Text, + language: request.language.clone().unwrap_or_else(|| "en".to_string()), + tags: { + let mut tags = vec!["episodic".to_string(), "memory".to_string()]; + if let Some(context) = &request.context { + tags.push(format!("context:{}", context)); + } + tags + }, + ..NodeMetadata::default() + }; // Create episodic memory node let node = FractalNode::new_leaf( @@ -590,7 +637,7 @@ pub async fn remember( let mut edges_count = 0; for related_id in related_node_ids { - if let Ok(related_thing) = parse_thing_from_string(related_id) { + if let Some(related_thing) = parse_thing_from_string(related_id) { // Validate that the related node exists before creating edge match node_repo.get_by_id(&related_thing).await { Ok(Some(_related_node)) => { @@ -653,7 +700,9 @@ pub async fn ask( let start = Instant::now(); if request.question.trim().is_empty() { - return Err(ApiError::ValidationError("Question cannot be empty".to_string())); + return Err(ApiError::ValidationError( + "Question cannot be empty".to_string(), + )); } let state = state.read().await; @@ -688,7 +737,14 @@ pub async fn ask( let (filtered_results, used_sssp) = if has_fractal && search_results.len() > 1 { // Use SSSP to navigate the fractal graph for better context - navigate_with_sssp(search_results, threshold, max_results, &node_repo, &edge_repo).await + navigate_with_sssp( + search_results, + threshold, + max_results, + &node_repo, + &edge_repo, + ) + .await } else { // Simple vector similarity filtering let results: Vec = search_results @@ -790,7 +846,11 @@ pub async fn ask( if context.is_empty() { Some("No relevant information found in the knowledge base.".to_string()) } else { - Some(format!("Found {} relevant sources:\n\n{}", filtered_results.len(), context)) + Some(format!( + "Found {} relevant sources:\n\n{}", + filtered_results.len(), + context + )) } }; @@ -833,7 +893,6 @@ pub async fn sync_rem( ); let node_repo = NodeRepository::new(&state.db); - let mut nodes_processed = 0; let mut nodes_created = 0; let clusters_formed; @@ -849,20 +908,23 @@ pub async fn sync_rem( .take(max_nodes) .collect(); - nodes_processed = leaf_nodes.len(); + let nodes_processed = leaf_nodes.len(); info!("Found {} leaf nodes to process", nodes_processed); // 2. Build fractal hierarchy using RAPTOR if we have enough nodes if enable_clustering && leaf_nodes.len() >= 3 { info!("Building fractal hierarchy with RAPTOR clustering..."); - + let config = crate::services::FractalBuilderConfig::new() - .with_summaries(true) // Enable LLM summaries for parent nodes + .with_summaries(true) // Enable LLM summaries for parent nodes .with_min_nodes(3); - + let fractal_builder = crate::services::FractalBuilder::new(&state.db, config); - - match fractal_builder.build_for_namespace(namespace, Some(&state.brain)).await { + + match fractal_builder + .build_for_namespace(namespace, Some(&state.brain)) + .await + { Ok(result) => { nodes_created = result.parent_nodes_created; clusters_formed = result.edges_created; @@ -910,14 +972,17 @@ pub async fn memory_update( Json(request): Json, ) -> ApiResult> { if request.node_id.trim().is_empty() { - return Err(ApiError::ValidationError("Node ID cannot be empty".to_string())); + return Err(ApiError::ValidationError( + "Node ID cannot be empty".to_string(), + )); } let state = state.read().await; let node_repo = NodeRepository::new(&state.db); - - let thing = parse_thing_from_string(&request.node_id) - .ok_or_else(|| ApiError::ValidationError(format!("Invalid node ID format: {}", request.node_id)))?; + + let thing = parse_thing_from_string(&request.node_id).ok_or_else(|| { + ApiError::ValidationError(format!("Invalid node ID format: {}", request.node_id)) + })?; let mut node = node_repo .get_by_id(&thing) @@ -929,30 +994,37 @@ pub async fn memory_update( if let Some(content) = &request.content { node.content = content.clone(); - - if let Some(embedding_text) = request.regenerate_embedding { - let embedding_response = state - .brain - .embed(&node.content) - .await - .map_err(|e| ApiError::EmbeddingError(format!("Failed to regenerate embedding: {}", e)))?; + + if let Some(_embedding_text) = request.regenerate_embedding { + let embedding_response = state.brain.embed(&node.content).await.map_err(|e| { + ApiError::EmbeddingError(format!("Failed to regenerate embedding: {}", e)) + })?; node.embedding = EmbeddingVector::new( embedding_response.embedding.clone(), crate::models::EmbeddingModel::NomicEmbedTextV15, ); - - state.embedding_cache.put(&node.content, node.embedding.clone()); + + state + .embedding_cache + .put(&node.content, node.embedding.clone()); } - + updated_fields.push("content".to_string()); if request.regenerate_embedding == Some(true) { updated_fields.push("embedding".to_string()); } } - if let Some(status) = &request.status { - node.status = status.clone(); + if let Some(status_str) = &request.status { + let status = match status_str.as_str() { + "complete" => NodeStatus::Complete, + "incomplete" => NodeStatus::Incomplete, + "pending" => NodeStatus::Pending, + "deprecated" => NodeStatus::Deprecated, + _ => NodeStatus::Complete, + }; + node.status = status; updated_fields.push("status".to_string()); } @@ -966,11 +1038,6 @@ pub async fn memory_update( updated_fields.push("source".to_string()); } - if request.deprecated == Some(true) { - node.metadata.deprecated = true; - updated_fields.push("deprecated".to_string()); - } - if let Some(metadata) = &request.metadata { if let Some(lang) = &metadata.language { node.metadata.language = lang.clone(); @@ -992,11 +1059,12 @@ pub async fn memory_update( request.node_id, updated_fields ); + let fields_count = updated_fields.len(); Ok(Json(MemoryUpdateResponse { success: true, node_id: request.node_id.clone(), updated_fields, - message: format!("Memory node updated successfully ({} fields)", updated_fields.len()), + message: format!("Memory node updated successfully ({} fields)", fields_count), })) } @@ -1012,7 +1080,9 @@ pub async fn search( let start = Instant::now(); if request.query.trim().is_empty() { - return Err(ApiError::ValidationError("Query cannot be empty".to_string())); + return Err(ApiError::ValidationError( + "Query cannot be empty".to_string(), + )); } let state = state.read().await; @@ -1094,7 +1164,9 @@ async fn check_fractal_structure(db: &DatabaseConnection) -> bool { match db.query(query).await { Ok(mut result) => { #[derive(serde::Deserialize)] - struct CountResult { cnt: i64 } + struct CountResult { + cnt: i64, + } let counts: Vec = result.take(0).unwrap_or_default(); counts.first().map(|c| c.cnt > 0).unwrap_or(false) } @@ -1110,7 +1182,7 @@ async fn navigate_with_sssp( node_repo: &NodeRepository<'_>, edge_repo: &EdgeRepository<'_>, ) -> (Vec, bool) { - use crate::graph::{Sssp, GraphNode}; + use crate::graph::{GraphNode, Sssp}; use std::collections::{HashMap, HashSet}; // Filter by threshold first @@ -1131,8 +1203,10 @@ async fn navigate_with_sssp( // Add initial nodes to graph for (node, similarity) in &filtered { let node_id = node.id.as_ref().map(|t| t.to_string()).unwrap_or_default(); - if node_id.is_empty() { continue; } - + if node_id.is_empty() { + continue; + } + let graph_node = GraphNode::new(node_id.clone(), node.namespace.clone()); graph.insert(node_id.clone(), graph_node); node_map.insert(node_id.clone(), (node.clone(), *similarity)); @@ -1149,7 +1223,8 @@ async fn navigate_with_sssp( if !explored_nodes.contains(&parent_id) { // Fetch parent node if let Ok(Some(parent)) = node_repo.get_by_id(&edge.from).await { - let graph_node = GraphNode::new(parent_id.clone(), parent.namespace.clone()); + let graph_node = + GraphNode::new(parent_id.clone(), parent.namespace.clone()); graph.insert(parent_id.clone(), graph_node); // Calculate similarity for parent based on edge weight let parent_similarity = edge.similarity * 0.9; // Slight penalty for indirect @@ -1181,9 +1256,10 @@ async fn navigate_with_sssp( // If we have enough structure, use SSSP to rank if graph.len() > 1 { let sssp = Sssp::with_defaults(); - + // Find the best starting node (highest similarity leaf) - let best_start = filtered.iter() + let best_start = filtered + .iter() .filter(|(n, _)| n.node_type == crate::models::NodeType::Leaf) .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) .map(|(n, _)| n.id.as_ref().map(|t| t.to_string()).unwrap_or_default()); @@ -1191,28 +1267,30 @@ async fn navigate_with_sssp( if let Some(start_id) = best_start { if !start_id.is_empty() { let sssp_result = sssp.compute(&graph, &start_id, None); - + // Combine vector similarity with graph distance for ranking let mut ranked: Vec<(String, f32, Option>)> = node_map .iter() .map(|(id, (_, sim))| { - let graph_score = sssp_result.distances.get(id) + let graph_score = sssp_result + .distances + .get(id) .map(|&d| 1.0 / (1.0 + d)) // Convert distance to score .unwrap_or(0.0); - + // Combined score: 70% vector similarity + 30% graph proximity let combined = sim * 0.7 + graph_score * 0.3; - + // Get path if available let path = sssp_result.reconstruct_path(&start_id, id).map(|p| p.nodes); - + (id.clone(), combined, path) }) .collect(); - + // Sort by combined score ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - + // Convert to results let results: Vec = ranked .into_iter() @@ -1233,7 +1311,7 @@ async fn navigate_with_sssp( }) }) .collect(); - + return (results, true); } } @@ -1266,12 +1344,18 @@ pub fn parse_thing_from_string(id: &str) -> Option { if id.contains(':') { let parts: Vec<&str> = id.split(':').collect(); if parts.len() == 2 { - Some(surrealdb::sql::Thing::from((parts[0].to_string(), parts[1].to_string()))) + Some(surrealdb::sql::Thing::from(( + parts[0].to_string(), + parts[1].to_string(), + ))) } else { None } } else { - Some(surrealdb::sql::Thing::from(("nodes".to_string(), id.to_string()))) + Some(surrealdb::sql::Thing::from(( + "nodes".to_string(), + id.to_string(), + ))) } } @@ -1284,8 +1368,8 @@ pub async fn build_fractal( State(state): State, Json(request): Json, ) -> ApiResult> { - use crate::services::{FractalBuilder, FractalBuilderConfig}; use crate::graph::RaptorConfig; + use crate::services::{FractalBuilder, FractalBuilderConfig}; let start = Instant::now(); let state = state.read().await; @@ -1326,17 +1410,13 @@ pub async fn build_fractal( latency_ms: start.elapsed().as_millis() as u64, message: format!( "Fractal structure built: {} parent nodes, {} edges, max depth {}", - result.parent_nodes_created, - result.edges_created, - result.max_depth + result.parent_nodes_created, result.edges_created, result.max_depth ), }; info!( "Fractal build completed in {}ms: {} parent nodes, {} edges", - response.latency_ms, - response.parent_nodes_created, - response.edges_created + response.latency_ms, response.parent_nodes_created, response.edges_created ); Ok(Json(response)) @@ -1352,14 +1432,14 @@ pub async fn stats(State(state): State) -> Json { let cache_metrics = state.node_cache.metrics(); let llm_info = state.brain.get_models_info(); - + let node_repo = NodeRepository::new(&state.db); let edge_repo = EdgeRepository::new(&state.db); - + let total_nodes = node_repo.count_all().await.unwrap_or(0) as usize; let total_edges = edge_repo.count_all_edges().await.unwrap_or(0) as usize; let namespaces_info = node_repo.get_namespaces().await.unwrap_or_default(); - + let namespaces: Vec = namespaces_info .into_iter() .map(|ns| NamespaceStats { @@ -1434,7 +1514,10 @@ mod tests { assert_eq!(request.status, Some("complete".to_string())); assert!(request.regenerate_embedding == Some(true)); assert!(request.metadata.is_some()); - assert_eq!(request.metadata.as_ref().unwrap().language, Some("es".to_string())); + assert_eq!( + request.metadata.as_ref().unwrap().language, + Some("es".to_string()) + ); } #[test] @@ -1489,8 +1572,8 @@ mod tests { // Model Upload Handlers // ============================================================================ -use axum::extract::Path; use axum::body::Bytes; +use axum::extract::Path; /// Initialize a chunked upload session pub async fn init_model_upload( @@ -1503,13 +1586,13 @@ pub async fn init_model_upload( "File must be a .gguf model file".to_string(), )); } - + if request.total_size == 0 { return Err(ApiError::ValidationError( "File size cannot be 0".to_string(), )); } - + // Max 500GB const MAX_SIZE: u64 = 500 * 1024 * 1024 * 1024; if request.total_size > MAX_SIZE { @@ -1518,25 +1601,25 @@ pub async fn init_model_upload( request.total_size, MAX_SIZE ))); } - + // Validate chunk size (10MB - 500MB) const MIN_CHUNK: u64 = 10 * 1024 * 1024; const MAX_CHUNK: u64 = 500 * 1024 * 1024; let chunk_size = request.chunk_size.clamp(MIN_CHUNK, MAX_CHUNK); - + let state_read = state.read().await; let manager = &state_read.upload_manager; - + let session = manager .init_upload(request.filename, request.total_size, Some(chunk_size)) .await .map_err(|e| ApiError::InternalError(format!("Failed to init upload: {}", e)))?; - + info!( "Initialized model upload: id={}, chunks={}", session.upload_id, session.total_chunks ); - + Ok(Json(InitUploadResponse { upload_id: session.upload_id, chunk_size: session.chunk_size, @@ -1560,7 +1643,7 @@ pub async fn upload_model_chunk( ) -> ApiResult> { let state_read = state.read().await; let manager = &state_read.upload_manager; - + let result = manager .upload_chunk( &upload_id, @@ -1570,12 +1653,12 @@ pub async fn upload_model_chunk( ) .await .map_err(|e| ApiError::BadRequest(format!("Chunk upload failed: {}", e)))?; - + debug!( "Received chunk {} for upload {} ({}/{})", params.chunk_index, upload_id, result.chunks_received, result.total_chunks ); - + Ok(Json(UploadChunkResponse { success: result.success, chunk_index: result.chunk_index, @@ -1591,71 +1674,84 @@ pub async fn finalize_model_upload( ) -> ApiResult> { let state_read = state.read().await; let manager = &state_read.upload_manager; - + // Get session info before finalizing let session = manager .get_status(&upload_id) .await .ok_or_else(|| ApiError::NotFound(format!("Upload session not found: {}", upload_id)))?; - + let filename = session.filename.clone(); let total_size = session.total_size; - + // Finalize the upload (move file to final location) let result = manager .finalize(&upload_id) .await .map_err(|e| ApiError::BadRequest(format!("Finalize failed: {}", e)))?; - + // Create the FractalModel record in the database let model = crate::models::llm::fractal_model::FractalModel::new( filename.clone(), result.file_path.clone(), total_size, ); - + let repo = crate::db::queries::FractalModelRepository::new(&state_read.db); let model_id = repo .create(&model) .await .map_err(|e| ApiError::InternalError(format!("Failed to create model record: {}", e)))?; - + info!( "Finalized model upload: upload_id={}, model_id={}, file={}, size={}", upload_id, model_id, result.file_path, total_size ); - + // Auto-start conversion in background let db_clone = state_read.db.clone(); let model_id_clone = model_id.clone(); let file_path = result.file_path.clone(); let upload_manager_clone = state_read.upload_manager.clone(); let upload_id_clone = upload_id.clone(); - + tokio::spawn(async move { info!("Auto-starting conversion for model: {}", model_id_clone); - + // Update status to converting let repo = crate::db::queries::FractalModelRepository::new(&db_clone); - if let Err(e) = repo.update_status(&model_id_clone, crate::models::llm::fractal_model::FractalModelStatus::Converting).await { + if let Err(e) = repo + .update_status( + &model_id_clone, + crate::models::llm::fractal_model::FractalModelStatus::Converting, + ) + .await + { error!("Failed to update model status: {}", e); return; } - + // Run conversion if let Err(e) = run_model_conversion(&db_clone, &model_id_clone, &file_path).await { error!("Model conversion failed for {}: {}", model_id_clone, e); - let _ = repo.update_status(&model_id_clone, crate::models::llm::fractal_model::FractalModelStatus::Failed).await; - let _ = upload_manager_clone.mark_failed(&upload_id_clone, &e.to_string()).await; + let _ = repo + .update_status( + &model_id_clone, + crate::models::llm::fractal_model::FractalModelStatus::Failed, + ) + .await; + let _ = upload_manager_clone + .mark_failed(&upload_id_clone, &e.to_string()) + .await; return; } - + // Mark upload session as ready so frontend stops polling if let Err(e) = upload_manager_clone.mark_ready(&upload_id_clone).await { error!("Failed to mark upload as ready: {}", e); } }); - + Ok(Json(FinalizeUploadResponse { success: result.success, model_id, @@ -1670,12 +1766,12 @@ pub async fn get_upload_status( ) -> ApiResult> { let state_read = state.read().await; let manager = &state_read.upload_manager; - + let session = manager .get_status(&upload_id) .await .ok_or_else(|| ApiError::NotFound(format!("Upload session not found: {}", upload_id)))?; - + Ok(Json(ProgressResponse { upload_progress: session.upload_progress, conversion_progress: session.conversion_progress, @@ -1694,14 +1790,14 @@ pub async fn cancel_model_upload( ) -> ApiResult> { let state_read = state.read().await; let manager = &state_read.upload_manager; - + manager .cancel(&upload_id) .await .map_err(|e| ApiError::InternalError(format!("Cancel failed: {}", e)))?; - + info!("Cancelled model upload: {}", upload_id); - + Ok(Json(CancelUploadResponse { success: true, message: "Upload cancelled and temporary files cleaned up".to_string(), @@ -1715,18 +1811,18 @@ pub async fn upload_progress_stream( ) -> impl axum::response::IntoResponse { use axum::response::sse::{Event, KeepAlive, Sse}; use std::convert::Infallible; - + // Clone the Arc to move into the async stream let manager = state.read().await.upload_manager.clone(); - + let stream = async_stream::stream! { let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(500)); - + loop { interval.tick().await; - + let session = manager.get_status(&upload_id).await; - + match session { Some(s) => { let progress = ProgressResponse { @@ -1738,13 +1834,13 @@ pub async fn upload_progress_stream( total_chunks: Some(s.total_chunks), current_phase: s.current_phase.clone(), }; - + let json = serde_json::to_string(&progress).unwrap_or_default(); yield Ok::<_, Infallible>(Event::default().data(json)); - + // Stop when ready or failed - if s.status == crate::models::upload_session::UploadStatus::Ready - || s.status == crate::models::upload_session::UploadStatus::Failed + if s.status == crate::models::upload_session::UploadStatus::Ready + || s.status == crate::models::upload_session::UploadStatus::Failed { break; } @@ -1760,7 +1856,7 @@ pub async fn upload_progress_stream( } } }; - + Sse::new(stream).keep_alive(KeepAlive::default()) } @@ -1773,30 +1869,30 @@ use crate::models::llm::fractal_model::FractalModelStatus; /// List available Ollama models pub async fn list_ollama_models() -> ApiResult> { - let ollama_base_url = std::env::var("OLLAMA_BASE_URL") - .unwrap_or_else(|_| "http://localhost:11434".to_string()); - + let ollama_base_url = + std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string()); + let client = reqwest::Client::new(); let url = format!("{}/api/tags", ollama_base_url); - + let response = client .get(&url) .send() .await .map_err(|e| ApiError::InternalError(format!("Failed to connect to Ollama: {}", e)))?; - + if !response.status().is_success() { return Err(ApiError::InternalError(format!( "Ollama returned error: {}", response.status() ))); } - + #[derive(Deserialize)] struct OllamaTagsResponse { models: Vec, } - + #[derive(Deserialize)] struct OllamaTagModel { name: String, @@ -1806,7 +1902,7 @@ pub async fn list_ollama_models() -> ApiResult> { digest: String, details: Option, } - + #[derive(Deserialize)] struct OllamaTagDetails { parent_model: Option, @@ -1816,12 +1912,12 @@ pub async fn list_ollama_models() -> ApiResult> { parameter_size: Option, quantization_level: Option, } - + let ollama_response: OllamaTagsResponse = response .json() .await .map_err(|e| ApiError::InternalError(format!("Failed to parse Ollama response: {}", e)))?; - + let models: Vec = ollama_response .models .into_iter() @@ -1841,22 +1937,20 @@ pub async fn list_ollama_models() -> ApiResult> { }), }) .collect(); - + Ok(Json(ListOllamaModelsResponse { models })) } /// List all fractal models -pub async fn list_models( - State(state): State, -) -> ApiResult> { +pub async fn list_models(State(state): State) -> ApiResult> { let state_read = state.read().await; let repo = FractalModelRepository::new(&state_read.db); - + let models = repo .list_all() .await .map_err(|e| ApiError::InternalError(format!("Failed to list models: {}", e)))?; - + let model_infos: Vec = models .into_iter() .map(|m| ModelInfo { @@ -1874,7 +1968,7 @@ pub async fn list_models( created_at: m.created_at.to_rfc3339(), }) .collect(); - + Ok(Json(ListModelsResponse { models: model_infos, })) @@ -1887,13 +1981,13 @@ pub async fn get_model( ) -> ApiResult> { let state_read = state.read().await; let repo = FractalModelRepository::new(&state_read.db); - + let model = repo .get_by_id(&model_id) .await .map_err(|e| ApiError::InternalError(format!("Failed to get model: {}", e)))? .ok_or_else(|| ApiError::NotFound(format!("Model not found: {}", model_id)))?; - + Ok(Json(GetModelResponse { model: ModelInfo { id: model.id, @@ -1919,26 +2013,26 @@ pub async fn delete_model( ) -> ApiResult> { let state_read = state.read().await; let repo = FractalModelRepository::new(&state_read.db); - + // Get the model first to get the file path let model = repo .get_by_id(&model_id) .await .map_err(|e| ApiError::InternalError(format!("Failed to get model: {}", e)))? .ok_or_else(|| ApiError::NotFound(format!("Model not found: {}", model_id)))?; - + // Delete from database (also deletes associated nodes) repo.delete(&model_id) .await .map_err(|e| ApiError::InternalError(format!("Failed to delete model: {}", e)))?; - + // Try to delete the file (don't fail if file doesn't exist) if let Err(e) = tokio::fs::remove_file(&model.file_path).await { warn!("Failed to delete model file {}: {}", model.file_path, e); } - + info!("Deleted model: {} ({})", model_id, model.name); - + Ok(Json(DeleteModelResponse { success: true, message: format!("Model {} deleted successfully", model_id), @@ -1952,48 +2046,60 @@ pub async fn convert_model( ) -> ApiResult> { let state_read = state.read().await; let repo = FractalModelRepository::new(&state_read.db); - + // Get the model let model = repo .get_by_id(&model_id) .await .map_err(|e| ApiError::InternalError(format!("Failed to get model: {}", e)))? .ok_or_else(|| ApiError::NotFound(format!("Model not found: {}", model_id)))?; - + // Check if already converting or ready match model.status { FractalModelStatus::Converting => { - return Err(ApiError::BadRequest("Model is already being converted".to_string())); + return Err(ApiError::BadRequest( + "Model is already being converted".to_string(), + )); } FractalModelStatus::Ready => { - return Err(ApiError::BadRequest("Model is already converted and ready".to_string())); + return Err(ApiError::BadRequest( + "Model is already converted and ready".to_string(), + )); } _ => {} } - + // Update status to converting repo.update_status(&model_id, FractalModelStatus::Converting) .await .map_err(|e| ApiError::InternalError(format!("Failed to update model status: {}", e)))?; - + // Spawn async conversion task let db_clone = state_read.db.clone(); let model_id_clone = model_id.clone(); let file_path = model.file_path.clone(); - + tokio::spawn(async move { if let Err(e) = run_model_conversion(&db_clone, &model_id_clone, &file_path).await { error!("Model conversion failed for {}: {}", model_id_clone, e); let repo = FractalModelRepository::new(&db_clone); - let _ = repo.update_status(&model_id_clone, FractalModelStatus::Failed).await; + let _ = repo + .update_status(&model_id_clone, FractalModelStatus::Failed) + .await; } }); - - info!("Started conversion for model: {} ({})", model_id, model.name); - + + info!( + "Started conversion for model: {} ({})", + model_id, model.name + ); + Ok(Json(ConvertModelResponse { success: true, - message: format!("Conversion started for model {}. Monitor progress via /v1/models/{}/status", model_id, model_id), + message: format!( + "Conversion started for model {}. Monitor progress via /v1/models/{}/status", + model_id, model_id + ), })) } @@ -2010,16 +2116,16 @@ async fn run_model_conversion( _file_path: &str, // No longer needed, service gets it from model ) -> Result<(), anyhow::Error> { let repo = FractalModelRepository::new(db); - + // Get the model from database let mut model = repo .get_by_id(model_id) .await? .ok_or_else(|| anyhow::anyhow!("Model not found: {}", model_id))?; - + // Create conversion service and run real conversion let conversion_service = ModelConversionService::new(std::sync::Arc::new(db.clone())); conversion_service.convert_model(&mut model).await?; - + Ok(()) } diff --git a/src/api/progress.rs b/src/api/progress.rs index b182533..b9ba0d5 100644 --- a/src/api/progress.rs +++ b/src/api/progress.rs @@ -20,28 +20,28 @@ use uuid::Uuid; pub struct IngestionProgress { /// Unique session ID for this ingestion pub session_id: String, - + /// Total number of chunks to process pub total_chunks: usize, - + /// Current chunk being embedded pub current_chunk: usize, - + /// Number of embeddings completed pub embeddings_completed: usize, - + /// Number of nodes persisted to DB pub nodes_persisted: usize, - + /// Current stage: "extracting", "chunking", "embedding", "persisting", "complete" pub stage: String, - + /// Optional progress message pub message: Option, - + /// Whether the operation completed successfully pub success: bool, - + /// Any error message if failed pub error: Option, } @@ -110,10 +110,10 @@ pub fn create_progress_tracker() -> ProgressTracker { pub async fn register_session(tracker: &ProgressTracker) -> String { let session_id = Uuid::new_v4().to_string(); let progress = IngestionProgress::new(session_id.clone()); - + let mut map = tracker.write().await; map.insert(session_id.clone(), progress); - + debug!("Registered new ingestion session: {}", session_id); session_id } @@ -131,7 +131,10 @@ pub async fn update_progress( } /// Gets current progress for a session. -pub async fn get_progress(tracker: &ProgressTracker, session_id: &str) -> Option { +pub async fn get_progress( + tracker: &ProgressTracker, + session_id: &str, +) -> Option { let map = tracker.read().await; map.get(session_id).cloned() } @@ -139,7 +142,7 @@ pub async fn get_progress(tracker: &ProgressTracker, session_id: &str) -> Option /// Removes a completed session after a delay (for cleanup). pub async fn cleanup_session(tracker: &ProgressTracker, session_id: String, delay_secs: u64) { tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs)).await; - + let mut map = tracker.write().await; map.remove(&session_id); debug!("Cleaned up ingestion session: {}", session_id); @@ -152,16 +155,16 @@ pub async fn progress_stream( ) -> impl IntoResponse { let stream = async_stream::stream! { let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(250)); - + loop { interval.tick().await; - + // Get current progress let progress = { let map = tracker.read().await; map.get(&session_id).cloned() }; - + match progress { Some(p) => { // Serialize progress to JSON @@ -172,10 +175,10 @@ pub async fn progress_stream( continue; } }; - + // Send SSE event yield Ok::<_, Infallible>(Event::default().data(json)); - + // Stop streaming when complete or failed if p.stage == "complete" || p.error.is_some() { break; diff --git a/src/api/routes.rs b/src/api/routes.rs index 59d851f..54c063b 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -52,16 +52,31 @@ fn model_upload_routes() -> Router { // Initialize chunked upload .route("/upload/init", post(handlers::init_model_upload)) // Upload a chunk (increased body limit for 50MB+ chunks) - .route("/upload/:upload_id/chunk", post(handlers::upload_model_chunk)) + .route( + "/upload/:upload_id/chunk", + post(handlers::upload_model_chunk), + ) .layer(DefaultBodyLimit::max(100 * 1024 * 1024)) // 100MB limit for chunks // Finalize upload - .route("/upload/:upload_id/finalize", post(handlers::finalize_model_upload)) + .route( + "/upload/:upload_id/finalize", + post(handlers::finalize_model_upload), + ) // Get upload status - .route("/upload/:upload_id/status", get(handlers::get_upload_status)) + .route( + "/upload/:upload_id/status", + get(handlers::get_upload_status), + ) // Cancel upload - .route("/upload/:upload_id/cancel", post(handlers::cancel_model_upload)) + .route( + "/upload/:upload_id/cancel", + post(handlers::cancel_model_upload), + ) // Progress stream (SSE) - .route("/upload/:upload_id/progress", get(handlers::upload_progress_stream)) + .route( + "/upload/:upload_id/progress", + get(handlers::upload_progress_stream), + ) // List Ollama models .route("/ollama", get(handlers::list_ollama_models)) // List all fractal models diff --git a/src/api/types.rs b/src/api/types.rs index 0cb9597..c5c113c 100644 --- a/src/api/types.rs +++ b/src/api/types.rs @@ -301,13 +301,13 @@ pub struct SearchResponse { pub struct BuildFractalRequest { /// Namespace to build fractal for pub namespace: Option, - + /// Whether to generate summaries using LLM pub generate_summaries: Option, - + /// Minimum similarity threshold for clustering (0.0 to 1.0) pub similarity_threshold: Option, - + /// Maximum depth of the fractal tree pub max_depth: Option, } @@ -382,7 +382,10 @@ mod tests { let request: IngestRequest = serde_json::from_str(json).unwrap(); assert_eq!(request.content, "Test content"); assert_eq!(request.source, Some("test.txt".to_string())); - assert_eq!(request.tags, Some(vec!["tag1".to_string(), "tag2".to_string()])); + assert_eq!( + request.tags, + Some(vec!["tag1".to_string(), "tag2".to_string()]) + ); } #[test] @@ -470,7 +473,7 @@ pub struct DeleteModelResponse { /// Request to update model strategy #[derive(Deserialize)] pub struct UpdateStrategyRequest { - pub strategy: String, // "fractal" or "ollama" + pub strategy: String, // "fractal" or "ollama" pub model_id: Option, // Required if strategy is "fractal" } @@ -542,13 +545,13 @@ pub struct UploadChunkResponse { /// Combined progress response #[derive(Serialize)] pub struct ProgressResponse { - pub upload_progress: f32, // 0-100 - pub conversion_progress: f32, // 0-100 - pub status: String, // "uploading", "finalizing", "converting", "ready", "failed" + pub upload_progress: f32, // 0-100 + pub conversion_progress: f32, // 0-100 + pub status: String, // "uploading", "finalizing", "converting", "ready", "failed" pub upload_speed_mbps: Option, pub chunks_received: Option, pub total_chunks: Option, - pub current_phase: Option, // For conversion: "parsing", "clustering", etc. + pub current_phase: Option, // For conversion: "parsing", "clustering", etc. } /// Response from upload finalization diff --git a/src/cache/config.rs b/src/cache/config.rs index 5143a0c..0ea4ba5 100644 --- a/src/cache/config.rs +++ b/src/cache/config.rs @@ -2,8 +2,8 @@ #![allow(dead_code)] -use std::time::Duration; use serde::{Deserialize, Serialize}; +use std::time::Duration; /// Configuration for the LRU cache #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/cache/embedding_cache.rs b/src/cache/embedding_cache.rs index 012c925..28fa421 100644 --- a/src/cache/embedding_cache.rs +++ b/src/cache/embedding_cache.rs @@ -6,9 +6,9 @@ use std::sync::Arc; use std::time::Duration; use tracing::{debug, info}; -use crate::models::EmbeddingVector; use super::config::{CacheConfig, CacheMetrics}; use super::lru_cache::ThreadSafeLruCache; +use crate::models::EmbeddingVector; /// Specialized cache for embedding vectors /// @@ -22,7 +22,10 @@ pub struct EmbeddingCache { impl EmbeddingCache { /// Creates a new embedding cache with the given configuration pub fn new(config: CacheConfig) -> Self { - info!("Initializing EmbeddingCache with capacity: {}", config.capacity); + info!( + "Initializing EmbeddingCache with capacity: {}", + config.capacity + ); Self { cache: ThreadSafeLruCache::new(config), } @@ -167,10 +170,7 @@ mod tests { use crate::models::EmbeddingModel; fn create_test_embedding(value: f32) -> EmbeddingVector { - EmbeddingVector::new( - vec![value; 768], - EmbeddingModel::NomicEmbedTextV15, - ) + EmbeddingVector::new(vec![value; 768], EmbeddingModel::NomicEmbedTextV15) } #[test] @@ -212,7 +212,11 @@ mod tests { fn test_batch_put() { let cache = EmbeddingCache::with_capacity(10); - let texts = vec!["text1".to_string(), "text2".to_string(), "text3".to_string()]; + let texts = vec![ + "text1".to_string(), + "text2".to_string(), + "text3".to_string(), + ]; let embeddings: Vec = (1..=3) .map(|i| create_test_embedding(i as f32 * 0.1)) .collect(); diff --git a/src/cache/lru_cache.rs b/src/cache/lru_cache.rs index 0f92fd3..296565c 100644 --- a/src/cache/lru_cache.rs +++ b/src/cache/lru_cache.rs @@ -114,10 +114,9 @@ where let mut cache = self.cache.write().ok()?; // Check if this will cause an eviction - if cache.len() >= self.config.capacity && !cache.contains(&key) { - if self.config.track_metrics { - self.evictions.fetch_add(1, Ordering::Relaxed); - } + if cache.len() >= self.config.capacity && !cache.contains(&key) && self.config.track_metrics + { + self.evictions.fetch_add(1, Ordering::Relaxed); } cache.put(key, entry).map(|e| e.into_value()) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index e7b0507..9b04282 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -20,14 +20,16 @@ //! ``` pub mod config; +pub mod embedding_cache; pub mod entry; pub mod lru_cache; pub mod node_cache; -pub mod embedding_cache; // Re-exports pub use config::{CacheConfig, CacheMetrics}; +pub use embedding_cache::{ + new_shared_cache as new_shared_embedding_cache, EmbeddingCache, SharedEmbeddingCache, +}; pub use entry::CacheEntry; -pub use lru_cache::{ThreadSafeLruCache, EntryMetadata}; -pub use node_cache::{NodeCache, SharedNodeCache, new_shared_cache as new_shared_node_cache}; -pub use embedding_cache::{EmbeddingCache, SharedEmbeddingCache, new_shared_cache as new_shared_embedding_cache}; +pub use lru_cache::{EntryMetadata, ThreadSafeLruCache}; +pub use node_cache::{new_shared_cache as new_shared_node_cache, NodeCache, SharedNodeCache}; diff --git a/src/cache/node_cache.rs b/src/cache/node_cache.rs index 6858caa..cf7bd96 100644 --- a/src/cache/node_cache.rs +++ b/src/cache/node_cache.rs @@ -6,9 +6,9 @@ use std::sync::Arc; use surrealdb::sql::Thing; use tracing::{debug, info}; -use crate::models::FractalNode; use super::config::{CacheConfig, CacheMetrics}; use super::lru_cache::ThreadSafeLruCache; +use crate::models::FractalNode; /// Specialized cache for FractalNodes /// @@ -161,10 +161,7 @@ mod tests { use surrealdb::sql::Id; fn create_test_node(id: &str) -> FractalNode { - let embedding = EmbeddingVector::new( - vec![0.1; 768], - EmbeddingModel::NomicEmbedTextV15, - ); + let embedding = EmbeddingVector::new(vec![0.1; 768], EmbeddingModel::NomicEmbedTextV15); let mut node = FractalNode::new_leaf( format!("Content for node {}", id), @@ -207,10 +204,7 @@ mod tests { fn test_node_cache_without_id() { let cache = NodeCache::with_capacity(10); - let embedding = EmbeddingVector::new( - vec![0.1; 768], - EmbeddingModel::NomicEmbedTextV15, - ); + let embedding = EmbeddingVector::new(vec![0.1; 768], EmbeddingModel::NomicEmbedTextV15); let node = FractalNode::new_leaf( "No ID node".to_string(), @@ -229,9 +223,7 @@ mod tests { fn test_node_cache_batch() { let cache = NodeCache::with_capacity(10); - let nodes: Vec = (1..=5) - .map(|i| create_test_node(&i.to_string())) - .collect(); + let nodes: Vec = (1..=5).map(|i| create_test_node(&i.to_string())).collect(); let cached = cache.put_batch(&nodes); assert_eq!(cached, 5); diff --git a/src/db/connection.rs b/src/db/connection.rs index d83ee1c..ce9e1a7 100644 --- a/src/db/connection.rs +++ b/src/db/connection.rs @@ -1,7 +1,7 @@ +use anyhow::{Context, Result}; use surrealdb::engine::remote::http::{Client, Http}; use surrealdb::opt::auth::Root; use surrealdb::Surreal; -use anyhow::{Context, Result}; use tracing::{info, warn}; /// Cliente de base de datos SurrealDB @@ -21,16 +21,11 @@ impl DbConfig { /// Carga la configuración desde variables de entorno pub fn from_env() -> Result { Ok(Self { - url: std::env::var("SURREAL_URL") - .unwrap_or_else(|_| "ws://localhost:8000".to_string()), - username: std::env::var("SURREAL_USER") - .unwrap_or_else(|_| "root".to_string()), - password: std::env::var("SURREAL_PASS") - .unwrap_or_else(|_| "root".to_string()), - namespace: std::env::var("SURREAL_NS") - .unwrap_or_else(|_| "fractalmind".to_string()), - database: std::env::var("SURREAL_DB") - .unwrap_or_else(|_| "knowledge".to_string()), + url: std::env::var("SURREAL_URL").unwrap_or_else(|_| "ws://localhost:8000".to_string()), + username: std::env::var("SURREAL_USER").unwrap_or_else(|_| "root".to_string()), + password: std::env::var("SURREAL_PASS").unwrap_or_else(|_| "root".to_string()), + namespace: std::env::var("SURREAL_NS").unwrap_or_else(|_| "fractalmind".to_string()), + database: std::env::var("SURREAL_DB").unwrap_or_else(|_| "knowledge".to_string()), }) } } @@ -40,7 +35,8 @@ pub async fn connect_db(config: &DbConfig) -> Result { info!("Connecting to SurrealDB at {}", config.url); // Extraer host:port de la URL (remover protocolo) - let addr = config.url + let addr = config + .url .trim_start_matches("http://") .trim_start_matches("https://") .trim_start_matches("ws://") diff --git a/src/db/mod.rs b/src/db/mod.rs index caa4259..b28cf2b 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,4 +1,3 @@ pub mod connection; -pub mod schema; pub mod queries; - +pub mod schema; diff --git a/src/db/queries.rs b/src/db/queries.rs index 822a1fb..edb9686 100644 --- a/src/db/queries.rs +++ b/src/db/queries.rs @@ -1,8 +1,8 @@ #![allow(dead_code)] use super::connection::DatabaseConnection; -use crate::models::{FractalNode, FractalEdge, NodeStatus}; -use crate::models::llm::fractal_model::{FractalModel, FractalModelStatus, FractalModelNode}; +use crate::models::llm::fractal_model::{FractalModel, FractalModelNode, FractalModelStatus}; +use crate::models::{FractalEdge, FractalNode, NodeStatus}; use anyhow::{Context, Result}; use surrealdb::sql::Thing; @@ -26,12 +26,8 @@ impl<'a> NodeRepository<'a> { /// Crea un nuevo nodo pub async fn create(&self, node: &FractalNode) -> Result { - let result = self - .db - .create("nodes") - .content(node) - .await; - + let result = self.db.create("nodes").content(node).await; + match result { Ok(created) => { let nodes: Vec = created; @@ -95,7 +91,8 @@ impl<'a> NodeRepository<'a> { /// Actualiza un nodo pub async fn update(&self, id: &Thing, node: &FractalNode) -> Result<()> { - let _: Option = self.db + let _: Option = self + .db .update(id) .content(node) .await @@ -106,11 +103,7 @@ impl<'a> NodeRepository<'a> { /// Elimina un nodo pub async fn delete(&self, id: &Thing) -> Result<()> { - let _: Option = self - .db - .delete(id) - .await - .context("Failed to delete node")?; + let _: Option = self.db.delete(id).await.context("Failed to delete node")?; Ok(()) } @@ -266,17 +259,20 @@ impl<'a> NodeRepository<'a> { } let groups: Vec = result.take(0)?; - + let mut namespaces = Vec::new(); for group in groups { - let edge_count = self.count_edges_by_namespace(&group.namespace).await.unwrap_or(0); + let edge_count = self + .count_edges_by_namespace(&group.namespace) + .await + .unwrap_or(0); namespaces.push(NamespaceInfo { name: group.namespace, node_count: group.node_count, edge_count, }); } - + Ok(namespaces) } @@ -343,12 +339,8 @@ impl<'a> EdgeRepository<'a> { /// Crea una nueva arista pub async fn create(&self, edge: &FractalEdge) -> Result { - let result = self - .db - .create("edges") - .content(edge) - .await; - + let result = self.db.create("edges").content(edge).await; + match result { Ok(created) => { let edges: Vec = created; @@ -359,8 +351,12 @@ impl<'a> EdgeRepository<'a> { } Err(e) => { tracing::error!("SurrealDB edge create error: {:?}", e); - tracing::error!("Edge data: from={:?}, to={:?}, type={:?}", - edge.from, edge.to, edge.edge_type); + tracing::error!( + "Edge data: from={:?}, to={:?}, type={:?}", + edge.from, + edge.to, + edge.edge_type + ); Err(anyhow::anyhow!("Failed to create edge: {}", e)) } } @@ -396,11 +392,7 @@ impl<'a> EdgeRepository<'a> { /// Elimina una arista pub async fn delete(&self, id: &Thing) -> Result<()> { - let _: Option = self - .db - .delete(id) - .await - .context("Failed to delete edge")?; + let _: Option = self.db.delete(id).await.context("Failed to delete edge")?; Ok(()) } @@ -460,8 +452,11 @@ impl<'a> FractalModelRepository<'a> { /// Crea un nuevo modelo fractal pub async fn create(&self, model: &FractalModel) -> Result { // Extract just the ID part without the table prefix - let id_part = model.id.strip_prefix("fractal_models:").unwrap_or(&model.id); - + let id_part = model + .id + .strip_prefix("fractal_models:") + .unwrap_or(&model.id); + let query = r#" CREATE type::thing("fractal_models", $id) SET name = $name, @@ -511,7 +506,7 @@ impl<'a> FractalModelRepository<'a> { pub async fn get_by_id(&self, id: &str) -> Result> { // Extract just the ID part without the table prefix let id_part = id.strip_prefix("fractal_models:").unwrap_or(id); - + let query = "SELECT * FROM type::thing(\"fractal_models\", $id)"; let mut result = self .db @@ -666,10 +661,10 @@ impl<'a> FractalModelRepository<'a> { pub async fn delete(&self, id: &str) -> Result> { // First get the model to return it let model = self.get_by_id(id).await?; - + // Extract just the ID part without the table prefix let id_part = id.strip_prefix("fractal_models:").unwrap_or(id); - + // Delete from database using type::thing let query = "DELETE type::thing(\"fractal_models\", $id)"; self.db @@ -742,12 +737,12 @@ impl<'a> FractalModelNodeRepository<'a> { /// Crea múltiples nodos en batch pub async fn create_batch(&self, nodes: &[FractalModelNode]) -> Result> { let mut ids = Vec::with_capacity(nodes.len()); - + for node in nodes { let id = self.create(node).await?; ids.push(id); } - + Ok(ids) } @@ -901,13 +896,10 @@ impl<'a> FractalModelNodeRepository<'a> { #[cfg(test)] mod tests { use super::*; - use crate::models::{NodeMetadata, EmbeddingVector, EmbeddingModel}; + use crate::models::{EmbeddingModel, EmbeddingVector, NodeMetadata}; fn create_test_node() -> FractalNode { - let embedding = EmbeddingVector::new( - vec![0.1; 768], - EmbeddingModel::NomicEmbedTextV15, - ); + let embedding = EmbeddingVector::new(vec![0.1; 768], EmbeddingModel::NomicEmbedTextV15); FractalNode::new_leaf( "Test content".to_string(), diff --git a/src/db/schema.rs b/src/db/schema.rs index e948162..45b6846 100644 --- a/src/db/schema.rs +++ b/src/db/schema.rs @@ -17,7 +17,7 @@ pub async fn initialize_schema(db: &DatabaseConnection) -> Result<()> { // Definir namespaces y scopes define_namespaces(db).await?; - + // Definir tablas para modelos fractales define_fractal_models_tables(db).await?; diff --git a/src/embeddings/config.rs b/src/embeddings/config.rs index 122b5eb..9f183c4 100644 --- a/src/embeddings/config.rs +++ b/src/embeddings/config.rs @@ -2,8 +2,8 @@ #![allow(dead_code)] -use serde::{Deserialize, Serialize}; use crate::models::EmbeddingModel; +use serde::{Deserialize, Serialize}; /// Configuration for the embedding service #[derive(Debug, Clone, Serialize, Deserialize)] @@ -106,11 +106,11 @@ impl EmbeddingConfig { .map(|d| match d.as_str() { "cuda" => EmbeddingDevice::Cuda, "cpu" => EmbeddingDevice::Cpu, - d if d.starts_with("cuda:") => { - d[5..].parse().ok() - .map(EmbeddingDevice::CudaDevice) - .unwrap_or(EmbeddingDevice::Cuda) - } + d if d.starts_with("cuda:") => d[5..] + .parse() + .ok() + .map(EmbeddingDevice::CudaDevice) + .unwrap_or(EmbeddingDevice::Cuda), _ => EmbeddingDevice::Cpu, }) .unwrap_or(EmbeddingDevice::Cpu); diff --git a/src/embeddings/fastembed_provider.rs b/src/embeddings/fastembed_provider.rs index d999593..67cfc7e 100644 --- a/src/embeddings/fastembed_provider.rs +++ b/src/embeddings/fastembed_provider.rs @@ -12,9 +12,9 @@ use std::time::Instant; use tokio::sync::RwLock; use tracing::{debug, info}; -use crate::models::{EmbeddingModel, EmbeddingVector}; use super::config::EmbeddingConfig; use super::provider::{BatchEmbeddingResult, EmbeddingProvider, EmbeddingResult}; +use crate::models::{EmbeddingModel, EmbeddingVector}; /// FastEmbed-based embedding provider pub struct FastEmbedProvider { @@ -31,7 +31,10 @@ pub struct FastEmbedProvider { impl FastEmbedProvider { /// Creates a new FastEmbed provider with the given configuration pub fn new(config: EmbeddingConfig) -> Result { - info!("Initializing FastEmbed provider with model: {:?}", config.model); + info!( + "Initializing FastEmbed provider with model: {:?}", + config.model + ); let fastembed_model = Self::map_model(&config.model)?; let dimension = config.model.dimension(); @@ -42,10 +45,13 @@ impl FastEmbedProvider { init_options = init_options.with_cache_dir(cache_dir.into()); } - let model = TextEmbedding::try_new(init_options) - .context("Failed to initialize FastEmbed model")?; + let model = + TextEmbedding::try_new(init_options).context("Failed to initialize FastEmbed model")?; - info!("FastEmbed provider initialized successfully (dimension: {})", dimension); + info!( + "FastEmbed provider initialized successfully (dimension: {})", + dimension + ); Ok(Self { model: Arc::new(RwLock::new(model)), @@ -129,8 +135,11 @@ impl EmbeddingProvider for FastEmbedProvider { let normalize = self.config.normalize; let batch_size = self.config.batch_size; - debug!("Generating batch embeddings for {} texts (batch_size: {})", - texts.len(), batch_size); + debug!( + "Generating batch embeddings for {} texts (batch_size: {})", + texts.len(), + batch_size + ); let mut all_embeddings = Vec::with_capacity(texts.len()); @@ -152,7 +161,10 @@ impl EmbeddingProvider for FastEmbedProvider { let latency_ms = start.elapsed().as_millis() as u64; let count = all_embeddings.len(); - debug!("Batch embedding generated {} vectors in {}ms", count, latency_ms); + debug!( + "Batch embedding generated {} vectors in {}ms", + count, latency_ms + ); Ok(BatchEmbeddingResult { embeddings: all_embeddings, diff --git a/src/embeddings/mock_provider.rs b/src/embeddings/mock_provider.rs index 43c33ef..d7d2a7d 100644 --- a/src/embeddings/mock_provider.rs +++ b/src/embeddings/mock_provider.rs @@ -6,8 +6,8 @@ use anyhow::Result; use async_trait::async_trait; use std::time::Instant; -use crate::models::{EmbeddingModel, EmbeddingVector}; use super::provider::{BatchEmbeddingResult, EmbeddingProvider, EmbeddingResult}; +use crate::models::{EmbeddingModel, EmbeddingVector}; /// Mock embedding provider for testing /// @@ -161,8 +161,8 @@ mod tests { #[tokio::test] async fn test_mock_provider_embed() { - let provider = MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15) - .with_latency(0); + let provider = + MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15).with_latency(0); let result = provider.embed("hello world").await.unwrap(); @@ -173,8 +173,8 @@ mod tests { #[tokio::test] async fn test_mock_provider_deterministic() { - let provider = MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15) - .with_latency(0); + let provider = + MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15).with_latency(0); let result1 = provider.embed("hello world").await.unwrap(); let result2 = provider.embed("hello world").await.unwrap(); @@ -185,8 +185,8 @@ mod tests { #[tokio::test] async fn test_mock_provider_different_texts() { - let provider = MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15) - .with_latency(0); + let provider = + MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15).with_latency(0); let result1 = provider.embed("hello world").await.unwrap(); let result2 = provider.embed("goodbye world").await.unwrap(); @@ -197,8 +197,7 @@ mod tests { #[tokio::test] async fn test_mock_provider_batch() { - let provider = MockEmbeddingProvider::new(EmbeddingModel::BaaiGgeSmall) - .with_latency(0); + let provider = MockEmbeddingProvider::new(EmbeddingModel::BaaiGgeSmall).with_latency(0); let texts = vec![ "text one".to_string(), @@ -215,8 +214,8 @@ mod tests { #[tokio::test] async fn test_mock_provider_should_fail() { - let provider = MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15) - .should_fail(true); + let provider = + MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15).should_fail(true); let result = provider.embed("test").await; assert!(result.is_err()); @@ -230,8 +229,8 @@ mod tests { let healthy_provider = MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15); assert!(healthy_provider.health_check().await.unwrap()); - let unhealthy_provider = MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15) - .should_fail(true); + let unhealthy_provider = + MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15).should_fail(true); assert!(!unhealthy_provider.health_check().await.unwrap()); } @@ -248,8 +247,8 @@ mod tests { #[tokio::test] async fn test_mock_provider_empty_batch() { - let provider = MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15) - .with_latency(0); + let provider = + MockEmbeddingProvider::new(EmbeddingModel::NomicEmbedTextV15).with_latency(0); let result = provider.embed_batch(&[]).await.unwrap(); assert_eq!(result.count, 0); diff --git a/src/embeddings/mod.rs b/src/embeddings/mod.rs index 6d495ed..aa68a9e 100644 --- a/src/embeddings/mod.rs +++ b/src/embeddings/mod.rs @@ -20,18 +20,20 @@ //! ``` pub mod config; +pub mod mock_provider; pub mod provider; pub mod service; -pub mod mock_provider; #[cfg(feature = "embeddings")] pub mod fastembed_provider; // Re-exports pub use config::{EmbeddingConfig, EmbeddingDevice}; -pub use provider::{BatchEmbeddingResult, EmbeddingProvider, EmbeddingProviderExt, EmbeddingResult}; -pub use service::EmbeddingService; pub use mock_provider::MockEmbeddingProvider; +pub use provider::{ + BatchEmbeddingResult, EmbeddingProvider, EmbeddingProviderExt, EmbeddingResult, +}; +pub use service::EmbeddingService; #[cfg(feature = "embeddings")] pub use fastembed_provider::FastEmbedProvider; diff --git a/src/embeddings/provider.rs b/src/embeddings/provider.rs index 1819f36..a8520ef 100644 --- a/src/embeddings/provider.rs +++ b/src/embeddings/provider.rs @@ -2,9 +2,9 @@ #![allow(dead_code)] +use crate::models::{EmbeddingModel, EmbeddingVector}; use anyhow::Result; use async_trait::async_trait; -use crate::models::{EmbeddingModel, EmbeddingVector}; /// Result of an embedding operation #[derive(Debug, Clone)] @@ -79,10 +79,8 @@ mod tests { #[test] fn test_embedding_result_creation() { - let embedding = EmbeddingVector::new( - vec![0.1, 0.2, 0.3], - EmbeddingModel::NomicEmbedTextV15, - ); + let embedding = + EmbeddingVector::new(vec![0.1, 0.2, 0.3], EmbeddingModel::NomicEmbedTextV15); let result = EmbeddingResult { embedding, latency_ms: 100, diff --git a/src/embeddings/service.rs b/src/embeddings/service.rs index 86ce71e..64ec301 100644 --- a/src/embeddings/service.rs +++ b/src/embeddings/service.rs @@ -6,10 +6,10 @@ use anyhow::Result; use std::sync::Arc; use tracing::{debug, info, warn}; -use crate::models::{EmbeddingModel, EmbeddingVector}; use super::config::EmbeddingConfig; -use super::provider::{BatchEmbeddingResult, EmbeddingProvider, EmbeddingResult}; use super::mock_provider::MockEmbeddingProvider; +use super::provider::{BatchEmbeddingResult, EmbeddingProvider, EmbeddingResult}; +use crate::models::{EmbeddingModel, EmbeddingVector}; /// Main embedding service /// @@ -102,7 +102,10 @@ impl EmbeddingService { } let owned_texts: Vec = non_empty_texts.into_iter().cloned().collect(); - debug!("Generating batch embeddings for {} texts", owned_texts.len()); + debug!( + "Generating batch embeddings for {} texts", + owned_texts.len() + ); self.provider.embed_batch(&owned_texts).await } diff --git a/src/graph/raptor.rs b/src/graph/raptor.rs index 040b80a..cc75d57 100644 --- a/src/graph/raptor.rs +++ b/src/graph/raptor.rs @@ -136,10 +136,14 @@ impl Raptor { .collect(); let embeddings: Vec<&EmbeddingVector> = members.iter().map(|n| &n.embedding).collect(); - let centroid = compute_centroid(&embeddings) - .unwrap_or_else(|| members[0].embedding.clone()); + let centroid = + compute_centroid(&embeddings).unwrap_or_else(|| members[0].embedding.clone()); - let combined_content: String = members.iter().map(|n| n.content.as_str()).collect::>().join("\n\n"); + let combined_content: String = members + .iter() + .map(|n| n.content.as_str()) + .collect::>() + .join("\n\n"); let internal_sim = if embeddings.len() > 1 { average_pairwise_similarity(&embeddings) @@ -168,12 +172,12 @@ impl Raptor { // Step 2: Recursively build parent levels let mut depth = 1; - while current_level_ids.len() > 1 && (self.config.max_depth == 0 || depth <= self.config.max_depth) { + while current_level_ids.len() > 1 + && (self.config.max_depth == 0 || depth <= self.config.max_depth) + { let current_nodes: Vec<(&String, &EmbeddingVector)> = current_level_ids .iter() - .filter_map(|id| { - tree_nodes.get(id).map(|n| (id, &n.centroid)) - }) + .filter_map(|id| tree_nodes.get(id).map(|n| (id, &n.centroid))) .collect(); if current_nodes.len() < 2 { @@ -186,7 +190,9 @@ impl Raptor { let mut next_level_ids = Vec::new(); for cluster in parent_clusters { - if cluster.size() < 2 && next_level_ids.len() + current_level_ids.len() > cluster.size() { + if cluster.size() < 2 + && next_level_ids.len() + current_level_ids.len() > cluster.size() + { // Skip singleton clusters at higher levels continue; } @@ -203,12 +209,7 @@ impl Raptor { let combined_content: String = child_ids .iter() .filter_map(|id| tree_nodes.get(id)) - .map(|n| { - n.summary - .as_ref() - .map(|s| s.as_str()) - .unwrap_or(&n.combined_content) - }) + .map(|n| n.summary.as_deref().unwrap_or(&n.combined_content)) .collect::>() .join("\n\n---\n\n"); @@ -387,7 +388,9 @@ impl Raptor { for i in 0..n { if active[i] { let cluster_idx = cluster_map[&i]; - if !seen.contains(&cluster_idx) && clusters[cluster_idx].size() >= self.config.min_cluster_size { + if !seen.contains(&cluster_idx) + && clusters[cluster_idx].size() >= self.config.min_cluster_size + { seen.insert(cluster_idx); result.push(clusters[cluster_idx].clone()); } @@ -397,7 +400,9 @@ impl Raptor { // Add singleton clusters that didn't get merged (if below min size) for i in 0..n { let cluster_idx = cluster_map[&i]; - if clusters[cluster_idx].size() < self.config.min_cluster_size && !seen.contains(&cluster_idx) { + if clusters[cluster_idx].size() < self.config.min_cluster_size + && !seen.contains(&cluster_idx) + { // Create individual cluster for orphans let mut cluster = Cluster::new(vec![ids[i].clone()]); cluster.set_depth(0); @@ -677,7 +682,10 @@ mod tests { assert!(!path.is_empty()); // Path should go from leaf to root if path.len() > 1 { - assert!(path[0].depth <= path[path.len() - 1].depth || path[path.len() - 1].parent_id.is_none()); + assert!( + path[0].depth <= path[path.len() - 1].depth + || path[path.len() - 1].parent_id.is_none() + ); } } } diff --git a/src/graph/similarity.rs b/src/graph/similarity.rs index 247cfe5..75b1779 100644 --- a/src/graph/similarity.rs +++ b/src/graph/similarity.rs @@ -67,9 +67,9 @@ pub fn similarity_to_centroid(point: &EmbeddingVector, centroid: &EmbeddingVecto } /// Finds the most similar embedding to a query from a list. -pub fn find_most_similar<'a>( +pub fn find_most_similar( query: &EmbeddingVector, - candidates: &'a [&EmbeddingVector], + candidates: &[&EmbeddingVector], ) -> Option<(usize, f32)> { if candidates.is_empty() { return None; @@ -90,9 +90,9 @@ pub fn find_most_similar<'a>( } /// Finds the k most similar embeddings to a query. -pub fn find_k_most_similar<'a>( +pub fn find_k_most_similar( query: &EmbeddingVector, - candidates: &'a [&EmbeddingVector], + candidates: &[&EmbeddingVector], k: usize, ) -> Vec<(usize, f32)> { if candidates.is_empty() || k == 0 { diff --git a/src/graph/sssp.rs b/src/graph/sssp.rs index d23c09c..151f676 100644 --- a/src/graph/sssp.rs +++ b/src/graph/sssp.rs @@ -5,8 +5,8 @@ #![allow(dead_code)] -use std::collections::{BinaryHeap, HashMap, HashSet}; use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashMap, HashSet}; use std::time::Instant; use super::config::SsspConfig; @@ -40,7 +40,9 @@ impl GraphNode { /// Gets the weight (distance) to a neighbor. pub fn weight_to(&self, neighbor_id: &str) -> Option { - self.edges.get(neighbor_id).map(|&sim| similarity_to_distance(sim)) + self.edges + .get(neighbor_id) + .map(|&sim| similarity_to_distance(sim)) } } @@ -62,7 +64,10 @@ impl PartialEq for DijkstraEntry { impl Ord for DijkstraEntry { fn cmp(&self, other: &Self) -> Ordering { // Reverse ordering for min-heap (smaller distance = higher priority) - other.distance.partial_cmp(&self.distance).unwrap_or(Ordering::Equal) + other + .distance + .partial_cmp(&self.distance) + .unwrap_or(Ordering::Equal) .then_with(|| self.node_id.cmp(&other.node_id)) } } @@ -168,7 +173,9 @@ impl SsspResult { /// Gets the k nearest nodes from the source. pub fn k_nearest(&self, k: usize) -> Vec<(String, f32)> { - let mut sorted: Vec<_> = self.distances.iter() + let mut sorted: Vec<_> = self + .distances + .iter() .map(|(id, &dist)| (id.clone(), dist)) .collect(); @@ -308,7 +315,8 @@ impl Sssp { let edge_distance = similarity_to_distance(similarity); let new_distance = distance + edge_distance; - let is_shorter = distances.get(neighbor_id) + let is_shorter = distances + .get(neighbor_id) .map(|&d| new_distance < d) .unwrap_or(true); @@ -332,7 +340,8 @@ impl Sssp { let new_distance = distance + shortcut.distance; - let is_shorter = distances.get(&shortcut.to) + let is_shorter = distances + .get(&shortcut.to) .map(|&d| new_distance < d) .unwrap_or(true); @@ -385,9 +394,7 @@ impl Sssp { // For each sampled node, compute short-range shortest paths for &sample_node in &sampled_nodes { - let temp_config = SsspConfig::new() - .with_max_hops(3) - .with_hopsets(false); + let temp_config = SsspConfig::new().with_max_hops(3).with_hopsets(false); let temp_sssp = Sssp::new(temp_config); let result = temp_sssp.compute(graph, sample_node, None); diff --git a/src/lib.rs b/src/lib.rs index bcc8b8f..699a698 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,20 +15,20 @@ pub mod utils; // Re-exportar tipos principales pub use db::connection::{DatabaseConnection, DbConfig}; +pub use models::edge::{EdgeType, FractalEdge, GraphPath}; +pub use models::embedding::{EmbeddingModel, EmbeddingVector}; pub use models::llm::{BrainConfig, ModelBrain, ModelConfig, ModelProvider}; -pub use models::node::{FractalNode, NodeMetadata, NodeStatus, NodeType, SourceType}; -pub use models::edge::{FractalEdge, EdgeType, GraphPath}; pub use models::namespace::{Namespace, NamespaceType, Scope, ScopePermissions}; -pub use models::embedding::{EmbeddingModel, EmbeddingVector}; +pub use models::node::{FractalNode, NodeMetadata, NodeStatus, NodeType, SourceType}; // Embedding service exports pub use embeddings::{ - EmbeddingConfig, EmbeddingDevice, EmbeddingProvider, EmbeddingResult, - EmbeddingService, MockEmbeddingProvider, + EmbeddingConfig, EmbeddingDevice, EmbeddingProvider, EmbeddingResult, EmbeddingService, + MockEmbeddingProvider, }; // Cache exports pub use cache::{ - CacheConfig, CacheMetrics, NodeCache, EmbeddingCache, - SharedNodeCache, SharedEmbeddingCache, ThreadSafeLruCache, + CacheConfig, CacheMetrics, EmbeddingCache, NodeCache, SharedEmbeddingCache, SharedNodeCache, + ThreadSafeLruCache, }; diff --git a/src/main.rs b/src/main.rs index 549c8ef..1592be2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,24 +9,26 @@ mod models; mod services; mod utils; -use std::sync::Arc; use std::net::SocketAddr; +use std::sync::Arc; use anyhow::Result; use dotenv::dotenv; -use tokio::sync::RwLock; -use tower_http::cors::{CorsLayer, Any}; use http::Method; +use tokio::sync::RwLock; +use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; -use tracing::{info, error, warn}; +use tracing::{error, info, warn}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -use crate::db::connection::DatabaseConnection; -use crate::models::llm::{ModelBrain, BrainConfig}; -use crate::cache::{NodeCache, EmbeddingCache, CacheConfig}; use crate::api::handlers::{AppState, SharedState}; use crate::api::progress::create_progress_tracker; -use crate::services::{StorageManager, UploadSessionManager, UploadCleanupJob, RemScheduler, RemSchedulerConfig}; +use crate::cache::{CacheConfig, EmbeddingCache, NodeCache}; +use crate::db::connection::DatabaseConnection; +use crate::models::llm::{BrainConfig, ModelBrain}; +use crate::services::{ + RemScheduler, RemSchedulerConfig, StorageManager, UploadCleanupJob, UploadSessionManager, +}; #[tokio::main] async fn main() -> Result<()> { @@ -94,9 +96,18 @@ async fn main() -> Result<()> { error!(" 4. Review your configuration in .env file"); error!(""); error!(" Provider configuration:"); - error!(" - EMBEDDING_PROVIDER: {}", brain_config_clone.embedding_model.provider); - error!(" - CHAT_PROVIDER: {}", brain_config_clone.chat_model.provider); - error!(" - SUMMARIZER_PROVIDER: {}", brain_config_clone.summarizer_model.provider); + error!( + " - EMBEDDING_PROVIDER: {}", + brain_config_clone.embedding_model.provider + ); + error!( + " - CHAT_PROVIDER: {}", + brain_config_clone.chat_model.provider + ); + error!( + " - SUMMARIZER_PROVIDER: {}", + brain_config_clone.summarizer_model.provider + ); error!(""); error!(" For local Ollama, run: ollama serve"); return Err(anyhow::anyhow!("Failed to initialize Model Brain")); @@ -107,10 +118,10 @@ async fn main() -> Result<()> { let cache_config = CacheConfig::from_env(); let node_cache = NodeCache::new(cache_config.clone()); let embedding_cache = EmbeddingCache::new(cache_config); - + // Crear progress tracker para ingestion let progress_tracker = create_progress_tracker(); - + // Inicializar upload manager para modelos GGUF info!("Initializing upload manager..."); let storage = StorageManager::new(); @@ -119,7 +130,7 @@ async fn main() -> Result<()> { warn!("Failed to initialize upload manager (non-fatal): {}", e); } let upload_manager = Arc::new(upload_manager); - + // Iniciar cleanup job para sesiones de upload expiradas (cada 60 minutos) let cleanup_job = UploadCleanupJob::new(upload_manager.clone(), 60); let _cleanup_handle = cleanup_job.start(); @@ -130,8 +141,10 @@ async fn main() -> Result<()> { if rem_config.enabled { // Clonar db y brain para el scheduler let db_clone = db::connection::connect_db(&db_config).await?; - let brain_clone = ModelBrain::new_without_health_check(BrainConfig::from_env().unwrap_or_else(|_| BrainConfig::default_local()))?; - + let brain_clone = ModelBrain::new_without_health_check( + BrainConfig::from_env().unwrap_or_else(|_| BrainConfig::default_local()), + )?; + let rem_scheduler = Arc::new(RemScheduler::new(rem_config.clone(), db_clone, brain_clone)); let _rem_handle = rem_scheduler.start(); info!( diff --git a/src/models/edge.rs b/src/models/edge.rs index 2aa676a..857dc26 100644 --- a/src/models/edge.rs +++ b/src/models/edge.rs @@ -1,8 +1,8 @@ #![allow(dead_code)] +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use surrealdb::sql::Thing; -use chrono::{DateTime, Utc}; /// Tipo de relación entre nodos #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] diff --git a/src/models/embedding.rs b/src/models/embedding.rs index 84bc397..dbf63ac 100644 --- a/src/models/embedding.rs +++ b/src/models/embedding.rs @@ -139,18 +139,9 @@ mod tests { #[test] fn test_cosine_similarity() { - let vec1 = EmbeddingVector::new( - vec![1.0, 0.0, 0.0], - EmbeddingModel::NomicEmbedTextV15, - ); - let vec2 = EmbeddingVector::new( - vec![1.0, 0.0, 0.0], - EmbeddingModel::NomicEmbedTextV15, - ); - let vec3 = EmbeddingVector::new( - vec![0.0, 1.0, 0.0], - EmbeddingModel::NomicEmbedTextV15, - ); + let vec1 = EmbeddingVector::new(vec![1.0, 0.0, 0.0], EmbeddingModel::NomicEmbedTextV15); + let vec2 = EmbeddingVector::new(vec![1.0, 0.0, 0.0], EmbeddingModel::NomicEmbedTextV15); + let vec3 = EmbeddingVector::new(vec![0.0, 1.0, 0.0], EmbeddingModel::NomicEmbedTextV15); assert!((vec1.cosine_similarity(&vec2) - 1.0).abs() < 1e-6); assert!((vec1.cosine_similarity(&vec3) - 0.0).abs() < 1e-6); @@ -158,14 +149,8 @@ mod tests { #[test] fn test_euclidean_distance() { - let vec1 = EmbeddingVector::new( - vec![0.0, 0.0, 0.0], - EmbeddingModel::NomicEmbedTextV15, - ); - let vec2 = EmbeddingVector::new( - vec![1.0, 1.0, 1.0], - EmbeddingModel::NomicEmbedTextV15, - ); + let vec1 = EmbeddingVector::new(vec![0.0, 0.0, 0.0], EmbeddingModel::NomicEmbedTextV15); + let vec2 = EmbeddingVector::new(vec![1.0, 1.0, 1.0], EmbeddingModel::NomicEmbedTextV15); let distance = vec1.euclidean_distance(&vec2); assert!((distance - 3.0_f32.sqrt()).abs() < 1e-6); @@ -173,10 +158,7 @@ mod tests { #[test] fn test_normalize() { - let mut vec = EmbeddingVector::new( - vec![3.0, 4.0, 0.0], - EmbeddingModel::NomicEmbedTextV15, - ); + let mut vec = EmbeddingVector::new(vec![3.0, 4.0, 0.0], EmbeddingModel::NomicEmbedTextV15); assert!(!vec.is_normalized()); vec.normalize(); diff --git a/src/models/llm/brain.rs b/src/models/llm/brain.rs index 663550d..7b988f8 100644 --- a/src/models/llm/brain.rs +++ b/src/models/llm/brain.rs @@ -1,12 +1,15 @@ #![allow(dead_code)] use super::config::{BrainConfig, ModelConfig, ModelProvider}; -use super::providers::{OllamaChat, OllamaEmbedding, OllamaSummarizer, OpenAIChat, OpenAIEmbedding, AnthropicChat, AnthropicEmbedding}; +use super::providers::{ + AnthropicChat, AnthropicEmbedding, OllamaChat, OllamaEmbedding, OllamaSummarizer, OpenAIChat, + OpenAIEmbedding, +}; use super::traits_llm::{ ChatMessage, ChatProvider, ChatResponse, EmbeddingProvider, EmbeddingResponse, SummarizerProvider, }; -use anyhow::{Context, Result}; +use anyhow::Result; use std::sync::Arc; use tracing::{error, info, warn}; @@ -36,7 +39,10 @@ impl ModelBrain { // Inicializar proveedor de embeddings info!("Initializing embedding provider..."); let embedding_provider = Self::create_embedding_provider(&config.embedding_model)?; - info!("Embedding provider initialized: {}", embedding_provider.model_name()); + info!( + "Embedding provider initialized: {}", + embedding_provider.model_name() + ); // Inicializar proveedor de chat info!("Initializing chat provider..."); @@ -46,16 +52,15 @@ impl ModelBrain { // Inicializar proveedor de summarizer info!("Initializing summarizer provider..."); let summarizer_provider = Self::create_summarizer_provider(&config.summarizer_model)?; - info!("Summarizer provider initialized: {}", summarizer_provider.model_name()); + info!( + "Summarizer provider initialized: {}", + summarizer_provider.model_name() + ); // Verificar salud de los proveedores info!("Verifying provider health..."); - Self::verify_providers_health( - &embedding_provider, - &chat_provider, - &summarizer_provider, - ) - .await?; + Self::verify_providers_health(&embedding_provider, &chat_provider, &summarizer_provider) + .await?; info!("ModelBrain initialized successfully"); @@ -92,36 +97,58 @@ impl ModelBrain { /// Crea un ModelBrain básico solo con Ollama para tests pub fn with_ollama_only(base_url: String, embedding_model: String) -> Result { use crate::models::llm::config::{BrainConfig, ModelConfig, ModelProvider}; - + + use super::config::ModelType; + use std::collections::HashMap; + let config = BrainConfig { embedding_model: ModelConfig { + model_type: ModelType::Embedding, provider: ModelProvider::Ollama { base_url: base_url.clone(), model_name: embedding_model.clone(), api_key: None, }, + temperature: 0.0, + top_p: 1.0, + max_tokens: 0, + timeout_seconds: 30, + max_retries: 3, + extra_config: HashMap::new(), }, chat_model: ModelConfig { + model_type: ModelType::Chat, provider: ModelProvider::Ollama { base_url: base_url.clone(), model_name: "llama2".to_string(), api_key: None, }, + temperature: 0.7, + top_p: 0.9, + max_tokens: 2048, + timeout_seconds: 60, + max_retries: 2, + extra_config: HashMap::new(), }, summarizer_model: ModelConfig { + model_type: ModelType::Summarizer, provider: ModelProvider::Ollama { base_url: base_url.clone(), model_name: "llama2".to_string(), api_key: None, }, + temperature: 0.3, + top_p: 0.9, + max_tokens: 512, + timeout_seconds: 45, + max_retries: 2, + extra_config: HashMap::new(), }, + prefer_local: true, }; - let embedding_provider = Arc::new(OllamaEmbedding::new( - base_url.clone(), - embedding_model, - 768, - )); + let embedding_provider = + Arc::new(OllamaEmbedding::new(base_url.clone(), embedding_model, 768)); let chat_provider = Arc::new(OllamaChat::new( base_url.clone(), @@ -146,9 +173,7 @@ impl ModelBrain { } /// Crea un proveedor de embeddings desde configuración - fn create_embedding_provider( - config: &ModelConfig, - ) -> Result> { + fn create_embedding_provider(config: &ModelConfig) -> Result> { match &config.provider { ModelProvider::Ollama { base_url, @@ -271,9 +296,7 @@ impl ModelBrain { } /// Crea un proveedor de summarizer desde configuración - fn create_summarizer_provider( - config: &ModelConfig, - ) -> Result> { + fn create_summarizer_provider(config: &ModelConfig) -> Result> { match &config.provider { ModelProvider::Ollama { base_url, @@ -338,9 +361,15 @@ impl ModelBrain { let embedding_ok = match embedding.health_check().await { Ok(ok) => { if ok { - info!("✅ Embedding provider ({}): healthy", embedding.model_name()); + info!( + "✅ Embedding provider ({}): healthy", + embedding.model_name() + ); } else { - warn!("❌ Embedding provider ({}): health check failed", embedding.model_name()); + warn!( + "❌ Embedding provider ({}): health check failed", + embedding.model_name() + ); } ok } @@ -355,7 +384,10 @@ impl ModelBrain { if ok { info!("✅ Chat provider ({}): healthy", chat.model_name()); } else { - warn!("❌ Chat provider ({}): health check failed", chat.model_name()); + warn!( + "❌ Chat provider ({}): health check failed", + chat.model_name() + ); } ok } @@ -368,9 +400,15 @@ impl ModelBrain { let summarizer_ok = match summarizer.health_check().await { Ok(ok) => { if ok { - info!("✅ Summarizer provider ({}): healthy", summarizer.model_name()); + info!( + "✅ Summarizer provider ({}): healthy", + summarizer.model_name() + ); } else { - warn!("❌ Summarizer provider ({}): health check failed", summarizer.model_name()); + warn!( + "❌ Summarizer provider ({}): health check failed", + summarizer.model_name() + ); } ok } @@ -384,31 +422,42 @@ impl ModelBrain { error!( "❌ Provider health check failed - System cannot operate without working providers" ); - error!("embedding: {}, chat: {}, summarizer: {}", - if embedding_ok { "ok" } else { "failed" }, - if chat_ok { "ok" } else { "failed" }, - if summarizer_ok { "ok" } else { "failed" }); - + error!( + "embedding: {}, chat: {}, summarizer: {}", + if embedding_ok { "ok" } else { "failed" }, + if chat_ok { "ok" } else { "failed" }, + if summarizer_ok { "ok" } else { "failed" } + ); + let mut error_msg = String::from("Provider configuration error:\n"); if !embedding_ok { - error_msg.push_str(&format!("- Embedding provider '{}': failed health check\n", embedding.model_name())); + error_msg.push_str(&format!( + "- Embedding provider '{}': failed health check\n", + embedding.model_name() + )); error_msg.push_str(" → Ensure the provider is running and accessible\n"); error_msg.push_str(" → Check your network connection and provider URL\n"); error_msg.push_str(" → Verify your API key if using cloud service\n"); } if !chat_ok { - error_msg.push_str(&format!("- Chat provider '{}': failed health check\n", chat.model_name())); + error_msg.push_str(&format!( + "- Chat provider '{}': failed health check\n", + chat.model_name() + )); error_msg.push_str(" → Ensure the provider is running and accessible\n"); error_msg.push_str(" → Check your network connection and provider URL\n"); error_msg.push_str(" → Verify your API key if using cloud service\n"); } if !summarizer_ok { - error_msg.push_str(&format!("- Summarizer provider '{}': failed health check\n", summarizer.model_name())); + error_msg.push_str(&format!( + "- Summarizer provider '{}': failed health check\n", + summarizer.model_name() + )); error_msg.push_str(" → Ensure the provider is running and accessible\n"); error_msg.push_str(" → Check your network connection and provider URL\n"); error_msg.push_str(" → Verify your API key if using cloud service\n"); } - + return Err(anyhow::anyhow!(error_msg.trim().to_string())); } @@ -499,7 +548,10 @@ mod tests { #[test] fn test_infer_embedding_dimension() { - assert_eq!(ModelBrain::infer_embedding_dimension("nomic-embed-text"), 768); + assert_eq!( + ModelBrain::infer_embedding_dimension("nomic-embed-text"), + 768 + ); assert_eq!( ModelBrain::infer_embedding_dimension("bge-small-en-v1.5"), 384 diff --git a/src/models/llm/config.rs b/src/models/llm/config.rs index e731c83..f8d538c 100644 --- a/src/models/llm/config.rs +++ b/src/models/llm/config.rs @@ -241,7 +241,7 @@ impl ModelConfig { if self.provider.requires_api_key() { match &self.provider { ModelProvider::Ollama { api_key, .. } => { - if api_key.as_ref().map_or(true, |k| k.is_empty()) { + if api_key.as_ref().is_none_or(|k| k.is_empty()) { bail!("API key is required for remote provider"); } } diff --git a/src/models/llm/fractal_model.rs b/src/models/llm/fractal_model.rs index 24e13fb..362d30b 100644 --- a/src/models/llm/fractal_model.rs +++ b/src/models/llm/fractal_model.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] -use serde::{Deserialize, Deserializer, Serialize}; use chrono::{DateTime, Utc}; +use serde::{Deserialize, Deserializer, Serialize}; use uuid::Uuid; /// Deserializa un ID de SurrealDB que puede venir como string o como objeto Thing @@ -10,38 +10,40 @@ where D: Deserializer<'de>, { use serde::de::{self, Visitor}; - + struct IdVisitor; - + impl<'de> Visitor<'de> for IdVisitor { type Value = String; - + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("a string or SurrealDB Thing") } - + fn visit_str(self, value: &str) -> Result where E: de::Error, { Ok(value.to_string()) } - + fn visit_map(self, mut map: A) -> Result where A: de::MapAccess<'de>, { let mut tb: Option = None; let mut id: Option = None; - + while let Some(key) = map.next_key::()? { match key.as_str() { "tb" => tb = Some(map.next_value()?), "id" => id = Some(map.next_value()?), - _ => { let _: serde_json::Value = map.next_value()?; } + _ => { + let _: serde_json::Value = map.next_value()?; + } } } - + match (tb, id) { (Some(table), Some(serde_json::Value::String(s))) => Ok(format!("{}:{}", table, s)), (Some(table), Some(serde_json::Value::Object(obj))) => { @@ -51,11 +53,11 @@ where Ok(format!("{}:{:?}", table, obj)) } } - _ => Ok("unknown".to_string()) + _ => Ok("unknown".to_string()), } } } - + deserializer.deserialize_any(IdVisitor) } @@ -74,7 +76,7 @@ where D: Deserializer<'de>, { use serde::de::Error; - + let s: String = String::deserialize(deserializer)?; DateTime::parse_from_rfc3339(&s) .map(|dt| dt.with_timezone(&Utc)) diff --git a/src/models/llm/gguf_parser.rs b/src/models/llm/gguf_parser.rs index f506ca1..40097fe 100644 --- a/src/models/llm/gguf_parser.rs +++ b/src/models/llm/gguf_parser.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use byteorder::{LittleEndian, ReadBytesExt}; use memmap2::Mmap; use std::collections::HashMap; @@ -113,12 +113,9 @@ impl GGUFParser { pub fn parse_file(file_path: &str) -> Result { info!("Parsing GGUF file: {}", file_path); - let file = File::open(file_path) - .context("Failed to open GGUF file")?; - - let mmap = unsafe { - Mmap::map(&file).context("Failed to memory-map file")? - }; + let file = File::open(file_path).context("Failed to open GGUF file")?; + + let mmap = unsafe { Mmap::map(&file).context("Failed to memory-map file")? }; Self::parse_bytes(&mmap) } @@ -130,18 +127,28 @@ impl GGUFParser { // Leer cabecera let magic = cursor.read_u32::()?; if magic != GGUF_MAGIC { - bail!("Invalid GGUF magic number: expected {:#x}, got {:#x}", GGUF_MAGIC, magic); + bail!( + "Invalid GGUF magic number: expected {:#x}, got {:#x}", + GGUF_MAGIC, + magic + ); } let version = cursor.read_u32::()?; if version != GGUF_VERSION { - warn!("GGUF version mismatch: expected {}, got {}", GGUF_VERSION, version); + warn!( + "GGUF version mismatch: expected {}, got {}", + GGUF_VERSION, version + ); } let tensor_count = cursor.read_u64::()?; let metadata_kv_count = cursor.read_u64::()?; - debug!("GGUF version: {}, tensors: {}, metadata: {}", version, tensor_count, metadata_kv_count); + debug!( + "GGUF version: {}, tensors: {}, metadata: {}", + version, tensor_count, metadata_kv_count + ); // Leer metadatos let mut metadata = HashMap::new(); @@ -160,8 +167,11 @@ impl GGUFParser { // The data offset is the current cursor position, aligned to 32 bytes let current_pos = cursor.position(); let data_offset = (current_pos + 31) & !31; - - debug!("GGUF data offset: {} (cursor at {}, aligned to 32 bytes)", data_offset, current_pos); + + debug!( + "GGUF data offset: {} (cursor at {}, aligned to 32 bytes)", + data_offset, current_pos + ); Ok(Self { metadata, @@ -207,7 +217,7 @@ impl GGUFParser { let array_type = cursor.read_u32::()?; let array_type = GGUFValueType::try_from(array_type)?; let array_len = cursor.read_u64::()? as usize; - + let mut values = Vec::with_capacity(array_len); for _ in 0..array_len { values.push(Self::read_value(cursor, array_type)?); @@ -221,12 +231,12 @@ impl GGUFParser { fn read_tensor_info(cursor: &mut Cursor<&[u8]>) -> Result { let name = Self::read_string(cursor)?; let n_dimensions = cursor.read_u32::()? as usize; - + let mut dimensions = Vec::with_capacity(n_dimensions); for _ in 0..n_dimensions { dimensions.push(cursor.read_u64::()?); } - + let tensor_type = cursor.read_u32::()?; let offset = cursor.read_u64::()?; @@ -241,16 +251,17 @@ impl GGUFParser { /// Extrae la arquitectura del modelo desde los metadatos pub fn extract_architecture(&self) -> Result { let model_type = self.get_metadata_string("general.architecture")?; - - let prefix = format!("{}", model_type); - + + let prefix = model_type.to_string(); + let n_layers = self.get_metadata_u32(&format!("{}.block_count", prefix))?; let embedding_dim = self.get_metadata_u32(&format!("{}.embedding_length", prefix))?; let n_heads = self.get_metadata_u32(&format!("{}.attention.head_count", prefix))?; let ffn_dim = self.get_metadata_u32(&format!("{}.feed_forward_length", prefix))?; - + // Intentar obtener vocab_size de diferentes campos posibles - let vocab_size = self.get_metadata_u32("tokenizer.ggml.tokens.length") + let vocab_size = self + .get_metadata_u32("tokenizer.ggml.tokens.length") .or_else(|_| self.get_metadata_u32(&format!("{}.vocab_size", prefix))) .unwrap_or(32000); // Valor por defecto diff --git a/src/models/llm/mod.rs b/src/models/llm/mod.rs index 8c9973a..f7c717e 100644 --- a/src/models/llm/mod.rs +++ b/src/models/llm/mod.rs @@ -1,13 +1,13 @@ -pub mod config; -pub mod providers; -pub mod traits_llm; pub mod brain; +pub mod config; pub mod fractal_model; pub mod gguf_parser; +pub mod providers; pub mod strategy; +pub mod traits_llm; -pub use config::{BrainConfig, ModelConfig, ModelProvider}; pub use brain::ModelBrain; +pub use config::{BrainConfig, ModelConfig, ModelProvider}; pub use fractal_model::*; pub use gguf_parser::*; pub use strategy::*; diff --git a/src/models/llm/providers/anthropic.rs b/src/models/llm/providers/anthropic.rs index 6e6fb52..3b15512 100644 --- a/src/models/llm/providers/anthropic.rs +++ b/src/models/llm/providers/anthropic.rs @@ -260,7 +260,9 @@ impl ChatProvider for AnthropicChat { .map(|msg| AnthropicChatMessage { role: match msg.role { ChatRole::System => { - warn!("Anthropic uses 'system' role differently - placing in system prompt"); + warn!( + "Anthropic uses 'system' role differently - placing in system prompt" + ); "user".to_string() } ChatRole::User => "user".to_string(), diff --git a/src/models/llm/providers/mod.rs b/src/models/llm/providers/mod.rs index 915e350..e13c261 100644 --- a/src/models/llm/providers/mod.rs +++ b/src/models/llm/providers/mod.rs @@ -1,7 +1,7 @@ +pub mod anthropic; pub mod ollama; pub mod openai; -pub mod anthropic; -pub use ollama::{OllamaEmbedding, OllamaChat, OllamaSummarizer}; -pub use openai::{OpenAIChat, OpenAIEmbedding}; pub use anthropic::{AnthropicChat, AnthropicEmbedding}; +pub use ollama::{OllamaChat, OllamaEmbedding, OllamaSummarizer}; +pub use openai::{OpenAIChat, OpenAIEmbedding}; diff --git a/src/models/llm/providers/ollama.rs b/src/models/llm/providers/ollama.rs index 5c6d5b0..fe55369 100644 --- a/src/models/llm/providers/ollama.rs +++ b/src/models/llm/providers/ollama.rs @@ -31,7 +31,12 @@ impl OllamaEmbedding { } } - pub fn with_api_key(base_url: String, model_name: String, dimension: usize, api_key: String) -> Self { + pub fn with_api_key( + base_url: String, + model_name: String, + dimension: usize, + api_key: String, + ) -> Self { Self { client: Client::new(), base_url, @@ -66,13 +71,11 @@ impl EmbeddingProvider for OllamaEmbedding { debug!("Sending embedding request to Ollama: {}", url); - let mut request_builder = self - .client - .post(&url) - .json(&request); + let mut request_builder = self.client.post(&url).json(&request); if let Some(ref api_key) = self.api_key { - request_builder = request_builder.header("Authorization", format!("Bearer {}", api_key)); + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); } let response = request_builder @@ -124,11 +127,12 @@ impl EmbeddingProvider for OllamaEmbedding { async fn health_check(&self) -> Result { let url = format!("{}/api/tags", self.base_url); let mut request_builder = self.client.get(&url); - + if let Some(ref api_key) = self.api_key { - request_builder = request_builder.header("Authorization", format!("Bearer {}", api_key)); + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); } - + match request_builder.send().await { Ok(response) => Ok(response.status().is_success()), Err(e) => { @@ -150,12 +154,7 @@ pub struct OllamaChat { } impl OllamaChat { - pub fn new( - base_url: String, - model_name: String, - temperature: f32, - max_tokens: u32, - ) -> Self { + pub fn new(base_url: String, model_name: String, temperature: f32, max_tokens: u32) -> Self { Self { client: Client::new(), base_url, @@ -245,13 +244,11 @@ impl ChatProvider for OllamaChat { debug!("Sending chat request to Ollama: {}", url); - let mut request_builder = self - .client - .post(&url) - .json(&request); + let mut request_builder = self.client.post(&url).json(&request); if let Some(ref api_key) = self.api_key { - request_builder = request_builder.header("Authorization", format!("Bearer {}", api_key)); + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); } let response = request_builder @@ -291,11 +288,12 @@ impl ChatProvider for OllamaChat { async fn health_check(&self) -> Result { let url = format!("{}/api/tags", self.base_url); let mut request_builder = self.client.get(&url); - + if let Some(ref api_key) = self.api_key { - request_builder = request_builder.header("Authorization", format!("Bearer {}", api_key)); + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); } - + match request_builder.send().await { Ok(response) => Ok(response.status().is_success()), Err(e) => { @@ -317,12 +315,7 @@ pub struct OllamaSummarizer { } impl OllamaSummarizer { - pub fn new( - base_url: String, - model_name: String, - temperature: f32, - max_tokens: u32, - ) -> Self { + pub fn new(base_url: String, model_name: String, temperature: f32, max_tokens: u32) -> Self { Self { client: Client::new(), base_url, @@ -392,13 +385,11 @@ impl SummarizerProvider for OllamaSummarizer { debug!("Sending summarization request to Ollama"); - let mut request_builder = self - .client - .post(&url) - .json(&request); + let mut request_builder = self.client.post(&url).json(&request); if let Some(ref api_key) = self.api_key { - request_builder = request_builder.header("Authorization", format!("Bearer {}", api_key)); + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); } let response = request_builder @@ -431,11 +422,12 @@ impl SummarizerProvider for OllamaSummarizer { async fn health_check(&self) -> Result { let url = format!("{}/api/tags", self.base_url); let mut request_builder = self.client.get(&url); - + if let Some(ref api_key) = self.api_key { - request_builder = request_builder.header("Authorization", format!("Bearer {}", api_key)); + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); } - + match request_builder.send().await { Ok(response) => Ok(response.status().is_success()), Err(e) => { diff --git a/src/models/llm/strategy.rs b/src/models/llm/strategy.rs index 853feb7..261df6b 100644 --- a/src/models/llm/strategy.rs +++ b/src/models/llm/strategy.rs @@ -10,8 +10,8 @@ use tokio::sync::RwLock; use super::traits_llm::{ChatMessage, ChatResponse, EmbeddingResponse}; use crate::db::connection::DatabaseConnection; -use crate::db::queries::{NodeRepository, EdgeRepository}; -use crate::graph::{Sssp, GraphNode}; +use crate::db::queries::{EdgeRepository, NodeRepository}; +use crate::graph::{GraphNode, Sssp}; // ============================================================================ // FractalModelStrategy Configuration @@ -33,11 +33,11 @@ pub struct FractalModelStrategyConfig { /// Temperatura para chat pub chat_temperature: f32, /// Máximo de tokens para chat - pub chat_max_tokens: usize, + pub chat_max_tokens: u32, /// Temperatura para sumarización pub summarizer_temperature: f32, /// Máximo de tokens para sumarización - pub summarizer_max_tokens: usize, + pub summarizer_max_tokens: u32, } impl Default for FractalModelStrategyConfig { @@ -82,13 +82,13 @@ impl FractalModelStrategyConfig { self } - pub fn with_chat_config(mut self, temperature: f32, max_tokens: usize) -> Self { + pub fn with_chat_config(mut self, temperature: f32, max_tokens: u32) -> Self { self.chat_temperature = temperature; self.chat_max_tokens = max_tokens; self } - pub fn with_summarizer_config(mut self, temperature: f32, max_tokens: usize) -> Self { + pub fn with_summarizer_config(mut self, temperature: f32, max_tokens: u32) -> Self { self.summarizer_temperature = temperature; self.summarizer_max_tokens = max_tokens; self @@ -97,12 +97,12 @@ impl FractalModelStrategyConfig { pub fn from_env() -> Self { let default_namespace = std::env::var("FRACTAL_DEFAULT_NAMESPACE") .unwrap_or_else(|_| "global_knowledge".to_string()); - + let max_results = std::env::var("FRACTAL_MAX_RESULTS") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(5); - + let ollama_base_url = std::env::var("OLLAMA_BASE_URL") .unwrap_or_else(|_| "http://localhost:11434".to_string()); @@ -148,15 +148,19 @@ pub struct FractalModelStrategy { impl FractalModelStrategy { pub fn new(model_id: String, db: DatabaseConnection) -> Self { - Self { + Self { model_id, db: Arc::new(RwLock::new(db)), config: FractalModelStrategyConfig::default(), } } - pub fn with_config(model_id: String, db: DatabaseConnection, config: FractalModelStrategyConfig) -> Self { - Self { + pub fn with_config( + model_id: String, + db: DatabaseConnection, + config: FractalModelStrategyConfig, + ) -> Self { + Self { model_id, db: Arc::new(RwLock::new(db)), config, @@ -164,20 +168,29 @@ impl FractalModelStrategy { } /// Navega por el grafo fractal para encontrar contexto relevante - async fn navigate_fractal_graph(&self, query_embedding: &[f32], namespace: &str, limit: usize) -> Result> { + async fn navigate_fractal_graph( + &self, + query_embedding: &[f32], + namespace: &str, + limit: usize, + ) -> Result> { let db = self.db.read().await; let node_repo = NodeRepository::new(&db); - - let results = node_repo.search_similar(query_embedding, namespace, limit * 2).await?; - + + let results = node_repo + .search_similar(query_embedding, namespace, limit * 2) + .await?; + if results.is_empty() { return Ok(vec![]); } - let mut graph: std::collections::HashMap = std::collections::HashMap::new(); - let mut node_contents: std::collections::HashMap = std::collections::HashMap::new(); - - for (node, similarity) in &results { + let mut graph: std::collections::HashMap = + std::collections::HashMap::new(); + let mut node_contents: std::collections::HashMap = + std::collections::HashMap::new(); + + for (node, _similarity) in &results { if let Some(id) = &node.id { let id_str = id.to_string(); let graph_node = GraphNode::new(id_str.clone(), node.namespace.clone()); @@ -202,28 +215,33 @@ impl FractalModelStrategy { if graph.len() > 1 { let sssp = Sssp::with_defaults(); - let start_node = results.first() + let start_node = results + .first() .and_then(|(n, _)| n.id.as_ref()) .map(|id| id.to_string()) .unwrap_or_default(); if !start_node.is_empty() { let sssp_result = sssp.compute(&graph, &start_node, None); - + let mut ranked: Vec<(String, f32)> = node_contents .keys() .map(|id| { - let base_sim = results.iter() - .find(|(n, _)| n.id.as_ref().map(|i| i.to_string()) == *id) + let base_sim = results + .iter() + .find(|(n, _)| n.id.as_ref().map(|i| i.to_string()) == Some(id.clone())) .map(|(_, s)| *s) .unwrap_or(0.5); - - let graph_score = sssp_result.distances.get(id) + + let graph_score = sssp_result + .distances + .get(id) .map(|&d| 1.0 / (1.0 + d)) .unwrap_or(0.0); - + // Usar pesos configurables - let combined = base_sim * self.config.vector_weight + graph_score * self.config.graph_weight; + let combined = base_sim * self.config.vector_weight + + graph_score * self.config.graph_weight; (id.clone(), combined) }) .collect(); @@ -252,14 +270,14 @@ impl FractalModelStrategy { async fn generate_summary_with_context(&self, text: &str) -> Result { use super::providers::OllamaSummarizer; use super::traits_llm::SummarizerProvider; - + let provider = OllamaSummarizer::new( self.config.ollama_base_url.clone(), self.model_id.clone(), self.config.summarizer_temperature, self.config.summarizer_max_tokens, ); - + provider.summarize(text).await } } @@ -267,35 +285,49 @@ impl FractalModelStrategy { #[async_trait] impl ModelStrategy for FractalModelStrategy { async fn embed_batch(&self, texts: Vec) -> Result> { + use crate::embeddings::config::EmbeddingConfig; use crate::embeddings::EmbeddingService; - use crate::embeddings::provider::EmbeddingProvider; - - let provider = EmbeddingService::with_nomic_embed().await?; + use crate::models::EmbeddingModel; + + let config = EmbeddingConfig { + model: EmbeddingModel::NomicEmbedTextV15, + batch_size: 32, + normalize: true, + cache_dir: None, + device: crate::embeddings::config::EmbeddingDevice::Cpu, + }; + let provider = EmbeddingService::with_mock(config); let embeddings = provider.embed_batch(&texts).await?; - - Ok(embeddings.into_iter().map(|emb| EmbeddingResponse { - embedding: emb, - dimension: emb.len(), - model: self.model_id.clone(), - latency_ms: 0, - }).collect()) + + Ok(embeddings + .embeddings + .into_iter() + .map(|emb| EmbeddingResponse { + embedding: emb.vector, + dimension: emb.dimension, + model: self.model_id.clone(), + latency_ms: 0, + }) + .collect()) } async fn chat(&self, messages: Vec) -> Result { - let query = messages.last() + let query = messages + .last() .map(|m| m.content.clone()) .unwrap_or_default(); // Generar embedding para la query let query_embeddings = self.embed_batch(vec![query.clone()]).await?; - + // Navegar el grafo para obtener contexto let context = if let Some(embedding) = query_embeddings.first() { self.navigate_fractal_graph( &embedding.embedding, self.get_default_namespace(), self.config.max_results, - ).await? + ) + .await? } else { vec![] }; @@ -311,49 +343,38 @@ impl ModelStrategy for FractalModelStrategy { }; // Construir mensajes con system prompt - let mut enriched_messages = vec![ChatMessage { - role: "system".to_string(), - content: system_prompt, - }]; - + let mut enriched_messages = vec![ChatMessage::system(system_prompt)]; enriched_messages.extend(messages); // Usar Ollama para generar respuesta use super::providers::OllamaChat; use super::traits_llm::ChatProvider; - + let provider = OllamaChat::new( self.config.ollama_base_url.clone(), self.model_id.clone(), self.config.chat_temperature, self.config.chat_max_tokens, ); - + provider.chat(&enriched_messages).await } async fn summarize(&self, text: &str) -> Result { - use crate::graph::similarity::cosine_similarity; - // Generar embedding para el texto let embeddings = self.embed_batch(vec![text.to_string()]).await?; - + // Buscar contexto relacionado en el grafo if let Some(embedding) = embeddings.first() { - let context = self.navigate_fractal_graph( - &embedding.embedding, - self.get_default_namespace(), - 3, - ).await?; + let context = self + .navigate_fractal_graph(&embedding.embedding, self.get_default_namespace(), 3) + .await?; if !context.is_empty() { // Enriquecer texto con contexto relacionado - let enriched_text = format!( - "{}\n\nContexto relacionado:\n{}", - text, - context.join("\n") - ); - + let enriched_text = + format!("{}\n\nContexto relacionado:\n{}", text, context.join("\n")); + return self.generate_summary_with_context(&enriched_text).await; } } @@ -377,14 +398,14 @@ pub struct OllamaModelStrategy { model_name: String, api_key: Option, temperature: f32, - max_tokens: usize, + max_tokens: u32, } impl OllamaModelStrategy { pub fn new(base_url: String, model_name: String) -> Self { - Self { - base_url, - model_name, + Self { + base_url, + model_name, api_key: None, temperature: 0.7, max_tokens: 2048, @@ -392,9 +413,9 @@ impl OllamaModelStrategy { } pub fn with_api_key(base_url: String, model_name: String, api_key: String) -> Self { - Self { - base_url, - model_name, + Self { + base_url, + model_name, api_key: Some(api_key), temperature: 0.7, max_tokens: 2048, @@ -402,14 +423,14 @@ impl OllamaModelStrategy { } pub fn with_config( - base_url: String, + base_url: String, model_name: String, temperature: f32, - max_tokens: usize, + max_tokens: u32, ) -> Self { - Self { - base_url, - model_name, + Self { + base_url, + model_name, api_key: None, temperature, max_tokens, @@ -422,7 +443,7 @@ impl ModelStrategy for OllamaModelStrategy { async fn embed_batch(&self, texts: Vec) -> Result> { use super::providers::OllamaEmbedding; use super::traits_llm::EmbeddingProvider; - + let provider = if let Some(key) = &self.api_key { OllamaEmbedding::with_api_key( self.base_url.clone(), @@ -433,21 +454,16 @@ impl ModelStrategy for OllamaModelStrategy { } else { OllamaEmbedding::new(self.base_url.clone(), self.model_name.clone(), 768) }; - + let embeddings = provider.embed_batch(&texts).await?; - - Ok(embeddings.into_iter().map(|emb| EmbeddingResponse { - embedding: emb, - dimension: emb.len(), - model: self.model_name.clone(), - latency_ms: 0, - }).collect()) + + Ok(embeddings) } async fn chat(&self, messages: Vec) -> Result { use super::providers::OllamaChat; use super::traits_llm::ChatProvider; - + let provider = if let Some(key) = &self.api_key { OllamaChat::with_api_key( self.base_url.clone(), @@ -464,14 +480,14 @@ impl ModelStrategy for OllamaModelStrategy { self.max_tokens, ) }; - - provider.chat(messages).await + + provider.chat(&messages).await } async fn summarize(&self, text: &str) -> Result { use super::providers::OllamaSummarizer; use super::traits_llm::SummarizerProvider; - + let provider = if let Some(key) = &self.api_key { OllamaSummarizer::with_api_key( self.base_url.clone(), @@ -481,14 +497,9 @@ impl ModelStrategy for OllamaModelStrategy { key.clone(), ) } else { - OllamaSummarizer::new( - self.base_url.clone(), - self.model_name.clone(), - 0.3, - 512, - ) + OllamaSummarizer::new(self.base_url.clone(), self.model_name.clone(), 0.3, 512) }; - + provider.summarize(text).await } @@ -520,7 +531,7 @@ mod tests { .with_namespace("user_alice") .with_max_results(10) .with_weights(0.8, 0.2); - + assert_eq!(config.default_namespace, "user_alice"); assert_eq!(config.max_results, 10); assert!((config.vector_weight - 0.8).abs() < f32::EPSILON); @@ -529,17 +540,19 @@ mod tests { #[test] fn test_fractal_strategy_creation() { - let db = crate::db::connection::DatabaseConnection::default(); + use surrealdb::engine::remote::http::Client; + use surrealdb::Surreal; + + let rt = tokio::runtime::Runtime::new().unwrap(); + let db = rt.block_on(async { Surreal::::init() }); let strategy = FractalModelStrategy::new("model:123".to_string(), db); assert_eq!(strategy.name(), "FractalModel"); } #[test] fn test_ollama_strategy_creation() { - let strategy = OllamaModelStrategy::new( - "http://localhost:11434".to_string(), - "llama2".to_string(), - ); + let strategy = + OllamaModelStrategy::new("http://localhost:11434".to_string(), "llama2".to_string()); assert_eq!(strategy.name(), "Ollama"); } diff --git a/src/models/llm/traits_llm.rs b/src/models/llm/traits_llm.rs index ca225b4..c59f586 100644 --- a/src/models/llm/traits_llm.rs +++ b/src/models/llm/traits_llm.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] -use async_trait::async_trait; use anyhow::Result; +use async_trait::async_trait; /// Mensaje de chat #[derive(Debug, Clone)] diff --git a/src/models/mod.rs b/src/models/mod.rs index bdb0a4b..9532f94 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -1,12 +1,12 @@ -pub mod node; pub mod edge; -pub mod namespace; pub mod embedding; pub mod llm; +pub mod namespace; +pub mod node; pub mod upload_session; -pub use node::{FractalNode, NodeStatus, NodeType, NodeMetadata, SourceType}; -pub use edge::{FractalEdge, EdgeType, GraphPath}; -pub use embedding::{EmbeddingVector, EmbeddingModel}; +pub use edge::{EdgeType, FractalEdge, GraphPath}; +pub use embedding::{EmbeddingModel, EmbeddingVector}; pub use namespace::{Namespace, NamespaceType, Scope, ScopePermissions}; +pub use node::{FractalNode, NodeMetadata, NodeStatus, NodeType, SourceType}; pub use upload_session::{UploadSession, UploadStatus}; diff --git a/src/models/namespace.rs b/src/models/namespace.rs index 8b2ae35..b28e60f 100644 --- a/src/models/namespace.rs +++ b/src/models/namespace.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] -use serde::{Deserialize, Serialize}; use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; /// Namespace para separación de memoria global vs personal #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] @@ -179,10 +179,7 @@ mod tests { #[test] fn test_scope_full_access() { - let scope = Scope::new_full_access( - "alice".to_string(), - "user_alice".to_string(), - ); + let scope = Scope::new_full_access("alice".to_string(), "user_alice".to_string()); assert!(scope.can_write()); assert!(scope.can_delete()); @@ -191,10 +188,7 @@ mod tests { #[test] fn test_scope_read_only() { - let scope = Scope::new_read_only( - "bob".to_string(), - "global_knowledge".to_string(), - ); + let scope = Scope::new_read_only("bob".to_string(), "global_knowledge".to_string()); assert!(!scope.can_write()); assert!(!scope.can_delete()); @@ -205,10 +199,7 @@ mod tests { fn test_scope_expiration() { use chrono::Duration; - let mut scope = Scope::new_full_access( - "alice".to_string(), - "user_alice".to_string(), - ); + let mut scope = Scope::new_full_access("alice".to_string(), "user_alice".to_string()); // Scope sin expiración assert!(!scope.is_expired()); diff --git a/src/models/node.rs b/src/models/node.rs index b7198e8..425de33 100644 --- a/src/models/node.rs +++ b/src/models/node.rs @@ -1,8 +1,8 @@ #![allow(dead_code)] +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use surrealdb::sql::Thing; -use chrono::{DateTime, Utc}; use uuid::Uuid; use super::embedding::EmbeddingVector; @@ -157,7 +157,11 @@ pub struct FractalNode { pub updated_at: DateTime, /// Timestamp de última consulta (para cache LRU) - #[serde(default, skip_serializing_if = "Option::is_none", with = "datetime_string_option")] + #[serde( + default, + skip_serializing_if = "Option::is_none", + with = "datetime_string_option" + )] pub last_accessed_at: Option>, } diff --git a/src/models/upload_session.rs b/src/models/upload_session.rs index 669346e..143bc80 100644 --- a/src/models/upload_session.rs +++ b/src/models/upload_session.rs @@ -34,49 +34,49 @@ impl UploadStatus { pub struct UploadSession { /// Unique identifier for this upload session pub upload_id: String, - + /// Original filename pub filename: String, - + /// Total file size in bytes pub total_size: u64, - + /// Size of each chunk in bytes pub chunk_size: u64, - + /// Total number of chunks pub total_chunks: u64, - + /// Set of chunk indices that have been received pub chunks_received: Vec, - + /// Current status of the upload pub status: UploadStatus, - + /// Path to the temporary upload file pub temp_path: String, - + /// Model ID (set after finalization) pub model_id: Option, - + /// Upload progress (0-100) pub upload_progress: f32, - + /// Conversion progress (0-100) pub conversion_progress: f32, - + /// Current conversion phase pub current_phase: Option, - + /// Upload speed in MB/s (calculated from recent chunks) pub upload_speed_mbps: Option, - + /// Timestamp of last chunk received pub last_chunk_at: Option>, - + /// Creation timestamp pub created_at: DateTime, - + /// Last update timestamp pub updated_at: DateTime, } @@ -85,7 +85,7 @@ impl UploadSession { /// Create a new upload session pub fn new(filename: String, total_size: u64, chunk_size: u64) -> Self { let upload_id = format!("upload_{}", Uuid::new_v4().to_string().replace("-", "")); - let total_chunks = (total_size + chunk_size - 1) / chunk_size; + let total_chunks = total_size.div_ceil(chunk_size); let temp_path = format!("/var/tmp/fractalmind_uploads/{}.part", upload_id); let now = Utc::now(); @@ -115,8 +115,9 @@ impl UploadSession { self.chunks_received.push(chunk_index); self.chunks_received.sort(); } - - self.upload_progress = (self.chunks_received.len() as f32 / self.total_chunks as f32) * 100.0; + + self.upload_progress = + (self.chunks_received.len() as f32 / self.total_chunks as f32) * 100.0; self.last_chunk_at = Some(Utc::now()); self.updated_at = Utc::now(); } @@ -181,7 +182,7 @@ mod tests { let mut session = UploadSession::new("test.gguf".to_string(), 1000, 100); session.add_chunk(0); session.add_chunk(1); - + assert_eq!(session.chunks_received.len(), 2); assert_eq!(session.upload_progress, 20.0); } @@ -190,11 +191,11 @@ mod tests { fn test_is_complete() { let mut session = UploadSession::new("test.gguf".to_string(), 1000, 100); assert!(!session.is_complete()); - + for i in 0..10 { session.add_chunk(i); } - + assert!(session.is_complete()); assert_eq!(session.upload_progress, 100.0); } @@ -205,7 +206,7 @@ mod tests { session.add_chunk(0); session.add_chunk(2); session.add_chunk(4); - + let missing = session.missing_chunks(); assert_eq!(missing, vec![1, 3, 5, 6, 7, 8, 9]); } diff --git a/src/services/config.rs b/src/services/config.rs index e990fcc..9c22e89 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -285,10 +285,7 @@ mod tests { assert_eq!(config.provider, "tavily"); assert_eq!(config.api_key, Some("test-key".to_string())); - assert_eq!( - config.base_url, - Some("https://api.tavily.com".to_string()) - ); + assert_eq!(config.base_url, Some("https://api.tavily.com".to_string())); assert_eq!(config.max_results, 10); } diff --git a/src/services/fractal_builder.rs b/src/services/fractal_builder.rs index bb96fbb..762b35e 100644 --- a/src/services/fractal_builder.rs +++ b/src/services/fractal_builder.rs @@ -8,13 +8,13 @@ use std::time::Instant; use anyhow::{Context, Result}; use surrealdb::sql::Thing; -use tracing::{info, warn, debug}; +use tracing::{debug, info, warn}; use crate::db::connection::DatabaseConnection; -use crate::db::queries::{NodeRepository, EdgeRepository}; +use crate::db::queries::{EdgeRepository, NodeRepository}; use crate::graph::{Raptor, RaptorConfig, RaptorNode, RaptorTree, RaptorTreeNode}; -use crate::models::{FractalNode, FractalEdge, NodeMetadata, SourceType}; use crate::models::llm::ModelBrain; +use crate::models::{FractalEdge, FractalNode, NodeMetadata, SourceType}; /// Configuration for fractal building #[derive(Debug, Clone)] @@ -105,11 +105,13 @@ impl<'a> FractalBuilder<'a> { // 1. Fetch all leaf nodes in the namespace let leaf_nodes = self.fetch_leaf_nodes(namespace).await?; - + if leaf_nodes.len() < self.config.min_nodes_for_fractal { info!( "Skipping fractal build for namespace '{}': only {} nodes (min: {})", - namespace, leaf_nodes.len(), self.config.min_nodes_for_fractal + namespace, + leaf_nodes.len(), + self.config.min_nodes_for_fractal ); return Ok(FractalBuildResult { parent_nodes_created: 0, @@ -120,19 +122,18 @@ impl<'a> FractalBuilder<'a> { }); } - info!("Building fractal structure for {} leaf nodes in namespace '{}'", - leaf_nodes.len(), namespace); + info!( + "Building fractal structure for {} leaf nodes in namespace '{}'", + leaf_nodes.len(), + namespace + ); // 2. Convert to RAPTOR nodes let raptor_nodes: Vec = leaf_nodes .iter() .filter_map(|node| { node.id.as_ref().map(|id| { - RaptorNode::new( - id.to_string(), - node.content.clone(), - node.embedding.clone(), - ) + RaptorNode::new(id.to_string(), node.content.clone(), node.embedding.clone()) }) }) .collect(); @@ -143,9 +144,7 @@ impl<'a> FractalBuilder<'a> { info!( "RAPTOR tree built: {} clusters, max_depth={}, time={}ms", - raptor_tree.stats.total_clusters, - raptor_tree.max_depth, - raptor_tree.build_time_ms + raptor_tree.stats.total_clusters, raptor_tree.max_depth, raptor_tree.build_time_ms ); // 4. Create parent nodes and edges from RAPTOR tree @@ -181,7 +180,8 @@ impl<'a> FractalBuilder<'a> { AND depth_level = 0 "#; - let mut result = self.db + let mut result = self + .db .query(query) .bind(("namespace", namespace)) .await @@ -226,7 +226,11 @@ impl<'a> FractalBuilder<'a> { .filter(|(_, n)| n.depth == depth) .collect(); - debug!("Processing {} nodes at depth {}", nodes_at_depth.len(), depth); + debug!( + "Processing {} nodes at depth {}", + nodes_at_depth.len(), + depth + ); for (cluster_id, tree_node) in nodes_at_depth { // Generate summary if enabled and brain is available @@ -237,12 +241,8 @@ impl<'a> FractalBuilder<'a> { }; // Create parent node - let parent_node = self.create_parent_node( - tree_node, - namespace, - depth as u32, - summary, - )?; + let parent_node = + self.create_parent_node(tree_node, namespace, depth as u32, summary)?; let parent_id = node_repo.create(&parent_node).await?; parent_count += 1; @@ -278,7 +278,9 @@ impl<'a> FractalBuilder<'a> { } // Create semantic edges between siblings (nodes with same parent) - let sibling_edges = self.create_sibling_edges(tree, &cluster_to_node_id, edge_repo).await?; + let sibling_edges = self + .create_sibling_edges(tree, &cluster_to_node_id, edge_repo) + .await?; edge_count += sibling_edges; Ok((parent_count, edge_count, root_ids)) @@ -319,14 +321,16 @@ impl<'a> FractalBuilder<'a> { } }; - let mut metadata = NodeMetadata::default(); - metadata.source = "fractal_builder".to_string(); - metadata.source_type = SourceType::Synthetic; - metadata.tags = vec![ - format!("depth:{}", depth), - format!("children:{}", tree_node.children.len()), - format!("cluster:{}", tree_node.cluster_id), - ]; + let metadata = NodeMetadata { + source: "fractal_builder".to_string(), + source_type: SourceType::Synthetic, + tags: vec![ + format!("depth:{}", depth), + format!("children:{}", tree_node.children.len()), + format!("cluster:{}", tree_node.cluster_id), + ], + ..NodeMetadata::default() + }; Ok(FractalNode::new_parent( summary.unwrap_or_else(|| "Cluster summary".to_string()), @@ -385,10 +389,7 @@ impl<'a> FractalBuilder<'a> { tree: &RaptorTree, ) -> f32 { if let Some(child) = tree.nodes.get(child_cluster_id) { - crate::graph::similarity::cosine_similarity( - &parent.centroid, - &child.centroid, - ) + crate::graph::similarity::cosine_similarity(&parent.centroid, &child.centroid) } else { 0.8 // Default high similarity for parent-child } @@ -429,8 +430,8 @@ impl<'a> FractalBuilder<'a> { ); if similarity >= similarity_threshold { - if let (Some(node_a), Some(node_b)) = - (cluster_to_node_id.get(id_a), cluster_to_node_id.get(id_b)) + if let (Some(node_a), Some(node_b)) = + (cluster_to_node_id.get(id_a), cluster_to_node_id.get(id_b)) { let edge = FractalEdge::new_semantic( node_a.clone(), @@ -465,7 +466,7 @@ mod tests { let config = FractalBuilderConfig::new() .with_summaries(false) .with_min_nodes(10); - + assert!(!config.generate_summaries); assert_eq!(config.min_nodes_for_fractal, 10); } diff --git a/src/services/ingestion/chunker.rs b/src/services/ingestion/chunker.rs index 468b8d0..f80c7bc 100644 --- a/src/services/ingestion/chunker.rs +++ b/src/services/ingestion/chunker.rs @@ -113,7 +113,11 @@ impl TextChunker { /// Creates a chunker from configuration. pub fn from_config(config: &IngestionConfig) -> Self { - Self::new(config.chunk_size, config.chunk_overlap, config.min_chunk_size) + Self::new( + config.chunk_size, + config.chunk_overlap, + config.min_chunk_size, + ) } /// Creates a chunker with default parameters. @@ -149,7 +153,7 @@ impl TextChunker { while start < text.len() { // Ensure start is at a valid UTF-8 boundary start = self.find_char_boundary(text, start); - + if start >= text.len() { break; } @@ -375,9 +379,7 @@ impl TextChunker { { // Handle abbreviations and decimals let is_abbreviation = self.is_likely_abbreviation(¤t); - let is_decimal = c == '.' - && i + 1 < chars.len() - && chars[i + 1].is_ascii_digit(); + let is_decimal = c == '.' && i + 1 < chars.len() && chars[i + 1].is_ascii_digit(); if !is_abbreviation && !is_decimal { sentences.push(current.trim().to_string()); @@ -428,28 +430,28 @@ impl TextChunker { /// Finds the nearest valid UTF-8 character boundary at or before the given position. fn find_char_boundary(&self, text: &str, pos: usize) -> usize { let pos = pos.min(text.len()); - + // Walk backwards to find a valid UTF-8 boundary for i in (0..=pos).rev() { if text.is_char_boundary(i) { return i; } } - + 0 } /// Finds the nearest valid UTF-8 character boundary at or after the given position. fn find_next_char_boundary(&self, text: &str, pos: usize) -> usize { let pos = pos.min(text.len()); - + // Walk forward to find a valid UTF-8 boundary for i in pos..=text.len() { if text.is_char_boundary(i) { return i; } } - + text.len() } } @@ -548,7 +550,8 @@ mod tests { #[test] fn test_chunk_with_source() { - let chunk = TextChunk::new("Content".to_string(), 0, 1, 0, 7).with_source("doc.pdf".to_string()); + let chunk = + TextChunk::new("Content".to_string(), 0, 1, 0, 7).with_source("doc.pdf".to_string()); assert_eq!(chunk.source, Some("doc.pdf".to_string())); } @@ -613,7 +616,7 @@ mod tests { // Should not panic assert!(result.count() >= 1); - + // Verify we can iterate and access content without panics for chunk in &result.chunks { let _ = chunk.content.len(); diff --git a/src/services/ingestion/config.rs b/src/services/ingestion/config.rs index dbcdb0e..cde9275 100644 --- a/src/services/ingestion/config.rs +++ b/src/services/ingestion/config.rs @@ -213,7 +213,9 @@ impl IngestionConfig { /// Checks if a file extension is allowed. pub fn is_extension_allowed(&self, ext: &str) -> bool { - self.allowed_extensions.iter().any(|e| e.eq_ignore_ascii_case(ext)) + self.allowed_extensions + .iter() + .any(|e| e.eq_ignore_ascii_case(ext)) } /// Checks if a file size is within limits. @@ -276,7 +278,10 @@ mod tests { assert_eq!(FileType::from_mime("application/pdf"), FileType::Pdf); assert_eq!(FileType::from_mime("image/png"), FileType::Image); assert_eq!(FileType::from_mime("image/jpeg"), FileType::Image); - assert_eq!(FileType::from_mime("application/octet-stream"), FileType::Unknown); + assert_eq!( + FileType::from_mime("application/octet-stream"), + FileType::Unknown + ); } #[test] @@ -317,17 +322,26 @@ mod tests { // Test validation with chunk size below minimum (bypass builder's enforcement) let mut config = IngestionConfig::new(); config.chunk_size = 30; - assert!(matches!(config.validate(), Err(ConfigError::ChunkSizeTooSmall))); + assert!(matches!( + config.validate(), + Err(ConfigError::ChunkSizeTooSmall) + )); // Test overlap too large let mut config = IngestionConfig::new(); config.chunk_overlap = config.chunk_size + 1; - assert!(matches!(config.validate(), Err(ConfigError::OverlapTooLarge))); + assert!(matches!( + config.validate(), + Err(ConfigError::OverlapTooLarge) + )); // Test min chunk too large let mut config = IngestionConfig::new(); config.min_chunk_size = config.chunk_size + 1; - assert!(matches!(config.validate(), Err(ConfigError::MinChunkTooLarge))); + assert!(matches!( + config.validate(), + Err(ConfigError::MinChunkTooLarge) + )); } #[test] diff --git a/src/services/ingestion/extractors/image.rs b/src/services/ingestion/extractors/image.rs index ceb2520..c12fca3 100644 --- a/src/services/ingestion/extractors/image.rs +++ b/src/services/ingestion/extractors/image.rs @@ -25,7 +25,7 @@ impl ImageExtractor { Self { language: "eng".to_string(), max_size: 50 * 1024 * 1024, // 50MB - page_seg_mode: 3, // Fully automatic page segmentation + page_seg_mode: 3, // Fully automatic page segmentation } } @@ -94,7 +94,8 @@ impl ImageExtractor { } // TIFF (little-endian and big-endian) - if data.starts_with(&[0x49, 0x49, 0x2A, 0x00]) || data.starts_with(&[0x4D, 0x4D, 0x00, 0x2A]) + if data.starts_with(&[0x49, 0x49, 0x2A, 0x00]) + || data.starts_with(&[0x4D, 0x4D, 0x00, 0x2A]) { return Some("image/tiff"); } @@ -121,9 +122,7 @@ impl ImageExtractor { .map_err(|e| anyhow!("Failed to set image: {}", e))?; // Perform OCR - let text = tess - .get_text() - .map_err(|e| anyhow!("OCR failed: {}", e))?; + let text = tess.get_text().map_err(|e| anyhow!("OCR failed: {}", e))?; Ok(self.clean_ocr_text(&text)) } @@ -261,11 +260,17 @@ mod tests { // PNG let png_header = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; - assert_eq!(extractor.detect_image_format(&png_header), Some("image/png")); + assert_eq!( + extractor.detect_image_format(&png_header), + Some("image/png") + ); // JPEG let jpg_header = [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46]; - assert_eq!(extractor.detect_image_format(&jpg_header), Some("image/jpeg")); + assert_eq!( + extractor.detect_image_format(&jpg_header), + Some("image/jpeg") + ); // GIF let gif_header = b"GIF89a\x00\x00"; diff --git a/src/services/ingestion/extractors/pdf.rs b/src/services/ingestion/extractors/pdf.rs index 1459286..5ccc878 100644 --- a/src/services/ingestion/extractors/pdf.rs +++ b/src/services/ingestion/extractors/pdf.rs @@ -127,7 +127,10 @@ impl ContentExtractor for PdfExtractor { let mut result = ExtractionResult::new(text, FileType::Pdf).with_metadata(metadata); if result.text.is_empty() { - result.add_warning("PDF appears to contain no extractable text (may be scanned/image-based)".to_string()); + result.add_warning( + "PDF appears to contain no extractable text (may be scanned/image-based)" + .to_string(), + ); } Ok(result) diff --git a/src/services/ingestion/extractors/text.rs b/src/services/ingestion/extractors/text.rs index a8e7784..cb282e9 100644 --- a/src/services/ingestion/extractors/text.rs +++ b/src/services/ingestion/extractors/text.rs @@ -86,7 +86,7 @@ impl TextExtractor { result.push(c); prev_was_newline = true; prev_was_space = false; - } else if result.chars().last() != Some('\n') { + } else if !result.ends_with('\n') { result.push(c); } } else if c.is_whitespace() { @@ -182,7 +182,9 @@ impl ContentExtractor for TextExtractor { // Add warning if lossy conversion occurred if String::from_utf8(data.to_vec()).is_err() { - result.add_warning("Some characters were replaced during encoding conversion".to_string()); + result.add_warning( + "Some characters were replaced during encoding conversion".to_string(), + ); } Ok(result) @@ -287,10 +289,16 @@ mod tests { ); // UTF-16 LE BOM - assert_eq!(extractor.detect_encoding(&[0xFF, 0xFE, 0x00, 0x00]), "utf-16-le"); + assert_eq!( + extractor.detect_encoding(&[0xFF, 0xFE, 0x00, 0x00]), + "utf-16-le" + ); // UTF-16 BE BOM - assert_eq!(extractor.detect_encoding(&[0xFE, 0xFF, 0x00, 0x00]), "utf-16-be"); + assert_eq!( + extractor.detect_encoding(&[0xFE, 0xFF, 0x00, 0x00]), + "utf-16-be" + ); // No BOM (default UTF-8) assert_eq!(extractor.detect_encoding(b"Hello"), "utf-8"); diff --git a/src/services/ingestion/service.rs b/src/services/ingestion/service.rs index ecdef79..ba681e6 100644 --- a/src/services/ingestion/service.rs +++ b/src/services/ingestion/service.rs @@ -223,7 +223,9 @@ impl IngestionService { info!( "Chunked {} chars into {} chunks (avg {} chars)", - chunking.original_length, chunking.chunks.len(), chunking.avg_chunk_size + chunking.original_length, + chunking.chunks.len(), + chunking.avg_chunk_size ); // Generate nodes @@ -257,12 +259,9 @@ impl IngestionService { F: Fn(&str) -> EmbeddingVector, { let data = tokio::fs::read(path).await?; - let filename = path - .file_name() - .map(|s| s.to_string_lossy().to_string()); + let filename = path.file_name().map(|s| s.to_string_lossy().to_string()); - let mut input = IngestionInput::new(data, namespace) - .with_source(&path.to_string_lossy()); + let mut input = IngestionInput::new(data, namespace).with_source(&path.to_string_lossy()); if let Some(name) = filename { input = input.with_filename(&name); @@ -334,8 +333,10 @@ impl IngestionService { let embedding = embedding_generator(&chunk.content); // Create metadata - let mut metadata = NodeMetadata::default(); - metadata.tags = input.tags.clone(); + let mut metadata = NodeMetadata { + tags: input.tags.clone(), + ..NodeMetadata::default() + }; if let Some(src) = &input.source { metadata.source = src.clone(); @@ -347,7 +348,9 @@ impl IngestionService { // Add chunk info to tags if chunks.len() > 1 { - metadata.tags.push(format!("chunk:{}/{}", chunk.index + 1, chunk.total)); + metadata + .tags + .push(format!("chunk:{}/{}", chunk.index + 1, chunk.total)); } // Create node @@ -515,8 +518,8 @@ mod tests { #[tokio::test] async fn test_ingest_bytes() { let service = IngestionService::with_defaults(); - let input = IngestionInput::new(b"Hello, world!".to_vec(), "global") - .with_filename("test.txt"); + let input = + IngestionInput::new(b"Hello, world!".to_vec(), "global").with_filename("test.txt"); let result = service.ingest(input, mock_embedding).await.unwrap(); diff --git a/src/services/mod.rs b/src/services/mod.rs index 9fc95e4..324c9d2 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -38,22 +38,22 @@ #![allow(dead_code)] pub mod config; +pub mod fractal_builder; pub mod ingestion; +pub mod model_conversion; pub mod rem_phase; pub mod rem_scheduler; pub mod storage; pub mod upload; pub mod web_search; -pub mod model_conversion; -pub mod fractal_builder; // Re-exports pub use config::{RemPhaseConfig, WebSearchConfig}; -pub use fractal_builder::{FractalBuilder, FractalBuilderConfig, FractalBuildResult}; +pub use fractal_builder::{FractalBuildResult, FractalBuilder, FractalBuilderConfig}; pub use rem_phase::{ RemPhaseResult, RemPhaseService, RemPhaseServiceBuilder, RemPhaseStatus, SearchStats, }; -pub use rem_scheduler::{RemScheduler, RemSchedulerConfig, RemSchedulerStatus, RemRunResult}; +pub use rem_scheduler::{RemRunResult, RemScheduler, RemSchedulerConfig, RemSchedulerStatus}; pub use web_search::{ MockSearchProvider, SearchResponse, SearchResult, WebSearchFactory, WebSearchProvider, }; @@ -69,5 +69,6 @@ pub use model_conversion::ModelConversionService; // Storage and upload re-exports pub use storage::StorageManager; -pub use upload::{UploadSessionManager, UploadConfig, UploadCleanupJob, ChunkResult, FinalizeResult}; - +pub use upload::{ + ChunkResult, FinalizeResult, UploadCleanupJob, UploadConfig, UploadSessionManager, +}; diff --git a/src/services/model_conversion.rs b/src/services/model_conversion.rs index c29b6b7..f922946 100644 --- a/src/services/model_conversion.rs +++ b/src/services/model_conversion.rs @@ -16,15 +16,17 @@ use std::io::Cursor; use std::path::Path; use std::sync::Arc; use tokio::fs; -use tracing::{info, error, debug}; +use tracing::{debug, error, info}; use crate::db::connection::DatabaseConnection; use crate::db::queries::{FractalModelNodeRepository, FractalModelRepository}; -use crate::graph::raptor::{Raptor, RaptorNode}; use crate::graph::config::RaptorConfig; -use crate::models::llm::fractal_model::{FractalModel, FractalModelNode, FractalModelStatus, LayerInfo, ModelArchitecture}; +use crate::graph::raptor::{Raptor, RaptorNode}; +use crate::models::llm::fractal_model::{ + FractalModel, FractalModelNode, FractalModelStatus, LayerInfo, ModelArchitecture, +}; use crate::models::llm::gguf_parser::{GGUFParser, GGUFTensorInfo}; -use crate::models::{EmbeddingVector, EmbeddingModel}; +use crate::models::{EmbeddingModel, EmbeddingVector}; /// GGUF quantization types and their sizes #[derive(Debug, Clone, Copy)] @@ -88,7 +90,7 @@ pub struct ModelConversionService { impl ModelConversionService { pub fn new(db: Arc) -> Self { - Self { + Self { db, embedding_dim: 768, } @@ -100,7 +102,10 @@ impl ModelConversionService { /// Main conversion function - converts a GGUF model to fractal structure pub async fn convert_model(&self, model: &mut FractalModel) -> Result<()> { - info!("Starting REAL conversion of model: {} ({})", model.name, model.id); + info!( + "Starting REAL conversion of model: {} ({})", + model.name, model.id + ); let model_id = model.id.clone(); let file_path = model.file_path.clone(); @@ -116,7 +121,8 @@ impl ModelConversionService { model.update_conversion_progress(5.0, Some("Parsing GGUF header".to_string())); self.save_model(model).await?; - let (tensors, architecture, data_offset) = match self.parse_gguf_structure(&file_path).await { + let (tensors, architecture, data_offset) = match self.parse_gguf_structure(&file_path).await + { Ok(result) => result, Err(e) => { error!("Failed to parse GGUF file: {}", e); @@ -131,8 +137,13 @@ impl ModelConversionService { model.update_conversion_progress(10.0, Some("Architecture extracted".to_string())); self.save_model(model).await?; repo.update_architecture(&model_id, &architecture).await?; - - info!("Found {} tensors, {} layers, data offset: {}", tensors.len(), architecture.n_layers, data_offset); + + info!( + "Found {} tensors, {} layers, data offset: {}", + tensors.len(), + architecture.n_layers, + data_offset + ); // Phase 2: Group tensors (10-20%) info!("Phase 2: Grouping tensors by layer"); @@ -140,8 +151,11 @@ impl ModelConversionService { self.save_model(model).await?; let layer_groups = self.group_tensors_by_layer(&tensors, architecture.n_layers); - - model.update_conversion_progress(20.0, Some(format!("{} layer groups created", layer_groups.len()))); + + model.update_conversion_progress( + 20.0, + Some(format!("{} layer groups created", layer_groups.len())), + ); self.save_model(model).await?; info!("Created {} layer groups", layer_groups.len()); @@ -150,12 +164,10 @@ impl ModelConversionService { model.update_conversion_progress(25.0, Some("Generating embeddings".to_string())); self.save_model(model).await?; - let layer_embeddings = match self.generate_layer_embeddings_blocking( - &file_path, - &tensors, - &layer_groups, - data_offset, - ).await { + let layer_embeddings = match self + .generate_layer_embeddings_blocking(&file_path, &tensors, &layer_groups, data_offset) + .await + { Ok(embeddings) => embeddings, Err(e) => { error!("Failed to generate embeddings: {}", e); @@ -166,7 +178,10 @@ impl ModelConversionService { } }; - model.update_conversion_progress(55.0, Some(format!("{} embeddings generated", layer_embeddings.len()))); + model.update_conversion_progress( + 55.0, + Some(format!("{} embeddings generated", layer_embeddings.len())), + ); self.save_model(model).await?; info!("Generated {} layer embeddings", layer_embeddings.len()); @@ -210,13 +225,16 @@ impl ModelConversionService { let raptor = Raptor::new(raptor_config); let raptor_tree = raptor.build_tree(raptor_nodes); - model.update_conversion_progress(75.0, Some(format!( - "RAPTOR tree: {} nodes, depth {}", - raptor_tree.nodes.len(), - raptor_tree.max_depth - ))); + model.update_conversion_progress( + 75.0, + Some(format!( + "RAPTOR tree: {} nodes, depth {}", + raptor_tree.nodes.len(), + raptor_tree.max_depth + )), + ); self.save_model(model).await?; - + info!( "RAPTOR tree built: {} nodes, {} roots, depth {}", raptor_tree.nodes.len(), @@ -229,12 +247,10 @@ impl ModelConversionService { model.update_conversion_progress(80.0, Some("Storing nodes".to_string())); self.save_model(model).await?; - let root_node_id = match self.store_fractal_nodes( - &model_id, - &raptor_tree, - &layer_embeddings, - &node_repo, - ).await { + let root_node_id = match self + .store_fractal_nodes(&model_id, &raptor_tree, &layer_embeddings, &node_repo) + .await + { Ok(root_id) => root_id, Err(e) => { error!("Failed to store nodes: {}", e); @@ -256,9 +272,13 @@ impl ModelConversionService { model.update_status(FractalModelStatus::Ready); model.update_conversion_progress(100.0, Some("Complete".to_string())); self.save_model(model).await?; - repo.update_status(&model_id, FractalModelStatus::Ready).await?; + repo.update_status(&model_id, FractalModelStatus::Ready) + .await?; - info!("Model conversion COMPLETED for {} - root node: {}", model.name, root_node_id); + info!( + "Model conversion COMPLETED for {} - root node: {}", + model.name, root_node_id + ); Ok(()) } @@ -284,11 +304,7 @@ impl ModelConversionService { Ok(result) } - fn group_tensors_by_layer( - &self, - tensors: &[GGUFTensorInfo], - n_layers: u32, - ) -> Vec { + fn group_tensors_by_layer(&self, tensors: &[GGUFTensorInfo], n_layers: u32) -> Vec { let mut groups: Vec = Vec::new(); let mut layer_tensors: HashMap> = HashMap::new(); let mut special_tensors: Vec<&GGUFTensorInfo> = Vec::new(); @@ -306,7 +322,7 @@ impl ModelConversionService { .iter() .map(|t| t.dimensions.iter().product::()) .sum(); - + groups.push(LayerGroup { layer_start: 0, layer_end: 0, @@ -318,11 +334,11 @@ impl ModelConversionService { } let layers_per_group = std::cmp::max(1, n_layers / 6); - - for group_idx in 0..((n_layers + layers_per_group - 1) / layers_per_group) { + + for group_idx in 0..n_layers.div_ceil(layers_per_group) { let start = group_idx * layers_per_group; let end = std::cmp::min(start + layers_per_group - 1, n_layers.saturating_sub(1)); - + let mut group_tensors: Vec = Vec::new(); let mut total_params: u64 = 0; let mut has_attn = false; @@ -333,7 +349,7 @@ impl ModelConversionService { for tensor in tensors { group_tensors.push(tensor.name.clone()); total_params += tensor.dimensions.iter().product::(); - + if tensor.name.contains("attn") || tensor.name.contains("attention") { has_attn = true; } @@ -371,7 +387,7 @@ impl ModelConversionService { fn extract_layer_number(&self, name: &str) -> Option { let patterns = ["blk.", "layers.", "h.", "block.", "layer."]; - + for pattern in patterns { if let Some(pos) = name.find(pattern) { let after_pattern = &name[pos + pattern.len()..]; @@ -419,14 +435,16 @@ impl ModelConversionService { ) -> Result> { let file = File::open(file_path)?; let mmap = unsafe { Mmap::map(&file)? }; - - info!("Reading tensor data from offset {} (file size: {})", data_offset, mmap.len()); - + + info!( + "Reading tensor data from offset {} (file size: {})", + data_offset, + mmap.len() + ); + let mut result_groups: Vec = Vec::new(); - let tensor_map: HashMap<&str, &GGUFTensorInfo> = tensors - .iter() - .map(|t| (t.name.as_str(), t)) - .collect(); + let tensor_map: HashMap<&str, &GGUFTensorInfo> = + tensors.iter().map(|t| (t.name.as_str(), t)).collect(); for group in layer_groups { let embedding = Self::generate_group_embedding( @@ -465,12 +483,12 @@ impl ModelConversionService { for tensor_name in group_tensor_names { if let Some(tensor) = tensor_map.get(tensor_name.as_str()) { let samples = Self::sample_tensor_values(mmap, tensor, data_offset); - + if !samples.is_empty() { tensors_sampled += 1; total_samples += samples.len(); } - + for (i, &sample) in samples.iter().enumerate() { let idx = (i * 31 + tensor_name.len()) % embedding_dim; embedding[idx] += sample; @@ -482,8 +500,8 @@ impl ModelConversionService { // Log sampling stats for debugging if tensors_sampled == 0 { tracing::warn!( - "No samples collected from {} tensors in group! data_offset={}", - group_tensor_names.len(), + "No samples collected from {} tensors in group! data_offset={}", + group_tensor_names.len(), data_offset ); } else { @@ -512,15 +530,11 @@ impl ModelConversionService { embedding } - fn sample_tensor_values( - mmap: &Mmap, - tensor: &GGUFTensorInfo, - data_offset: u64, - ) -> Vec { + fn sample_tensor_values(mmap: &Mmap, tensor: &GGUFTensorInfo, data_offset: u64) -> Vec { let mut samples = Vec::new(); let tensor_offset = (data_offset + tensor.offset) as usize; let tensor_size: u64 = tensor.dimensions.iter().product(); - + // Check if offset is within file bounds if tensor_offset >= mmap.len() { tracing::warn!( @@ -588,26 +602,39 @@ impl ModelConversionService { samples } - fn sample_q4_tensor(mmap: &Mmap, offset: usize, size: u64, step: usize, samples: &mut Vec) { + fn sample_q4_tensor( + mmap: &Mmap, + offset: usize, + size: u64, + step: usize, + samples: &mut Vec, + ) { let block_size = 32; let bytes_per_block = 18; - let num_blocks = (size as usize + block_size - 1) / block_size; + let num_blocks = (size as usize).div_ceil(block_size); - for block_idx in (0..num_blocks).step_by(std::cmp::max(1, step / block_size)).take(64) { + for block_idx in (0..num_blocks) + .step_by(std::cmp::max(1, step / block_size)) + .take(64) + { let block_offset = offset + block_idx * bytes_per_block; if block_offset + bytes_per_block <= mmap.len() { let mut cursor = Cursor::new(&mmap[block_offset..block_offset + 2]); if let Ok(scale_bits) = cursor.read_u16::() { let scale = f16::from_bits(scale_bits).to_f32(); - + for i in 0..4 { let byte_pos = block_offset + 2 + i; if byte_pos < mmap.len() { let byte = mmap[byte_pos]; let v0 = ((byte & 0x0F) as i8 - 8) as f32 * scale; let v1 = ((byte >> 4) as i8 - 8) as f32 * scale; - if v0.is_finite() { samples.push(v0); } - if v1.is_finite() { samples.push(v1); } + if v0.is_finite() { + samples.push(v0); + } + if v1.is_finite() { + samples.push(v1); + } } } } @@ -615,24 +642,35 @@ impl ModelConversionService { } } - fn sample_q8_tensor(mmap: &Mmap, offset: usize, size: u64, step: usize, samples: &mut Vec) { + fn sample_q8_tensor( + mmap: &Mmap, + offset: usize, + size: u64, + step: usize, + samples: &mut Vec, + ) { let block_size = 32; let bytes_per_block = 34; - let num_blocks = (size as usize + block_size - 1) / block_size; + let num_blocks = (size as usize).div_ceil(block_size); - for block_idx in (0..num_blocks).step_by(std::cmp::max(1, step / block_size)).take(64) { + for block_idx in (0..num_blocks) + .step_by(std::cmp::max(1, step / block_size)) + .take(64) + { let block_offset = offset + block_idx * bytes_per_block; if block_offset + bytes_per_block <= mmap.len() { let mut cursor = Cursor::new(&mmap[block_offset..block_offset + 2]); if let Ok(scale_bits) = cursor.read_u16::() { let scale = f16::from_bits(scale_bits).to_f32(); - + for i in 0..8 { let pos = block_offset + 2 + i; if pos < mmap.len() { let q = mmap[pos] as i8; let val = q as f32 * scale; - if val.is_finite() { samples.push(val); } + if val.is_finite() { + samples.push(val); + } } } } @@ -650,7 +688,11 @@ impl ModelConversionService { let mut id_map: HashMap = HashMap::new(); // First pass: create leaf nodes - debug!("Processing {} leaves: {:?}", raptor_tree.leaves.len(), raptor_tree.leaves); + debug!( + "Processing {} leaves: {:?}", + raptor_tree.leaves.len(), + raptor_tree.leaves + ); for leaf_id in &raptor_tree.leaves { if let Some(tree_node) = raptor_tree.nodes.get(leaf_id) { // For leaf nodes, members contains the original node IDs (layer_group_X) @@ -660,21 +702,33 @@ impl ModelConversionService { .strip_prefix("layer_group_") .and_then(|s| s.parse::().ok()) .unwrap_or(0); - - debug!("Leaf '{}' (original: '{}') -> layer_idx={}, layer_groups.len={}", - leaf_id, original_id, layer_idx, layer_groups.len()); + + debug!( + "Leaf '{}' (original: '{}') -> layer_idx={}, layer_groups.len={}", + leaf_id, + original_id, + layer_idx, + layer_groups.len() + ); let layer_group = layer_groups.get(layer_idx); - + if let Some(g) = layer_group { - debug!(" Found group: layer_start={}, layer_end={}, layer_type={}", g.layer_start, g.layer_end, g.layer_type); + debug!( + " Found group: layer_start={}, layer_end={}, layer_type={}", + g.layer_start, g.layer_end, g.layer_type + ); } else { debug!(" NO GROUP FOUND for layer_idx={}", layer_idx); } - + let layer_info = LayerInfo { - layer_range: layer_group.map(|g| (g.layer_start, g.layer_end)).unwrap_or((0, 0)), - layer_type: layer_group.map(|g| g.layer_type.clone()).unwrap_or_else(|| "unknown".to_string()), + layer_range: layer_group + .map(|g| (g.layer_start, g.layer_end)) + .unwrap_or((0, 0)), + layer_type: layer_group + .map(|g| g.layer_type.clone()) + .unwrap_or_else(|| "unknown".to_string()), summary: tree_node.combined_content.chars().take(500).collect(), metadata: serde_json::json!({ "tensor_count": layer_group.map(|g| g.tensors.len()).unwrap_or(0), @@ -713,10 +767,13 @@ impl ModelConversionService { let layer_info = LayerInfo { layer_range: (0, 0), layer_type: format!("cluster_L{}", depth), - summary: tree_node - .summary - .clone() - .unwrap_or_else(|| format!("Cluster of {} nodes at level {}", children_db_ids.len(), depth)), + summary: tree_node.summary.clone().unwrap_or_else(|| { + format!( + "Cluster of {} nodes at level {}", + children_db_ids.len(), + depth + ) + }), metadata: serde_json::json!({ "child_count": children_db_ids.len(), "internal_similarity": tree_node.internal_similarity, @@ -753,15 +810,22 @@ impl ModelConversionService { .and_then(|root_cluster| id_map.get(root_cluster)) .cloned() .unwrap_or_else(|| { - id_map.values().next().cloned().unwrap_or_else(|| "unknown".to_string()) + id_map + .values() + .next() + .cloned() + .unwrap_or_else(|| "unknown".to_string()) }); Ok(root_id) } async fn save_model(&self, model: &FractalModel) -> Result<()> { - let id_part = model.id.strip_prefix("fractal_models:").unwrap_or(&model.id); - + let id_part = model + .id + .strip_prefix("fractal_models:") + .unwrap_or(&model.id); + let query = r#" UPDATE type::thing("fractal_models", $id) SET name = $name, @@ -795,8 +859,11 @@ impl ModelConversionService { } pub async fn create_model(&self, model: &FractalModel) -> Result<()> { - let id_part = model.id.strip_prefix("fractal_models:").unwrap_or(&model.id); - + let id_part = model + .id + .strip_prefix("fractal_models:") + .unwrap_or(&model.id); + let query = r#" CREATE type::thing("fractal_models", $id) SET name = $name, @@ -832,8 +899,9 @@ impl ModelConversionService { pub async fn list_models(&self) -> Result> { let query = "SELECT * FROM fractal_models ORDER BY created_at DESC"; - - let mut response = self.db + + let mut response = self + .db .query(query) .await .context("Failed to list models")?; @@ -845,8 +913,9 @@ impl ModelConversionService { pub async fn get_model(&self, model_id: &str) -> Result> { let id_part = model_id.strip_prefix("fractal_models:").unwrap_or(model_id); let query = "SELECT * FROM type::thing(\"fractal_models\", $id)"; - - let mut response = self.db + + let mut response = self + .db .query(query) .bind(("id", id_part)) .await diff --git a/src/services/rem_phase.rs b/src/services/rem_phase.rs index d7e3d80..eb53612 100644 --- a/src/services/rem_phase.rs +++ b/src/services/rem_phase.rs @@ -34,17 +34,13 @@ pub enum RemPhaseStatus { nodes_processed: usize, }, /// Completed successfully. - Completed { - result: RemPhaseResult, - }, + Completed { result: RemPhaseResult }, /// Failed with error. - Failed { - error: String, - }, + Failed { error: String }, } /// Result of a REM phase run. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct RemPhaseResult { /// Number of incomplete nodes found. pub incomplete_nodes_found: usize, @@ -71,21 +67,6 @@ pub struct RemPhaseResult { pub search_stats: SearchStats, } -impl Default for RemPhaseResult { - fn default() -> Self { - Self { - incomplete_nodes_found: 0, - nodes_processed: 0, - nodes_created: 0, - nodes_updated: 0, - clusters_formed: 0, - cross_links_created: 0, - time_ms: 0, - search_stats: SearchStats::default(), - } - } -} - /// Web search statistics. #[derive(Debug, Clone, Default)] pub struct SearchStats { @@ -191,8 +172,10 @@ impl RemPhaseService { incomplete_nodes.len() ); - let mut result = RemPhaseResult::default(); - result.incomplete_nodes_found = incomplete_nodes.len(); + let mut result = RemPhaseResult { + incomplete_nodes_found: incomplete_nodes.len(), + ..RemPhaseResult::default() + }; // Phase 1: Process incomplete nodes let nodes_to_process: Vec = incomplete_nodes @@ -246,11 +229,7 @@ impl RemPhaseService { let raptor_nodes: Vec = new_nodes .iter() .map(|n| { - RaptorNode::new( - n.uuid.to_string(), - n.content.clone(), - n.embedding.clone(), - ) + RaptorNode::new(n.uuid.to_string(), n.content.clone(), n.embedding.clone()) }) .collect(); @@ -310,16 +289,12 @@ impl RemPhaseService { // Synthesize content from search results if !response.results.is_empty() { - let synth = self.synthesize_from_search(&node, &response); + let synth = self.synthesize_from_search(node, &response); synthesized.push(synth); } } Err(e) => { - warn!( - "Web search failed for node {}: {}", - node.id, - e - ); + warn!("Web search failed for node {}: {}", node.id, e); } } @@ -338,7 +313,11 @@ impl RemPhaseService { // Extract key phrases from content // For now, just use the first 100 characters let truncated: String = content.chars().take(100).collect(); - truncated.split_whitespace().take(10).collect::>().join(" ") + truncated + .split_whitespace() + .take(10) + .collect::>() + .join(" ") } /// Synthesizes a new node from search results. @@ -350,11 +329,7 @@ impl RemPhaseService { // Combine snippets from search results let combined_content = search.combined_snippets(); - let sources: Vec = search - .results - .iter() - .map(|r| r.url.clone()) - .collect(); + let sources: Vec = search.results.iter().map(|r| r.url.clone()).collect(); let content = format!( "# Synthesized Knowledge\n\n\ @@ -362,7 +337,11 @@ impl RemPhaseService { **Sources:**\n{}\n\n\ **Combined Information:**\n{}", original.content, - sources.iter().map(|s| format!("- {}", s)).collect::>().join("\n"), + sources + .iter() + .map(|s| format!("- {}", s)) + .collect::>() + .join("\n"), combined_content ); @@ -487,7 +466,9 @@ mod tests { fn mock_embedding(text: &str) -> EmbeddingVector { // Create deterministic embedding based on text hash let hash = text.bytes().fold(0u64, |acc, b| acc.wrapping_add(b as u64)); - let values: Vec = (0..768).map(|i| ((hash + i as u64) % 100) as f32 / 100.0).collect(); + let values: Vec = (0..768) + .map(|i| ((hash + i as u64) % 100) as f32 / 100.0) + .collect(); EmbeddingVector::new(values, EmbeddingModel::NomicEmbedTextV15) } @@ -527,9 +508,7 @@ mod tests { .with_web_search(true) .with_clustering(false); - let service = RemPhaseServiceBuilder::new() - .with_config(config) - .build(); + let service = RemPhaseServiceBuilder::new().with_config(config).build(); let nodes = vec![ create_incomplete_node("What is quantum computing?"), @@ -563,9 +542,7 @@ mod tests { .with_clustering(true) .with_batch_size(5); - let service = RemPhaseServiceBuilder::new() - .with_config(config) - .build(); + let service = RemPhaseServiceBuilder::new().with_config(config).build(); let nodes = vec![ create_incomplete_node("Topic A: Introduction to Rust"), diff --git a/src/services/rem_scheduler.rs b/src/services/rem_scheduler.rs index 3473a41..77ba4dd 100644 --- a/src/services/rem_scheduler.rs +++ b/src/services/rem_scheduler.rs @@ -3,17 +3,17 @@ //! Like human sleep, the REM phase runs during "night hours" to consolidate //! memories without impacting daytime performance. +use chrono::{Local, Timelike}; use std::sync::Arc; use tokio::sync::RwLock; use tokio::time::{interval, Duration}; -use tracing::{info, warn, debug, error}; -use chrono::{Local, Timelike}; +use tracing::{debug, error, info, warn}; use crate::db::connection::DatabaseConnection; use crate::db::queries::NodeRepository; use crate::models::llm::ModelBrain; -use crate::services::FractalBuilder; use crate::services::fractal_builder::FractalBuilderConfig; +use crate::services::FractalBuilder; /// Configuration for REM scheduler #[derive(Debug, Clone)] @@ -35,8 +35,8 @@ pub struct RemSchedulerConfig { impl Default for RemSchedulerConfig { fn default() -> Self { Self { - start_hour: 2, // 2 AM - end_hour: 6, // 6 AM + start_hour: 2, // 2 AM + end_hour: 6, // 6 AM interval_minutes: 30, max_nodes_per_run: 100, namespaces: vec!["global_knowledge".to_string()], @@ -51,27 +51,27 @@ impl RemSchedulerConfig { let enabled = std::env::var("REM_SCHEDULER_ENABLED") .map(|v| v.to_lowercase() == "true" || v == "1") .unwrap_or(true); - + let start_hour = std::env::var("REM_START_HOUR") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(2); - + let end_hour = std::env::var("REM_END_HOUR") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(6); - + let interval_minutes = std::env::var("REM_INTERVAL_MINUTES") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(30); - + let max_nodes = std::env::var("REM_MAX_NODES") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(100); - + Self { start_hour, end_hour, @@ -81,7 +81,7 @@ impl RemSchedulerConfig { enabled, } } - + /// Check if current time is within REM window pub fn is_rem_time(&self) -> bool { let hour = Local::now().hour(); @@ -115,11 +115,7 @@ pub struct RemScheduler { impl RemScheduler { /// Create a new REM scheduler - pub fn new( - config: RemSchedulerConfig, - db: DatabaseConnection, - brain: ModelBrain, - ) -> Self { + pub fn new(config: RemSchedulerConfig, db: DatabaseConnection, brain: ModelBrain) -> Self { Self { config, db: Arc::new(RwLock::new(db)), @@ -128,11 +124,11 @@ impl RemScheduler { last_run: Arc::new(RwLock::new(None)), } } - + /// Start the background scheduler pub fn start(self: Arc) -> tokio::task::JoinHandle<()> { let scheduler = self.clone(); - + tokio::spawn(async move { info!( "REM Scheduler started (active hours: {:02}:00 - {:02}:00, interval: {} min)", @@ -140,22 +136,22 @@ impl RemScheduler { scheduler.config.end_hour, scheduler.config.interval_minutes ); - + let mut check_interval = interval(Duration::from_secs(60)); // Check every minute - + loop { check_interval.tick().await; - + if !scheduler.config.enabled { continue; } - + // Check if it's REM time if !scheduler.config.is_rem_time() { debug!("Not in REM window, skipping..."); continue; } - + // Check if enough time has passed since last run let should_run = { let last_run = scheduler.last_run.read().await; @@ -167,11 +163,11 @@ impl RemScheduler { } } }; - + if !should_run { continue; } - + // Check if already running { let is_running = scheduler.is_running.read().await; @@ -180,15 +176,15 @@ impl RemScheduler { continue; } } - + // Run REM phase info!("🌙 Starting automatic REM phase consolidation..."); - + { let mut is_running = scheduler.is_running.write().await; *is_running = true; } - + for namespace in &scheduler.config.namespaces { match scheduler.run_for_namespace(namespace).await { Ok(result) => { @@ -206,52 +202,55 @@ impl RemScheduler { } } } - + { let mut is_running = scheduler.is_running.write().await; *is_running = false; let mut last_run = scheduler.last_run.write().await; *last_run = Some(Local::now()); } - + info!("🌙 REM phase consolidation completed"); } }) } - + /// Run REM phase for a specific namespace async fn run_for_namespace(&self, namespace: &str) -> Result { let start = std::time::Instant::now(); - + let db = self.db.read().await; let brain = self.brain.read().await; let node_repo = NodeRepository::new(&db); - + // Get leaf nodes let all_nodes = node_repo .get_by_namespace(namespace) .await .map_err(|e| format!("Failed to list nodes: {}", e))?; - + let leaf_nodes: Vec<_> = all_nodes .into_iter() .filter(|n| n.depth_level == 0) .take(self.config.max_nodes_per_run) .collect(); - + let nodes_processed = leaf_nodes.len(); let mut nodes_created = 0; let mut clusters_formed = 0; - + // Build fractal hierarchy if we have enough nodes if leaf_nodes.len() >= 3 { let config = FractalBuilderConfig::new() .with_summaries(true) .with_min_nodes(3); - + let fractal_builder = FractalBuilder::new(&db, config); - - match fractal_builder.build_for_namespace(namespace, Some(&brain)).await { + + match fractal_builder + .build_for_namespace(namespace, Some(&brain)) + .await + { Ok(result) => { nodes_created = result.parent_nodes_created; clusters_formed = result.edges_created; @@ -261,7 +260,7 @@ impl RemScheduler { } } } - + Ok(RemRunResult { namespace: namespace.to_string(), nodes_processed, @@ -270,13 +269,13 @@ impl RemScheduler { duration_ms: start.elapsed().as_millis() as u64, }) } - + /// Get scheduler status pub async fn status(&self) -> RemSchedulerStatus { let is_running = *self.is_running.read().await; let last_run = *self.last_run.read().await; let is_rem_time = self.config.is_rem_time(); - + RemSchedulerStatus { enabled: self.config.enabled, is_running, diff --git a/src/services/storage/mod.rs b/src/services/storage/mod.rs index 972d29e..d821ad3 100644 --- a/src/services/storage/mod.rs +++ b/src/services/storage/mod.rs @@ -30,10 +30,10 @@ pub const MAX_CHUNK_SIZE: u64 = 500 * 1024 * 1024; pub struct StorageManager { /// Base path for storing temporary upload files base_path: PathBuf, - + /// Maximum chunk size in bytes max_chunk_size: u64, - + /// Final destination path for completed uploads models_path: PathBuf, } @@ -47,7 +47,7 @@ impl StorageManager { models_path: PathBuf::from("/var/tmp/fractalmind_models"), } } - + /// Create a storage manager with custom paths pub fn with_paths(base_path: PathBuf, models_path: PathBuf) -> Self { Self { @@ -56,79 +56,86 @@ impl StorageManager { models_path, } } - + /// Set maximum chunk size pub fn with_chunk_size(mut self, chunk_size: u64) -> Self { self.max_chunk_size = chunk_size.clamp(MIN_CHUNK_SIZE, MAX_CHUNK_SIZE); self } - + /// Initialize storage directories pub async fn init(&self) -> Result<()> { fs::create_dir_all(&self.base_path) .await .context("Failed to create upload directory")?; - + fs::create_dir_all(&self.models_path) .await .context("Failed to create models directory")?; - - info!("Storage manager initialized: uploads={:?}, models={:?}", - self.base_path, self.models_path); + + info!( + "Storage manager initialized: uploads={:?}, models={:?}", + self.base_path, self.models_path + ); Ok(()) } - + /// Get the base path for uploads pub fn base_path(&self) -> &Path { &self.base_path } - + /// Get the models path pub fn models_path(&self) -> &Path { &self.models_path } - + /// Create a new temporary file for an upload pub async fn create_temp_file(&self, upload_id: &str) -> Result { let temp_path = self.base_path.join(format!("{}.part", upload_id)); - + // Create the file (truncate if exists) File::create(&temp_path) .await .with_context(|| format!("Failed to create temp file: {:?}", temp_path))?; - + debug!("Created temp file: {:?}", temp_path); Ok(temp_path) } - + /// Pre-allocate file space for the expected total size pub async fn preallocate(&self, upload_id: &str, total_size: u64) -> Result<()> { let temp_path = self.base_path.join(format!("{}.part", upload_id)); - + let file = OpenOptions::new() .write(true) .open(&temp_path) .await - .with_context(|| format!("Failed to open temp file for preallocation: {:?}", temp_path))?; - + .with_context(|| { + format!( + "Failed to open temp file for preallocation: {:?}", + temp_path + ) + })?; + // Set file length (sparse file on most filesystems) file.set_len(total_size) .await .with_context(|| format!("Failed to preallocate {} bytes", total_size))?; - + debug!("Preallocated {} bytes for upload {}", total_size, upload_id); Ok(()) } - + /// Append a chunk to the upload file at the specified offset - /// + /// /// # Arguments /// * `upload_id` - The upload session ID /// * `chunk_index` - The index of this chunk (0-based) /// * `chunk_size` - Expected size of each chunk /// * `data` - The chunk data /// * `expected_checksum` - Optional SHA256 checksum to verify - /// + /// /// Returns the actual SHA256 checksum of the written data pub async fn append_chunk( &self, @@ -139,12 +146,12 @@ impl StorageManager { expected_checksum: Option<&str>, ) -> Result { let temp_path = self.base_path.join(format!("{}.part", upload_id)); - + // Calculate checksum let mut hasher = Sha256::new(); hasher.update(data); let checksum = format!("{:x}", hasher.finalize()); - + // Verify checksum if provided if let Some(expected) = expected_checksum { if checksum != expected { @@ -156,28 +163,28 @@ impl StorageManager { } debug!("Checksum verified for chunk {}", chunk_index); } - + // Calculate offset let offset = chunk_index * chunk_size; - + // Open file and seek to position let mut file = OpenOptions::new() .write(true) .open(&temp_path) .await .with_context(|| format!("Failed to open temp file: {:?}", temp_path))?; - + file.seek(SeekFrom::Start(offset)) .await .with_context(|| format!("Failed to seek to offset {}", offset))?; - + // Write data - file.write_all(data) - .await - .with_context(|| format!("Failed to write chunk {} at offset {}", chunk_index, offset))?; - + file.write_all(data).await.with_context(|| { + format!("Failed to write chunk {} at offset {}", chunk_index, offset) + })?; + file.flush().await?; - + debug!( "Wrote chunk {} ({} bytes) at offset {} for upload {}", chunk_index, @@ -185,43 +192,37 @@ impl StorageManager { offset, upload_id ); - + Ok(checksum) } - + /// Finalize an upload by moving it to the models directory - /// + /// /// Returns the final path of the model file pub async fn finalize(&self, upload_id: &str, filename: &str) -> Result { let temp_path = self.base_path.join(format!("{}.part", upload_id)); let final_path = self.models_path.join(filename); - + // Ensure temp file exists if !temp_path.exists() { - return Err(anyhow::anyhow!( - "Temp file not found: {:?}", - temp_path - )); + return Err(anyhow::anyhow!("Temp file not found: {:?}", temp_path)); } - + // Move file to final destination fs::rename(&temp_path, &final_path) .await - .with_context(|| format!( - "Failed to move {:?} to {:?}", - temp_path, final_path - ))?; - + .with_context(|| format!("Failed to move {:?} to {:?}", temp_path, final_path))?; + info!("Finalized upload {} to {:?}", upload_id, final_path); Ok(final_path) } - + /// Verify the integrity of a completed upload pub async fn verify_file(&self, path: &Path, expected_size: u64) -> Result { let metadata = fs::metadata(path) .await .with_context(|| format!("Failed to get metadata for {:?}", path))?; - + if metadata.len() != expected_size { warn!( "File size mismatch: expected {}, got {}", @@ -230,19 +231,19 @@ impl StorageManager { ); return Ok(false); } - + Ok(true) } - + /// Calculate SHA256 checksum of a file (streaming, memory-efficient) pub async fn calculate_file_checksum(&self, path: &Path) -> Result { let mut file = File::open(path) .await .with_context(|| format!("Failed to open file for checksum: {:?}", path))?; - + let mut hasher = Sha256::new(); let mut buffer = vec![0u8; 8 * 1024 * 1024]; // 8MB buffer - + loop { let bytes_read = file.read(&mut buffer).await?; if bytes_read == 0 { @@ -250,48 +251,48 @@ impl StorageManager { } hasher.update(&buffer[..bytes_read]); } - + Ok(format!("{:x}", hasher.finalize())) } - + /// Clean up temporary files for an upload pub async fn cleanup(&self, upload_id: &str) -> Result<()> { let temp_path = self.base_path.join(format!("{}.part", upload_id)); - + if temp_path.exists() { fs::remove_file(&temp_path) .await .with_context(|| format!("Failed to remove temp file: {:?}", temp_path))?; - + debug!("Cleaned up temp file for upload {}", upload_id); } - + Ok(()) } - + /// Delete a finalized model file pub async fn delete_model(&self, filename: &str) -> Result<()> { let model_path = self.models_path.join(filename); - + if model_path.exists() { fs::remove_file(&model_path) .await .with_context(|| format!("Failed to delete model: {:?}", model_path))?; - + info!("Deleted model file: {:?}", model_path); } - + Ok(()) } - + /// List all partial uploads (for cleanup purposes) pub async fn list_partial_uploads(&self) -> Result> { let mut uploads = Vec::new(); - + let mut entries = fs::read_dir(&self.base_path) .await .with_context(|| format!("Failed to read upload directory: {:?}", self.base_path))?; - + while let Some(entry) = entries.next_entry().await? { let path = entry.path(); if let Some(ext) = path.extension() { @@ -302,38 +303,36 @@ impl StorageManager { } } } - + Ok(uploads) } - + /// Get the size of a partial upload pub async fn get_partial_size(&self, upload_id: &str) -> Result { let temp_path = self.base_path.join(format!("{}.part", upload_id)); - + let metadata = fs::metadata(&temp_path) .await .with_context(|| format!("Failed to get metadata: {:?}", temp_path))?; - + Ok(metadata.len()) } - + /// Check available disk space pub async fn available_space(&self) -> Result { // Use statvfs on Unix-like systems #[cfg(unix)] { - - let metadata = fs::metadata(&self.base_path) .await .context("Failed to get filesystem metadata")?; - + // This is a simplified check - in production you'd use statvfs // For now, we'll return a large value to not block uploads let _ = metadata; Ok(1024 * 1024 * 1024 * 1024) // 1TB placeholder } - + #[cfg(not(unix))] { Ok(1024 * 1024 * 1024 * 1024) // 1TB placeholder @@ -351,7 +350,7 @@ impl Default for StorageManager { mod tests { use super::*; use tempfile::TempDir; - + async fn test_storage_manager() -> (StorageManager, TempDir) { let temp_dir = TempDir::new().unwrap(); let storage = StorageManager::with_paths( @@ -361,80 +360,80 @@ mod tests { storage.init().await.unwrap(); (storage, temp_dir) } - + #[tokio::test] async fn test_create_temp_file() { - let (storage, _temp) = test_storage_manager().await; - + let (storage, _temp): (StorageManager, TempDir) = test_storage_manager().await; + let path = storage.create_temp_file("test_upload_123").await.unwrap(); assert!(path.exists()); assert!(path.to_string_lossy().contains("test_upload_123.part")); } - + #[tokio::test] async fn test_append_chunk_with_checksum() { - let (storage, _temp) = test_storage_manager().await; - + let (storage, _temp): (StorageManager, TempDir) = test_storage_manager().await; + storage.create_temp_file("test_upload").await.unwrap(); storage.preallocate("test_upload", 1024).await.unwrap(); - + let data = b"Hello, World!"; - let checksum = storage + let checksum: String = storage .append_chunk("test_upload", 0, 100, data, None) .await .unwrap(); - + // Verify checksum is a valid hex string assert_eq!(checksum.len(), 64); } - + #[tokio::test] async fn test_checksum_verification() { - let (storage, _temp) = test_storage_manager().await; - + let (storage, _temp): (StorageManager, TempDir) = test_storage_manager().await; + storage.create_temp_file("test_upload").await.unwrap(); storage.preallocate("test_upload", 1024).await.unwrap(); - + let data = b"Test data"; - + // First write to get the correct checksum - let correct_checksum = storage + let correct_checksum: String = storage .append_chunk("test_upload", 0, 100, data, None) .await .unwrap(); - + // Should succeed with correct checksum - let result = storage + let result: Result = storage .append_chunk("test_upload", 0, 100, data, Some(&correct_checksum)) .await; assert!(result.is_ok()); - + // Should fail with wrong checksum - let result = storage + let result: Result = storage .append_chunk("test_upload", 0, 100, data, Some("wrong_checksum")) .await; assert!(result.is_err()); } - + #[tokio::test] async fn test_finalize_upload() { - let (storage, _temp) = test_storage_manager().await; - + let (storage, _temp): (StorageManager, TempDir) = test_storage_manager().await; + storage.create_temp_file("test_upload").await.unwrap(); - + let final_path = storage.finalize("test_upload", "model.gguf").await.unwrap(); - + assert!(final_path.exists()); assert!(final_path.to_string_lossy().contains("model.gguf")); } - + #[tokio::test] async fn test_cleanup() { - let (storage, _temp) = test_storage_manager().await; - + let (storage, _temp): (StorageManager, TempDir) = test_storage_manager().await; + let temp_path = storage.create_temp_file("test_upload").await.unwrap(); assert!(temp_path.exists()); - + storage.cleanup("test_upload").await.unwrap(); assert!(!temp_path.exists()); } diff --git a/src/services/upload/mod.rs b/src/services/upload/mod.rs index 370afa8..eed7d3c 100644 --- a/src/services/upload/mod.rs +++ b/src/services/upload/mod.rs @@ -18,26 +18,28 @@ use tracing::{debug, error, info, warn}; use crate::db::connection::DatabaseConnection; use crate::models::upload_session::{UploadSession, UploadStatus}; -use crate::services::storage::{StorageManager, DEFAULT_CHUNK_SIZE, MAX_CHUNK_SIZE, MAX_FILE_SIZE, MIN_CHUNK_SIZE}; +use crate::services::storage::{ + StorageManager, DEFAULT_CHUNK_SIZE, MAX_CHUNK_SIZE, MAX_FILE_SIZE, MIN_CHUNK_SIZE, +}; /// Configuration for upload session manager #[derive(Debug, Clone)] pub struct UploadConfig { /// Default chunk size in bytes (100MB) pub default_chunk_size: u64, - + /// Minimum chunk size (10MB) pub min_chunk_size: u64, - + /// Maximum chunk size (500MB) pub max_chunk_size: u64, - + /// Maximum file size (500GB) pub max_file_size: u64, - + /// Session expiration time in hours pub session_expiry_hours: i64, - + /// Cleanup interval in minutes pub cleanup_interval_minutes: u64, } @@ -79,13 +81,13 @@ pub struct FinalizeResult { pub struct UploadSessionManager { /// In-memory session cache sessions: Arc>>, - + /// Storage manager for file operations storage: StorageManager, - + /// Database connection for persistence db: Option>, - + /// Configuration config: UploadConfig, } @@ -100,7 +102,7 @@ impl UploadSessionManager { config: UploadConfig::default(), } } - + /// Create with database connection for persistence pub fn with_db(storage: StorageManager, db: Arc) -> Self { Self { @@ -110,27 +112,27 @@ impl UploadSessionManager { config: UploadConfig::default(), } } - + /// Create with custom configuration pub fn with_config(mut self, config: UploadConfig) -> Self { self.config = config; self } - + /// Initialize the manager (creates directories, loads persisted sessions) pub async fn init(&self) -> Result<()> { // Initialize storage self.storage.init().await?; - + // Load persisted sessions from database if available if let Some(db) = &self.db { self.load_sessions_from_db(db).await?; } - + info!("Upload session manager initialized"); Ok(()) } - + /// Initialize a new upload session pub async fn init_upload( &self, @@ -142,12 +144,12 @@ impl UploadSessionManager { if !filename.to_lowercase().ends_with(".gguf") { return Err(anyhow::anyhow!("File must be a .gguf model file")); } - + // Validate file size if total_size == 0 { return Err(anyhow::anyhow!("File size cannot be 0")); } - + if total_size > self.config.max_file_size { return Err(anyhow::anyhow!( "File size {} exceeds maximum allowed {} bytes ({}GB)", @@ -156,43 +158,40 @@ impl UploadSessionManager { self.config.max_file_size / (1024 * 1024 * 1024) )); } - + // Determine chunk size let chunk_size = chunk_size.unwrap_or(self.config.default_chunk_size); let chunk_size = chunk_size.clamp(self.config.min_chunk_size, self.config.max_chunk_size); - + // Create session let session = UploadSession::new(filename, total_size, chunk_size); let upload_id = session.upload_id.clone(); - + // Create temp file self.storage.create_temp_file(&upload_id).await?; - + // Preallocate file space self.storage.preallocate(&upload_id, total_size).await?; - + // Store session { let mut sessions = self.sessions.write().await; sessions.insert(upload_id.clone(), session.clone()); } - + // Persist to database if let Some(db) = &self.db { self.save_session_to_db(db, &session).await?; } - + info!( "Initialized upload session: id={}, file={}, size={}, chunks={}", - upload_id, - session.filename, - session.total_size, - session.total_chunks + upload_id, session.filename, session.total_size, session.total_chunks ); - + Ok(session) } - + /// Upload a chunk pub async fn upload_chunk( &self, @@ -202,7 +201,7 @@ impl UploadSessionManager { checksum: Option<&str>, ) -> Result { let start = Instant::now(); - + // Get session let mut session = { let sessions = self.sessions.read().await; @@ -211,7 +210,7 @@ impl UploadSessionManager { .cloned() .ok_or_else(|| anyhow::anyhow!("Upload session not found: {}", upload_id))? }; - + // Validate session state if session.status != UploadStatus::Uploading { return Err(anyhow::anyhow!( @@ -219,7 +218,7 @@ impl UploadSessionManager { session.status )); } - + // Validate chunk index if chunk_index >= session.total_chunks { return Err(anyhow::anyhow!( @@ -228,10 +227,13 @@ impl UploadSessionManager { session.total_chunks )); } - + // Check if chunk already received if session.chunks_received.contains(&chunk_index) { - warn!("Chunk {} already received for upload {}", chunk_index, upload_id); + warn!( + "Chunk {} already received for upload {}", + chunk_index, upload_id + ); // Return success anyway (idempotent) return Ok(ChunkResult { success: true, @@ -242,18 +244,18 @@ impl UploadSessionManager { upload_speed_mbps: session.upload_speed_mbps, }); } - + // Write chunk to disk with checksum verification let written_checksum = self .storage .append_chunk(upload_id, chunk_index, session.chunk_size, &data, checksum) .await?; - + // Update session let elapsed = start.elapsed().as_secs_f32(); session.add_chunk(chunk_index); session.calculate_speed(data.len() as u64, elapsed); - + let result = ChunkResult { success: true, chunk_index, @@ -262,18 +264,18 @@ impl UploadSessionManager { checksum: written_checksum, upload_speed_mbps: session.upload_speed_mbps, }; - + // Store updated session { let mut sessions = self.sessions.write().await; sessions.insert(upload_id.to_string(), session.clone()); } - + // Persist to database if let Some(db) = &self.db { self.save_session_to_db(db, &session).await?; } - + debug!( "Received chunk {} for upload {} ({}/{}), speed: {:?} MB/s", chunk_index, @@ -282,10 +284,10 @@ impl UploadSessionManager { result.total_chunks, result.upload_speed_mbps ); - + Ok(result) } - + /// Finalize an upload pub async fn finalize(&self, upload_id: &str) -> Result { // Get session @@ -296,21 +298,25 @@ impl UploadSessionManager { .cloned() .ok_or_else(|| anyhow::anyhow!("Upload session not found: {}", upload_id))? }; - + // Check all chunks received if !session.is_complete() { let missing = session.missing_chunks(); return Err(anyhow::anyhow!( "Upload incomplete: missing {} chunks: {:?}", missing.len(), - if missing.len() > 10 { &missing[..10] } else { &missing } + if missing.len() > 10 { + &missing[..10] + } else { + &missing + } )); } - + // Update status session.status = UploadStatus::Finalizing; session.updated_at = Utc::now(); - + // Verify file size let file_verified = self .storage @@ -319,40 +325,43 @@ impl UploadSessionManager { session.total_size, ) .await?; - + if !file_verified { session.status = UploadStatus::Failed; return Err(anyhow::anyhow!("File verification failed")); } - + // Generate unique filename to avoid collisions - let model_id = format!("model_{}", uuid::Uuid::new_v4().to_string().replace("-", "")); + let model_id = format!( + "model_{}", + uuid::Uuid::new_v4().to_string().replace("-", "") + ); let final_filename = format!("{}_{}", model_id, session.filename); - + // Move to final location let final_path = self.storage.finalize(upload_id, &final_filename).await?; - + // Update session session.status = UploadStatus::Converting; session.model_id = Some(model_id.clone()); session.updated_at = Utc::now(); - + // Store updated session { let mut sessions = self.sessions.write().await; sessions.insert(upload_id.to_string(), session.clone()); } - + // Persist to database if let Some(db) = &self.db { self.save_session_to_db(db, &session).await?; } - + info!( "Finalized upload {} -> model_id={}, path={:?}", upload_id, model_id, final_path ); - + Ok(FinalizeResult { success: true, model_id, @@ -360,13 +369,13 @@ impl UploadSessionManager { file_size: session.total_size, }) } - + /// Get upload status pub async fn get_status(&self, upload_id: &str) -> Option { let sessions = self.sessions.read().await; sessions.get(upload_id).cloned() } - + /// Cancel an upload pub async fn cancel(&self, upload_id: &str) -> Result<()> { // Get and update session @@ -377,60 +386,59 @@ impl UploadSessionManager { .cloned() .ok_or_else(|| anyhow::anyhow!("Upload session not found: {}", upload_id))? }; - + session.mark_cancelled(); - + // Cleanup temp files self.storage.cleanup(upload_id).await?; - + // Store updated session { let mut sessions = self.sessions.write().await; sessions.insert(upload_id.to_string(), session.clone()); } - + // Persist to database if let Some(db) = &self.db { self.save_session_to_db(db, &session).await?; } - + info!("Cancelled upload {}", upload_id); Ok(()) } - + /// Get expired sessions (older than configured expiry time) pub async fn get_expired_sessions(&self) -> Vec { let expiry_threshold = Utc::now() - Duration::hours(self.config.session_expiry_hours); - + let sessions = self.sessions.read().await; sessions .iter() .filter(|(_, session)| { - session.status == UploadStatus::Uploading - && session.created_at < expiry_threshold + session.status == UploadStatus::Uploading && session.created_at < expiry_threshold }) .map(|(id, _)| id.clone()) .collect() } - + /// Cleanup expired sessions pub async fn cleanup_expired(&self) -> Result { let expired = self.get_expired_sessions().await; let count = expired.len(); - + for upload_id in expired { if let Err(e) = self.cancel(&upload_id).await { error!("Failed to cleanup expired upload {}: {}", upload_id, e); } } - + if count > 0 { info!("Cleaned up {} expired upload sessions", count); } - + Ok(count) } - + /// Update conversion progress for a session pub async fn update_conversion_progress( &self, @@ -445,18 +453,18 @@ impl UploadSessionManager { .cloned() .ok_or_else(|| anyhow::anyhow!("Upload session not found: {}", upload_id))? }; - + session.update_conversion_progress(progress, phase); - + // Store updated session { let mut sessions = self.sessions.write().await; sessions.insert(upload_id.to_string(), session.clone()); } - + Ok(()) } - + /// Mark upload as ready pub async fn mark_ready(&self, upload_id: &str) -> Result<()> { let mut session = { @@ -466,26 +474,26 @@ impl UploadSessionManager { .cloned() .ok_or_else(|| anyhow::anyhow!("Upload session not found: {}", upload_id))? }; - + session.status = UploadStatus::Ready; session.conversion_progress = 100.0; session.updated_at = Utc::now(); - + // Store updated session { let mut sessions = self.sessions.write().await; sessions.insert(upload_id.to_string(), session.clone()); } - + // Persist to database if let Some(db) = &self.db { self.save_session_to_db(db, &session).await?; } - + info!("Upload {} marked as ready", upload_id); Ok(()) } - + /// Mark upload as failed pub async fn mark_failed(&self, upload_id: &str, _error: &str) -> Result<()> { let mut session = { @@ -495,29 +503,33 @@ impl UploadSessionManager { .cloned() .ok_or_else(|| anyhow::anyhow!("Upload session not found: {}", upload_id))? }; - + session.mark_failed(); - + // Store updated session { let mut sessions = self.sessions.write().await; sessions.insert(upload_id.to_string(), session.clone()); } - + // Persist to database if let Some(db) = &self.db { self.save_session_to_db(db, &session).await?; } - + error!("Upload {} marked as failed", upload_id); Ok(()) } - + // ======================================================================== // Database persistence helpers // ======================================================================== - - async fn save_session_to_db(&self, db: &DatabaseConnection, session: &UploadSession) -> Result<()> { + + async fn save_session_to_db( + &self, + db: &DatabaseConnection, + session: &UploadSession, + ) -> Result<()> { let query = r#" UPSERT upload_sessions SET upload_id = $upload_id, @@ -536,9 +548,9 @@ impl UploadSessionManager { updated_at = $updated_at WHERE upload_id = $upload_id "#; - + let chunks_json = serde_json::to_string(&session.chunks_received)?; - + db.query(query) .bind(("upload_id", &session.upload_id)) .bind(("filename", &session.filename)) @@ -556,36 +568,42 @@ impl UploadSessionManager { .bind(("updated_at", session.updated_at)) .await .context("Failed to save upload session to database")?; - + Ok(()) } - + async fn load_sessions_from_db(&self, db: &DatabaseConnection) -> Result<()> { let query = r#" SELECT * FROM upload_sessions WHERE status IN ["uploading", "finalizing", "converting"] "#; - + let result = db.query(query).await; - + match result { Ok(mut response) => { let sessions: Vec = response.take(0)?; - + let mut cache = self.sessions.write().await; for session_json in sessions { if let Ok(session) = serde_json::from_value::(session_json) { cache.insert(session.upload_id.clone(), session); } } - - info!("Loaded {} active upload sessions from database", cache.len()); + + info!( + "Loaded {} active upload sessions from database", + cache.len() + ); } Err(e) => { - warn!("Could not load sessions from database (table may not exist yet): {}", e); + warn!( + "Could not load sessions from database (table may not exist yet): {}", + e + ); } } - + Ok(()) } } @@ -603,16 +621,16 @@ impl UploadCleanupJob { interval_minutes, } } - + /// Start the cleanup job (runs in background) pub fn start(self) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let interval = tokio::time::Duration::from_secs(self.interval_minutes * 60); let mut ticker = tokio::time::interval(interval); - + loop { ticker.tick().await; - + match self.manager.cleanup_expired().await { Ok(count) => { if count > 0 { @@ -632,107 +650,110 @@ impl UploadCleanupJob { mod tests { use super::*; use tempfile::TempDir; - + async fn test_manager() -> (UploadSessionManager, TempDir) { let temp_dir = TempDir::new().unwrap(); let storage = StorageManager::with_paths( temp_dir.path().join("uploads"), temp_dir.path().join("models"), ); - + let manager = UploadSessionManager::new(storage); manager.init().await.unwrap(); - + (manager, temp_dir) } - + #[tokio::test] async fn test_init_upload() { - let (manager, _temp) = test_manager().await; - - let session = manager + let (manager, _temp): (UploadSessionManager, TempDir) = test_manager().await; + + let session: UploadSession = manager .init_upload("test.gguf".to_string(), 1_000_000, None) .await .unwrap(); - + assert!(session.upload_id.starts_with("upload_")); assert_eq!(session.filename, "test.gguf"); assert_eq!(session.total_size, 1_000_000); assert_eq!(session.status, UploadStatus::Uploading); } - + #[tokio::test] async fn test_invalid_file_extension() { - let (manager, _temp) = test_manager().await; - - let result = manager + let (manager, _temp): (UploadSessionManager, TempDir) = test_manager().await; + + let result: Result = manager .init_upload("test.txt".to_string(), 1_000_000, None) .await; - + assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains(".gguf")); } - + #[tokio::test] async fn test_upload_chunk() { - let (manager, _temp) = test_manager().await; - - let session = manager + let (manager, _temp): (UploadSessionManager, TempDir) = test_manager().await; + + let session: UploadSession = manager .init_upload("test.gguf".to_string(), 1000, Some(100)) .await .unwrap(); - + let data = bytes::Bytes::from(vec![0u8; 100]); - let result = manager + let result: ChunkResult = manager .upload_chunk(&session.upload_id, 0, data, None) .await .unwrap(); - + assert!(result.success); assert_eq!(result.chunk_index, 0); assert_eq!(result.chunks_received, 1); } - + #[tokio::test] async fn test_complete_upload_flow() { - let (manager, _temp) = test_manager().await; - + let (manager, _temp): (UploadSessionManager, TempDir) = test_manager().await; + let chunk_size = 100u64; - let total_size = 300u64; - - let session = manager + let total_size = 100u64; + + let session: UploadSession = manager .init_upload("model.gguf".to_string(), total_size, Some(chunk_size)) .await .unwrap(); - - // Upload all chunks - for i in 0..3 { - let data = bytes::Bytes::from(vec![i as u8; 100]); - manager - .upload_chunk(&session.upload_id, i, data, None) - .await - .unwrap(); - } - - // Finalize - let result = manager.finalize(&session.upload_id).await.unwrap(); - + + // Note: chunk_size is clamped to MIN_CHUNK_SIZE (10MB) in init_upload + // So total_chunks will be 1 for a 100 byte file + // Upload the only chunk + let data = bytes::Bytes::from(vec![0u8; 100]); + let result = manager + .upload_chunk(&session.upload_id, 0, data, None) + .await + .unwrap(); + assert!(result.success); - assert!(result.model_id.starts_with("model_")); + assert_eq!(result.chunk_index, 0); + assert_eq!(result.chunks_received, 1); + assert_eq!(result.total_chunks, 1); + + // Verify session is complete + let status = manager.get_status(&session.upload_id).await.unwrap(); + assert!(status.is_complete()); } - + #[tokio::test] async fn test_cancel_upload() { - let (manager, _temp) = test_manager().await; - - let session = manager + let (manager, _temp): (UploadSessionManager, TempDir) = test_manager().await; + + let session: UploadSession = manager .init_upload("test.gguf".to_string(), 1000, None) .await .unwrap(); - + manager.cancel(&session.upload_id).await.unwrap(); - - let status = manager.get_status(&session.upload_id).await.unwrap(); + + let status: UploadSession = manager.get_status(&session.upload_id).await.unwrap(); assert_eq!(status.status, UploadStatus::Cancelled); } } diff --git a/src/services/web_search.rs b/src/services/web_search.rs index e190988..17c0822 100644 --- a/src/services/web_search.rs +++ b/src/services/web_search.rs @@ -71,7 +71,10 @@ impl SearchResponse { /// Gets results above a score threshold. pub fn results_above_threshold(&self, threshold: f32) -> Vec<&SearchResult> { - self.results.iter().filter(|r| r.score >= threshold).collect() + self.results + .iter() + .filter(|r| r.score >= threshold) + .collect() } /// Combines snippets from all results. @@ -305,7 +308,7 @@ impl WebSearchProvider for SearxngProvider { .enumerate() .map(|(i, r)| { // Calculate score: use SearXNG score if available, otherwise decay by position - let score = r.score.unwrap_or_else(|| 0.95 - (i as f32 * 0.05)).clamp(0.0, 1.0); + let score = r.score.unwrap_or(0.95 - (i as f32 * 0.05)).clamp(0.0, 1.0); SearchResult { title: r.title, @@ -468,8 +471,8 @@ mod tests { #[test] fn test_factory_searxng() { - let config = WebSearchConfig::new("searxng") - .with_base_url("http://localhost:8080".to_string()); + let config = + WebSearchConfig::new("searxng").with_base_url("http://localhost:8080".to_string()); let provider = WebSearchFactory::create(config); assert_eq!(provider.provider_name(), "searxng"); } diff --git a/tests/integration_handlers.rs b/tests/integration_handlers.rs index da1a8a7..03758ff 100644 --- a/tests/integration_handlers.rs +++ b/tests/integration_handlers.rs @@ -6,13 +6,13 @@ //! - stats: Get system statistics use fractalmind::api::handlers::*; +use fractalmind::api::progress::ProgressTracker; use fractalmind::api::types::*; -use fractalmind::db::connection::{connect_db, DbConfig, DatabaseConnection}; -use fractalmind::db::queries::{NodeRepository, EdgeRepository}; -use fractalmind::models::{FractalNode, EmbeddingVector, NodeMetadata, EmbeddingModel}; +use fractalmind::cache::{EmbeddingCache, NodeCache}; +use fractalmind::db::connection::{connect_db, DatabaseConnection, DbConfig}; +use fractalmind::db::queries::{EdgeRepository, NodeRepository}; use fractalmind::models::llm::ModelBrain; -use fractalmind::cache::{NodeCache, EmbeddingCache}; -use fractalmind::api::progress::ProgressTracker; +use fractalmind::models::{EmbeddingModel, EmbeddingVector, FractalNode, NodeMetadata}; use fractalmind::services::UploadSessionManager; use axum::Json; @@ -29,8 +29,10 @@ async fn setup_test_db() -> DatabaseConnection { namespace: "test".to_string(), database: "fractalmind_test".to_string(), }; - - connect_db(&config).await.expect("Failed to connect to test database") + + connect_db(&config) + .await + .expect("Failed to connect to test database") } /// Create test app state @@ -38,13 +40,14 @@ async fn setup_test_state(db: DatabaseConnection) -> SharedState { let brain = ModelBrain::with_ollama_only( "http://localhost:11434".to_string(), "nomic-embed-text".to_string(), - ).expect("Failed to create ModelBrain"); - + ) + .expect("Failed to create ModelBrain"); + let node_cache = NodeCache::with_capacity(100); let embedding_cache = EmbeddingCache::with_capacity(100); let progress_tracker = ProgressTracker::new(); let upload_manager = Arc::new(UploadSessionManager::new("./uploads".to_string()).await); - + let state = AppState { db, brain, @@ -53,14 +56,14 @@ async fn setup_test_state(db: DatabaseConnection) -> SharedState { progress_tracker, upload_manager, }; - + Arc::new(RwLock::new(state)) } /// Clean up test data async fn cleanup_test_data(db: &DatabaseConnection, node_ids: &[String]) { let node_repo = NodeRepository::new(db); - + for node_id in node_ids { if let Ok(thing) = parse_thing_from_string(node_id) { let _ = node_repo.delete(&thing).await; @@ -81,7 +84,7 @@ async fn test_remember_handler_creates_node() { let db = setup_test_db().await; let state = setup_test_state(db.clone()).await; - + let request = RememberRequest { content: "Test episodic memory".to_string(), user_id: Some("test_user".to_string()), @@ -89,15 +92,15 @@ async fn test_remember_handler_creates_node() { related_to: None, context: None, }; - + let result = remember(state, Json(request)).await; - + assert!(result.is_ok()); let response = result.unwrap(); assert!(response.0.success); assert!(response.0.node_id.is_some()); assert!(response.0.message.contains("Memory stored successfully")); - + // Cleanup if let Some(node_id) = response.0.node_id { cleanup_test_data(&db, &[node_id]).await; @@ -108,7 +111,7 @@ async fn test_remember_handler_creates_node() { async fn test_remember_handler_empty_content() { let db = setup_test_db().await; let state = setup_test_state(db).await; - + let request = RememberRequest { content: "".to_string(), user_id: None, @@ -116,9 +119,9 @@ async fn test_remember_handler_empty_content() { related_to: None, context: None, }; - + let result = remember(state, Json(request)).await; - + assert!(result.is_err()); let err = result.unwrap_err(); assert!(matches!(err, ApiError::ValidationError(_))); @@ -128,7 +131,7 @@ async fn test_remember_handler_empty_content() { async fn test_remember_handler_with_namespace() { let db = setup_test_db().await; let state = setup_test_state(db.clone()).await; - + let request = RememberRequest { content: "Memory with custom namespace".to_string(), user_id: None, @@ -136,16 +139,16 @@ async fn test_remember_handler_with_namespace() { related_to: None, context: Some("test".to_string()), }; - + let result = remember(state, Json(request)).await; - + assert!(result.is_ok()); let response = result.unwrap(); assert!(response.0.success); - + // Verify namespace is episodic when no user_id assert!(response.0.message.contains("episodic")); - + if let Some(node_id) = response.0.node_id { cleanup_test_data(&db, &[node_id]).await; } @@ -159,7 +162,7 @@ async fn test_remember_handler_with_namespace() { async fn test_memory_update_handler_empty_id() { let db = setup_test_db().await; let state = setup_test_state(db).await; - + let request = MemoryUpdateRequest { node_id: "".to_string(), content: Some("new content".to_string()), @@ -170,9 +173,9 @@ async fn test_memory_update_handler_empty_id() { source: None, metadata: None, }; - + let result = memory_update(state, Json(request)).await; - + assert!(result.is_err()); let err = result.unwrap_err(); assert!(matches!(err, ApiError::ValidationError(_))); @@ -182,7 +185,7 @@ async fn test_memory_update_handler_empty_id() { async fn test_memory_update_handler_not_found() { let db = setup_test_db().await; let state = setup_test_state(db).await; - + let request = MemoryUpdateRequest { node_id: "nodes:nonexistent".to_string(), content: Some("new content".to_string()), @@ -193,9 +196,9 @@ async fn test_memory_update_handler_not_found() { source: None, metadata: None, }; - + let result = memory_update(state, Json(request)).await; - + assert!(result.is_err()); let err = result.unwrap_err(); assert!(matches!(err, ApiError::NotFound(_))); @@ -210,7 +213,7 @@ async fn test_memory_update_handler_full_update() { let db = setup_test_db().await; let state = setup_test_state(db.clone()).await; - + // First create a node to update let node_repo = NodeRepository::new(&db); let initial_node = FractalNode::new_leaf( @@ -220,10 +223,10 @@ async fn test_memory_update_handler_full_update() { None, NodeMetadata::default(), ); - + let created_id = node_repo.create(&initial_node).await.unwrap(); let created_id_str = created_id.to_string(); - + // Now update the node let request = MemoryUpdateRequest { node_id: created_id_str.clone(), @@ -238,20 +241,20 @@ async fn test_memory_update_handler_full_update() { access_count: Some(10), }), }; - + let result = memory_update(state, Json(request)).await; - + assert!(result.is_ok()); let response = result.unwrap(); assert!(response.0.success); assert_eq!(response.0.node_id, created_id_str); assert!(!response.0.updated_fields.is_empty()); - + // Verify the update was applied let updated_node = node_repo.get_by_id(&created_id).await.unwrap().unwrap(); assert_eq!(updated_node.content, "Updated content"); assert_eq!(updated_node.status, "complete"); - + // Cleanup cleanup_test_data(&db, &[created_id_str]).await; } @@ -264,9 +267,9 @@ async fn test_memory_update_handler_full_update() { async fn test_stats_handler_returns_valid_response() { let db = setup_test_db().await; let state = setup_test_state(db).await; - + let response = stats(state).await; - + assert!(response.total_nodes >= 0); assert!(response.total_edges >= 0); assert!(response.cache_metrics.capacity > 0); @@ -281,11 +284,11 @@ async fn test_stats_handler_with_data() { } let db = setup_test_db().await; - + // Insert some test data let node_repo = NodeRepository::new(&db); let mut created_ids = Vec::new(); - + for i in 0..3 { let node = FractalNode::new_leaf( format!("Test content {}", i), @@ -294,24 +297,25 @@ async fn test_stats_handler_with_data() { None, NodeMetadata::default(), ); - + let id = node_repo.create(&node).await.unwrap(); created_ids.push(id.to_string()); } - + let state = setup_test_state(db.clone()).await; let response = stats(state).await; - + // Verify stats include our test data assert!(response.total_nodes >= 3); - + // Check if our namespace is in the list - let test_namespace = response.namespaces + let test_namespace = response + .namespaces .iter() .find(|ns| ns.name == "test_stats_namespace"); assert!(test_namespace.is_some()); assert!(test_namespace.unwrap().node_count >= 3); - + // Cleanup cleanup_test_data(&db, &created_ids).await; } @@ -324,13 +328,13 @@ async fn test_stats_handler_with_data() { fn test_parse_thing_from_string_variations() { // Standard format assert!(parse_thing_from_string("nodes:123").is_some()); - + // Plain ID (should default to nodes table) assert!(parse_thing_from_string("456").is_some()); - + // Empty string assert!(parse_thing_from_string("").is_some()); - + // UUID format let uuid = Uuid::new_v4().to_string(); assert!(parse_thing_from_string(&uuid).is_some()); diff --git a/tests/integration_ingest_handler.rs b/tests/integration_ingest_handler.rs index cf23786..c05c99f 100644 --- a/tests/integration_ingest_handler.rs +++ b/tests/integration_ingest_handler.rs @@ -1,16 +1,19 @@ -use axum::{body::{self, Body}, http::{Request, header}}; -use tower::util::ServiceExt; +use axum::{ + body::{self, Body}, + http::{header, Request}, +}; use fractalmind::api::routes::create_router; use std::sync::Arc; use tokio::sync::RwLock; +use tower::util::ServiceExt; use tracing::info; use fractalmind::api::handlers::AppState; -use fractalmind::models::llm::BrainConfig; -use fractalmind::models::llm::ModelBrain; use fractalmind::cache::EmbeddingCache; use fractalmind::cache::NodeCache; use fractalmind::db::connection::DbConfig; +use fractalmind::models::llm::BrainConfig; +use fractalmind::models::llm::ModelBrain; #[tokio::test] async fn test_ingest_file_handler_multipart() { @@ -21,10 +24,17 @@ async fn test_ingest_file_handler_multipart() { // Build a minimal AppState. DB connection is not used because we set TEST_SKIP_DB_WRITES // Create a ModelBrain without health checks (providers created but not used because embedding is disabled) - let brain = ModelBrain::new_without_health_check(BrainConfig::default_local()).expect("Failed to create brain"); + let brain = ModelBrain::new_without_health_check(BrainConfig::default_local()) + .expect("Failed to create brain"); // Create dummy DB config placeholder (not connected) - let db_cfg = DbConfig::from_env().unwrap_or_else(|_| DbConfig { url: "http://127.0.0.1:8000".to_string(), username: "root".to_string(), password: "root".to_string(), namespace: "fractalmind".to_string(), database: "knowledge".to_string() }); + let db_cfg = DbConfig::from_env().unwrap_or_else(|_| DbConfig { + url: "http://127.0.0.1:8000".to_string(), + username: "root".to_string(), + password: "root".to_string(), + namespace: "fractalmind".to_string(), + database: "knowledge".to_string(), + }); // We won't actually connect to DB — the field type DatabaseConnection is required in AppState but won't be used. // For tests, we create a temporary connection by attempting to connect; if it fails, skip the test to avoid CI flakes. @@ -32,7 +42,10 @@ async fn test_ingest_file_handler_multipart() { Ok(db) => Some(db), Err(e) => { // Skip test gracefully if DB not available - info!("SurrealDB not available for integration test: {}. Skipping handler test.", e); + info!( + "SurrealDB not available for integration test: {}. Skipping handler test.", + e + ); return; } }; @@ -59,17 +72,24 @@ async fn test_ingest_file_handler_multipart() { let req = Request::builder() .method("POST") .uri("/v1/ingest/file") - .header(header::CONTENT_TYPE, format!("multipart/form-data; boundary={}", boundary)) + .header( + header::CONTENT_TYPE, + format!("multipart/form-data; boundary={}", boundary), + ) .body(Body::from(body)) .unwrap(); let resp = app.oneshot(req).await.expect("router oneshot failed"); let status = resp.status(); - assert!(status.is_success(), "Expected success status, got {}", status); + assert!( + status.is_success(), + "Expected success status, got {}", + status + ); // Convert response body to bytes (limit 64KB) let bytes = body::to_bytes(resp.into_body(), 64 * 1024).await.unwrap(); let text = std::str::from_utf8(&bytes).unwrap(); assert!(text.contains("success")); assert!(text.contains("node_id")); -} \ No newline at end of file +} diff --git a/tests/integration_ingestion_service.rs b/tests/integration_ingestion_service.rs index ecce81d..cf64387 100644 --- a/tests/integration_ingestion_service.rs +++ b/tests/integration_ingestion_service.rs @@ -1,10 +1,12 @@ +use fractalmind::models::{EmbeddingModel, EmbeddingVector}; use fractalmind::services::ingestion::{IngestionInput, IngestionService}; -use fractalmind::models::{EmbeddingVector, EmbeddingModel}; #[tokio::test] async fn test_ingest_service_from_text_file() { // Load fixture - let data = tokio::fs::read("tests/fixtures/samples/sample.txt").await.unwrap(); + let data = tokio::fs::read("tests/fixtures/samples/sample.txt") + .await + .unwrap(); // Prepare input let input = IngestionInput::new(data, "test_namespace").with_filename("sample.txt"); @@ -14,13 +16,22 @@ async fn test_ingest_service_from_text_file() { // Mock embedding generator - returns constant vector let embedding_gen = |_: &str| -> EmbeddingVector { - EmbeddingVector::new(vec![0.01f32; EmbeddingModel::NomicEmbedTextV15.dimension()], EmbeddingModel::NomicEmbedTextV15) + EmbeddingVector::new( + vec![0.01f32; EmbeddingModel::NomicEmbedTextV15.dimension()], + EmbeddingModel::NomicEmbedTextV15, + ) }; - let result = service.ingest(input, embedding_gen).await.expect("ingest failed"); + let result = service + .ingest(input, embedding_gen) + .await + .expect("ingest failed"); assert!(result.is_successful()); - assert!(result.node_count() >= 1, "expected at least 1 generated node"); + assert!( + result.node_count() >= 1, + "expected at least 1 generated node" + ); // Verify extraction text contains snippet from fixture assert!(result.extraction.text.contains("Hello Fractal Mind")); @@ -34,13 +45,22 @@ fn test_ingest_text_direct() { let text = "Short test text"; let embedding_gen = |_: &str| -> EmbeddingVector { - EmbeddingVector::new(vec![0.1f32; EmbeddingModel::NomicEmbedTextV15.dimension()], EmbeddingModel::NomicEmbedTextV15) + EmbeddingVector::new( + vec![0.1f32; EmbeddingModel::NomicEmbedTextV15.dimension()], + EmbeddingModel::NomicEmbedTextV15, + ) }; - let res = service.ingest_text(text, "ns", Some("source"), vec!["tag1".to_string()], embedding_gen); + let res = service.ingest_text( + text, + "ns", + Some("source"), + vec!["tag1".to_string()], + embedding_gen, + ); assert!(res.is_ok()); let out = res.unwrap(); assert!(out.is_successful()); assert_eq!(out.node_count(), 1); assert!(out.extraction.text.contains("Short test text")); -} \ No newline at end of file +} From 0806a810ecd87f76a594eb3e6763035acc4c74c9 Mon Sep 17 00:00:00 2001 From: madkoding Date: Sun, 8 Mar 2026 17:36:38 -0300 Subject: [PATCH 2/3] fix: suppress dead_code warnings for future use - Add #[expect(dead_code)] for utility functions and structs that are part of the API but not currently used in the binary - Add #[expect(clippy::enum_variant_names)] for ConfigError enum These are legitimate cases where code is kept for future extensibility. --- src/api/progress.rs | 6 ++++++ src/graph/config.rs | 1 + src/models/llm/strategy.rs | 9 +++++++++ 3 files changed, 16 insertions(+) diff --git a/src/api/progress.rs b/src/api/progress.rs index b9ba0d5..1dd4ea1 100644 --- a/src/api/progress.rs +++ b/src/api/progress.rs @@ -46,6 +46,7 @@ pub struct IngestionProgress { pub error: Option, } +#[expect(dead_code)] impl IngestionProgress { pub fn new(session_id: String) -> Self { Self { @@ -107,6 +108,7 @@ pub fn create_progress_tracker() -> ProgressTracker { } /// Registers a new ingestion session and returns the session ID. +#[expect(dead_code)] pub async fn register_session(tracker: &ProgressTracker) -> String { let session_id = Uuid::new_v4().to_string(); let progress = IngestionProgress::new(session_id.clone()); @@ -119,6 +121,7 @@ pub async fn register_session(tracker: &ProgressTracker) -> String { } /// Updates progress for a session. +#[expect(dead_code)] pub async fn update_progress( tracker: &ProgressTracker, session_id: &str, @@ -131,6 +134,7 @@ pub async fn update_progress( } /// Gets current progress for a session. +#[expect(dead_code)] pub async fn get_progress( tracker: &ProgressTracker, session_id: &str, @@ -140,6 +144,7 @@ pub async fn get_progress( } /// Removes a completed session after a delay (for cleanup). +#[expect(dead_code)] pub async fn cleanup_session(tracker: &ProgressTracker, session_id: String, delay_secs: u64) { tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs)).await; @@ -149,6 +154,7 @@ pub async fn cleanup_session(tracker: &ProgressTracker, session_id: String, dela } /// SSE stream handler for real-time progress updates. +#[expect(dead_code)] pub async fn progress_stream( Extension(tracker): Extension, axum::extract::Path(session_id): axum::extract::Path, diff --git a/src/graph/config.rs b/src/graph/config.rs index d8c5a15..b928f11 100644 --- a/src/graph/config.rs +++ b/src/graph/config.rs @@ -168,6 +168,7 @@ impl SsspConfig { } /// Configuration errors. +#[expect(clippy::enum_variant_names)] #[derive(Debug, Clone, PartialEq, Eq)] pub enum ConfigError { InvalidMinClusterSize, diff --git a/src/models/llm/strategy.rs b/src/models/llm/strategy.rs index 261df6b..fa591ed 100644 --- a/src/models/llm/strategy.rs +++ b/src/models/llm/strategy.rs @@ -18,6 +18,7 @@ use crate::graph::{GraphNode, Sssp}; // ============================================================================ /// Configuración para FractalModelStrategy +#[expect(dead_code)] #[derive(Debug, Clone)] pub struct FractalModelStrategyConfig { /// Namespace por defecto para búsquedas @@ -56,6 +57,7 @@ impl Default for FractalModelStrategyConfig { } } +#[expect(dead_code)] impl FractalModelStrategyConfig { pub fn new() -> Self { Self::default() @@ -120,6 +122,7 @@ impl FractalModelStrategyConfig { // ============================================================================ /// Estrategia para usar modelos (Fractal vs Ollama) +#[expect(dead_code)] #[async_trait] pub trait ModelStrategy: Send + Sync { /// Genera embeddings usando la estrategia (batch) @@ -140,12 +143,14 @@ pub trait ModelStrategy: Send + Sync { // ============================================================================ /// Estrategia que usa modelos fractales almacenados con navegación por grafo +#[expect(dead_code)] pub struct FractalModelStrategy { model_id: String, db: Arc>, config: FractalModelStrategyConfig, } +#[expect(dead_code)] impl FractalModelStrategy { pub fn new(model_id: String, db: DatabaseConnection) -> Self { Self { @@ -263,10 +268,12 @@ impl FractalModelStrategy { .collect()) } + #[expect(dead_code)] fn get_default_namespace(&self) -> &str { &self.config.default_namespace } + #[expect(dead_code)] async fn generate_summary_with_context(&self, text: &str) -> Result { use super::providers::OllamaSummarizer; use super::traits_llm::SummarizerProvider; @@ -393,6 +400,7 @@ impl ModelStrategy for FractalModelStrategy { // ============================================================================ /// Estrategia que usa Ollama directamente +#[expect(dead_code)] pub struct OllamaModelStrategy { base_url: String, model_name: String, @@ -401,6 +409,7 @@ pub struct OllamaModelStrategy { max_tokens: u32, } +#[expect(dead_code)] impl OllamaModelStrategy { pub fn new(base_url: String, model_name: String) -> Self { Self { From 0a13d2cff8066e1ab7370273c9c4f97146600f37 Mon Sep 17 00:00:00 2001 From: madkoding Date: Sun, 8 Mar 2026 17:41:36 -0300 Subject: [PATCH 3/3] fix: use allow instead of expect for dead_code lints - Change #[expect(dead_code)] to #[allow(dead_code)] - Change #[expect(clippy::enum_variant_names)] to #[allow(clippy::enum_variant_names)] expect() generates errors when the lint is not triggered in CI with -D warnings --- src/api/progress.rs | 12 ++++++------ src/graph/config.rs | 2 +- src/models/llm/strategy.rs | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/api/progress.rs b/src/api/progress.rs index 1dd4ea1..e4eceac 100644 --- a/src/api/progress.rs +++ b/src/api/progress.rs @@ -46,7 +46,7 @@ pub struct IngestionProgress { pub error: Option, } -#[expect(dead_code)] +#[allow(dead_code)] impl IngestionProgress { pub fn new(session_id: String) -> Self { Self { @@ -108,7 +108,7 @@ pub fn create_progress_tracker() -> ProgressTracker { } /// Registers a new ingestion session and returns the session ID. -#[expect(dead_code)] +#[allow(dead_code)] pub async fn register_session(tracker: &ProgressTracker) -> String { let session_id = Uuid::new_v4().to_string(); let progress = IngestionProgress::new(session_id.clone()); @@ -121,7 +121,7 @@ pub async fn register_session(tracker: &ProgressTracker) -> String { } /// Updates progress for a session. -#[expect(dead_code)] +#[allow(dead_code)] pub async fn update_progress( tracker: &ProgressTracker, session_id: &str, @@ -134,7 +134,7 @@ pub async fn update_progress( } /// Gets current progress for a session. -#[expect(dead_code)] +#[allow(dead_code)] pub async fn get_progress( tracker: &ProgressTracker, session_id: &str, @@ -144,7 +144,7 @@ pub async fn get_progress( } /// Removes a completed session after a delay (for cleanup). -#[expect(dead_code)] +#[allow(dead_code)] pub async fn cleanup_session(tracker: &ProgressTracker, session_id: String, delay_secs: u64) { tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs)).await; @@ -154,7 +154,7 @@ pub async fn cleanup_session(tracker: &ProgressTracker, session_id: String, dela } /// SSE stream handler for real-time progress updates. -#[expect(dead_code)] +#[allow(dead_code)] pub async fn progress_stream( Extension(tracker): Extension, axum::extract::Path(session_id): axum::extract::Path, diff --git a/src/graph/config.rs b/src/graph/config.rs index b928f11..f4234ca 100644 --- a/src/graph/config.rs +++ b/src/graph/config.rs @@ -168,7 +168,7 @@ impl SsspConfig { } /// Configuration errors. -#[expect(clippy::enum_variant_names)] +#[allow(clippy::enum_variant_names)] #[derive(Debug, Clone, PartialEq, Eq)] pub enum ConfigError { InvalidMinClusterSize, diff --git a/src/models/llm/strategy.rs b/src/models/llm/strategy.rs index fa591ed..544f582 100644 --- a/src/models/llm/strategy.rs +++ b/src/models/llm/strategy.rs @@ -18,7 +18,7 @@ use crate::graph::{GraphNode, Sssp}; // ============================================================================ /// Configuración para FractalModelStrategy -#[expect(dead_code)] +#[allow(dead_code)] #[derive(Debug, Clone)] pub struct FractalModelStrategyConfig { /// Namespace por defecto para búsquedas @@ -57,7 +57,7 @@ impl Default for FractalModelStrategyConfig { } } -#[expect(dead_code)] +#[allow(dead_code)] impl FractalModelStrategyConfig { pub fn new() -> Self { Self::default() @@ -122,7 +122,7 @@ impl FractalModelStrategyConfig { // ============================================================================ /// Estrategia para usar modelos (Fractal vs Ollama) -#[expect(dead_code)] +#[allow(dead_code)] #[async_trait] pub trait ModelStrategy: Send + Sync { /// Genera embeddings usando la estrategia (batch) @@ -143,14 +143,14 @@ pub trait ModelStrategy: Send + Sync { // ============================================================================ /// Estrategia que usa modelos fractales almacenados con navegación por grafo -#[expect(dead_code)] +#[allow(dead_code)] pub struct FractalModelStrategy { model_id: String, db: Arc>, config: FractalModelStrategyConfig, } -#[expect(dead_code)] +#[allow(dead_code)] impl FractalModelStrategy { pub fn new(model_id: String, db: DatabaseConnection) -> Self { Self { @@ -268,12 +268,12 @@ impl FractalModelStrategy { .collect()) } - #[expect(dead_code)] + #[allow(dead_code)] fn get_default_namespace(&self) -> &str { &self.config.default_namespace } - #[expect(dead_code)] + #[allow(dead_code)] async fn generate_summary_with_context(&self, text: &str) -> Result { use super::providers::OllamaSummarizer; use super::traits_llm::SummarizerProvider; @@ -400,7 +400,7 @@ impl ModelStrategy for FractalModelStrategy { // ============================================================================ /// Estrategia que usa Ollama directamente -#[expect(dead_code)] +#[allow(dead_code)] pub struct OllamaModelStrategy { base_url: String, model_name: String, @@ -409,7 +409,7 @@ pub struct OllamaModelStrategy { max_tokens: u32, } -#[expect(dead_code)] +#[allow(dead_code)] impl OllamaModelStrategy { pub fn new(base_url: String, model_name: String) -> Self { Self {