diff --git a/CHANGELOG.md b/CHANGELOG.md index 25f4966..11428a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [3.2.1] - 2026-08-30 + +### 🧹 Cleanup des ontologies S3 orphelines + +- **`storage_check`** ne considère plus automatiquement tous les objets contenant `_ontology_` comme légitimes : seules les ontologies encore référencées par une mémoire existante sont protégées. +- **`storage_cleanup`** peut désormais détecter et supprimer les ontologies laissées sur S3 après la suppression d'une mémoire, y compris les copies obsolètes d'une ontologie. +- **Compatibilité legacy** : les mémoires anciennes sans `ontology_uri` restent protégées par une correspondance stricte sur leur identifiant et leur nom d'ontologie. +- Correctif de l'issue [#31](https://github.com/Cloud-Temple/graph-memory/issues/31), validé sur Docker local : 5 ontologies orphelines détectées puis supprimées, second contrôle à 0 orphelin. + ## [3.2.0] - 2026-06-04 ### 🧭 `source_path` exposé dans la recherche Graph-first diff --git a/Dockerfile b/Dockerfile index 74932ee..5fb45ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ FROM python:3.11.12-slim # Métadonnées LABEL maintainer="Cloud Temple" LABEL description="MCP Memory Service - Knowledge Graph Memory for AI Agents" -LABEL version="3.2.0" +LABEL version="3.2.1" # Variables d'environnement Python ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/README.en.md b/README.en.md index de29c72..9bc2a99 100644 --- a/README.en.md +++ b/README.en.md @@ -14,9 +14,9 @@ Built by **[Cloud Temple](https://www.cloud-temple.com)**. ## 📋 Changelog -See **[CHANGELOG.md](CHANGELOG.md)** for the full version history (v0.5.0 → v3.2.0). +See **[CHANGELOG.md](CHANGELOG.md)** for the full version history (v0.5.0 → v3.2.1). -**Latest**: v3.2.0 (June 4, 2026) — `source_path` exposed in Graph-first search: `memory_search` and `memory_query` now return the canonical source path (`source_path`) and a derived `repo_path` for every document/chunk, letting an agent open the Git file immediately without a full `document_list`. Enriched via a retroactive graph join (no re-ingestion). `document_get`/`document_list` tools aligned. Previously: v3.1.1 (`/admin` "⚡ Ingest Jobs" console). +**Latest**: v3.2.1 (August 30, 2026) — `storage_check` and `storage_cleanup` now detect S3 ontology objects orphaned after memory deletion, while preserving ontologies still referenced by existing memories and legacy memories without an `ontology_uri`. Fixes [#31](https://github.com/Cloud-Temple/graph-memory/issues/31). Previously: v3.2.0 (`source_path` exposed in Graph-first search). --- diff --git a/README.md b/README.md index bac6c4b..19d4440 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,9 @@ Développé par **[Cloud Temple](https://www.cloud-temple.com)**. ## 📋 Changelog -Voir **[CHANGELOG.md](CHANGELOG.md)** pour l'historique complet des versions (v0.5.0 → v3.2.0). +Voir **[CHANGELOG.md](CHANGELOG.md)** pour l'historique complet des versions (v0.5.0 → v3.2.1). -**Dernière version** : v3.2.0 (4 juin 2026) — `source_path` exposé dans la recherche Graph-first : `memory_search` et `memory_query` renvoient désormais le chemin source canonique (`source_path`) et un `repo_path` dérivé pour chaque document/chunk, permettant à un agent d'ouvrir immédiatement le fichier Git sans `document_list` complet. Enrichissement par jointure graphe rétroactive (aucune ré-ingestion). Outils `document_get`/`document_list` alignés. Précédemment : v3.1.1 (console `/admin` « ⚡ Ingest Jobs »). +**Dernière version** : v3.2.1 (30 août 2026) — `storage_check` et `storage_cleanup` détectent désormais les ontologies S3 devenues orphelines après la suppression d'une mémoire, tout en protégeant les ontologies encore référencées et les mémoires legacy sans `ontology_uri`. Correctif de l'issue [#31](https://github.com/Cloud-Temple/graph-memory/issues/31). Précédemment : v3.2.0 (`source_path` exposé dans la recherche Graph-first). --- @@ -932,4 +932,4 @@ Développé par **[Cloud Temple](https://www.cloud-temple.com)**. --- -*Graph Memory v3.2.0 — Juin 2026* +*Graph Memory v3.2.1 — Août 2026* diff --git a/VERSION b/VERSION index a4f52a5..e4604e3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0 \ No newline at end of file +3.2.1 diff --git a/src/mcp_memory/__init__.py b/src/mcp_memory/__init__.py index d2c3ec8..0d14752 100644 --- a/src/mcp_memory/__init__.py +++ b/src/mcp_memory/__init__.py @@ -15,5 +15,5 @@ python -m src.mcp_memory.server --port 8002 """ -__version__ = "3.2.0" +__version__ = "3.2.1" __author__ = "Cloud Temple" diff --git a/src/mcp_memory/server.py b/src/mcp_memory/server.py index 81b7b7b..ea47f8d 100644 --- a/src/mcp_memory/server.py +++ b/src/mcp_memory/server.py @@ -26,6 +26,7 @@ from .auth.middleware import AuthMiddleware, LoggingMiddleware, StaticFilesMiddleware from .auth.context import check_memory_access, check_write_permission, check_admin_permission, get_allowed_memory_ids, current_auth from .core.validators import validate_memory_id, validate_filename, validate_document_size, validate_entity_name, validate_backup_id as validate_backup_id_format, check_bootstrap_key_safety +from .storage_consistency import collect_referenced_ontology_keys, is_referenced_ontology_key # ============================================================================= @@ -2226,13 +2227,21 @@ async def storage_check( return admin_err # 1. Récupérer les mémoires à vérifier + all_memories = await get_graph().list_memories() if memory_id: - memory = await get_graph().get_memory(memory_id) + memory = next((mem for mem in all_memories if mem.id == memory_id), None) if not memory: return {"status": "error", "message": f"Mémoire '{memory_id}' non trouvée"} memories = [memory] else: - memories = await get_graph().list_memories() + memories = all_memories + + # Les ontologies ne sont légitimes que si une mémoire existante les + # référence. Les mémoires legacy sans ontology_uri sont protégées par + # un fallback strict sur leur préfixe et leur nom d'ontologie. + referenced_ontology_keys, legacy_ontology_patterns = collect_referenced_ontology_keys( + all_memories, get_storage()._parse_key + ) # 2. Collecter toutes les URIs des documents référencés dans le graphe graph_uris = set() # URIs référencées dans Neo4j @@ -2275,7 +2284,6 @@ async def storage_check( all_graph_uris = set(graph_uris) # Commencer avec celles du scope if memory_id: # Charger les URIs des autres mémoires aussi - all_memories = await get_graph().list_memories() for mem in all_memories: if mem.id == memory_id: continue # Déjà chargé @@ -2310,9 +2318,11 @@ async def storage_check( if key.startswith("_backups/"): continue - # Ignorer les ontologies (fichiers légitimes) - # Le pattern est {hash[:8]}__ontology_{name}.yaml (double _ car hash + _ontology) - if "_ontology_" in key: + # Ignorer uniquement les ontologies encore référencées par une + # mémoire existante. Une ontologie de mémoire supprimée est orpheline. + if is_referenced_ontology_key( + key, referenced_ontology_keys, legacy_ontology_patterns + ): continue # Si la clé n'est pas référencée dans le graphe → orphelin diff --git a/src/mcp_memory/static/admin.html b/src/mcp_memory/static/admin.html index bd94be1..fddb168 100644 --- a/src/mcp_memory/static/admin.html +++ b/src/mcp_memory/static/admin.html @@ -4,7 +4,7 @@ Cloud Temple — Graph Memory Admin - +
@@ -45,7 +45,7 @@

Graph Memory

- - + + diff --git a/src/mcp_memory/storage_consistency.py b/src/mcp_memory/storage_consistency.py new file mode 100644 index 0000000..0673594 --- /dev/null +++ b/src/mcp_memory/storage_consistency.py @@ -0,0 +1,41 @@ +"""Règles pures de classification des objets S3 liés aux mémoires.""" + + +def collect_referenced_ontology_keys(memories, parse_key): + """Retourne les clés d'ontologie légitimes et les fallbacks legacy. + + Les mémoires récentes référencent exactement leur objet via ``ontology_uri``. + Pour une mémoire legacy sans URI, on protège uniquement son propre préfixe + et le nom de son ontologie ; les autres objets ``_ontology_`` restent + éligibles au nettoyage. + """ + referenced_keys = set() + legacy_patterns = [] + + for memory in memories: + ontology_uri = getattr(memory, "ontology_uri", None) + if ontology_uri: + try: + referenced_keys.add(parse_key(ontology_uri)) + continue + except ValueError: + pass + + memory_id = getattr(memory, "id", "") + ontology = getattr(memory, "ontology", "") + if memory_id and ontology: + legacy_patterns.append((f"{memory_id}/", f"__ontology_{ontology}.yaml")) + + return referenced_keys, legacy_patterns + + +def is_referenced_ontology_key(key, referenced_keys, legacy_patterns): + """Indique si une clé d'ontologie est encore rattachée à une mémoire.""" + if "_ontology_" not in key: + return False + if key in referenced_keys: + return True + return any( + key.startswith(prefix) and key.endswith(suffix) + for prefix, suffix in legacy_patterns + ) diff --git a/tests/test_storage_consistency.py b/tests/test_storage_consistency.py new file mode 100644 index 0000000..9898a68 --- /dev/null +++ b/tests/test_storage_consistency.py @@ -0,0 +1,68 @@ +from types import SimpleNamespace + +from src.mcp_memory.storage_consistency import ( + collect_referenced_ontology_keys, + is_referenced_ontology_key, +) + + +def _parse_key(value: str) -> str: + if value.startswith("s3://"): + parts = value[5:].split("/", 1) + if len(parts) != 2: + raise ValueError(value) + return parts[1] + return value + + +def test_referenced_ontology_is_protected_but_duplicate_is_orphan(): + memory = SimpleNamespace( + id="active-memory", + ontology="software-development", + ontology_uri=( + "s3://bucket/active-memory/documents/aaaa1111_" + "_ontology_software-development.yaml" + ), + ) + referenced, legacy = collect_referenced_ontology_keys([memory], _parse_key) + + assert is_referenced_ontology_key( + "active-memory/documents/aaaa1111__ontology_software-development.yaml", + referenced, + legacy, + ) + assert not is_referenced_ontology_key( + "active-memory/documents/bbbb2222__ontology_software-development.yaml", + referenced, + legacy, + ) + + +def test_deleted_memory_ontology_is_orphan(): + referenced, legacy = collect_referenced_ontology_keys([], _parse_key) + + assert not is_referenced_ontology_key( + "deleted-memory/documents/aaaa1111__ontology_general.yaml", + referenced, + legacy, + ) + + +def test_legacy_memory_without_uri_protects_only_its_expected_ontology(): + memory = SimpleNamespace( + id="legacy-memory", + ontology="legal", + ontology_uri=None, + ) + referenced, legacy = collect_referenced_ontology_keys([memory], _parse_key) + + assert is_referenced_ontology_key( + "legacy-memory/documents/aaaa1111__ontology_legal.yaml", + referenced, + legacy, + ) + assert not is_referenced_ontology_key( + "deleted-memory/documents/aaaa1111__ontology_legal.yaml", + referenced, + legacy, + ) diff --git a/tools/graph-memory-ingest/.gitignore b/tools/graph-memory-ingest/.gitignore new file mode 100644 index 0000000..ae3c172 --- /dev/null +++ b/tools/graph-memory-ingest/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/tools/graph-memory-ingest/README.md b/tools/graph-memory-ingest/README.md new file mode 100644 index 0000000..0ed28a5 --- /dev/null +++ b/tools/graph-memory-ingest/README.md @@ -0,0 +1,172 @@ +# graph-memory-ingest + +CLI Go d’ingestion de fichiers vers **Graph Memory** ou **Hivemind**, porté depuis +`hivemind-private/tools/hivemind-ingest` le 31 août 2026. Il utilise MCP Streamable +HTTP, avec initialisation de session, réponses JSON ou SSE et suivi des jobs. + +## Compilation et tests + +Go 1.27 ou supérieur ; aucune dépendance au SDK Python du serveur. + +```bash +cd tools/graph-memory-ingest +go build -o bin/graph-memory-ingest . +go test -count=1 ./... +go test -count=1 -race ./... +git diff --check +``` + +Le binaire fonctionne sous macOS et Linux. Aucun serveur n’est nécessaire pour +les tests unitaires : ils utilisent des serveurs HTTP locaux et des fichiers +temporaires. L’extraction réelle nécessite les accès LLM et embeddings du serveur. + +## Configuration + +Aucun endpoint, token ou espace n’est prédéfini. Priorité : **flags CLI**, puis +**variables d’environnement**, puis **fichier YAML**. Les variables `GRAPH_MEMORY_*` +priment sur leurs alias `HIVEMIND_*`. + +```bash +export GRAPH_MEMORY_ENDPOINT='http://127.0.0.1:8070/mcp' +export GRAPH_MEMORY_TOKEN='votre-jeton' +export GRAPH_MEMORY_SPACE='ma-memoire' + +./bin/graph-memory-ingest config test --json +./bin/graph-memory-ingest config get --json # token masqué +``` + +Le fichier par défaut est `~/.config/graph-memory/config.yaml` : + +```yaml +endpoint: http://127.0.0.1:8070/mcp +space_id: ma-memoire +batch_size_mb: 50 +allowed_extensions: [.md, .txt, .json, .yaml, .yml, .py, .go, .ts, .js, .pdf] +watch_jobs: true +timeout_seconds: 600 +threshold_other: 25 +``` + +`config set --endpoint URL --space ID` enregistre la configuration atomiquement, +avec fichier en mode `0600` et répertoire en `0700`. Préférer une variable +d’environnement au flag `--token` pour éviter de laisser le jeton dans l’historique +shell ou la liste des processus. + +| Variable `GRAPH_MEMORY_*` | Usage | +|---|---| +| `CONFIG_PATH` ou `CONFIG` | Chemin du fichier YAML | +| `ENDPOINT`, `TOKEN` | Connexion MCP et authentification Bearer | +| `SPACE_ID` ou `SPACE` | Mémoire Graph Memory / espace Hivemind | +| `ONTOLOGY` | Ontologie par défaut | +| `BATCH_SIZE_MB` | Taille maximale des données brutes d’un lot, défaut 50 Mio | +| `TIMEOUT_SECONDS` | Délai HTTP / suivi d’un job, défaut 600 s | +| `EXTENSIONS` | Extensions autorisées séparées par des virgules | +| `THRESHOLD_OTHER` | Tolérance d’évaluation de 0 à 100 %, défaut 25 | + +Les mêmes suffixes existent sous `HIVEMIND_*`. Le fichier Hivemind n’est pas chargé +implicitement ; on peut le sélectionner explicitement avec `HIVEMIND_CONFIG_PATH`. + +## Ingestion + +```bash +# Simulation entièrement locale, sans connexion ni écriture serveur +./bin/graph-memory-ingest run --path ./docs --space ma-memoire --dry-run --json + +# Ingestion et attente de la fin des jobs +./bin/graph-memory-ingest run --path ./docs --space ma-memoire --watch + +# Création d’une mémoire Graph Memory avec une ontologie nommée du serveur +./bin/graph-memory-ingest run --path ./docs --space nouvelle-memoire \ + --create-space-if-missing --ontology technical --watch + +# Automatisation : JSON uniquement sur stdout, sans interaction +./bin/graph-memory-ingest run --path ./docs --space ma-memoire \ + --non-interactive --json +``` + +Le scan est récursif. Il exclut les entrées cachées, les liens symboliques et les +fichiers spéciaux. Un contenu binaire n’est accepté que pour un format documentaire +explicitement autorisé : PDF, DOCX, XLSX, PPTX, ODT, ODS ou ODP. Les extensions hors +liste sont ignorées. Les fichiers dépassant la limite d’un lot sont refusés. + +La déduplication SHA-256 s’applique aux fichiers locaux et au catalogue distant. +`--replace` la désactive. Une erreur de catalogue (droits, transport, réponse +invalide) interrompt l’exécution ; seul un outil explicitement absent permet de +continuer sans catalogue. Les documents dont l’ingestion a échoué restent éligibles. + +Chaque lot envoie `content_base64`, `sha256`, `filename`, `source_path` et les +métadonnées de format. Le hash est recalculé sur les octets envoyés. La taille du +lot désigne les données **avant** base64 : prévoir environ un tiers de plus, plus +l’enveloppe JSON, dans les limites HTTP du serveur et du proxy. + +| Fonction | Hivemind | Graph Memory standalone | +|---|---|---| +| Vérification / création | `space_info` / `space_create` | `memory_list` / `memory_create` | +| Catalogue | `long_document_list` paginé | `document_list` sans pagination | +| Lot | `long_ingest_async` | `memory_ingest_batch_async` | +| Jobs actifs | `long_ingest_list` | `ingest_job_list` | +| Suivi | `long_ingest_status`, sinon `long_ingest_job_status` | `ingest_job_status` | + +Les replis ne sont tentés que si l’outil précédent est absent. Un refus d’accès ou +un échec serveur ne déclenche pas une deuxième soumission. `--ontology` choisit +l’ontologie lors de la **création d’une mémoire Graph Memory** ; il ne modifie pas +une mémoire existante. Hivemind utilise l’ontologie de son espace. + +Chaque fichier doit recevoir un résultat unique. Les chemins inconnus, réponses +manquantes, noms ambigus, job IDs dupliqués et statuts inconnus sont refusés. +Les statuts `succeeded`, `completed`, `skipped`, `changed_skipped`, `queued`, +`running`, `failed`, `error`, `queue_full` et `cancelled` sont traités explicitement. + +Avec `--no-poll` (ou `--watch=false`), le code 0 signifie que les soumissions ont +été acceptées. Les jobs encore en attente ne sont **pas** comptés dans +`total_succeeded`. Un timeout client n’annule pas les jobs déjà soumis au serveur. +Le terminal affiche la progression ; `--json` fournit un rapport structuré sans +séquences ANSI. `NO_COLOR` désactive les couleurs. + +## Évaluation d’ontologie + +`eval`, `test` et `test-ontology` désignent la même commande : + +```bash +./bin/graph-memory-ingest eval --path ./docs --ontology ./ontology.yaml \ + --sample-size 5 --threshold-other 20 --json +``` + +L’échantillon correspond aux premiers fichiers éligibles, dans l’ordre du scan. +Le CLI crée un espace `tmp-eval-...`, valide le YAML par `ontology_validate`, +soumet les documents avec `options.ontology_yaml`, suit les jobs, puis lit +`long_status(include_graph=true)`. Il calcule la proportion d’entités **Other + +Generic**, détaille les types et propose d’enrichir l’ontologie si le seuil est +dépassé. Une distribution absente ou incohérente est une erreur, jamais un score +positif par défaut. + +L’espace créé est supprimé après succès ou erreur, sauf avec `--keep-memory`. +Un échec de nettoyage est signalé avec l’identifiant de l’espace à reprendre. +`--space` peut fixer le nom de cet espace ; il doit être nouveau pour éviter de +mélanger les statistiques avec un corpus existant. `--timeout` borne l’évaluation +complète ; le nettoyage dispose ensuite de son propre délai de 30 secondes. + +**Limite d’intégration constatée au portage :** Graph Memory standalone ne fournit +pas ce workflow (`space_create`, `ontology_validate`, `long_status`). Le backend +Hivemind consulté ignore encore `options.ontology_yaml` et n’expose pas +`graph_stats.entity_types`. Le module CLI et ses tests sont portés, mais une +évaluation réelle exige que le backend applique le YAML demandé et fournisse la +distribution des types. La version consultée est refusée au moment du calcul, +sans produire de faux score. Ce portage ne modifie pas le serveur Hivemind. + +## Protocole et codes de retour + +L’initialisation suit `initialize` puis `notifications/initialized`. Les versions +acceptées sont strictement `2024-11-05` et `2024-10-07`, conformément à la source +transmise. Le client conserve `mcp-session-id` et `mcp-protocol-version`. Les flux +SSE peuvent contenir des notifications avant la réponse et plusieurs lignes +`data:` par événement ; le client termine à réception du résultat correspondant. +La réponse est limitée à 16 Mio. La reconnexion avec rejeu automatique et l’ancien +transport SSE à deux endpoints ne sont pas pris en charge. + +| Code | Signification | +|---|---| +| 0 | Succès ; avec `--no-poll`, soumission acceptée seulement | +| 1 | Arguments ou fichiers invalides | +| 2 | Connexion, authentification ou vérification serveur impossible | +| 3 | Échec d’ingestion, d’évaluation, de nettoyage ou seuil dépassé | diff --git a/tools/graph-memory-ingest/go.mod b/tools/graph-memory-ingest/go.mod new file mode 100644 index 0000000..8739d0b --- /dev/null +++ b/tools/graph-memory-ingest/go.mod @@ -0,0 +1,5 @@ +module graph-memory-ingest + +go 1.27.0 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/tools/graph-memory-ingest/go.sum b/tools/graph-memory-ingest/go.sum new file mode 100644 index 0000000..4bc0337 --- /dev/null +++ b/tools/graph-memory-ingest/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/graph-memory-ingest/internal/cli/app.go b/tools/graph-memory-ingest/internal/cli/app.go new file mode 100644 index 0000000..cb611a2 --- /dev/null +++ b/tools/graph-memory-ingest/internal/cli/app.go @@ -0,0 +1,579 @@ +package cli + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "time" + + "graph-memory-ingest/internal/config" + "graph-memory-ingest/internal/ingest" + "graph-memory-ingest/internal/mcpclient" + "graph-memory-ingest/internal/ontology" + "graph-memory-ingest/internal/tui" +) + +// Exit codes +const ( + ExitSuccess = 0 + ExitValidationError = 1 + ExitNetworkError = 2 + ExitIngestionFailure = 3 +) + +// App is the CLI application router +type App struct { + ui *tui.UI + cfg *config.Config +} + +// NewApp creates a new CLI App +func NewApp(ui *tui.UI, cfg *config.Config) *App { + return &App{ui: ui, cfg: cfg} +} + +// Run executes the application given command-line arguments +func (a *App) Run(ctx context.Context, args []string) int { + if len(args) < 1 { + a.printUsage() + return ExitValidationError + } + + cmd := args[0] + cmdArgs := args[1:] + + switch cmd { + case "run": + return a.handleRun(ctx, cmdArgs) + case "test-ontology", "test", "eval": + return a.handleTestOntology(ctx, cmdArgs) + case "config": + return a.handleConfig(ctx, cmdArgs) + case "help", "--help", "-h": + a.printUsage() + return ExitSuccess + case "version", "--version", "-v": + fmt.Println("graph-memory-ingest v1.0.0") + return ExitSuccess + default: + fmt.Fprintf(os.Stderr, "Unknown command: %s\n", cmd) + a.printUsage() + return ExitValidationError + } +} + +func (a *App) printUsage() { + a.ui.Banner() + fmt.Println("Usage:") + fmt.Println(" graph-memory-ingest [flags]") + fmt.Println() + fmt.Println("Commands:") + fmt.Println(" run Ingest documents into Hivemind Graph Memory") + fmt.Println(" test-ontology Evaluate ontology extraction fit against sample files") + fmt.Println(" config Manage server endpoint and authentication token") + fmt.Println(" help Display this help message") + fmt.Println(" version Display version information") + fmt.Println() + fmt.Println("Examples:") + fmt.Println(" graph-memory-ingest run --path ./docs --space my-team-space --ontology software --watch") + fmt.Println(" graph-memory-ingest test-ontology --path ./docs --ontology ./ontology.yaml --threshold-other 20") + fmt.Println(" graph-memory-ingest config set --endpoint $GRAPH_MEMORY_ENDPOINT --token $GRAPH_MEMORY_TOKEN") + fmt.Println(" graph-memory-ingest config test") +} + +func classifyError(err error) int { + if err == nil { + return ExitSuccess + } + s := strings.ToLower(err.Error()) + if strings.Contains(s, "http error") || + strings.Contains(s, "connection") || + strings.Contains(s, "connect:") || + strings.Contains(s, "network") || + strings.Contains(s, "mcp error") || + strings.Contains(s, "access denied") || + strings.Contains(s, "failed to verify or create space") || + strings.Contains(s, "failed to retrieve document catalog") || + strings.Contains(s, "failed to fetch remote document catalog") || + strings.Contains(s, "space verification failed") || + strings.Contains(s, "timeout") { + return ExitNetworkError + } + if strings.Contains(s, "required") || + strings.Contains(s, "invalid") || + strings.Contains(s, "threshold_other must be") || + strings.Contains(s, "not found") || + strings.Contains(s, "syntax") { + return ExitValidationError + } + return ExitIngestionFailure +} + +func (a *App) handleRun(ctx context.Context, args []string) int { + fs := flag.NewFlagSet("run", flag.ContinueOnError) + + var ( + pathFlag string + spaceFlag string + ontologyFlag string + createSpaceFlag bool + rulesFlag string + batchSizeMBFlag int + extensionsFlag string + replaceFlag bool + watchFlag bool + noPollFlag bool + dryRunFlag bool + jsonFlag bool + nonInteractive bool + endpointFlag string + tokenFlag string + timeoutFlag int + ) + + fs.StringVar(&pathFlag, "path", "", "Target file or directory to ingest") + fs.StringVar(&pathFlag, "p", "", "Target file or directory (shorthand)") + fs.StringVar(&spaceFlag, "space", a.cfg.SpaceID, "Target Graph Memory memory / Hivemind space ID") + fs.StringVar(&spaceFlag, "s", a.cfg.SpaceID, "Target Graph Memory memory / Hivemind space ID (shorthand)") + fs.StringVar(&ontologyFlag, "ontology", a.cfg.DefaultOntology, "Named ontology for Graph Memory memory creation") + fs.StringVar(&ontologyFlag, "o", a.cfg.DefaultOntology, "Named ontology for Graph Memory memory creation (shorthand)") + fs.BoolVar(&createSpaceFlag, "create-space-if-missing", false, "Automatically create the space if missing") + fs.StringVar(&rulesFlag, "rules", "standard", "Rules template if creating space (standard, software, minimal)") + fs.IntVar(&batchSizeMBFlag, "batch-size-mb", a.cfg.BatchSizeMB, "Max batch size in MB") + fs.StringVar(&extensionsFlag, "extensions", "", "Comma-separated allowed extensions") + fs.StringVar(&extensionsFlag, "x", "", "Comma-separated allowed extensions (shorthand)") + fs.BoolVar(&replaceFlag, "replace", false, "Re-ingest documents even if SHA256 matches") + fs.BoolVar(&watchFlag, "watch", a.cfg.WatchJobs, "Wait for asynchronous background ingestion jobs to complete") + fs.BoolVar(&noPollFlag, "no-poll", false, "Do not poll background job status") + fs.BoolVar(&dryRunFlag, "dry-run", false, "Simulate ingestion without server calls") + fs.BoolVar(&jsonFlag, "json", false, "Output results as formatted JSON") + fs.BoolVar(&nonInteractive, "non-interactive", false, "Disable interactive prompts") + fs.StringVar(&endpointFlag, "endpoint", a.cfg.Endpoint, "Override MCP endpoint URL") + fs.StringVar(&tokenFlag, "token", "", "Override auth token") + fs.IntVar(&timeoutFlag, "timeout", a.cfg.TimeoutSeconds, "Execution timeout in seconds") + + if err := fs.Parse(args); err != nil { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": err.Error()}) + } + return ExitValidationError + } + + if noPollFlag { + watchFlag = false + } + + // Interactive Wizard if missing essential parameters and TTY available + isInteractive := a.ui.IsInteractive() && !nonInteractive && !jsonFlag + if isInteractive { + if pathFlag == "" { + a.ui.Banner() + p, err := a.ui.Prompt("Enter target file or directory path", ".") + if err != nil { + return ExitValidationError + } + pathFlag = p + } + if spaceFlag == "" { + s, err := a.ui.Prompt("Enter target Hivemind space ID", "default-space") + if err != nil { + return ExitValidationError + } + spaceFlag = s + } + } + + if pathFlag == "" { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": "--path is required"}) + } else { + fmt.Fprintln(os.Stderr, "Error: --path is required") + } + return ExitValidationError + } + if spaceFlag == "" { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": "--space is required"}) + } else { + fmt.Fprintln(os.Stderr, "Error: --space is required") + } + return ExitValidationError + } + + endpoint := endpointFlag + if endpoint == "" { + endpoint = a.cfg.Endpoint + } + token := tokenFlag + if token == "" { + token = a.cfg.Token + } + + if !dryRunFlag && endpoint == "" { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": "MCP endpoint is not configured"}) + } else { + fmt.Fprintln(os.Stderr, "Error: MCP endpoint is not configured. Run 'graph-memory-ingest config set --endpoint ' or set GRAPH_MEMORY_ENDPOINT.") + } + return ExitNetworkError + } + + var allowedExts []string + if extensionsFlag != "" { + for _, e := range strings.Split(extensionsFlag, ",") { + trimmed := strings.TrimSpace(e) + if trimmed != "" { + if !strings.HasPrefix(trimmed, ".") { + trimmed = "." + trimmed + } + allowedExts = append(allowedExts, strings.ToLower(trimmed)) + } + } + } else { + allowedExts = a.cfg.AllowedExtensions + } + + if timeoutFlag <= 0 { + timeoutFlag = a.cfg.TimeoutSeconds + } + if timeoutFlag <= 0 { + timeoutFlag = 600 + } + + client := mcpclient.NewClient(endpoint, token, time.Duration(timeoutFlag)*time.Second) + engine := ingest.NewEngine(client) + + opts := ingest.IngestOptions{ + Path: pathFlag, + SpaceID: spaceFlag, + Ontology: ontologyFlag, + CreateSpaceIfMissing: createSpaceFlag, + RulesTemplate: rulesFlag, + BatchSizeMB: batchSizeMBFlag, + AllowedExtensions: allowedExts, + ForceReplace: replaceFlag, + WatchJobs: watchFlag, + DryRun: dryRunFlag, + Timeout: time.Duration(timeoutFlag) * time.Second, + } + + var cb ingest.ProgressCallback + if !jsonFlag { + cb = func(stage string, message string, current int, total int, file string, jobID string, status string, errStr string) { + if total > 0 { + a.ui.ProgressBar(current, total, message) + } else { + fmt.Println(a.ui.Dim("»"), message) + } + } + } + + res, err := engine.Run(ctx, opts, cb) + if err != nil { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{ + "status": "error", + "error": err.Error(), + }) + } else { + fmt.Fprintf(os.Stderr, "%s %s\n", a.ui.Red("Error:"), err.Error()) + } + return classifyError(err) + } + + if jsonFlag { + _ = a.ui.PrintJSON(res) + if res.Success { + return ExitSuccess + } + return ExitIngestionFailure + } + + fmt.Println() + if res.Success { + if watchFlag { + fmt.Printf("%s Ingestion completed successfully in %d ms.\n", a.ui.Green("✓"), res.DurationMs) + } else { + fmt.Printf("%s Submission accepted in %d ms; pending jobs have not been checked.\n", a.ui.Green("✓"), res.DurationMs) + } + fmt.Printf(" Space: %s | Uploaded: %d | Skipped: %d | Total Succeeded: %d\n", + a.ui.Bold(res.SpaceID), res.TotalUploaded, res.TotalSkipped, res.TotalSucceeded) + return ExitSuccess + } else { + fmt.Printf("%s Ingestion finished with failures (%d succeeded, %d failed).\n", + a.ui.Red("✗"), res.TotalSucceeded, res.TotalFailed) + return ExitIngestionFailure + } +} + +func (a *App) handleTestOntology(ctx context.Context, args []string) int { + fs := flag.NewFlagSet("test-ontology", flag.ContinueOnError) + + var ( + pathFlag string + ontologyFlag string + spaceFlag string + sampleSizeFlag int + thresholdFlag int + keepMemoryFlag bool + jsonFlag bool + endpointFlag string + tokenFlag string + timeoutFlag int + ) + + fs.StringVar(&pathFlag, "path", ".", "Target file or directory of samples") + fs.StringVar(&pathFlag, "p", ".", "Target file or directory (shorthand)") + fs.StringVar(&ontologyFlag, "ontology", a.cfg.DefaultOntology, "Ontology YAML path or raw YAML") + fs.StringVar(&ontologyFlag, "o", a.cfg.DefaultOntology, "Ontology YAML path or raw YAML (shorthand)") + fs.StringVar(&spaceFlag, "space", "", "Optional space ID (ephemeral space created if omitted)") + fs.IntVar(&sampleSizeFlag, "sample-size", 3, "Number of sample documents to evaluate") + fs.IntVar(&thresholdFlag, "threshold-other", a.cfg.ThresholdOther, "Max allowable percentage of untyped 'Other' entities") + fs.BoolVar(&keepMemoryFlag, "keep-memory", false, "Do not delete temporary evaluation space after testing") + fs.BoolVar(&jsonFlag, "json", false, "Output results as JSON") + fs.StringVar(&endpointFlag, "endpoint", a.cfg.Endpoint, "Override MCP endpoint URL") + fs.StringVar(&tokenFlag, "token", "", "Override auth token") + fs.IntVar(&timeoutFlag, "timeout", a.cfg.TimeoutSeconds, "Timeout in seconds") + + if err := fs.Parse(args); err != nil { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": err.Error()}) + } + return ExitValidationError + } + + endpoint := endpointFlag + if endpoint == "" { + endpoint = a.cfg.Endpoint + } + token := tokenFlag + if token == "" { + token = a.cfg.Token + } + + if timeoutFlag <= 0 { + timeoutFlag = a.cfg.TimeoutSeconds + } + if timeoutFlag <= 0 { + timeoutFlag = 600 + } + + client := mcpclient.NewClient(endpoint, token, time.Duration(timeoutFlag)*time.Second) + eval := ontology.NewEvaluator(client) + + opts := ontology.EvalOptions{ + SpaceID: spaceFlag, + Ontology: ontologyFlag, + SampleSize: sampleSizeFlag, + ThresholdOther: thresholdFlag, + KeepMemory: keepMemoryFlag, + AllowedExts: a.cfg.AllowedExtensions, + } + + evalCtx, cancel := context.WithTimeout(ctx, time.Duration(timeoutFlag)*time.Second) + defer cancel() + res, err := eval.TestOntology(evalCtx, pathFlag, opts) + if err != nil { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": err.Error()}) + } else { + fmt.Fprintf(os.Stderr, "%s %s\n", a.ui.Red("Error:"), err.Error()) + } + return classifyError(err) + } + + if jsonFlag { + _ = a.ui.PrintJSON(res) + if res.PassedThreshold { + return ExitSuccess + } + return ExitIngestionFailure + } + + fmt.Println() + a.ui.Banner() + if res.PassedThreshold { + fmt.Printf("%s %s\n", a.ui.Green("✓"), res.Message) + } else { + fmt.Printf("%s %s\n", a.ui.Yellow("⚠"), res.Message) + } + fmt.Printf(" Ontology: %s | Evaluated Files: %d | Space: %s\n", + a.ui.Bold(res.OntologyPath), res.SampleCount, a.ui.Bold(res.SpaceID)) + fmt.Printf(" Relevance Tier: %s\n", a.ui.Bold(res.RelevanceTier)) + fmt.Printf(" Total Entities: %d (Typed: %d, Other: %d -> %.1f%%)\n", + res.TotalEntities, res.TypedEntities, res.OtherEntities, res.OtherPercentage) + + if len(res.EntityTypesBreakdown) > 0 { + fmt.Println("\n Discovered Entity Types:") + for tName, count := range res.EntityTypesBreakdown { + fmt.Printf(" • %-20s: %d\n", tName, count) + } + } + + if len(res.UnclassifiedConcepts) > 0 { + fmt.Println("\n Unclassified 'Other' Concepts:") + for _, concept := range res.UnclassifiedConcepts { + fmt.Printf(" • %s\n", concept.Name) + } + } + + if len(res.Suggestions) > 0 { + fmt.Println("\n Recommendations:") + for _, sug := range res.Suggestions { + fmt.Printf(" → %s\n", sug) + } + } + + if res.IsTemporarySpace { + if keepMemoryFlag { + fmt.Printf("\n %s Temporary space '%s' preserved (--keep-memory).\n", a.ui.Dim("»"), res.SpaceID) + } else { + fmt.Printf("\n %s Ephemeral space '%s' cleaned up.\n", a.ui.Dim("»"), res.SpaceID) + } + } + + if res.PassedThreshold { + return ExitSuccess + } + return ExitIngestionFailure +} + +func (a *App) handleConfig(ctx context.Context, args []string) int { + if len(args) < 1 { + fmt.Println("Usage: graph-memory-ingest config ") + return ExitValidationError + } + + subCmd := args[0] + subArgs := args[1:] + + switch subCmd { + case "get": + var jsonFlag bool + fs := flag.NewFlagSet("config get", flag.ContinueOnError) + fs.BoolVar(&jsonFlag, "json", false, "Output config as JSON") + if err := fs.Parse(subArgs); err != nil { + return ExitValidationError + } + + if jsonFlag { + safeCfg := *a.cfg + safeCfg.Token = config.MaskToken(safeCfg.Token) + _ = a.ui.PrintJSON(safeCfg) + return ExitSuccess + } + + fmt.Printf("Config File: %s\n", config.GetConfigPath()) + fmt.Printf(" Endpoint: %s\n", a.cfg.Endpoint) + fmt.Printf(" Token: %s\n", config.MaskToken(a.cfg.Token)) + fmt.Printf(" Default Space: %s\n", a.cfg.SpaceID) + fmt.Printf(" Default Ontology: %s\n", a.cfg.DefaultOntology) + fmt.Printf(" Batch Size (MB): %d\n", a.cfg.BatchSizeMB) + fmt.Printf(" Threshold Other: %d%%\n", a.cfg.ThresholdOther) + fmt.Printf(" Allowed Exts: %s\n", strings.Join(a.cfg.AllowedExtensions, ", ")) + fmt.Printf(" Timeout (s): %d\n", a.cfg.TimeoutSeconds) + return ExitSuccess + + case "set": + fs := flag.NewFlagSet("config set", flag.ContinueOnError) + var ( + endpointFlag string + tokenFlag string + spaceFlag string + ontologyFlag string + batchSizeMBFlag int + thresholdFlag int + timeoutFlag int + ) + fs.StringVar(&endpointFlag, "endpoint", "", "MCP endpoint URL") + fs.StringVar(&tokenFlag, "token", "", "Authentication token") + fs.StringVar(&spaceFlag, "space", "", "Default space ID") + fs.StringVar(&ontologyFlag, "ontology", "", "Default ontology") + fs.IntVar(&batchSizeMBFlag, "batch-size-mb", 0, "Max batch size in MB") + fs.IntVar(&thresholdFlag, "threshold-other", -1, "Default Other threshold percentage") + fs.IntVar(&timeoutFlag, "timeout", 0, "Default timeout in seconds") + + if err := fs.Parse(subArgs); err != nil { + return ExitValidationError + } + + if endpointFlag != "" { + a.cfg.Endpoint = endpointFlag + } + if tokenFlag != "" { + a.cfg.Token = tokenFlag + } + if spaceFlag != "" { + a.cfg.SpaceID = spaceFlag + } + if ontologyFlag != "" { + a.cfg.DefaultOntology = ontologyFlag + } + if batchSizeMBFlag > 0 { + a.cfg.BatchSizeMB = batchSizeMBFlag + } + if thresholdFlag >= 0 && thresholdFlag <= 100 { + a.cfg.ThresholdOther = thresholdFlag + } + if timeoutFlag > 0 { + a.cfg.TimeoutSeconds = timeoutFlag + } + + if err := config.Save(a.cfg); err != nil { + fmt.Fprintf(os.Stderr, "%s Failed to save configuration: %v\n", a.ui.Red("Error:"), err) + return ExitValidationError + } + fmt.Printf("%s Configuration saved to %s (permissions 0600)\n", a.ui.Green("✓"), config.GetConfigPath()) + return ExitSuccess + + case "test": + var jsonFlag bool + fs := flag.NewFlagSet("config test", flag.ContinueOnError) + fs.BoolVar(&jsonFlag, "json", false, "Output test result as JSON") + if err := fs.Parse(subArgs); err != nil { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": err.Error()}) + } + return ExitValidationError + } + + if a.cfg.Endpoint == "" { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": "Endpoint is not configured"}) + } else { + fmt.Fprintln(os.Stderr, "Error: Endpoint is not configured.") + } + return ExitNetworkError + } + + client := mcpclient.NewClient(a.cfg.Endpoint, a.cfg.Token, time.Duration(a.cfg.TimeoutSeconds)*time.Second) + res, err := client.CallTool(ctx, "system_whoami", map[string]interface{}{}) + if err == nil && (res["isError"] == true || res["status"] == "error") { + err = fmt.Errorf("system_whoami failed: %v", res["message"]) + } + if err != nil { + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "error", "error": err.Error()}) + } else { + fmt.Fprintf(os.Stderr, "%s Connection failed: %v\n", a.ui.Red("✗"), err) + } + return ExitNetworkError + } + + if jsonFlag { + _ = a.ui.PrintJSON(map[string]interface{}{"status": "ok", "whoami": res}) + } else { + fmt.Printf("%s Connected to MCP endpoint %s\n", a.ui.Green("✓"), a.cfg.Endpoint) + if user, ok := res["user"].(string); ok && user != "" { + fmt.Printf(" Authenticated as: %s\n", a.ui.Bold(user)) + } + } + return ExitSuccess + + default: + fmt.Fprintf(os.Stderr, "Unknown config subcommand: %s\n", subCmd) + return ExitValidationError + } +} diff --git a/tools/graph-memory-ingest/internal/cli/app_test.go b/tools/graph-memory-ingest/internal/cli/app_test.go new file mode 100644 index 0000000..54560f5 --- /dev/null +++ b/tools/graph-memory-ingest/internal/cli/app_test.go @@ -0,0 +1,158 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "graph-memory-ingest/internal/config" + "graph-memory-ingest/internal/mcpclient" + "graph-memory-ingest/internal/tui" +) + +func TestAppCLIEndToEnd(t *testing.T) { + tempDir := t.TempDir() + docFile := filepath.Join(tempDir, "manual.md") + _ = os.WriteFile(docFile, []byte("# Project Manual\nArchitecture guide"), 0644) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req mcpclient.JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + w.Header().Set("Content-Type", "application/json") + + if req.Method == "initialize" { + w.Header().Set("mcp-session-id", "mock-session-id") + w.Header().Set("mcp-protocol-version", "2024-11-05") + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + + if req.Method == "notifications/initialized" { + w.WriteHeader(http.StatusOK) + return + } + + params, _ := req.Params.(map[string]interface{}) + name, _ := params["name"].(string) + + var resBody string + switch name { + case "system_whoami": + resBody = `{"status":"ok","user":"ci-runner"}` + case "space_info": + resBody = `{"status":"ok","space_id":"test-space"}` + case "space_create": + resBody = `{"status":"created"}` + case "space_delete": + resBody = `{"status":"ok"}` + case "long_ingest_list", "long_document_list": + resBody = `{"status":"ok","documents":[]}` + case "long_ingest_async", "long_ingest_document": + resBody = `{"status":"ok","batch_id":"batch-1","total":1,"counts":{"queued":1},"items":[{"index":0,"source_path":"manual.md","job_id":"job-test-1","status":"queued"}],"errors":[]}` + case "long_ingest_status", "long_ingest_job_status": + resBody = `{"status":"succeeded","job_id":"job-test-1"}` + case "ontology_validate": + resBody = `{"status":"ok","valid":true}` + case "long_status": + resBody = `{"status":"ok","graph_stats":{"entities_count":2,"entity_types":{"Document":1,"Concept":1}}}` + case "long_test_ontology": + resBody = `{"status":"ok","entities":[{"name":"Project Manual","type":"Document"},{"name":"Architecture","type":"Concept"}],"relations":[]}` + default: + resBody = `{"status":"ok"}` + } + + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"content":[{"type":"text","text":` + string(mustJSON(resBody)) + `}],"isError":false}`), + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + cfg := &config.Config{ + Endpoint: server.URL, + Token: "ci-token", + SpaceID: "test-space", + BatchSizeMB: 50, + TimeoutSeconds: 10, + WatchJobs: true, + ThresholdOther: 25, + AllowedExtensions: []string{".md"}, + } + + var in bytes.Buffer + var out bytes.Buffer + ui := tui.NewUI(&in, &out) + app := NewApp(ui, cfg) + + // 1. Test config test (code 0) + out.Reset() + code := app.Run(context.Background(), []string{"config", "test", "--json"}) + if code != ExitSuccess { + t.Fatalf("config test failed with exit code %d, output: %s", code, out.String()) + } + if !strings.Contains(out.String(), "ci-runner") { + t.Errorf("expected output to contain ci-runner, got %s", out.String()) + } + + // 2. Test test-ontology (code 0) + out.Reset() + code = app.Run(context.Background(), []string{"test-ontology", "--path", tempDir, "--ontology", "software", "--json"}) + if code != ExitSuccess { + t.Fatalf("test-ontology failed with exit code %d, output: %s", code, out.String()) + } + if !strings.Contains(out.String(), `"passed_threshold": true`) { + t.Errorf("expected passed_threshold true, got %s", out.String()) + } + + // 3. Test run dry-run (code 0) + out.Reset() + code = app.Run(context.Background(), []string{"run", "--path", tempDir, "--space", "test-space", "--dry-run", "--json"}) + if code != ExitSuccess { + t.Fatalf("run --dry-run failed with code %d, output: %s", code, out.String()) + } + if !strings.Contains(out.String(), `"total_scanned": 1`) { + t.Errorf("expected total_scanned 1, got %s", out.String()) + } + + // 4. Test run real ingest with timeout flag (code 0) + out.Reset() + code = app.Run(context.Background(), []string{"run", "--path", tempDir, "--space", "test-space", "--timeout", "5", "--json"}) + if code != ExitSuccess { + t.Fatalf("run failed with code %d, output: %s", code, out.String()) + } + if !strings.Contains(out.String(), `"total_succeeded": 1`) { + t.Errorf("expected total_succeeded 1, got %s", out.String()) + } + + // 5. Test validation error (code 1) + out.Reset() + code = app.Run(context.Background(), []string{"run", "--invalid-flag-123"}) + if code != ExitValidationError { + t.Errorf("expected ExitValidationError (1) on invalid flag, got %d", code) + } + + // 6. Test network error (code 2) + out.Reset() + code = app.Run(context.Background(), []string{"run", "--path", tempDir, "--space", "test-space", "--endpoint", "http://127.0.0.1:54321/down", "--non-interactive", "--json"}) + if code != ExitNetworkError { + t.Errorf("expected ExitNetworkError (2) on network failure, got %d", code) + } +} + +func mustJSON(s string) []byte { + b, _ := json.Marshal(s) + return b +} diff --git a/tools/graph-memory-ingest/internal/cli/port_test.go b/tools/graph-memory-ingest/internal/cli/port_test.go new file mode 100644 index 0000000..9ca6a66 --- /dev/null +++ b/tools/graph-memory-ingest/internal/cli/port_test.go @@ -0,0 +1,64 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "graph-memory-ingest/internal/config" + "graph-memory-ingest/internal/mcpclient" + "graph-memory-ingest/internal/tui" +) + +func TestHelpDoesNotExposeConfiguredToken(t *testing.T) { + cfg := config.DefaultConfig() + cfg.Token = "secret-that-must-not-appear" + var output bytes.Buffer + app := NewApp(tui.NewUI(strings.NewReader(""), &output), cfg) + file, err := os.CreateTemp(t.TempDir(), "stderr") + if err != nil { + t.Fatal(err) + } + defer file.Close() + original := os.Stderr + os.Stderr = file + defer func() { os.Stderr = original }() + app.Run(context.Background(), []string{"run", "--help"}) + data, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data)+output.String(), cfg.Token) { + t.Fatal("help leaked the configured token") + } +} + +func TestConfigTestRejectsToolError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req mcpclient.JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if req.Method == "notifications/initialized" { + w.WriteHeader(202) + return + } + result := json.RawMessage(`{"content":[{"type":"text","text":"denied"}],"isError":true}`) + if req.Method == "initialize" { + result = json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test"}}`) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(mcpclient.JSONRPCResponse{JSONRPC: "2.0", ID: req.ID, Result: result}) + })) + defer server.Close() + cfg := config.DefaultConfig() + cfg.Endpoint = server.URL + var output bytes.Buffer + code := NewApp(tui.NewUI(strings.NewReader(""), &output), cfg).Run(context.Background(), []string{"config", "test", "--json"}) + if code != ExitNetworkError || !strings.Contains(output.String(), `"status": "error"`) { + t.Fatalf("false positive connection test: code=%d %s", code, output.String()) + } +} diff --git a/tools/graph-memory-ingest/internal/config/config.go b/tools/graph-memory-ingest/internal/config/config.go new file mode 100644 index 0000000..5406348 --- /dev/null +++ b/tools/graph-memory-ingest/internal/config/config.go @@ -0,0 +1,209 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +const ( + DefaultDirMode = 0700 + DefaultFileMode = 0600 +) + +// Config holds all parameters for graph-memory-ingest +type Config struct { + Endpoint string `yaml:"endpoint"` + Token string `yaml:"token"` + SpaceID string `yaml:"space_id"` + DefaultOntology string `yaml:"default_ontology"` + BatchSizeMB int `yaml:"batch_size_mb"` + ThresholdOther int `yaml:"threshold_other"` + AllowedExtensions []string `yaml:"allowed_extensions"` + WatchJobs bool `yaml:"watch_jobs"` + TimeoutSeconds int `yaml:"timeout_seconds"` +} + +// DefaultConfig returns reasonable defaults +func DefaultConfig() *Config { + return &Config{ + Endpoint: "", + Token: "", + SpaceID: "", + DefaultOntology: "", + BatchSizeMB: 50, + ThresholdOther: 25, + AllowedExtensions: []string{".md", ".txt", ".json", ".yaml", ".yml", ".py", ".go", ".ts", ".js", ".pdf"}, + WatchJobs: true, + TimeoutSeconds: 600, + } +} + +// GetConfigPath returns the canonical path ~/.config/graph-memory/config.yaml +func env(key string) string { + if value := os.Getenv("GRAPH_MEMORY_" + key); value != "" { + return value + } + return os.Getenv("HIVEMIND_" + key) +} + +func GetConfigPath() string { + for _, key := range []string{"GRAPH_MEMORY_CONFIG_PATH", "GRAPH_MEMORY_CONFIG", "HIVEMIND_CONFIG_PATH", "HIVEMIND_CONFIG"} { + if custom := os.Getenv(key); custom != "" { + return custom + } + } + home, err := os.UserHomeDir() + if err != nil { + home = "." + } + return filepath.Join(home, ".config", "graph-memory", "config.yaml") +} + +// Load reads config from file, then overrides with environment variables +func Load() (*Config, error) { + cfg := DefaultConfig() + path := GetConfigPath() + + if _, err := os.Stat(path); err == nil { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read config file %s: %w", path, err) + } + if err := yaml.Unmarshal(data, cfg); err != nil { + return nil, fmt.Errorf("failed to parse config file %s: %w", path, err) + } + } + + // Environment variable overrides + if env := env("ENDPOINT"); env != "" { + cfg.Endpoint = env + } + if env := env("TOKEN"); env != "" { + cfg.Token = env + } + for _, key := range []string{"GRAPH_MEMORY_SPACE_ID", "GRAPH_MEMORY_SPACE", "HIVEMIND_SPACE_ID", "HIVEMIND_SPACE"} { + if value := os.Getenv(key); value != "" { + cfg.SpaceID = value + break + } + } + if env := env("ONTOLOGY"); env != "" { + cfg.DefaultOntology = env + } + if env := env("BATCH_SIZE_MB"); env != "" { + if val, err := strconv.Atoi(env); err == nil && val > 0 { + cfg.BatchSizeMB = val + } + } + if env := env("THRESHOLD_OTHER"); env != "" { + if val, err := strconv.Atoi(env); err == nil && val >= 0 && val <= 100 { + cfg.ThresholdOther = val + } + } + if env := env("EXTENSIONS"); env != "" { + parts := strings.Split(env, ",") + var exts []string + for _, p := range parts { + trimmed := strings.TrimSpace(p) + if trimmed != "" { + if !strings.HasPrefix(trimmed, ".") { + trimmed = "." + trimmed + } + exts = append(exts, strings.ToLower(trimmed)) + } + } + if len(exts) > 0 { + cfg.AllowedExtensions = exts + } + } + if env := env("TIMEOUT_SECONDS"); env != "" { + if val, err := strconv.Atoi(env); err == nil && val > 0 { + cfg.TimeoutSeconds = val + } + } + + return cfg, nil +} + +// Save writes the configuration to disk with strict 0600 permissions atomically +func Save(cfg *Config) error { + path := GetConfigPath() + dir := filepath.Dir(path) + + if err := os.MkdirAll(dir, DefaultDirMode); err != nil { + return fmt.Errorf("failed to create config directory %s: %w", dir, err) + } + if err := os.Chmod(dir, DefaultDirMode); err != nil { + return fmt.Errorf("failed to enforce %v permissions on %s: %w", DefaultDirMode, dir, err) + } + + data, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + + tmpFile, err := os.CreateTemp(dir, ".config-*.tmp") + if err != nil { + return fmt.Errorf("failed to create temporary config file in %s: %w", dir, err) + } + tmpPath := tmpFile.Name() + + if err := os.Chmod(tmpPath, DefaultFileMode); err != nil { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("failed to set 0600 permissions on %s: %w", tmpPath, err) + } + + if _, err := tmpFile.Write(data); err != nil { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("failed to write config data to %s: %w", tmpPath, err) + } + + if err := tmpFile.Sync(); err != nil { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("failed to sync config file %s: %w", tmpPath, err) + } + + if err := tmpFile.Close(); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("failed to close temporary config file %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, path); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("failed to atomically rename config file %s to %s: %w", tmpPath, path, err) + } + + if err := os.Chmod(path, DefaultFileMode); err != nil { + return fmt.Errorf("failed to set %v permissions on %s: %w", DefaultFileMode, path, err) + } + + d, err := os.Open(dir) + if err != nil { + return fmt.Errorf("failed to open parent directory %s for sync: %w", dir, err) + } + if err := d.Sync(); err != nil { + _ = d.Close() + return fmt.Errorf("failed to sync parent directory %s: %w", dir, err) + } + if err := d.Close(); err != nil { + return fmt.Errorf("failed to close parent directory %s: %w", dir, err) + } + + return nil +} + +// MaskToken returns a masked representation of the token for secure display +func MaskToken(token string) string { + if token == "" { + return "" + } + return "tok_***" +} diff --git a/tools/graph-memory-ingest/internal/config/config_test.go b/tools/graph-memory-ingest/internal/config/config_test.go new file mode 100644 index 0000000..68e7b02 --- /dev/null +++ b/tools/graph-memory-ingest/internal/config/config_test.go @@ -0,0 +1,100 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestConfigLoadAndSave(t *testing.T) { + tempDir := t.TempDir() + customPath := filepath.Join(tempDir, "test-config.yaml") + + t.Setenv("HIVEMIND_CONFIG_PATH", customPath) + t.Setenv("HIVEMIND_ENDPOINT", "http://localhost:9999/mcp") + t.Setenv("HIVEMIND_TOKEN", "test-secret-token") + t.Setenv("HIVEMIND_SPACE_ID", "custom-space") + t.Setenv("HIVEMIND_BATCH_SIZE_MB", "100") + t.Setenv("HIVEMIND_TIMEOUT_SECONDS", "300") + t.Setenv("HIVEMIND_THRESHOLD_OTHER", "15") + t.Setenv("HIVEMIND_EXTENSIONS", ".txt,.md,.custom") + + cfg, err := Load() + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + + if cfg.Endpoint != "http://localhost:9999/mcp" { + t.Errorf("expected endpoint http://localhost:9999/mcp, got %s", cfg.Endpoint) + } + if cfg.Token != "test-secret-token" { + t.Errorf("expected token test-secret-token, got %s", cfg.Token) + } + if cfg.SpaceID != "custom-space" { + t.Errorf("expected space custom-space, got %s", cfg.SpaceID) + } + if cfg.BatchSizeMB != 100 { + t.Errorf("expected BatchSizeMB 100, got %d", cfg.BatchSizeMB) + } + if cfg.TimeoutSeconds != 300 { + t.Errorf("expected TimeoutSeconds 300, got %d", cfg.TimeoutSeconds) + } + if cfg.ThresholdOther != 15 { + t.Errorf("expected ThresholdOther 15, got %d", cfg.ThresholdOther) + } + if len(cfg.AllowedExtensions) != 3 { + t.Errorf("expected 3 extensions, got %d", len(cfg.AllowedExtensions)) + } + + // Test Save (creates 0600 file) + err = Save(cfg) + if err != nil { + t.Fatalf("Save() failed: %v", err) + } + + info, err := os.Stat(customPath) + if err != nil { + t.Fatalf("stat failed: %v", err) + } + if info.Mode().Perm() != 0600 { + t.Errorf("expected 0600 permissions, got %o", info.Mode().Perm()) + } + + // Test Atomic Save over existing file + cfg.BatchSizeMB = 200 + err = Save(cfg) + if err != nil { + t.Fatalf("atomic Save() failed on existing file: %v", err) + } + info2, err := os.Stat(customPath) + if err != nil || info2.Mode().Perm() != 0600 { + t.Errorf("expected 0600 permissions after update, got %o (err=%v)", info2.Mode().Perm(), err) + } + + // Test MaskToken + masked := MaskToken("secret-bearer-token-12345") + if masked != "tok_***" { + t.Errorf("unexpected masked token: %s", masked) + } + if emptyMask := MaskToken(""); emptyMask != "" { + t.Errorf("expected for empty token, got %s", emptyMask) + } +} + +func TestGraphMemoryOverridesHivemind(t *testing.T) { + t.Setenv("GRAPH_MEMORY_CONFIG_PATH", filepath.Join(t.TempDir(), "config.yaml")) + t.Setenv("GRAPH_MEMORY_ENDPOINT", "https://graph.example/mcp") + t.Setenv("HIVEMIND_ENDPOINT", "https://hive.example/mcp") + t.Setenv("GRAPH_MEMORY_TOKEN", "graph-token") + t.Setenv("HIVEMIND_TOKEN", "hive-token") + t.Setenv("GRAPH_MEMORY_SPACE", "graph-space") + t.Setenv("HIVEMIND_SPACE_ID", "hive-space") + cfg, err := Load() + if err != nil || cfg.Endpoint != "https://graph.example/mcp" || cfg.Token != "graph-token" || cfg.SpaceID != "graph-space" { + t.Fatalf("cfg=%+v err=%v", cfg, err) + } + defaults := DefaultConfig() + if defaults.Endpoint != "" || defaults.SpaceID != "" || defaults.Token != "" { + t.Fatal("deployment values must not be hard-coded") + } +} diff --git a/tools/graph-memory-ingest/internal/ingest/engine.go b/tools/graph-memory-ingest/internal/ingest/engine.go new file mode 100644 index 0000000..b62c220 --- /dev/null +++ b/tools/graph-memory-ingest/internal/ingest/engine.go @@ -0,0 +1,590 @@ +package ingest + +import ( + "context" + "fmt" + "strings" + "time" + + "graph-memory-ingest/internal/mcpclient" + "graph-memory-ingest/internal/scanner" +) + +// ProgressCallback is invoked during scanning and ingestion for TUI/CLI updates +type ProgressCallback func(stage string, message string, current int, total int, file string, jobID string, status string, errStr string) + +// IngestOptions configures an ingestion run +type IngestOptions struct { + Path string + SpaceID string + Ontology string + BatchSizeMB int + MaxFileBytes int64 + AllowedExtensions []string + DryRun bool + ForceReplace bool + CreateSpaceIfMissing bool + RulesTemplate string + WatchJobs bool + Timeout time.Duration + NonInteractive bool +} + +// IngestResult captures the overall outcome of an ingestion execution +type IngestResult struct { + SpaceID string `json:"space_id"` + TotalScanned int `json:"total_scanned"` + TotalUploaded int `json:"total_uploaded"` + TotalSkipped int `json:"total_skipped"` + TotalSucceeded int `json:"total_succeeded"` + TotalFailed int `json:"total_failed"` + BatchesCount int `json:"batches_count"` + DurationMs int64 `json:"duration_ms"` + Jobs []*JobRecord `json:"jobs"` + Success bool `json:"success"` +} + +// JobRecord represents the status of an ingested file/job +type JobRecord struct { + JobID string `json:"job_id"` + Filename string `json:"filename"` + SHA256 string `json:"sha256"` + Status string `json:"status"` // queued, running, succeeded, failed, skipped + Error string `json:"error,omitempty"` +} + +// Engine coordinates scanning, space verification, batching, and async ingestion +type Engine struct { + client *mcpclient.Client +} + +// NewEngine creates a new Engine instance +func NewEngine(client *mcpclient.Client) *Engine { + return &Engine{client: client} +} + +// Run executes the ingestion workflow +func (e *Engine) Run(ctx context.Context, opts IngestOptions, callback ProgressCallback) (*IngestResult, error) { + start := time.Now() + res := &IngestResult{ + SpaceID: opts.SpaceID, + Jobs: make([]*JobRecord, 0), + } + + report := func(stage, msg string, current, total int, file, jobID, status, errStr string) { + if callback != nil { + callback(stage, msg, current, total, file, jobID, status, errStr) + } + } + + // 1. Dry-run is 100% offline and local + if opts.DryRun { + report("scanning", fmt.Sprintf("Scanning %s locally (dry-run mode)...", opts.Path), 0, 0, "", "", "", "") + scanRes, err := scanner.Scan(scanner.ScanOptions{ + RootPath: opts.Path, + AllowedExtensions: opts.AllowedExtensions, + BatchSizeMB: opts.BatchSizeMB, + MaxFileBytes: opts.MaxFileBytes, + ForceReplace: opts.ForceReplace, + }) + if err != nil { + return nil, fmt.Errorf("scanning failed: %w", err) + } + + res.TotalScanned = scanRes.TotalFiles + scanRes.SkippedCount + res.TotalSkipped = scanRes.SkippedCount + res.BatchesCount = len(scanRes.Batches) + + for _, skippedPath := range scanRes.SkippedFiles { + res.Jobs = append(res.Jobs, &JobRecord{ + Filename: skippedPath, + Status: "skipped", + }) + } + for _, b := range scanRes.Batches { + for _, f := range b.Files { + res.Jobs = append(res.Jobs, &JobRecord{ + Filename: f.RelPath, + SHA256: f.SHA256, + Status: "dry_run", + }) + } + } + + res.Success = true + res.DurationMs = time.Since(start).Milliseconds() + report("dry_run", fmt.Sprintf("Dry-run complete. Discovered %d files in %d batches.", scanRes.TotalFiles, len(scanRes.Batches)), scanRes.TotalFiles, scanRes.TotalFiles, "", "", "dry_run", "") + return res, nil + } + + // 2. Ensure space exists or auto-create if requested + report("verifying_space", fmt.Sprintf("Verifying space '%s'...", opts.SpaceID), 0, 0, "", "", "", "") + if err := e.EnsureSpace(ctx, opts.SpaceID, opts.CreateSpaceIfMissing, opts.RulesTemplate, opts.Ontology); err != nil { + return nil, fmt.Errorf("space verification failed: %w", err) + } + + // 3. Inspect active running jobs in space to notify operator + activeJobs, _ := e.FetchActiveJobs(ctx, opts.SpaceID) + if activeJobs > 0 { + report("active_jobs", fmt.Sprintf("Notice: %d active ingestion job(s) in space '%s'.", activeJobs, opts.SpaceID), 0, 0, "", "", "", "") + } + + // 4. Fetch known hashes for deduplication (unless force replace is specified) + var knownHashes map[string]bool + if !opts.ForceReplace { + report("fetching_hashes", "Checking remote document catalog for SHA-256 deduplication...", 0, 0, "", "", "", "") + hashes, err := e.FetchKnownHashes(ctx, opts.SpaceID) + if err != nil { + return nil, fmt.Errorf("failed to fetch remote document catalog: %w", err) + } + if len(hashes) > 0 { + knownHashes = hashes + report("hashes_loaded", fmt.Sprintf("Loaded %d existing document fingerprint(s).", len(hashes)), 0, 0, "", "", "", "") + } + } + + // 5. Scan directory + report("scanning", fmt.Sprintf("Scanning %s...", opts.Path), 0, 0, "", "", "", "") + scanRes, err := scanner.Scan(scanner.ScanOptions{ + RootPath: opts.Path, + AllowedExtensions: opts.AllowedExtensions, + BatchSizeMB: opts.BatchSizeMB, + MaxFileBytes: opts.MaxFileBytes, + KnownHashes: knownHashes, + ForceReplace: opts.ForceReplace, + }) + if err != nil { + return nil, fmt.Errorf("scanning failed: %w", err) + } + + res.TotalScanned = scanRes.TotalFiles + scanRes.SkippedCount + res.TotalSkipped = scanRes.SkippedCount + res.BatchesCount = len(scanRes.Batches) + + for _, skippedPath := range scanRes.SkippedFiles { + res.Jobs = append(res.Jobs, &JobRecord{ + Filename: skippedPath, + Status: "skipped", + }) + } + + report("scanned", fmt.Sprintf("Discovered %d new files to ingest (%d skipped as unchanged).", scanRes.TotalFiles, scanRes.SkippedCount), scanRes.TotalFiles, scanRes.TotalFiles, "", "", "", "") + + if scanRes.TotalFiles == 0 { + res.Success = true + res.DurationMs = time.Since(start).Milliseconds() + report("done", "No new files to ingest. Everything is up to date.", 0, 0, "", "", "finished", "") + return res, nil + } + + // 6. Ingestion per batch + totalProcessed := 0 + for batchIdx, batch := range scanRes.Batches { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + report("batch_starting", fmt.Sprintf("Processing batch [%d/%d] (%d files)...", batchIdx+1, len(scanRes.Batches), len(batch.Files)), totalProcessed, scanRes.TotalFiles, "", "", "uploading", "") + + batchJobs, err := e.IngestBatch(ctx, opts.SpaceID, batch, opts.Ontology, opts.MaxFileBytes, opts.ForceReplace) + if err != nil { + for _, file := range batch.Files { + totalProcessed++ + res.TotalFailed++ + rec := &JobRecord{ + Filename: file.RelPath, + SHA256: file.SHA256, + Status: "failed", + Error: err.Error(), + } + res.Jobs = append(res.Jobs, rec) + report("upload_failed", fmt.Sprintf("Failed batch on %s: %v", file.RelPath, err), totalProcessed, scanRes.TotalFiles, file.RelPath, "", "failed", err.Error()) + } + continue + } + + for _, jobRec := range batchJobs { + totalProcessed++ + st := strings.ToLower(jobRec.Status) + switch st { + case "failed", "error", "queue_full", "cancelled", "rejected": + jobRec.Status = "failed" + res.TotalFailed++ + res.Jobs = append(res.Jobs, jobRec) + report("upload_failed", fmt.Sprintf("Failed to upload %s: %s", jobRec.Filename, jobRec.Error), totalProcessed, scanRes.TotalFiles, jobRec.Filename, "", "failed", jobRec.Error) + case "skipped", "changed_skipped": + jobRec.Status = "skipped" + res.TotalSkipped++ + res.Jobs = append(res.Jobs, jobRec) + report("skipped", fmt.Sprintf("Skipped %s (%s)", jobRec.Filename, jobRec.Error), totalProcessed, scanRes.TotalFiles, jobRec.Filename, "", "skipped", "") + case "succeeded", "completed": + jobRec.Status = "succeeded" + res.TotalSucceeded++ + res.TotalUploaded++ + res.Jobs = append(res.Jobs, jobRec) + report("uploaded", fmt.Sprintf("Uploaded %s", jobRec.Filename), totalProcessed, scanRes.TotalFiles, jobRec.Filename, jobRec.JobID, "succeeded", "") + case "queued", "running", "processing", "pending", "in_progress": + if jobRec.JobID == "" { + jobRec.Status = "failed" + jobRec.Error = "server returned pending status without job_id" + res.TotalFailed++ + res.Jobs = append(res.Jobs, jobRec) + report("upload_failed", fmt.Sprintf("Failed %s: missing job_id", jobRec.Filename), totalProcessed, scanRes.TotalFiles, jobRec.Filename, "", "failed", jobRec.Error) + } else { + res.TotalUploaded++ + res.Jobs = append(res.Jobs, jobRec) + report("uploaded", fmt.Sprintf("Uploaded %s (Job ID: %s)", jobRec.Filename, jobRec.JobID), totalProcessed, scanRes.TotalFiles, jobRec.Filename, jobRec.JobID, jobRec.Status, "") + } + default: + jobRec.Status = "failed" + jobRec.Error = fmt.Sprintf("unrecognized job status '%s'", st) + res.TotalFailed++ + res.Jobs = append(res.Jobs, jobRec) + report("upload_failed", fmt.Sprintf("Failed %s: %s", jobRec.Filename, jobRec.Error), totalProcessed, scanRes.TotalFiles, jobRec.Filename, "", "failed", jobRec.Error) + } + } + } + + // 7. Watch background jobs if requested + if opts.WatchJobs && res.TotalUploaded > 0 { + report("watching", "Monitoring background ingestion jobs...", 0, len(res.Jobs), "", "", "", "") + for i, job := range res.Jobs { + if job.Status == "skipped" || job.Status == "failed" || job.JobID == "" || job.Status == "succeeded" { + continue + } + + report("job_polling", fmt.Sprintf("Checking status for job %s (%s)...", job.JobID, job.Filename), i+1, len(res.Jobs), job.Filename, job.JobID, "polling", "") + status, err := e.PollJob(ctx, opts.SpaceID, job.JobID, opts.Timeout) + if err != nil { + job.Status = "failed" + job.Error = err.Error() + res.TotalFailed++ + report("job_failed", fmt.Sprintf("Job %s failed: %v", job.JobID, err), i+1, len(res.Jobs), job.Filename, job.JobID, "failed", err.Error()) + } else { + job.Status = status + switch status { + case "succeeded", "completed": + res.TotalSucceeded++ + report("job_succeeded", fmt.Sprintf("Job %s succeeded (%s).", job.JobID, job.Filename), i+1, len(res.Jobs), job.Filename, job.JobID, "succeeded", "") + case "skipped", "changed_skipped": + res.TotalSkipped++ + report("skipped", fmt.Sprintf("Job %s skipped (%s).", job.JobID, job.Filename), i+1, len(res.Jobs), job.Filename, job.JobID, "skipped", "") + default: + res.TotalFailed++ + report("job_failed", fmt.Sprintf("Job %s ended with status: %s", job.JobID, status), i+1, len(res.Jobs), job.Filename, job.JobID, status, "") + } + } + } + } + + res.Success = res.TotalFailed == 0 && (!opts.WatchJobs || res.TotalSucceeded+res.TotalSkipped == res.TotalScanned) + res.DurationMs = time.Since(start).Milliseconds() + report("done", fmt.Sprintf("Run finished in %d ms (%d succeeded, %d failed).", res.DurationMs, res.TotalSucceeded, res.TotalFailed), res.TotalScanned, res.TotalScanned, "", "", "finished", "") + return res, nil +} + +// FetchActiveJobs inspects running and queued ingestion jobs on the space +func (e *Engine) FetchActiveJobs(ctx context.Context, spaceID string) (int, error) { + res, err := e.client.CallTool(ctx, "long_ingest_list", map[string]interface{}{ + "space_id": spaceID, + "status": "running", + "limit": 50, + }) + if mcpclient.IsToolMissing(res, err, "long_ingest_list") { + res, err = e.client.CallTool(ctx, "ingest_job_list", map[string]interface{}{"memory_id": spaceID, "status": "running"}) + } + if err != nil || res["status"] == "error" || res["isError"] == true { + return 0, nil + } + jobs, ok := res["jobs"].([]interface{}) + if !ok { + return 0, nil + } + return len(jobs), nil +} + +// EnsureSpace checks if space exists, creating it only on explicit not-found +func (e *Engine) EnsureSpace(ctx context.Context, spaceID string, createIfMissing bool, rulesTemplate string, ontology ...string) error { + res, err := e.client.CallTool(ctx, "space_info", map[string]interface{}{"space_id": spaceID}) + if mcpclient.IsToolMissing(res, err, "space_info") { + name := "" + if len(ontology) > 0 { + name = ontology[0] + } + return e.ensureGraphMemory(ctx, spaceID, createIfMissing, name) + } + if err == nil { + if status, ok := res["status"].(string); ok && status == "ok" && res["isError"] != true { + return nil + } + if status, ok := res["status"].(string); ok && status == "not_found" { + if !createIfMissing { + return fmt.Errorf("space %s does not exist (use --create-space-if-missing to create it automatically)", spaceID) + } + var rulesParam string + if rulesTemplate != "" && rulesTemplate != "standard" { + rulesParam = rulesTemplate + } + createRes, err := e.client.CallTool(ctx, "space_create", map[string]interface{}{ + "space_id": spaceID, + "description": fmt.Sprintf("Space for %s created by graph-memory-ingest", spaceID), + "rules": rulesParam, + }) + if err != nil { + return fmt.Errorf("space_create failed: %w", err) + } + createStatus, _ := createRes["status"].(string) + if createRes["isError"] == true || (createStatus != "created" && createStatus != "already_exists") { + return fmt.Errorf("space_create failed with status '%s': %v", createStatus, createRes["message"]) + } + return nil + } + return fmt.Errorf("space verification returned status '%v': %v", res["status"], res["message"]) + } + return fmt.Errorf("failed to verify space %s: %w", spaceID, err) +} + +// FetchKnownHashes retrieves the list of SHA256 hashes already indexed in the space +func (e *Engine) FetchKnownHashes(ctx context.Context, spaceID string) (map[string]bool, error) { + known := make(map[string]bool) + limit := 100 + offset := 0 + standalone := false + + for { + // Attempt to list documents using long_document_list (Issue #464) + res, err := e.client.CallTool(ctx, "long_document_list", map[string]interface{}{ + "space_id": spaceID, + "limit": limit, + "offset": offset, + }) + if mcpclient.IsToolMissing(res, err, "long_document_list") { + standalone = true + res, err = e.client.CallTool(ctx, "document_list", map[string]interface{}{"memory_id": spaceID}) + if mcpclient.IsToolMissing(res, err, "document_list") { + return known, nil + } + } + if err != nil { + return nil, fmt.Errorf("failed to fetch document catalog: %w", err) + } + if res["status"] != "ok" || res["isError"] == true { + return nil, fmt.Errorf("server error from document catalog: %v", res["message"]) + } + + rawDocs, ok := res["documents"].([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid or missing 'documents' array in space %s catalog response", spaceID) + } + if len(rawDocs) == 0 { + break + } + + for idx, item := range rawDocs { + docMap, ok := item.(map[string]interface{}) + if !ok || docMap == nil { + return nil, fmt.Errorf("malformed document entry at index %d in space %s catalog", idx, spaceID) + } + // Failed or unfinished documents must remain eligible for ingestion. + if status, _ := docMap["ingestion_status"].(string); status != "" && status != "unknown" && status != "succeeded" && status != "completed" { + continue + } + sha, ok := docMap["sha256"].(string) + if standalone && sha == "" { + sha, ok = docMap["hash"].(string) + } + if !ok || len(sha) != 64 || !isValidHex(sha) { + return nil, fmt.Errorf("invalid sha256 checksum '%v' at index %d in space %s catalog", docMap["sha256"], idx, spaceID) + } + known[strings.ToLower(sha)] = true + } + + if standalone || len(rawDocs) < limit { + break + } + offset += len(rawDocs) + } + + return known, nil +} + +func isValidHex(s string) bool { + for _, c := range s { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + return false + } + } + return true +} + +// IngestBatch streams an entire batch of documents in a single long_ingest_async call +func (e *Engine) IngestBatch(ctx context.Context, spaceID string, batch scanner.Batch, ontology string, maxBytes int64, replace bool) ([]*JobRecord, error) { + if len(batch.Files) == 0 { + return nil, nil + } + + docsPayload := make([]map[string]interface{}, 0, len(batch.Files)) + fileMap := make(map[string]scanner.FileItem) + + for _, file := range batch.Files { + contentObj, err := scanner.LoadFileContent(file.Path, maxBytes) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", file.Path, err) + } + + // Graph Memory backend strictly requires content_base64, filename, source_path, and sha256 + doc := map[string]interface{}{ + "source_path": file.RelPath, + "filename": file.Filename, + "sha256": contentObj.SHA256, + "content_base64": contentObj.Base64Data, + "metadata": map[string]interface{}{ + "content_type": file.ContentType, + }, + } + + docsPayload = append(docsPayload, doc) + file.SHA256 = contentObj.SHA256 + fileMap[file.RelPath] = file + } + + options := map[string]interface{}{ + "replace_existing": replace, + } + + res, err := e.client.CallTool(ctx, "long_ingest_async", map[string]interface{}{ + "space_id": spaceID, + "documents": docsPayload, + "options": options, + }) + if mcpclient.IsToolMissing(res, err, "long_ingest_async") { + res, err = e.client.CallTool(ctx, "memory_ingest_batch_async", map[string]interface{}{ + "memory_id": spaceID, "documents": docsPayload, "replace_existing": replace, + }) + } + if err != nil { + return nil, fmt.Errorf("long_ingest_async failed: %w", err) + } + + if res["status"] == "error" || res["isError"] == true { + msg, _ := res["message"].(string) + if msg == "" { + msg = "unknown error from long_ingest_async" + } + return nil, fmt.Errorf("%s", msg) + } + + // Reconcile with the hashes of the bytes actually sent, even if a file changed after scanning. + for i := range batch.Files { + batch.Files[i] = fileMap[batch.Files[i].RelPath] + } + return ReconcileBatch(batch.Files, res) +} + +// IngestFile streams a single file (used for single-file operations) +func (e *Engine) IngestFile(ctx context.Context, spaceID string, file scanner.FileItem, ontology string, maxBytes int64, replace bool) (*JobRecord, error) { + batch := scanner.Batch{ + Files: []scanner.FileItem{file}, + } + jobs, err := e.IngestBatch(ctx, spaceID, batch, ontology, maxBytes, replace) + if err != nil { + return nil, err + } + if len(jobs) == 0 { + return nil, fmt.Errorf("no job returned for file %s", file.RelPath) + } + return jobs[0], nil +} + +// PollJob monitors canonical terminal states. A real failure never triggers a compatibility retry. +func (e *Engine) PollJob(ctx context.Context, spaceID string, jobID string, timeout time.Duration) (string, error) { + if timeout <= 0 { + timeout = 10 * time.Minute + } + pollCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + for { + args := map[string]interface{}{"space_id": spaceID, "job_id": jobID} + res, err := e.client.CallTool(pollCtx, "long_ingest_status", args) + if mcpclient.IsToolMissing(res, err, "long_ingest_status") { + res, err = e.client.CallTool(pollCtx, "long_ingest_job_status", args) + if mcpclient.IsToolMissing(res, err, "long_ingest_job_status") { + res, err = e.client.CallTool(pollCtx, "ingest_job_status", map[string]interface{}{"job_id": jobID}) + } + } + if err != nil { + return "failed", err + } + status, _ := res["status"].(string) + if res["isError"] == true || status == "error" { + return "failed", fmt.Errorf("job status error: %v", res["message"]) + } + if state, _ := res["state"].(string); state != "" { + status = state + } + switch strings.ToLower(status) { + case "succeeded", "completed": + return "succeeded", nil + case "skipped", "changed_skipped": + return status, nil + case "failed", "error", "cancelled", "queue_full": + message, _ := res["error"].(string) + if message == "" { + message, _ = res["message"].(string) + } + if message == "" { + message = "ingestion job " + status + } + return status, fmt.Errorf("%s", message) + case "queued", "running", "in_progress", "processing", "pending": + default: + return "failed", fmt.Errorf("unrecognized job status %q", status) + } + timer := time.NewTimer(time.Second) + select { + case <-pollCtx.Done(): + timer.Stop() + return "cancelled", pollCtx.Err() + case <-timer.C: + } + } +} + +func (e *Engine) ensureGraphMemory(ctx context.Context, id string, create bool, ontology string) error { + res, err := e.client.CallTool(ctx, "memory_list", map[string]interface{}{}) + if err != nil { + return err + } + if res["status"] != "ok" || res["isError"] == true { + return fmt.Errorf("memory_list failed: %v", res["message"]) + } + memories, ok := res["memories"].([]interface{}) + if !ok { + return fmt.Errorf("invalid memory_list response") + } + for _, raw := range memories { + memory, ok := raw.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid memory_list entry") + } + if memory["id"] == id { + return nil + } + } + if !create { + return fmt.Errorf("memory %s not found (use --create-space-if-missing)", id) + } + if ontology == "" { + return fmt.Errorf("--ontology is required to create a Graph Memory memory") + } + res, err = e.client.CallTool(ctx, "memory_create", map[string]interface{}{"memory_id": id, "name": id, "ontology": ontology}) + if err != nil { + return err + } + if res["isError"] == true || (res["status"] != "created" && res["status"] != "already_exists") { + return fmt.Errorf("memory_create failed: %v", res["message"]) + } + return nil +} diff --git a/tools/graph-memory-ingest/internal/ingest/engine_test.go b/tools/graph-memory-ingest/internal/ingest/engine_test.go new file mode 100644 index 0000000..38767dc --- /dev/null +++ b/tools/graph-memory-ingest/internal/ingest/engine_test.go @@ -0,0 +1,423 @@ +package ingest + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "graph-memory-ingest/internal/mcpclient" + "graph-memory-ingest/internal/scanner" +) + +func newMockServer(toolHandler func(name string, args map[string]interface{}) (string, bool)) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req mcpclient.JSONRPCRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + + if req.Method == "initialize" { + w.Header().Set("mcp-session-id", "mock-session-id") + w.Header().Set("mcp-protocol-version", "2024-11-05") + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + + if req.Method == "notifications/initialized" { + w.WriteHeader(http.StatusOK) + return + } + + if req.Method == "tools/call" { + params, _ := req.Params.(map[string]interface{}) + name, _ := params["name"].(string) + args, _ := params["arguments"].(map[string]interface{}) + resBody, isErr := toolHandler(name, args) + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"content":[{"type":"text","text":` + string(mustJSON(resBody)) + `}],"isError":` + fmt.Sprintf("%t", isErr) + `}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"status":"ok"}`), + } + _ = json.NewEncoder(w).Encode(resp) + })) +} + +func TestEngineRunSuccess(t *testing.T) { + tempDir := t.TempDir() + docPath := filepath.Join(tempDir, "sample.md") + _ = os.WriteFile(docPath, []byte("# Test Document\nSome content"), 0644) + + jobPollCount := 0 + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + switch name { + case "space_info": + return `{"status":"ok","space_id":"demo-space"}`, false + case "long_ingest_list", "long_document_list": + return `{"status":"ok","documents":[]}`, false + case "long_ingest_async": + return `{"status":"ok","batch_id":"batch-123","total":1,"counts":{"queued":1},"items":[{"index":0,"source_path":"sample.md","job_id":"job-12345","status":"queued"}],"errors":[]}`, false + case "long_ingest_status", "long_ingest_job_status": + jobPollCount++ + if jobPollCount >= 2 { + return `{"status":"succeeded","job_id":"job-12345"}`, false + } + return `{"status":"running","job_id":"job-12345"}`, false + default: + return `{"status":"ok"}`, false + } + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + opts := IngestOptions{ + Path: tempDir, + SpaceID: "demo-space", + AllowedExtensions: []string{".md"}, + WatchJobs: true, + Timeout: 5 * time.Second, + } + + res, err := engine.Run(context.Background(), opts, nil) + if err != nil { + t.Fatalf("engine.Run failed: %v", err) + } + + if !res.Success { + t.Errorf("expected success true, got false") + } + if res.TotalUploaded != 1 { + t.Errorf("expected 1 uploaded file, got %d", res.TotalUploaded) + } + if res.TotalSucceeded != 1 { + t.Errorf("expected 1 succeeded file, got %d", res.TotalSucceeded) + } +} + +func TestEngineAsyncJobFailure(t *testing.T) { + tempDir := t.TempDir() + docPath := filepath.Join(tempDir, "corrupted.md") + _ = os.WriteFile(docPath, []byte("# Corrupted Document"), 0644) + + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + switch name { + case "space_info": + return `{"status":"ok","space_id":"demo-space"}`, false + case "long_ingest_list", "long_document_list": + return `{"status":"ok","documents":[]}`, false + case "long_ingest_async": + return `{"status":"ok","batch_id":"batch-123","total":1,"counts":{"queued":1},"items":[{"index":0,"source_path":"corrupted.md","job_id":"job-fail-99","status":"queued"}],"errors":[]}`, false + case "long_ingest_status", "long_ingest_job_status": + return `{"status":"failed","job_id":"job-fail-99","error":"graph extraction memory limit exceeded"}`, false + default: + return `{"status":"ok"}`, false + } + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + opts := IngestOptions{ + Path: tempDir, + SpaceID: "demo-space", + AllowedExtensions: []string{".md"}, + WatchJobs: true, + Timeout: 5 * time.Second, + } + + res, err := engine.Run(context.Background(), opts, nil) + if err != nil { + t.Fatalf("engine.Run should not error at top-level on partial job failure, got: %v", err) + } + + if res.Success { + t.Errorf("expected success false on failed job, got true") + } + if res.TotalFailed != 1 { + t.Errorf("expected 1 failed job, got %d", res.TotalFailed) + } + if len(res.Jobs) != 1 || res.Jobs[0].Error != "graph extraction memory limit exceeded" { + t.Errorf("expected job error captured, got: %+v", res.Jobs) + } +} + +func TestEngineDeduplicationAndForceReplace(t *testing.T) { + tempDir := t.TempDir() + docPath := filepath.Join(tempDir, "existing.md") + content := []byte("# Existing Document") + _ = os.WriteFile(docPath, content, 0644) + + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + switch name { + case "space_info": + return `{"status":"ok","space_id":"demo-space"}`, false + case "long_ingest_list", "long_document_list": + // Valid 64-char SHA-256 of "# Existing Document" + return `{"status":"ok","documents":[{"filename":"existing.md","sha256":"88e62a89c10acc7f3ec78ff2df1b2a47386b7a36310a1372ee50a95125857486"}]}`, false + case "long_ingest_async": + return `{"status":"ok","batch_id":"batch-123","total":1,"counts":{"queued":1},"items":[{"index":0,"source_path":"existing.md","job_id":"job-replace","status":"queued"}],"errors":[]}`, false + case "long_ingest_status": + return `{"status":"succeeded","job_id":"job-replace"}`, false + default: + return `{"status":"ok"}`, false + } + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + // 1. Without ForceReplace: should be skipped + opts := IngestOptions{ + Path: tempDir, + SpaceID: "demo-space", + AllowedExtensions: []string{".md"}, + ForceReplace: false, + } + res, err := engine.Run(context.Background(), opts, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + if res.TotalSkipped != 1 { + t.Errorf("expected 1 skipped file, got %d", res.TotalSkipped) + } + if res.TotalUploaded != 0 { + t.Errorf("expected 0 uploaded files, got %d", res.TotalUploaded) + } + + // 2. With ForceReplace: should upload even if hash is known + opts.ForceReplace = true + res2, err := engine.Run(context.Background(), opts, nil) + if err != nil { + t.Fatalf("Run with ForceReplace failed: %v", err) + } + if res2.TotalUploaded != 1 { + t.Errorf("expected 1 uploaded file with ForceReplace, got %d", res2.TotalUploaded) + } +} + +func TestEngineMissingSpaceFailClosed(t *testing.T) { + tempDir := t.TempDir() + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + return `{"status":"not_found","message":"space not found"}`, false + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + opts := IngestOptions{ + Path: tempDir, + SpaceID: "non-existent-space", + CreateSpaceIfMissing: false, + } + + _, err := engine.Run(context.Background(), opts, nil) + if err == nil { + t.Fatal("expected error on missing space when CreateSpaceIfMissing is false, got nil") + } +} + +func TestEngineAuthErrorDoesNotCreateSpace(t *testing.T) { + tempDir := t.TempDir() + spaceCreateCalled := false + + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + if name == "space_create" { + spaceCreateCalled = true + } + return `{"status":"error","message":"Access denied to space 'secret-space'"}`, false + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + opts := IngestOptions{ + Path: tempDir, + SpaceID: "secret-space", + CreateSpaceIfMissing: true, // Even if requested, must fail closed on auth error without calling space_create + } + + _, err := engine.Run(context.Background(), opts, nil) + if err == nil { + t.Fatal("expected error on auth denied space, got nil") + } + if spaceCreateCalled { + t.Fatal("space_create was called on auth error; expected fail-closed behavior") + } +} + +func TestEngineSpaceCreatePartialFails(t *testing.T) { + tempDir := t.TempDir() + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + switch name { + case "space_info": + return `{"status":"not_found","message":"space does not exist"}`, false + case "space_create": + // Return partial status (recovery required) + return `{"status":"partial","message":"Space creation incomplete, recovery required"}`, false + default: + return `{"status":"ok"}`, false + } + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + opts := IngestOptions{ + Path: tempDir, + SpaceID: "partial-space", + CreateSpaceIfMissing: true, + } + + _, err := engine.Run(context.Background(), opts, nil) + if err == nil { + t.Fatal("expected Run to fail when space_create returns status: partial, got nil") + } +} + +func TestEngineMalformedCatalogEntryFailClosed(t *testing.T) { + tempDir := t.TempDir() + docPath := filepath.Join(tempDir, "doc.md") + _ = os.WriteFile(docPath, []byte("# Doc"), 0644) + + ingestCalled := false + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + switch name { + case "space_info": + return `{"status":"ok","space_id":"demo-space"}`, false + case "long_ingest_list", "long_document_list": + // Catalog entry with invalid (non-64 hex) sha256 checksum + return `{"status":"ok","documents":[{"filename":"bad.md","sha256":"invalid_short_hash"}]}`, false + case "long_ingest_async": + ingestCalled = true + return `{"status":"ok"}`, false + default: + return `{"status":"ok"}`, false + } + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + opts := IngestOptions{ + Path: tempDir, + SpaceID: "demo-space", + } + + _, err := engine.Run(context.Background(), opts, nil) + if err == nil { + t.Fatal("expected Run to fail closed on malformed catalog sha256, got nil") + } + if ingestCalled { + t.Fatal("long_ingest_async was called despite malformed catalog SHA256; expected fail-closed") + } +} + +func TestEngineBatchIngestionContractItemsAndErrors(t *testing.T) { + tempDir := t.TempDir() + doc1 := filepath.Join(tempDir, "ok.md") + doc2 := filepath.Join(tempDir, "fail.md") + _ = os.WriteFile(doc1, []byte("# OK file"), 0644) + _ = os.WriteFile(doc2, []byte("# Fail file"), 0644) + + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + if name == "long_ingest_async" { + // Verify payload contains content_base64 + docs, _ := args["documents"].([]interface{}) + if len(docs) != 2 { + return fmt.Sprintf(`{"status":"error","message":"expected 2 docs, got %d"}`, len(docs)), false + } + for _, dRaw := range docs { + d, _ := dRaw.(map[string]interface{}) + if d["content_base64"] == nil || d["content_base64"] == "" { + return `{"status":"error","message":"missing content_base64 in document payload"}`, false + } + } + + // Return canonical memory_ingest_batch_async response with mixed items & errors + return `{ + "status": "ok", + "batch_id": "batch-mix", + "total": 2, + "counts": {"queued": 1, "failed": 1}, + "items": [ + {"index": 0, "source_path": "ok.md", "job_id": "job-ok-1", "status": "queued"}, + {"index": 1, "source_path": "fail.md", "status": "error", "message": "unsupported binary format"} + ], + "errors": [ + {"source_path": "fail.md", "filename": "fail.md", "error": "unsupported binary format"} + ] + }`, false + } + return `{"status":"ok"}`, false + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + engine := NewEngine(client) + + batch := scanner.Batch{ + Files: []scanner.FileItem{ + {Path: doc1, RelPath: "ok.md", Filename: "ok.md", SHA256: "abc1"}, + {Path: doc2, RelPath: "fail.md", Filename: "fail.md", SHA256: "abc2"}, + }, + } + + records, err := engine.IngestBatch(context.Background(), "demo-space", batch, "", 10*1024*1024, false) + if err != nil { + t.Fatalf("IngestBatch failed: %v", err) + } + + if len(records) != 2 { + t.Fatalf("expected 2 job records, got %d", len(records)) + } + + var okRec, failRec *JobRecord + for _, r := range records { + if r.Filename == "ok.md" { + okRec = r + } else if r.Filename == "fail.md" { + failRec = r + } + } + + if okRec == nil || okRec.JobID != "job-ok-1" || okRec.Status != "queued" { + t.Errorf("okRec mismatch: %+v", okRec) + } + if failRec == nil || failRec.Status != "error" || failRec.Error != "unsupported binary format" { + t.Errorf("failRec mismatch: %+v", failRec) + } +} + +func mustJSON(s string) []byte { + b, _ := json.Marshal(s) + return b +} diff --git a/tools/graph-memory-ingest/internal/ingest/port_test.go b/tools/graph-memory-ingest/internal/ingest/port_test.go new file mode 100644 index 0000000..af89f06 --- /dev/null +++ b/tools/graph-memory-ingest/internal/ingest/port_test.go @@ -0,0 +1,186 @@ +package ingest + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "graph-memory-ingest/internal/mcpclient" + "graph-memory-ingest/internal/scanner" +) + +func TestReconcileBatchRejectsAmbiguity(t *testing.T) { + files := []scanner.FileItem{{RelPath: "a/doc.md", Filename: "doc.md"}, {RelPath: "b/doc.md", Filename: "doc.md"}} + cases := map[string]string{ + "duplicate source": `{"items":[{"source_path":"a/doc.md","status":"succeeded"},{"source_path":"a/doc.md","status":"succeeded"}]}`, + "duplicate job": `{"items":[{"source_path":"a/doc.md","job_id":"1","status":"queued"},{"source_path":"b/doc.md","job_id":"1","status":"queued"}]}`, + "unknown source": `{"items":[{"source_path":"x/doc.md","status":"succeeded"}]}`, + "missing source": `{"items":[{"source_path":"a/doc.md","status":"succeeded"}]}`, + "ambiguous filename": `{"jobs":[{"filename":"doc.md","status":"succeeded"}]}`, + "single job multiple files": `{"job_id":"1","status":"queued"}`, + "missing job": `{"items":[{"source_path":"a/doc.md","status":"queued"}]}`, + "unknown status": `{"items":[{"source_path":"a/doc.md","status":"mystery"}]}`, + "contradictory error": `{"items":[{"source_path":"a/doc.md","status":"succeeded"}],"errors":[{"source_path":"a/doc.md","error":"failure"}]}`, + "malformed errors": `{"items":[],"errors":"invalid"}`, + "duplicate errors": `{"errors":[{"source_path":"a/doc.md","error":"x"},{"source_path":"a/doc.md","error":"x"}]}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + var response map[string]interface{} + if err := json.Unmarshal([]byte(body), &response); err != nil { + t.Fatal(err) + } + if _, err := ReconcileBatch(files, response); err == nil { + t.Fatal("accepted an ambiguous batch") + } + }) + } +} + +func TestReconcileCanonicalStatuses(t *testing.T) { + statuses := []string{"succeeded", "completed", "skipped", "changed_skipped", "queued", "running", "failed", "error", "queue_full", "cancelled"} + for _, status := range statuses { + t.Run(status, func(t *testing.T) { + files := []scanner.FileItem{{RelPath: "a/doc.md", Filename: "doc.md", SHA256: "fresh"}} + res := map[string]interface{}{"jobs": []interface{}{map[string]interface{}{"filename": "doc.md", "job_id": "job", "status": status}}} + records, err := ReconcileBatch(files, res) + if err != nil || len(records) != 1 || records[0].Status != status || records[0].SHA256 != "fresh" { + t.Fatalf("records=%+v err=%v", records, err) + } + }) + } + var res map[string]interface{} + _ = json.Unmarshal([]byte(`{"items":[{"source_path":"a","status":"completed"}],"errors":[{"source_path":"b","error":"queue full"}]}`), &res) + records, err := ReconcileBatch([]scanner.FileItem{{RelPath: "a"}, {RelPath: "b"}}, res) + if err != nil || len(records) != 2 || records[1].Error != "queue full" { + t.Fatalf("errors-only outcome lost: %+v %v", records, err) + } +} + +func TestEngineStandaloneAndNoPollCounts(t *testing.T) { + for _, status := range []string{"queued", "succeeded"} { + t.Run(status, func(t *testing.T) { + dir := t.TempDir() + data := []byte("standalone ingestion") + if err := os.WriteFile(filepath.Join(dir, "doc.md"), data, 0600); err != nil { + t.Fatal(err) + } + hash := sha256.Sum256(data) + expectedSHA := hex.EncodeToString(hash[:]) + created, submitted := false, false + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + if strings.HasPrefix(name, "long_") || name == "space_info" { + return "Unknown tool: " + name, true + } + switch name { + case "memory_list": + return `{"status":"ok","memories":[]}`, false + case "memory_create": + if len(args) != 3 || args["memory_id"] != "demo" || args["ontology"] != "technical" { + t.Errorf("incorrect create args: %v", args) + } + created = true + return `{"status":"created"}`, false + case "ingest_job_list": + return `{"status":"ok","jobs":[]}`, false + case "document_list": + if len(args) != 1 || args["memory_id"] != "demo" { + t.Errorf("incorrect catalog args: %v", args) + } + return fmt.Sprintf(`{"status":"ok","documents":[{"sha256":%q,"ingestion_status":"failed"}]}`, expectedSHA), false + case "memory_ingest_batch_async": + if len(args) != 3 || args["memory_id"] != "demo" || args["replace_existing"] != false { + t.Errorf("incorrect batch args: %v", args) + } + docs := args["documents"].([]interface{}) + doc := docs[0].(map[string]interface{}) + if doc["content_base64"] != base64.StdEncoding.EncodeToString(data) || doc["sha256"] != expectedSHA || doc["source_path"] != "doc.md" || doc["filename"] != "doc.md" { + t.Errorf("bad payload: %v", doc) + } + submitted = true + return fmt.Sprintf(`{"status":"ok","items":[{"source_path":"doc.md","job_id":"j1","status":%q}]}`, status), false + default: + t.Errorf("unexpected call %s", name) + return "unexpected", true + } + }) + defer server.Close() + engine := NewEngine(mcpclient.NewClient(server.URL, "", time.Second)) + result, err := engine.Run(context.Background(), IngestOptions{Path: dir, SpaceID: "demo", Ontology: "technical", CreateSpaceIfMissing: true, AllowedExtensions: []string{".md"}}, nil) + wantSucceeded := 0 + if status == "succeeded" { + wantSucceeded = 1 + } + if err != nil || !created || !submitted || !result.Success || result.TotalSucceeded != wantSucceeded || result.TotalUploaded != 1 { + t.Fatalf("incorrect result: %+v err=%v", result, err) + } + }) + } +} + +func TestCatalogFailClosedAndMissingFallback(t *testing.T) { + for _, missing := range []bool{false, true} { + t.Run(fmt.Sprint(missing), func(t *testing.T) { + calls := 0 + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + calls++ + if missing { + return "Unknown tool: " + name, true + } + return `{"status":"error","message":"access denied"}`, false + }) + defer server.Close() + _, err := NewEngine(mcpclient.NewClient(server.URL, "", time.Second)).FetchKnownHashes(context.Background(), "demo") + if missing && (err != nil || calls != 2) { + t.Fatalf("missing tool fallback: %d %v", calls, err) + } + if !missing && (err == nil || calls != 1) { + t.Fatalf("real error retried or masked: %d %v", calls, err) + } + }) + } +} + +func TestPollJobFallbackAndErrors(t *testing.T) { + for _, test := range []struct { + name, status string + missing bool + wantErr bool + }{ + {"compatibility", "completed", true, false}, {"failure", "error", false, true}, {"unknown", "mystery", false, true}, {"cancelled", "cancelled", false, true}, {"skip", "changed_skipped", false, false}, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + calls++ + if test.missing && name != "ingest_job_status" { + return "Unknown tool: " + name, true + } + if name == "ingest_job_status" && len(args) != 1 { + t.Errorf("standalone status args: %v", args) + } + return fmt.Sprintf(`{"status":%q,"message":"test"}`, test.status), false + }) + defer server.Close() + _, err := NewEngine(mcpclient.NewClient(server.URL, "", time.Second)).PollJob(context.Background(), "demo", "j1", time.Second) + if (err != nil) != test.wantErr { + t.Fatalf("wrong error: %v", err) + } + wantCalls := 1 + if test.missing { + wantCalls = 3 + } + if calls != wantCalls { + t.Fatalf("calls=%d want %d", calls, wantCalls) + } + }) + } +} diff --git a/tools/graph-memory-ingest/internal/ingest/reconcile.go b/tools/graph-memory-ingest/internal/ingest/reconcile.go new file mode 100644 index 0000000..044c6ec --- /dev/null +++ b/tools/graph-memory-ingest/internal/ingest/reconcile.go @@ -0,0 +1,161 @@ +package ingest + +import ( + "fmt" + "strings" + + "graph-memory-ingest/internal/scanner" +) + +// ReconcileBatch requires exactly one outcome per submitted file and unique job IDs. +// errors[] may supplement a failed item, but cannot contradict an accepted item. +func ReconcileBatch(files []scanner.FileItem, res map[string]interface{}) ([]*JobRecord, error) { + fileMap := make(map[string]scanner.FileItem, len(files)) + for _, f := range files { + if _, duplicate := fileMap[f.RelPath]; duplicate || f.RelPath == "" { + return nil, fmt.Errorf("duplicate or empty submitted source_path %q", f.RelPath) + } + fileMap[f.RelPath] = f + } + resolve := func(item map[string]interface{}) (string, error) { + if path, _ := item["source_path"].(string); path != "" { + if _, exists := fileMap[path]; !exists { + return "", fmt.Errorf("unknown source_path %q in batch response", path) + } + return path, nil + } + name, _ := item["filename"].(string) + path := "" + for _, f := range files { + if name != "" && (f.Filename == name || f.RelPath == name) { + if path != "" { + return "", fmt.Errorf("ambiguous filename %q in batch response", name) + } + path = f.RelPath + } + } + if path == "" { + return "", fmt.Errorf("unidentified file in batch response") + } + return path, nil + } + readArray := func(key string) ([]interface{}, error) { + raw, exists := res[key] + if !exists { + return nil, nil + } + items, ok := raw.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid %s array in batch response", key) + } + return items, nil + } + items, err := readArray("items") + if err != nil { + return nil, err + } + jobs, err := readArray("jobs") + if err != nil { + return nil, err + } + if len(items) > 0 && len(jobs) > 0 { + return nil, fmt.Errorf("ambiguous items and jobs arrays") + } + if len(items) == 0 { + items = jobs + } + if id, _ := res["job_id"].(string); id != "" { + if len(files) != 1 || len(items) != 0 { + return nil, fmt.Errorf("single job_id cannot identify this batch") + } + items = []interface{}{map[string]interface{}{"source_path": files[0].RelPath, "job_id": id, "status": res["status"]}} + } + errItems, err := readArray("errors") + if err != nil { + return nil, err + } + errorsByPath := make(map[string]string) + for _, raw := range errItems { + item, ok := raw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid batch error entry") + } + path, err := resolve(item) + if err != nil { + return nil, err + } + if _, duplicate := errorsByPath[path]; duplicate { + return nil, fmt.Errorf("duplicate error for %s", path) + } + message, _ := item["error"].(string) + if message == "" { + message, _ = item["message"].(string) + } + if message == "" { + return nil, fmt.Errorf("missing error detail for %s", path) + } + errorsByPath[path] = message + } + records := make(map[string]*JobRecord, len(files)) + jobIDs := make(map[string]bool) + for _, raw := range items { + item, ok := raw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid batch item") + } + path, err := resolve(item) + if err != nil { + return nil, err + } + if records[path] != nil { + return nil, fmt.Errorf("duplicate outcome for %s", path) + } + id, _ := item["job_id"].(string) + if id != "" && jobIDs[id] { + return nil, fmt.Errorf("duplicate job_id %s", id) + } + if id != "" { + jobIDs[id] = true + } + status, _ := item["status"].(string) + status = strings.ToLower(status) + rec := &JobRecord{Filename: path, SHA256: fileMap[path].SHA256, JobID: id, Status: status} + switch status { + case "succeeded", "completed", "skipped", "changed_skipped": + case "queued", "running", "pending", "processing", "in_progress": + if id == "" { + return nil, fmt.Errorf("pending file %s without job_id", path) + } + case "failed", "error", "queue_full", "cancelled", "rejected": + rec.Error, _ = item["error"].(string) + if rec.Error == "" { + rec.Error, _ = item["message"].(string) + } + if rec.Error == "" { + rec.Error = errorsByPath[path] + } + if rec.Error == "" { + rec.Error = "ingestion " + status + } + default: + return nil, fmt.Errorf("unrecognized status %q for %s", status, path) + } + if errorsByPath[path] != "" && rec.Error == "" { + return nil, fmt.Errorf("contradictory error and accepted outcome for %s", path) + } + records[path] = rec + } + ordered := make([]*JobRecord, 0, len(files)) + for _, file := range files { + rec := records[file.RelPath] + if rec == nil { + message := errorsByPath[file.RelPath] + if message == "" { + return nil, fmt.Errorf("file %s unacknowledged in batch response", file.RelPath) + } + rec = &JobRecord{Filename: file.RelPath, SHA256: file.SHA256, Status: "failed", Error: message} + } + ordered = append(ordered, rec) + } + return ordered, nil +} diff --git a/tools/graph-memory-ingest/internal/mcpclient/client.go b/tools/graph-memory-ingest/internal/mcpclient/client.go new file mode 100644 index 0000000..c4bc8da --- /dev/null +++ b/tools/graph-memory-ingest/internal/mcpclient/client.go @@ -0,0 +1,513 @@ +package mcpclient + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + maxResponseBytes = 16 * 1024 * 1024 // 16 MB maximum HTTP response +) + +// JSONRPCRequest represents a JSON-RPC 2.0 request to an MCP server +type JSONRPCRequest struct { + JSONRPC string `json:"jsonrpc"` + ID uint64 `json:"id"` + Method string `json:"method"` + Params interface{} `json:"params,omitempty"` +} + +// ToolCallParams defines parameters for tools/call +type ToolCallParams struct { + Name string `json:"name"` + Arguments map[string]interface{} `json:"arguments"` +} + +// JSONRPCResponse represents a JSON-RPC 2.0 response from an MCP server +type JSONRPCResponse struct { + JSONRPC string `json:"jsonrpc"` + ID uint64 `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *JSONRPCError `json:"error,omitempty"` +} + +// JSONRPCError represents a standard JSON-RPC error +type JSONRPCError struct { + Code int `json:"code"` + Message string `json:"message"` + Data json.RawMessage `json:"data,omitempty"` +} + +func (e *JSONRPCError) Error() string { + return fmt.Sprintf("mcp error (code %d): %s", e.Code, e.Message) +} + +// ContentItem represents a content block returned by MCP tools +type ContentItem struct { + Type string `json:"type"` + Text string `json:"text"` +} + +// ToolCallResult represents the payload inside JSON-RPC result +type ToolCallResult struct { + Content []ContentItem `json:"content"` + StructuredContent map[string]interface{} `json:"structuredContent,omitempty"` + IsError bool `json:"isError,omitempty"` +} + +// Client interacts with Hivemind via Streamable HTTP (JSON-RPC 2.0) +type Client struct { + endpoint string + token string + httpClient *http.Client + requestID atomic.Uint64 + initMu sync.Mutex + initialized bool + sessionID string + protocolVersion string +} + +// NewClient initializes a new MCP streamable HTTP client +func NewClient(endpoint string, token string, timeout time.Duration) *Client { + if timeout <= 0 { + timeout = 120 * time.Second + } + return &Client{ + endpoint: endpoint, + token: token, + httpClient: &http.Client{ + Timeout: timeout, + }, + protocolVersion: "2024-11-05", + } +} + +// SupportedMCPProtocols defines the strictly supported MCP protocol versions +var SupportedMCPProtocols = map[string]bool{ + "2024-11-05": true, + "2024-10-07": true, +} + +// Streamable HTTP media types per MCP specification +const ( + StreamableHTTPAccept = "application/json, text/event-stream" +) + +// InitializeResult represents the expected JSON-RPC result from initialize +type InitializeResult struct { + ProtocolVersion string `json:"protocolVersion"` + Capabilities ServerCapabilities `json:"capabilities"` + ServerInfo ServerInfo `json:"serverInfo"` + Instructions string `json:"instructions,omitempty"` +} + +// ServerCapabilities lists the MCP capabilities offered by the server +type ServerCapabilities struct { + Tools *ToolsCapability `json:"tools,omitempty"` + Resources *ResourcesCapability `json:"resources,omitempty"` + Prompts *PromptsCapability `json:"prompts,omitempty"` + Logging map[string]interface{} `json:"logging,omitempty"` +} + +// ToolsCapability indicates support for MCP tools +type ToolsCapability struct { + ListChanged bool `json:"listChanged,omitempty"` +} + +// ResourcesCapability indicates support for MCP resources +type ResourcesCapability struct { + Subscribe bool `json:"subscribe,omitempty"` + ListChanged bool `json:"listChanged,omitempty"` +} + +// PromptsCapability indicates support for MCP prompts +type PromptsCapability struct { + ListChanged bool `json:"listChanged,omitempty"` +} + +// ServerInfo describes the remote MCP server +type ServerInfo struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// readJSONRPCResponse consumes one matching response, not the lifetime of an SSE stream. +func readJSONRPCResponse(contentType string, body io.Reader, requestID uint64) (*JSONRPCResponse, error) { + limited := &io.LimitedReader{R: body, N: maxResponseBytes + 1} + decode := func(data []byte) (*JSONRPCResponse, error) { + var response JSONRPCResponse + if err := json.Unmarshal(data, &response); err != nil { + return nil, fmt.Errorf("invalid json-rpc response: %w", err) + } + if response.ID != requestID { + return nil, fmt.Errorf("mcp response ID mismatch: expected %d, got %d", requestID, response.ID) + } + return &response, nil + } + if !strings.Contains(contentType, "text/event-stream") { + data, err := io.ReadAll(limited) + if err != nil { + return nil, err + } + if limited.N == 0 { + return nil, fmt.Errorf("mcp response exceeded maximum allowed size") + } + return decode(data) + } + lines := bufio.NewScanner(limited) + lines.Buffer(make([]byte, 4096), int(maxResponseBytes)+1) + var data []string + flush := func() (*JSONRPCResponse, error) { + payload := strings.Join(data, "\n") + data = nil + if payload == "" || payload == "[DONE]" { + return nil, nil + } + var envelope struct { + Method string `json:"method"` + } + if err := json.Unmarshal([]byte(payload), &envelope); err != nil { + return nil, fmt.Errorf("invalid SSE JSON: %w", err) + } + if envelope.Method != "" { + return nil, nil + } // Notifications may precede the response. + return decode([]byte(payload)) + } + for lines.Scan() { + if limited.N == 0 { + return nil, fmt.Errorf("mcp response exceeded maximum allowed size") + } + line := lines.Text() + if line == "" { + response, err := flush() + if err != nil || response != nil { + return response, err + } + } else if strings.HasPrefix(line, "data:") { + data = append(data, strings.TrimPrefix(strings.TrimPrefix(line, "data:"), " ")) + } + } + if limited.N == 0 { + return nil, fmt.Errorf("mcp response exceeded maximum allowed size") + } + if err := lines.Err(); err != nil { + return nil, err + } + if response, err := flush(); err != nil || response != nil { + return response, err + } + return nil, fmt.Errorf("SSE stream ended without a JSON-RPC response") +} + +// IsToolMissing allows compatibility fallbacks only for an explicitly absent tool. +func IsToolMissing(result map[string]interface{}, err error, name string) bool { + var rpcErr *JSONRPCError + if errors.As(err, &rpcErr) && rpcErr.Code == -32601 { + return true + } + message, _ := result["message"].(string) + if err != nil { + message = err.Error() + } else if result["status"] != "error" && result["isError"] != true { + return false + } + message = strings.ToLower(strings.TrimSpace(message)) + for _, absent := range []string{"unknown tool: " + name, "unknown tool '" + name + "'", "tool '" + name + "' not found", "tool not found: " + name} { + if message == absent || strings.HasSuffix(message, ": "+absent) { + return true + } + } + return false +} + +// ensureInitialized performs the MCP initialize handshake and initialized notification +func (c *Client) ensureInitialized(ctx context.Context) error { + c.initMu.Lock() + defer c.initMu.Unlock() + + if c.initialized { + return nil + } + + reqID := c.requestID.Add(1) + initReq := JSONRPCRequest{ + JSONRPC: "2.0", + ID: reqID, + Method: "initialize", + Params: map[string]interface{}{ + "protocolVersion": "2024-11-05", + "capabilities": map[string]interface{}{ + "roots": map[string]interface{}{ + "listChanged": false, + }, + }, + "clientInfo": map[string]interface{}{ + "name": "graph-memory-ingest", + "version": "1.0.0", + }, + }, + } + + reqBody, err := json.Marshal(initReq) + if err != nil { + return fmt.Errorf("failed to encode initialize request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create initialize http request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", StreamableHTTPAccept) + if c.token != "" { + httpReq.Header.Set("Authorization", "Bearer "+c.token) + } + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return fmt.Errorf("initialize http request to %s failed: %w", c.endpoint, err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 256)) + return fmt.Errorf("http error %d from server: %s", resp.StatusCode, sanitizeErrSnippet(body, 256)) + } + if sid := resp.Header.Get("mcp-session-id"); sid != "" { + c.sessionID = sid + } + rpcResp, err := readJSONRPCResponse(resp.Header.Get("Content-Type"), resp.Body, reqID) + if err != nil { + return fmt.Errorf("failed to parse response: %w", err) + } + + if rpcResp.JSONRPC != "2.0" { + return fmt.Errorf("invalid jsonrpc version '%s' in initialize, expected '2.0'", rpcResp.JSONRPC) + } + if rpcResp.ID != reqID { + return fmt.Errorf("mcp initialize response ID mismatch: expected %d, got %d", reqID, rpcResp.ID) + } + if rpcResp.Error != nil { + return rpcResp.Error + } + + // Decode and validate InitializeResult + var initResult InitializeResult + if err := json.Unmarshal(rpcResp.Result, &initResult); err != nil { + return fmt.Errorf("failed to unmarshal initialize result: %w", err) + } + + if !SupportedMCPProtocols[initResult.ProtocolVersion] { + return fmt.Errorf("unsupported mcp protocol version '%s' (supported: 2024-11-05, 2024-10-07)", initResult.ProtocolVersion) + } + + // If header is provided, it must be consistent with result + if protoHeader := resp.Header.Get("mcp-protocol-version"); protoHeader != "" { + if protoHeader != initResult.ProtocolVersion { + return fmt.Errorf("mcp-protocol-version header '%s' contradicts initialize protocolVersion '%s'", protoHeader, initResult.ProtocolVersion) + } + } + + c.protocolVersion = initResult.ProtocolVersion + + if initResult.ServerInfo.Name == "" { + return fmt.Errorf("server did not provide valid serverInfo in initialize result") + } + if initResult.Capabilities.Tools == nil { + return fmt.Errorf("server does not declare 'tools' capability in initialize result") + } + + // Send notification initialized + notifReq := map[string]interface{}{ + "jsonrpc": "2.0", + "method": "notifications/initialized", + } + notifBody, err := json.Marshal(notifReq) + if err != nil { + return fmt.Errorf("failed to marshal notifications/initialized: %w", err) + } + notifHttpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(notifBody)) + if err != nil { + return fmt.Errorf("failed to create notifications/initialized request: %w", err) + } + notifHttpReq.Header.Set("Content-Type", "application/json") + notifHttpReq.Header.Set("Accept", StreamableHTTPAccept) + if c.token != "" { + notifHttpReq.Header.Set("Authorization", "Bearer "+c.token) + } + if c.sessionID != "" { + notifHttpReq.Header.Set("mcp-session-id", c.sessionID) + } + if c.protocolVersion != "" { + notifHttpReq.Header.Set("mcp-protocol-version", c.protocolVersion) + } + + notifResp, err := c.httpClient.Do(notifHttpReq) + if err != nil { + return fmt.Errorf("notifications/initialized http request failed: %w", err) + } + defer notifResp.Body.Close() + _, _ = io.Copy(io.Discard, notifResp.Body) + + if notifResp.StatusCode < 200 || notifResp.StatusCode >= 300 { + return fmt.Errorf("notifications/initialized rejected with http status %d", notifResp.StatusCode) + } + + c.initialized = true + return nil +} + +// CallTool invokes an MCP tool over Streamable HTTP (JSON-RPC 2.0) +func (c *Client) CallTool(ctx context.Context, toolName string, args map[string]interface{}) (map[string]interface{}, error) { + if c.endpoint == "" { + return nil, fmt.Errorf("mcp endpoint is not configured") + } + + if err := c.ensureInitialized(ctx); err != nil { + return nil, fmt.Errorf("mcp initialization failed: %w", err) + } + + reqID := c.requestID.Add(1) + rpcReq := JSONRPCRequest{ + JSONRPC: "2.0", + ID: reqID, + Method: "tools/call", + Params: ToolCallParams{ + Name: toolName, + Arguments: args, + }, + } + + reqBody, err := json.Marshal(rpcReq) + if err != nil { + return nil, fmt.Errorf("failed to encode request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("failed to create http request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", StreamableHTTPAccept) + if c.token != "" { + httpReq.Header.Set("Authorization", "Bearer "+c.token) + } + if c.sessionID != "" { + httpReq.Header.Set("mcp-session-id", c.sessionID) + } + if c.protocolVersion != "" { + httpReq.Header.Set("mcp-protocol-version", c.protocolVersion) + } + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("http request to %s failed: %w", c.endpoint, err) + } + defer resp.Body.Close() + + // Read with limit+1 to strictly detect oversized responses + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 256)) + return nil, fmt.Errorf("http error %d from server: %s", resp.StatusCode, sanitizeErrSnippet(body, 256)) + } + + rpcResp, err := readJSONRPCResponse(resp.Header.Get("Content-Type"), resp.Body, reqID) + if err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + if rpcResp.JSONRPC != "2.0" { + return nil, fmt.Errorf("invalid jsonrpc version '%s', expected '2.0'", rpcResp.JSONRPC) + } + + if rpcResp.ID != reqID { + return nil, fmt.Errorf("mcp response ID mismatch: expected %d, got %d", reqID, rpcResp.ID) + } + + if rpcResp.Error != nil { + return nil, rpcResp.Error + } + + if len(rpcResp.Result) == 0 || string(rpcResp.Result) == "null" { + return nil, fmt.Errorf("empty or null result in json-rpc response") + } + + // 1. Try MCP ToolCallResult envelope (which has non-nil content array or isError == true) + var toolRes ToolCallResult + if err := json.Unmarshal(rpcResp.Result, &toolRes); err == nil && (toolRes.Content != nil || toolRes.StructuredContent != nil || toolRes.IsError) { + var combined string + for _, item := range toolRes.Content { + if item.Type == "text" { + combined += item.Text + } + } + combined = strings.TrimSpace(combined) + + if toolRes.IsError { + if combined != "" { + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(combined), &parsed); err == nil { + parsed["status"] = "error" + parsed["isError"] = true + return parsed, nil + } + return map[string]interface{}{ + "status": "error", + "message": combined, + "isError": true, + }, nil + } + return map[string]interface{}{ + "status": "error", + "message": "mcp tool returned error (isError: true)", + "isError": true, + }, nil + } + + if toolRes.StructuredContent != nil { + return toolRes.StructuredContent, nil + } + + if combined != "" { + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(combined), &parsed); err == nil { + return parsed, nil + } + return map[string]interface{}{ + "status": "ok", + "message": combined, + }, nil + } + + return map[string]interface{}{"status": "ok"}, nil + } + + // 2. Direct map in Result (e.g. {"status":"ok", ...} or {"status":"error", ...}) + var directResult map[string]interface{} + if err := json.Unmarshal(rpcResp.Result, &directResult); err == nil { + return directResult, nil + } + + return nil, fmt.Errorf("unrecognized mcp result format: %s", sanitizeErrSnippet(rpcResp.Result, 256)) +} + +func sanitizeErrSnippet(data []byte, maxLen int) string { + s := strings.TrimSpace(string(data)) + if len(s) > maxLen { + return s[:maxLen] + "..." + } + return s +} diff --git a/tools/graph-memory-ingest/internal/mcpclient/client_test.go b/tools/graph-memory-ingest/internal/mcpclient/client_test.go new file mode 100644 index 0000000..a84a292 --- /dev/null +++ b/tools/graph-memory-ingest/internal/mcpclient/client_test.go @@ -0,0 +1,327 @@ +package mcpclient + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func handleMockMCP(w http.ResponseWriter, r *http.Request, toolHandler func(req JSONRPCRequest) (json.RawMessage, bool, *JSONRPCError)) { + var req JSONRPCRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/json") + + if req.Method == "initialize" { + w.Header().Set("mcp-session-id", "mock-session-12345") + w.Header().Set("mcp-protocol-version", "2024-11-05") + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + + if req.Method == "notifications/initialized" { + w.WriteHeader(http.StatusOK) + return + } + + if req.Method == "tools/call" { + res, _, rpcErr := toolHandler(req) + if rpcErr != nil { + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Error: rpcErr, + } + _ = json.NewEncoder(w).Encode(resp) + return + } + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: res, + } + _ = json.NewEncoder(w).Encode(resp) + return + } + + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"status":"ok"}`), + } + _ = json.NewEncoder(w).Encode(resp) +} + +func TestClientCallToolSuccess(t *testing.T) { + sessionReceived := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer test-token" { + t.Errorf("missing or invalid Authorization header: %s", r.Header.Get("Authorization")) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Errorf("invalid Content-Type: %s", r.Header.Get("Content-Type")) + } + + if r.Header.Get("mcp-session-id") == "mock-session-12345" { + sessionReceived = true + } + + handleMockMCP(w, r, func(req JSONRPCRequest) (json.RawMessage, bool, *JSONRPCError) { + return json.RawMessage(`{"content":[{"type":"text","text":"{\"status\":\"ok\",\"user\":\"operator\"}"}],"isError":false}`), false, nil + }) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + res, err := client.CallTool(context.Background(), "system_whoami", nil) + if err != nil { + t.Fatalf("CallTool failed: %v", err) + } + + if res["status"] != "ok" { + t.Errorf("expected status ok, got %v", res["status"]) + } + if res["user"] != "operator" { + t.Errorf("expected user operator, got %v", res["user"]) + } + if !sessionReceived { + t.Errorf("expected mcp-session-id header to be sent on tool calls after initialize") + } +} + +func TestClientCallToolServerError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handleMockMCP(w, r, func(req JSONRPCRequest) (json.RawMessage, bool, *JSONRPCError) { + return nil, true, &JSONRPCError{ + Code: -32000, + Message: "Internal tool failure", + } + }) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + _, err := client.CallTool(context.Background(), "fail_tool", nil) + if err == nil { + t.Fatalf("expected error from failed tool call, got nil") + } +} + +func TestClientCallToolIsError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handleMockMCP(w, r, func(req JSONRPCRequest) (json.RawMessage, bool, *JSONRPCError) { + return json.RawMessage(`{"content":[{"type":"text","text":"Failed extraction on invalid document"}],"isError":true}`), true, nil + }) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + res, err := client.CallTool(context.Background(), "long_ingest_async", nil) + if err != nil { + t.Fatalf("unexpected transport error: %v", err) + } + if res["status"] != "error" || res["isError"] != true { + t.Errorf("expected status=error and isError=true, got %+v", res) + } +} + +func TestClientCallToolEmptyContentIsError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handleMockMCP(w, r, func(req JSONRPCRequest) (json.RawMessage, bool, *JSONRPCError) { + return json.RawMessage(`{"content":[],"isError":true}`), true, nil + }) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + res, err := client.CallTool(context.Background(), "some_tool", nil) + if err != nil { + t.Fatalf("unexpected transport error: %v", err) + } + if res["status"] != "error" || res["isError"] != true { + t.Errorf("expected status=error and isError=true for empty content error, got %+v", res) + } +} + +func TestClientCallToolIDMismatch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + w.Header().Set("Content-Type", "application/json") + if req.Method == "initialize" { + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: 99999, // Mismatched ID + Result: json.RawMessage(`{"status":"ok"}`), + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + _, err := client.CallTool(context.Background(), "some_tool", nil) + if err == nil { + t.Fatalf("expected ID mismatch error, got nil") + } +} + +func TestClientInitializeIncompatibleVersion(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + w.Header().Set("Content-Type", "application/json") + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"1.0.0-legacy","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + _, err := client.CallTool(context.Background(), "some_tool", nil) + if err == nil { + t.Fatalf("expected incompatible protocol version error, got nil") + } + if !strings.Contains(err.Error(), "unsupported mcp protocol version") { + t.Errorf("expected unsupported protocol version error message, got %v", err) + } +} + +func TestClientInitializeMissingToolsCapability(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + w.Header().Set("Content-Type", "application/json") + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + _, err := client.CallTool(context.Background(), "some_tool", nil) + if err == nil { + t.Fatalf("expected missing tools capability error, got nil") + } + if !strings.Contains(err.Error(), "does not declare 'tools' capability") { + t.Errorf("expected missing tools capability error message, got %v", err) + } +} + +func TestClientInitializeNotificationRejected(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + w.Header().Set("Content-Type", "application/json") + if req.Method == "initialize" { + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + if req.Method == "notifications/initialized" { + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"status":"ok"}`), + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + _, err := client.CallTool(context.Background(), "some_tool", nil) + if err == nil { + t.Fatalf("expected rejected notification error, got nil") + } + if !strings.Contains(err.Error(), "notifications/initialized rejected") { + t.Errorf("expected rejected notification error message, got %v", err) + } +} + +func TestClientCallToolOversizedResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + // Write 17MB response (exceeding 16MB limit) + chunk := strings.Repeat("A", 1024*1024) + for i := 0; i < 17; i++ { + _, _ = w.Write([]byte(chunk)) + } + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + _, err := client.CallTool(context.Background(), "oversized_tool", nil) + if err == nil { + t.Fatalf("expected oversized response error, got nil") + } + if !strings.Contains(err.Error(), "exceeded maximum allowed size") { + t.Errorf("expected maximum allowed size error, got %v", err) + } +} + +func TestClientCallToolSSEStreamResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if req.Method == "initialize" { + w.Header().Set("Content-Type", "application/json") + resp := JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + if req.Method == "notifications/initialized" { + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("Content-Type", "text/event-stream") + sseData := fmt.Sprintf("event: message\ndata: {\"jsonrpc\":\"2.0\",\"id\":%d,\"result\":{\"status\":\"ok\",\"job_id\":\"job-sse-123\"}}\n\n", req.ID) + _, _ = w.Write([]byte(sseData)) + })) + defer server.Close() + + client := NewClient(server.URL, "test-token", 10*time.Second) + res, err := client.CallTool(context.Background(), "test_sse", nil) + if err != nil { + t.Fatalf("unexpected error calling SSE tool: %v", err) + } + if res["status"] != "ok" || res["job_id"] != "job-sse-123" { + t.Fatalf("unexpected result from SSE tool: %+v", res) + } +} diff --git a/tools/graph-memory-ingest/internal/mcpclient/port_test.go b/tools/graph-memory-ingest/internal/mcpclient/port_test.go new file mode 100644 index 0000000..edba729 --- /dev/null +++ b/tools/graph-memory-ingest/internal/mcpclient/port_test.go @@ -0,0 +1,56 @@ +package mcpclient + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestSSEStopsAtResponseAfterNotifications(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req JSONRPCRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if req.Method == "notifications/initialized" { + w.WriteHeader(202) + return + } + w.Header().Set("Content-Type", "text/event-stream") + result := `{"content":[],"structuredContent":{"status":"ok","value":42}}` + if req.Method == "initialize" { + w.Header().Set("Mcp-Session-Id", "sse-session") + w.Header().Set("Mcp-Protocol-Version", "2024-10-07") + result = `{"protocolVersion":"2024-10-07","capabilities":{"tools":{}},"serverInfo":{"name":"test"}}` + } else if r.Header.Get("Mcp-Session-Id") != "sse-session" || r.Header.Get("Mcp-Protocol-Version") != "2024-10-07" { + t.Error("negotiated headers missing") + } + fmt.Fprint(w, ": heartbeat\r\ndata: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/message\"}\r\n\r\n") + fmt.Fprintf(w, "event: message\r\ndata: {\"jsonrpc\":\"2.0\",\"id\":%d,\r\ndata: \"result\":%s}\r\n\r\n", req.ID, result) + w.(http.Flusher).Flush() + // Stream stays open: waiting for EOF instead of the response would time out. + <-r.Context().Done() + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result, err := NewClient(server.URL, "", time.Second).CallTool(ctx, "example", map[string]interface{}{}) + if err != nil || result["value"] != float64(42) { + t.Fatalf("result=%v err=%v", result, err) + } +} + +func TestMissingToolDoesNotMaskOperationalErrors(t *testing.T) { + for _, message := range []string{"access denied", "Unknown tool: another_tool", "long_document_list backend not found", "database error: unknown tool: long_document_list is unavailable"} { + if IsToolMissing(map[string]interface{}{"status": "error", "message": message}, nil, "long_document_list") { + t.Errorf("masked %s", message) + } + } + _, err := readJSONRPCResponse("text/event-stream", strings.NewReader("data: {\"jsonrpc\":\"2.0\",\"id\":9,\"result\":{}}\n\n"), 2) + if err == nil { + t.Fatal("accepted wrong SSE response id") + } +} diff --git a/tools/graph-memory-ingest/internal/ontology/evaluator.go b/tools/graph-memory-ingest/internal/ontology/evaluator.go new file mode 100644 index 0000000..f97cc61 --- /dev/null +++ b/tools/graph-memory-ingest/internal/ontology/evaluator.go @@ -0,0 +1,393 @@ +package ontology + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "math" + "math/big" + "os" + "strings" + "time" + + "graph-memory-ingest/internal/ingest" + "graph-memory-ingest/internal/mcpclient" + "graph-memory-ingest/internal/scanner" +) + +// EvalOptions configures the ontology adequacy evaluation +type EvalOptions struct { + RootPath string `json:"root_path"` + SpaceID string `json:"space_id"` + Ontology string `json:"ontology"` + SampleSize int `json:"sample_size"` + ThresholdOther int `json:"threshold_other"` + KeepMemory bool `json:"keep_memory"` + AllowedExtensions []string `json:"allowed_extensions"` + AllowedExts []string `json:"allowed_exts"` // alias + MaxFileBytes int64 `json:"max_file_bytes"` +} + +// UnclassifiedConcept holds information about an untyped entity +type UnclassifiedConcept struct { + Name string `json:"name"` + Frequency int `json:"frequency,omitempty"` + Sources []string `json:"sources,omitempty"` +} + +// EvalResult contains the complete adequacy assessment and actionable report +type EvalResult struct { + OntologyPath string `json:"ontology_path"` + SpaceID string `json:"space_id"` + IsTemporarySpace bool `json:"is_temporary_space"` + SampleCount int `json:"sample_count"` + SampleFiles []string `json:"sample_files"` + TotalEntities int `json:"total_entities"` + TypedEntities int `json:"typed_entities"` + OtherEntities int `json:"other_entities"` + OtherPercentage float64 `json:"other_percentage"` + RelevanceTier string `json:"relevance_tier"` // Excellent (<1%), Very Good (<5%), Moderate (<10%), Inadequate (>=10%) + EntityTypesBreakdown map[string]int `json:"entity_types_breakdown,omitempty"` + UnclassifiedConcepts []UnclassifiedConcept `json:"unclassified_concepts,omitempty"` + ThresholdOther int `json:"threshold_other"` + PassedThreshold bool `json:"passed_threshold"` + ExtractionStatus string `json:"extraction_status"` + Message string `json:"message"` + Suggestions []string `json:"suggestions,omitempty"` +} + +// Evaluator evaluates ontology fit against document samples +type Evaluator struct { + client *mcpclient.Client +} + +// NewEvaluator creates a new Evaluator instance +func NewEvaluator(client *mcpclient.Client) *Evaluator { + return &Evaluator{client: client} +} + +// TestOntology is the primary entry point for ontology evaluation +func (e *Evaluator) TestOntology(ctx context.Context, rootPath string, opts EvalOptions) (*EvalResult, error) { + opts.RootPath = rootPath + if len(opts.AllowedExtensions) == 0 && len(opts.AllowedExts) > 0 { + opts.AllowedExtensions = opts.AllowedExts + } + return e.Evaluate(ctx, opts) +} + +// Evaluate performs real end-to-end evaluation using an ephemeral space and graph_status +func (e *Evaluator) Evaluate(ctx context.Context, opts EvalOptions) (result *EvalResult, resultErr error) { + if opts.SampleSize <= 0 { + opts.SampleSize = 5 + } + if opts.ThresholdOther < 0 || opts.ThresholdOther > 100 { + return nil, fmt.Errorf("threshold-other must be between 0 and 100, got %d", opts.ThresholdOther) + } + + // 1. Read ontology YAML (from file path or raw string) + var ontologyYAML string + if opts.Ontology != "" { + if fileInfo, err := os.Stat(opts.Ontology); err == nil && fileInfo.Mode().IsRegular() { + data, err := os.ReadFile(opts.Ontology) + if err != nil { + return nil, fmt.Errorf("failed to read ontology YAML file %s: %w", opts.Ontology, err) + } + ontologyYAML = string(data) + } else { + ontologyYAML = opts.Ontology + } + } + + allowedExts := opts.AllowedExtensions + if len(allowedExts) == 0 && len(opts.AllowedExts) > 0 { + allowedExts = opts.AllowedExts + } + + // 2. Discover sample documents in the path + scanRes, err := scanner.Scan(scanner.ScanOptions{ + RootPath: opts.RootPath, + AllowedExtensions: allowedExts, + BatchSizeMB: 50, + MaxFileBytes: opts.MaxFileBytes, + ForceReplace: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to scan path %s: %w", opts.RootPath, err) + } + + if scanRes.TotalFiles == 0 { + return nil, fmt.Errorf("no valid files found in path %s for ontology evaluation", opts.RootPath) + } + + // Select sample files + var sampleItems []scanner.FileItem + var sampleNames []string + for _, b := range scanRes.Batches { + for _, f := range b.Files { + sampleItems = append(sampleItems, f) + sampleNames = append(sampleNames, f.RelPath) + if len(sampleItems) >= opts.SampleSize { + break + } + } + if len(sampleItems) >= opts.SampleSize { + break + } + } + + // 3. Create target space (ephemeral by default) + targetSpace := opts.SpaceID + isTemp := false + if targetSpace == "" { + isTemp = true + for attempt := 0; attempt < 5; attempt++ { + rVal, err := rand.Int(rand.Reader, big.NewInt(100000)) + if err != nil { + return nil, err + } + candidate := fmt.Sprintf("tmp-eval-%d-%05d", time.Now().Unix(), rVal.Int64()) + createRes, err := e.client.CallTool(ctx, "space_create", map[string]interface{}{ + "space_id": candidate, + "description": fmt.Sprintf("Ephemeral ontology evaluation space for %s", opts.RootPath), + "rules": "", + }) + if mcpclient.IsToolMissing(createRes, err, "space_create") { + return nil, fmt.Errorf("ontology evaluation requires Hivemind space_create, ontology_validate and long_status; Graph Memory standalone does not expose this workflow") + } + if err != nil { + return nil, fmt.Errorf("space_create failed for %s: %w", candidate, err) + } + if createRes["status"] == "created" && createRes["isError"] != true { + targetSpace = candidate + break + } + if createRes["status"] != "already_exists" { + return nil, fmt.Errorf("space_create failed: %v", createRes["message"]) + } + } + if targetSpace == "" { + return nil, fmt.Errorf("failed to create unique ephemeral space after multiple attempts") + } + } else { + createRes, err := e.client.CallTool(ctx, "space_create", map[string]interface{}{ + "space_id": targetSpace, + "description": fmt.Sprintf("Evaluation space for %s", opts.RootPath), + "rules": "", + }) + if err != nil { + return nil, fmt.Errorf("failed to create evaluation space %s: %w", targetSpace, err) + } + createStatus, _ := createRes["status"].(string) + if createStatus != "created" || createRes["isError"] == true { + return nil, fmt.Errorf("evaluation requires a new, empty space; space %s returned status '%s': %v", targetSpace, createStatus, createRes["message"]) + } + } + + // Install cleanup hook immediately upon space creation + defer func() { + if !opts.KeepMemory { + cleanCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cleanRes, cleanErr := e.client.CallTool(cleanCtx, "space_delete", map[string]interface{}{ + "space_id": targetSpace, + "confirm": true, + }) + if cleanErr == nil && (cleanRes["isError"] == true || (cleanRes["status"] != "ok" && cleanRes["status"] != "deleted")) { + cleanErr = fmt.Errorf("%v", cleanRes["message"]) + } + if cleanErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("cleanup failed for space %s: %w", targetSpace, cleanErr)) + } + } + }() + + // Validate ontology YAML with ontology_validate in context of the created space + if ontologyYAML != "" { + valRes, err := e.client.CallTool(ctx, "ontology_validate", map[string]interface{}{ + "space_id": targetSpace, + "content_yaml": ontologyYAML, + }) + if err != nil { + return nil, fmt.Errorf("ontology validation request failed: %w", err) + } + if valRes["status"] == "error" || valRes["isError"] == true { + return nil, fmt.Errorf("invalid ontology YAML: %v", valRes["message"]) + } + if valid, ok := valRes["valid"].(bool); !ok || !valid { + return nil, fmt.Errorf("ontology schema is invalid: %v", valRes["errors"]) + } + } + + // 4. Load sample contents and submit ingestion + var docPayloads []map[string]interface{} + for _, item := range sampleItems { + contentObj, err := scanner.LoadFileContent(item.Path, opts.MaxFileBytes) + if err != nil { + return nil, fmt.Errorf("failed to load sample %s: %w", item.Path, err) + } + + doc := map[string]interface{}{ + "source_path": item.RelPath, + "filename": item.Filename, + "sha256": contentObj.SHA256, + "content_base64": contentObj.Base64Data, + "metadata": map[string]interface{}{ + "content_type": item.ContentType, + }, + } + docPayloads = append(docPayloads, doc) + } + + ingestOpts := map[string]interface{}{} + if ontologyYAML != "" { + ingestOpts["ontology_yaml"] = ontologyYAML + } + + ingestRes, err := e.client.CallTool(ctx, "long_ingest_async", map[string]interface{}{ + "space_id": targetSpace, + "documents": docPayloads, + "options": ingestOpts, + }) + if err != nil { + return nil, fmt.Errorf("failed to submit sample ingestion: %w", err) + } + if ingestRes["status"] == "error" || ingestRes["isError"] == true { + return nil, fmt.Errorf("ingestion error on evaluation: %v", ingestRes["message"]) + } + + records, err := ingest.ReconcileBatch(sampleItems, ingestRes) + if err != nil { + return nil, fmt.Errorf("invalid sample ingestion response: %w", err) + } + engine := ingest.NewEngine(e.client) + for _, job := range records { + if job.Error != "" { + return nil, fmt.Errorf("sample ingestion failed for %s: %s", job.Filename, job.Error) + } + switch job.Status { + case "succeeded", "completed", "skipped", "changed_skipped": + default: + if _, err := engine.PollJob(ctx, targetSpace, job.JobID, 0); err != nil { + return nil, fmt.Errorf("sample ingestion job %s failed: %w", job.JobID, err) + } + } + } + + // 5. Read graph statistics directly from long_status + statusRes, err := e.client.CallTool(ctx, "long_status", map[string]interface{}{ + "space_id": targetSpace, + "include_graph": true, + }) + if err != nil { + return nil, fmt.Errorf("failed to fetch graph status for evaluation: %w", err) + } + if statusRes["status"] != "ok" || statusRes["isError"] == true { + return nil, fmt.Errorf("failed to get valid graph status: %v", statusRes["message"]) + } + + res := &EvalResult{ + OntologyPath: opts.Ontology, + SpaceID: targetSpace, + IsTemporarySpace: isTemp, + SampleCount: len(sampleItems), + SampleFiles: sampleNames, + ThresholdOther: opts.ThresholdOther, + EntityTypesBreakdown: make(map[string]int), + Suggestions: make([]string, 0), + } + + // Extract graph stats (supports document_count, entity_count/entities_count, relation_count) + if statsMap, ok := statusRes["graph_stats"].(map[string]interface{}); ok { + if totalEnts, ok := statsMap["entity_count"].(float64); ok { + res.TotalEntities = int(totalEnts) + } else if totalEnts, ok := statsMap["entities_count"].(float64); ok { + res.TotalEntities = int(totalEnts) + } else if totalEntsInt, ok := statsMap["entity_count"].(int); ok { + res.TotalEntities = totalEntsInt + } + + if typeDist, ok := statsMap["entity_types"].(map[string]interface{}); ok { + for typeName, countRaw := range typeDist { + c := 0 + if cFloat, ok := countRaw.(float64); ok { + if cFloat < 0 || cFloat != math.Trunc(cFloat) { + return nil, fmt.Errorf("invalid entity count for %s", typeName) + } + c = int(cFloat) + } else if cInt, ok := countRaw.(int); ok { + if cInt < 0 { + return nil, fmt.Errorf("invalid entity count for %s", typeName) + } + c = cInt + } else { + return nil, fmt.Errorf("invalid entity count for %s", typeName) + } + res.EntityTypesBreakdown[typeName] = c + if strings.EqualFold(typeName, "Other") || strings.EqualFold(typeName, "Generic") { + res.OtherEntities += c + } else { + res.TypedEntities += c + } + } + } + } + + // Extract top unclassified entities + if topEnts, ok := statusRes["top_entities"].([]interface{}); ok { + for _, entRaw := range topEnts { + if entMap, ok := entRaw.(map[string]interface{}); ok { + eType, _ := entMap["type"].(string) + eName, _ := entMap["name"].(string) + if eName == "" { + eName, _ = entMap["entity"].(string) + } + if strings.EqualFold(eType, "Other") || strings.EqualFold(eType, "Generic") { + res.UnclassifiedConcepts = append(res.UnclassifiedConcepts, UnclassifiedConcept{ + Name: eName, + }) + } + } + } + } + + // If TotalEntities is derived from sum + if res.TotalEntities == 0 { + res.TotalEntities = res.TypedEntities + res.OtherEntities + } + + if res.TotalEntities == 0 { + return nil, fmt.Errorf("no entities were extracted from the sample documents during ontology evaluation") + } + + if len(res.EntityTypesBreakdown) == 0 || res.TotalEntities != res.TypedEntities+res.OtherEntities { + return nil, fmt.Errorf("long_status lacks a complete entity_types distribution; ontology evaluation cannot be scored reliably") + } + + res.OtherPercentage = (float64(res.OtherEntities) / float64(res.TotalEntities)) * 100.0 + + // Determine relevance tier + switch { + case res.OtherPercentage < 1.0: + res.RelevanceTier = "Excellent (<1%)" + case res.OtherPercentage < 5.0: + res.RelevanceTier = "Very Good (<5%)" + case res.OtherPercentage < 10.0: + res.RelevanceTier = "Moderate (<10%)" + default: + res.RelevanceTier = "Inadequate (>=10%)" + } + + if res.OtherPercentage > float64(opts.ThresholdOther) { + res.PassedThreshold = false + res.ExtractionStatus = "warning" + res.Message = fmt.Sprintf("Ontology adequacy alert: %.1f%% of extracted entities are unclassified 'Other' (threshold: %d%%).", res.OtherPercentage, opts.ThresholdOther) + res.Suggestions = append(res.Suggestions, "Enrich ontology YAML with dedicated entity types for the unclassified concepts listed in the report.") + } else { + res.PassedThreshold = true + res.ExtractionStatus = "success" + res.Message = fmt.Sprintf("Ontology adequacy verified: %.1f%% unclassified entities (Tier: %s, threshold: %d%%).", res.OtherPercentage, res.RelevanceTier, opts.ThresholdOther) + } + + return res, nil +} diff --git a/tools/graph-memory-ingest/internal/ontology/evaluator_test.go b/tools/graph-memory-ingest/internal/ontology/evaluator_test.go new file mode 100644 index 0000000..61669d1 --- /dev/null +++ b/tools/graph-memory-ingest/internal/ontology/evaluator_test.go @@ -0,0 +1,184 @@ +package ontology + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "graph-memory-ingest/internal/mcpclient" +) + +func newMockServer(toolHandler func(name string, args map[string]interface{}) (string, bool)) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req mcpclient.JSONRPCRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + + if req.Method == "initialize" { + w.Header().Set("mcp-session-id", "mock-session-id") + w.Header().Set("mcp-protocol-version", "2024-11-05") + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"test","version":"1.0"}}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + + if req.Method == "notifications/initialized" { + w.WriteHeader(http.StatusOK) + return + } + + if req.Method == "tools/call" { + params, _ := req.Params.(map[string]interface{}) + name, _ := params["name"].(string) + args, _ := params["arguments"].(map[string]interface{}) + resBody, isErr := toolHandler(name, args) + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"content":[{"type":"text","text":` + string(mustJSON(resBody)) + `}],"isError":` + fmt.Sprintf("%t", isErr) + `}`), + } + _ = json.NewEncoder(w).Encode(resp) + return + } + + resp := mcpclient.JSONRPCResponse{ + JSONRPC: "2.0", + ID: req.ID, + Result: json.RawMessage(`{"status":"ok"}`), + } + _ = json.NewEncoder(w).Encode(resp) + })) +} + +func TestEvaluator(t *testing.T) { + tempDir := t.TempDir() + docPath := filepath.Join(tempDir, "doc.md") + _ = os.WriteFile(docPath, []byte("# Software Spec\nServer and Database"), 0644) + + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + switch name { + case "ontology_validate": + return `{"status":"ok","valid":true}`, false + case "space_create": + return `{"status":"created"}`, false + case "long_ingest_async": + return `{"status":"ok","batch_id":"batch-eval-1","total":1,"counts":{"queued":1},"items":[{"index":0,"source_path":"doc.md","job_id":"job-123","status":"queued"}],"errors":[]}`, false + case "long_ingest_status": + return `{"status":"succeeded"}`, false + case "long_status": + return `{"status":"ok","graph_stats":{"entities_count":3,"entity_types":{"Component":1,"Database":1,"Other":1}},"top_entities":[{"name":"Misc","type":"Other"}]}`, false + case "space_delete": + return `{"status":"ok"}`, false + default: + return `{"status":"ok"}`, false + } + }) + defer server.Close() + + client := mcpclient.NewClient(server.URL, "tok", 10*time.Second) + eval := NewEvaluator(client) + + opts := EvalOptions{ + Ontology: "entities:\n - name: Component", + SampleSize: 1, + ThresholdOther: 40, // 1 Other out of 3 = 33.3% -> passes + } + + res, err := eval.TestOntology(context.Background(), tempDir, opts) + if err != nil { + t.Fatalf("TestOntology failed: %v", err) + } + + if !res.PassedThreshold { + t.Errorf("expected PassedThreshold true, got false") + } + if res.TotalEntities != 3 { + t.Errorf("expected 3 entities, got %d", res.TotalEntities) + } + if res.OtherEntities != 1 { + t.Errorf("expected 1 other entity, got %d", res.OtherEntities) + } + if len(res.UnclassifiedConcepts) != 1 || res.UnclassifiedConcepts[0].Name != "Misc" { + t.Errorf("expected 1 unclassified concept 'Misc', got %+v", res.UnclassifiedConcepts) + } + + // Test failing threshold + opts.ThresholdOther = 20 // 33.3% > 20% -> fails + res2, err := eval.TestOntology(context.Background(), tempDir, opts) + if err != nil { + t.Fatalf("TestOntology2 failed: %v", err) + } + if res2.PassedThreshold { + t.Errorf("expected PassedThreshold false, got true") + } +} + +func TestEvaluatorFailClosedOnServerError(t *testing.T) { + tempDir := t.TempDir() + docPath := filepath.Join(tempDir, "doc.md") + _ = os.WriteFile(docPath, []byte("# Content"), 0644) + + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + return `{"status":"error","message":"LLM rate limit reached"}`, false + }) + defer server.Close() + + client := NewEvaluator(mcpclient.NewClient(server.URL, "tok", 10*time.Second)) + _, err := client.TestOntology(context.Background(), tempDir, EvalOptions{ + Ontology: "entities:\n - name: Server", + SampleSize: 1, + ThresholdOther: 25, + }) + if err == nil { + t.Fatal("expected TestOntology to fail closed on server error, got nil") + } +} + +func TestEvaluatorRejectsInvalidYAML(t *testing.T) { + tempDir := t.TempDir() + docPath := filepath.Join(tempDir, "doc.md") + _ = os.WriteFile(docPath, []byte("# Content"), 0644) + + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + return `{"status":"ok","valid":false,"errors":["missing entities section"]}`, false + }) + defer server.Close() + + client := NewEvaluator(mcpclient.NewClient(server.URL, "tok", 10*time.Second)) + _, err := client.TestOntology(context.Background(), tempDir, EvalOptions{ + Ontology: "invalid-yaml", + SampleSize: 1, + ThresholdOther: 25, + }) + if err == nil { + t.Fatal("expected TestOntology to fail closed on invalid YAML, got nil") + } +} + +func TestEvaluatorRejectsInvalidThreshold(t *testing.T) { + client := NewEvaluator(mcpclient.NewClient("http://mock", "tok", 10*time.Second)) + _, err := client.TestOntology(context.Background(), ".", EvalOptions{ + ThresholdOther: 150, // Invalid > 100 + }) + if err == nil { + t.Fatal("expected error on threshold > 100, got nil") + } +} + +func mustJSON(s string) []byte { + b, _ := json.Marshal(s) + return b +} diff --git a/tools/graph-memory-ingest/internal/ontology/port_test.go b/tools/graph-memory-ingest/internal/ontology/port_test.go new file mode 100644 index 0000000..0538424 --- /dev/null +++ b/tools/graph-memory-ingest/internal/ontology/port_test.go @@ -0,0 +1,98 @@ +package ontology + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "graph-memory-ingest/internal/mcpclient" +) + +func TestEvaluationCleanupAndFailClosed(t *testing.T) { + for _, tc := range []struct { + name string + keep bool + badStage string + wantError bool + }{ + {"success", false, "", false}, {"keep", true, "", false}, + {"invalid YAML", false, "ontology_validate", true}, {"missing breakdown", false, "long_status", true}, + {"duplicate outcome", false, "long_ingest_async", true}, {"cleanup failure", false, "space_delete", true}, + {"creation denied", false, "space_create", true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "doc.md"), []byte("evaluation"), 0600); err != nil { + t.Fatal(err) + } + deleted, created := 0, 0 + yaml := "entities:\n - name: Server" + server := newMockServer(func(name string, args map[string]interface{}) (string, bool) { + if name == "space_create" { + created++ + if !strings.HasPrefix(args["space_id"].(string), "tmp-eval-") { + t.Error("space is not ephemeral") + } + } + if name == "space_delete" { + deleted++ + } + if name == tc.badStage { + switch name { + case "ontology_validate": + return `{"status":"ok","valid":false,"errors":["bad YAML"]}`, false + case "long_status": + return `{"status":"ok","graph_stats":{"entity_count":2}}`, false + case "long_ingest_async": + return `{"status":"ok","items":[{"source_path":"doc.md","status":"succeeded"},{"source_path":"doc.md","status":"succeeded"}]}`, false + default: + return `{"status":"error","message":"denied"}`, false + } + } + switch name { + case "space_create": + return `{"status":"created"}`, false + case "ontology_validate": + if args["content_yaml"] != yaml { + t.Error("YAML not passed to validator") + } + return `{"status":"ok","valid":true}`, false + case "long_ingest_async": + if args["options"].(map[string]interface{})["ontology_yaml"] != yaml { + t.Error("YAML not passed to ingestion") + } + return `{"status":"ok","items":[{"source_path":"doc.md","status":"completed"}]}`, false + case "long_status": + return `{"status":"ok","graph_stats":{"entity_count":4,"entity_types":{"Server":2,"Other":1,"Generic":1}}}`, false + case "space_delete": + return `{"status":"deleted"}`, false + default: + t.Errorf("unexpected call %s", name) + return "unexpected", true + } + }) + defer server.Close() + res, err := NewEvaluator(mcpclient.NewClient(server.URL, "", time.Second)).Evaluate(context.Background(), EvalOptions{RootPath: dir, Ontology: yaml, ThresholdOther: 50, KeepMemory: tc.keep}) + if (err != nil) != tc.wantError { + t.Fatalf("result=%+v err=%v", res, err) + } + if !tc.wantError && (res.OtherPercentage != 50 || !res.PassedThreshold || res.OtherEntities != 2) { + t.Fatalf("wrong ratio %+v", res) + } + wantDeletes := 1 + if tc.keep || tc.badStage == "space_create" { + wantDeletes = 0 + } + if deleted != wantDeletes || created != 1 { + t.Fatalf("created=%d deleted=%d want deletes=%d", created, deleted, wantDeletes) + } + if tc.badStage == "space_delete" && !strings.Contains(fmt.Sprint(err), "cleanup failed for space tmp-eval-") { + t.Fatal("missing cleanup recovery information") + } + }) + } +} diff --git a/tools/graph-memory-ingest/internal/scanner/port_test.go b/tools/graph-memory-ingest/internal/scanner/port_test.go new file mode 100644 index 0000000..5dc0dd4 --- /dev/null +++ b/tools/graph-memory-ingest/internal/scanner/port_test.go @@ -0,0 +1,26 @@ +package scanner + +import ( + "os" + "path/filepath" + "testing" +) + +func TestScanLocalDedupAndBinaryAllowlist(t *testing.T) { + dir := t.TempDir() + for name, data := range map[string][]byte{"a.md": []byte("text"), "b.md": []byte("text"), "binary.md": {0, 1, 2}, "doc.pdf": {'%', 'P', 'D', 'F', 0}, ".secret.md": []byte("secret")} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(filepath.Join(dir, "a.md"), filepath.Join(dir, "link.md")); err != nil { + t.Fatal(err) + } + res, err := Scan(ScanOptions{RootPath: dir, AllowedExtensions: []string{".md", ".pdf"}}) + if err != nil || res.TotalFiles != 2 || res.SkippedCount != 1 { + t.Fatalf("scan=%+v err=%v", res, err) + } + if res.Batches[0].Files[0].RelPath != "a.md" || res.Batches[0].Files[1].RelPath != "doc.pdf" { + t.Fatalf("wrong files: %+v", res.Batches) + } +} diff --git a/tools/graph-memory-ingest/internal/scanner/scanner.go b/tools/graph-memory-ingest/internal/scanner/scanner.go new file mode 100644 index 0000000..ef44755 --- /dev/null +++ b/tools/graph-memory-ingest/internal/scanner/scanner.go @@ -0,0 +1,346 @@ +package scanner + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "strings" + "unicode/utf8" +) + +// FileItem represents a discovered file with its metadata +type FileItem struct { + Path string `json:"path"` + RelPath string `json:"rel_path"` + Filename string `json:"filename"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + ContentType string `json:"content_type"` + IsText bool `json:"is_text"` +} + +// Batch represents a chunk of files grouped by size limit +type Batch struct { + Index int `json:"index"` + Files []FileItem `json:"files"` + TotalSize int64 `json:"total_size"` +} + +// ScanOptions configures the directory traversal and file selection +type ScanOptions struct { + RootPath string + AllowedExtensions []string + BatchSizeMB int + MaxFileBytes int64 + KnownHashes map[string]bool // map of sha256 -> exists + ForceReplace bool +} + +// ScanResult contains discovered items and batches +type ScanResult struct { + RootPath string `json:"root_path"` + TotalFiles int `json:"total_files"` + TotalBytes int64 `json:"total_bytes"` + Batches []Batch `json:"batches"` + SkippedCount int `json:"skipped_count"` + SkippedFiles []string `json:"skipped_files,omitempty"` +} + +// FileContent holds loaded data for a single file on-demand with strictly bound SHA-256 +type FileContent struct { + TextContent string + Base64Data string + IsText bool + SHA256 string +} + +// LoadFileContent reads and encodes file content on-demand, refusing symlinks and computing SHA-256 directly on the opened descriptor. +func LoadFileContent(path string, maxBytes int64) (*FileContent, error) { + if maxBytes <= 0 { + maxBytes = 50 * 1024 * 1024 + } + + lstat, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("failed to stat file %s: %w", path, err) + } + if lstat.Mode()&os.ModeSymlink != 0 || !lstat.Mode().IsRegular() { + return nil, fmt.Errorf("file %s is not a regular file (symlinks are forbidden)", path) + } + if lstat.Size() > maxBytes { + return nil, fmt.Errorf("file %s size (%d bytes) exceeds maximum allowed limit (%d bytes)", path, lstat.Size(), maxBytes) + } + + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: %w", path, err) + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, fmt.Errorf("failed to stat descriptor %s: %w", path, err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("file %s descriptor is not a regular file", path) + } + if !os.SameFile(lstat, info) { + return nil, fmt.Errorf("file %s was replaced during open (TOCTOU detected)", path) + } + + limitR := io.LimitReader(f, maxBytes+1) + data, err := io.ReadAll(limitR) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", path, err) + } + if int64(len(data)) > maxBytes { + return nil, fmt.Errorf("file %s size exceeded maximum allowed limit (%d bytes) during read", path, maxBytes) + } + + sha := sha256.Sum256(data) + hashHex := hex.EncodeToString(sha[:]) + b64 := base64.StdEncoding.EncodeToString(data) + + // Empty file is treated as empty text + if len(data) == 0 { + return &FileContent{ + TextContent: "", + Base64Data: "", + IsText: true, + SHA256: hashHex, + }, nil + } + + // Check for binary NUL bytes or invalid UTF-8 + isBinary := bytes.IndexByte(data, 0) != -1 || !utf8.Valid(data) + if isBinary { + return &FileContent{ + TextContent: "", + Base64Data: b64, + IsText: false, + SHA256: hashHex, + }, nil + } + + return &FileContent{ + TextContent: string(data), + Base64Data: b64, + IsText: true, + SHA256: hashHex, + }, nil +} + +// Scan traverses the target path and prepares file batches (metadata only, no persistent buffer OOM) +func Scan(opts ScanOptions) (*ScanResult, error) { + if opts.RootPath == "" { + return nil, fmt.Errorf("root path is required") + } + + info, err := os.Lstat(opts.RootPath) + if err != nil { + return nil, fmt.Errorf("failed to access %s: %w", opts.RootPath, err) + } + + extMap := make(map[string]bool) + for _, ext := range opts.AllowedExtensions { + e := strings.ToLower(strings.TrimSpace(ext)) + if e != "" { + if !strings.HasPrefix(e, ".") { + e = "." + e + } + extMap[e] = true + } + } + + var discovered []FileItem + var skipped []string + localHashes := make(map[string]bool) + + maxBatchBytes := int64(opts.BatchSizeMB) * 1024 * 1024 + if maxBatchBytes <= 0 { + maxBatchBytes = 50 * 1024 * 1024 + } + maxFileBytes := opts.MaxFileBytes + if maxFileBytes <= 0 { + maxFileBytes = maxBatchBytes + } + + processFile := func(fullPath string, fInfo os.FileInfo, relPath string) error { + // Strict regular file check: reject symlinks, FIFOs, devices, sockets + if !fInfo.Mode().IsRegular() { + return nil + } + + base := fInfo.Name() + if strings.HasPrefix(base, ".") { + return nil + } + + ext := strings.ToLower(filepath.Ext(base)) + if len(extMap) > 0 && !extMap[ext] { + return nil + } + + if fInfo.Size() > maxFileBytes { + return fmt.Errorf("file %s size (%d bytes) exceeds maximum allowed limit (%d bytes)", fullPath, fInfo.Size(), maxFileBytes) + } + + f, err := os.Open(fullPath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", fullPath, err) + } + defer f.Close() + + st, err := f.Stat() + if err != nil || !st.Mode().IsRegular() { + return nil + } + if !os.SameFile(fInfo, st) { + return fmt.Errorf("file %s was replaced during open (TOCTOU detected)", fullPath) + } + + // Stream SHA-256 computation and header sniffing with bounded reader + hasher := sha256.New() + headerBuf := make([]byte, 512) + nHeader, _ := io.ReadFull(f, headerBuf) + if nHeader > 0 { + hasher.Write(headerBuf[:nHeader]) + } + limitR := io.LimitReader(f, maxFileBytes+1-int64(nHeader)) + nCopied, err := io.Copy(hasher, limitR) + if err != nil { + return fmt.Errorf("failed to compute hash for %s: %w", fullPath, err) + } + if int64(nHeader)+nCopied > maxFileBytes { + return fmt.Errorf("file %s size exceeded maximum allowed limit (%d bytes) during read", fullPath, maxFileBytes) + } + hashHex := hex.EncodeToString(hasher.Sum(nil)) + + // Check if known and not forcing replace + if !opts.ForceReplace && opts.KnownHashes != nil && opts.KnownHashes[hashHex] { + skipped = append(skipped, relPath) + return nil + } + + cType := mime.TypeByExtension(ext) + if cType == "" && nHeader > 0 { + cType = http.DetectContentType(headerBuf[:nHeader]) + } + if cType == "" { + cType = "application/octet-stream" + } + + isText := true + if nHeader > 0 { + if bytes.IndexByte(headerBuf[:nHeader], 0) != -1 || !utf8.Valid(headerBuf[:nHeader]) { + isText = false + } + } + + // Binary content needs an explicitly allowed document extension. + if !isText && (!extMap[ext] || (ext != ".pdf" && ext != ".docx" && ext != ".xlsx" && ext != ".pptx" && ext != ".odt" && ext != ".ods" && ext != ".odp")) { + return nil + } + if !opts.ForceReplace && localHashes[hashHex] { + skipped = append(skipped, relPath) + return nil + } + localHashes[hashHex] = true + + item := FileItem{ + Path: fullPath, + RelPath: relPath, + Filename: base, + Size: int64(nHeader) + nCopied, + SHA256: hashHex, + ContentType: cType, + IsText: isText, + } + + discovered = append(discovered, item) + return nil + } + + if !info.IsDir() { + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("target path %s is not a regular file", opts.RootPath) + } + relPath := filepath.Base(opts.RootPath) + if err := processFile(opts.RootPath, info, relPath); err != nil { + return nil, err + } + } else { + err := filepath.Walk(opts.RootPath, func(path string, fInfo os.FileInfo, err error) error { + if err != nil { + return err + } + // Skip hidden directories explicitly with SkipDir + if fInfo.IsDir() { + base := fInfo.Name() + if strings.HasPrefix(base, ".") && path != opts.RootPath { + return filepath.SkipDir + } + return nil + } + + relPath, errRel := filepath.Rel(opts.RootPath, path) + if errRel != nil { + relPath = path + } + return processFile(path, fInfo, relPath) + }) + if err != nil { + return nil, fmt.Errorf("error walking path %s: %w", opts.RootPath, err) + } + } + + // Partition into batches + var batches []Batch + var currentBatch []FileItem + var currentSize int64 + batchIndex := 0 + + var totalBytes int64 + for _, item := range discovered { + if item.Size > maxBatchBytes { + return nil, fmt.Errorf("file %s exceeds batch size limit", item.RelPath) + } + totalBytes += item.Size + if len(currentBatch) > 0 && (currentSize+item.Size) > maxBatchBytes { + batches = append(batches, Batch{ + Index: batchIndex, + Files: currentBatch, + TotalSize: currentSize, + }) + batchIndex++ + currentBatch = nil + currentSize = 0 + } + currentBatch = append(currentBatch, item) + currentSize += item.Size + } + + if len(currentBatch) > 0 { + batches = append(batches, Batch{ + Index: batchIndex, + Files: currentBatch, + TotalSize: currentSize, + }) + } + + return &ScanResult{ + RootPath: opts.RootPath, + TotalFiles: len(discovered), + TotalBytes: totalBytes, + Batches: batches, + SkippedCount: len(skipped), + SkippedFiles: skipped, + }, nil +} diff --git a/tools/graph-memory-ingest/internal/scanner/scanner_test.go b/tools/graph-memory-ingest/internal/scanner/scanner_test.go new file mode 100644 index 0000000..e6b39f3 --- /dev/null +++ b/tools/graph-memory-ingest/internal/scanner/scanner_test.go @@ -0,0 +1,167 @@ +package scanner + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" +) + +func TestScanner(t *testing.T) { + tempDir := t.TempDir() + + f1 := filepath.Join(tempDir, "doc1.md") + f2 := filepath.Join(tempDir, "doc2.txt") + f3 := filepath.Join(tempDir, "ignored.bin") + subDir := filepath.Join(tempDir, "nested") + _ = os.MkdirAll(subDir, 0755) + f4 := filepath.Join(subDir, "doc3.md") + + _ = os.WriteFile(f1, []byte("# Title 1\nHello world"), 0644) + _ = os.WriteFile(f2, []byte("Simple plain text"), 0644) + _ = os.WriteFile(f3, []byte("binary data"), 0644) + _ = os.WriteFile(f4, []byte("# Nested doc"), 0644) + + opts := ScanOptions{ + RootPath: tempDir, + AllowedExtensions: []string{".md", ".txt"}, + BatchSizeMB: 1, + } + + res, err := Scan(opts) + if err != nil { + t.Fatalf("Scan failed: %v", err) + } + + if res.TotalFiles != 3 { + t.Errorf("expected 3 files, got %d", res.TotalFiles) + } + if len(res.Batches) == 0 { + t.Fatalf("expected at least 1 batch") + } + + // Test skip unchanged hashes + known := make(map[string]bool) + for _, batch := range res.Batches { + for _, file := range batch.Files { + if file.Filename == "doc1.md" { + known[file.SHA256] = true + } + } + } + + opts.KnownHashes = known + res2, err := Scan(opts) + if err != nil { + t.Fatalf("Scan2 failed: %v", err) + } + if res2.TotalFiles != 2 { + t.Errorf("expected 2 files after skipping doc1.md, got %d", res2.TotalFiles) + } + if res2.SkippedCount != 1 { + t.Errorf("expected 1 skipped file, got %d", res2.SkippedCount) + } +} + +func TestScannerRejectsSymlinksAndHiddenDirs(t *testing.T) { + tempDir := t.TempDir() + + // 1. Regular valid file + validFile := filepath.Join(tempDir, "valid.md") + _ = os.WriteFile(validFile, []byte("# Valid doc"), 0644) + + // 2. Hidden dir with file inside + hiddenDir := filepath.Join(tempDir, ".hidden") + _ = os.MkdirAll(hiddenDir, 0755) + hiddenFile := filepath.Join(hiddenDir, "secret.md") + _ = os.WriteFile(hiddenFile, []byte("# Secret in hidden dir"), 0644) + + // 3. Target outside dir and symlink to it + outsideDir := t.TempDir() + outsideFile := filepath.Join(outsideDir, "outside.md") + _ = os.WriteFile(outsideFile, []byte("# Outside secret file"), 0644) + + symlinkFile := filepath.Join(tempDir, "symlink.md") + _ = os.Symlink(outsideFile, symlinkFile) + + res, err := Scan(ScanOptions{ + RootPath: tempDir, + AllowedExtensions: []string{".md"}, + }) + if err != nil { + t.Fatalf("Scan failed: %v", err) + } + + // Only valid.md should be discovered (symlink and hidden dir skipped) + if res.TotalFiles != 1 { + t.Fatalf("expected exactly 1 discovered file, got %d", res.TotalFiles) + } + if res.Batches[0].Files[0].Filename != "valid.md" { + t.Errorf("expected valid.md, got %s", res.Batches[0].Files[0].Filename) + } +} + +func TestLoadFileContent(t *testing.T) { + tempDir := t.TempDir() + + // Text file + txtFile := filepath.Join(tempDir, "test.txt") + txtData := []byte("Hello UTF-8 text") + _ = os.WriteFile(txtFile, txtData, 0644) + c1, err := LoadFileContent(txtFile, 1024*1024) + if err != nil || !c1.IsText || c1.TextContent != "Hello UTF-8 text" { + t.Errorf("unexpected LoadFileContent on text file: %+v, err=%v", c1, err) + } + expectedHash := sha256.Sum256(txtData) + if c1.SHA256 != hex.EncodeToString(expectedHash[:]) { + t.Errorf("expected hash %s, got %s", hex.EncodeToString(expectedHash[:]), c1.SHA256) + } + + // Empty file + emptyFile := filepath.Join(tempDir, "empty.txt") + _ = os.WriteFile(emptyFile, []byte(""), 0644) + c2, err := LoadFileContent(emptyFile, 1024*1024) + if err != nil || !c2.IsText || c2.TextContent != "" { + t.Errorf("unexpected LoadFileContent on empty file: %+v, err=%v", c2, err) + } + + // Binary file with NUL byte + binFile := filepath.Join(tempDir, "bin.dat") + binData := []byte{0x00, 0xFF, 0xFE, 0x12} + _ = os.WriteFile(binFile, binData, 0644) + c3, err := LoadFileContent(binFile, 1024*1024) + if err != nil || c3.IsText || c3.Base64Data == "" { + t.Errorf("unexpected LoadFileContent on binary file: %+v, err=%v", c3, err) + } + + // Oversized file rejection + oversizedFile := filepath.Join(tempDir, "large.txt") + _ = os.WriteFile(oversizedFile, make([]byte, 2048), 0644) + _, err = LoadFileContent(oversizedFile, 1024) + if err == nil { + t.Errorf("expected error for oversized file loading, got nil") + } + + // Symlink rejection on load + symlink := filepath.Join(tempDir, "link_to_txt.txt") + _ = os.Symlink(txtFile, symlink) + _, err = LoadFileContent(symlink, 1024*1024) + if err == nil { + t.Errorf("expected error when loading symlink, got nil") + } +} + +func TestScanOversizedFile(t *testing.T) { + tempDir := t.TempDir() + largeFile := filepath.Join(tempDir, "huge.txt") + _ = os.WriteFile(largeFile, make([]byte, 5000), 0644) + + _, err := Scan(ScanOptions{ + RootPath: tempDir, + MaxFileBytes: 1000, + }) + if err == nil { + t.Errorf("expected error during scan of oversized file, got nil") + } +} diff --git a/tools/graph-memory-ingest/internal/tui/ioctl_darwin.go b/tools/graph-memory-ingest/internal/tui/ioctl_darwin.go new file mode 100644 index 0000000..76c74b2 --- /dev/null +++ b/tools/graph-memory-ingest/internal/tui/ioctl_darwin.go @@ -0,0 +1,9 @@ +//go:build darwin + +package tui + +import "syscall" + +func getTermiosIoctl() uint { + return syscall.TIOCGETA +} diff --git a/tools/graph-memory-ingest/internal/tui/ioctl_linux.go b/tools/graph-memory-ingest/internal/tui/ioctl_linux.go new file mode 100644 index 0000000..340f5cc --- /dev/null +++ b/tools/graph-memory-ingest/internal/tui/ioctl_linux.go @@ -0,0 +1,9 @@ +//go:build linux + +package tui + +import "syscall" + +func getTermiosIoctl() uint { + return syscall.TCGETS +} diff --git a/tools/graph-memory-ingest/internal/tui/ioctl_other.go b/tools/graph-memory-ingest/internal/tui/ioctl_other.go new file mode 100644 index 0000000..3cd532e --- /dev/null +++ b/tools/graph-memory-ingest/internal/tui/ioctl_other.go @@ -0,0 +1,7 @@ +//go:build !darwin && !linux + +package tui + +func getTermiosIoctl() uint { + return 0 +} diff --git a/tools/graph-memory-ingest/internal/tui/tui.go b/tools/graph-memory-ingest/internal/tui/tui.go new file mode 100644 index 0000000..e7fd493 --- /dev/null +++ b/tools/graph-memory-ingest/internal/tui/tui.go @@ -0,0 +1,180 @@ +package tui + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "syscall" + "unsafe" +) + +// UI manages terminal output and interactive prompts +type UI struct { + in io.Reader + out io.Writer + reader *bufio.Reader + isTTY bool + noColor bool +} + +// NewUI initializes terminal UI with automatic TTY and NO_COLOR detection +func NewUI(in io.Reader, out io.Writer) *UI { + isTTY := false + if f, ok := out.(*os.File); ok { + isTTY = isTerminal(f.Fd()) + } + noColor := os.Getenv("NO_COLOR") != "" || os.Getenv("TERM") == "dumb" || !isTTY + + return &UI{ + in: in, + out: out, + reader: bufio.NewReader(in), + isTTY: isTTY, + noColor: noColor, + } +} + +// IsInteractive returns true if running in an interactive terminal +func (u *UI) IsInteractive() bool { + return u.isTTY +} + +// Color formatting constants +const ( + colorReset = "\033[0m" + colorBold = "\033[1m" + colorDim = "\033[2m" + colorRed = "\033[31m" + colorGreen = "\033[32m" + colorYellow = "\033[33m" + colorBlue = "\033[34m" + colorMagenta = "\033[35m" + colorCyan = "\033[36m" + colorWhite = "\033[37m" +) + +func (u *UI) style(text, code string) string { + if u.noColor { + return text + } + return code + text + colorReset +} + +func (u *UI) Bold(text string) string { return u.style(text, colorBold) } +func (u *UI) Dim(text string) string { return u.style(text, colorDim) } +func (u *UI) Red(text string) string { return u.style(text, colorRed) } +func (u *UI) Green(text string) string { return u.style(text, colorGreen) } +func (u *UI) Yellow(text string) string { return u.style(text, colorYellow) } +func (u *UI) Cyan(text string) string { return u.style(text, colorCyan) } + +// Banner prints the stylish Graph Memory CLI header +func (u *UI) Banner() { + if u.noColor { + fmt.Fprintln(u.out, "=== GRAPH MEMORY INGESTION ===") + return + } + fmt.Fprintf(u.out, "%s%s⚡ GRAPH MEMORY INGESTION ⚡%s\n", colorBold, colorCyan, colorReset) + fmt.Fprintf(u.out, "%sHigh-throughput asynchronous knowledge graph ingestion%s\n\n", colorDim, colorReset) +} + +// PrintJSON outputs any struct as formatted JSON +func (u *UI) PrintJSON(v interface{}) error { + enc := json.NewEncoder(u.out) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +// Prompt asks the user for text input with a default value +func (u *UI) Prompt(label, defaultVal string) (string, error) { + promptText := label + if defaultVal != "" { + promptText = fmt.Sprintf("%s [%s]", label, u.Dim(defaultVal)) + } + fmt.Fprintf(u.out, "%s: ", promptText) + + line, err := u.reader.ReadString('\n') + if err != nil && err != io.EOF { + return "", err + } + val := strings.TrimSpace(line) + if val == "" { + return defaultVal, nil + } + return val, nil +} + +// Confirm asks a yes/no question +func (u *UI) Confirm(label string, defaultYes bool) (bool, error) { + opts := "[y/N]" + if defaultYes { + opts = "[Y/n]" + } + fmt.Fprintf(u.out, "%s %s: ", label, u.Dim(opts)) + + line, err := u.reader.ReadString('\n') + if err != nil && err != io.EOF { + return defaultYes, err + } + val := strings.ToLower(strings.TrimSpace(line)) + if val == "" { + return defaultYes, nil + } + return val == "y" || val == "yes" || val == "o" || val == "oui", nil +} + +// ProgressBar renders a progress indicator +func (u *UI) ProgressBar(current, total int, message string) { + if !u.isTTY { + fmt.Fprintf(u.out, "[%d/%d] %s\n", current, total, message) + return + } + + percent := 0 + if total > 0 { + percent = (current * 100) / total + } + if percent > 100 { + percent = 100 + } + + width := 24 + filled := (percent * width) / 100 + empty := width - filled + + bar := strings.Repeat("=", filled) + if filled > 0 && empty > 0 { + bar = bar[:filled-1] + ">" + } + space := strings.Repeat(" ", empty) + + fmt.Fprintf(u.out, "\r[%s%s] %3d%% (%d/%d) %-30s", u.Cyan(bar), space, percent, current, total, u.Dim(truncate(message, 30))) + if current >= total { + fmt.Fprintln(u.out) + } +} + +func truncate(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + if maxLen <= 3 { + return s[:maxLen] + } + return s[:maxLen-3] + "..." +} + +// isTerminal checks if fd is a real terminal on POSIX/Darwin +func isTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := syscall.Syscall6( + syscall.SYS_IOCTL, + fd, + uintptr(getTermiosIoctl()), + uintptr(unsafe.Pointer(&termios)), + 0, 0, 0, + ) + return err == 0 +} diff --git a/tools/graph-memory-ingest/internal/tui/tui_test.go b/tools/graph-memory-ingest/internal/tui/tui_test.go new file mode 100644 index 0000000..841d9c8 --- /dev/null +++ b/tools/graph-memory-ingest/internal/tui/tui_test.go @@ -0,0 +1,64 @@ +package tui + +import ( + "bytes" + "strings" + "testing" +) + +func TestUIFormatting(t *testing.T) { + var in bytes.Buffer + var out bytes.Buffer + + ui := NewUI(&in, &out) + + // Test styles + bold := ui.Bold("hello") + if !strings.Contains(bold, "hello") { + t.Errorf("expected string to contain hello, got %s", bold) + } + + // Test Banner + ui.Banner() + if !strings.Contains(strings.ToUpper(out.String()), "GRAPH MEMORY") { + t.Errorf("expected banner to contain GRAPH MEMORY, got %s", out.String()) + } + + // Test JSON + data := map[string]string{"key": "value"} + out.Reset() + _ = ui.PrintJSON(data) + if !strings.Contains(out.String(), `"key": "value"`) { + t.Errorf("expected json output, got %s", out.String()) + } + + // Test ProgressBar (headless) + out.Reset() + ui.ProgressBar(5, 10, "processing") + if !strings.Contains(out.String(), "[5/10]") { + t.Errorf("expected progress bar output, got %s", out.String()) + } +} + +func TestUIPromptAndConfirm(t *testing.T) { + in := bytes.NewBufferString("custom-val\ny\n") + var out bytes.Buffer + + ui := NewUI(in, &out) + + val, err := ui.Prompt("Enter name", "default-name") + if err != nil { + t.Fatalf("Prompt failed: %v", err) + } + if val != "custom-val" { + t.Errorf("expected custom-val, got %s", val) + } + + confirmed, err := ui.Confirm("Are you sure?", false) + if err != nil { + t.Fatalf("Confirm failed: %v", err) + } + if !confirmed { + t.Errorf("expected confirmed true, got false") + } +} diff --git a/tools/graph-memory-ingest/main.go b/tools/graph-memory-ingest/main.go new file mode 100644 index 0000000..dfb25a8 --- /dev/null +++ b/tools/graph-memory-ingest/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "graph-memory-ingest/internal/cli" + "graph-memory-ingest/internal/config" + "graph-memory-ingest/internal/tui" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + cfg, err := config.Load() + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to load config file: %v. Using defaults.\n", err) + cfg = config.DefaultConfig() + } + + ui := tui.NewUI(os.Stdin, os.Stdout) + app := cli.NewApp(ui, cfg) + + exitCode := app.Run(ctx, os.Args[1:]) + os.Exit(exitCode) +}