From 62597b52d378a72817f98f6fefbad30f7a3ca2cf Mon Sep 17 00:00:00 2001 From: Bortlesboat Date: Mon, 23 Mar 2026 12:00:42 -0400 Subject: [PATCH 01/64] fix: retry model deletion on Windows when files are locked After unloading a model, Windows may still hold file locks briefly. Add await on unload response and retry logic with backoff for EBUSY/EPERM errors during directory removal. Fixes #12 Co-Authored-By: Claude Opus 4.6 (1M context) --- electron/main/ipc-handlers.ts | 36 ++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 8083694..59411a4 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -145,17 +145,39 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe ipcMain.handle('model:delete', async (_, modelId: string): Promise<{ success: boolean; error?: string }> => { const modelDir = join(getSettings(app.getPath('userData')).modelsDir, modelId) + + // Unload the model and wait for confirmation so file handles are released try { - await axios.post(`${API_BASE_URL}/model/unload/${encodeURIComponent(modelId)}`, {}, { timeout: 5000 }) + await axios.post(`${API_BASE_URL}/model/unload/${encodeURIComponent(modelId)}`, {}, { timeout: 10_000 }) + // Give the OS a moment to release file locks (Windows holds handles briefly after close) + await new Promise(resolve => setTimeout(resolve, 1_500)) } catch { - // unload is best-effort — proceed with deletion anyway + // Unload failed (model may not be loaded) — still attempt deletion } - try { - await rmAsync(modelDir, { recursive: true, force: true }) - return { success: true } - } catch (err) { - return { success: false, error: String(err) } + + // Retry removal — Windows may return EBUSY/EPERM if handles linger + const maxRetries = 3 + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await rmAsync(modelDir, { recursive: true, force: true }) + return { success: true } + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code + const isLocked = code === 'EBUSY' || code === 'EPERM' + if (isLocked && attempt < maxRetries) { + await new Promise(resolve => setTimeout(resolve, 1_000 * attempt)) + continue + } + return { + success: false, + error: isLocked + ? `Model files are still locked after ${maxRetries} attempts. Close any programs using the model and try again.` + : String(err), + } + } } + + return { success: false, error: 'Unexpected error during deletion' } }) ipcMain.handle('model:showInFolder', (_, modelId: string) => { From da2137f8e927ea5a8115b5f0416f33881efbaaa2 Mon Sep 17 00:00:00 2001 From: Daniel Wahlgren Date: Sat, 25 Apr 2026 10:30:41 +0200 Subject: [PATCH 02/64] fix(security): correct typo in trusted extensions registry URL --- electron/main/ipc-handlers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 3871b97..4c2292b 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -488,7 +488,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe }) // Remote registry — list of trusted GitHub repo URLs - const REGISTRY_URL = 'https://raw.githubusercontent.com/liightnig125/modly-official-extension/main/registry.json' + const REGISTRY_URL = 'https://raw.githubusercontent.com/lightningpixel/modly-official-extension/main/registry.json' const REGISTRY_TTL = 5 * 60 * 1000 // 5 minutes let registryCache: { repos: Set; fetchedAt: number } | null = null From f98d89872267b4427ba393e04970f95d8845e27b Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 27 Apr 2026 08:01:52 +0200 Subject: [PATCH 03/64] feat(workflow): add more base node --- api/routers/generation.py | 6 +- api/routers/optimize.py | 32 +++- src/areas/generate/components/Viewer3D.tsx | 8 +- .../nodes/mesh-optimizer/processor.ts | 9 +- .../workflows/nodes/mesh-repair/manifest.json | 56 +++++++ .../workflows/nodes/mesh-repair/processor.py | 153 ++++++++++++++++++ .../nodes/mesh-smoother/manifest.json | 49 ++++++ .../nodes/mesh-smoother/processor.py | 124 ++++++++++++++ 8 files changed, 425 insertions(+), 12 deletions(-) create mode 100644 src/areas/workflows/nodes/mesh-repair/manifest.json create mode 100644 src/areas/workflows/nodes/mesh-repair/processor.py create mode 100644 src/areas/workflows/nodes/mesh-smoother/manifest.json create mode 100644 src/areas/workflows/nodes/mesh-smoother/processor.py diff --git a/api/routers/generation.py b/api/routers/generation.py index 2c4eb18..101ad24 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -174,6 +174,10 @@ def progress_cb(pct: int, step: str = "") -> None: if job_id in _cancelled: return tb = traceback.format_exc() - print(f"[Generation ERROR] {exc}\n{tb}") + msg = f"[Generation ERROR] {exc}\n{tb}" + try: + print(msg) + except UnicodeEncodeError: + print(msg.encode("ascii", errors="replace").decode("ascii")) job.status = "error" job.error = tb.strip() diff --git a/api/routers/optimize.py b/api/routers/optimize.py index db7152c..f68e261 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -69,11 +69,27 @@ def optimize_mesh(body: OptimizeRequest): def _has_texture(geom: trimesh.Trimesh) -> bool: - return ( - isinstance(geom.visual, trimesh.visual.TextureVisuals) - and geom.visual.material is not None - and getattr(geom.visual.material, "image", None) is not None - ) + if not isinstance(geom.visual, trimesh.visual.TextureVisuals): + return False + mat = geom.visual.material + if mat is None: + return False + # Simple material (SimpleMaterial / Material) + if getattr(mat, "image", None) is not None: + return True + # PBR material (from Trellis2 SLaT texturing and GLB imports) + if getattr(mat, "baseColorTexture", None) is not None: + return True + return False + + +def _get_texture_image(geom: trimesh.Trimesh): + """Return the base color texture image regardless of material type.""" + mat = geom.visual.material + img = getattr(mat, "image", None) + if img is not None: + return img + return getattr(mat, "baseColorTexture", None) def _decimate(input_path: str, target_faces: int, tmp_dir: str) -> trimesh.Trimesh: @@ -93,8 +109,8 @@ def _decimate(input_path: str, target_faces: int, tmp_dir: str) -> trimesh.Trime tex_in = os.path.join(tmp_dir, "texture.png") obj_out = os.path.join(tmp_dir, "output.obj") - # Save texture image under a known filename - geom.visual.material.image.save(tex_in) + # Save texture image under a known filename (handles PBR and simple materials) + _get_texture_image(geom).save(tex_in) # Export OBJ (trimesh writes UV coords + MTL) geom.export(obj_in) @@ -183,7 +199,7 @@ def _smooth(input_path: str, iterations: int, tmp_dir: str) -> trimesh.Trimesh: tex_in = os.path.join(tmp_dir, "texture.png") obj_out = os.path.join(tmp_dir, "output.obj") - geom.visual.material.image.save(tex_in) + _get_texture_image(geom).save(tex_in) geom.export(obj_in) if os.path.exists(mtl_in): diff --git a/src/areas/generate/components/Viewer3D.tsx b/src/areas/generate/components/Viewer3D.tsx index 01c055d..c39f4cd 100644 --- a/src/areas/generate/components/Viewer3D.tsx +++ b/src/areas/generate/components/Viewer3D.tsx @@ -1,7 +1,7 @@ import { Component, Suspense, useEffect, useMemo, useRef, useState } from 'react' import type { ReactNode, ErrorInfo } from 'react' import { Canvas, useThree } from '@react-three/fiber' -import { GizmoHelper, OrbitControls, useGizmoContext, useGLTF } from '@react-three/drei' +import { Environment, GizmoHelper, Lightformer, OrbitControls, useGizmoContext, useGLTF } from '@react-three/drei' import * as THREE from 'three' import { computeBoundsTree, disposeBoundsTree, acceleratedRaycast } from 'three-mesh-bvh' @@ -404,6 +404,12 @@ export default function Viewer3D({ lightSettings = DEFAULT_LIGHT_SETTINGS }: { l > + + + + + + diff --git a/src/areas/workflows/nodes/mesh-optimizer/processor.ts b/src/areas/workflows/nodes/mesh-optimizer/processor.ts index a8ecf66..d248c43 100644 --- a/src/areas/workflows/nodes/mesh-optimizer/processor.ts +++ b/src/areas/workflows/nodes/mesh-optimizer/processor.ts @@ -59,8 +59,13 @@ const processor = async ( // error tolerance scales with aggressiveness: tighter simplification needs more room const error = Math.max(0.001, 1 - ratio) - context.progress(25, 'Welding vertices…') - await doc.transform(weld()) + // Skip weld on large meshes — deduplication is O(N²) and stalls for millions of faces + if (currentFaces < 500_000) { + context.progress(25, 'Welding vertices…') + await doc.transform(weld()) + } else { + context.log(`Skipping weld (${currentFaces} faces > 500k threshold)`) + } context.progress(55, 'Simplifying mesh…') await doc.transform( diff --git a/src/areas/workflows/nodes/mesh-repair/manifest.json b/src/areas/workflows/nodes/mesh-repair/manifest.json new file mode 100644 index 0000000..fb44ef5 --- /dev/null +++ b/src/areas/workflows/nodes/mesh-repair/manifest.json @@ -0,0 +1,56 @@ +{ + "id": "mesh-repair", + "name": "Mesh Repair", + "type": "process", + "entry": "processor.py", + "version": "1.2.0", + "author": "Modly", + "description": "Repairs mesh topology: removes duplicates, degenerate faces, fixes non-manifold edges, fills simple boundary holes.", + "nodes": [ + { + "id": "repair", + "name": "Repair", + "input": "mesh", + "output": "mesh", + "params_schema": [ + { + "id": "remove_duplicates", + "label": "Remove Duplicates", + "type": "boolean", + "default": true, + "tooltip": "Remove duplicate vertices and faces." + }, + { + "id": "remove_degenerate", + "label": "Remove Degenerate Faces", + "type": "boolean", + "default": true, + "tooltip": "Remove zero-area faces and collapsed edges." + }, + { + "id": "fix_non_manifold", + "label": "Fix Non-Manifold", + "type": "boolean", + "default": true, + "tooltip": "Detach faces causing non-manifold edges." + }, + { + "id": "fill_holes", + "label": "Fill Holes", + "type": "boolean", + "default": true, + "tooltip": "Fill simple boundary holes. Structural holes from AI generation may not be fillable in post-processing." + }, + { + "id": "max_hole_size", + "label": "Max Hole Size", + "type": "int", + "default": 2000, + "min": 10, + "max": 10000, + "tooltip": "Maximum number of boundary edges of a hole to be filled. Increase if large holes remain open." + } + ] + } + ] +} diff --git a/src/areas/workflows/nodes/mesh-repair/processor.py b/src/areas/workflows/nodes/mesh-repair/processor.py new file mode 100644 index 0000000..b09979e --- /dev/null +++ b/src/areas/workflows/nodes/mesh-repair/processor.py @@ -0,0 +1,153 @@ +""" +Mesh Repair — built-in process extension. + +Fixes common topology issues in AI-generated meshes: + - Duplicate vertices and faces + - Non-manifold edges + - Degenerate (zero-area) faces + - Simple boundary holes + +Note: structural holes from FlexiCubes/TRELLIS voxel extraction cannot be +reliably closed in post-processing. Increase the generator's remesh resolution +to reduce them at the source. + +Protocol: reads one JSON line from stdin, writes JSON lines to stdout. + stdin : { input, params, workspaceDir, tempDir } + stdout: { type: "progress"|"log"|"done"|"error", ... } +""" +import json +import os +import shutil +import sys +import tempfile +from pathlib import Path + + +def emit(obj: dict) -> None: + print(json.dumps(obj), flush=True) + + +def progress(pct: int, label: str) -> None: + emit({"type": "progress", "percent": pct, "label": label}) + + +def log(msg: str) -> None: + emit({"type": "log", "message": msg}) + + +def done(file_path: str) -> None: + emit({"type": "done", "result": {"filePath": file_path}}) + + +def error(msg: str) -> None: + emit({"type": "error", "message": msg}) + + +def main() -> None: + raw = sys.stdin.readline() + data = json.loads(raw) + + input_data = data.get("input", {}) + params = data.get("params", {}) + workspace_dir = data.get("workspaceDir", "") + + input_path = input_data.get("filePath") + if not input_path or not Path(input_path).is_file(): + error(f"mesh-repair: input file not found: {input_path}") + return + + do_remove_dupes = bool(params.get("remove_duplicates", True)) + do_fix_non_manifold = bool(params.get("fix_non_manifold", True)) + do_remove_degen = bool(params.get("remove_degenerate", True)) + do_fill_holes = bool(params.get("fill_holes", True)) + max_hole_size = int(params.get("max_hole_size", 2000)) + + out_dir = Path(workspace_dir) / "Workflows" + out_dir.mkdir(parents=True, exist_ok=True) + from time import time + out_path = str(out_dir / f"mesh-repair-{int(time() * 1000)}.glb") + + try: + import pymeshlab + except ImportError: + error("mesh-repair: pymeshlab is not available on this system") + return + + import trimesh + + progress(10, "Loading mesh…") + loaded = trimesh.load(input_path) + if isinstance(loaded, trimesh.Scene): + geoms = list(loaded.geometry.values()) + geom = trimesh.util.concatenate(geoms) if len(geoms) > 1 else geoms[0] + else: + geom = loaded + + tmp_dir = tempfile.mkdtemp() + try: + ply_in = os.path.join(tmp_dir, "input.ply") + ply_out = os.path.join(tmp_dir, "output.ply") + geom.export(ply_in) + + ms = pymeshlab.MeshSet() + ms.load_new_mesh(ply_in) + + log(f"Input: {ms.current_mesh().vertex_number()} verts, {ms.current_mesh().face_number()} faces") + + if do_remove_dupes: + progress(20, "Removing duplicates…") + ms.meshing_remove_duplicate_vertices() + ms.meshing_remove_duplicate_faces() + + if do_remove_degen: + progress(40, "Removing degenerate faces…") + ms.meshing_remove_null_faces() + ms.meshing_remove_folded_faces() + + if do_fix_non_manifold: + progress(60, "Fixing non-manifold edges…") + # method=0 removes offending faces (low memory); method=1 detaches (OOMs on dense meshes) + try: + ms.meshing_repair_non_manifold_edges(method=0) + except Exception as e: + log(f"Non-manifold edge repair skipped: {e}") + try: + ms.meshing_repair_non_manifold_vertices() + except Exception as e: + log(f"Non-manifold vertex repair skipped: {e}") + + if do_fill_holes: + progress(75, "Filling holes…") + try: + ms.meshing_close_holes( + maxholesize=max_hole_size, + newfaceselected=False, + selfintersection=False, + ) + except Exception as e: + log(f"Hole fill skipped (mesh may still be non-manifold): {e}") + + after = ms.current_mesh().face_number() + log(f"Output: {ms.current_mesh().vertex_number()} verts, {after} faces") + + progress(85, "Exporting…") + ms.save_current_mesh(ply_out) + _loaded = trimesh.load(ply_out, process=False) + if isinstance(_loaded, trimesh.Scene): + _geoms = list(_loaded.geometry.values()) + _loaded = _geoms[0] if len(_geoms) == 1 else trimesh.util.concatenate(_geoms) + result = trimesh.Trimesh(vertices=_loaded.vertices, faces=_loaded.faces, process=False) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + result.export(out_path) + progress(100, "Done") + done(out_path) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + import traceback + error(f"{exc}\n{traceback.format_exc()}") diff --git a/src/areas/workflows/nodes/mesh-smoother/manifest.json b/src/areas/workflows/nodes/mesh-smoother/manifest.json new file mode 100644 index 0000000..4a84aff --- /dev/null +++ b/src/areas/workflows/nodes/mesh-smoother/manifest.json @@ -0,0 +1,49 @@ +{ + "id": "mesh-smoother", + "name": "Mesh Smoother", + "type": "process", + "entry": "processor.py", + "version": "1.0.0", + "author": "Modly", + "description": "Smooths mesh vertices to reduce sharp artifacts (e.g. zipper triangles from AI-generated meshes) using Taubin smoothing.", + "nodes": [ + { + "id": "smooth", + "name": "Smooth", + "input": "mesh", + "output": "mesh", + "params_schema": [ + { + "id": "iterations", + "label": "Iterations", + "type": "int", + "default": 5, + "min": 1, + "max": 50, + "tooltip": "Number of smoothing passes. More iterations = smoother result but may lose fine details." + }, + { + "id": "lambda_", + "label": "Smoothing Strength", + "type": "float", + "default": 0.5, + "min": 0.1, + "max": 1.0, + "step": 0.05, + "tooltip": "Controls how far each vertex moves toward its neighbours per iteration. Lower = more conservative." + }, + { + "id": "mode", + "label": "Mode", + "type": "select", + "default": "taubin", + "options": [ + { "value": "taubin", "label": "Taubin (volume-preserving)" }, + { "value": "laplacian", "label": "Laplacian (stronger, may shrink)" } + ], + "tooltip": "Taubin alternates positive/negative steps to prevent mesh shrinkage. Laplacian is simpler but tends to shrink the mesh over many iterations." + } + ] + } + ] +} diff --git a/src/areas/workflows/nodes/mesh-smoother/processor.py b/src/areas/workflows/nodes/mesh-smoother/processor.py new file mode 100644 index 0000000..3ae78b6 --- /dev/null +++ b/src/areas/workflows/nodes/mesh-smoother/processor.py @@ -0,0 +1,124 @@ +""" +Mesh Smoother — built-in process extension. + +Reduces sharp artifacts (zipper triangles, sawtooth edges) produced by +AI mesh generators via Taubin or Laplacian smoothing. + +Protocol: reads one JSON line from stdin, writes JSON lines to stdout. + stdin : { input, params, workspaceDir, tempDir } + stdout: { type: "progress"|"log"|"done"|"error", ... } +""" +import json +import os +import shutil +import sys +import tempfile +from pathlib import Path + + +def emit(obj: dict) -> None: + print(json.dumps(obj), flush=True) + + +def progress(pct: int, label: str) -> None: + emit({"type": "progress", "percent": pct, "label": label}) + + +def log(msg: str) -> None: + emit({"type": "log", "message": msg}) + + +def done(file_path: str) -> None: + emit({"type": "done", "result": {"filePath": file_path}}) + + +def error(msg: str) -> None: + emit({"type": "error", "message": msg}) + + +def main() -> None: + raw = sys.stdin.readline() + data = json.loads(raw) + + input_data = data.get("input", {}) + params = data.get("params", {}) + workspace_dir = data.get("workspaceDir", "") + + input_path = input_data.get("filePath") + if not input_path or not Path(input_path).is_file(): + error(f"mesh-smoother: input file not found: {input_path}") + return + + iterations = int(params.get("iterations", 5)) + lambda_ = float(params.get("lambda_", 0.5)) + mode = str(params.get("mode", "taubin")) + + out_dir = Path(workspace_dir) / "Workflows" + out_dir.mkdir(parents=True, exist_ok=True) + from time import time + out_path = str(out_dir / f"mesh-smoother-{int(time() * 1000)}.glb") + + log(f"Mode: {mode}, iterations: {iterations}, strength: {lambda_}") + + try: + import pymeshlab + except ImportError: + error("mesh-smoother: pymeshlab is not available on this system") + return + + import trimesh + + progress(10, "Loading mesh…") + loaded = trimesh.load(input_path) + if isinstance(loaded, trimesh.Scene): + geoms = list(loaded.geometry.values()) + geom = trimesh.util.concatenate(geoms) if len(geoms) > 1 else geoms[0] + else: + geom = loaded + + tmp_dir = tempfile.mkdtemp() + try: + ply_in = os.path.join(tmp_dir, "input.ply") + ply_out = os.path.join(tmp_dir, "output.ply") + geom.export(ply_in) + + ms = pymeshlab.MeshSet() + ms.load_new_mesh(ply_in) + + progress(30, f"Smoothing ({mode})…") + + if mode == "taubin": + ms.apply_coord_taubin_smoothing( + lambda_=lambda_, + mu=-lambda_ - 0.01, + stepsmoothnum=iterations, + ) + else: + ms.apply_coord_laplacian_smoothing( + stepsmoothnum=iterations, + cotangentweight=False, + ) + + progress(80, "Exporting…") + ms.save_current_mesh(ply_out) + # Load raw geometry only — avoids scipy dependency triggered by face→vertex color conversion + _loaded = trimesh.load(ply_out, process=False) + if isinstance(_loaded, trimesh.Scene): + _geoms = list(_loaded.geometry.values()) + _loaded = _geoms[0] if len(_geoms) == 1 else trimesh.util.concatenate(_geoms) + result = trimesh.Trimesh(vertices=_loaded.vertices, faces=_loaded.faces, process=False) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + result.export(out_path) + log(f"Output: {out_path} ({len(result.faces)} faces)") + progress(100, "Done") + done(out_path) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + import traceback + error(f"{exc}\n{traceback.format_exc()}") From c5c2a03d5818d95fec88b68d3ac3e6ea1752feb9 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 27 Apr 2026 14:51:41 +0200 Subject: [PATCH 04/64] feat(generate): add modly agent --- api/main.py | 3 +- api/mcp_server.py | 265 +++++++ api/requirements.txt | 6 +- api/routers/agent.py | 359 ++++++++++ src/areas/generate/components/ChatPanel.tsx | 678 ++++++++++++++++++ .../generate/components/WorkflowPanel.tsx | 112 ++- src/areas/workflows/workflowRunStore.ts | 11 +- 7 files changed, 1392 insertions(+), 42 deletions(-) create mode 100644 api/mcp_server.py create mode 100644 api/routers/agent.py create mode 100644 src/areas/generate/components/ChatPanel.tsx diff --git a/api/main.py b/api/main.py index eda77fa..bba2ad7 100644 --- a/api/main.py +++ b/api/main.py @@ -9,7 +9,7 @@ from fastapi.responses import FileResponse from fastapi import HTTPException -from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs +from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent @asynccontextmanager @@ -50,6 +50,7 @@ def filter(self, record): app.include_router(extensions.router, prefix="/extensions") app.include_router(export.router, prefix="/export") app.include_router(workflow_runs.router, prefix="/workflow-runs") +app.include_router(agent.router) # Serve generated files from workspace — dynamic so path changes take effect immediately @app.get("/workspace/{full_path:path}") diff --git a/api/mcp_server.py b/api/mcp_server.py new file mode 100644 index 0000000..fa9127c --- /dev/null +++ b/api/mcp_server.py @@ -0,0 +1,265 @@ +""" +Modly MCP Server +Exposes Modly's capabilities as MCP tools for external agents (Claude Desktop, Codex CLI, etc.). + +Usage: + python mcp_server.py + +Configuration in Claude Desktop (~/.config/claude/claude_desktop_config.json): + { + "mcpServers": { + "modly": { + "command": "python", + "args": ["C:/path/to/modly/desktop/api/mcp_server.py"] + } + } + } + +Requires Modly's FastAPI backend to be running on http://localhost:8765. +""" + +import asyncio +import mimetypes +import httpx +from mcp.server import Server +from mcp.server.stdio import stdio_server +from mcp.types import Tool, TextContent + +API_BASE = "http://localhost:8765" + +server = Server("modly") + + +@server.list_tools() +async def list_tools() -> list[Tool]: + return [ + Tool( + name="modly_list_models", + description="List all 3D generation models available in Modly (downloaded and ready to use).", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="modly_switch_model", + description="Switch the active 3D generation model in Modly.", + inputSchema={ + "type": "object", + "properties": { + "model_id": {"type": "string", "description": "The model ID to activate."}, + }, + "required": ["model_id"], + }, + ), + Tool( + name="modly_generate_from_image", + description="Generate a 3D mesh from a 2D image file. Returns a job_id to track progress.", + inputSchema={ + "type": "object", + "properties": { + "image_path": { + "type": "string", + "description": "Absolute path to the image file on disk.", + }, + "model_id": { + "type": "string", + "description": "Which model to use. If omitted, uses the currently active model.", + }, + "remesh": { + "type": "string", + "enum": ["quad", "triangle", "none"], + "description": "Remesh strategy after generation. Default: quad.", + }, + }, + "required": ["image_path"], + }, + ), + Tool( + name="modly_get_generation_status", + description="Poll the status of a 3D generation job. Call repeatedly until status is 'done' or 'error'.", + inputSchema={ + "type": "object", + "properties": { + "job_id": {"type": "string", "description": "Job ID returned by modly_generate_from_image."}, + }, + "required": ["job_id"], + }, + ), + Tool( + name="modly_decimate_mesh", + description="Reduce the polygon count of a mesh using quadric edge collapse decimation.", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "target_faces": { + "type": "integer", + "description": "Target number of faces after decimation (minimum 100).", + }, + }, + "required": ["path", "target_faces"], + }, + ), + Tool( + name="modly_smooth_mesh", + description="Apply Laplacian smoothing to a mesh. More iterations = smoother surface but less detail.", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "iterations": { + "type": "integer", + "description": "Number of smoothing iterations (1–20).", + }, + }, + "required": ["path", "iterations"], + }, + ), + Tool( + name="modly_import_mesh", + description="Import a mesh file from disk into Modly's workspace (.glb, .obj, .stl, .ply).", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Absolute path to the mesh file on disk.", + }, + }, + "required": ["path"], + }, + ), + Tool( + name="modly_unload_models", + description="Unload all 3D generation models from GPU VRAM. Useful before running VRAM-intensive tasks.", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="modly_get_settings", + description="Get the current Modly settings (models directory, workspace directory).", + inputSchema={"type": "object", "properties": {}}, + ), + ] + + +@server.call_tool() +async def call_tool(name: str, arguments: dict) -> list[TextContent]: + async with httpx.AsyncClient(timeout=60.0) as client: + try: + result = await _dispatch(client, name, arguments) + except httpx.ConnectError: + result = ( + "Cannot connect to Modly API at http://localhost:8765. " + "Make sure Modly is running." + ) + except httpx.HTTPStatusError as e: + result = f"Modly API error {e.response.status_code}: {e.response.text[:300]}" + except Exception as e: + result = f"Error: {e}" + + return [TextContent(type="text", text=result)] + + +async def _dispatch(client: httpx.AsyncClient, name: str, args: dict) -> str: + if name == "modly_list_models": + r = await client.get(f"{API_BASE}/model/all") + r.raise_for_status() + models = [m for m in r.json() if m.get("downloaded")] + if not models: + return "No models downloaded yet. Download one from the Models tab in Modly." + return "\n".join(f"- {m['id']}: {m.get('name', m['id'])}" for m in models) + + elif name == "modly_switch_model": + r = await client.post(f"{API_BASE}/model/switch", params={"model_id": args["model_id"]}) + r.raise_for_status() + return f"Switched active model to: {args['model_id']}" + + elif name == "modly_generate_from_image": + image_path: str = args["image_path"] + with open(image_path, "rb") as f: + img_bytes = f.read() + mime = mimetypes.guess_type(image_path)[0] or "image/png" + filename = image_path.replace("\\", "/").split("/")[-1] + + form_data = { + "remesh": args.get("remesh", "quad"), + } + if args.get("model_id"): + form_data["model_id"] = args["model_id"] + + r = await client.post( + f"{API_BASE}/generate/from-image", + files={"image": (filename, img_bytes, mime)}, + data=form_data, + timeout=30.0, + ) + r.raise_for_status() + job_id = r.json()["job_id"] + return ( + f"Generation started. Job ID: {job_id}\n" + f"Use modly_get_generation_status with this ID to track progress." + ) + + elif name == "modly_get_generation_status": + r = await client.get(f"{API_BASE}/generate/status/{args['job_id']}") + r.raise_for_status() + s = r.json() + parts = [f"Status: {s['status']}", f"Progress: {s.get('progress', 0)}%"] + if s.get("step"): + parts.append(f"Step: {s['step']}") + if s.get("output_url"): + parts.append(f"Output: {s['output_url']}") + if s.get("error"): + parts.append(f"Error: {s['error']}") + return " | ".join(parts) + + elif name == "modly_decimate_mesh": + r = await client.post( + f"{API_BASE}/optimize/mesh", + json={"path": args["path"], "target_faces": args["target_faces"]}, + ) + r.raise_for_status() + data = r.json() + return f"Decimated mesh to {data.get('face_count', '?')} faces. New file: {data.get('url', '')}" + + elif name == "modly_smooth_mesh": + r = await client.post( + f"{API_BASE}/optimize/smooth", + json={"path": args["path"], "iterations": args["iterations"]}, + ) + r.raise_for_status() + data = r.json() + return f"Smoothed mesh ({args['iterations']} iterations). New file: {data.get('url', '')}" + + elif name == "modly_import_mesh": + r = await client.post(f"{API_BASE}/optimize/import-by-path", json={"path": args["path"]}) + r.raise_for_status() + data = r.json() + return f"Mesh imported. URL: {data.get('url', '')}" + + elif name == "modly_unload_models": + r = await client.post(f"{API_BASE}/model/unload-all") + r.raise_for_status() + return "All 3D generation models unloaded from VRAM." + + elif name == "modly_get_settings": + r = await client.get(f"{API_BASE}/settings/paths") + r.raise_for_status() + data = r.json() + return f"Models directory: {data.get('models_dir')}\nWorkspace directory: {data.get('workspace_dir')}" + + else: + return f"Unknown tool: {name}" + + +async def main(): + async with stdio_server() as (read_stream, write_stream): + await server.run(read_stream, write_stream, server.create_initialization_options()) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/api/requirements.txt b/api/requirements.txt index a3888d5..473e37f 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,8 +1,12 @@ # Web server -fastapi==0.115.6 +fastapi>=0.115.6 uvicorn[standard]==0.34.0 python-multipart==0.0.20 +# Agent & MCP +httpx>=0.27.0 +mcp>=1.0.0 + # Mesh processing (optimize + export endpoints) trimesh>=4.5.0 pymeshlab>=2023.12 diff --git a/api/routers/agent.py b/api/routers/agent.py new file mode 100644 index 0000000..9d886f2 --- /dev/null +++ b/api/routers/agent.py @@ -0,0 +1,359 @@ +""" +Agent chat endpoint — runs an Ollama-powered tool-use loop against Modly's API. +""" +import re +import httpx +from fastapi import APIRouter +from pydantic import BaseModel + +router = APIRouter(prefix="/agent", tags=["agent"]) + +MODLY_API = "http://localhost:8765" + +SYSTEM_PROMPT = """\ +You are Modly's built-in AI assistant, specialized in 3D modeling and workflow automation. +You help users generate 3D models from images, optimize meshes, and manage workflows directly inside the Modly application. + +## Available tools + +- **list_models** — List all downloaded 3D generation models ready to use. +- **unload_models** — Unload all 3D generation models from GPU VRAM to free memory. +- **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). +- **decimate_mesh(path, target_faces)** — Reduce the polygon count of a mesh. +- **smooth_mesh(path, iterations)** — Apply Laplacian smoothing to a mesh. +- **get_generation_status(job_id)** — Poll the status of an ongoing 3D generation job. +- **list_workflows** — List all available workflows in Modly. +- **run_workflow(workflow_id)** — Execute a workflow in Modly by its ID. If the user attached an image in their message, it will automatically be used as the workflow's input image. + +## Rules + +- Always use tools to act on the scene — never just describe what you would do. +- If you need the current mesh path, call get_mesh_info first. +- If you need to run a workflow but don't know the ID, call list_workflows first. +- After each tool call, give a short one-sentence summary of what was done. +- Always reply in the same language the user is writing in. +- Be concise. No unnecessary explanations.\ +""" + +TOOLS = [ + { + "type": "function", + "function": { + "name": "list_models", + "description": "List all available 3D generation models that are downloaded and ready.", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "unload_models", + "description": "Unload all 3D generation models from VRAM to free GPU memory.", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "get_mesh_info", + "description": "Get information about the current mesh loaded in the 3D viewer (triangle count, path, etc.).", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "decimate_mesh", + "description": "Reduce the polygon count of the current mesh using quadric edge collapse.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file (e.g. 'Default/mesh.glb'). Use get_mesh_info to obtain it.", + }, + "target_faces": { + "type": "integer", + "description": "Target number of faces after decimation.", + }, + }, + "required": ["path", "target_faces"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "smooth_mesh", + "description": "Apply Laplacian smoothing to the current mesh.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + "iterations": { + "type": "integer", + "description": "Number of smoothing iterations (1–20). More = smoother but loses detail.", + }, + }, + "required": ["path", "iterations"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_generation_status", + "description": "Poll the status of an ongoing 3D generation job.", + "parameters": { + "type": "object", + "properties": { + "job_id": {"type": "string", "description": "Job ID returned by a previous generation call."}, + }, + "required": ["job_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "list_workflows", + "description": "List all workflows available in Modly.", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "run_workflow", + "description": "Execute a Modly workflow by its ID. The workflow runs in the background; progress is shown in the app.", + "parameters": { + "type": "object", + "properties": { + "workflow_id": {"type": "string", "description": "The workflow ID to execute. Use list_workflows to get available IDs."}, + }, + "required": ["workflow_id"], + }, + }, + }, +] + + +async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, dict | None]: + """Execute a tool and return (result_text, action_payload). + action_payload carries data the frontend needs to react (e.g. new mesh URL). + """ + async with httpx.AsyncClient(timeout=60.0) as client: + try: + if name == "list_models": + r = await client.get(f"{MODLY_API}/model/all") + r.raise_for_status() + models = [m for m in r.json() if m.get("downloaded")] + if not models: + return "No models downloaded yet.", None + lines = "\n".join(f"- {m['id']}: {m.get('name', m['id'])}" for m in models) + return f"Available models:\n{lines}", None + + elif name == "unload_models": + await client.post(f"{MODLY_API}/model/unload-all") + return "All 3D generation models have been unloaded from VRAM.", None + + elif name == "get_mesh_info": + mesh_path = context.get("currentMeshPath") + mesh_triangles = context.get("meshTriangles") + if not mesh_path: + return "No mesh currently loaded in the viewer.", None + info = f"Current mesh: {mesh_path}" + if mesh_triangles: + info += f" ({mesh_triangles:,} triangles)" + return info, None + + elif name == "decimate_mesh": + r = await client.post( + f"{MODLY_API}/optimize/mesh", + json={"path": arguments["path"], "target_faces": arguments["target_faces"]}, + ) + r.raise_for_status() + data = r.json() + payload = {"type": "mesh_update", "url": data["url"], "face_count": data.get("face_count")} + return f"Decimated to {data.get('face_count', '?')} faces.", payload + + elif name == "smooth_mesh": + r = await client.post( + f"{MODLY_API}/optimize/smooth", + json={"path": arguments["path"], "iterations": arguments["iterations"]}, + ) + r.raise_for_status() + data = r.json() + payload = {"type": "mesh_update", "url": data["url"]} + return f"Smoothed mesh ({arguments['iterations']} iterations).", payload + + elif name == "get_generation_status": + r = await client.get(f"{MODLY_API}/generate/status/{arguments['job_id']}") + r.raise_for_status() + s = r.json() + text = f"Status: {s['status']}, Progress: {s.get('progress', 0)}%" + if s.get("step"): + text += f", Step: {s['step']}" + if s.get("output_url"): + text += f", Output: {s['output_url']}" + return text, None + + elif name == "list_workflows": + workflows = context.get("workflows", []) + if not workflows: + return "No workflows found. Create one in the Workflows tab.", None + lines = "\n".join(f"- {w['id']}: {w['name']}" for w in workflows) + return f"Available workflows:\n{lines}", None + + elif name == "run_workflow": + workflow_id = arguments["workflow_id"] + workflows = context.get("workflows", []) + match = next((w for w in workflows if w["id"] == workflow_id), None) + if not match: + return f"Workflow '{workflow_id}' not found. Use list_workflows to see available workflows.", None + payload = {"type": "run_workflow", "workflow_id": workflow_id, "workflow_name": match["name"]} + return f"Executing workflow '{match['name']}'…", payload + + else: + return f"Unknown tool: {name}", None + + except httpx.HTTPStatusError as e: + return f"API error {e.response.status_code}: {e.response.text[:200]}", None + except Exception as e: + return f"Error: {e}", None + + +class ChatMessage(BaseModel): + role: str + content: str + images: list[str] = [] + + +class AgentChatRequest(BaseModel): + messages: list[ChatMessage] + ollama_url: str = "http://localhost:11434" + model: str = "qwen2.5:3b" + context: dict = {} + thinking: str = "auto" # "auto" | "on" | "off" + + +class ActionDone(BaseModel): + tool: str + result: str + payload: dict | None = None + + +class AgentChatResponse(BaseModel): + message: str + actions: list[ActionDone] = [] + thinking: str | None = None + + +def _extract_thinking(msg: dict) -> tuple[str, str | None]: + """Return (clean_content, thinking_text). Handles both Ollama native field and tags.""" + content = msg.get("content", "") + thinking = msg.get("thinking") or None + if not thinking: + match = re.search(r"(.*?)", content, re.DOTALL) + if match: + thinking = match.group(1).strip() + content = (content[: match.start()] + content[match.end() :]).strip() + return content, thinking + + +@router.get("/models") +async def list_ollama_models(ollama_url: str = "http://localhost:11434"): + async with httpx.AsyncClient(timeout=5.0) as client: + try: + r = await client.get(f"{ollama_url}/api/tags") + r.raise_for_status() + models = [m["name"] for m in r.json().get("models", [])] + return {"models": models} + except Exception: + return {"models": []} + + +@router.post("/chat", response_model=AgentChatResponse) +async def agent_chat(request: AgentChatRequest): + messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] + + # Inject scene context so the LLM knows current state + if request.context: + ctx_lines = [] + if request.context.get("currentMeshPath"): + ctx_lines.append(f"Current mesh path: {request.context['currentMeshPath']}") + if request.context.get("meshTriangles"): + ctx_lines.append(f"Current mesh triangles: {request.context['meshTriangles']:,}") + if ctx_lines: + messages.append({ + "role": "system", + "content": "Scene context:\n" + "\n".join(ctx_lines), + }) + + for m in request.messages: + entry: dict = {"role": m.role, "content": m.content} + if m.images: + entry["images"] = m.images + messages.append(entry) + + actions_done: list[ActionDone] = [] + all_thinking: list[str] = [] + + # Build Ollama think param + ollama_extra: dict = {} + if request.thinking == "on": + ollama_extra["think"] = True + elif request.thinking == "off": + ollama_extra["think"] = False + + async with httpx.AsyncClient(timeout=120.0) as client: + for _ in range(10): # max tool-call rounds + r = await client.post( + f"{request.ollama_url}/api/chat", + json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **ollama_extra}, + ) + + if r.status_code != 200: + return AgentChatResponse( + message=f"Ollama error ({r.status_code}). Is Ollama running at {request.ollama_url}?", + ) + + msg = r.json()["message"] + messages.append(msg) + + clean_content, thinking_text = _extract_thinking(msg) + if thinking_text: + all_thinking.append(thinking_text) + + tool_calls = msg.get("tool_calls") or [] + if not tool_calls: + combined_thinking = "\n\n---\n\n".join(all_thinking) if all_thinking else None + return AgentChatResponse( + message=clean_content, + actions=actions_done, + thinking=combined_thinking, + ) + + for tc in tool_calls: + fn = tc["function"] + result_text, payload = await execute_tool(fn["name"], fn.get("arguments") or {}, request.context) + actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload)) + messages.append({"role": "tool", "content": result_text}) + + has_workflow = any(a.tool == "run_workflow" for a in actions_done) + if has_workflow: + # Unload LLM from VRAM immediately so the workflow has full GPU memory + try: + await client.post( + f"{request.ollama_url}/api/generate", + json={"model": request.model, "keep_alive": 0}, + timeout=5.0, + ) + except Exception: + pass + + combined_thinking = "\n\n---\n\n".join(all_thinking) if all_thinking else None + return AgentChatResponse(message="Reached maximum tool iterations.", actions=actions_done, thinking=combined_thinking) diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/areas/generate/components/ChatPanel.tsx new file mode 100644 index 0000000..5f6d76a --- /dev/null +++ b/src/areas/generate/components/ChatPanel.tsx @@ -0,0 +1,678 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { useAppStore } from '@shared/stores/appStore' +import { useWorkflowsStore } from '@shared/stores/workflowsStore' +import { useExtensionsStore } from '@shared/stores/extensionsStore' +import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' +import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' + +// ─── Types ──────────────────────────────────────────────────────────────────── + +type ThinkingMode = 'auto' | 'on' | 'off' + +interface Message { + id: string + role: 'user' | 'assistant' + content: string + thinking?: string + imageDataUrls?: string[] + actions?: ActionDone[] +} + +interface ActionDone { + tool: string + result: string + payload?: { + type: string + url?: string + face_count?: number + workflow_id?: string + workflow_name?: string + } | null +} + +// ─── Constants ──────────────────────────────────────────────────────────────── + +const OLLAMA_URL = 'http://localhost:11434' +const OLLAMA_MODEL = 'gemma4:e4b' +const COLLAPSE_AFTER = 4 + +// ─── Prose renderer — basic markdown-like ──────────────────────────────────── + +function ProseMessage({ content }: { content: string }): JSX.Element { + const blocks = content.split(/\n\n+/) + return ( +
+ {blocks.map((block, i) => { + const lines = block.split('\n') + const isList = lines.every((l) => /^[-•*]\s/.test(l.trim()) || l.trim() === '') + if (isList) { + return ( +
    + {lines.filter(Boolean).map((l, j) => ( +
  • + • + {l.replace(/^[-•*]\s/, '')} +
    • + ))} +
+ ) + } + return ( +

+ {block} +

+ ) + })} +
+ ) +} + +// ─── Actions card ───────────────────────────────────────────────────────────── + +const TOOL_LABELS: Record = { + decimate_mesh: 'Decimated mesh', + smooth_mesh: 'Smoothed mesh', + list_models: 'Listed models', + unload_models: 'Unloaded models', + get_mesh_info: 'Inspected mesh', + get_generation_status:'Checked generation', + list_workflows: 'Listed workflows', + run_workflow: 'Ran workflow', +} + +function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () => void }): JSX.Element { + const [expanded, setExpanded] = useState(false) + const meshActions = actions.filter((a) => a.payload?.type === 'mesh_update') + const canUndo = meshActions.length > 0 && !!onUndo + + return ( +
+ {/* Header */} +
+ + {actions.length} action{actions.length > 1 ? 's' : ''} performed + +
+ {canUndo && ( + + )} + +
+
+ + {/* Rows */} + {expanded && ( +
+ {actions.map((a, i) => ( +
+ {TOOL_LABELS[a.tool] ?? a.tool.replace(/_/g, ' ')} + {a.payload?.type === 'mesh_update' && a.payload.face_count && ( + {a.payload.face_count.toLocaleString()} faces + )} + {a.payload?.type === 'run_workflow' && ( + {a.payload.workflow_name} + )} +
+ ))} +
+ )} +
+ ) +} + +// ─── Feedback row ───────────────────────────────────────────────────────────── + +function FeedbackRow({ content }: { content: string }): JSX.Element { + const [copied, setCopied] = useState(false) + + function handleCopy() { + navigator.clipboard.writeText(content) + setCopied(true) + setTimeout(() => setCopied(false), 1500) + } + + return ( +
+ + + +
+ ) +} + +// ─── Thinking block ─────────────────────────────────────────────────────────── + +function ThinkingBlock({ content }: { content: string }): JSX.Element { + const [open, setOpen] = useState(false) + return ( +
+ + {open && ( +
+

{content}

+
+ )} +
+ ) +} + +// ─── Workflow progress card ──────────────────────────────────────────────────── + +function WorkflowProgressCard({ name }: { name: string }): JSX.Element { + const runState = useWorkflowRunStore((s) => s.runState) + const pct = runState.blockProgress + return ( +
+
+
+ + {name} +
+ {pct}% +
+
+
+
+ {runState.blockStep && ( +

{runState.blockStep}

+ )} +
+ ) +} + +// ─── Main component ────────────────────────────────────────────────────────── + +export default function ChatPanel(): JSX.Element { + const [messages, setMessages] = useState([]) + const [input, setInput] = useState('') + const [isLoading, setIsLoading] = useState(false) + const [error, setError] = useState(null) + const [showAll, setShowAll] = useState(false) + const [model, setModel] = useState(OLLAMA_MODEL) + const [showModelPicker, setShowModelPicker] = useState(false) + const [ollamaModels, setOllamaModels] = useState([]) + const [pendingWorkflow, setPendingWorkflow] = useState<{ id: string; name: string } | null>(null) + const [attachments, setAttachments] = useState([]) // data URLs + const [isDragging, setIsDragging] = useState(false) + const [thinkingMode, setThinkingMode] = useState('auto') + const endRef = useRef(null) + const textareaRef = useRef(null) + const modelPickerRef = useRef(null) + const fileInputRef = useRef(null) + const messagesRef = useRef([]) + messagesRef.current = messages + + const apiUrl = useAppStore((s) => s.apiUrl) + const currentJob = useAppStore((s) => s.currentJob) + const meshStats = useAppStore((s) => s.meshStats) + const updateCurrentJob = useAppStore((s) => s.updateCurrentJob) + const pushMeshUrl = useAppStore((s) => s.pushMeshUrl) + const undoMesh = useAppStore((s) => s.undoMesh) + + const workflows = useWorkflowsStore((s) => s.workflows) + const { modelExtensions, processExtensions } = useExtensionsStore() + const runWorkflow = useWorkflowRunStore((s) => s.run) + const runState = useWorkflowRunStore((s) => s.runState) + const allExtensions = useMemo( + () => buildAllWorkflowExtensions(modelExtensions, processExtensions), + [modelExtensions, processExtensions], + ) + + // Close model picker on outside click + useEffect(() => { + if (!showModelPicker) return + const handler = (e: MouseEvent) => { + if (modelPickerRef.current && !modelPickerRef.current.contains(e.target as Node)) + setShowModelPicker(false) + } + document.addEventListener('mousedown', handler) + return () => document.removeEventListener('mousedown', handler) + }, [showModelPicker]) + + // Watch workflow completion → send follow-up to agent + useEffect(() => { + if (!pendingWorkflow) return + if (runState.status !== 'done' && runState.status !== 'error') return + + const wf = pendingWorkflow + setPendingWorkflow(null) + + if (runState.status === 'error') { + setMessages((prev) => [...prev, { + id: `sys-${Date.now()}`, + role: 'assistant', + content: `The workflow '${wf.name}' failed: ${runState.error ?? 'Unknown error'}`, + }]) + return + } + + // Update viewer with the generated mesh + if (runState.outputUrl) { + updateCurrentJob({ outputUrl: runState.outputUrl, status: 'done', progress: 100 }) + pushMeshUrl(runState.outputUrl) + } + + // Send automatic follow-up to agent + const completionCtx = `Workflow '${wf.name}' just completed.${runState.outputUrl ? ` Output mesh: ${runState.outputUrl}` : ''} Ask the user what they'd like to do next.` + callAgent(messagesRef.current, { workflowCompletion: completionCtx }) + }, [runState.status, pendingWorkflow]) + + useEffect(() => { + endRef.current?.scrollIntoView({ behavior: 'smooth' }) + }, [messages, isLoading, pendingWorkflow]) + + function buildContext(): Record { + const ctx: Record = {} + if (currentJob?.outputUrl) ctx.currentMeshPath = currentJob.outputUrl.replace('/workspace/', '') + if (meshStats?.triangles) ctx.meshTriangles = meshStats.triangles + if (workflows.length > 0) ctx.workflows = workflows.map((w) => ({ id: w.id, name: w.name })) + return ctx + } + + async function callAgent(msgs: Message[], extraContext: Record = {}) { + setIsLoading(true) + setError(null) + try { + const context = { ...buildContext(), ...extraContext } + + // Inject workflow completion as a system hint if present + const apiMessages = msgs.map((m) => { + const entry: { role: string; content: string; images?: string[] } = { + role: m.role, + content: m.content, + } + if (m.imageDataUrls?.length) { + entry.images = m.imageDataUrls.map((url) => url.split(',')[1]) + } + return entry + }) + if (extraContext.workflowCompletion) { + apiMessages.push({ role: 'user', content: `[System] ${extraContext.workflowCompletion}` }) + delete context.workflowCompletion + } + + const res = await fetch(`${apiUrl}/agent/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ messages: apiMessages, ollama_url: OLLAMA_URL, model, context, thinking: thinkingMode }), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + + const data: { message: string; actions: ActionDone[]; thinking?: string } = await res.json() + + setMessages((prev) => [...prev, { + id: `a-${Date.now()}`, + role: 'assistant', + content: data.message, + thinking: data.thinking ?? undefined, + actions: data.actions?.length ? data.actions : undefined, + }]) + + // Extract base64 from the most recent user message that had an image attached + const latestImageDataUrl = [...msgs].reverse() + .find((m) => m.role === 'user' && m.imageDataUrls?.length) + ?.imageDataUrls?.[0] + const overrideImageData = latestImageDataUrl ? latestImageDataUrl.split(',')[1] : undefined + + for (const action of data.actions ?? []) { + if (action.payload?.type === 'mesh_update' && action.payload.url) { + updateCurrentJob({ outputUrl: action.payload.url }) + pushMeshUrl(action.payload.url) + } + if (action.payload?.type === 'run_workflow' && action.payload.workflow_id) { + const wf = workflows.find((w) => w.id === action.payload!.workflow_id) + if (wf) { runWorkflow(wf, allExtensions, overrideImageData); setPendingWorkflow({ id: wf.id, name: wf.name }) } + } + } + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e) + setError(msg.includes('fetch') ? 'Cannot reach Modly API. Is the backend running?' : msg) + } finally { + setIsLoading(false) + } + } + + async function fetchOllamaModels() { + try { + const res = await fetch(`${apiUrl}/agent/models?ollama_url=${encodeURIComponent(OLLAMA_URL)}`) + const data = await res.json() + setOllamaModels(data.models ?? []) + } catch { + setOllamaModels([]) + } + } + + function handleFiles(files: File[]) { + files.forEach((file) => { + if (!file.type.startsWith('image/')) return + const reader = new FileReader() + reader.onload = (e) => { + const dataUrl = e.target?.result as string + setAttachments((prev) => [...prev, dataUrl]) + } + reader.readAsDataURL(file) + }) + } + + function handleDragOver(e: React.DragEvent) { + e.preventDefault() + setIsDragging(true) + } + + function handleDragLeave(e: React.DragEvent) { + if (!(e.currentTarget as HTMLElement).contains(e.relatedTarget as Node)) setIsDragging(false) + } + + function handleDrop(e: React.DragEvent) { + e.preventDefault() + setIsDragging(false) + handleFiles(Array.from(e.dataTransfer.files)) + } + + function adjustHeight() { + const el = textareaRef.current + if (!el) return + el.style.height = 'auto' + el.style.height = `${Math.min(el.scrollHeight, 160)}px` + } + + async function handleSend() { + const text = input.trim() + if (!text || isLoading || pendingWorkflow) return + + const userMsg: Message = { + id: `u-${Date.now()}`, + role: 'user', + content: text, + ...(attachments.length ? { imageDataUrls: [...attachments] } : {}), + } + const nextMessages = [...messages, userMsg] + setMessages(nextMessages) + setInput('') + setAttachments([]) + if (textareaRef.current) textareaRef.current.style.height = 'auto' + await callAgent(nextMessages) + } + + function handleKeyDown(e: React.KeyboardEvent) { + if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSend() } + } + + // Collapsed history + const collapsed = !showAll && messages.length > COLLAPSE_AFTER + const hidden = collapsed ? messages.length - COLLAPSE_AFTER : 0 + const visible = collapsed ? messages.slice(-COLLAPSE_AFTER) : messages + + return ( +
+ {/* Drag overlay */} + {isDragging && ( +
+

Drop image here

+
+ )} + + {/* Messages */} +
+ + {/* Empty state */} + {messages.length === 0 && ( +
+
+ + + + +
+

+ Ask me to generate, optimize,
or run a workflow. +

+
+ )} + + {/* Previous messages pill */} + {collapsed && ( + + )} + + {/* Message list */} +
+ {visible.map((msg) => ( +
+ {msg.role === 'user' ? ( + /* User message */ +
+ {msg.imageDataUrls && msg.imageDataUrls.length > 0 && ( +
+ {msg.imageDataUrls.map((url, i) => ( + + ))} +
+ )} +
+ {msg.content} +
+
+ ) : ( + /* Assistant message */ +
+ {msg.thinking && } + + {msg.actions && msg.actions.length > 0 && ( + + )} + +
+ )} +
+ ))} + + {/* Workflow progress card — visible while agent waits for workflow */} + {pendingWorkflow && } + + {/* Loading indicator */} + {isLoading && ( +
+ {[0, 1, 2].map((i) => ( + + ))} +
+ )} + + {/* Error */} + {error && ( +
+

{error}

+
+ )} + +
+
+
+ + {/* Input bar */} +
+ { if (e.target.files) handleFiles(Array.from(e.target.files)); e.target.value = '' }} + /> +
+ {/* Attachment previews */} + {attachments.length > 0 && ( +
+ {attachments.map((url, i) => ( +
+ + +
+ ))} +
+ )} +