From 73b26ffb8812dd740a6ccfb406de8fa2bf190f7a Mon Sep 17 00:00:00 2001 From: Aditya Sanjeev Date: Thu, 6 Aug 2026 16:31:58 -0700 Subject: [PATCH] Fix 3D meshes never loading for uploaded scans The 3D pane hung on "Loading 3D segmentation..." for every uploaded scan. MeshViewer fetches /api/cases//mesh-manifest, which only serves pre-baked dataset meshes via get_panTS_id + MESH_PATH; for a session id (a UUID) it 404s, so the manifest stayed null and the pane never rendered. Inference never generated session meshes. Add on-demand session meshes, reusing the existing mesh machinery: - GET /api/sessions//mesh-manifest: builds the manifest from the session's combined_labels (generate_mesh_manifest with route_base="sessions") - GET /api/sessions//render_only/: marching-cubes the requested organ from combined_labels (generate_organ_glb_bytes), cached next to the mask - generate_mesh_manifest gains a route_base arg so organ URLs point at the session route (default "cases" keeps dataset behaviour unchanged) - MeshViewer/fetchMeshManifest take isSession; VisualizationPage passes it for session routes - register liver/kidney/colon_lesion (33/34/35) in mesh LABELS so the four LesionSegmenter lesions also get 3D meshes (absent labels are skipped) --- .../src/components/viewer/MeshViewer.tsx | 16 ++++-- PanTS-Demo/src/routes/VisualizationPage.tsx | 2 +- flask-server/api/api_blueprint.py | 50 +++++++++++++++++++ flask-server/services/mesh_generation.py | 9 +++- 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/PanTS-Demo/src/components/viewer/MeshViewer.tsx b/PanTS-Demo/src/components/viewer/MeshViewer.tsx index 66081da..a53354f 100644 --- a/PanTS-Demo/src/components/viewer/MeshViewer.tsx +++ b/PanTS-Demo/src/components/viewer/MeshViewer.tsx @@ -19,15 +19,21 @@ type SegmentationMeshViewerProps = { crosshairMm: Vec3 | null customOrgans?: CheckBoxData[]; labelColorMap?: { [key: number]: Color }; + // Uploaded scans have no pre-baked meshes; fetch from the session route, which + // builds them on demand from the session's combined_labels. + isSession?: boolean; }; -export async function fetchMeshManifest(caseId: string): Promise { - const res = await fetch(`${APP_CONSTANTS.API_ORIGIN}/api/cases/${caseId}/mesh-manifest`); +export async function fetchMeshManifest(caseId: string, isSession = false): Promise { + const base = isSession + ? `${APP_CONSTANTS.API_ORIGIN}/api/sessions/${caseId}/mesh-manifest` + : `${APP_CONSTANTS.API_ORIGIN}/api/cases/${caseId}/mesh-manifest`; + const res = await fetch(base); if (!res.ok) throw new Error(`Failed to fetch mesh manifest: ${res.status}`); return res.json(); } -export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, crosshairMm, customOrgans = [], labelColorMap = {}}: SegmentationMeshViewerProps) { +export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, crosshairMm, customOrgans = [], labelColorMap = {}, isSession = false}: SegmentationMeshViewerProps) { const [manifest, setManifest] = useState(null); const [loaded, setLoaded] = useState>({}); // Bumped on every mask edit so editedSegments below is recomputed — the 3D pane @@ -50,7 +56,7 @@ export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, c useEffect(() => { let alive = true; - fetchMeshManifest(caseId) + fetchMeshManifest(caseId, isSession) .then((data) => { if (!alive) return; setManifest(data); @@ -60,7 +66,7 @@ export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, c }) .catch((err) => console.error(err)); return () => { alive = false; }; - }, [caseId]); + }, [caseId, isSession]); const organs = useMemo(() => manifest?.organs ?? [], [manifest]); diff --git a/PanTS-Demo/src/routes/VisualizationPage.tsx b/PanTS-Demo/src/routes/VisualizationPage.tsx index f0b9556..c667489 100644 --- a/PanTS-Demo/src/routes/VisualizationPage.tsx +++ b/PanTS-Demo/src/routes/VisualizationPage.tsx @@ -3445,7 +3445,7 @@ const aiAvailableOrgans = useMemo(() => { (switch to Volume rendering above) ) : ( - + )} {!loading && ( diff --git a/flask-server/api/api_blueprint.py b/flask-server/api/api_blueprint.py index 99e1458..f572548 100644 --- a/flask-server/api/api_blueprint.py +++ b/flask-server/api/api_blueprint.py @@ -1527,6 +1527,56 @@ def get_session_segmentation(session_id): return response +def _session_seg_path(session_id): + """Return the combined_labels.nii.gz path for a finished session, or None.""" + job = _get_inference_job(session_id) or {} + output_mask_dir = job.get("output_mask_dir") + if not output_mask_dir: + return None + seg_path = os.path.join(output_mask_dir, "combined_labels.nii.gz") + return seg_path if os.path.exists(seg_path) else None + + +# 3D organ meshes for an UPLOADED scan. Unlike dataset cases (whose meshes are +# pre-baked into MESH_PATH by preprocess_meshes.py), a session has no pre-baked +# meshes, so we build them on demand from the session's combined_labels and cache +# them next to it. Without these routes the 3D pane hangs on "Loading 3D +# segmentation..." because /cases//mesh-manifest 404s for a session id. +@api_blueprint.route('/sessions//mesh-manifest', methods=['GET']) +def get_session_mesh_manifest(session_id): + if not _is_safe_id(session_id): + return jsonify({"error": "Invalid id"}), 400 + seg_path = _session_seg_path(session_id) + if not seg_path: + return jsonify({"error": "Segmentation not ready for session"}), 404 + manifest = generate_mesh_manifest(session_id, seg_path, route_base="sessions") + return jsonify(manifest) + + +@api_blueprint.route('/sessions//render_only/', methods=['GET']) +def get_session_mesh_file(session_id, filename): + if not _is_safe_id(session_id): + return jsonify({"error": "Invalid id"}), 400 + filename = secure_filename(filename) + seg_path = _session_seg_path(session_id) + if not seg_path: + return jsonify({"error": "Segmentation not ready for session"}), 404 + # Cache the generated GLB next to the mask so repeat views / organ toggles + # don't re-run marching cubes. + cache_dir = os.path.join(os.path.dirname(seg_path), "render_only") + os.makedirs(cache_dir, exist_ok=True) + glb_path = os.path.join(cache_dir, filename) + if not os.path.exists(glb_path): + organ_key = os.path.splitext(filename)[0] + try: + glb_bytes = generate_organ_glb_bytes(organ_key, seg_path) + except ValueError as e: + return jsonify({"error": str(e)}), 404 + with open(glb_path, "wb") as f: + f.write(glb_bytes) + return send_file(glb_path, mimetype="model/gltf-binary", conditional=False) + + @api_blueprint.route('/session-reconstruction/', methods=['GET']) def get_session_reconstruction(session_id): """Serves the OpenVAE reconstructed CT for a session.""" diff --git a/flask-server/services/mesh_generation.py b/flask-server/services/mesh_generation.py index eedc438..84fb717 100644 --- a/flask-server/services/mesh_generation.py +++ b/flask-server/services/mesh_generation.py @@ -47,6 +47,9 @@ 30: {"key": "renal_vein_left", "name": "Left Renal Vein"}, 31: {"key": "renal_vein_right", "name": "Right Renal Vein"}, 32: {"key": "cbd_stent", "name": "Common Bile Duct Stent"}, + 33: {"key": "liver_lesion", "name": "Liver Lesion"}, + 34: {"key": "kidney_lesion", "name": "Kidney Lesion"}, + 35: {"key": "colon_lesion", "name": "Colon Lesion"}, } @@ -176,7 +179,11 @@ def generate_organ_glb_bytes( def generate_mesh_manifest( case_id: str, label_nifti_path: str, + route_base: str = "cases", ) -> dict: + # route_base selects which serving route the per-organ GLB URLs point at: + # "cases" for pre-baked dataset meshes, "sessions" for on-demand meshes + # generated from an uploaded scan's combined_labels. img, data = load_clean_label_data(label_nifti_path) present_labels = set(np.unique(data).astype(int).tolist()) @@ -200,7 +207,7 @@ def generate_mesh_manifest( "id": key, "key": meta["key"], "name": meta["name"], - "url": f"{os.getenv('API_ORIGIN', 'http://localhost:5001')}/api/cases/{case_id}/render_only/{filename}", + "url": f"{os.getenv('API_ORIGIN', 'http://localhost:5001')}/api/{route_base}/{case_id}/render_only/{filename}", } )