fix(codegen): clean up orphaned scaffold on cancelled/failed generation

src/youtube_extension/backend/code_generator.py

@@ -10,17 +10,117 @@
 template-based generation that still produces unique, video-specific output.
 """
 
+import asyncio
+import contextlib
 import hashlib
 import json
 import logging
 import os
+import shutil
 import tempfile
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Optional, TypeVar
 from urllib.parse import parse_qs, urlparse
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+
+
+async def _run_offloop(func: Callable[..., _T], /, *args: Any, **kwargs: Any) -> _T:
+    """Run a blocking callable off the loop without abandoning it on cancellation.
+
+    ``asyncio.to_thread`` cannot stop a worker thread that has already started,
+    so a cancellation delivered while the worker runs would return control to the
+    caller while the thread keeps mutating the scaffold on disk. Shield the worker
+    and wait for it to settle before re-raising ``CancelledError`` -- mirroring
+    ``_run_sync_rpc`` in ``services/cloud/cloud_tasks_queue.py`` -- so any
+    higher-level cleanup runs against a quiescent filesystem, never a live writer.
+    """
+    task = asyncio.ensure_future(asyncio.to_thread(func, *args, **kwargs))
+    cancelled = False
+    while not task.done():
+        try:
+            await asyncio.shield(task)
+        except asyncio.CancelledError:
+            cancelled = True
+        except Exception:
+            if not cancelled:
+                raise
+    if cancelled:
+        # Surface the worker's own failure if it raised; otherwise honour the
+        # cancellation now that the thread has finished touching the disk.
+        with contextlib.suppress(Exception):
+            task.result()
+        raise asyncio.CancelledError
+    return task.result()
+
+
+def _safe_rmtree(path: Path) -> None:
+    """Best-effort removal of a scaffold tree, tolerating absence and races."""
+    shutil.rmtree(path, ignore_errors=True)
+
+
+async def _make_scaffold_dir(prefix: str) -> Path:
+    """Create a temp scaffold directory off the loop, cleaning it up if the
+    caller is cancelled while ``mkdtemp`` is still running.
+
+    ``asyncio.to_thread`` cannot interrupt ``mkdtemp`` once it has started, so a
+    cancellation delivered during the hop can still leave a directory on disk
+    whose path never reaches the caller -- the assignment that would hand it to
+    a higher-level cleanup scope never happens. This helper owns that window: it
+    waits for the worker to settle, removes any directory the worker created,
+    and only then propagates the ``CancelledError``. The removal is synchronous
+    because the directory is freshly created and empty, and cleanup on the
+    cancellation path must complete rather than risk a second cancellation.
+    """
+    task = asyncio.ensure_future(asyncio.to_thread(tempfile.mkdtemp, prefix=prefix))
+    cancelled = False
+    while not task.done():
+        try:
+            await asyncio.shield(task)
+        except asyncio.CancelledError:
+            cancelled = True
+        except Exception:
+            if not cancelled:
+                raise
+    if cancelled:
+        created: Optional[str] = None
+        with contextlib.suppress(Exception):
+            created = task.result()
+        if created is not None:
+            _safe_rmtree(Path(created))
+        raise asyncio.CancelledError
+    return Path(task.result())
+
+
+#: One scaffolding step. ``(path, None)`` creates a directory; ``(path, text)``
+#: writes a file. Steps are applied in list order.
+WritePlan = list[tuple[Path, Optional[str]]]
+
+
+def _apply_write_plan(plan: WritePlan) -> None:
+    """Apply an ordered scaffolding plan on the calling thread.
+
+    This is the only place the generators touch the filesystem. It is written
+    as a plain synchronous function so callers can hand the whole batch to
+    ``asyncio.to_thread`` in a single hop, rather than paying a hop per file.
+
+    Steps are applied strictly in order, so a directory step always lands
+    before the file steps that depend on it and the on-disk result matches
+    what the equivalent inline sequence produced.
+
+    No exception is suppressed. A failing step raises exactly the error the
+    equivalent inline call would have raised, on the same step, leaving the
+    preceding steps applied -- identical to the previous behaviour.
+    """
+    for path, content in plan:
+        if content is None:
+            path.mkdir(exist_ok=True)
+        else:
+            with open(path, "w") as handle:
+                handle.write(content)
+
 
 def _extract_video_id(video_url: str) -> Optional[str]:
     """Extract the YouTube video ID from a URL, returning None if not found."""
@@ -149,31 +249,41 @@ async def generate_project(self, video_analysis: dict[str, Any], project_config:
             if build_plan:
                 video_analysis["build_plan"] = build_plan
 
-            # Create temporary project directory
-            temp_dir = tempfile.mkdtemp(prefix="uvai_project_")
-            project_path = Path(temp_dir)
-
-            # Generate project structure based on type
-            if project_type == "web":
-                result = await self._generate_web_project(project_path, video_analysis, technologies, features)
-            elif project_type == "api":
-                result = await self._generate_api_project(project_path, video_analysis, technologies, features)
-            elif project_type == "mobile":
-                result = await self._generate_mobile_project(project_path, video_analysis, technologies, features)
-            else:
-                result = await self._generate_web_project(project_path, video_analysis, technologies, features)
-
-            result["project_path"] = str(project_path)
-            result["project_type"] = project_type
-            result["technologies"] = technologies
-            result["features"] = features
-            # Include the structured BuildPlan artifact in the result so that
-            # callers (e.g. API endpoints and tests) can inspect it.
-            if build_plan is not None:
-                result["build_plan"] = build_plan
-
-            logger.info(f"✅ Project generated successfully at {project_path}")
-            return result
+            # Create temporary project directory. ``_make_scaffold_dir`` owns
+            # cleanup for the window where cancellation lands *during* mkdtemp:
+            # the directory can already exist on disk before its path reaches
+            # the cleanup scope below, so the helper removes it itself.
+            project_path = await _make_scaffold_dir("uvai_project_")
+
+            # Past this point the path is known, so a cancelled or failed
+            # generation removes the scaffold here. The write hops are shielded
+            # (see ``_run_offloop``), so by the time cleanup runs the filesystem
+            # is settled and rmtree does not race a live writer.
+            try:
+                # Generate project structure based on type
+                if project_type == "web":
+                    result = await self._generate_web_project(project_path, video_analysis, technologies, features)
+                elif project_type == "api":
+                    result = await self._generate_api_project(project_path, video_analysis, technologies, features)
+                elif project_type == "mobile":
+                    result = await self._generate_mobile_project(project_path, video_analysis, technologies, features)
+                else:
+                    result = await self._generate_web_project(project_path, video_analysis, technologies, features)
+
+                result["project_path"] = str(project_path)
+                result["project_type"] = project_type
+                result["technologies"] = technologies
+                result["features"] = features
+                # Include the structured BuildPlan artifact in the result so that
+                # callers (e.g. API endpoints and tests) can inspect it.
+                if build_plan is not None:
+                    result["build_plan"] = build_plan
+
+                logger.info(f"✅ Project generated successfully at {project_path}")
+                return result
+            except (Exception, asyncio.CancelledError):
+                await _run_offloop(_safe_rmtree, project_path)
+                raise
 
         except Exception as e:
             logger.error(f"❌ Project generation failed: {e}")
@@ -234,25 +344,13 @@ async def _generate_react_project(self, project_path: Path, video_analysis: dict
             package_json["dependencies"]["tailwindcss"] = "^3.3.0"
             package_json["devDependencies"] = {"autoprefixer": "^10.4.14", "postcss": "^8.4.24"}
 
-        # Write package.json
-        with open(project_path / "package.json", "w") as f:
-            json.dump(package_json, f, indent=2)
-
-        # Create src directory
         src_dir = project_path / "src"
-        src_dir.mkdir(exist_ok=True)
-
-        # Create public directory
         public_dir = project_path / "public"
-        public_dir.mkdir(exist_ok=True)
 
-        # Generate index.html
-        index_html = self._generate_index_html(title)
-        with open(public_dir / "index.html", "w") as f:
-            f.write(index_html)
-
-        # Generate main App component
+        # Build every artifact in memory first. This is pure string work and
+        # stays on the event loop; only the disk I/O below is offloaded.
         technologies = extracted_info.get("technologies", [])
+        index_html = self._generate_index_html(title)
         app_component = self._generate_react_app_component(
             title,
             technologies,
@@ -261,23 +359,23 @@ async def _generate_react_project(self, project_path: Path, video_analysis: dict
             summary,
             key_concepts
         )
-        with open(src_dir / "App.js", "w") as f:
-            f.write(app_component)
-
-        # Generate index.js
         index_js = self._generate_react_index_js()
-        with open(src_dir / "index.js", "w") as f:
-            f.write(index_js)
-
-        # Generate CSS
         app_css = self._generate_app_css(features)
-        with open(src_dir / "App.css", "w") as f:
-            f.write(app_css)
-
-        # Generate README
         readme = self._generate_readme(title, "React", video_analysis)
-        with open(project_path / "README.md", "w") as f:
-            f.write(readme)
+
+        # ``json.dumps`` produces exactly the bytes ``json.dump`` would have
+        # written for the same arguments; neither appends a trailing newline.
+        plan: WritePlan = [
+            (project_path / "package.json", json.dumps(package_json, indent=2)),
+            (src_dir, None),
+            (public_dir, None),
+            (public_dir / "index.html", index_html),
+            (src_dir / "App.js", app_component),
+            (src_dir / "index.js", index_js),
+            (src_dir / "App.css", app_css),
+            (project_path / "README.md", readme),
+        ]
+        await _run_offloop(_apply_write_plan, plan)
 
         return {
             "framework": "react",
@@ -313,25 +411,25 @@ async def _generate_vanilla_js_project(self, project_path: Path, video_analysis:
             summary,
             key_concepts
         )
-        with open(project_path / "index.html", "w") as f:
-            f.write(index_html)
 
         # Generate main.js — NOW uses video-specific content
         main_js = self._generate_vanilla_main_js(
             title, tutorial_steps, features, key_concepts, fingerprint
         )
-        with open(project_path / "main.js", "w") as f:
-            f.write(main_js)
 
         # Generate styles.css — NOW uses video-derived accent color
         styles_css = self._generate_vanilla_styles_css(title, features, fingerprint)
-        with open(project_path / "styles.css", "w") as f:
-            f.write(styles_css)
 
         # Generate README
         readme = self._generate_readme(title, "Vanilla JavaScript", video_analysis)
-        with open(project_path / "README.md", "w") as f:
-            f.write(readme)
+
+        plan: WritePlan = [
+            (project_path / "index.html", index_html),
+            (project_path / "main.js", main_js),
+            (project_path / "styles.css", styles_css),
+            (project_path / "README.md", readme),
+        ]
+        await _run_offloop(_apply_write_plan, plan)
 
         return {
             "framework": "vanilla",
@@ -369,18 +467,19 @@ async def _generate_python_api(self, project_path: Path, video_analysis: dict, f
             requirements.append("sqlalchemy==2.0.23")
         if "authentication" in features:
             requirements.append("python-jose[cryptography]==3.3.0")
-        with open(project_path / "requirements.txt", "w") as f:
-            f.write("\n".join(requirements))
 
         # Generate main.py
         main_py = self._generate_fastapi_main(title, features)
-        with open(project_path / "main.py", "w") as f:
-            f.write(main_py)
 
         # Generate README
         readme = self._generate_readme(title, "Python FastAPI", video_analysis)
-        with open(project_path / "README.md", "w") as f:
-            f.write(readme)
+
+        plan: WritePlan = [
+            (project_path / "requirements.txt", "\n".join(requirements)),
+            (project_path / "main.py", main_py),
+            (project_path / "README.md", readme),
+        ]
+        await _run_offloop(_apply_write_plan, plan)
 
         return {
             "framework": "fastapi",