diff --git a/api/hastefuncapi/function_app.py b/api/hastefuncapi/function_app.py index 707cd6c3..466f741f 100644 --- a/api/hastefuncapi/function_app.py +++ b/api/hastefuncapi/function_app.py @@ -8,6 +8,7 @@ import os import re import tempfile +import time import traceback import azure.functions as func # type: ignore @@ -72,6 +73,7 @@ PublishingSourceNotFoundError, PublishingSourceResolver, ) +from hastegeo.core.utils import perf from hastegeo.core.utils.blob import ( download_blob_to_tempfile, parse_byte_range, @@ -735,6 +737,12 @@ async def GetProjectDetails(req: func.HttpRequest) -> func.HttpResponse: f"GetProjectDetails HTTP trigger function processed a request for project id: {project_id} with includeModels: {include_models}" ) + # Phase 0 baseline instrumentation (spec/features/perf-layer-loading). + # Opt-in via HASTE_PERF=true; zero overhead when disabled. + _perf_on = os.environ.get("HASTE_PERF", "false").lower() == "true" + _perf = perf.begin(_perf_on) + _perf_wall = time.perf_counter() + project = await asyncio.to_thread( MetadataProcessor( data_type=config.get_metadata_types().PROJECT.value, @@ -839,7 +847,21 @@ async def GetProjectDetails(req: func.HttpRequest) -> func.HttpResponse: project["imageLayer"].sort( key=lambda x: x["creationDate"], reverse=True ) - return func.HttpResponse(json.dumps(project), status_code=200) + _payload = json.dumps(project) + _perf_headers = perf.headers(_perf, _perf_wall) + perf.log_summary( + logger, + "GetProjectDetails", + _perf, + _perf_wall, + project_id=project_id, + include_models=include_models, + layers=len(image_layers), + payload_bytes=len(_payload), + ) + return func.HttpResponse( + _payload, status_code=200, headers=_perf_headers or None + ) except FileNotFoundError as e: logger.error(f"Project not found: {e}\n{traceback.format_exc()}") @@ -1470,9 +1492,9 @@ async def GetModelArtifact(req: func.HttpRequest) -> func.HttpResponse: # interactive labeler's other artifacts are fetched by range and parsed # in-browser, so they must NOT be forced as downloads). if kind == "gpkg": - headers[ - "Content-Disposition" - ] = f'attachment; filename="building_predictions_{model_id}.gpkg"' + headers["Content-Disposition"] = "; ".join( + ["attachment", f'filename="building_predictions_{model_id}.gpkg"'] + ) if result.etag: headers["ETag"] = ( result.etag if result.etag.startswith('"') else f'"{result.etag}"' diff --git a/docker/docker-compose.perf.yml b/docker/docker-compose.perf.yml new file mode 100644 index 00000000..1706a16a --- /dev/null +++ b/docker/docker-compose.perf.yml @@ -0,0 +1,20 @@ +# Perf-baseline overlay for the perf-layer-loading spec (Phase 0). +# +# Enables the opt-in HASTE_PERF instrumentation and bind-mounts the working-tree +# copies of the API handler + hastegeo library over the image's runtime paths, so +# the current branch's code runs without an image rebuild. +# +# Usage: +# docker compose -f docker/docker-compose.yml -f docker/docker-compose.perf.yml \ +# up -d hastefuncapi api-proxy +services: + hastefuncapi: + environment: + HASTE_PERF: "true" + volumes: + # hastegeo exists in two places (a site-packages install and a wwwroot + # copy); sys.path order varies by process, so overlay the working-tree + # source over BOTH to guarantee the worker imports the updated code. + - ../hastelib/src/hastegeo:/usr/local/lib/python3.11/site-packages/hastegeo:ro + - ../hastelib/src/hastegeo:/home/site/wwwroot/hastegeo:ro + - ../api/hastefuncapi/function_app.py:/home/site/wwwroot/function_app.py:ro diff --git a/hastelib/src/hastegeo/core/processors/metadata.py b/hastelib/src/hastegeo/core/processors/metadata.py index d8f42102..06ba67d4 100644 --- a/hastelib/src/hastegeo/core/processors/metadata.py +++ b/hastelib/src/hastegeo/core/processors/metadata.py @@ -5,6 +5,7 @@ from hastegeo.core.config import Config from ..data_layer.unified import UnifiedDataLayer +from ..utils.perf import timed class MetadataProcessor: @@ -96,9 +97,12 @@ def load(self, key, data_format="json"): """ Load metadata from the backend storage. """ - metadata = self.storage.load( - identifier=key, data_type=self.data_type, data_format=data_format - ) + with timed("load"): + metadata = self.storage.load( + identifier=key, + data_type=self.data_type, + data_format=data_format, + ) return metadata def load_all(self, data_format="json"): @@ -106,9 +110,10 @@ def load_all(self, data_format="json"): Load all metadata from the backend storage. """ metadata = [] - metadata_list = self.storage.load_all( - data_type=self.data_type, data_format=data_format - ) + with timed("load_all"): + metadata_list = self.storage.load_all( + data_type=self.data_type, data_format=data_format + ) for each_metadata in metadata_list: metadata.append(each_metadata) return metadata @@ -118,9 +123,10 @@ def load_all_from_partition(self, data_format="json"): Load all metadata from the backend storage. """ metadata = [] - metadata_list = self.storage.load_all_from_partition( - data_type=self.data_type, data_format=data_format - ) + with timed("load_all_from_partition"): + metadata_list = self.storage.load_all_from_partition( + data_type=self.data_type, data_format=data_format + ) for each_metadata in metadata_list: metadata.append(each_metadata) return metadata @@ -225,8 +231,9 @@ def export(self, key, data_format="json"): """ # NOTE: This is a quick method to make the export work for Azure blob storage layer. # Rework needed to handle different storage types and formats properly. - return self.storage.get_file_remote_path( - identifier=key, - data_type=self.data_type, - data_format=data_format, - ) + with timed("export"): + return self.storage.get_file_remote_path( + identifier=key, + data_type=self.data_type, + data_format=data_format, + ) diff --git a/hastelib/src/hastegeo/core/utils/perf.py b/hastelib/src/hastegeo/core/utils/perf.py new file mode 100644 index 00000000..c454ef8a --- /dev/null +++ b/hastelib/src/hastegeo/core/utils/perf.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Lightweight, opt-in performance instrumentation. + +Counts and times backend storage round-trips for a single logical request so we +can establish a baseline (Phase 0 of the perf-layer-loading spec) and later prove +the O(layers x models) -> O(1) improvement. + +Design notes: +- A ``ContextVar`` holds a shared ``PerfCounter`` *object*. ``asyncio.to_thread`` + copies the current context into the worker thread, so a read method running in + the thread sees the *same* counter instance and its (lock-guarded) mutations are + visible back in the calling coroutine. The ContextVar value (the reference) is + never reassigned inside the thread, only the object it points to is mutated. +- When tracking is not enabled, ``timed()`` is a no-op with no measurable overhead, + so instrumenting the shared data layer is safe for every caller (API + queues). +""" +import contextvars +import threading +import time +from contextlib import contextmanager + +_current: "contextvars.ContextVar[PerfCounter | None]" = ( + contextvars.ContextVar("haste_perf_counter", default=None) +) + + +class PerfCounter: + """Thread-safe accumulator of storage round-trip count and duration.""" + + def __init__(self): + self.calls = 0 + self.seconds = 0.0 + self.by_op = {} + self._lock = threading.Lock() + + def record(self, op, elapsed): + with self._lock: + self.calls += 1 + self.seconds += elapsed + entry = self.by_op.get(op) + if entry is None: + self.by_op[op] = {"calls": 1, "seconds": elapsed} + else: + entry["calls"] += 1 + entry["seconds"] += elapsed + + +def begin(enabled=True): + """Start tracking for the current context. Returns the counter (or None).""" + if not enabled: + _current.set(None) + return None + counter = PerfCounter() + _current.set(counter) + return counter + + +def end(): + """Stop tracking for the current context.""" + _current.set(None) + + +def get_counter(): + return _current.get() + + +@contextmanager +def timed(op): + """Time an ``op`` and record it on the active counter, if any. + + Zero-overhead when tracking is disabled (no active counter). + """ + counter = _current.get() + if counter is None: + yield + return + start = time.perf_counter() + try: + yield + finally: + counter.record(op, time.perf_counter() - start) + + +def headers(counter, wall_start): + """Response headers exposing round-trip count/timing for benchmarking.""" + if counter is None: + return {} + storage_ms = counter.seconds * 1000.0 + wall_ms = (time.perf_counter() - wall_start) * 1000.0 + return { + "X-Haste-Storage-Calls": str(counter.calls), + "X-Haste-Storage-Ms": f"{storage_ms:.1f}", + "X-Haste-Wall-Ms": f"{wall_ms:.1f}", + "Server-Timing": ", ".join( + [ + ";".join(["storage", "desc=storage", f"dur={storage_ms:.1f}"]), + ";".join(["wall", "desc=wall", f"dur={wall_ms:.1f}"]), + ] + ), + } + + +def log_summary(logger, name, counter, wall_start, **fields): + """Emit a single structured ``PERF`` line and stop tracking.""" + if counter is None: + return + wall_ms = (time.perf_counter() - wall_start) * 1000.0 + ops = { + op: {"calls": e["calls"], "ms": round(e["seconds"] * 1000, 1)} + for op, e in counter.by_op.items() + } + extra = " ".join(f"{k}={v}" for k, v in fields.items()) + logger.info( + "PERF %s %s storage_calls=%d storage_ms=%.1f wall_ms=%.1f ops=%s", + name, + extra, + counter.calls, + counter.seconds * 1000.0, + wall_ms, + ops, + ) + end() diff --git a/spec/features/perf-layer-loading/README.md b/spec/features/perf-layer-loading/README.md new file mode 100644 index 00000000..02177e93 --- /dev/null +++ b/spec/features/perf-layer-loading/README.md @@ -0,0 +1,70 @@ +# Feature: Image Layer & Model Run Loading Performance + +**Status:** draft +**Author:** prbatero +**Date:** 2026-08-03 +**Target Release:** TBD +**Priority:** P1 + +## Summary + +The HASTE UI takes too long to load a project's image layers and their associated +model runs, and the delay grows roughly linearly (in places quadratically) with the +number of image layers on a project. The root cause is a set of N+1 storage access +patterns in the `GetProjectDetails` API path, fully-sequential (never parallelized) +blob I/O in `hastelib`, full-container blob scans without prefix filtering, and a UI +that re-fetches the entire project — every model of every layer — every 20 seconds +while re-rendering the whole component tree. This spec catalogs the verified +bottlenecks and lays out a phased plan to make layer/run loading fast and roughly +constant-time regardless of project size. + +## Motivation + +- **Problem:** Disaster-response users open a project and wait many seconds for the + layer list to appear; the wait scales with project size, so the most active + (large) projects are the slowest — exactly when responders can least afford it. +- **Trigger:** Direct user report — "the HASTE UI takes too long to load image + layers and their model runs when there are multiple image layers on a project." +- **Cost of inaction:** Load time degrades as projects accumulate layers and models; + the 20s polling loop multiplies backend load and cost, and the app feels + progressively slower the more it is used. + +## Success Criteria + +- [ ] `GET GetProjectDetails?includeModels=True` for a 50-layer / ~5-models-per-layer + project returns in **< 1.5s p95** (from a current baseline measured in Phase 0). +- [ ] Backend storage round-trips for that request drop from **O(layers × models)** + (~600 for the 50×5 case) to **O(1) small constant** (≤ ~6 partition reads). +- [ ] UI time-to-interactive for the project page is **< 2s p95** on the same project + and no longer scales linearly with layer count. +- [ ] Background refresh no longer refetches unchanged data or re-renders the full + tree; idle CPU and network on an open project page drop measurably. + +## HASTE Components Affected + +| Component | Impact | +|---|---| +| `hastelib/src/hastegeo/core/data_layer/` | Add prefix-scoped listing, parallel/metadata-only reads, fix double-deserialize | +| `hastelib/src/hastegeo/core/processors/` | `MetadataProcessor` filtered load + optional request-scoped cache | +| `hastelib/src/hastegeo/core/artifact_storage/` | Parallelize multi-blob fetch | +| `api/hastefuncapi/` | Rewrite `GetProjectDetails` layer loop; add cache headers | +| `api/hastefuncqueues/` | Parallelize independent loads; fix N+1 label lookup; batch saves | +| `ui/src/Components/` | Smart polling, memoization, split context, lazy model expansion | + +## Related Specs + +| Spec | Relationship | +|---|---| +| [../batch-config-drift/](../batch-config-drift/) | related (queue/Batch path) | + +## Document Index + +| Document | Purpose | Status | +|---|---|---| +| [findings.md](findings.md) | Verified bottleneck inventory with file:line evidence | draft | +| [design.md](design.md) | Technical design of each fix | draft | +| [plan.md](plan.md) | Phased execution plan | Phase 0 done | +| [impact-analysis.md](impact-analysis.md) | Risk, blast radius, backward compat | draft | +| [test-plan.md](test-plan.md) | Benchmark harness & regression coverage | draft | +| [results.md](results.md) | **Phase 0 measured baseline** (603 round-trips, 20.8 s API, 40.3 s UI TTI @ 50×5) | done | +| [tools/](tools/) | Seed + benchmarks: `phase0_baseline.py`, `bench_api_http.py`, `ui_bench.cjs`; `docker/docker-compose.perf.yml` overlay | done | diff --git a/spec/features/perf-layer-loading/design.md b/spec/features/perf-layer-loading/design.md new file mode 100644 index 00000000..56d3f49c --- /dev/null +++ b/spec/features/perf-layer-loading/design.md @@ -0,0 +1,165 @@ +# Technical Design: Image Layer & Model Run Loading Performance + +## Overview + +Collapse the `GetProjectDetails` request from `O(layers × models)` sequential storage +round-trips to a small constant by (1) hoisting redundant reads out of loops, +(2) batch-loading each metadata type **once per partition** and joining in memory, +(3) parallelizing genuinely independent I/O, and (4) pushing filtering into the data +layer. Add HTTP caching so unchanged data isn't recomputed. On the UI, stop the +20-second full-project poll from thrashing the render tree by adding change-detection, +context splitting, and memoization. Backend fixes are sequenced first because they are +the dominant cost and are transparent to the UI. + +## Architecture + +``` +┌──────────────┐ GetProjectDetails ┌────────────────────┐ 1 read / type ┌──────────────┐ +│ React UI │───────────────────────▶│ hastefuncapi │──────────────────▶│ Blob / Cosmos │ +│ Project.jsx │ (ETag / 304 aware) │ GetProjectDetails │ (batched+joined) │ data layer │ +└──────────────┘ └─────────┬──────────┘ └──────────────┘ + ▲ smart poll (304 fast-path) │ uses + │ ┌─────────▼──────────┐ + └── memoized rows, split context │ MetadataProcessor │ load_filtered() / + │ (+ load_map cache) │ load_all_from_partition (prefix) + └────────────────────┘ +``` + +## API Design + +### `GET /api/GetProjectDetails` — reworked internals (contract unchanged by default) + +Same request/response shape by default, so the UI keeps working during rollout. New +**optional** query params enable the lighter paths incrementally: + +| Param | Type | Default | Effect | +|---|---|---|---| +| `includeModels` | bool | existing | unchanged | +| `includeArtifacts` | bool | `true` (compat) | when `false`, skip B2's per-model artifact/label expansion; return `modelCount` + model summaries only | +| `summary` | bool | `false` | return per-layer counts (`modelCount`, `labelProjectCount`, `validationLabelCount`) without nested `models[]` — the shape the list view actually needs | + +**New response headers:** `ETag` (hash of the serialized payload) and +`Cache-Control: private, max-age=15`. On a conditional request whose `If-None-Match` +matches, return `304` with an empty body. + +### Reworked handler logic (replaces [function_app.py:534-638](../../../api/hastefuncapi/function_app.py#L534-L638)) + +```python +# 1. Independent top-level reads in parallel +project, image_layers, models, label_projects = await asyncio.gather( + _load(PROJECT.load, project_id), + _load(IMAGELAYER.load_all_from_partition), + _load(MODEL.load_all_from_partition) if include_models else _none(), + _load(LABELS.load_all_from_partition), # ONCE, not per-layer (fixes B1) +) + +# 2. Index once, join in memory — no per-item I/O +models_by_layer = group_by(models or [], "imageLayerId") +labels_by_layer = index_by(label_projects, "imageLayerId") + +# 3. Validation + (optional) artifacts: batch in parallel, bounded concurrency +validation_by_layer = await gather_map( + {l["imageLayerId"]: VALIDATION.load for l in image_layers} +) # fixes B3 +if include_models and include_artifacts: + artifacts_by_model, labelsurl_by_model = await gather_models(models) # fixes B2 + +# 4. Assemble response purely in memory, then sort. +``` + +Key point: every metadata **type** is read at most once per partition (a handful of +`load_all_from_partition` calls), plus one bounded-parallel batch for the per-key +`VALIDATION`/artifact reads. Total round-trips become a small constant + at most two +bounded-concurrency fan-outs, independent of `L × M` sequencing. + +## Internal Interfaces (hastegeo) + +| Module | Change | Signature | Purpose | +|---|---|---|---| +| `core/processors/metadata.py` | **new** | `load_filtered(self, predicate: dict) -> list` | Filter in the data layer, not the caller (fixes H3, B1, Q2). Backed by prefix scan + property match. | +| `core/processors/metadata.py` | **new** | `load_map(self, keys: list[str], max_workers=8) -> dict` | Parallel multi-key load with bounded `ThreadPoolExecutor` (powers B2/B3 batches). | +| `core/data_layer/azure_blob_storage_data_layer.py` | **fix** | `load_all(..., name_starts_with=None)` | Always pass a prefix; add optional metadata-only listing (fixes H1). | +| `core/data_layer/azure_blob_storage_data_layer.py` | **fix** | parallel download loop | Bounded `ThreadPoolExecutor` in `load_all*` and artifact `fetch_artifact` (fixes H2). | +| `core/data_layer/azure_blob_storage_data_layer.py` | **fix** | remove double `json.loads` | Single-serialize on save; delete re-parse (fixes H4). | +| `core/blob.py` | **fix** | module-level `BlobServiceClient` singleton keyed by conn-string | Reuse client (fixes H5). | +| `core/artifact_storage/azure_blob_artifact_storage.py` | **fix** | parallel `fetch_artifact` | Same `ThreadPoolExecutor` pattern (fixes H2). | + +`load_map` sketch: + +```python +def load_map(self, keys, max_workers=8): + from concurrent.futures import ThreadPoolExecutor + out = {} + with ThreadPoolExecutor(max_workers=max_workers) as ex: + futs = {ex.submit(self._safe_load, k): k for k in keys} + for fut in futs: + out[futs[fut]] = fut.result() # None on FileNotFoundError + return out +``` + +The API layer calls `await asyncio.to_thread(processor.load_map, keys)` so one thread +hop wraps the whole bounded-parallel batch instead of one hop per key. + +## Queue design (Phase 3) + +- `GetCreateModelRunQueueTrigger`: replace `load_all_from_partition` + Python filter + with `load_filtered({"imageLayerId": ...})` (Q2). +- Inference / image triggers: wrap independent loads in `asyncio.gather` (Q3). +- Add `MetadataProcessor.save_batch(items)` and collect intermediate writes (Q4). +- `host.json`: evaluate `batchSize` > 1 for I/O-bound triggers and + `maxDequeueCount` ≥ 3 with a real poison path (Q1); raise `visibilityTimeout` for + long steps (Q5). These are config changes gated on load testing. + +## UI design (Phase 4) + +> **Measured priority (Phase 0):** TTI is API-bound (~2 s render over the API call), so +> the backend phases carry the TTI win. Within Phase 4, the highest-value items are the +> single-flight guard (U7) and the poll guard (U1) — they roughly halve effective +> latency and stop request pile-up. Memoization/virtualization (U3/U5) help poll-time +> re-render churn, not first paint; `summary` payload mode (B6) is a minor win. + +- **Single-flight + cancellation (U7):** guard `fetchProjectDetails` so only one request + is in flight at a time — track the in-flight promise and reuse it, and use an + `AbortController` to cancel a superseded request on re-fetch/unmount. Removes the + duplicate concurrent load call (measured ~2× latency amplification). +- **Smart poll (U1, U5):** send `If-None-Match` with the last `ETag`; on `304`, do + nothing. Otherwise shallow-compare (or hash-compare) before `setComponentState`. + **Do not start a poll while a request is in flight** (the measured response can exceed + the 20 s interval); pause polling when the tab is hidden (`document.visibilityState`) + and consider an interval that backs off toward the observed response time. +- **Split context (U2):** move volatile fields (`isLoading`, `dialogParams`, + `bootstrapBreakpoint`) into a separate context/provider so a loading toggle doesn't + re-render layer rows. +- **Memoization (U3):** wrap `LayerRow`, `ModelRow`, `ModelRowMobile` in `React.memo`; + `useMemo` derived arrays; `useCallback` handlers passed as props. +- **Lazy expansion / virtualization (U5):** render a layer's `models[]` only when + expanded; introduce windowing (e.g. `react-window`) if list sizes warrant. +- **Parallel fetches (U4):** `Promise.all` the independent GETs in + `CreateEditImageLayerHelper.js` and `LabelingTool.jsx`. +- **Tiles (U6):** request an overview/thumbnail first; defer full-res second map. + +## Configuration + +| Config Key | Type | Default | Where | Description | +|---|---|---|---|---| +| `HASTE_METADATA_LOAD_WORKERS` | int | 8 | App Settings / `local.settings.json` | Bounded parallelism for `load_map` fan-out | +| `HASTE_PROJECTDETAILS_CACHE_SECONDS` | int | 15 | App Settings | `Cache-Control: max-age` for `GetProjectDetails` | +| queue `batchSize` | int | 1 → TBD | `host.json` | Concurrent messages per instance (load-test gated) | + +## Observability + +- Log per-request storage round-trip **count** and total storage time in + `GetProjectDetails` (proves the O(N)→O(1) win and guards against regression). +- Emit request duration to App Insights; add a synthetic 50-layer project to the perf + test to track p95 over time. +- Track queue depth and dequeue/poison counts when changing `host.json`. + +## Open Questions + +- [ ] Does the storage backend (Blob vs Cosmos, via `UnifiedDataLayer`) support a + server-side predicate for `load_filtered`, or is prefix-scan + in-layer filter + the best available? Determines how much H3 actually saves on the Blob path. +- [ ] Is the double-serialization (H4) safe to fix in place, or are there existing + blobs already double-encoded that need a migration/back-compat read path? +- [ ] Acceptable default polling interval / should we move to server-push (SignalR) + instead of polling for run-status updates? diff --git a/spec/features/perf-layer-loading/findings.md b/spec/features/perf-layer-loading/findings.md new file mode 100644 index 00000000..980bd205 --- /dev/null +++ b/spec/features/perf-layer-loading/findings.md @@ -0,0 +1,210 @@ +# Findings: Verified Performance Bottlenecks + +Each finding was confirmed by reading the referenced code. Severity reflects impact +on the reported symptom (slow layer/run loading that scales with layer count). + +## Cost model (why it scales) + +For a project with **L** image layers and an average of **M** models per layer, the +`GetProjectDetails?includeModels=True` request currently issues approximately: + +``` +3 (project + image_layers + all-models partition reads) ++ L × 1 (LABELS full-partition scan — once PER layer, see B1) ++ L × 1 (VALIDATION load per layer) ++ L × M × 2 (MODEL_ARTIFACTS load + TRAIN_LABELS export per model) += 3 + 2L + 2LM sequential, blocking storage round-trips +``` + +For L=50, M=5 that is **~603 sequential round-trips**, each an `await asyncio.to_thread(...)` +that blocks the next. None are parallelized. The target is a **small constant** (≤ ~6). + +--- + +## Backend — `api/hastefuncapi/function_app.py` + +### B1 — CRITICAL: full LABELS partition scan re-run once per layer +[function_app.py:594-599](../../../api/hastefuncapi/function_app.py#L594-L599) + +Inside `for image_layer in image_layers:` the code calls +`MetadataProcessor(LABELS, partition_key=project_id).load_all_from_partition`. The call +takes **no per-layer argument** — it returns the identical full label set every +iteration, then filters in Python by `imageLayerId`. This is a pure redundancy bug: +the download is repeated L times when it should happen **once** before the loop. +`load_all_from_partition` downloads and deserializes every label blob in the +partition, so this is L full-partition downloads. + +**Measured (Phase 0):** per-round-trip cost is **super-linear** — 8.5 → 12.5 → 22 ms +as the partition grows (small → medium → large), because each redundant scan lists + +downloads an ever-larger label set. So B1 costs *more* the bigger the project, on top +of running L times. See [results.md](results.md). + +### B2 — CRITICAL: per-model artifact + labels loads, sequential +[function_app.py:565-591](../../../api/hastefuncapi/function_app.py#L565-L591) + +For every model of every layer, two sequential awaits: `MODEL_ARTIFACTS.load(modelId)` +and `TRAIN_LABELS.export(modelId)`. That is `L × M × 2` blocking round-trips executed +one at a time. This is the single largest contributor for model-heavy projects. + +### B3 — HIGH: per-layer VALIDATION load, sequential +[function_app.py:621-632](../../../api/hastefuncapi/function_app.py#L621-L632) + +`VALIDATION.load(image_layer_id)` per layer, sequential — `L` more blocking round-trips +that could be one batched/parallel read. + +### B4 — HIGH: no parallelism anywhere in the handler +The handler uses `await asyncio.to_thread(...)` for each I/O but never +`asyncio.gather`. Even the independent initial loads (project → image_layers → +all-models) are chained. Every storage call waits for the previous one. + +### B5 — HIGH: same N+1 repeated in `GenerateProjectStats` +[function_app.py:2602-2624](../../../api/hastefuncapi/function_app.py#L2602-L2624) + +The identical per-layer `LABELS.load_all_from_partition` pattern exists in the stats +path, so dashboards/stats regeneration degrade the same way. + +### B6 — MEDIUM: unbounded response, no pagination +[function_app.py:638](../../../api/hastefuncapi/function_app.py#L638) returns the entire +nested project (`imageLayer[].models[].artifacts`) in one payload. Serialization and +transfer grow with total model count; there is no page/limit and no lightweight +"summary" shape for the initial list render. + +### B7 — MEDIUM: no HTTP caching on list/detail endpoints +`GetModelArtifact` sets `Cache-Control`/`ETag`, but `GetProjectDetails`, +`GetProjectStats`, `GetLayerModelsDetails`, `GetLayerDetailView` set none — every poll +is a full recompute + full transfer even when nothing changed. + +--- + +## Backend — `hastelib/src/hastegeo/` + +### H1 — HIGH: `load_all` scans the whole container, no prefix +[data_layer/azure_blob_storage_data_layer.py:268-323](../../../hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py#L268-L323) + +`walk_blobs()` is called with **no `name_starts_with`**, listing the entire container, +then filtering in Python. `load_all_from_partition` does pass a prefix (good), but +`load_all` does not. All matched blobs are then fully downloaded even when only counts +/ metadata are needed. + +### H2 — HIGH: sequential per-blob download in listing + artifact fetch +[artifact_storage/azure_blob_artifact_storage.py:179-187](../../../hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py#L179-L187) +and the download loops in `load_all` / `load_all_from_partition`. Each blob is +downloaded and read to completion before the next starts — `N` files ⇒ `N ×` latency. +A bounded `ThreadPoolExecutor` would collapse this to roughly one round-trip of +latency. + +### H3 — HIGH: no filtered query — everything filtered in Python +`MetadataProcessor.load_all_from_partition` has no `imageLayerId`/predicate parameter, +forcing callers (B1, and the queue in Q2) to pull the whole partition and filter +in-process. A `load_filtered(...)` method would let callers request only what they need. + +### H4 — MEDIUM: double JSON deserialization +[data_layer/azure_blob_storage_data_layer.py:254-259](../../../hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py#L254-L259) +(and 3 sibling paths) — `json.loads` result is `json.loads`-ed again for +projects/image_layers because they were double-serialized on save. CPU cost on every +read; the real fix is single-serialize on write, then drop the re-parse. + +### H5 — MEDIUM: `BlobServiceClient` created per call +[core/blob.py:79-81, 150-151](../../../hastelib/src/hastegeo/core/blob.py#L79-L81) — +`BlobServiceClient.from_connection_string(...)` on every `download_blob_to_tempfile` / +`read_blob_range` call; no module-level reuse, so credential parse + connection setup +repeat. Same theme: `MetadataProcessor`/`UnifiedDataLayer` are re-instantiated dozens +of times per request/message rather than reused. + +### H6 — MEDIUM: no caching layer anywhere except `footprints.get_latest_release` +[core/utils/footprints.py:155](../../../hastelib/src/hastegeo/core/utils/footprints.py#L155) +is the only `lru_cache` in the core library. There is no request-scoped memoization, +so `load_and_combine_sub_data_types` and similar re-fetch identical data within a +single request. + +--- + +## Queue — `api/hastefuncqueues/function_app.py` + +### Q1 — HIGH: `batchSize: 1`, `maxDequeueCount: 1` +[host.json](../../../api/hastefuncqueues/host.json) processes one message per instance +and dead-letters after a single failure (no retry). Throughput depends entirely on +instance scale-out; a transient error poisons the message immediately. + +### Q2 — HIGH: N+1 label lookup in training trigger +[function_app.py:354-367](../../../api/hastefuncqueues/function_app.py#L354-L367) — +`LABELS.load_all_from_partition` then Python `next(...)` filter; should use H3's +filtered load. + +### Q3 — MEDIUM: independent loads not parallelized +Inference trigger loads image layer then experiment config sequentially +([function_app.py:695-712](../../../api/hastefuncqueues/function_app.py#L695-L712)); +image-processing trigger does store-artifact then get-URL sequentially. `asyncio.gather` +would halve these. + +### Q4 — MEDIUM: 3–5 sequential `MetadataProcessor.save` calls per message, no batching. +Every trigger re-instantiates `MetadataProcessor` several times and awaits each save +serially. A `save_batch` would cut round-trips. + +### Q5 — LOW: `visibilityTimeout: 30s` risks re-enqueue for slow checkpoints; consider +raising or extending visibility on long steps. + +--- + +## UI — `ui/src/` + +### U1 — CRITICAL: 20s poll refetches the entire project incl. all models +[Project.jsx:149-157](../../../ui/src/Components/Project.jsx#L149-L157) → +[Project.jsx:102-147](../../../ui/src/Components/Project.jsx#L102-L147) calls +`GetProjectDetails?includeModels=True` every 20s, always `setComponentState(...)` even +when data is unchanged. This drives B1–B4 repeatedly and re-renders the whole tree on +a timer. + +**Measured (Phase 0):** the large-project response is **~21–38 s — longer than the 20 s +poll interval**, so a new poll fires before the previous one returns and overlapping +603-round-trip requests pile up on the backend. The poll must not fire while a request +is in flight (single-flight guard) and/or the interval must adapt to response time. + +### U7 — HIGH (new, measured): duplicate concurrent `GetProjectDetails` on load +Playwright observed **two concurrent** `GetProjectDetails` calls on the initial project +load, so browser-observed latency is ~2× the isolated API call (large: ~38 s vs 20.8 s). +Dev-mode React StrictMode double-invokes effects, but the root cause that survives a +production build is the **absence of in-flight de-duplication / request cancellation**: +[Project.jsx:102-147](../../../ui/src/Components/Project.jsx#L102-L147) has no +single-flight guard or `AbortController`, so overlapping mounts/effects/polls each issue +a full request. Fix: dedupe in-flight requests and abort superseded ones. + +### U2 — HIGH: monolithic `AppContext` → broad re-renders +[AppContext.jsx:266-281](../../../ui/src/AppContext.jsx#L266-L281) — a single +`appParams` object (loading flag, dialog, breakpoint, …) is consumed by ~38 +components. Any `setIsLoading` (fired by every poll) re-renders all of them. + +### U3 — HIGH: no memoization +No `React.memo` / `useMemo` / `useCallback` anywhere under `ui/src/Components/`. Every +parent render re-renders all `LayerRow` / `ModelRow` children with freshly-created prop +objects. + +### U4 — MEDIUM: sequential independent API calls +[CreateEditImageLayerHelper.js:28-37](../../../ui/src/Components/CreateEditImageLayerHelper.js#L28-L37) +and [LabelingTool.jsx:174-181](../../../ui/src/Components/LabelingTool/LabelingTool.jsx#L174-L181) +`await` two independent GETs in series; should be `Promise.all`. + +### U5 — MEDIUM: no change detection on poll; eager model-row rendering on expand; +no virtualization for large layer/model lists +([Project.jsx:193-228](../../../ui/src/Components/Project.jsx#L193-L228), +[ProjectManagement/LayerRow.jsx:359-404](../../../ui/src/Components/ProjectManagement/LayerRow.jsx#L359-L404)). + +### U6 — MEDIUM-LOW: Visualizer requests full-res pre+post tiles eagerly with no +overview/thumbnail +([Visualizer/Visualizer.jsx:315-345](../../../ui/src/Components/Visualizer/Visualizer.jsx#L315-L345)). + +## Measured priority adjustments (Phase 0) + +The browser baseline reshuffles a few priorities relative to first estimates: + +- **Backend-first is strongly validated.** UI time-to-interactive is **API-bound**: + TTI ≈ the `GetProjectDetails` duration + only ~2 s of render (large: 40.3 s TTI vs + 38.4 s call). So Phase 1/2 (backend) captures the overwhelming majority of the TTI + win; no UI change can hide a 20 s API call. +- **U7 + poll-overlap (U1) rise in priority** — they roughly *double* effective latency + and pile requests on the backend. Cheap to fix (single-flight guard), high impact. +- **Render-side items (U3 memoization, U5 virtualization) drop for TTI** — with only + ~2 s of render at 50 layers, they matter for *poll-time smoothness / re-render churn*, + not first paint. Still worth doing, but not the TTI lever. +- **B6 payload is minor** — the large default payload is only ~83 KB; `summary` mode is + a nice-to-have, not a latency driver. Round-trip count, not payload size, dominates. diff --git a/spec/features/perf-layer-loading/impact-analysis.md b/spec/features/perf-layer-loading/impact-analysis.md new file mode 100644 index 00000000..50c0a4c7 --- /dev/null +++ b/spec/features/perf-layer-loading/impact-analysis.md @@ -0,0 +1,77 @@ +# Impact Analysis: Image Layer & Model Run Loading Performance + +## Scope of Change + +| Component | Path | Type | Severity | +|---|---|---|---| +| REST API | `api/hastefuncapi/function_app.py` | modified (`GetProjectDetails`, `GenerateProjectStats`) | high | +| Core library | `hastelib/src/hastegeo/core/processors/metadata.py` | modified (new methods) | medium | +| Core library | `hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py` | modified | medium | +| Core library | `hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py` | modified | low | +| Core library | `hastelib/src/hastegeo/core/blob.py` | modified (client reuse) | low | +| Queue workers | `api/hastefuncqueues/function_app.py` + `host.json` | modified | medium | +| React UI | `ui/src/Components/Project.jsx`, `AppContext.jsx`, `ProjectManagement/*` | modified | medium | + +## Azure Service Impact + +| Service | Change | Cost Impact | +|---|---|---| +| Blob Storage / Cosmos | Far fewer read round-trips per `GetProjectDetails`; prefix-scoped listings | **Lower** transaction count & egress (especially with 20s poll × N clients) | +| Azure Functions | Lower per-request CPU/wall-time; `gather` uses more threads briefly per request | Net **lower** consumption; watch thread-pool sizing under load | +| Queue Storage | `batchSize`>1 raises concurrency per instance | Neutral–lower; validate scale behavior | + +## Dependency Analysis + +### Upstream (needed by this work) +| Dependency | Status | Risk | +|---|---|---| +| `UnifiedDataLayer` predicate support | to confirm | If Blob path can't filter server-side, `load_filtered` = prefix scan + in-layer filter (still removes caller N+1, smaller win) | +| Existing blob serialization format | available | H4 fix depends on whether legacy blobs are double-encoded | + +### Downstream (affected) +| Consumer | How | Breaking? | Migration? | +|---|---|---|---| +| UI `GetProjectDetails` callers | Same default shape; new optional params | no | no | +| UI polling | Gains `304` fast-path; old client still works | no | no | +| Existing blobs (if double-encoded) | H4 read-path change | **maybe** | Add back-compat read (accept single- or double-encoded) before removing re-parse | +| Queue message format | unchanged | no | no | + +## Risk Assessment + +| Risk | Likelihood | Impact | Mitigation | +|---|---|---|---| +| H4 double-serialize fix breaks reads of legacy blobs | med | high | Keep tolerant read (try single, fall back to double) during transition; migrate lazily on next save | +| `asyncio.gather` + `ThreadPoolExecutor` fan-out exhausts Functions thread pool under concurrency | med → **confirmed relevant** | med | **Phase 0 evidence:** two *concurrent* `GetProjectDetails` calls already ~2× the latency (38 s vs 21 s) — the worker contends today. Bound workers via `HASTE_METADATA_LOAD_WORKERS`, cap total in-flight, and fix the UI single-flight (U7) so fewer concurrent requests hit the API; load-test the fan-out under ≥2 concurrent requests | +| `batchSize`>1 changes ordering/poison behavior | med | med | Gate behind load test; keep separate from Phase 1; revert via config | +| Cache headers serve stale run-status to UI | low | med | Short `max-age` (≤15s); UI still change-detects; runs move to terminal states, not backwards | +| UI context split / memo introduces render regressions | med | low | Ship incrementally; visual + interaction QA per component | + +## Performance Impact (measured baseline → target) + +Phase 0 baselines (50×5, Azurite/dev — lower bound; see [results.md](results.md)): + +- **API latency:** `GetProjectDetails` **20.8 s p50 / 21.8 s p95** → target **< 1.5 s**; + round-trips **603 → ≤ ~6 + 2 bounded fan-outs**. +- **UI time-to-interactive:** **40.3 s** (large) → target **< 2 s**. TTI is API-bound + (~2 s render over the API call), so the backend fix drives most of this; the UI + single-flight/poll guards remove the ~2× amplification and request pile-up. +- **Backend load:** the 20 s poll currently re-issues the full 603-round-trip call + (measured 36.5 s per poll) — worse, response > interval so polls overlap. Fixing the + poll guard + `304` collapses idle per-open-project storage transactions dramatically. +- **Queue throughput:** higher with `batchSize`>1; training/inference lose partition scans. + +## Security Impact + +- [x] No new endpoints exposed; `GetProjectDetails` auth level unchanged. +- [x] No new data classification; same imagery/metadata. +- [x] No auth/CORS changes. +- [ ] `ETag` is a payload hash — ensure it doesn't leak across tenants (it's per-project, + auth already scopes access). Confirm no cross-user cache reuse. + +## Rollback Assessment + +- **Reversibility:** fully reversible — behavioral/perf changes, no schema migration + required (H4 handled with a tolerant read path, so no data rewrite). +- **API:** contract preserved; new params are opt-in. Revert = redeploy prior build. +- **Config:** `host.json` / env knobs revert independently. +- **Estimated rollback time:** one redeploy (minutes). diff --git a/spec/features/perf-layer-loading/plan.md b/spec/features/perf-layer-loading/plan.md new file mode 100644 index 00000000..dbc45281 --- /dev/null +++ b/spec/features/perf-layer-loading/plan.md @@ -0,0 +1,142 @@ +# Execution Plan: Image Layer & Model Run Loading Performance + +Sequenced by impact-per-risk. Phase 0 establishes a baseline so every later phase is +measured, not guessed. Backend before UI: the backend N+1 is the dominant cost and its +fixes are transparent to the UI. Each phase is independently shippable. + +## Phase 0: Baseline & Instrumentation — DONE (2026-08-03) + +**Goal:** Make the problem measurable before changing it. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Add opt-in round-trip counter + timing (`hastelib/.../utils/perf.py`, wired into `MetadataProcessor` reads + `GetProjectDetails` via `HASTE_PERF`) | `backend-dev` | — | B1–B4 | **done** | +| Seed script for synthetic L×M project (`tools/seed_synthetic_project.py`) | `backend-dev` | — | cost-model | **done** | +| Capture baseline round-trips + payload (`tools/phase0_baseline.py` → `results.md`) | `backend-dev` | above | success-criteria | **done** | +| HTTP latency harness for running stack (`tools/bench_api_http.py`) | `backend-dev` | above | success-criteria | **done** | +| Capture real API p50/p95 latency against running stack (Docker + Azurite) | `backend-dev` | stack up + seed | success-criteria | **done** | +| Compose overlay + reproducible run (`docker/docker-compose.perf.yml`) | `backend-dev` | — | — | **done** | +| Record UI time-to-interactive + poll cost (Playwright, `tools/ui_bench.cjs`) | `ui` | stack up | U1 | **done** | + +**Exit Criteria:** +- [x] Baseline round-trips (33 / 243 / **603**) + payload captured in + [results.md](results.md) and `test-plan.md`; per-op breakdown quantifies B1/B2. +- [x] Real API latency captured (large **20.8 s p50 / 21.8 s p95** on Azurite; + storage = 13.4 s of that). Confirms the reported symptom. +- [x] Browser-side UI TTI captured (large **40.3 s**; API-bound + fired twice + concurrently; 20 s poll re-does the full call). Phase 0 complete. + +## Phase 1: Backend API hot path (`hastefuncapi`) — highest ROI + +**Goal:** Collapse `GetProjectDetails` to a constant number of storage reads. No API +contract change; UI untouched. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Hoist `LABELS.load_all_from_partition` out of the layer loop (load once) | `backend-dev` | P0 | B1 | not-started | +| Parallelize top-level reads (project/layers/models/labels) with `asyncio.gather` | `backend-dev` | P0 | B4 | not-started | +| Batch per-model artifacts + `labelsUrl` via `load_map` (bounded parallel) | `backend-dev` | Ph2 `load_map` | B2 | not-started | +| Batch per-layer VALIDATION via `load_map` | `backend-dev` | Ph2 `load_map` | B3 | not-started | +| Apply the same hoist to `GenerateProjectStats` | `backend-dev` | B1 | B5 | not-started | +| Add `ETag` + `Cache-Control` + `304` handling to `GetProjectDetails` | `backend-dev` | — | B7 | not-started | +| Add optional `summary` / `includeArtifacts=false` response modes | `backend-dev` | above | B6 | not-started | + +**Exit Criteria:** +- [ ] Round-trip count for 50×5 project drops from ~600 to ≤ ~6 + 2 bounded fan-outs. +- [ ] `GetProjectDetails` p95 < 1.5s on the synthetic project. +- [ ] Existing UI still works unchanged (contract preserved). + +## Phase 2: Core library (`hastelib`) — enables Phase 1 batch/filter + +**Goal:** Give the API layer the primitives it needs and remove data-layer waste. +(Ships alongside Phase 1; `load_map` is a prerequisite for B2/B3.) + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Add `MetadataProcessor.load_map(keys, max_workers)` (bounded `ThreadPoolExecutor`) | `backend-dev` | — | H2 | not-started | +| Add `MetadataProcessor.load_filtered(predicate)` | `backend-dev` | — | H3 | not-started | +| Pass `name_starts_with` prefix in `load_all`; add metadata-only listing | `backend-dev` | — | H1 | not-started | +| Parallelize download loops in `load_all*` and `fetch_artifact` | `backend-dev` | — | H2 | not-started | +| Fix double-serialization on save; drop redundant `json.loads` | `backend-dev` | migration Q | H4 | not-started | +| Reuse module-level `BlobServiceClient` (keyed by conn-string) | `backend-dev` | — | H5 | not-started | +| Unit tests in `hastelib/tests/` for new methods + parity of old behavior | `backend-dev` | all above | — | not-started | + +**Exit Criteria:** +- [ ] `hastelib` unit tests pass; `load_filtered`/`load_map` covered. +- [ ] No container-wide scans without a prefix remain in `load_all`. + +## Phase 3: Queue workers (`hastefuncqueues`) — throughput + +**Goal:** Remove the queue-side N+1 and unlock concurrency. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Replace label N+1 in training trigger with `load_filtered` | `backend-dev` | Ph2 H3 | Q2 | not-started | +| Parallelize independent loads (inference, image) with `asyncio.gather` | `backend-dev` | — | Q3 | not-started | +| Add `save_batch`; batch intermediate saves | `backend-dev` | — | Q4 | not-started | +| Load-test `batchSize`>1 / `maxDequeueCount`≥3 / `visibilityTimeout` in `host.json` | `backend-dev` | — | Q1,Q5 | not-started | + +**Exit Criteria:** +- [ ] Training/inference triggers issue no full-partition scans. +- [ ] `host.json` changes validated under load without poison-queue regressions. + +## Phase 4: UI (`ui/src`) — perceived performance + +**Goal:** Stop the poll from thrashing the tree; render only what's needed. + +> **Measured (Phase 0):** TTI is API-bound, so backend phases own the TTI win. Rank the +> UI work by impact: **single-flight guard (U7) + poll guard (U1) first** (halve +> effective latency, stop pile-up), then context split (U2); memoization/virtualization +> (U3/U5) and `summary` payload (B6) are lower-value follow-ups. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Single-flight guard + `AbortController` on `fetchProjectDetails` (dedupe/cancel concurrent calls) | `ui` | — | U7 | not-started | +| Smart poll: send `If-None-Match`, handle `304`, skip `setState` if unchanged | `ui` | Ph1 B7 | U1 | not-started | +| Poll guard: don't fire while a request is in flight (response can exceed 20 s interval) | `ui` | — | U1 | not-started | +| Pause polling when tab hidden; make interval configurable/adaptive | `ui` | — | U1 | not-started | +| Split volatile fields out of `AppContext` into a lighter provider | `ui` | — | U2 | not-started | +| `React.memo` / `useMemo` / `useCallback` on `LayerRow`/`ModelRow`/`ModelRowMobile` | `ui` | — | U3 | not-started | +| Lazy-render model rows on expand; evaluate list virtualization | `ui` | — | U5 | not-started | +| `Promise.all` the independent GETs in edit/labeling helpers | `ui` | — | U4 | not-started | +| Use `summary` mode for list view; fetch full models on expand | `ui` | Ph1 B6 | B6,U5 | not-started | +| Thumbnail/overview-first in Visualizer; defer full-res second map | `ui` | — | U6 | not-started | + +**Exit Criteria:** +- [ ] Project page time-to-interactive < 2s p95 on the synthetic project. +- [ ] Idle open-project CPU/network drops measurably (no full refetch/re-render per poll). + +## Phase 5: Integration & Validation + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| End-to-end run on Docker Compose stack with synthetic large project | `backend-dev` | Ph1–4 | — | not-started | +| Compare against Phase 0 baseline; record deltas | `backend-dev` | above | success-criteria | not-started | +| Update `docs/` (api-overview, architecture) with new params/caching | `backend-dev` | — | — | not-started | + +**Exit Criteria:** +- [ ] All success criteria in [README.md](README.md#success-criteria) met and recorded. +- [ ] CI passes (secret-scan, deploy-apps). + +## Milestones + +| Milestone | Deliverable | +|---|---| +| Baseline captured | Phase 0 numbers in test-plan.md | +| Backend hot path fixed | `GetProjectDetails` O(1) reads, cache headers | +| Core lib primitives | `load_map` / `load_filtered` / prefix scans merged | +| Queue optimized | N+1 removed, config load-tested | +| UI responsive | smart poll + memoization shipped | +| Validated | End-to-end deltas meet success criteria | + +## Agent Summary + +| Agent | Phases | +|---|---| +| `backend-dev` | 0, 1, 2, 3, 5 | +| `ui` | 0, 4 | + +## Open Questions + +- [ ] Ship Phase 1 alone first (backend-only, invisible) to bank the win before UI work? +- [ ] Poll vs push (SignalR) for run-status — defer or fold into Phase 4? diff --git a/spec/features/perf-layer-loading/results.md b/spec/features/perf-layer-loading/results.md new file mode 100644 index 00000000..756851a1 --- /dev/null +++ b/spec/features/perf-layer-loading/results.md @@ -0,0 +1,128 @@ +# Phase 0 Baseline — Measured Results + +**Date:** 2026-08-03 +**Branch:** `prbatero/feat/performance-improvements` +**Method:** `tools/phase0_baseline.py` — seeds synthetic projects into the real +`local` filesystem backend and replays the exact `GetProjectDetails` read+assemble +sequence ([function_app.py:534-638](../../../api/hastefuncapi/function_app.py#L534-L638)) +with `HASTE_PERF` instrumentation on. Models seeded **without** `labelsUrl` (worst-case +N+1 that triggers the per-model `TRAIN_LABELS` export). + +## Headline metric — storage round-trips per request + +| Fixture | Layers × Models | **Round-trips** | Payload | Formula `3 + L·(2M+2)` | +|---|---|---|---|---| +| small | 5 × 2 | **33** | 5.8 KB | 33 ✓ | +| medium | 20 × 5 | **243** | 49.2 KB | 243 ✓ | +| large | 50 × 5 | **603** | 122.4 KB | 603 ✓ | + +Round-trip counts match the derived cost formula exactly, confirming the replay is +faithful to the handler and that cost scales as **O(layers × models)**. + +## Per-op breakdown (large, 50 × 5) + +| Op | Count | Source | +|---|---|---| +| `load` | 301 | 1 project + 250 per-model `MODEL_ARTIFACTS` (B2) + 50 per-layer `VALIDATION` (B3) | +| `load_all_from_partition` | 52 | 1 imagelayer + 1 model + **50 redundant full `LABELS` scans** (B1 — should be 1) | +| `export` | 250 | per-model `TRAIN_LABELS` export (B2) | +| **total** | **603** | | + +**B1 is quantified:** 50 of the 52 partition scans are the identical full `LABELS` +download re-run once per layer. **B2 is quantified:** 500 of 603 round-trips (83%) are +the per-model artifact + labels-export pair. + +## Latency — MEASURED against the real API (Docker + Azurite blob backend) + +Captured via `tools/bench_api_http.py` against the running stack (compose overlay +`docker/docker-compose.perf.yml`, `HASTE_PERF=true`, `blob` backend on Azurite), +after seeding each project into Azurite. End-to-end `GetProjectDetails?includeModels=True`: + +| Fixture | **latency p50** | latency p95 | storage calls | storage time p50 | payload | per-call cost | +|---|---|---|---|---|---|---| +| small (5×2) | **0.70 s** | 0.71 s | 33 | 0.28 s | 4.2 KB | ~8.5 ms/call | +| medium (20×5) | **6.18 s** | 6.71 s | 243 | 3.05 s | 33.2 KB | ~12.5 ms/call | +| large (50×5) | **20.77 s** | 21.78 s | 603 | 13.39 s | 82.8 KB | ~22 ms/call | + +**A single large-project load takes ~21 seconds** — and this is a *localhost* emulator; +real Azure Blob has higher per-op latency, so production is worse. Storage accounts for +~65% of wall time (13.4 s of 20.8 s); the rest is Python serialization, the double-JSON +deserialize (H4), and per-blob client creation inside the blob layer. + +**B1 is super-linear:** per-call cost climbs 8.5 → 12.5 → 22 ms as the partition grows, +because each of the 50 redundant full-`LABELS` partition scans lists + downloads *every* +label blob, and that set grows with the project. So the redundant scans cost more the +bigger the project gets — compounding the O(layers × models) round-trip growth. + +For reference, the local-FS replay wall (no network, relative floor only) was +6.7 / 51.0 / 356.5 ms for small / medium / large. + +## Browser-side (UI) baseline — MEASURED (Playwright vs the real React app) + +Captured via `tools/ui_bench.cjs` driving the real UI (swa-cli emulator in Docker) +with Playwright. The cheap auth/user bootstrap is mocked (crafted SWA admin cookie + +route interception of `/.auth/me`, `GetUserById`, `PutUser`); **`GetProjectDetails` +hits the real API**. TTI = navigation → first image-layer row visible in the DOM. + +| Fixture | **TTI** (open → layers) | initial `GetProjectDetails` | GPD calls on load | 20 s poll | +|---|---|---|---|---| +| small (5×2) | 3.10 s | 1.27 s | 2 | (interval not reached) | +| medium (20×5) | 12.36 s | 10.40 s | 2 | 1 call, 6.20 s | +| large (50×5) | **40.27 s** | 38.44 s | 2 | 1 call, **36.50 s** | + +**TTI is API-bound:** it tracks the `GetProjectDetails` duration almost exactly (large +40.3 s TTI vs 38.4 s call — only ~2 s of render). So the layer-loading fix is +fundamentally the backend fix; UI work reduces the *amplifiers* below. + +Two UI-specific amplifiers the browser run exposed: +- **Duplicate concurrent fetch (U1-adjacent):** the page issues `GetProjectDetails` + **twice concurrently** on load (React StrictMode in dev + no request dedup). The two + 603-round-trip requests contend, so browser-observed latency is **~2× the isolated + API call** (large: 38.4 s browser vs 20.8 s isolated `curl`). Production build drops + the StrictMode double, but the absence of dedup/caching is real. +- **Poll re-does everything (U1):** the 20 s poll fires a fresh full + `GetProjectDetails` (large poll call = 36.5 s), re-incurring all 603 round-trips and + re-rendering the whole tree (no memoization, monolithic context). Because the + response (~21–38 s) is **longer than the 20 s interval**, polls overlap and pile up. + +**Caveats (inflate absolute numbers, not the structural findings):** UI ran under Vite +**dev mode** (unminified + StrictMode double-invoke); the API image is amd64 emulated on +Apple Silicon; storage is a localhost Azurite emulator. A production build + real Azure +would shift the constants but not the O(layers × models) scaling or the amplifiers. +- To capture TTI/poll cost precisely once the stack is up: Chrome DevTools Performance + trace on the project page; `Server-Timing` (now emitted by the API) surfaces + server storage time directly in the Network panel. + +## How to reproduce + +```bash +# Headline round-trip baseline (no infra needed): +PYTHONPATH=hastelib/src python3 \ + spec/features/perf-layer-loading/tools/phase0_baseline.py + +# Real latency (requires running stack + HASTE_PERF=true on the API + seeded project): +METADATA_STORAGE_TYPE=local DATA_PATH=/tmp/haste-bench PYTHONPATH=hastelib/src \ + python3 spec/features/perf-layer-loading/tools/seed_synthetic_project.py \ + --project-id --layers 50 --models 5 +python3 spec/features/perf-layer-loading/tools/bench_api_http.py \ + --base-url http://localhost:7071/api --project-id --repeats 30 +``` + +## Targets to beat (from [README.md](README.md#success-criteria)) + +| Metric | Baseline (large, measured) | Target | +|---|---|---| +| Round-trips / request | **603** | ≤ ~6 + 2 bounded fan-outs | +| API p50 / p95 latency | **20.8 s / 21.8 s** (Azurite) | < 1.5 s | +| Payload (default shape) | 82.8 KB | smaller via `summary` mode | + +## Environment notes + +- Backend: Azurite blob emulator on `localhost` (compose). Real Azure Blob per-op + latency is higher, so these numbers are a **lower bound** on production. +- Host: Docker Desktop on Apple Silicon; the amd64 API image runs emulated, inflating + the non-storage (CPU) portion somewhat. The storage-round-trip count (603) is + hardware-independent and is the primary target metric. +- Reproduce: `docker compose -f docker/docker-compose.yml -f docker/docker-compose.perf.yml + up -d hastefuncapi api-proxy`, seed via `seed_synthetic_project.py` inside the + `hastefuncapi` container, then run `bench_api_http.py` from the host. diff --git a/spec/features/perf-layer-loading/test-plan.md b/spec/features/perf-layer-loading/test-plan.md new file mode 100644 index 00000000..7f381818 --- /dev/null +++ b/spec/features/perf-layer-loading/test-plan.md @@ -0,0 +1,98 @@ +# Test Plan: Image Layer & Model Run Loading Performance + +## Strategy + +Performance work must be **measured, not asserted**. Every phase compares against the +Phase 0 baseline on a fixed synthetic project. Correctness tests guard that the +refactors preserve behavior (same data, new speed). + +## Benchmark fixture + +A synthetic project used everywhere in this spec: + +- **Small:** 5 layers × 2 models +- **Medium:** 20 layers × 5 models +- **Large (headline):** 50 layers × ~5 models, plus labels + validation per layer + +Provide a seed script (`hastelib/tests` helper or a queue-message replay) that +populates the local Docker Compose storage emulator (Azurite) with these shapes. + +## Metrics captured (per fixture size) + +| Metric | How | Target (Large) | +|---|---|---| +| `GetProjectDetails` p50 / p95 latency | timed API calls (warm) | p95 < 1.5s | +| Storage round-trip count per request | Phase 0 counter log | ≤ ~6 + 2 bounded fan-outs | +| Response payload size | `Content-Length` | tracked; smaller in `summary` mode | +| UI time-to-interactive | DevTools performance trace | p95 < 2s | +| Idle open-project network/CPU over 60s | DevTools, no data change | no full refetch; `304` on poll | + +## Baseline capture (Phase 0 — measured 2026-08-03) + +Full detail in [results.md](results.md). Captured via `tools/phase0_baseline.py` +(real code, real seeded data, local FS backend). + +| Fixture | round-trips | API p50 | API p95 | payload | **UI TTI** | +|---|---|---|---|---|---| +| Small (5×2) | **33** | 0.70 s | 0.71 s | 4.2 KB | 3.10 s | +| Medium (20×5) | **243** | 6.18 s | 6.71 s | 33.2 KB | 12.36 s | +| Large (50×5) | **603** | **20.77 s** | **21.78 s** | 82.8 KB | **40.27 s** | + +Round-trips match `3 + L·(2M+2)` exactly. Large breakdown: 301 `load`, 52 +`load_all_from_partition` (50 redundant `LABELS` scans = B1), 250 `export` (B2). +API latency measured via `tools/bench_api_http.py`; UI TTI via `tools/ui_bench.cjs` +(Playwright vs the real app). TTI is API-bound (~2 s render over the API call) and the +UI fires the call twice concurrently on load. All numbers are a lower bound (localhost +Azurite, amd64 emulation, Vite dev mode). See [results.md](results.md). + +## Correctness / regression tests + +### Backend (`hastelib/tests/`) +- `load_map`: returns one entry per key; `None` for missing (parity with old + per-key `try/except FileNotFoundError`); respects `max_workers`. +- `load_filtered`: returns the same subset the old "load all + Python filter" produced, + for `imageLayerId` predicate. +- `load_all` with prefix: identical results to prior no-prefix scan for a given + partition; no cross-partition leakage. +- H4 tolerant read: correctly decodes both legacy double-encoded and new + single-encoded blobs. +- `BlobServiceClient` reuse: same client instance returned for same conn-string. + +### API (`hastefuncapi`) +- `GetProjectDetails` default response is **byte-for-byte equivalent** (post-sort) to + the pre-refactor response for Small/Medium/Large fixtures (golden-file compare). +- `summary` mode omits `models[]` but keeps counts. +- `includeArtifacts=false` skips artifact expansion, keeps `modelCount`. +- `ETag` stable across identical requests; `If-None-Match` match ⇒ `304` empty body. +- `GenerateProjectStats` parity after the B5 hoist. + +### Queue (`hastefuncqueues`) +- Training trigger with `load_filtered` selects the same label project as before. +- `save_batch` persists all items; partial-failure behavior defined and tested. +- With `batchSize`>1: N messages processed, no dropped/duplicated results; poison + path exercised at `maxDequeueCount`. + +### UI (`ui`) +- **Single-flight (U7):** initial project load issues **exactly one** `GetProjectDetails` + (not two) — assert via `tools/ui_bench.cjs` `getprojectdetails_calls_during_load == 1` + (production build) and a network spy; a superseded fetch is aborted. +- **Poll guard (U1):** no new poll fires while a request is in flight; assert no + overlapping `GetProjectDetails` when response time > interval. +- Poll receiving `304` (or byte-identical body) does **not** call `setComponentState` + (assert via spy/render-count). +- `LayerRow`/`ModelRow` memoized: unchanged props ⇒ no re-render (React Profiler). +- Expanding a layer renders its model rows; collapsed layers render none. +- Edit/labeling helpers issue independent GETs concurrently (`Promise.all`). + +## Load test (Phase 3 gate) + +- Enqueue a burst (e.g. 100 image/inference messages); compare end-to-end drain time + and poison count for `batchSize` 1 vs candidate value. +- Confirm no `visibilityTimeout` re-enqueue storms on long-running steps. + +## Exit gate + +- [ ] Large-fixture targets met and recorded in the baseline table (before/after). +- [ ] All correctness/parity tests green. +- [ ] `docker compose up` runs the full stack with the synthetic project without error. +- [ ] CI (secret-scan, deploy-apps) passes. diff --git a/spec/features/perf-layer-loading/tools/bench_api_http.py b/spec/features/perf-layer-loading/tools/bench_api_http.py new file mode 100644 index 00000000..9c7224fa --- /dev/null +++ b/spec/features/perf-layer-loading/tools/bench_api_http.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Benchmark GetProjectDetails against a running HASTE API (real latency). + +Use this once the full stack is up (Docker Compose or Azure) and a synthetic +project has been seeded into that backend. It captures end-to-end p50/p95 latency +plus the server-side round-trip count and storage time exposed by the Phase 0 +headers (requires HASTE_PERF=true on the API). + +Run: + python spec/features/perf-layer-loading/tools/bench_api_http.py \ + --base-url http://localhost:7071/api \ + --project-id --repeats 30 [--code ] + +Reads only the standard library so it can run anywhere. +""" +import argparse +import json +import statistics +import time +import urllib.request + + +def _pct(values, p): + if not values: + return None + s = sorted(values) + k = max(0, min(len(s) - 1, int(round((p / 100.0) * (len(s) - 1))))) + return s[k] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:7071/api") + ap.add_argument("--project-id", required=True) + ap.add_argument("--repeats", type=int, default=30) + ap.add_argument("--warmup", type=int, default=3) + ap.add_argument("--code", default=None, help="Azure Functions key, if required") + args = ap.parse_args() + + qs = f"?projectId={args.project_id}&includeModels=True" + if args.code: + qs += f"&code={args.code}" + url = f"{args.base_url}/GetProjectDetails{qs}" + + latencies, storage_calls, storage_ms, payloads = [], [], [], [] + for i in range(args.warmup + args.repeats): + t0 = time.perf_counter() + with urllib.request.urlopen(url) as resp: + body = resp.read() + hdrs = resp.headers + dt = (time.perf_counter() - t0) * 1000.0 + if i < args.warmup: + continue + latencies.append(dt) + payloads.append(len(body)) + if hdrs.get("X-Haste-Storage-Calls"): + storage_calls.append(int(hdrs["X-Haste-Storage-Calls"])) + if hdrs.get("X-Haste-Storage-Ms"): + storage_ms.append(float(hdrs["X-Haste-Storage-Ms"])) + + result = { + "project_id": args.project_id, + "repeats": args.repeats, + "latency_p50_ms": round(statistics.median(latencies), 1), + "latency_p95_ms": round(_pct(latencies, 95), 1), + "latency_max_ms": round(max(latencies), 1), + "payload_kb": round(statistics.median(payloads) / 1024, 1), + "server_storage_calls": storage_calls[0] if storage_calls else None, + "server_storage_ms_p50": ( + round(statistics.median(storage_ms), 1) if storage_ms else None + ), + } + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/spec/features/perf-layer-loading/tools/phase0_baseline.py b/spec/features/perf-layer-loading/tools/phase0_baseline.py new file mode 100644 index 00000000..1748b322 --- /dev/null +++ b/spec/features/perf-layer-loading/tools/phase0_baseline.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Phase 0 baseline: measure GetProjectDetails storage round-trips. + +Seeds synthetic projects (small / medium / large) into a temporary local-FS +backend and replays the *exact* read sequence of the ``GetProjectDetails`` handler +(``api/hastefuncapi/function_app.py`` lines ~534-638) with perf instrumentation on. + +Reports, per size, the number of backend storage round-trips (the headline +success metric: ~O(layers x models) today -> O(1) target) plus a per-op breakdown +and a local-FS wall-clock (for relative comparison only; absolute latency p50/p95 +must be measured against the running Azure/Docker stack via bench_api_http.py). + +Run: + PYTHONPATH=hastelib/src \ + python spec/features/perf-layer-loading/tools/phase0_baseline.py +""" +import json +import os +import statistics +import sys +import tempfile + +# Force the local filesystem backend for a self-contained, infra-free baseline. +os.environ.setdefault("METADATA_STORAGE_TYPE", "local") + +sys.path.insert(0, os.path.join(os.path.dirname(__file__))) # for seed import +from seed_synthetic_project import seed # noqa: E402 + +from hastegeo.core.config import Config # noqa: E402 +from hastegeo.core.processors.metadata import MetadataProcessor # noqa: E402 +from hastegeo.core.utils import perf # noqa: E402 + +SIZES = [ + ("small", 5, 2), + ("medium", 20, 5), + ("large", 50, 5), +] +REPEATS = 5 + + +def _mp(data_type, project_id): + return MetadataProcessor(data_type=data_type, partition_key=project_id) + + +def replay_get_project_details(project_id, include_models=True): + """Faithful replay of the GetProjectDetails read+assemble sequence. + + Mirrors api/hastefuncapi/function_app.py:534-638. Returns the serialized + payload length so we can report response size alongside round-trips. + """ + types = Config.get_metadata_types() + + project = _mp(types.PROJECT.value, project_id).load(project_id) + image_layers = _mp(types.IMAGELAYER.value, project_id).load_all_from_partition() + models = [] + if include_models: + models = _mp(types.MODEL.value, project_id).load_all_from_partition() + + for image_layer in image_layers: + image_layer_id = image_layer["imageLayerId"] + if include_models: + match_models = [ + m for m in models if m["imageLayerId"] == image_layer_id + ] + match_models.sort(key=lambda x: x["creationDate"], reverse=True) + for model in match_models: + try: + model["artifacts"] = _mp( + types.MODEL_ARTIFACTS.value, project_id + ).load(model["modelId"]) + except FileNotFoundError: + model["artifacts"] = None + try: + if not model.get("labelsUrl"): + model["labelsUrl"] = _mp( + types.TRAIN_LABELS.value, project_id + ).export(key=model["modelId"], data_format="geojson") + except FileNotFoundError: + model["labelsUrl"] = None + image_layer["models"] = match_models + image_layer["modelCount"] = len(match_models) + + label_projects = _mp( + types.LABELS.value, project_id + ).load_all_from_partition() + match = next( + (lp for lp in label_projects + if lp["imageLayerId"] == image_layer_id), + None, + ) + if match is not None and match.get("labels") is not None: + image_layer["labelProjectCount"] = len(match["labels"]) + + try: + validation = _mp(types.VALIDATION.value, project_id).load( + image_layer_id + ) + image_layer["validationLabelCount"] = len( + validation.get("labels") or {} + ) + except FileNotFoundError: + image_layer["validationLabelCount"] = 0 + + project["imageLayer"] = image_layers + project["imageLayerCount"] = len(image_layers) + project["imageLayer"].sort(key=lambda x: x["creationDate"], reverse=True) + return len(json.dumps(project)) + + +def run(): + rows = [] + for name, layers, models_per in SIZES: + with tempfile.TemporaryDirectory(prefix=f"haste-bench-{name}-") as d: + os.environ["DATA_PATH"] = d + project_id = f"00000000-0000-4000-8000-{layers:06d}{models_per:06d}" + total_models = seed( + project_id, layers, models_per, + labels_per_layer=20, validation_per_layer=10, + with_labels_url=False, + ) + + walls, calls, payload, ops = [], None, None, None + for _ in range(REPEATS): + counter = perf.begin(True) + import time + t0 = time.perf_counter() + payload = replay_get_project_details(project_id) + walls.append((time.perf_counter() - t0) * 1000.0) + calls = counter.calls + ops = {k: v["calls"] for k, v in counter.by_op.items()} + perf.end() + + rows.append({ + "size": name, "layers": layers, "models_per": models_per, + "total_models": total_models, "round_trips": calls, + "ops": ops, "payload_bytes": payload, + "wall_p50_ms": round(statistics.median(walls), 1), + "wall_p95_ms": round(max(walls), 1), + }) + + hdr = (f"{'size':7} {'layers':6} {'mdl/l':5} {'round_trips':11} " + f"{'payload_kb':10} {'localfs_p50ms':13} {'ops (by type)'}") + print(hdr) + print("-" * len(hdr)) + for r in rows: + print(f"{r['size']:7} {r['layers']:<6} {r['models_per']:<5} " + f"{r['round_trips']:<11} {r['payload_bytes']/1024:<10.1f} " + f"{r['wall_p50_ms']:<13} {r['ops']}") + print() + for r in rows: + L, M = r["layers"], r["models_per"] + print(f" {r['size']}: round_trips={r['round_trips']} " + f"formula 3 + L*(2M+2) = {3 + L * (2 * M + 2)}") + + +if __name__ == "__main__": + run() diff --git a/spec/features/perf-layer-loading/tools/seed_synthetic_project.py b/spec/features/perf-layer-loading/tools/seed_synthetic_project.py new file mode 100644 index 00000000..05b17e7d --- /dev/null +++ b/spec/features/perf-layer-loading/tools/seed_synthetic_project.py @@ -0,0 +1,155 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Seed a synthetic HASTE project for perf baselining (Phase 0). + +Writes a project with ``--layers`` image layers, each with ``--models`` models, +plus per-layer LABELS + VALIDATION records and per-model MODEL_ARTIFACTS, using the +real ``MetadataProcessor`` against whatever backend the environment is configured +for (defaults to the ``local`` filesystem layer, so no Azure/Docker is required). + +Usage (local FS, no infra): + METADATA_STORAGE_TYPE=local DATA_PATH=/tmp/haste-bench \ + PYTHONPATH=hastelib/src \ + python spec/features/perf-layer-loading/tools/seed_synthetic_project.py \ + --project-id 00000000-0000-4000-8000-000000000050 --layers 50 --models 5 + +The project id is a canonical GUID so the same seed is reusable against the real +HTTP API (which validates ``projectId`` as a GUID). +""" +import argparse +import uuid + +from hastegeo.core.config import Config +from hastegeo.core.processors.metadata import MetadataProcessor + +# Fixed base date so seeds are deterministic (no wall-clock dependency). +_BASE_DATE = "2026-01-01T00:00:00Z" + + +def _iso(seq): + # Distinct, sortable creationDate values without importing time. + return f"2026-01-{(seq % 27) + 1:02d}T00:00:00Z" + + +def _mp(data_type, project_id): + return MetadataProcessor(data_type=data_type, partition_key=project_id) + + +def seed(project_id, layers, models, labels_per_layer, validation_per_layer, + with_labels_url): + types = Config.get_metadata_types() + + _mp(types.PROJECT.value, project_id).save( + project_id, + { + "projectId": project_id, + "name": f"Perf baseline {layers}x{models}", + "description": "Synthetic project for perf-layer-loading Phase 0.", + "creationDate": _BASE_DATE, + }, + ) + + model_total = 0 + for li in range(layers): + layer_id = f"layer-{li:04d}" + _mp(types.IMAGELAYER.value, project_id).save( + layer_id, + { + "imageLayerId": layer_id, + "projectId": project_id, + "name": f"Layer {li}", + "creationDate": _iso(li), + }, + ) + + # One label project per layer, with N labels. + _mp(types.LABELS.value, project_id).save( + f"labelproj-{li:04d}", + { + "labelprojectId": f"labelproj-{li:04d}", + "imageLayerId": layer_id, + "labels": [{"id": j} for j in range(labels_per_layer)], + }, + ) + + # Validation record keyed by imageLayerId. + _mp(types.VALIDATION.value, project_id).save( + layer_id, + {"imageLayerId": layer_id, + "labels": {str(j): {"id": j} for j in range(validation_per_layer)}}, + ) + + for mi in range(models): + model_id = f"{li:04d}{mi:02d}" + # Fields below mirror what the UI's ModelRow / ModelResultsButton read + # so the real React tree renders without crashing (e.g. inferenceJobs + # must be an array). This enriches the record but does NOT change the + # GetProjectDetails round-trip count. + model = { + "modelId": model_id, + "imageLayerId": layer_id, + "projectId": project_id, + "name": f"Model {li}-{mi}", + "userId": "bench@example.com", + "modelType": "segmentation", + "status": "Trained", + "statusMessage": "", + "trainDate": _iso(mi), + "creationDate": _iso(mi), + "labelsCount": 100, + "currentStep": 10, + "totalSteps": 10, + "progressPct": 100, + "inferenceJobs": [], + "inferenceStatus": None, + "inferenceStatusMessage": "", + "inferenceCurrentStep": 0, + "inferenceTotalSteps": 0, + "inferenceProgressPct": 0, + "labelsUrl": None, + "artifacts": None, + } + if with_labels_url: + model["labelsUrl"] = f"https://example/{model_id}.geojson" + _mp(types.MODEL.value, project_id).save(model_id, model) + + _mp(types.MODEL_ARTIFACTS.value, project_id).save( + model_id, + {"modelId": model_id, "metrics": {"iou": 0.5, "f1": 0.6}}, + ) + model_total += 1 + + return model_total + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--project-id", default=str(uuid.uuid4())) + ap.add_argument("--layers", type=int, default=50) + ap.add_argument("--models", type=int, default=5) + ap.add_argument("--labels-per-layer", type=int, default=20) + ap.add_argument("--validation-per-layer", type=int, default=10) + ap.add_argument( + "--with-labels-url", + action="store_true", + help="Seed models with labelsUrl set (skips the per-model TRAIN_LABELS " + "export round-trip; omit to reproduce the worst-case N+1).", + ) + args = ap.parse_args() + + total = seed( + args.project_id, + args.layers, + args.models, + args.labels_per_layer, + args.validation_per_layer, + args.with_labels_url, + ) + print( + f"Seeded project {args.project_id}: {args.layers} layers, " + f"{total} models, with_labels_url={args.with_labels_url}" + ) + + +if __name__ == "__main__": + main() diff --git a/spec/features/perf-layer-loading/tools/ui_bench.cjs b/spec/features/perf-layer-loading/tools/ui_bench.cjs new file mode 100644 index 00000000..62a208c5 --- /dev/null +++ b/spec/features/perf-layer-loading/tools/ui_bench.cjs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// Browser-side Phase 0 baseline for the project page (perf-layer-loading spec). +// +// Drives the real React UI (served by the swa-cli emulator in Docker) with +// Playwright and measures, for a seeded project: +// - time-to-interactive: navigation start -> first image-layer row visible +// - the real GetProjectDetails request duration (the expensive call) +// - the 20s background poll: whether it fires and its cost +// +// The cheap auth/user bootstrap is mocked (crafted SWA admin cookie + route +// interception of /.auth/me, GetUserById, PutUser) so the page renders without a +// real login; GetProjectDetails hits the REAL API and is what we measure. +// +// Run (playwright installed in a scratch dir): +// NODE_PATH=/tmp/haste-uibench/node_modules \ +// node spec/features/perf-layer-loading/tools/ui_bench.cjs \ +// --ui http://localhost:4280 --api http://localhost:7071 \ +// --project 00000000-0000-4000-8000-000050000005 +const { chromium } = require("playwright"); + +function arg(name, def) { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : def; +} + +const UI = arg("ui", "http://localhost:4280"); +const API = arg("api", "http://localhost:7071"); +const PROJECT = arg("project", "00000000-0000-4000-8000-000050000005"); +const POLL_WAIT_MS = parseInt(arg("pollwait", "26000"), 10); + +const principal = { + identityProvider: "aad", + userId: "benchuser", + userDetails: "bench@example.com", + userRoles: ["authenticated", "administrators", "contributors"], + claims: [], +}; +const mockUser = { + userId: "bench@example.com", + email: "bench@example.com", + name: "Bench User", + status: "Active", + userRoles: ["administrators"], + identityProvider: "aad", + settings: { itemsPerPage: 10 }, +}; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ bypassCSP: true }); + + // Crafted SWA auth cookie: base64(JSON(clientPrincipal)), no signing locally. + const cookieVal = Buffer.from(JSON.stringify(principal)).toString("base64"); + await context.addCookies([ + { name: "StaticWebAppsAuthCookie", value: cookieVal, domain: "localhost", path: "/" }, + ]); + + const page = await context.newPage(); + const consoleErrors = []; + page.on("console", (m) => { if (m.type() === "error") consoleErrors.push(m.text()); }); + + // Mock cheap bootstrap calls (not what we measure). + await context.route("**/.auth/me", (r) => + r.fulfill({ contentType: "application/json", body: JSON.stringify({ clientPrincipal: principal }) }) + ); + await context.route("**/GetUserById**", (r) => + r.fulfill({ contentType: "application/json", body: JSON.stringify(mockUser) }) + ); + await context.route("**/PutUser**", (r) => + r.fulfill({ contentType: "application/json", body: JSON.stringify(mockUser) }) + ); + + // Time every GetProjectDetails call (the real, expensive one). + const gpd = []; + const starts = new Map(); + page.on("request", (req) => { + if (req.url().includes("GetProjectDetails")) starts.set(req.url() + req.method() + Date.now(), Date.now()); + }); + page.on("requestfinished", async (req) => { + if (!req.url().includes("GetProjectDetails")) return; + const t = req.timing(); + // responseEnd is ms since request start (fetchStart); use it as duration. + gpd.push({ url: req.url(), ms: Math.round(t.responseEnd) }); + }); + + const observedApiOrigins = new Set(); + page.on("request", (req) => { + const u = req.url(); + if (u.includes("/api/GetProjectDetails")) observedApiOrigins.add(new URL(u).origin); + }); + + const t0 = Date.now(); + await page.goto(`${UI}/project/${PROJECT}`, { waitUntil: "commit", timeout: 60000 }); + + // TTI: first image-layer row (seed names layers "Layer "). + const rowTimeout = parseInt(arg("rowtimeout", "90000"), 10); + let tti = null, rowError = null; + try { + await page.waitForFunction( + () => !!document.body && /Layer \d+/.test(document.body.innerText), + null, + { timeout: rowTimeout, polling: 50 } + ); + tti = Date.now() - t0; + } catch (e) { + rowError = String(e).split("\n")[0]; + } + + let bodyText = null; + try { + bodyText = (await page.locator("body").innerText()).replace(/\s+/g, " ").slice(0, 600); + } catch (e) { bodyText = ""; } + try { await page.screenshot({ path: arg("shot", "/tmp/haste-uibench/shot.png"), fullPage: true }); } catch (e) {} + + const initialGpdMs = gpd.length ? gpd[0].ms : null; + const gpdCountAfterLoad = gpd.length; + + // Observe the 20s background poll. + const pollStart = Date.now(); + await page.waitForTimeout(POLL_WAIT_MS); + const pollCalls = gpd.slice(gpdCountAfterLoad); + + const result = { + project: PROJECT, + time_to_interactive_ms: tti, + initial_getprojectdetails_ms: initialGpdMs, + getprojectdetails_calls_during_load: gpdCountAfterLoad, + poll_window_ms: POLL_WAIT_MS, + poll_getprojectdetails_calls: pollCalls.length, + poll_getprojectdetails_ms: pollCalls.map((c) => c.ms), + api_origins_observed: [...observedApiOrigins], + row_wait_error: rowError, + body_text_sample: bodyText, + console_errors: consoleErrors.slice(0, 4), + }; + console.log(JSON.stringify(result, null, 2)); + + await browser.close(); +})().catch((e) => { console.error("FATAL", e); process.exit(1); });