diff --git a/api/hastefuncapi/README.md b/api/hastefuncapi/README.md index 909efe18..cfd5bac2 100644 --- a/api/hastefuncapi/README.md +++ b/api/hastefuncapi/README.md @@ -21,11 +21,22 @@ All functions are defined in `function_app.py` as a single Azure Functions app. |--------|-------|-------------| | GET | `GetDashboardData` | Aggregated dashboard stats: project summaries, layer info, model status, and system-wide metrics. | | GET | `GetProjects` | All projects with aggregated layer and model counts. | -| GET | `GetProjectDetails` | Full project details including image layers, models, and processing status. Requires `projectId`. | +| GET | `GetProjectDetails` | Project, layer, validation, and optional model details. Supports `ETag`/`If-None-Match`; requires `projectId`. | | PUT | `PutProject` | Create or update a project. Auto-generates `projectId` and `creationDate` if not provided. | | DELETE | `DeleteProject` | Delete a project by `projectId`. | | GET | `GenerateProjectStats` | Regenerates project stats from raw data — useful if stats fall out of sync. | +#### Project Detail Caching + +`GetProjectDetails` returns `ETag`, `Cache-Control`, and `X-Haste-Cache` headers. +Send `If-None-Match` to receive an empty `304` when the fresh cached representation is +unchanged. Send `Cache-Control: no-cache` after a mutation to force storage refresh and +then compare the ETag. + +The response cache is bounded and process-local. It deduplicates concurrent requests but +does not provide coherence across scaled-out Function workers. Performance headers named +`X-Haste-Data-Layer-*` count logical calls, not Azure Storage REST transactions. + ### Image Layers | Method | Route | Description | diff --git a/api/hastefuncapi/function_app.py b/api/hastefuncapi/function_app.py index 707cd6c3..39fa3bdc 100644 --- a/api/hastefuncapi/function_app.py +++ b/api/hastefuncapi/function_app.py @@ -4,10 +4,12 @@ import asyncio import base64 import binascii +import hashlib import json import os import re import tempfile +import time import traceback import azure.functions as func # type: ignore @@ -43,6 +45,7 @@ from hastegeo.core.processors.imagery import ImageryPreProcessor from hastegeo.core.processors.inference import InferencePreprocessor from hastegeo.core.processors.metadata import MetadataProcessor +from hastegeo.core.processors.project_details import ProjectDetailsProcessor from hastegeo.core.processors.publishing import ( PublishingDependencyError, PublishingDisabledError, @@ -72,6 +75,11 @@ PublishingSourceNotFoundError, PublishingSourceResolver, ) +from hastegeo.core.utils import perf +from hastegeo.core.utils.async_cache import ( + AsyncTTLCache, + configured_cache_value, +) from hastegeo.core.utils.blob import ( download_blob_to_tempfile, parse_byte_range, @@ -104,6 +112,17 @@ ) app = func.FunctionApp() +_PROJECT_DETAILS_CACHE_SECONDS = configured_cache_value( + "HASTE_PROJECTDETAILS_CACHE_SECONDS", 15, 0, 300 +) +_PROJECT_DETAILS_CACHE_ENTRIES = configured_cache_value( + "HASTE_PROJECTDETAILS_CACHE_ENTRIES", 64, 1, 512 +) +_project_details_cache = AsyncTTLCache( + ttl_seconds=_PROJECT_DETAILS_CACHE_SECONDS, + max_entries=_PROJECT_DETAILS_CACHE_ENTRIES, +) + # Development mode check - when running locally with Docker/Azurite # Set DEVELOPMENT_MODE=true to disable function key authentication DEVELOPMENT_MODE = ( @@ -140,6 +159,25 @@ def _require_guid_param(req: func.HttpRequest, name: str) -> str: return value +def _etag_matches(if_none_match: str | None, etag: str) -> bool: + if not if_none_match: + return False + for candidate in if_none_match.split(","): + candidate = candidate.strip() + if candidate == "*" or candidate.removeprefix("W/") == etag: + return True + return False + + +def _cache_refresh_requested(cache_control: str | None) -> bool: + if not cache_control: + return False + directives = { + directive.strip().lower() for directive in cache_control.split(",") + } + return "no-cache" in directives or "max-age=0" in directives + + def _require_short_int_id_param(req: func.HttpRequest, name: str) -> str: """Return a request parameter validated as a short integer id, or raise ValueError. @@ -735,116 +773,69 @@ async def GetProjectDetails(req: func.HttpRequest) -> func.HttpResponse: f"GetProjectDetails HTTP trigger function processed a request for project id: {project_id} with includeModels: {include_models}" ) - project = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().PROJECT.value, - partition_key=project_id, - ).load, - project_id, - ) - image_layers = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().IMAGELAYER.value, - partition_key=project_id, - ).load_all_from_partition - ) - if include_models: - models = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().MODEL.value, - partition_key=project_id, - ).load_all_from_partition - ) - for image_layer in image_layers: - image_layer_id = image_layer["imageLayerId"] - if include_models: - match_models = [ - model - for model in models - if model["imageLayerId"] == image_layer_id - ] - match_models.sort( - key=lambda x: x["creationDate"], reverse=True - ) - for model in match_models: - try: - artifacts = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().MODEL_ARTIFACTS.value, - partition_key=project_id, - ).load, - model["modelId"], - ) - model["artifacts"] = artifacts - except FileNotFoundError: - model["artifacts"] = None - try: - if not model.get("labelsUrl"): - # Older models may not have labelsUrl - model["labelsUrl"] = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().TRAIN_LABELS.value, - partition_key=project_id, - ).export, - key=model["modelId"], - data_format="geojson", - ) - except FileNotFoundError: - # This is a noop until the export method is properly - # implemented - model["labelsUrl"] = None - image_layer["models"] = match_models - image_layer["modelCount"] = len(match_models) - label_projects = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().LABELS.value, - partition_key=project_id, - ).load_all_from_partition - ) - match_label_projects = next( - ( - label_project - for label_project in label_projects - if label_project["imageLayerId"] == image_layer_id - ), - None, - ) - if match_label_projects is not None: - if ( - "labels" in match_label_projects - and match_label_projects["labels"] is not None - ): - image_layer["labelProjectCount"] = len( - match_label_projects["labels"] - ) - else: - image_layer["labelProjectCount"] = 0 - if not image_layer.get("labelsUrl"): - # Older image layers will not have the generated geoJSON - image_layer["labelsUrl"] = None - try: - validation_data = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().VALIDATION.value, - partition_key=project_id, - ).load, - image_layer_id, - ) - labels = validation_data.get("labels") or {} - image_layer["validationLabelCount"] = len(labels) - except FileNotFoundError: - image_layer["validationLabelCount"] = 0 - project["imageLayer"] = image_layers - project["imageLayerCount"] = len(image_layers) - project["imageLayer"].sort( - key=lambda x: x["creationDate"], reverse=True + # Phase 0 baseline instrumentation (spec/features/perf-layer-loading). + # Opt-in via HASTE_PERF=true; zero overhead when disabled. + _perf_on = os.environ.get("HASTE_PERF", "false").lower() == "true" + _perf = perf.begin(_perf_on) + _perf_wall = time.perf_counter() + + async def _load_response(): + project = await ProjectDetailsProcessor( + project_id=project_id, config=config + ).load(include_models=include_models) + payload = json.dumps(project) + return { + "payload": payload, + "etag": '"' + + hashlib.sha256(payload.encode()).hexdigest()[:32] + + '"', + "layer_count": len(project["imageLayer"]), + } + + cache_key = (project_id, include_models) + ( + cached_response, + cache_hit, + ) = await _project_details_cache.get_or_create( + cache_key, + _load_response, + refresh=_cache_refresh_requested(req.headers.get("Cache-Control")), + ) + _payload = cached_response["payload"] + etag = cached_response["etag"] + _perf_headers = perf.headers(_perf, _perf_wall) + perf.log_summary( + logger, + "GetProjectDetails", + _perf, + _perf_wall, + project_id=project_id, + include_models=include_models, + layers=cached_response["layer_count"], + payload_bytes=len(_payload), + cache_hit=cache_hit, + ) + cache_headers = { + "Cache-Control": ( + f"private, max-age={_PROJECT_DETAILS_CACHE_SECONDS}" + ), + "ETag": etag, + "X-Haste-Cache": "HIT" if cache_hit else "MISS", + } + if _perf_headers: + cache_headers.update(_perf_headers) + if _etag_matches(req.headers.get("If-None-Match"), etag): + return func.HttpResponse(status_code=304, headers=cache_headers) + return func.HttpResponse( + _payload, status_code=200, headers=cache_headers ) - return func.HttpResponse(json.dumps(project), status_code=200) except FileNotFoundError as e: + perf.end() logger.error(f"Project not found: {e}\n{traceback.format_exc()}") return func.HttpResponse("Project not found.", status_code=404) except Exception as e: + perf.end() logger.error( f"Error loading project details: {e}\n{traceback.format_exc()}" ) @@ -1470,9 +1461,9 @@ async def GetModelArtifact(req: func.HttpRequest) -> func.HttpResponse: # interactive labeler's other artifacts are fetched by range and parsed # in-browser, so they must NOT be forced as downloads). if kind == "gpkg": - headers[ - "Content-Disposition" - ] = f'attachment; filename="building_predictions_{model_id}.gpkg"' + headers["Content-Disposition"] = "; ".join( + ["attachment", f'filename="building_predictions_{model_id}.gpkg"'] + ) if result.etag: headers["ETag"] = ( result.etag if result.etag.startswith('"') else f'"{result.etag}"' @@ -2828,6 +2819,19 @@ async def GenerateProjectStats(req: func.HttpRequest) -> func.HttpResponse: partition_key=project_id, ).load_all_from_partition ) + # PERF (Phase 1, B5): load LABELS once per project and index by + # imageLayerId, instead of a full partition scan per image layer. + label_projects = await asyncio.to_thread( + MetadataProcessor( + data_type=config.get_metadata_types().LABELS.value, + partition_key=project_id, + ).load_all_from_partition + ) + labels_by_layer = {} + for label_project in label_projects: + lid = label_project.get("imageLayerId") + if lid is not None and lid not in labels_by_layer: + labels_by_layer[lid] = label_project for image_layer in image_layers: image_layer_id = image_layer["imageLayerId"] match_models = [ @@ -2837,20 +2841,8 @@ async def GenerateProjectStats(req: func.HttpRequest) -> func.HttpResponse: ] image_layer["models"] = match_models image_layer["modelCount"] = len(match_models) - label_projects = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().LABELS.value, - partition_key=project_id, - ).load_all_from_partition - ) - match_label_projects = next( - ( - label_project - for label_project in label_projects - if label_project["imageLayerId"] == image_layer_id - ), - None, - ) + image_layer["labelProjectCount"] = 0 + match_label_projects = labels_by_layer.get(image_layer_id) if match_label_projects is not None: if ( "labels" in match_label_projects diff --git a/api/hastefuncapi/tests/test_generate_project_stats.py b/api/hastefuncapi/tests/test_generate_project_stats.py new file mode 100644 index 00000000..e0d10379 --- /dev/null +++ b/api/hastefuncapi/tests/test_generate_project_stats.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import io +import json +import os +import unittest +from contextlib import redirect_stderr +from unittest.mock import Mock, patch + +import azure.functions as func + +os.environ.setdefault("DEVELOPMENT_MODE", "true") +os.environ.setdefault("METADATA_STORAGE_TYPE", "local") +os.environ.setdefault("ARTIFACT_STORAGE_TYPE", "local") +os.environ.setdefault("DATA_PATH", "/tmp/haste-project-stats-api-tests") +os.environ.setdefault("TEMP_DATA_PATH", "/tmp/haste-project-stats-api-tests") + +with redirect_stderr(io.StringIO()): + from api.hastefuncapi import function_app + + +class TestGenerateProjectStats(unittest.IsolatedAsyncioTestCase): + async def test_unlabeled_layer_counts_zero_and_labels_load_once( + self, + ) -> None: + types = function_app.config.get_metadata_types() + processors = { + (types.PROJECT.value, None): Mock(), + (types.IMAGELAYER.value, "project-1"): Mock(), + (types.MODEL.value, "project-1"): Mock(), + (types.LABELS.value, "project-1"): Mock(), + } + processors[(types.PROJECT.value, None)].load_all.return_value = [ + { + "projectId": "project-1", + "name": "Project", + "description": "Description", + "creationDate": "2026-01-01T00:00:00Z", + "affectedCountries": [], + } + ] + processors[ + (types.IMAGELAYER.value, "project-1") + ].load_all_from_partition.return_value = [ + {"imageLayerId": "labeled"}, + {"imageLayerId": "unlabeled"}, + ] + processors[ + (types.MODEL.value, "project-1") + ].load_all_from_partition.return_value = [] + labels = processors[(types.LABELS.value, "project-1")] + labels.load_all_from_partition.return_value = [ + {"imageLayerId": "labeled", "labels": [{"id": "label-1"}]} + ] + + def processor_factory(*, data_type, partition_key=None): + return processors[(data_type, partition_key)] + + request = func.HttpRequest( + method="GET", + url="http://localhost/api/GenerateProjectStats", + headers={}, + params={}, + route_params={}, + body=b"", + ) + with patch.object( + function_app, "MetadataProcessor", side_effect=processor_factory + ): + response = await function_app.GenerateProjectStats(request) + + self.assertEqual(response.status_code, 200) + project = json.loads(response.get_body())["projects"][0] + self.assertEqual(project["labelsCount"], 1) + self.assertEqual( + project["imageLayerStats"], + [ + {"imageLayerId": "labeled", "labelsCount": 1}, + {"imageLayerId": "unlabeled", "labelsCount": 0}, + ], + ) + labels.load_all_from_partition.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/api/hastefuncapi/tests/test_model_artifact_route.py b/api/hastefuncapi/tests/test_model_artifact_route.py new file mode 100644 index 00000000..d0a82ff4 --- /dev/null +++ b/api/hastefuncapi/tests/test_model_artifact_route.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import io +import os +import unittest +from contextlib import redirect_stderr +from unittest.mock import AsyncMock, Mock, patch + +import azure.functions as func +from hastegeo.core.utils.blob import BlobRange + +os.environ.setdefault("DEVELOPMENT_MODE", "true") +os.environ.setdefault("METADATA_STORAGE_TYPE", "local") +os.environ.setdefault("ARTIFACT_STORAGE_TYPE", "local") +os.environ.setdefault("DATA_PATH", "/tmp/haste-model-artifact-api-tests") +os.environ.setdefault("TEMP_DATA_PATH", "/tmp/haste-model-artifact-api-tests") + +with redirect_stderr(io.StringIO()): + from api.hastefuncapi import function_app + +PROJECT_ID = "123e4567-e89b-12d3-a456-426614174000" + + +class TestGetModelArtifact(unittest.IsolatedAsyncioTestCase): + async def test_gpkg_response_is_a_named_download(self) -> None: + request = func.HttpRequest( + method="GET", + url="http://localhost/api/GetModelArtifact", + headers={}, + params={ + "projectId": PROJECT_ID, + "modelId": "42", + "kind": "gpkg", + }, + route_params={}, + body=b"", + ) + processor = Mock() + processor.load.return_value = { + "gpkgUrl": "https://account.test/model.gpkg" + } + blob = BlobRange( + data=b"gpkg", + total_size=4, + content_type="application/octet-stream", + etag='"etag"', + ) + + with patch.object( + function_app, "MetadataProcessor", return_value=processor + ), patch.object( + function_app, + "read_blob_range", + new=AsyncMock(return_value=blob), + ): + response = await function_app.GetModelArtifact(request) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.headers["Content-Disposition"], + 'attachment; filename="building_predictions_42.gpkg"', + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/hastefuncapi/tests/test_project_details_route.py b/api/hastefuncapi/tests/test_project_details_route.py new file mode 100644 index 00000000..1d2c75c6 --- /dev/null +++ b/api/hastefuncapi/tests/test_project_details_route.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import io +import json +import os +import unittest +from contextlib import redirect_stderr +from unittest.mock import AsyncMock, patch + +import azure.functions as func +from hastegeo.core.utils.async_cache import AsyncTTLCache + +os.environ.setdefault("DEVELOPMENT_MODE", "true") +os.environ.setdefault("METADATA_STORAGE_TYPE", "local") +os.environ.setdefault("ARTIFACT_STORAGE_TYPE", "local") +os.environ.setdefault("DATA_PATH", "/tmp/haste-project-details-api-tests") +os.environ.setdefault("TEMP_DATA_PATH", "/tmp/haste-project-details-api-tests") + +with redirect_stderr(io.StringIO()): + from api.hastefuncapi import function_app + +PROJECT_ID = "123e4567-e89b-12d3-a456-426614174000" + + +def make_request( + *, include_models: str | None = None, headers: dict | None = None +) -> func.HttpRequest: + params = {"projectId": PROJECT_ID} + if include_models is not None: + params["includeModels"] = include_models + return func.HttpRequest( + method="GET", + url="http://localhost/api/GetProjectDetails", + headers=headers or {}, + params=params, + route_params={}, + body=b"", + ) + + +class TestGetProjectDetails(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.project = { + "projectId": PROJECT_ID, + "name": "Project", + "imageLayer": [], + "imageLayerCount": 0, + } + self.loader = AsyncMock() + self.loader.load.return_value = self.project + patcher = patch.object( + function_app, + "ProjectDetailsProcessor", + return_value=self.loader, + ) + self.addCleanup(patcher.stop) + self.processor_class = patcher.start() + cache_patcher = patch.object( + function_app, + "_project_details_cache", + AsyncTTLCache(ttl_seconds=15, max_entries=8), + ) + self.addCleanup(cache_patcher.stop) + cache_patcher.start() + + async def test_returns_project_and_cache_headers(self) -> None: + response = await function_app.GetProjectDetails( + make_request(include_models="True") + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.get_body()), self.project) + self.assertEqual( + response.headers["Cache-Control"], "private, max-age=15" + ) + self.assertEqual(response.headers["X-Haste-Cache"], "MISS") + self.assertTrue(response.headers["ETag"].startswith('"')) + self.processor_class.assert_called_once_with( + project_id=PROJECT_ID, config=function_app.config + ) + self.loader.load.assert_awaited_once_with(include_models=True) + + async def test_include_models_defaults_to_false(self) -> None: + response = await function_app.GetProjectDetails(make_request()) + + self.assertEqual(response.status_code, 200) + self.loader.load.assert_awaited_once_with(include_models=False) + + async def test_matching_etag_returns_empty_304(self) -> None: + first = await function_app.GetProjectDetails(make_request()) + etag = first.headers["ETag"] + self.loader.reset_mock() + + response = await function_app.GetProjectDetails( + make_request(headers={"If-None-Match": etag}) + ) + + self.assertEqual(response.status_code, 304) + self.assertEqual(response.get_body(), b"") + self.assertEqual(response.headers["ETag"], etag) + self.assertEqual(response.headers["X-Haste-Cache"], "HIT") + self.loader.load.assert_not_awaited() + + async def test_weak_etag_in_a_list_matches(self) -> None: + first = await function_app.GetProjectDetails(make_request()) + etag = first.headers["ETag"] + + response = await function_app.GetProjectDetails( + make_request(headers={"If-None-Match": f'"different", W/{etag}'}) + ) + + self.assertEqual(response.status_code, 304) + + async def test_include_models_uses_a_separate_cache_entry(self) -> None: + await function_app.GetProjectDetails(make_request()) + await function_app.GetProjectDetails( + make_request(include_models="true") + ) + + self.assertEqual(self.loader.load.await_count, 2) + self.loader.load.assert_any_await(include_models=False) + self.loader.load.assert_any_await(include_models=True) + + async def test_no_cache_request_refreshes_cached_response(self) -> None: + await function_app.GetProjectDetails(make_request()) + self.loader.load.return_value = dict(self.project, name="Updated") + + response = await function_app.GetProjectDetails( + make_request(headers={"Cache-Control": "no-cache"}) + ) + cached = await function_app.GetProjectDetails(make_request()) + + self.assertEqual(json.loads(response.get_body())["name"], "Updated") + self.assertEqual(json.loads(cached.get_body())["name"], "Updated") + self.assertEqual(self.loader.load.await_count, 2) + + def test_cache_refresh_directives(self) -> None: + self.assertTrue(function_app._cache_refresh_requested("no-cache")) + self.assertTrue( + function_app._cache_refresh_requested("public, max-age=0") + ) + self.assertFalse(function_app._cache_refresh_requested("max-age=15")) + + async def test_missing_project_returns_404(self) -> None: + self.loader.load.side_effect = FileNotFoundError("missing") + + first = await function_app.GetProjectDetails(make_request()) + second = await function_app.GetProjectDetails(make_request()) + + self.assertEqual(first.status_code, 404) + self.assertEqual(second.status_code, 404) + self.assertEqual(self.loader.load.await_count, 2) + + async def test_unexpected_error_returns_500(self) -> None: + self.loader.load.side_effect = RuntimeError("storage unavailable") + + response = await function_app.GetProjectDetails(make_request()) + + self.assertEqual(response.status_code, 500) + + async def test_invalid_project_id_returns_400_without_loading( + self, + ) -> None: + request = func.HttpRequest( + method="GET", + url="http://localhost/api/GetProjectDetails", + headers={}, + params={"projectId": "not-a-guid"}, + route_params={}, + body=b"", + ) + + response = await function_app.GetProjectDetails(request) + + self.assertEqual(response.status_code, 400) + self.processor_class.assert_not_called() + + def test_non_matching_etag_is_rejected(self) -> None: + self.assertFalse(function_app._etag_matches('"other"', '"expected"')) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/docker-compose.perf.yml b/docker/docker-compose.perf.yml new file mode 100644 index 00000000..1706a16a --- /dev/null +++ b/docker/docker-compose.perf.yml @@ -0,0 +1,20 @@ +# Perf-baseline overlay for the perf-layer-loading spec (Phase 0). +# +# Enables the opt-in HASTE_PERF instrumentation and bind-mounts the working-tree +# copies of the API handler + hastegeo library over the image's runtime paths, so +# the current branch's code runs without an image rebuild. +# +# Usage: +# docker compose -f docker/docker-compose.yml -f docker/docker-compose.perf.yml \ +# up -d hastefuncapi api-proxy +services: + hastefuncapi: + environment: + HASTE_PERF: "true" + volumes: + # hastegeo exists in two places (a site-packages install and a wwwroot + # copy); sys.path order varies by process, so overlay the working-tree + # source over BOTH to guarantee the worker imports the updated code. + - ../hastelib/src/hastegeo:/usr/local/lib/python3.11/site-packages/hastegeo:ro + - ../hastelib/src/hastegeo:/home/site/wwwroot/hastegeo:ro + - ../api/hastefuncapi/function_app.py:/home/site/wwwroot/function_app.py:ro diff --git a/docker/nginx.conf b/docker/nginx.conf index 1f159c50..5ab747cb 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -61,7 +61,7 @@ http { if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; - add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,If-None-Match,Keep-Alive,Origin,User-Agent,X-Requested-With' always; add_header 'Access-Control-Max-Age' 1728000 always; add_header 'Content-Type' 'text/plain; charset=utf-8' always; add_header 'Content-Length' 0 always; @@ -71,7 +71,8 @@ http { # Add CORS headers to all responses add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; - add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,If-None-Match,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + add_header 'Access-Control-Expose-Headers' 'ETag,Server-Timing,X-Haste-Cache,X-Haste-Data-Layer-Calls,X-Haste-Data-Layer-Ms' always; # Proxy to Azure Functions - preserve full URI proxy_pass http://functions_backend$request_uri; diff --git a/docs/api-overview.md b/docs/api-overview.md index 03dde3c5..105f3187 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -5,6 +5,15 @@ poison-queue handler) for managing disaster assessment projects, processing sate imagery, and running AI models for damage assessment. A separate TiTiler-based tile server handles geospatial imagery visualization. +## Contents + +- [Architecture](#architecture) +- [Authentication](#authentication) +- [Base URLs](#base-urls) +- [Response Formats](#response-formats) +- [Rate Limits](#rate-limits) +- [HTTP Status Codes](#http-status-codes) + ## Architecture The API layer consists of three Azure Functions apps: @@ -61,6 +70,17 @@ Some write endpoints (e.g. the Model Catalog) return a small status wrapper: { "success": true, "message": "…", "catalogModel": {} } ``` +### Conditional Project Details + +`GetProjectDetails` returns a bounded process-local cached representation with `ETag`, +`Cache-Control`, and `X-Haste-Cache` headers. Clients send `If-None-Match` for normal +polls; a matching fresh representation returns `304` without storage work or a body. +Clients send `Cache-Control: no-cache` after mutations to force storage refresh. + +The cache is per Functions worker and is not a distributed consistency mechanism. The +`X-Haste-Data-Layer-Calls` and `X-Haste-Data-Layer-Ms` headers describe logical library +operations; Azure Storage metrics remain the source for REST transaction counts. + ## Rate Limits API throughput is bounded by the underlying platform: diff --git a/docs/api/hastefuncapi.md b/docs/api/hastefuncapi.md index 2c207c75..70370881 100644 --- a/docs/api/hastefuncapi.md +++ b/docs/api/hastefuncapi.md @@ -2,6 +2,13 @@ Azure Functions backend for the HASTE application. Provides REST endpoints for managing projects, image layers, ML model training/inference, labeling, user management, and geospatial data access. +## Contents + +- [Overview](#overview) +- [Endpoints](#endpoints) +- [Development Setup](#development-setup) +- [Auto-generated API Docs](#auto-generated-api-docs) + --- ## Overview @@ -21,7 +28,7 @@ All functions are defined in `function_app.py` as a single Azure Functions app. |--------|-------|-------------| | GET | `GetDashboardData` | Aggregated dashboard stats: project summaries, layer info, model status, and system-wide metrics. | | GET | `GetProjects` | All projects with aggregated layer and model counts. | -| GET | `GetProjectDetails` | Full project details including image layers, models, and processing status. Requires `projectId`. | +| GET | `GetProjectDetails` | Project, layer, validation, and optional model details. Supports `ETag`/`If-None-Match`; requires `projectId`. | | PUT | `PutProject` | Create or update a project. Auto-generates `projectId` and `creationDate` if not provided. | | DELETE | `DeleteProject` | Delete a project by `projectId`. | | GET | `GenerateProjectStats` | Regenerates project stats from raw data — useful if stats fall out of sync. | diff --git a/docs/configuration.md b/docs/configuration.md index 4a2f9a21..4fd874a5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -12,6 +12,7 @@ This guide documents each configuration mode. For the end-to-end workflow, see ## Contents - [Core settings](#core-settings) +- [Project detail performance](#project-detail-performance) - [Batch (create vs. bring-your-own)](#batch-create-vs-bring-your-own) - [Batch image tags and pool immutability](#batch-image-tags-and-pool-immutability) - [hastegeo wheel pinning](#hastegeo-wheel-pinning) @@ -38,6 +39,24 @@ This guide documents each configuration mode. For the end-to-end workflow, see Resource names are `HASTE_RESOURCE_PREFIX` + `HASTE_RANDOM_SUFFIX` based (not azd's `resourceToken`) so `what-if` stays clean against existing deployments. +## Project detail performance + +The API and queue workers use the following optional runtime settings. Defaults are +validated at process startup, so invalid values fail fast instead of creating an +unbounded executor or cache. + +| Function App setting | Default | Allowed | Purpose | +|---|---:|---:|---| +| `HASTE_BLOB_DOWNLOAD_WORKERS` | 16 | 1–64 | Process-wide blocking Blob I/O budget. | +| `HASTE_METADATA_LOAD_WORKERS` | 8 | 1–64 | Per-map limit within the process budget. | +| `HASTE_ARTIFACT_DOWNLOAD_WORKERS` | 8 | 1–64 | Per-artifact limit within the process budget. | +| `HASTE_PROJECTDETAILS_CACHE_SECONDS` | 15 | 0–300 | Process-local response freshness and HTTP `max-age`. | +| `HASTE_PROJECTDETAILS_CACHE_ENTRIES` | 64 | 1–512 | Maximum cached project response variants per worker. | + +These controls have code defaults and are not yet exposed as Bicep parameters. Treat +IaC parameterization of non-default values as a separate deployment change. Load-test +worker-count overrides with concurrent requests before production rollout. + ## Batch (create vs. bring-your-own) HASTE runs GPU workloads on Azure Batch. Both the Batch **account** and the GPU diff --git a/hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py b/hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py index adefea4c..76e23ee7 100644 --- a/hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py +++ b/hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import json import os +import tempfile import time from datetime import datetime, timedelta, timezone from hashlib import sha256 @@ -12,17 +13,20 @@ import yaml from azure.core import MatchConditions from azure.core.exceptions import ResourceExistsError -from azure.identity import DefaultAzureCredential # type: ignore from azure.storage.blob import BlobClient # type: ignore from azure.storage.blob import ( AccessPolicy, BlobSasPermissions, - BlobServiceClient, ContainerSasPermissions, generate_blob_sas, generate_container_sas, ) +from hastegeo.core.utils.blob import ( + get_blob_service_client, + get_cached_user_delegation_key, +) from hastegeo.core.utils.logs import Logger +from hastegeo.core.utils.parallel import configured_worker_count, parallel_map from .abstract_artifact_storage import AbstractArtifactStorage @@ -48,34 +52,22 @@ def __init__( # which need Storage Blob Delegator / Owner. Write access alone suffices. self.serves_read_sas = serves_read_sas if connection_string: - credential = connection_string - self.blob_service_client = ( - BlobServiceClient.from_connection_string(connection_string) + self.blob_service_client = get_blob_service_client( + connection_string=connection_string ) self.user_delegation_key = None self.account_key = self.blob_service_client.credential.account_key self.identity_blob_service_client = ( - BlobServiceClient( - account_url=account_url, - credential=DefaultAzureCredential(), - ) + get_blob_service_client(account_url=account_url) if account_url and urlparse(account_url).scheme == "https" else None ) else: - credential = DefaultAzureCredential() - self.blob_service_client = BlobServiceClient( - account_url=account_url, credential=credential + self.blob_service_client = get_blob_service_client( + account_url=account_url ) self.identity_blob_service_client = self.blob_service_client - self.user_delegation_key = ( - self.blob_service_client.get_user_delegation_key( - datetime.now(timezone.utc), - datetime.now(timezone.utc) + timedelta(hours=1), - ) - if serves_read_sas - else None - ) + self.user_delegation_key = None self.account_key = None self.container_read_policy = container_read_policy_name @@ -97,7 +89,9 @@ def __init__( f"Container '{container}' created successfully." ) except ResourceExistsError: - self.logger.info(f"Container '{container}' already exists.") + self.logger.info( + f"Container '{container}' already exists." + ) if self.serves_read_sas: self._create_or_update_managed_access_policy() _INITIALIZED_CONTAINERS.add(cache_key) @@ -184,12 +178,18 @@ def get_download_url( # Otherwise generate SAS token + user_delegation_key = self.user_delegation_key + if self.account_key is None: + user_delegation_key = get_cached_user_delegation_key( + self.blob_service_client + ) + sas_token = generate_container_sas( account_name=self.container_client.account_name, container_name=self.container_client.container_name, policy_id=self.container_read_policy, account_key=self.account_key, - user_delegation_key=self.user_delegation_key, + user_delegation_key=user_delegation_key, ) return str(f"{blob_client.url}?{sas_token}") @@ -210,15 +210,45 @@ def fetch_artifact( src_path = self.get_file_path( identifier, extra_partition_keys=extra_partition_keys ) - blobs = self.container_client.list_blobs(name_starts_with=src_path) - for blob in blobs: - file_path = os.path.join(dst_path, blob.name) + if not src_path: + raise ValueError("A source artifact path is required") + if not dst_path: + raise ValueError("A destination path is required") + src_path = self.resolve_artifact_path(src_path) + blob_names = [ + blob.name + for blob in self.container_client.list_blobs( + name_starts_with=src_path + ) + ] + + def _download_one(blob_name): + relative_path = self.resolve_artifact_path(blob_name) + file_path = os.path.join( + os.path.abspath(dst_path), *PurePosixPath(relative_path).parts + ) os.makedirs(os.path.dirname(file_path), exist_ok=True) - blob_client = self.container_client.get_blob_client(blob.name) + blob_client = self.container_client.get_blob_client(blob_name) stream = blob_client.download_blob() - with open(file_path, "wb") as f: - for chunk in stream.chunks(): - f.write(chunk) + temp_path = None + try: + with tempfile.NamedTemporaryFile( + dir=os.path.dirname(file_path), delete=False + ) as temp_file: + temp_path = temp_file.name + for chunk in stream.chunks(): + temp_file.write(chunk) + os.replace(temp_path, file_path) + except Exception: + if temp_path and os.path.exists(temp_path): + os.unlink(temp_path) + raise + + if blob_names: + workers = configured_worker_count( + "HASTE_ARTIFACT_DOWNLOAD_WORKERS", 8 + ) + parallel_map(_download_one, blob_names, max_workers=workers) self.logger.info(f"Downloaded {src_path} to {dst_path}") return dst_path @@ -302,14 +332,22 @@ def resolve_artifact_path(self, location: str) -> str: if parsed.scheme: container_url = urlparse(self.container_client.url) if parsed.netloc.lower() != container_url.netloc.lower(): - raise ValueError("Artifact URL does not belong to configured storage") + raise ValueError( + "Artifact URL does not belong to configured storage" + ) container_path = container_url.path.rstrip("/") + "/" if not parsed.path.startswith(container_path): - raise ValueError("Artifact URL does not belong to configured container") + raise ValueError( + "Artifact URL does not belong to configured container" + ) location = unquote(parsed.path[len(container_path) :]) normalized = str(PurePosixPath(location.lstrip("/"))) - if not normalized or normalized == "." or ".." in PurePosixPath(normalized).parts: + if ( + not normalized + or normalized == "." + or ".." in PurePosixPath(normalized).parts + ): raise ValueError("Invalid artifact path") return normalized @@ -440,11 +478,8 @@ def get_scoped_download_url( ) account_key = None if account_key is None: - user_delegation_key = ( - delegation_client.get_user_delegation_key( - now - timedelta(minutes=5), - expiry + timedelta(minutes=5), - ) + user_delegation_key = get_cached_user_delegation_key( + delegation_client, now=now ) sas_token = generate_blob_sas( diff --git a/hastelib/src/hastegeo/core/data_layer/abstract_data_layer.py b/hastelib/src/hastegeo/core/data_layer/abstract_data_layer.py index b6f743fb..92ad312b 100644 --- a/hastelib/src/hastegeo/core/data_layer/abstract_data_layer.py +++ b/hastelib/src/hastegeo/core/data_layer/abstract_data_layer.py @@ -184,14 +184,57 @@ def load_all_from_partition(self, data_type, data_format="json"): """ pass - def load_bounded( - self, data_type, max_records, data_format="json" - ): + def load_bounded(self, data_type, max_records, data_format="json"): """Load no more than ``max_records`` or fail before full materialization.""" raise NotImplementedError( f"{self.__class__.__name__} does not support bounded reads" ) + def load_map( + self, + identifiers, + data_type, + data_format="json", + max_workers=None, + ): + """Load keyed records in one backend-native operation when supported.""" + raise NotImplementedError( + f"{self.__class__.__name__} does not implement batch reads" + ) + + def list_identifiers(self, data_type, data_format="json"): + """List the identifiers of records of a type in the current partition. + + Unlike :meth:`load_all_from_partition`, this returns only the keys and + does not download record contents — a cheap way to test existence in + bulk. Not abstract so existing backends keep working; concrete backends + that support cheap listing (blob, local filesystem) override it. + + Args: + data_type (str): Type/category of the data to list. + data_format (str, optional): File format of the records. Defaults + to "json". + + Returns: + List[str]: Identifiers present in the current partition. + """ + raise NotImplementedError( + f"{self.__class__.__name__} does not implement list_identifiers" + ) + + def get_file_remote_path( + self, + identifier=None, + data_type=None, + data_format="json", + extra_partition_keys=None, + check_exists=True, + ): + """Build a remotely accessible path when the backend supports one.""" + raise NotImplementedError( + f"{self.__class__.__name__} does not expose remote file paths" + ) + @abstractmethod def delete(self, identifier, data_type, data_format="json"): """Delete a specific data record from the storage backend. diff --git a/hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py b/hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py index 582bcc2c..4876608b 100644 --- a/hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py +++ b/hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py @@ -4,21 +4,21 @@ import json import logging import os -from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone from threading import Lock import yaml from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError -from azure.identity import DefaultAzureCredential # type: ignore from azure.storage.blob import AccessPolicy # type: ignore from azure.storage.blob import BlobBlock # type: ignore -from azure.storage.blob import ( - BlobServiceClient, - ContainerSasPermissions, - generate_container_sas, -) +from azure.storage.blob import ContainerSasPermissions, generate_container_sas +from ..utils.blob import ( + get_blob_service_client, + get_cached_user_delegation_key, +) +from ..utils.metadata import matches_metadata_type +from ..utils.parallel import parallel_map from .abstract_data_layer import AbstractDataLayer _INITIALIZED_CONTAINERS = set() @@ -36,23 +36,16 @@ def __init__( ): super().__init__(partition_key) if connection_string: - credential = connection_string - self.blob_service_client = ( - BlobServiceClient.from_connection_string(connection_string) + self.blob_service_client = get_blob_service_client( + connection_string=connection_string ) self.user_delegation_key = None self.account_key = self.blob_service_client.credential.account_key else: - credential = DefaultAzureCredential() - self.blob_service_client = BlobServiceClient( - account_url=account_url, credential=credential - ) - self.user_delegation_key = ( - self.blob_service_client.get_user_delegation_key( - datetime.now(timezone.utc), - datetime.now(timezone.utc) + timedelta(hours=1), - ) + self.blob_service_client = get_blob_service_client( + account_url=account_url ) + self.user_delegation_key = None self.account_key = None self.container_read_policy = container_read_policy_name @@ -141,6 +134,7 @@ def get_file_remote_path( data_type=None, data_format="json", extra_partition_keys=None, + check_exists=True, ): blob_name = self.get_file_path( identifier, @@ -150,13 +144,22 @@ def get_file_remote_path( ) blob_client = self.container_client.get_blob_client(blob_name) - if blob_client.exists(): + # ``check_exists=False`` skips the per-blob existence HEAD request. Use + # it when the caller has already confirmed presence in bulk (e.g. via + # ``list_identifiers``) — the container read SAS is identical for every + # blob, so building the URL is a purely local operation. + if not check_exists or blob_client.exists(): + user_delegation_key = self.user_delegation_key + if self.account_key is None: + user_delegation_key = get_cached_user_delegation_key( + self.blob_service_client + ) # Generate SAS token with the policy sas_token = generate_container_sas( account_name=self.container_client.account_name, container_name=self.container_client.container_name, policy_id=self.container_read_policy, - user_delegation_key=self.user_delegation_key, + user_delegation_key=user_delegation_key, account_key=self.account_key, ) @@ -276,67 +279,64 @@ def load(self, identifier, data_type, data_format="json"): return contents elif data_format == "yaml": return yaml.safe_load(downloader.readall()) - except Exception as e: + raise ValueError(f"Unsupported data_format: {data_format}") + except ResourceNotFoundError as error: raise FileNotFoundError( f"{self.__class__.__name__}.load: No data found for identifier: {identifier} and data_type: {data_type}" - ) from e + ) from error def load_all(self, data_type, data_format="json"): - data = [] + # Collect the matching blobs first (a cheap listing pass), then download + # + deserialize them concurrently. + matched = [] + + def matches(blob_name): + in_partition = not self.partition_key or blob_name.startswith( + f"{self.partition_key}/" + ) + return in_partition and matches_metadata_type(blob_name, data_type) + blobs = self.container_client.walk_blobs() for blob in blobs: - logging.info(f"Blob name: {blob.name}") # Ignore stats file if "stats" in blob.name: continue # Check if the blob is a directory if blob.name.endswith("/"): - logging.info(f"Blob is a directory: {blob.name}") sub_blobs = self.container_client.walk_blobs( name_starts_with=blob.name ) for sub_blob in sub_blobs: - logging.info(f"SubBlob name: {sub_blob.name}") - if ( - sub_blob.name.startswith( - f"{self.partition_key}/{data_type}_" - ) - if self.partition_key - else sub_blob.name.startswith( - f"{blob.name}{data_type}_" - ) - ): - sub_blob_client = ( - self.container_client.get_blob_client(sub_blob) - ) - downloader = sub_blob_client.download_blob() - if data_format == "json": - contents = json.loads(downloader.readall()) - if isinstance(contents, str): - # Need to do this conversion again. TODO: Investigate why image_layers needs this converted twice - # but models does not - contents = json.loads(contents) - data.append(contents) - elif data_format == "yaml": - data.append(yaml.safe_load(downloader.readall())) + if matches(sub_blob.name): + matched.append(sub_blob) else: - if ( - blob.name.startswith(f"{self.partition_key}/{data_type}_") - if self.partition_key - else blob.name.startswith(f"{data_type}_") - ): - blob_client = self.container_client.get_blob_client(blob) - downloader = blob_client.download_blob() - if data_format == "json": - contents = json.loads(downloader.readall()) - if isinstance(contents, str): - # Need to do this conversion again. TODO: Investigate why image_layers needs this converted twice - # but models does not - contents = json.loads(contents) - data.append(contents) - elif data_format == "yaml": - data.append(yaml.safe_load(downloader.readall())) - return data + if matches(blob.name): + matched.append(blob) + return self._read_blobs_parallel(matched, data_format) + + def _read_blob_content(self, blob, data_format): + """Download and deserialize a single blob's content. + + Tolerates records that were double-serialized on save (parse again when + the first parse yields a string) so both legacy and current blobs read + correctly. + """ + blob_client = self.container_client.get_blob_client(blob) + raw = blob_client.download_blob().readall() + if data_format == "json": + contents = json.loads(raw) + if isinstance(contents, str): + contents = json.loads(contents) + return contents + elif data_format == "yaml": + return yaml.safe_load(raw) + raise ValueError(f"Unsupported data_format: {data_format}") + + def _read_blobs_parallel(self, blobs, data_format): + """Download+parse a list of blobs concurrently, preserving order.""" + return parallel_map( + lambda blob: self._read_blob_content(blob, data_format), blobs + ) def load_all_from_partition(self, data_type, data_format="json"): if not self.partition_key: @@ -344,23 +344,36 @@ def load_all_from_partition(self, data_type, data_format="json"): f"{self.__class__.__name__}.load_all_from_partition: Partition key is not set." ) - data = [] blobs = self.container_client.walk_blobs( name_starts_with=f"{self.partition_key}/{data_type}_" ) - for blob in blobs: - blob_client = self.container_client.get_blob_client(blob) - downloader = blob_client.download_blob() - if data_format == "json": - contents = json.loads(downloader.readall()) - if isinstance(contents, str): - # Need to do this conversion again. TODO: Investigate why image_layers needs this converted twice - # but models does not - contents = json.loads(contents) - data.append(contents) - elif data_format == "yaml": - data.append(yaml.safe_load(downloader.readall())) - return data + matching_blobs = ( + blob + for blob in blobs + if matches_metadata_type(blob.name, data_type) + ) + return self._read_blobs_parallel(matching_blobs, data_format) + + def load_map( + self, + identifiers, + data_type, + data_format="json", + max_workers=None, + ): + identifiers = list(dict.fromkeys(identifiers)) + + def load_one(identifier): + try: + return identifier, self.load( + identifier, data_type, data_format=data_format + ) + except FileNotFoundError: + return identifier, None + + return dict( + parallel_map(load_one, identifiers, max_workers=max_workers) + ) def load_bounded(self, data_type, max_records, data_format="json"): records, _ = self.load_page( @@ -415,12 +428,7 @@ def load_page( parts = blob.name.split("/") if "stats" in blob.name or len(parts) > 2: continue - if self.partition_key: - matches = blob.name.startswith( - f"{self.partition_key}/{data_type}_" - ) - else: - matches = parts[-1].startswith(f"{data_type}_") + matches = matches_metadata_type(blob.name, data_type) if not matches or not blob.name.endswith(f".{data_format}"): continue catalog_record_count += 1 @@ -471,6 +479,8 @@ def _index_metadata(data_type, data): } def _load_blob_names(self, blob_names, data_format): + """Like :meth:`_read_blobs_parallel`, but tolerates blobs deleted + between listing and download (they are dropped from the result).""" if not blob_names: return [] @@ -478,26 +488,30 @@ def _load_blob_names(self, blob_names, data_format): def load_blob(blob_name): try: - downloader = self.container_client.get_blob_client( - blob_name - ).download_blob() + return self._read_blob_content(blob_name, data_format) except ResourceNotFoundError: return missing_blob - contents = downloader.readall() - if data_format == "json": - contents = json.loads(contents) - if isinstance(contents, str): - contents = json.loads(contents) - return contents - if data_format == "yaml": - return yaml.safe_load(contents) - raise ValueError(f"Unsupported data format: {data_format}") - workers = min(32, len(blob_names)) - with ThreadPoolExecutor(max_workers=workers) as executor: - records = list(executor.map(load_blob, blob_names)) + records = parallel_map(load_blob, blob_names) return [record for record in records if record is not missing_blob] + def list_identifiers(self, data_type, data_format="json"): + if not self.partition_key: + raise ValueError( + f"{self.__class__.__name__}.list_identifiers: Partition key is not set." + ) + prefix = f"{self.partition_key}/{data_type}_" + suffix = f".{data_format}" + identifiers = [] + for name in self.container_client.list_blob_names( + name_starts_with=prefix + ): + if name.endswith(suffix) and matches_metadata_type( + name, data_type + ): + identifiers.append(name[len(prefix) : -len(suffix)]) + return identifiers + def delete(self, identifier, data_type, data_format="json"): blob_name = self.get_file_path(identifier, data_type, data_format) blob_client = self.container_client.get_blob_client(blob_name) diff --git a/hastelib/src/hastegeo/core/data_layer/azure_cosmos_db_data_layer.py b/hastelib/src/hastegeo/core/data_layer/azure_cosmos_db_data_layer.py index 29aeba16..4c144216 100644 --- a/hastelib/src/hastegeo/core/data_layer/azure_cosmos_db_data_layer.py +++ b/hastelib/src/hastegeo/core/data_layer/azure_cosmos_db_data_layer.py @@ -5,6 +5,7 @@ from azure.cosmos import CosmosClient, exceptions # type: ignore from azure.identity import DefaultAzureCredential # type: ignore +from ..utils.metadata import matches_metadata_type from .abstract_data_layer import AbstractDataLayer @@ -77,7 +78,9 @@ def finalize_save( "Method not implemented and supported for Azure Cosmos DB." ) - def load(self, identifier, data_type): + def load(self, identifier, data_type, data_format="json"): + if data_format != "json": + raise ValueError("Cosmos DB metadata supports only json") partition_key = ( self.partition_key if self.partition_key else identifier ) @@ -91,7 +94,9 @@ def load(self, identifier, data_type): f"No data found for identifier: {identifier} and data_type: {data_type}" ) - def load_all(self, data_type): + def load_all(self, data_type, data_format="json"): + if data_format != "json": + raise ValueError("Cosmos DB metadata supports only json") id_prefix = self._id_prefix(data_type) query = "SELECT * FROM c WHERE STARTSWITH(c.id, @id_prefix)" items = list( @@ -101,9 +106,15 @@ def load_all(self, data_type): enable_cross_partition_query=True, ) ) - return items + return [ + item + for item in items + if matches_metadata_type(item["id"], data_type) + ] - def load_all_from_partition(self, data_type): + def load_all_from_partition(self, data_type, data_format="json"): + if data_format != "json": + raise ValueError("Cosmos DB metadata supports only json") id_prefix = self._id_prefix(data_type) query = ( "SELECT * FROM c WHERE c.partition_key = @partition_key " @@ -123,7 +134,67 @@ def load_all_from_partition(self, data_type): partition_key=self.partition_key, ) ) - return items + return [ + item + for item in items + if matches_metadata_type(item["id"], data_type) + ] + + def list_identifiers(self, data_type, data_format="json"): + if data_format != "json": + return [] + id_prefix = self._id_prefix(data_type) + query = ( + "SELECT VALUE c.id FROM c WHERE c.partition_key = @partition_key " + "AND STARTSWITH(c.id, @id_prefix)" + ) + item_ids = self.container.query_items( + query=query, + parameters=[ + {"name": "@partition_key", "value": self.partition_key}, + {"name": "@id_prefix", "value": id_prefix}, + ], + enable_cross_partition_query=False, + partition_key=self.partition_key, + ) + return [ + item_id[len(id_prefix) :] + for item_id in item_ids + if matches_metadata_type(item_id, data_type) + ] + + def load_map( + self, + identifiers, + data_type, + data_format="json", + max_workers=None, + ): + if data_format != "json": + raise ValueError("Cosmos DB metadata supports only json") + identifiers = list(dict.fromkeys(identifiers)) + if not identifiers: + return {} + id_prefix = self._id_prefix(data_type) + item_ids = [f"{id_prefix}{identifier}" for identifier in identifiers] + query = ( + "SELECT * FROM c WHERE c.partition_key = @partition_key " + "AND ARRAY_CONTAINS(@item_ids, c.id)" + ) + items = self.container.query_items( + query=query, + parameters=[ + {"name": "@partition_key", "value": self.partition_key}, + {"name": "@item_ids", "value": item_ids}, + ], + enable_cross_partition_query=False, + partition_key=self.partition_key, + ) + by_identifier = {item["id"][len(id_prefix) :]: item for item in items} + return { + identifier: by_identifier.get(identifier) + for identifier in identifiers + } def load_bounded(self, data_type, max_records, data_format="json"): if ( diff --git a/hastelib/src/hastegeo/core/data_layer/azure_data_lake_data_layer.py b/hastelib/src/hastegeo/core/data_layer/azure_data_lake_data_layer.py index 5cf4cfa7..1ee5e2e3 100644 --- a/hastelib/src/hastegeo/core/data_layer/azure_data_lake_data_layer.py +++ b/hastelib/src/hastegeo/core/data_layer/azure_data_lake_data_layer.py @@ -5,6 +5,7 @@ from azure.identity import DefaultAzureCredential # type: ignore from azure.storage.filedatalake import DataLakeServiceClient # type: ignore +from ..utils.metadata import matches_metadata_type from .abstract_data_layer import AbstractDataLayer @@ -49,11 +50,14 @@ def get_file_remote_path( data_type=None, data_format="json", extra_partition_keys=None, + check_exists=True, ): file_name = self.get_file_path( identifier, data_type, data_format, extra_partition_keys ) file_client = self.file_system_client.get_file_client(file_name) + if check_exists and not file_client.exists(): + return None sas_url = file_client.url return str(sas_url) @@ -111,22 +115,25 @@ def finalize_save( def update(self, data, identifier, data_type): self.save(data, identifier, data_type) - def load(self, identifier, data_type): - file_name = self.get_file_path(identifier, data_type) + def load(self, identifier, data_type, data_format="json"): + if data_format != "json": + raise ValueError("Data Lake metadata reads support only json") + file_name = self.get_file_path(identifier, data_type, data_format) file_client = self.file_system_client.get_file_client(file_name) download = file_client.download_file() file_contents = download.readall() return json.loads(file_contents) - def load_all(self, data_type): + def load_all(self, data_type, data_format="json"): + if data_format != "json": + raise ValueError("Data Lake metadata reads support only json") data = [] paths = self.file_system_client.get_paths() for path in paths: - if ( - path.name.startswith(f"{self.partition_key}/{data_type}_") - if self.partition_key - else path.name.startswith(f"{data_type}_") - ): + in_partition = not self.partition_key or path.name.startswith( + f"{self.partition_key}/" + ) + if in_partition and matches_metadata_type(path.name, data_type): file_client = self.file_system_client.get_file_client( path.name ) @@ -135,10 +142,23 @@ def load_all(self, data_type): data.append(json.loads(file_contents)) return data - def load_all_from_partition(self, data_type): - data = self.load_all(data_type) + def load_all_from_partition(self, data_type, data_format="json"): + data = self.load_all(data_type, data_format=data_format) return data + def list_identifiers(self, data_type, data_format="json"): + prefix = f"{self.partition_key}/{data_type}_" + suffix = f".{data_format}" + identifiers = [] + for path in self.file_system_client.get_paths(path=self.partition_key): + if ( + path.name.startswith(prefix) + and path.name.endswith(suffix) + and matches_metadata_type(path.name, data_type) + ): + identifiers.append(path.name[len(prefix) : -len(suffix)]) + return identifiers + def load_bounded(self, data_type, max_records, data_format="json"): if data_format != "json" or max_records < 1: raise ValueError("Invalid bounded Data Lake read") @@ -150,7 +170,9 @@ def load_bounded(self, data_type, max_records, data_format="json"): if scanned_paths > scan_limit: raise ValueError("Metadata scan exceeds the bounded envelope") parts = path.name.split("/") - if len(parts) > 2 or not parts[-1].startswith(f"{data_type}_"): + if len(parts) > 2 or not matches_metadata_type( + path.name, data_type + ): continue file_contents = ( self.file_system_client.get_file_client(path.name) diff --git a/hastelib/src/hastegeo/core/data_layer/azure_postgresql_data_layer.py b/hastelib/src/hastegeo/core/data_layer/azure_postgresql_data_layer.py index 9481dcd8..61ee3018 100644 --- a/hastelib/src/hastegeo/core/data_layer/azure_postgresql_data_layer.py +++ b/hastelib/src/hastegeo/core/data_layer/azure_postgresql_data_layer.py @@ -55,6 +55,15 @@ def _build_table_identifier(table_name): def _table_identifier(self): return self._qualified_table_identifier + @staticmethod + def _require_json(data_format): + if data_format != "json": + raise ValueError("PostgreSQL metadata supports only json") + + @staticmethod + def _deserialize_json(value): + return value if isinstance(value, (dict, list)) else json.loads(value) + def _create_table_if_not_exists(self): connection_string = f"host={self.server_name} dbname={self.database_name} user={self.postgres_user} password={self.token} sslmode=require" with psycopg2.connect(connection_string) as connection: @@ -183,7 +192,8 @@ def finalize_save( def update(self, data, identifier, data_type): self.save(data, identifier, data_type) - def load(self, identifier, data_type): + def load(self, identifier, data_type, data_format="json"): + self._require_json(data_format) partition_key = ( self.partition_key if self.partition_key else identifier ) @@ -201,9 +211,10 @@ def load(self, identifier, data_type): raise FileNotFoundError( f"No data found for identifier: {identifier} and data_type: {data_type}" ) - return json.loads(result[0]) + return self._deserialize_json(result[0]) - def load_all(self, data_type): + def load_all(self, data_type, data_format="json"): + self._require_json(data_format) connection_string = f"host={self.server_name} dbname={self.database_name} user={self.postgres_user} password={self.token} sslmode=require" with psycopg2.connect(connection_string) as connection: with connection.cursor() as cursor: @@ -214,9 +225,12 @@ def load_all(self, data_type): (data_type,), ) results = cursor.fetchall() - return [json.loads(result[0]) for result in results] + return [ + self._deserialize_json(result[0]) for result in results + ] - def load_all_from_partition(self, data_type): + def load_all_from_partition(self, data_type, data_format="json"): + self._require_json(data_format) partition_key = self.partition_key connection_string = f"host={self.server_name} dbname={self.database_name} user={self.postgres_user} password={self.token} sslmode=require" with psycopg2.connect(connection_string) as connection: @@ -228,7 +242,51 @@ def load_all_from_partition(self, data_type): (data_type, partition_key), ) results = cursor.fetchall() - return [json.loads(result[0]) for result in results] + return [ + self._deserialize_json(result[0]) for result in results + ] + + def list_identifiers(self, data_type, data_format="json"): + if data_format != "json": + return [] + connection_string = f"host={self.server_name} dbname={self.database_name} user={self.postgres_user} password={self.token} sslmode=require" + with psycopg2.connect(connection_string) as connection: + with connection.cursor() as cursor: + cursor.execute( + sql.SQL( + "SELECT identifier FROM {} WHERE data_type = %s AND partition_key = %s" + ).format(self._table_identifier()), + (data_type, self.partition_key), + ) + return [result[0] for result in cursor.fetchall()] + + def load_map( + self, + identifiers, + data_type, + data_format="json", + max_workers=None, + ): + self._require_json(data_format) + identifiers = list(dict.fromkeys(identifiers)) + if not identifiers: + return {} + connection_string = f"host={self.server_name} dbname={self.database_name} user={self.postgres_user} password={self.token} sslmode=require" + with psycopg2.connect(connection_string) as connection: + with connection.cursor() as cursor: + cursor.execute( + sql.SQL( + "SELECT identifier, data FROM {} WHERE data_type = %s AND partition_key = %s AND identifier = ANY(%s)" + ).format(self._table_identifier()), + (data_type, self.partition_key, identifiers), + ) + records = { + identifier: self._deserialize_json(data) + for identifier, data in cursor.fetchall() + } + return { + identifier: records.get(identifier) for identifier in identifiers + } def load_bounded(self, data_type, max_records, data_format="json"): if data_format != "json" or max_records < 1: diff --git a/hastelib/src/hastegeo/core/data_layer/local_file_system_data_layer.py b/hastelib/src/hastegeo/core/data_layer/local_file_system_data_layer.py index a33ff2c0..d367f39c 100644 --- a/hastelib/src/hastegeo/core/data_layer/local_file_system_data_layer.py +++ b/hastelib/src/hastegeo/core/data_layer/local_file_system_data_layer.py @@ -6,6 +6,7 @@ import yaml +from ..utils.metadata import matches_metadata_type from .abstract_data_layer import AbstractDataLayer @@ -90,6 +91,7 @@ def get_file_remote_path( data_type=None, data_format="json", extra_partition_keys=None, + check_exists=True, ): """Get the remote path for a file (same as local path for filesystem layer). @@ -271,9 +273,9 @@ def load(self, identifier, data_type, data_format="json"): def load_all(self, data_type, data_format="json"): data = [] for file_name in os.listdir(self.directory): - if file_name.startswith(f"{data_type}_") and file_name.endswith( - f".{data_format}" - ): + if matches_metadata_type( + file_name, data_type + ) and file_name.endswith(f".{data_format}"): with open( os.path.join(self.directory, file_name), "r" ) as file: @@ -318,9 +320,9 @@ def load_bounded(self, data_type, max_records, data_format="json"): ) if not entry.is_file(follow_symlinks=False): continue - if not entry.name.startswith(f"{data_type}_") or not entry.name.endswith( - f".{data_format}" - ): + if not matches_metadata_type( + entry.name, data_type + ) or not entry.name.endswith(f".{data_format}"): continue with open(entry.path, "r") as file: records.append( @@ -334,6 +336,20 @@ def load_bounded(self, data_type, max_records, data_format="json"): ) return records + def list_identifiers(self, data_type, data_format="json"): + prefix = f"{data_type}_" + suffix = f".{data_format}" + identifiers = [] + if os.path.exists(self.directory): + for file_name in os.listdir(self.directory): + if ( + file_name.startswith(prefix) + and file_name.endswith(suffix) + and matches_metadata_type(file_name, data_type) + ): + identifiers.append(file_name[len(prefix) : -len(suffix)]) + return identifiers + def delete(self, identifier, data_type, data_format="json"): file_path = self.get_file_path(identifier, data_type, data_format) if not os.path.exists(file_path): diff --git a/hastelib/src/hastegeo/core/data_layer/unified.py b/hastelib/src/hastegeo/core/data_layer/unified.py index 20e35fe1..cc88258b 100644 --- a/hastelib/src/hastegeo/core/data_layer/unified.py +++ b/hastelib/src/hastegeo/core/data_layer/unified.py @@ -3,6 +3,7 @@ import importlib from ..utils.metadata import MetadataUtils +from .abstract_data_layer import AbstractDataLayer class UnifiedDataLayer: @@ -35,9 +36,7 @@ def __init__(self, storage_type, partition_key=None, **kwargs): if storage_type in storage_class_map: module_name, class_name = storage_class_map[storage_type] - module = importlib.import_module( - f"{__package__}.{module_name}" - ) + module = importlib.import_module(f"{__package__}.{module_name}") data_layer_class = getattr(module, class_name) self.data_layer = data_layer_class( partition_key=self.partition_key, **kwargs @@ -132,6 +131,24 @@ def load_bounded(self, data_type, max_records, data_format="json"): data_format=data_format, ) + def supports_load_map(self): + method = type(self.data_layer).load_map + return method is not AbstractDataLayer.load_map + + def load_map( + self, + identifiers, + data_type, + data_format="json", + max_workers=None, + ): + return self.data_layer.load_map( + identifiers=identifiers, + data_type=data_type, + data_format=data_format, + max_workers=max_workers, + ) + def load_page( self, data_type, @@ -154,6 +171,11 @@ def load_page( max_records=max_records, ) + def list_identifiers(self, data_type, data_format="json"): + return self.data_layer.list_identifiers( + data_type, data_format=data_format + ) + def delete(self, identifier, data_type, data_format="json"): self.data_layer.delete(identifier, data_type, data_format=data_format) @@ -183,12 +205,14 @@ def get_file_remote_path( data_type=None, data_format="json", extra_partition_keys=None, + check_exists=True, ): return self.data_layer.get_file_remote_path( identifier, data_type, data_format=data_format, extra_partition_keys=extra_partition_keys, + check_exists=check_exists, ) def get_base_url(self): diff --git a/hastelib/src/hastegeo/core/processors/metadata.py b/hastelib/src/hastegeo/core/processors/metadata.py index d8f42102..87eee3be 100644 --- a/hastelib/src/hastegeo/core/processors/metadata.py +++ b/hastelib/src/hastegeo/core/processors/metadata.py @@ -5,6 +5,12 @@ from hastegeo.core.config import Config from ..data_layer.unified import UnifiedDataLayer +from ..utils.parallel import ( + configured_worker_count, + parallel_map, + validate_worker_count, +) +from ..utils.perf import timed class MetadataProcessor: @@ -96,9 +102,12 @@ def load(self, key, data_format="json"): """ Load metadata from the backend storage. """ - metadata = self.storage.load( - identifier=key, data_type=self.data_type, data_format=data_format - ) + with timed("load"): + metadata = self.storage.load( + identifier=key, + data_type=self.data_type, + data_format=data_format, + ) return metadata def load_all(self, data_format="json"): @@ -106,9 +115,10 @@ def load_all(self, data_format="json"): Load all metadata from the backend storage. """ metadata = [] - metadata_list = self.storage.load_all( - data_type=self.data_type, data_format=data_format - ) + with timed("load_all"): + metadata_list = self.storage.load_all( + data_type=self.data_type, data_format=data_format + ) for each_metadata in metadata_list: metadata.append(each_metadata) return metadata @@ -118,9 +128,10 @@ def load_all_from_partition(self, data_format="json"): Load all metadata from the backend storage. """ metadata = [] - metadata_list = self.storage.load_all_from_partition( - data_type=self.data_type, data_format=data_format - ) + with timed("load_all_from_partition"): + metadata_list = self.storage.load_all_from_partition( + data_type=self.data_type, data_format=data_format + ) for each_metadata in metadata_list: metadata.append(each_metadata) return metadata @@ -155,6 +166,98 @@ def load_page( max_records=max_records, ) + def load_map( + self, + keys: list[str], + data_format: str = "json", + max_workers: int | None = None, + ) -> dict[str, dict | None]: + """Load many records by key concurrently. + + Returns ``{key: record}``; a key whose record is missing maps to + ``None`` (mirrors a per-key ``load`` that raises ``FileNotFoundError``). + Bounded parallelism overlaps per-key storage operations while the + process-wide executor caps aggregate concurrency. Preserves perf + instrumentation across worker threads by binding the active counter + inside each task. + """ + from ..utils.perf import bind, get_counter + + keys = list(dict.fromkeys(keys)) + if not keys: + return {} + workers = ( + configured_worker_count("HASTE_METADATA_LOAD_WORKERS", 8) + if max_workers is None + else validate_worker_count(max_workers) + ) + if self.storage.supports_load_map(): + with timed("load_map"): + return self.storage.load_map( + identifiers=keys, + data_type=self.data_type, + data_format=data_format, + max_workers=workers, + ) + counter = get_counter() + + def _one(key): + with bind(counter): + try: + return key, self.load(key, data_format=data_format) + except FileNotFoundError: + return key, None + + return dict(parallel_map(_one, keys, max_workers=workers)) + + def load_filtered( + self, predicate: dict[str, object], data_format: str = "json" + ) -> list[dict]: + """Load partition records matching every key/value in ``predicate``. + + This is an explicit client-side fallback: the partition is loaded once + and then filtered in process. Backends need a separate query primitive + before this can reduce transferred records. + """ + if not isinstance(predicate, dict) or not predicate: + raise ValueError("predicate must be a non-empty dictionary") + records = self.load_all_from_partition(data_format=data_format) + return [ + record + for record in records + if isinstance(record, dict) + and all( + key in record and record[key] == value + for key, value in predicate.items() + ) + ] + + def list_keys(self, data_format="json"): + """List the identifiers present for this data type in the partition. + + A cheap, metadata-only alternative to ``load_all_from_partition`` for + bulk existence checks (does not download record contents). + """ + with timed("list_keys"): + return self.storage.list_identifiers( + data_type=self.data_type, data_format=data_format + ) + + def build_url(self, key, data_format="json"): + """Build a remote URL for a record without a per-item existence check. + + Intended for callers that have already confirmed the record exists in + bulk via :meth:`list_keys`. On the blob backend this avoids a network + round-trip per key (the container read SAS is shared), so it is a local + operation and is not counted as a storage round-trip. + """ + return self.storage.get_file_remote_path( + identifier=key, + data_type=self.data_type, + data_format=data_format, + check_exists=False, + ) + def load_and_combine_sub_data_types(self, key, data_types): """ Load and combine metadata from multiple data types. @@ -225,8 +328,9 @@ def export(self, key, data_format="json"): """ # NOTE: This is a quick method to make the export work for Azure blob storage layer. # Rework needed to handle different storage types and formats properly. - return self.storage.get_file_remote_path( - identifier=key, - data_type=self.data_type, - data_format=data_format, - ) + with timed("export"): + return self.storage.get_file_remote_path( + identifier=key, + data_type=self.data_type, + data_format=data_format, + ) diff --git a/hastelib/src/hastegeo/core/processors/project_details.py b/hastelib/src/hastegeo/core/processors/project_details.py new file mode 100644 index 00000000..bb9cfd99 --- /dev/null +++ b/hastelib/src/hastegeo/core/processors/project_details.py @@ -0,0 +1,213 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Project-details loading and response assembly.""" + +import asyncio +from collections import defaultdict +from collections.abc import Callable, Mapping, Sequence +from typing import Any + +from ..config import Config +from .metadata import MetadataProcessor + + +class ProjectDetailsProcessor: + """Load and assemble one project's API detail response.""" + + def __init__( + self, + project_id: str, + config: Config | None = None, + processor_factory: Callable[ + ..., MetadataProcessor + ] = MetadataProcessor, + ) -> None: + self.project_id = project_id + self.config = config or Config() + self.processor_factory = processor_factory + + def _processor(self, data_type: str) -> MetadataProcessor: + return self.processor_factory( + data_type=data_type, + partition_key=self.project_id, + config=self.config, + ) + + def _load(self, data_type: str, key: str) -> dict[str, Any]: + return self._processor(data_type).load(key) + + def _load_partition(self, data_type: str) -> list[dict[str, Any]]: + return self._processor(data_type).load_all_from_partition() + + def _load_map( + self, data_type: str, keys: Sequence[str] + ) -> dict[str, dict[str, Any] | None]: + return self._processor(data_type).load_map(keys) + + def _load_train_label_urls( + self, models: Sequence[Mapping[str, Any]] + ) -> dict[str, str | None]: + model_ids = [ + model["modelId"] for model in models if not model.get("labelsUrl") + ] + if not model_ids: + return {} + + processor = self._processor( + self.config.get_metadata_types().TRAIN_LABELS.value + ) + try: + existing_keys = set(processor.list_keys(data_format="geojson")) + except NotImplementedError: + urls = {} + for model_id in model_ids: + try: + urls[model_id] = processor.export( + model_id, data_format="geojson" + ) + except (FileNotFoundError, NotImplementedError): + urls[model_id] = None + return urls + + urls = {} + for model_id in model_ids: + if model_id not in existing_keys: + urls[model_id] = None + continue + try: + urls[model_id] = processor.build_url( + model_id, data_format="geojson" + ) + except NotImplementedError: + urls[model_id] = None + return urls + + async def load(self, include_models: bool) -> dict[str, Any]: + """Load project metadata concurrently and assemble the response.""" + types = self.config.get_metadata_types() + project = await asyncio.to_thread( + self._load, types.PROJECT.value, self.project_id + ) + + partition_types = [types.IMAGELAYER.value, types.LABELS.value] + if include_models: + partition_types.append(types.MODEL.value) + partition_results = await asyncio.gather( + *( + asyncio.to_thread(self._load_partition, data_type) + for data_type in partition_types + ) + ) + image_layers = partition_results[0] + label_projects = partition_results[1] + models = partition_results[2] if include_models else [] + + image_layer_ids = [ + image_layer["imageLayerId"] for image_layer in image_layers + ] + validation_task = asyncio.to_thread( + self._load_map, types.VALIDATION.value, image_layer_ids + ) + + if include_models: + model_ids = [model["modelId"] for model in models] + ( + validation_by_layer, + artifacts_by_model, + train_label_urls_by_model, + ) = await asyncio.gather( + validation_task, + asyncio.to_thread( + self._load_map, types.MODEL_ARTIFACTS.value, model_ids + ), + asyncio.to_thread(self._load_train_label_urls, models), + ) + else: + validation_by_layer = await validation_task + artifacts_by_model = {} + train_label_urls_by_model = {} + + return assemble_project_details( + project=project, + image_layers=image_layers, + models=models, + label_projects=label_projects, + artifacts_by_model=artifacts_by_model, + validation_by_layer=validation_by_layer, + train_label_urls_by_model=train_label_urls_by_model, + include_models=include_models, + ) + + +def assemble_project_details( + project: Mapping[str, Any], + image_layers: Sequence[Mapping[str, Any]], + models: Sequence[Mapping[str, Any]], + label_projects: Sequence[Mapping[str, Any]], + artifacts_by_model: Mapping[str, Mapping[str, Any] | None], + validation_by_layer: Mapping[str, Mapping[str, Any] | None], + train_label_urls_by_model: Mapping[str, str | None], + include_models: bool, +) -> dict[str, Any]: + """Assemble the API response from already-loaded metadata records.""" + models_by_layer: dict[str | None, list[Mapping[str, Any]]] = defaultdict( + list + ) + for model in models: + models_by_layer[model.get("imageLayerId")].append(model) + + labels_by_layer: dict[str, Mapping[str, Any]] = {} + for label_project in label_projects: + image_layer_id = label_project.get("imageLayerId") + if ( + image_layer_id is not None + and image_layer_id not in labels_by_layer + ): + labels_by_layer[image_layer_id] = label_project + + assembled_layers: list[dict[str, Any]] = [] + for stored_image_layer in image_layers: + image_layer = dict(stored_image_layer) + image_layer_id = image_layer["imageLayerId"] + + if include_models: + layer_models = sorted( + models_by_layer.get(image_layer_id, []), + key=lambda model: model["creationDate"], + reverse=True, + ) + assembled_models = [] + for stored_model in layer_models: + model = dict(stored_model) + model_id = model["modelId"] + model["artifacts"] = artifacts_by_model.get(model_id) + if not model.get("labelsUrl"): + model["labelsUrl"] = train_label_urls_by_model.get( + model_id + ) + assembled_models.append(model) + image_layer["models"] = assembled_models + image_layer["modelCount"] = len(assembled_models) + + label_project = labels_by_layer.get(image_layer_id) + if label_project is not None: + labels = label_project.get("labels") + image_layer["labelProjectCount"] = ( + len(labels) if labels is not None else 0 + ) + if not image_layer.get("labelsUrl"): + image_layer["labelsUrl"] = None + + validation = validation_by_layer.get(image_layer_id) + validation_labels = validation.get("labels") if validation else None + image_layer["validationLabelCount"] = len(validation_labels or {}) + assembled_layers.append(image_layer) + + assembled_project = dict(project) + assembled_project["imageLayer"] = sorted( + assembled_layers, + key=lambda image_layer: image_layer["creationDate"], + reverse=True, + ) + assembled_project["imageLayerCount"] = len(assembled_layers) + return assembled_project diff --git a/hastelib/src/hastegeo/core/utils/async_cache.py b/hastelib/src/hastegeo/core/utils/async_cache.py new file mode 100644 index 00000000..72a7db95 --- /dev/null +++ b/hastelib/src/hastegeo/core/utils/async_cache.py @@ -0,0 +1,103 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Small bounded async cache with per-key single-flight loading.""" + +import asyncio +import os +import time +from collections import OrderedDict +from collections.abc import Awaitable, Callable +from typing import Generic, TypeVar + +KeyT = TypeVar("KeyT") +ValueT = TypeVar("ValueT") + + +def configured_cache_value( + name: str, default: int, minimum: int, maximum: int +) -> int: + """Read a bounded integer cache setting from the environment.""" + raw_value = os.environ.get(name, str(default)) + try: + value = int(raw_value) + except ValueError as error: + raise ValueError(f"{name} must be an integer") from error + if not minimum <= value <= maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return value + + +class AsyncTTLCache(Generic[KeyT, ValueT]): + """Cache successful async loads for a bounded freshness window.""" + + def __init__( + self, + ttl_seconds: float, + max_entries: int, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if ttl_seconds < 0: + raise ValueError("ttl_seconds cannot be negative") + if max_entries < 1: + raise ValueError("max_entries must be positive") + self.ttl_seconds = ttl_seconds + self.max_entries = max_entries + self._clock = clock + self._entries: OrderedDict[KeyT, tuple[float, ValueT]] = OrderedDict() + self._inflight: dict[KeyT, asyncio.Task[ValueT]] = {} + self._lock = asyncio.Lock() + + async def get_or_create( + self, + key: KeyT, + factory: Callable[[], Awaitable[ValueT]], + refresh: bool = False, + ) -> tuple[ValueT, bool]: + """Return ``(value, reused)`` and share one in-flight load per key.""" + async with self._lock: + cached = None if refresh else self._entries.get(key) + if cached is not None: + expires_at, value = cached + if self._clock() < expires_at: + self._entries.move_to_end(key) + return value, True + del self._entries[key] + elif refresh: + self._entries.pop(key, None) + + task = None if refresh else self._inflight.get(key) + reused = task is not None + if task is None: + task = asyncio.create_task(factory()) + self._inflight[key] = task + task.add_done_callback( + lambda completed, cache_key=key: asyncio.create_task( + self._complete(cache_key, completed) + ) + ) + + return await asyncio.shield(task), reused + + async def _complete(self, key: KeyT, task: asyncio.Task[ValueT]) -> None: + async with self._lock: + if self._inflight.get(key) is not task: + return + del self._inflight[key] + if task.cancelled() or task.exception() is not None: + return + self._entries[key] = ( + self._clock() + self.ttl_seconds, + task.result(), + ) + self._entries.move_to_end(key) + while len(self._entries) > self.max_entries: + self._entries.popitem(last=False) + + async def clear(self) -> None: + """Clear cached values and cancel unfinished loads.""" + async with self._lock: + tasks = list(self._inflight.values()) + self._inflight.clear() + self._entries.clear() + for task in tasks: + task.cancel() diff --git a/hastelib/src/hastegeo/core/utils/blob.py b/hastelib/src/hastegeo/core/utils/blob.py index d2f4affc..c43ccca5 100644 --- a/hastelib/src/hastegeo/core/utils/blob.py +++ b/hastelib/src/hastegeo/core/utils/blob.py @@ -20,12 +20,77 @@ from __future__ import annotations import asyncio +import functools import os import re import tempfile +import threading +from collections import OrderedDict +from datetime import datetime, timedelta, timezone from typing import NamedTuple, Optional, Tuple from urllib.parse import urlparse +_USER_DELEGATION_KEYS = OrderedDict() +_USER_DELEGATION_KEY_CACHE_SIZE = 8 +_USER_DELEGATION_KEYS_LOCK = threading.Lock() + + +@functools.lru_cache(maxsize=8) +def get_blob_service_client( + connection_string: str | None = None, + account_url: str | None = None, +): + """Return a process-wide ``BlobServiceClient`` for one storage account. + + Azure SDK clients are thread-safe and designed for reuse; creating one per + call re-parses credentials and re-establishes the connection pool. Caching + by credential target keeps that setup cost one-time per process. + """ + from azure.storage.blob import BlobServiceClient + + if connection_string: + return BlobServiceClient.from_connection_string(connection_string) + if account_url: + from azure.identity import DefaultAzureCredential + + return BlobServiceClient( + account_url=account_url, credential=DefaultAzureCredential() + ) + raise ValueError("A connection string or account URL is required") + + +def get_cached_user_delegation_key( + blob_service_client, + now: datetime | None = None, +): + """Return a reusable user-delegation key with a safe refresh margin.""" + current_time = now or datetime.now(timezone.utc) + cache_key = blob_service_client + with _USER_DELEGATION_KEYS_LOCK: + cached = _USER_DELEGATION_KEYS.get(cache_key) + if cached is not None: + delegation_key, expires_at = cached + if expires_at > current_time + timedelta(minutes=15): + _USER_DELEGATION_KEYS.move_to_end(cache_key) + return delegation_key + + expires_at = current_time + timedelta(hours=2) + delegation_key = blob_service_client.get_user_delegation_key( + current_time - timedelta(minutes=5), expires_at + ) + _USER_DELEGATION_KEYS[cache_key] = (delegation_key, expires_at) + _USER_DELEGATION_KEYS.move_to_end(cache_key) + while len(_USER_DELEGATION_KEYS) > _USER_DELEGATION_KEY_CACHE_SIZE: + _USER_DELEGATION_KEYS.popitem(last=False) + return delegation_key + + +def clear_blob_client_caches() -> None: + """Clear cached Blob clients and delegation keys for isolated tests.""" + get_blob_service_client.cache_clear() + with _USER_DELEGATION_KEYS_LOCK: + _USER_DELEGATION_KEYS.clear() + def split_blob_url(url: str) -> Tuple[str, str]: """Extract ``(container_name, blob_name)`` from a blob URL. @@ -99,14 +164,9 @@ async def download_blob_to_tempfile( caller is responsible for unlinking the returned path when done — use ``try/finally``. """ - # Imported here so this module stays cheap to import for callers that - # only need split_blob_url(): azure-storage-blob brings in tens of - # transitive imports. - from azure.storage.blob import BlobServiceClient - conn_str = os.environ.get("BLOB_CONNECTION_STRING", "") container_name, blob_name = split_blob_url(url) - bsc = BlobServiceClient.from_connection_string(conn_str) + bsc = get_blob_service_client(connection_string=conn_str) if max_bytes is not None and max_bytes < 1: raise ValueError("max_bytes must be positive") @@ -128,10 +188,7 @@ def download() -> str: downloaded_bytes = 0 for chunk in blob_client.download_blob().chunks(): downloaded_bytes += len(chunk) - if ( - max_bytes is not None - and downloaded_bytes > max_bytes - ): + if max_bytes is not None and downloaded_bytes > max_bytes: raise ValueError( "Blob exceeds the allowed download size" ) @@ -201,13 +258,11 @@ async def read_blob_range( reads to EOF. ``data`` is clamped to the blob size; an ``offset`` at or past EOF yields empty ``data`` (callers should answer ``416``). """ - from azure.storage.blob import BlobServiceClient - conn_str = os.environ.get("BLOB_CONNECTION_STRING", "") container_name, blob_name = split_blob_url(url) def _read() -> BlobRange: - bsc = BlobServiceClient.from_connection_string(conn_str) + bsc = get_blob_service_client(connection_string=conn_str) blob_client = bsc.get_container_client(container_name).get_blob_client( blob_name ) diff --git a/hastelib/src/hastegeo/core/utils/metadata.py b/hastelib/src/hastegeo/core/utils/metadata.py index 25e5570d..89804921 100644 --- a/hastelib/src/hastegeo/core/utils/metadata.py +++ b/hastelib/src/hastegeo/core/utils/metadata.py @@ -4,6 +4,42 @@ import random import uuid from datetime import datetime, timezone +from functools import lru_cache + + +@lru_cache(maxsize=1) +def _known_metadata_types() -> tuple[str, ...]: + from ..config import Config + + return tuple( + sorted( + ( + metadata_type.value + for metadata_type in Config.get_metadata_types() + ), + key=len, + reverse=True, + ) + ) + + +def matches_metadata_type(path: str, data_type: str) -> bool: + """Return whether a stored name belongs to the requested metadata type. + + Existing records use ``{type}_{identifier}``, while some type names are + prefixes of others (notably ``model`` and ``model_catalog``). Assigning a + name to the longest known matching type preserves the existing layout + without allowing broader scans to consume a narrower type. + """ + name = path.rsplit("/", 1)[-1] + matching_types = [ + known_type + for known_type in _known_metadata_types() + if name.startswith(f"{known_type}_") + ] + if not matching_types: + return name.startswith(f"{data_type}_") + return matching_types[0] == data_type class MetadataUtils: diff --git a/hastelib/src/hastegeo/core/utils/parallel.py b/hastelib/src/hastegeo/core/utils/parallel.py new file mode 100644 index 00000000..aa2b9ddc --- /dev/null +++ b/hastelib/src/hastegeo/core/utils/parallel.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Process-wide bounded execution for blocking I/O.""" + +import os +from collections.abc import Callable, Iterable +from concurrent.futures import ( + FIRST_COMPLETED, + Future, + ThreadPoolExecutor, + wait, +) +from threading import current_thread +from typing import TypeVar + +_MAX_CONFIGURED_WORKERS = 64 + +InputT = TypeVar("InputT") +OutputT = TypeVar("OutputT") + + +def validate_worker_count(value: int, name: str = "max_workers") -> int: + """Validate a worker count before constructing or scheduling a pool.""" + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if not 1 <= value <= _MAX_CONFIGURED_WORKERS: + raise ValueError( + f"{name} must be between 1 and {_MAX_CONFIGURED_WORKERS}" + ) + return value + + +def configured_worker_count(name: str, default: int) -> int: + """Read and validate a worker count from the process environment.""" + raw_value = os.environ.get(name) + if raw_value is None: + return validate_worker_count(default, name) + try: + value = int(raw_value) + except ValueError as error: + raise ValueError(f"{name} must be an integer") from error + return validate_worker_count(value, name) + + +class BoundedExecutor: + """Share one thread budget across all concurrent map operations.""" + + def __init__(self, max_workers: int) -> None: + self.max_workers = validate_worker_count(max_workers) + self._thread_prefix = f"haste-io-{id(self):x}" + self._executor = ThreadPoolExecutor( + max_workers=self.max_workers, + thread_name_prefix=self._thread_prefix, + ) + + def map( + self, + function: Callable[[InputT], OutputT], + values: Iterable[InputT], + max_workers: int | None = None, + ) -> list[OutputT]: + """Run an ordered map with bounded submissions and shared workers.""" + items = list(values) + if not items: + return [] + + requested_workers = ( + self.max_workers + if max_workers is None + else validate_worker_count(max_workers) + ) + worker_count = min(requested_workers, self.max_workers, len(items)) + if worker_count == 1 or current_thread().name.startswith( + self._thread_prefix + ): + return [function(item) for item in items] + + indexed_items = iter(enumerate(items)) + pending: dict[Future[OutputT], int] = {} + results: dict[int, OutputT] = {} + + def submit_next() -> bool: + try: + index, item = next(indexed_items) + except StopIteration: + return False + pending[self._executor.submit(function, item)] = index + return True + + for _ in range(worker_count): + submit_next() + + try: + while pending: + done, _ = wait(pending, return_when=FIRST_COMPLETED) + for future in done: + index = pending.pop(future) + results[index] = future.result() + for _ in done: + submit_next() + except Exception: + for future in pending: + future.cancel() + raise + + return [results[index] for index in range(len(items))] + + def shutdown(self) -> None: + """Release worker threads after a non-global executor is finished.""" + self._executor.shutdown(wait=True, cancel_futures=True) + + +PARALLEL_IO_EXECUTOR = BoundedExecutor( + configured_worker_count("HASTE_BLOB_DOWNLOAD_WORKERS", 16) +) + + +def parallel_map( + function: Callable[[InputT], OutputT], + values: Iterable[InputT], + max_workers: int | None = None, +) -> list[OutputT]: + """Map blocking I/O on the shared process-wide executor.""" + return PARALLEL_IO_EXECUTOR.map(function, values, max_workers=max_workers) diff --git a/hastelib/src/hastegeo/core/utils/perf.py b/hastelib/src/hastegeo/core/utils/perf.py new file mode 100644 index 00000000..e2456e51 --- /dev/null +++ b/hastelib/src/hastegeo/core/utils/perf.py @@ -0,0 +1,143 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Lightweight, opt-in performance instrumentation. + +Counts and times logical data-layer operations for a single request. A bulk +operation can issue multiple backend SDK requests, so these values are not +storage transaction counts. + +Design notes: +- A ``ContextVar`` holds a shared ``PerfCounter`` *object*. ``asyncio.to_thread`` + copies the current context into the worker thread, so a read method running in + the thread sees the *same* counter instance and its (lock-guarded) mutations are + visible back in the calling coroutine. The ContextVar value (the reference) is + never reassigned inside the thread, only the object it points to is mutated. +- When tracking is not enabled, ``timed()`` is a no-op with no measurable overhead, + so instrumenting the shared data layer is safe for every caller (API + queues). +""" +import contextvars +import threading +import time +from contextlib import contextmanager + +_current: "contextvars.ContextVar[PerfCounter | None]" = ( + contextvars.ContextVar("haste_perf_counter", default=None) +) + + +class PerfCounter: + """Thread-safe accumulator of logical data-layer calls and duration.""" + + def __init__(self): + self.calls = 0 + self.seconds = 0.0 + self.by_op = {} + self._lock = threading.Lock() + + def record(self, op, elapsed): + with self._lock: + self.calls += 1 + self.seconds += elapsed + entry = self.by_op.get(op) + if entry is None: + self.by_op[op] = {"calls": 1, "seconds": elapsed} + else: + entry["calls"] += 1 + entry["seconds"] += elapsed + + +def begin(enabled=True): + """Start tracking for the current context. Returns the counter (or None).""" + if not enabled: + _current.set(None) + return None + counter = PerfCounter() + _current.set(counter) + return counter + + +def end(): + """Stop tracking for the current context.""" + _current.set(None) + + +def get_counter(): + return _current.get() + + +@contextmanager +def bind(counter): + """Bind ``counter`` as the active counter for the current context. + + Used to propagate the active counter into worker threads (e.g. a + ``ThreadPoolExecutor``), which — unlike ``asyncio.to_thread`` — do not copy + the parent context. ``counter`` may be ``None`` (tracking disabled). + """ + token = _current.set(counter) + try: + yield + finally: + _current.reset(token) + + +@contextmanager +def timed(op): + """Time an ``op`` and record it on the active counter, if any. + + Zero-overhead when tracking is disabled (no active counter). + """ + counter = _current.get() + if counter is None: + yield + return + start = time.perf_counter() + try: + yield + finally: + counter.record(op, time.perf_counter() - start) + + +def headers(counter, wall_start): + """Response headers exposing data-layer call timing for benchmarking.""" + if counter is None: + return {} + storage_ms = counter.seconds * 1000.0 + wall_ms = (time.perf_counter() - wall_start) * 1000.0 + return { + "X-Haste-Data-Layer-Calls": str(counter.calls), + "X-Haste-Data-Layer-Ms": f"{storage_ms:.1f}", + # Keep the original names while benchmark consumers migrate. + "X-Haste-Storage-Calls": str(counter.calls), + "X-Haste-Storage-Ms": f"{storage_ms:.1f}", + "X-Haste-Wall-Ms": f"{wall_ms:.1f}", + "Server-Timing": ", ".join( + [ + ";".join( + ["data-layer", "desc=data-layer", f"dur={storage_ms:.1f}"] + ), + ";".join(["wall", "desc=wall", f"dur={wall_ms:.1f}"]), + ] + ), + } + + +def log_summary(logger, name, counter, wall_start, **fields): + """Emit a single structured ``PERF`` line and stop tracking.""" + if counter is None: + return + wall_ms = (time.perf_counter() - wall_start) * 1000.0 + ops = { + op: {"calls": e["calls"], "ms": round(e["seconds"] * 1000, 1)} + for op, e in counter.by_op.items() + } + extra = " ".join(f"{k}={v}" for k, v in fields.items()) + logger.info( + "PERF %s %s data_layer_calls=%d data_layer_ms=%.1f wall_ms=%.1f ops=%s", + name, + extra, + counter.calls, + counter.seconds * 1000.0, + wall_ms, + ops, + ) + end() diff --git a/hastelib/tests/core/artifact_storage/test_azure_blob_artifact_storage.py b/hastelib/tests/core/artifact_storage/test_azure_blob_artifact_storage.py new file mode 100644 index 00000000..beb23a2d --- /dev/null +++ b/hastelib/tests/core/artifact_storage/test_azure_blob_artifact_storage.py @@ -0,0 +1,238 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import os +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from azure.core.exceptions import ResourceExistsError +from hastegeo.core.artifact_storage.azure_blob_artifact_storage import ( + _INITIALIZED_CONTAINERS, + AzureBlobArtifactStorage, +) + + +class TestAzureBlobArtifactStorageFetch(unittest.TestCase): + def setUp(self) -> None: + self.storage = AzureBlobArtifactStorage.__new__( + AzureBlobArtifactStorage + ) + self.storage.partition_key = None + self.storage.logger = Mock() + self.storage.container_client = Mock() + self.storage.container_client.url = "https://account.test/container" + self.blob_client = ( + self.storage.container_client.get_blob_client.return_value + ) + self.stream = self.blob_client.download_blob.return_value + + def _set_blob_names(self, *names: str) -> None: + self.storage.container_client.list_blobs.return_value = [ + SimpleNamespace(name=name) for name in names + ] + + def test_fetch_downloads_each_blob_atomically(self) -> None: + self._set_blob_names("project/output.txt") + self.stream.chunks.return_value = [b"hello", b" world"] + + with tempfile.TemporaryDirectory() as destination: + result = self.storage.fetch_artifact( + src_path="project", dst_path=destination + ) + + output_path = os.path.join(destination, "project", "output.txt") + with open(output_path, "rb") as output: + self.assertEqual(output.read(), b"hello world") + self.assertEqual(result, destination) + self.assertEqual( + os.listdir(os.path.dirname(output_path)), ["output.txt"] + ) + + def test_fetch_rejects_parent_path_in_blob_name(self) -> None: + self._set_blob_names("../outside.txt") + + with tempfile.TemporaryDirectory() as destination: + with self.assertRaisesRegex(ValueError, "Invalid artifact path"): + self.storage.fetch_artifact( + src_path="project", dst_path=destination + ) + self.assertFalse( + os.path.exists(os.path.join(destination, "..", "outside.txt")) + ) + + def test_fetch_removes_partial_file_when_download_fails(self) -> None: + self._set_blob_names("project/output.txt") + + def failing_chunks(): + yield b"partial" + raise RuntimeError("download failed") + + self.stream.chunks.side_effect = failing_chunks + with tempfile.TemporaryDirectory() as destination: + with self.assertRaisesRegex(RuntimeError, "download failed"): + self.storage.fetch_artifact( + src_path="project", dst_path=destination + ) + output_directory = os.path.join(destination, "project") + self.assertEqual(os.listdir(output_directory), []) + + def test_fetch_requires_source_and_destination(self) -> None: + with self.assertRaisesRegex(ValueError, "source"): + self.storage.fetch_artifact(dst_path="destination") + with self.assertRaisesRegex(ValueError, "destination"): + self.storage.fetch_artifact(src_path="source") + + def test_fetch_rejects_invalid_worker_configuration(self) -> None: + self._set_blob_names("project/output.txt") + with patch.dict(os.environ, {"HASTE_ARTIFACT_DOWNLOAD_WORKERS": "0"}): + with self.assertRaisesRegex(ValueError, "between 1 and 64"): + self.storage.fetch_artifact( + src_path="project", dst_path="destination" + ) + + @patch( + "hastegeo.core.artifact_storage.azure_blob_artifact_storage.generate_container_sas", + return_value="sas", + ) + @patch( + "hastegeo.core.artifact_storage.azure_blob_artifact_storage.get_cached_user_delegation_key", + return_value="delegation-key", + ) + def test_download_url_fetches_delegation_key_lazily( + self, delegation_key, _generate_sas + ) -> None: + self.storage.blob_service_client = Mock() + self.storage.account_key = None + self.storage.user_delegation_key = None + self.storage.container_read_policy = "policy" + self.storage.container_client.account_name = "account" + self.storage.container_client.container_name = "artifacts" + self.blob_client.url = "https://account.test/artifacts/file.txt" + + result = self.storage.get_download_url(identifier="file.txt") + + self.assertEqual(result, f"{self.blob_client.url}?sas") + delegation_key.assert_called_once_with( + self.storage.blob_service_client + ) + + @patch( + "hastegeo.core.artifact_storage.azure_blob_artifact_storage.generate_blob_sas", + return_value="sas", + ) + @patch( + "hastegeo.core.artifact_storage.azure_blob_artifact_storage.get_cached_user_delegation_key", + return_value="delegation-key", + ) + def test_scoped_url_reuses_delegation_key( + self, delegation_key, _generate_sas + ) -> None: + self.storage.blob_service_client = Mock() + self.storage.identity_blob_service_client = ( + self.storage.blob_service_client + ) + self.storage.account_key = None + self.storage.container_client.account_name = "account" + self.storage.container_client.container_name = "artifacts" + self.storage.container_client.url = "https://account.test/artifacts" + self.blob_client.url = "https://account.test/artifacts/file.txt" + self.blob_client.exists.return_value = True + + result = self.storage.get_scoped_download_url("file.txt") + + self.assertEqual(result, f"{self.blob_client.url}?sas") + delegation_key.assert_called_once() + + def test_resolve_artifact_path_rejects_other_account(self) -> None: + self.storage.container_client.url = "https://account.test/artifacts" + + with self.assertRaisesRegex(ValueError, "configured storage"): + self.storage.resolve_artifact_path( + "https://other.test/artifacts/file.txt" + ) + + def test_resolve_artifact_path_rejects_other_container(self) -> None: + self.storage.container_client.url = "https://account.test/artifacts" + + with self.assertRaisesRegex(ValueError, "configured container"): + self.storage.resolve_artifact_path( + "https://account.test/other/file.txt" + ) + + +class TestAzureBlobArtifactStorageClientReuse(unittest.TestCase): + def setUp(self) -> None: + _INITIALIZED_CONTAINERS.clear() + + def tearDown(self) -> None: + _INITIALIZED_CONTAINERS.clear() + + @patch( + "hastegeo.core.artifact_storage.azure_blob_artifact_storage.get_blob_service_client" + ) + @patch.object( + AzureBlobArtifactStorage, + "_create_or_update_managed_access_policy", + ) + def test_connection_string_uses_cached_client( + self, create_policy, factory + ): + service = Mock(url="https://account.test") + service.credential.account_key = "key" # pragma: allowlist secret + factory.return_value = service + + storage = AzureBlobArtifactStorage( + account_url="", + container="artifacts", + connection_string="connection", + ) + + self.assertIs(storage.blob_service_client, service) + factory.assert_called_once_with(connection_string="connection") + create_policy.assert_called_once_with() + + @patch( + "hastegeo.core.artifact_storage.azure_blob_artifact_storage.get_blob_service_client" + ) + def test_managed_identity_uses_cached_client(self, factory): + service = Mock(url="https://account.test") + factory.return_value = service + + storage = AzureBlobArtifactStorage( + account_url="https://account.test", + container="artifacts", + connection_string=None, + serves_read_sas=False, + ) + + self.assertIs(storage.blob_service_client, service) + factory.assert_called_once_with(account_url="https://account.test") + service.get_user_delegation_key.assert_not_called() + + @patch( + "hastegeo.core.artifact_storage.azure_blob_artifact_storage.get_blob_service_client" + ) + @patch.object( + AzureBlobArtifactStorage, + "_create_or_update_managed_access_policy", + ) + def test_existing_container_is_reused(self, create_policy, factory): + service = Mock(url="https://account.test") + service.credential.account_key = "key" # pragma: allowlist secret + container = service.get_container_client.return_value + container.create_container.side_effect = ResourceExistsError("exists") + factory.return_value = service + + storage = AzureBlobArtifactStorage( + account_url="", + container="artifacts", + connection_string="connection", + ) + + self.assertIs(storage.container_client, container) + create_policy.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/hastelib/tests/core/data_layer/test_azure_blob_storage_data_layer.py b/hastelib/tests/core/data_layer/test_azure_blob_storage_data_layer.py new file mode 100644 index 00000000..cfaac2e4 --- /dev/null +++ b/hastelib/tests/core/data_layer/test_azure_blob_storage_data_layer.py @@ -0,0 +1,298 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json +import unittest +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from azure.core.exceptions import ResourceNotFoundError +from hastegeo.core.data_layer.azure_blob_storage_data_layer import ( + _INITIALIZED_CONTAINERS, + AzureBlobStorageDataLayer, +) + + +class TestAzureBlobStorageDataLayerLoad(unittest.TestCase): + def setUp(self) -> None: + self.layer = AzureBlobStorageDataLayer.__new__( + AzureBlobStorageDataLayer + ) + self.layer.partition_key = "partition" + self.layer.container_client = Mock() + self.blob_client = ( + self.layer.container_client.get_blob_client.return_value + ) + self.downloader = self.blob_client.download_blob.return_value + + def test_load_deserializes_json(self) -> None: + self.downloader.readall.return_value = b'{"value": 1}' + + result = self.layer.load("record", "model") + + self.assertEqual(result, {"value": 1}) + + def test_load_tolerates_legacy_double_serialized_json(self) -> None: + self.downloader.readall.return_value = json.dumps( + json.dumps({"value": 1}) + ).encode() + + result = self.layer.load("record", "model") + + self.assertEqual(result, {"value": 1}) + + def test_load_maps_only_resource_not_found_to_file_not_found(self) -> None: + self.blob_client.download_blob.side_effect = ResourceNotFoundError( + "missing" + ) + + with self.assertRaises(FileNotFoundError): + self.layer.load("record", "model") + + def test_load_preserves_transport_errors(self) -> None: + self.blob_client.download_blob.side_effect = RuntimeError( + "transport unavailable" + ) + + with self.assertRaisesRegex(RuntimeError, "transport unavailable"): + self.layer.load("record", "model") + + def test_load_preserves_json_errors(self) -> None: + self.downloader.readall.return_value = b"{" + + with self.assertRaises(json.JSONDecodeError): + self.layer.load("record", "model") + + def test_load_rejects_unsupported_format(self) -> None: + with self.assertRaisesRegex(ValueError, "Unsupported data_format"): + self.layer.load("record", "model", data_format="xml") + + def test_parallel_read_preserves_blob_order(self) -> None: + self.layer._read_blob_content = Mock( + side_effect=lambda blob, data_format: f"{blob}:{data_format}" + ) + + result = self.layer._read_blobs_parallel(["b", "a"], "json") + + self.assertEqual(result, ["b:json", "a:json"]) + + def test_parallel_read_handles_empty_listing(self) -> None: + self.layer._read_blob_content = Mock() + + result = self.layer._read_blobs_parallel([], "json") + + self.assertEqual(result, []) + self.layer._read_blob_content.assert_not_called() + + def test_load_blob_names_drops_blobs_deleted_after_listing(self) -> None: + self.layer._read_blob_content = Mock( + side_effect=[{"value": 1}, ResourceNotFoundError("missing")] + ) + + result = self.layer._load_blob_names(["first", "missing"], "json") + + self.assertEqual(result, [{"value": 1}]) + + def test_partition_scan_excludes_longer_type(self) -> None: + self.layer.container_client.walk_blobs.return_value = [ + SimpleNamespace(name="partition/model_a.json"), + SimpleNamespace(name="partition/model_catalog_index.json"), + ] + self.layer._read_blob_content = Mock( + side_effect=lambda blob, _: blob.name + ) + + result = self.layer.load_all_from_partition("model") + + self.assertEqual(result, ["partition/model_a.json"]) + + def test_load_all_does_not_cross_configured_partition(self) -> None: + self.layer.container_client.walk_blobs.return_value = [ + SimpleNamespace(name="partition/model_a.json"), + SimpleNamespace(name="other/model_b.json"), + ] + self.layer._read_blob_content = Mock( + side_effect=lambda blob, _: blob.name + ) + + result = self.layer.load_all("model") + + self.assertEqual(result, ["partition/model_a.json"]) + + def test_load_all_reads_matching_blobs_under_directory_markers( + self, + ) -> None: + directory = SimpleNamespace(name="partition/") + nested = SimpleNamespace(name="partition/model_a.json") + self.layer.container_client.walk_blobs.side_effect = [ + [directory], + [nested], + ] + self.layer._read_blob_content = Mock( + side_effect=lambda blob, _: blob.name + ) + + result = self.layer.load_all("model") + + self.assertEqual(result, ["partition/model_a.json"]) + + def test_load_page_skips_stats_and_deep_paths(self) -> None: + blobs = [ + SimpleNamespace( + name="partition/model_stats.json", + metadata={}, + last_modified=datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + SimpleNamespace( + name="partition/nested/model_a.json", + metadata={}, + last_modified=datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + SimpleNamespace( + name="partition/model_a.json", + metadata={}, + last_modified=datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + ] + pages = Mock() + pages.by_page.return_value = [blobs] + self.layer.container_client.list_blobs.return_value = pages + self.layer._load_blob_names = Mock(return_value=[{"modelId": "a"}]) + + records, count = self.layer.load_page("model", page=1, page_size=10) + + self.assertEqual(records, [{"modelId": "a"}]) + self.assertEqual(count, 1) + self.layer._load_blob_names.assert_called_once_with( + ["partition/model_a.json"], "json" + ) + + def test_identifier_listing_excludes_longer_type(self) -> None: + self.layer.container_client.list_blob_names.return_value = [ + "partition/model_a.json", + "partition/model_catalog_index.json", + ] + + result = self.layer.list_identifiers("model") + + self.assertEqual(result, ["a"]) + + @patch( + "hastegeo.core.data_layer.azure_blob_storage_data_layer.generate_container_sas", + return_value="sas", + ) + @patch( + "hastegeo.core.data_layer.azure_blob_storage_data_layer.get_cached_user_delegation_key", + return_value="delegation-key", + ) + def test_remote_path_fetches_delegation_key_lazily( + self, delegation_key, _generate_sas + ) -> None: + self.layer.blob_service_client = Mock() + self.layer.account_key = None + self.layer.user_delegation_key = None + self.layer.container_read_policy = "policy" + self.layer.container_client.account_name = "account" + self.layer.container_client.container_name = "metadata" + self.blob_client.url = "https://account.test/metadata/model_a.json" + + result = self.layer.get_file_remote_path( + "a", "model", check_exists=False + ) + + self.assertEqual(result, f"{self.blob_client.url}?sas") + delegation_key.assert_called_once_with(self.layer.blob_service_client) + + @patch( + "hastegeo.core.data_layer.azure_blob_storage_data_layer.generate_container_sas", + return_value="sas", + ) + @patch( + "hastegeo.core.data_layer.azure_blob_storage_data_layer.get_cached_user_delegation_key" + ) + def test_remote_path_with_account_key_skips_delegation_key( + self, delegation_key, _generate_sas + ) -> None: + self.layer.blob_service_client = Mock() + self.layer.account_key = "account-key" # pragma: allowlist secret + self.layer.user_delegation_key = None + self.layer.container_read_policy = "policy" + self.layer.container_client.account_name = "account" + self.layer.container_client.container_name = "metadata" + self.blob_client.url = "https://account.test/metadata/model_a.json" + + self.layer.get_file_remote_path("a", "model", check_exists=False) + + delegation_key.assert_not_called() + + def test_load_map_preserves_keys_and_missing_records(self) -> None: + self.layer.load = Mock( + side_effect=[{"modelId": "a"}, FileNotFoundError()] + ) + + result = self.layer.load_map( + ["a", "a", "missing"], "model", max_workers=2 + ) + + self.assertEqual(result, {"a": {"modelId": "a"}, "missing": None}) + self.assertEqual(self.layer.load.call_count, 2) + + +class TestAzureBlobStorageDataLayerClientReuse(unittest.TestCase): + def setUp(self) -> None: + _INITIALIZED_CONTAINERS.clear() + + def tearDown(self) -> None: + _INITIALIZED_CONTAINERS.clear() + + @patch( + "hastegeo.core.data_layer.azure_blob_storage_data_layer.get_blob_service_client" + ) + @patch.object( + AzureBlobStorageDataLayer, + "_create_or_update_managed_access_policy", + ) + def test_connection_string_uses_cached_client( + self, create_policy, factory + ): + service = Mock(url="https://account.test") + service.credential.account_key = "key" # pragma: allowlist secret + factory.return_value = service + + layer = AzureBlobStorageDataLayer( + account_url="", + container="metadata", + connection_string="connection", + ) + + self.assertIs(layer.blob_service_client, service) + factory.assert_called_once_with(connection_string="connection") + create_policy.assert_called_once_with() + + @patch( + "hastegeo.core.data_layer.azure_blob_storage_data_layer.get_blob_service_client" + ) + @patch.object( + AzureBlobStorageDataLayer, + "_create_or_update_managed_access_policy", + ) + def test_managed_identity_uses_cached_client(self, create_policy, factory): + service = Mock(url="https://account.test") + service.get_user_delegation_key.return_value = "delegation-key" + factory.return_value = service + + layer = AzureBlobStorageDataLayer( + account_url="https://account.test", + container="metadata", + connection_string=None, + ) + + self.assertIs(layer.blob_service_client, service) + self.assertIsNone(layer.user_delegation_key) + service.get_user_delegation_key.assert_not_called() + factory.assert_called_once_with(account_url="https://account.test") + create_policy.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/hastelib/tests/core/data_layer/test_read_contracts.py b/hastelib/tests/core/data_layer/test_read_contracts.py new file mode 100644 index 00000000..04a614f1 --- /dev/null +++ b/hastelib/tests/core/data_layer/test_read_contracts.py @@ -0,0 +1,367 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock, patch + +from hastegeo.core.data_layer.abstract_data_layer import AbstractDataLayer +from hastegeo.core.data_layer.azure_cosmos_db_data_layer import ( + AzureCosmosDBDataLayer, +) +from hastegeo.core.data_layer.azure_data_lake_data_layer import ( + AzureDataLakeDataLayer, +) +from hastegeo.core.data_layer.azure_postgresql_data_layer import ( + AzurePostgreSQLDataLayer, +) +from hastegeo.core.data_layer.local_file_system_data_layer import ( + LocalFileSystemDataLayer, +) +from hastegeo.core.data_layer.unified import UnifiedDataLayer +from psycopg2 import sql + + +class TestCosmosReadContract(unittest.TestCase): + def setUp(self) -> None: + self.layer = AzureCosmosDBDataLayer.__new__(AzureCosmosDBDataLayer) + self.layer.partition_key = "partition" + self.layer.container = Mock() + + def test_load_accepts_unified_data_format_keyword(self) -> None: + self.layer.container.read_item.return_value = {"value": 1} + + result = self.layer.load("record", "model", data_format="json") + + self.assertEqual(result, {"value": 1}) + + def test_list_identifiers_uses_partition_query(self) -> None: + self.layer.container.query_items.return_value = [ + "model_a", + "model_b", + "model_catalog_index", + ] + + result = self.layer.list_identifiers("model") + + self.assertEqual(result, ["a", "b"]) + call = self.layer.container.query_items.call_args.kwargs + self.assertEqual(call["partition_key"], "partition") + self.assertFalse(call["enable_cross_partition_query"]) + + def test_non_json_identifier_listing_is_empty(self) -> None: + self.assertEqual( + self.layer.list_identifiers("train_labels", "geojson"), [] + ) + self.layer.container.query_items.assert_not_called() + + def test_load_map_uses_one_partition_query_and_preserves_missing( + self, + ) -> None: + self.layer.container.query_items.return_value = [ + {"id": "model_a", "value": 1} + ] + + result = self.layer.load_map( + ["a", "a", "missing"], "model", max_workers=4 + ) + + self.assertEqual( + result, + {"a": {"id": "model_a", "value": 1}, "missing": None}, + ) + call = self.layer.container.query_items.call_args.kwargs + self.assertEqual(call["partition_key"], "partition") + self.assertEqual( + call["parameters"][1], + {"name": "@item_ids", "value": ["model_a", "model_missing"]}, + ) + + def test_partition_load_excludes_longer_metadata_type(self) -> None: + self.layer.container.query_items.return_value = [ + {"id": "model_a"}, + {"id": "model_catalog_index"}, + ] + + result = self.layer.load_all_from_partition("model") + + self.assertEqual(result, [{"id": "model_a"}]) + + def test_global_load_excludes_longer_metadata_type(self) -> None: + self.layer.container.query_items.return_value = [ + {"id": "model_a"}, + {"id": "model_catalog_index"}, + ] + + result = self.layer.load_all("model") + + self.assertEqual(result, [{"id": "model_a"}]) + + def test_load_map_handles_empty_and_non_json_inputs(self) -> None: + self.assertEqual(self.layer.load_map([], "model"), {}) + with self.assertRaisesRegex(ValueError, "only json"): + self.layer.load_map(["a"], "model", data_format="geojson") + + def test_read_methods_reject_non_json_format(self) -> None: + for method, args in ( + (self.layer.load, ("record", "model")), + (self.layer.load_all, ("model",)), + (self.layer.load_all_from_partition, ("model",)), + ): + with self.subTest(method=method.__name__): + with self.assertRaisesRegex(ValueError, "only json"): + method(*args, data_format="yaml") + + +class TestDataLakeReadContract(unittest.TestCase): + def setUp(self) -> None: + self.layer = AzureDataLakeDataLayer.__new__(AzureDataLakeDataLayer) + self.layer.partition_key = "partition" + self.layer.file_system_client = Mock() + + def test_remote_path_can_skip_exists_request(self) -> None: + file_client = ( + self.layer.file_system_client.get_file_client.return_value + ) + file_client.url = "https://account.test/file" + + result = self.layer.get_file_remote_path( + "record", "model", check_exists=False + ) + + self.assertEqual(result, "https://account.test/file") + file_client.exists.assert_not_called() + + def test_missing_remote_path_returns_none(self) -> None: + file_client = ( + self.layer.file_system_client.get_file_client.return_value + ) + file_client.exists.return_value = False + + result = self.layer.get_file_remote_path("record", "model") + + self.assertIsNone(result) + + def test_list_identifiers_strips_prefix_and_suffix(self) -> None: + self.layer.file_system_client.get_paths.return_value = [ + SimpleNamespace(name="partition/model_a.json"), + SimpleNamespace(name="partition/model_b.json"), + SimpleNamespace(name="partition/model_catalog_index.json"), + SimpleNamespace(name="partition/labels_c.json"), + ] + + result = self.layer.list_identifiers("model") + + self.assertEqual(result, ["a", "b"]) + self.layer.file_system_client.get_paths.assert_called_once_with( + path="partition" + ) + + def test_load_accepts_unified_data_format_keyword(self) -> None: + file_client = ( + self.layer.file_system_client.get_file_client.return_value + ) + file_client.download_file.return_value.readall.return_value = ( + b'{"value": 1}' + ) + + result = self.layer.load("record", "model", data_format="json") + + self.assertEqual(result, {"value": 1}) + + def test_load_all_and_partition_forward_json_format(self) -> None: + self.layer.file_system_client.get_paths.return_value = [] + self.assertEqual(self.layer.load_all("model", data_format="json"), []) + with patch.object(self.layer, "load_all", return_value=[]) as load_all: + self.assertEqual( + self.layer.load_all_from_partition( + "model", data_format="json" + ), + [], + ) + load_all.assert_called_once_with("model", data_format="json") + + def test_load_all_does_not_cross_configured_partition(self) -> None: + self.layer.file_system_client.get_paths.return_value = [ + SimpleNamespace(name="partition/model_a.json"), + SimpleNamespace(name="other/model_b.json"), + ] + file_client = ( + self.layer.file_system_client.get_file_client.return_value + ) + file_client.download_file.return_value.readall.return_value = ( + b'{"id": "a"}' + ) + + result = self.layer.load_all("model") + + self.assertEqual(result, [{"id": "a"}]) + self.layer.file_system_client.get_file_client.assert_called_once_with( + "partition/model_a.json" + ) + + def test_read_methods_reject_non_json_format(self) -> None: + for method, args in ( + (self.layer.load, ("record", "model")), + (self.layer.load_all, ("model",)), + ): + with self.subTest(method=method.__name__): + with self.assertRaisesRegex(ValueError, "only json"): + method(*args, data_format="yaml") + + def test_bounded_load_skips_deep_and_other_type_paths(self) -> None: + self.layer.file_system_client.get_paths.return_value = [ + SimpleNamespace(name="partition/nested/model_a.json"), + SimpleNamespace(name="partition/labels_a.json"), + ] + + self.assertEqual(self.layer.load_bounded("model", 2), []) + + +class TestPostgreSQLReadContract(unittest.TestCase): + def setUp(self) -> None: + self.layer = AzurePostgreSQLDataLayer.__new__(AzurePostgreSQLDataLayer) + self.layer.partition_key = "partition" + self.layer.server_name = "server" + self.layer.database_name = "database" + self.layer.postgres_user = "user" + self.layer.token = "token" + self.layer._qualified_table_identifier = sql.Identifier("metadata") + + @patch( + "hastegeo.core.data_layer.azure_postgresql_data_layer.psycopg2.connect" + ) + def test_load_accepts_jsonb_dictionary(self, connect) -> None: + cursor = self._cursor(connect) + cursor.fetchone.return_value = ({"value": 1},) + + result = self.layer.load("record", "model", data_format="json") + + self.assertEqual(result, {"value": 1}) + + @patch( + "hastegeo.core.data_layer.azure_postgresql_data_layer.psycopg2.connect" + ) + def test_list_identifiers_is_partition_scoped(self, connect) -> None: + cursor = self._cursor(connect) + cursor.fetchall.return_value = [("a",), ("b",)] + + result = self.layer.list_identifiers("model") + + self.assertEqual(result, ["a", "b"]) + self.assertEqual( + cursor.execute.call_args.args[1], ("model", "partition") + ) + + def test_non_json_identifier_listing_is_empty(self) -> None: + self.assertEqual( + self.layer.list_identifiers("train_labels", "geojson"), [] + ) + + @patch( + "hastegeo.core.data_layer.azure_postgresql_data_layer.psycopg2.connect" + ) + def test_load_map_uses_one_query_and_preserves_missing( + self, connect + ) -> None: + cursor = self._cursor(connect) + cursor.fetchall.return_value = [("a", {"value": 1})] + + result = self.layer.load_map( + ["a", "a", "missing"], "model", max_workers=4 + ) + + self.assertEqual(result, {"a": {"value": 1}, "missing": None}) + self.assertEqual( + cursor.execute.call_args.args[1], + ("model", "partition", ["a", "missing"]), + ) + + def test_load_map_handles_empty_and_non_json_inputs(self) -> None: + self.assertEqual(self.layer.load_map([], "model"), {}) + with self.assertRaisesRegex(ValueError, "only json"): + self.layer.load_map(["a"], "model", data_format="geojson") + + @patch( + "hastegeo.core.data_layer.azure_postgresql_data_layer.psycopg2.connect" + ) + def test_load_all_accepts_jsonb_values(self, connect) -> None: + cursor = self._cursor(connect) + cursor.fetchall.return_value = [({"value": 1},)] + + self.assertEqual( + self.layer.load_all("model", data_format="json"), + [{"value": 1}], + ) + + @patch( + "hastegeo.core.data_layer.azure_postgresql_data_layer.psycopg2.connect" + ) + def test_load_partition_accepts_serialized_values(self, connect) -> None: + cursor = self._cursor(connect) + cursor.fetchall.return_value = [('{"value": 1}',)] + + self.assertEqual( + self.layer.load_all_from_partition("model", data_format="json"), + [{"value": 1}], + ) + + def test_read_methods_reject_non_json_format(self) -> None: + for method, args in ( + (self.layer.load, ("record", "model")), + (self.layer.load_all, ("model",)), + (self.layer.load_all_from_partition, ("model",)), + ): + with self.subTest(method=method.__name__): + with self.assertRaisesRegex(ValueError, "only json"): + method(*args, data_format="yaml") + + @staticmethod + def _cursor(connect) -> MagicMock: + connection = MagicMock() + cursor = MagicMock() + connect.return_value.__enter__.return_value = connection + connection.cursor.return_value.__enter__.return_value = cursor + return cursor + + +class TestOptionalRemotePathContract(unittest.TestCase): + def test_default_remote_path_is_explicitly_unsupported(self) -> None: + with self.assertRaisesRegex(NotImplementedError, "remote file paths"): + AbstractDataLayer.get_file_remote_path(object()) + + def test_default_batch_read_is_explicitly_unsupported(self) -> None: + with self.assertRaisesRegex(NotImplementedError, "batch reads"): + AbstractDataLayer.load_map(object(), [], "model") + + +class TestLocalReadContract(unittest.TestCase): + def test_bounded_load_skips_nonmatching_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + layer = LocalFileSystemDataLayer(directory) + layer.save("index", "model_catalog", {"models": []}) + + self.assertEqual(layer.load_bounded("model", 1), []) + + +class TestUnifiedReadContract(unittest.TestCase): + def test_load_map_delegates_all_arguments(self) -> None: + unified = UnifiedDataLayer.__new__(UnifiedDataLayer) + unified.data_layer = Mock() + unified.data_layer.load_map.return_value = {"a": {"value": 1}} + + result = unified.load_map( + ["a"], "model", data_format="json", max_workers=4 + ) + + self.assertEqual(result, {"a": {"value": 1}}) + unified.data_layer.load_map.assert_called_once_with( + identifiers=["a"], + data_type="model", + data_format="json", + max_workers=4, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/hastelib/tests/core/processors/test_artifacts.py b/hastelib/tests/core/processors/test_artifacts.py index 999a56a1..eae42d9a 100644 --- a/hastelib/tests/core/processors/test_artifacts.py +++ b/hastelib/tests/core/processors/test_artifacts.py @@ -1,53 +1,25 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import os - -from azure.core.exceptions import ResourceExistsError -from azure.storage.blob import BlobServiceClient from hastegeo.core.processors.artifacts import ArtifactProcessor class TestArtifactProcessor: - def test_zip(self, mocker): - # Arrange - blob_service_client = BlobServiceClient.from_connection_string( - os.environ.get("BLOB_CONNECTION_STRING") + def test_fetch_artifact_delegates_to_storage(self, mocker): + processor = ArtifactProcessor.__new__(ArtifactProcessor) + processor.storage = mocker.Mock() + processor.storage.fetch_artifact.return_value = "/tmp/output" + + result = processor.fetch_artifact( + identifier="artifact", + extra_partition_keys=["model"], + src_path="source", + dst_path="/tmp/output", ) - try: - container_client = blob_service_client.create_container( - os.getenv("BLOB_CONTAINER") - ) - except ResourceExistsError: - container_client = blob_service_client.get_container_client( - os.getenv("BLOB_CONTAINER") - ) - except Exception as e: - print(e) - - test_artifacts = { - "folder1": ["test1.txt", "test2.txt"], - "folder2": ["test3.txt", "test4.txt"], - } - - for folder in test_artifacts.keys(): - for file in test_artifacts[folder]: - file_path = os.path.join(folder, file) - blob_client = container_client.get_blob_client(file_path) - blob_client.upload_blob( - r"This is a test file.", overwrite=True - ) - model_id = "1234" - model_name = "test_model_name" - - # Act - processor = ArtifactProcessor() - result = processor.zip( - artifact_paths=["folder1", "folder2"], - zip_path=f"model_{model_id}_artifacts/{model_name}.zip", + assert result == "/tmp/output" + processor.storage.fetch_artifact.assert_called_once_with( + identifier="artifact", + extra_partition_keys=["model"], + src_path="source", + dst_path="/tmp/output", ) - - # Assertions - assert result == f"model_{model_id}_artifacts/{model_name}.zip" - expected_blob_client = container_client.get_blob_client(result) - assert expected_blob_client.exists() is True diff --git a/hastelib/tests/core/processors/test_metadata_batch.py b/hastelib/tests/core/processors/test_metadata_batch.py new file mode 100644 index 00000000..fa6e0c98 --- /dev/null +++ b/hastelib/tests/core/processors/test_metadata_batch.py @@ -0,0 +1,171 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Tests for the batch metadata primitives added in the perf-layer-loading work: +``MetadataProcessor.load_map`` / ``load_filtered`` / ``list_keys`` / ``build_url``. + +Runs against the local filesystem backend so no Azure/Azurite is required. +""" +import importlib +from unittest.mock import Mock + +import pytest + + +@pytest.fixture() +def local_metadata(tmp_path, monkeypatch): + monkeypatch.setenv("METADATA_STORAGE_TYPE", "local") + monkeypatch.setenv("DATA_PATH", str(tmp_path)) + # Config reads env at construction; import fresh each test. + metadata = importlib.import_module("hastegeo.core.processors.metadata") + return metadata.MetadataProcessor + + +def _seed(MetadataProcessor, partition): + for i in range(5): + MetadataProcessor("model", partition).save( + f"m{i}", + {"modelId": f"m{i}", "imageLayerId": f"layer-{i % 2}"}, + ) + + +def test_list_keys_returns_all_identifiers(local_metadata): + MetadataProcessor = local_metadata + _seed(MetadataProcessor, "p1") + keys = set(MetadataProcessor("model", "p1").list_keys()) + assert keys == {"m0", "m1", "m2", "m3", "m4"} + + +def test_partition_scan_excludes_longer_metadata_type(local_metadata): + MetadataProcessor = local_metadata + MetadataProcessor("model", "prefix").save("1", {"modelId": "1"}) + MetadataProcessor("model_catalog", "prefix").save( + "index", {"modelCatalog": []} + ) + + models = MetadataProcessor("model", "prefix") + + assert models.load_all_from_partition() == [{"modelId": "1"}] + assert models.list_keys() == ["1"] + + +def test_load_map_parallel_matches_sequential(local_metadata): + MetadataProcessor = local_metadata + _seed(MetadataProcessor, "p2") + mp = MetadataProcessor("model", "p2") + keys = mp.list_keys() + + mapped = mp.load_map(keys, max_workers=4) + sequential = {k: mp.load(k) for k in keys} + assert mapped == sequential + + +def test_load_map_missing_key_is_none(local_metadata): + MetadataProcessor = local_metadata + _seed(MetadataProcessor, "p3") + mp = MetadataProcessor("model", "p3") + result = mp.load_map(["m0", "does-not-exist"]) + assert result["m0"]["modelId"] == "m0" + assert result["does-not-exist"] is None + + +def test_load_map_empty(local_metadata): + MetadataProcessor = local_metadata + assert MetadataProcessor("model", "p4").load_map([]) == {} + + +def test_load_map_rejects_invalid_worker_count(local_metadata): + MetadataProcessor = local_metadata + _seed(MetadataProcessor, "invalid-workers") + + with pytest.raises(ValueError, match="max_workers"): + MetadataProcessor("model", "invalid-workers").load_map( + ["m0"], max_workers=0 + ) + + +def test_load_map_deduplicates_keys(local_metadata, mocker): + MetadataProcessor = local_metadata + _seed(MetadataProcessor, "duplicate-keys") + processor = MetadataProcessor("model", "duplicate-keys") + load = mocker.spy(processor, "load") + + result = processor.load_map(["m0", "m0"], max_workers=2) + + assert result["m0"]["modelId"] == "m0" + load.assert_called_once_with("m0", data_format="json") + + +def test_load_map_prefers_backend_native_batch(local_metadata): + MetadataProcessor = local_metadata + processor = MetadataProcessor.__new__(MetadataProcessor) + processor.data_type = "model" + processor.storage = Mock() + processor.storage.supports_load_map.return_value = True + processor.storage.load_map.return_value = { + "m0": {"modelId": "m0"}, + "missing": None, + } + + result = processor.load_map(["m0", "m0", "missing"]) + + assert result == {"m0": {"modelId": "m0"}, "missing": None} + processor.storage.load_map.assert_called_once_with( + identifiers=["m0", "missing"], + data_type="model", + data_format="json", + max_workers=8, + ) + + +def test_native_load_map_rejects_invalid_worker_count(local_metadata): + MetadataProcessor = local_metadata + processor = MetadataProcessor.__new__(MetadataProcessor) + processor.data_type = "model" + processor.storage = Mock() + processor.storage.supports_load_map.return_value = True + + with pytest.raises(ValueError, match="max_workers"): + processor.load_map(["m0"], max_workers=0) + + processor.storage.load_map.assert_not_called() + + +def test_load_filtered_by_field(local_metadata): + MetadataProcessor = local_metadata + _seed(MetadataProcessor, "p5") + matched = MetadataProcessor("model", "p5").load_filtered( + {"imageLayerId": "layer-0"} + ) + assert {m["modelId"] for m in matched} == {"m0", "m2", "m4"} + + +def test_load_filtered_rejects_empty_predicate(local_metadata): + MetadataProcessor = local_metadata + + with pytest.raises(ValueError, match="non-empty"): + MetadataProcessor("model", "empty-predicate").load_filtered({}) + + +def test_load_filtered_does_not_treat_missing_field_as_none(local_metadata): + MetadataProcessor = local_metadata + processor = MetadataProcessor("model", "missing-field") + processor.save("missing", {"modelId": "missing"}) + processor.save("explicit", {"modelId": "explicit", "status": None}) + + matched = processor.load_filtered({"status": None}) + + assert [record["modelId"] for record in matched] == ["explicit"] + + +def test_load_map_counts_round_trips_across_threads(local_metadata): + MetadataProcessor = local_metadata + _seed(MetadataProcessor, "p6") + perf = importlib.import_module("hastegeo.core.utils.perf") + mp = MetadataProcessor("model", "p6") + keys = mp.list_keys() + + counter = perf.begin(True) + mp.load_map(keys, max_workers=4) + perf.end() + # Each threaded load records against the shared counter (context bound). + assert counter.calls >= len(keys) diff --git a/hastelib/tests/core/processors/test_project_details.py b/hastelib/tests/core/processors/test_project_details.py new file mode 100644 index 00000000..15369522 --- /dev/null +++ b/hastelib/tests/core/processors/test_project_details.py @@ -0,0 +1,463 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json +import os +import tempfile +import unittest +from unittest.mock import Mock, patch + +from hastegeo.core.config import Config +from hastegeo.core.processors.metadata import MetadataProcessor +from hastegeo.core.processors.project_details import ( + ProjectDetailsProcessor, + assemble_project_details, +) + + +class TestAssembleProjectDetails(unittest.TestCase): + def setUp(self) -> None: + self.project = {"projectId": "project-1", "name": "Project"} + self.layers = [ + { + "imageLayerId": "layer-old", + "creationDate": "2026-01-01T00:00:00Z", + }, + { + "imageLayerId": "layer-new", + "creationDate": "2026-02-01T00:00:00Z", + }, + ] + + def test_joins_records_by_storage_key_when_body_ids_are_missing( + self, + ) -> None: + models = [ + { + "modelId": "model-1", + "imageLayerId": "layer-new", + "creationDate": "2026-02-02T00:00:00Z", + } + ] + artifacts = {"model-1": {"metrics": {"iou": 0.8}}} + validations = {"layer-new": {"labels": {"building-1": {}}}} + + result = assemble_project_details( + self.project, + self.layers, + models, + [{"imageLayerId": "layer-new", "labels": [{"id": "a"}]}], + artifacts, + validations, + {"model-1": "https://example.test/model-1.geojson"}, + include_models=True, + ) + + newest_layer = result["imageLayer"][0] + self.assertEqual(newest_layer["imageLayerId"], "layer-new") + self.assertEqual( + newest_layer["models"][0]["artifacts"], artifacts["model-1"] + ) + self.assertEqual(newest_layer["validationLabelCount"], 1) + self.assertEqual(newest_layer["labelProjectCount"], 1) + self.assertEqual( + newest_layer["models"][0]["labelsUrl"], + "https://example.test/model-1.geojson", + ) + + def test_preserves_existing_labels_url(self) -> None: + models = [ + { + "modelId": "model-1", + "imageLayerId": "layer-new", + "creationDate": "2026-02-02T00:00:00Z", + "labelsUrl": "https://stored.test/labels.geojson", + } + ] + + result = assemble_project_details( + self.project, + self.layers, + models, + [], + {}, + {}, + {"model-1": "https://generated.test/labels.geojson"}, + include_models=True, + ) + + model = result["imageLayer"][0]["models"][0] + self.assertEqual( + model["labelsUrl"], "https://stored.test/labels.geojson" + ) + self.assertIsNone(model["artifacts"]) + + def test_preserves_existing_layer_labels_url(self) -> None: + layer = dict( + self.layers[1], labelsUrl="https://stored.test/layer.geojson" + ) + + result = assemble_project_details( + self.project, + [layer], + [], + [{"imageLayerId": "layer-new", "labels": []}], + {}, + {}, + {}, + include_models=False, + ) + + self.assertEqual( + result["imageLayer"][0]["labelsUrl"], + "https://stored.test/layer.geojson", + ) + + def test_uses_first_label_project_for_duplicate_layer(self) -> None: + result = assemble_project_details( + self.project, + [self.layers[1]], + [], + [ + {"imageLayerId": "layer-new", "labels": [{"id": "first"}]}, + { + "imageLayerId": "layer-new", + "labels": [{"id": "second"}, {"id": "third"}], + }, + {"labels": []}, + ], + {}, + {}, + {}, + include_models=False, + ) + + self.assertEqual(result["imageLayer"][0]["labelProjectCount"], 1) + + def test_missing_related_records_keep_legacy_defaults(self) -> None: + result = assemble_project_details( + self.project, + self.layers, + [], + [], + {}, + {}, + {}, + include_models=True, + ) + + for image_layer in result["imageLayer"]: + self.assertEqual(image_layer["models"], []) + self.assertEqual(image_layer["modelCount"], 0) + self.assertEqual(image_layer["validationLabelCount"], 0) + self.assertNotIn("labelProjectCount", image_layer) + + def test_excluding_models_does_not_add_model_fields(self) -> None: + result = assemble_project_details( + self.project, + self.layers, + [], + [], + {}, + {}, + {}, + include_models=False, + ) + + for image_layer in result["imageLayer"]: + self.assertNotIn("models", image_layer) + self.assertNotIn("modelCount", image_layer) + + def test_inputs_are_not_mutated(self) -> None: + assemble_project_details( + self.project, + self.layers, + [], + [], + {}, + {}, + {}, + include_models=False, + ) + + self.assertNotIn("imageLayer", self.project) + for image_layer in self.layers: + self.assertNotIn("validationLabelCount", image_layer) + + def test_complete_fixture_matches_expected_response_bytes(self) -> None: + result = assemble_project_details( + self.project, + [self.layers[1]], + [ + { + "modelId": "model-1", + "imageLayerId": "layer-new", + "creationDate": "2026-02-02T00:00:00Z", + } + ], + [{"imageLayerId": "layer-new", "labels": [{"id": "a"}]}], + {"model-1": {"metrics": {"iou": 0.8}}}, + {"layer-new": {"labels": {"building-1": {}}}}, + {"model-1": "https://example.test/model-1.geojson"}, + include_models=True, + ) + expected = { + "projectId": "project-1", + "name": "Project", + "imageLayer": [ + { + "imageLayerId": "layer-new", + "creationDate": "2026-02-01T00:00:00Z", + "models": [ + { + "modelId": "model-1", + "imageLayerId": "layer-new", + "creationDate": "2026-02-02T00:00:00Z", + "artifacts": {"metrics": {"iou": 0.8}}, + "labelsUrl": ( + "https://example.test/model-1.geojson" + ), + } + ], + "modelCount": 1, + "labelProjectCount": 1, + "labelsUrl": None, + "validationLabelCount": 1, + } + ], + "imageLayerCount": 1, + } + + self.assertEqual(json.dumps(result), json.dumps(expected)) + + +class TestProjectDetailsProcessor(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.types = Config.get_metadata_types() + self.config = Mock() + self.config.get_metadata_types.return_value = self.types + self.processors = {} + self.created_types = [] + + for data_type in ( + self.types.PROJECT.value, + self.types.IMAGELAYER.value, + self.types.LABELS.value, + self.types.MODEL.value, + self.types.VALIDATION.value, + self.types.MODEL_ARTIFACTS.value, + self.types.TRAIN_LABELS.value, + ): + self.processors[data_type] = Mock() + + self.processors[self.types.PROJECT.value].load.return_value = { + "projectId": "project-1", + "name": "Project", + } + self.processors[ + self.types.IMAGELAYER.value + ].load_all_from_partition.return_value = [ + { + "imageLayerId": "layer-1", + "creationDate": "2026-01-01T00:00:00Z", + } + ] + self.processors[ + self.types.LABELS.value + ].load_all_from_partition.return_value = [] + self.processors[ + self.types.MODEL.value + ].load_all_from_partition.return_value = [ + { + "modelId": "model-1", + "imageLayerId": "layer-1", + "creationDate": "2026-01-02T00:00:00Z", + } + ] + self.processors[self.types.VALIDATION.value].load_map.return_value = { + "layer-1": {"labels": {"building-1": {}}} + } + self.processors[ + self.types.MODEL_ARTIFACTS.value + ].load_map.return_value = {"model-1": {"metrics": {"iou": 0.8}}} + train_labels = self.processors[self.types.TRAIN_LABELS.value] + train_labels.list_keys.return_value = ["model-1"] + train_labels.build_url.return_value = ( + "https://example.test/model-1.geojson" + ) + + def factory(self, *, data_type, partition_key, config): + self.assertEqual(partition_key, "project-1") + self.assertIs(config, self.config) + self.created_types.append(data_type) + return self.processors[data_type] + + async def test_load_uses_keyed_maps_for_related_records(self) -> None: + processor = ProjectDetailsProcessor( + "project-1", self.config, self.factory + ) + + result = await processor.load(include_models=True) + + model = result["imageLayer"][0]["models"][0] + self.assertEqual(model["artifacts"], {"metrics": {"iou": 0.8}}) + self.assertEqual(result["imageLayer"][0]["validationLabelCount"], 1) + self.processors[ + self.types.VALIDATION.value + ].load_map.assert_called_once_with(["layer-1"]) + self.processors[ + self.types.MODEL_ARTIFACTS.value + ].load_map.assert_called_once_with(["model-1"]) + + async def test_load_without_models_skips_model_storage(self) -> None: + processor = ProjectDetailsProcessor( + "project-1", self.config, self.factory + ) + + result = await processor.load(include_models=False) + + self.assertNotIn(self.types.MODEL.value, self.created_types) + self.assertNotIn(self.types.MODEL_ARTIFACTS.value, self.created_types) + self.assertNotIn(self.types.TRAIN_LABELS.value, self.created_types) + self.assertNotIn("models", result["imageLayer"][0]) + + async def test_train_label_listing_falls_back_to_legacy_export( + self, + ) -> None: + train_labels = self.processors[self.types.TRAIN_LABELS.value] + train_labels.list_keys.side_effect = NotImplementedError + train_labels.export.return_value = ( + "https://fallback.test/model-1.geojson" + ) + processor = ProjectDetailsProcessor( + "project-1", self.config, self.factory + ) + + result = await processor.load(include_models=True) + + model = result["imageLayer"][0]["models"][0] + self.assertEqual( + model["labelsUrl"], "https://fallback.test/model-1.geojson" + ) + train_labels.export.assert_called_once_with( + "model-1", data_format="geojson" + ) + + async def test_missing_legacy_train_label_export_returns_none( + self, + ) -> None: + train_labels = self.processors[self.types.TRAIN_LABELS.value] + train_labels.list_keys.side_effect = NotImplementedError + train_labels.export.side_effect = FileNotFoundError + processor = ProjectDetailsProcessor( + "project-1", self.config, self.factory + ) + + result = await processor.load(include_models=True) + + self.assertIsNone(result["imageLayer"][0]["models"][0]["labelsUrl"]) + + async def test_existing_model_url_skips_train_label_storage(self) -> None: + model = self.processors[self.types.MODEL.value] + model.load_all_from_partition.return_value[0][ + "labelsUrl" + ] = "https://stored.test/model.geojson" + processor = ProjectDetailsProcessor( + "project-1", self.config, self.factory + ) + + result = await processor.load(include_models=True) + + self.assertEqual( + result["imageLayer"][0]["models"][0]["labelsUrl"], + "https://stored.test/model.geojson", + ) + self.assertNotIn(self.types.TRAIN_LABELS.value, self.created_types) + + async def test_unsupported_train_label_url_is_returned_as_none( + self, + ) -> None: + train_labels = self.processors[self.types.TRAIN_LABELS.value] + train_labels.build_url.side_effect = NotImplementedError + processor = ProjectDetailsProcessor( + "project-1", self.config, self.factory + ) + + result = await processor.load(include_models=True) + + model = result["imageLayer"][0]["models"][0] + self.assertIsNone(model["labelsUrl"]) + + async def test_project_load_failure_stops_partition_reads(self) -> None: + self.processors[ + self.types.PROJECT.value + ].load.side_effect = FileNotFoundError + processor = ProjectDetailsProcessor( + "project-1", self.config, self.factory + ) + + with self.assertRaises(FileNotFoundError): + await processor.load(include_models=True) + + self.assertEqual(self.created_types, [self.types.PROJECT.value]) + + +class TestProjectDetailsLocalStorage(unittest.IsolatedAsyncioTestCase): + async def test_load_preserves_keyed_legacy_records_end_to_end( + self, + ) -> None: + with tempfile.TemporaryDirectory() as data_path: + with patch.dict( + os.environ, + { + "METADATA_STORAGE_TYPE": "local", + "DATA_PATH": data_path, + }, + ): + config = Config() + types = config.get_metadata_types() + records = ( + ( + types.PROJECT.value, + "project-1", + {"projectId": "project-1"}, + ), + ( + types.IMAGELAYER.value, + "layer-1", + { + "imageLayerId": "layer-1", + "creationDate": "2026-01-01T00:00:00Z", + }, + ), + ( + types.MODEL.value, + "model-1", + { + "modelId": "model-1", + "imageLayerId": "layer-1", + "creationDate": "2026-01-02T00:00:00Z", + }, + ), + (types.MODEL_ARTIFACTS.value, "model-1", {"metrics": {}}), + (types.VALIDATION.value, "layer-1", {"labels": {"a": {}}}), + ) + for data_type, key, value in records: + MetadataProcessor( + data_type=data_type, + partition_key="project-1", + config=config, + ).save(key, value) + + result = await ProjectDetailsProcessor( + "project-1", config=config + ).load(include_models=True) + + image_layer = result["imageLayer"][0] + self.assertEqual( + image_layer["models"][0]["artifacts"], {"metrics": {}} + ) + self.assertEqual(image_layer["validationLabelCount"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/hastelib/tests/core/utils/test_async_cache.py b/hastelib/tests/core/utils/test_async_cache.py new file mode 100644 index 00000000..cf2dab3d --- /dev/null +++ b/hastelib/tests/core/utils/test_async_cache.py @@ -0,0 +1,180 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import asyncio +import os +import unittest +from unittest.mock import AsyncMock, patch + +from hastegeo.core.utils.async_cache import ( + AsyncTTLCache, + configured_cache_value, +) + + +class TestConfiguredCacheValue(unittest.TestCase): + def test_reads_bounded_integer(self) -> None: + with patch.dict(os.environ, {"TEST_CACHE_VALUE": "12"}): + self.assertEqual( + configured_cache_value("TEST_CACHE_VALUE", 5, 0, 20), 12 + ) + + def test_rejects_invalid_value(self) -> None: + for value in ("invalid", "-1", "21"): + with self.subTest(value=value): + with patch.dict(os.environ, {"TEST_CACHE_VALUE": value}): + with self.assertRaises(ValueError): + configured_cache_value("TEST_CACHE_VALUE", 5, 0, 20) + + +class TestAsyncTTLCache(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.now = 0.0 + self.cache = AsyncTTLCache( + ttl_seconds=10, + max_entries=2, + clock=lambda: self.now, + ) + + async def asyncTearDown(self) -> None: + await self.cache.clear() + + async def test_reuses_value_before_expiry(self) -> None: + factory = AsyncMock(return_value="value") + + first, first_reused = await self.cache.get_or_create("key", factory) + second, second_reused = await self.cache.get_or_create("key", factory) + + self.assertEqual((first, second), ("value", "value")) + self.assertFalse(first_reused) + self.assertTrue(second_reused) + factory.assert_awaited_once_with() + + async def test_reloads_value_after_expiry(self) -> None: + factory = AsyncMock(side_effect=["first", "second"]) + await self.cache.get_or_create("key", factory) + self.now = 10 + + value, reused = await self.cache.get_or_create("key", factory) + + self.assertEqual(value, "second") + self.assertFalse(reused) + self.assertEqual(factory.await_count, 2) + + async def test_refresh_replaces_unexpired_value(self) -> None: + factory = AsyncMock(side_effect=["first", "second"]) + await self.cache.get_or_create("key", factory) + + refreshed, reused = await self.cache.get_or_create( + "key", factory, refresh=True + ) + cached, cached_reused = await self.cache.get_or_create("key", factory) + + self.assertEqual((refreshed, cached), ("second", "second")) + self.assertFalse(reused) + self.assertTrue(cached_reused) + self.assertEqual(factory.await_count, 2) + + async def test_refresh_supersedes_an_older_inflight_load(self) -> None: + old_started = asyncio.Event() + old_release = asyncio.Event() + + async def old_factory() -> str: + old_started.set() + await old_release.wait() + return "old" + + old_request = asyncio.create_task( + self.cache.get_or_create("key", old_factory) + ) + await old_started.wait() + + refreshed, reused = await self.cache.get_or_create( + "key", lambda: asyncio.sleep(0, result="new"), refresh=True + ) + old_release.set() + old_value, _ = await old_request + cached, cached_reused = await self.cache.get_or_create( + "key", lambda: asyncio.sleep(0, result="unexpected") + ) + + self.assertEqual((old_value, refreshed, cached), ("old", "new", "new")) + self.assertFalse(reused) + self.assertTrue(cached_reused) + + async def test_concurrent_requests_share_one_load(self) -> None: + started = asyncio.Event() + release = asyncio.Event() + + async def factory() -> str: + started.set() + await release.wait() + return "value" + + first = asyncio.create_task(self.cache.get_or_create("key", factory)) + await started.wait() + second = asyncio.create_task(self.cache.get_or_create("key", factory)) + await asyncio.sleep(0) + release.set() + + self.assertEqual( + await asyncio.gather(first, second), + [("value", False), ("value", True)], + ) + + async def test_failed_load_is_not_cached(self) -> None: + factory = AsyncMock(side_effect=[RuntimeError("failed"), "recovered"]) + + with self.assertRaisesRegex(RuntimeError, "failed"): + await self.cache.get_or_create("key", factory) + value, reused = await self.cache.get_or_create("key", factory) + + self.assertEqual(value, "recovered") + self.assertFalse(reused) + self.assertEqual(factory.await_count, 2) + + async def test_evicts_least_recently_used_entry(self) -> None: + factories = { + key: AsyncMock(return_value=key) for key in ("a", "b", "c") + } + for key in ("a", "b", "a", "c"): + await self.cache.get_or_create(key, factories[key]) + + _, reused = await self.cache.get_or_create("b", factories["b"]) + + self.assertFalse(reused) + self.assertEqual(factories["a"].await_count, 1) + self.assertEqual(factories["b"].await_count, 2) + + async def test_clear_removes_cached_values(self) -> None: + factory = AsyncMock(return_value="value") + await self.cache.get_or_create("key", factory) + + await self.cache.clear() + _, reused = await self.cache.get_or_create("key", factory) + + self.assertFalse(reused) + self.assertEqual(factory.await_count, 2) + + async def test_clear_cancels_inflight_load(self) -> None: + started = asyncio.Event() + release = asyncio.Event() + + async def factory() -> str: + started.set() + await release.wait() + return "value" + + request = asyncio.create_task(self.cache.get_or_create("key", factory)) + await started.wait() + + await self.cache.clear() + + with self.assertRaises(asyncio.CancelledError): + await request + await asyncio.sleep(0) + + def test_rejects_invalid_configuration(self) -> None: + with self.assertRaises(ValueError): + AsyncTTLCache(ttl_seconds=-1, max_entries=1) + with self.assertRaises(ValueError): + AsyncTTLCache(ttl_seconds=1, max_entries=0) diff --git a/hastelib/tests/core/utils/test_blob.py b/hastelib/tests/core/utils/test_blob.py index 52f8a122..5bc88a56 100644 --- a/hastelib/tests/core/utils/test_blob.py +++ b/hastelib/tests/core/utils/test_blob.py @@ -13,16 +13,144 @@ existing test_artifacts.py. """ +import os import unittest from unittest.mock import MagicMock, patch from hastegeo.core.utils.blob import ( + clear_blob_client_caches, + download_blob_to_tempfile, fetch_url_text, + get_blob_service_client, + get_cached_user_delegation_key, parse_byte_range, + read_blob_range, split_blob_url, ) +class TestBlobServiceClientCache(unittest.TestCase): + def setUp(self): + clear_blob_client_caches() + + def tearDown(self): + clear_blob_client_caches() + + @patch("azure.storage.blob.BlobServiceClient.from_connection_string") + def test_reuses_client_for_connection_string(self, from_connection_string): + first = get_blob_service_client(connection_string="connection") + second = get_blob_service_client(connection_string="connection") + + self.assertIs(first, second) + from_connection_string.assert_called_once_with("connection") + + @patch("azure.storage.blob.BlobServiceClient") + @patch("azure.identity.DefaultAzureCredential") + def test_reuses_client_for_account_url(self, credential, client_class): + first = get_blob_service_client(account_url="https://account.test") + second = get_blob_service_client(account_url="https://account.test") + + self.assertIs(first, second) + credential.assert_called_once_with() + client_class.assert_called_once_with( + account_url="https://account.test", + credential=credential.return_value, + ) + + def test_requires_connection_target(self): + with self.assertRaises(ValueError): + get_blob_service_client() + + def test_reuses_unexpired_user_delegation_key(self): + from datetime import datetime, timedelta, timezone + + client = MagicMock(url="https://account.test") + client.get_user_delegation_key.return_value = "delegation-key" + now = datetime(2026, 9, 1, tzinfo=timezone.utc) + + first = get_cached_user_delegation_key(client, now=now) + second = get_cached_user_delegation_key( + client, now=now + timedelta(hours=1) + ) + + self.assertEqual(first, "delegation-key") + self.assertEqual(second, "delegation-key") + client.get_user_delegation_key.assert_called_once() + + def test_refreshes_expiring_user_delegation_key(self): + from datetime import datetime, timedelta, timezone + + client = MagicMock(url="https://account.test") + client.get_user_delegation_key.side_effect = ["first", "second"] + now = datetime(2026, 9, 1, tzinfo=timezone.utc) + get_cached_user_delegation_key(client, now=now) + + result = get_cached_user_delegation_key( + client, now=now + timedelta(hours=1, minutes=46) + ) + + self.assertEqual(result, "second") + self.assertEqual(client.get_user_delegation_key.call_count, 2) + + def test_user_delegation_cache_evicts_oldest_client(self): + from datetime import datetime, timezone + + now = datetime(2026, 9, 1, tzinfo=timezone.utc) + clients = [] + for index in range(9): + client = MagicMock(url=f"https://account-{index}.test") + client.get_user_delegation_key.return_value = f"key-{index}" + clients.append(client) + get_cached_user_delegation_key(client, now=now) + + get_cached_user_delegation_key(clients[0], now=now) + + self.assertEqual(clients[0].get_user_delegation_key.call_count, 2) + self.assertEqual(clients[-1].get_user_delegation_key.call_count, 1) + + +class TestAsyncBlobHelpers(unittest.IsolatedAsyncioTestCase): + @patch("hastegeo.core.utils.blob.get_blob_service_client") + async def test_download_blob_to_tempfile_uses_shared_client(self, factory): + blob_client = ( + factory.return_value.get_container_client.return_value.get_blob_client.return_value + ) + blob_client.download_blob.return_value.chunks.return_value = [ + b"hello", + b" world", + ] + + path = await download_blob_to_tempfile( + "https://account.blob.core.windows.net/container/file.txt" + ) + self.addCleanup(lambda: os.path.exists(path) and os.unlink(path)) + + with open(path, "rb") as downloaded: + self.assertEqual(downloaded.read(), b"hello world") + factory.assert_called_once() + + @patch("hastegeo.core.utils.blob.get_blob_service_client") + async def test_read_blob_range_uses_shared_client(self, factory): + blob_client = ( + factory.return_value.get_container_client.return_value.get_blob_client.return_value + ) + properties = blob_client.get_blob_properties.return_value + properties.size = 5 + properties.content_settings.content_type = "text/plain" + properties.etag = '"etag"' + blob_client.download_blob.return_value.readall.return_value = b"ell" + + result = await read_blob_range( + "https://account.blob.core.windows.net/container/file.txt", + offset=1, + length=3, + ) + + self.assertEqual(result.data, b"ell") + self.assertEqual(result.total_size, 5) + blob_client.download_blob.assert_called_once_with(offset=1, length=3) + + class TestSplitBlobUrl(unittest.TestCase): def test_azurite_docker_internal_host(self): # The case that motivated the helper in the first place: the diff --git a/hastelib/tests/core/utils/test_metadata_type_matching.py b/hastelib/tests/core/utils/test_metadata_type_matching.py new file mode 100644 index 00000000..18bfc37a --- /dev/null +++ b/hastelib/tests/core/utils/test_metadata_type_matching.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import unittest + +from hastegeo.core.utils.metadata import matches_metadata_type + + +class TestMetadataTypeMatching(unittest.TestCase): + def test_longest_known_metadata_type_wins(self) -> None: + self.assertTrue(matches_metadata_type("model_123.json", "model")) + self.assertFalse( + matches_metadata_type("model_catalog_index.json", "model") + ) + self.assertTrue( + matches_metadata_type( + "partition/model_catalog_index.json", "model_catalog" + ) + ) + + def test_unknown_type_uses_requested_prefix(self) -> None: + self.assertTrue(matches_metadata_type("custom_123.json", "custom")) + self.assertFalse(matches_metadata_type("other_123.json", "custom")) + + +if __name__ == "__main__": + unittest.main() diff --git a/hastelib/tests/core/utils/test_parallel.py b/hastelib/tests/core/utils/test_parallel.py new file mode 100644 index 00000000..602e5753 --- /dev/null +++ b/hastelib/tests/core/utils/test_parallel.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import os +import unittest +from concurrent.futures import ThreadPoolExecutor +from threading import Event, Lock +from unittest.mock import patch + +from hastegeo.core.utils.parallel import ( + BoundedExecutor, + configured_worker_count, +) + + +class TestConfiguredWorkerCount(unittest.TestCase): + def test_uses_default_when_environment_is_missing(self) -> None: + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(configured_worker_count("TEST_WORKERS", 4), 4) + + def test_reads_valid_environment_value(self) -> None: + with patch.dict(os.environ, {"TEST_WORKERS": "7"}): + self.assertEqual(configured_worker_count("TEST_WORKERS", 4), 7) + + def test_rejects_invalid_environment_values(self) -> None: + for value in ("not-an-int", "0", "65"): + with self.subTest(value=value): + with patch.dict(os.environ, {"TEST_WORKERS": value}): + with self.assertRaises(ValueError): + configured_worker_count("TEST_WORKERS", 4) + + def test_rejects_non_integer_direct_value(self) -> None: + for value in (True, "4"): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "integer"): + BoundedExecutor(max_workers=value) + + +class TestBoundedExecutor(unittest.TestCase): + def setUp(self) -> None: + self.executor = BoundedExecutor(max_workers=2) + + def tearDown(self) -> None: + self.executor.shutdown() + + def test_preserves_input_order(self) -> None: + result = self.executor.map(lambda value: value * 2, [3, 1, 2]) + + self.assertEqual(result, [6, 2, 4]) + + def test_propagates_worker_exceptions(self) -> None: + def fail_on_two(value: int) -> int: + if value == 2: + raise RuntimeError("failed") + return value + + with self.assertRaisesRegex(RuntimeError, "failed"): + self.executor.map(fail_on_two, [1, 2, 3]) + + def test_cancels_queued_work_after_failure(self) -> None: + release = Event() + + def fail_with_pending_work(value: int) -> int: + if value == 1: + raise RuntimeError("failed") + release.wait(timeout=1) + return value + + try: + with self.assertRaisesRegex(RuntimeError, "failed"): + self.executor.map(fail_with_pending_work, [1, 2, 3, 4]) + finally: + release.set() + + def test_rejects_invalid_per_call_limit(self) -> None: + with self.assertRaises(ValueError): + self.executor.map(str, [1], max_workers=0) + + def test_nested_map_does_not_deadlock(self) -> None: + single_worker = BoundedExecutor(max_workers=1) + self.addCleanup(single_worker.shutdown) + + result = single_worker.map( + lambda value: single_worker.map(lambda item: item, [value])[0], + [1], + ) + + self.assertEqual(result, [1]) + + def test_concurrent_maps_share_the_process_budget(self) -> None: + active = 0 + peak = 0 + lock = Lock() + two_workers_started = Event() + + def work(value: int) -> int: + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + if active == 2: + two_workers_started.set() + self.assertTrue(two_workers_started.wait(timeout=1)) + with lock: + active -= 1 + return value + + with ThreadPoolExecutor(max_workers=2) as callers: + first = callers.submit(self.executor.map, work, [1, 2, 3]) + second = callers.submit(self.executor.map, work, [4, 5, 6]) + self.assertEqual(first.result(), [1, 2, 3]) + self.assertEqual(second.result(), [4, 5, 6]) + + self.assertEqual(peak, 2) diff --git a/hastelib/tests/core/utils/test_perf.py b/hastelib/tests/core/utils/test_perf.py new file mode 100644 index 00000000..6a4bbdc7 --- /dev/null +++ b/hastelib/tests/core/utils/test_perf.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import unittest +from unittest.mock import Mock + +from hastegeo.core.utils import perf + + +class TestPerfInstrumentation(unittest.TestCase): + def tearDown(self) -> None: + perf.end() + + def test_timed_records_logical_data_layer_operation(self) -> None: + counter = perf.begin(True) + + with perf.timed("load"): + pass + + self.assertEqual(counter.calls, 1) + self.assertEqual(counter.by_op["load"]["calls"], 1) + + def test_headers_include_new_and_legacy_names(self) -> None: + counter = perf.begin(True) + + headers = perf.headers(counter, 0) + + self.assertEqual(headers["X-Haste-Data-Layer-Calls"], "0") + self.assertEqual(headers["X-Haste-Storage-Calls"], "0") + self.assertIn("data-layer", headers["Server-Timing"]) + + def test_disabled_instrumentation_emits_no_headers(self) -> None: + counter = perf.begin(False) + + self.assertIsNone(counter) + self.assertEqual(perf.headers(counter, 0), {}) + + def test_log_summary_clears_active_counter(self) -> None: + counter = perf.begin(True) + logger = Mock() + + perf.log_summary(logger, "operation", counter, 0, key="value") + + self.assertIsNone(perf.get_counter()) + logger.info.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/spec/features/perf-layer-loading/README.md b/spec/features/perf-layer-loading/README.md new file mode 100644 index 00000000..95d734d1 --- /dev/null +++ b/spec/features/perf-layer-loading/README.md @@ -0,0 +1,82 @@ +# Feature: Image Layer & Model Run Loading Performance + +**Status:** in-progress +**Author:** prbatero +**Date:** 2026-08-03 +**Priority:** P1 + +## Contents + +- [Summary](#summary) +- [Motivation](#motivation) +- [Success Criteria](#success-criteria) +- [HASTE Components Affected](#haste-components-affected) +- [Document Index](#document-index) + +## Summary + +The HASTE UI takes too long to load a project's image layers and their associated +model runs, and the delay grows roughly linearly (in places quadratically) with the +number of image layers on a project. The root cause is a set of N+1 storage access +patterns in the `GetProjectDetails` API path, fully-sequential (never parallelized) +blob I/O in `hastelib`, full-container blob scans without prefix filtering, and a UI +that re-fetches the entire project — every model of every layer — every 20 seconds +while re-rendering the whole component tree. This spec catalogs the verified +bottlenecks and lays out a phased plan to remove sequential amplification, bound +process-wide concurrency, and avoid duplicate or idle refresh work. + +## Motivation + +- **Problem:** Disaster-response users open a project and wait many seconds for the + layer list to appear; the wait scales with project size, so the most active + (large) projects are the slowest — exactly when responders can least afford it. +- **Trigger:** Direct user report — "the HASTE UI takes too long to load image + layers and their model runs when there are multiple image layers on a project." +- **Cost of inaction:** Load time degrades as projects accumulate layers and models; + the 20s polling loop multiplies backend load and cost, and the app feels + progressively slower the more it is used. + +## Success Criteria + +- [ ] `GET GetProjectDetails?includeModels=True` for a 50-layer / ~5-models-per-layer + project returns in **< 1.5s p95** (from a current baseline measured in Phase 0). +- [x] Logical data-layer calls for that request drop from **603 to 7**. This metric + counts processor operations, not Azure REST transactions; Blob downloads still + scale with the number of returned records. +- [x] Aggregate blocking Blob I/O is capped by one process-wide executor (16 workers + by default), including concurrent HTTP requests. +- [ ] UI time-to-interactive for the project page is **< 2s p95** on the same project + and no longer scales linearly with layer count. Current production-bundle + observation is **2.12 s** with 58 ms post-response rendering. +- [ ] Idle projects no longer poll and fresh conditional requests avoid storage via a + 15-second process-local cache. Measure the resulting browser CPU/network delta. + +## HASTE Components Affected + +| Component | Impact | +|---|---| +| `hastelib/src/hastegeo/core/data_layer/` | Add prefix-scoped listing, parallel/metadata-only reads, fix double-deserialize | +| `hastelib/src/hastegeo/core/processors/` | Keyed project-details loader and batch metadata reads | +| `hastelib/src/hastegeo/core/artifact_storage/` | Bounded, atomic multi-blob fetch | +| `api/hastefuncapi/` | Thin `GetProjectDetails` route; process-local TTL cache and ETags | +| `api/hastefuncqueues/` | Parallelize independent loads; fix N+1 label lookup; batch saves | +| `ui/src/Components/` | Smart polling, memoization, split context, lazy model expansion | + +## Related Specs + +| Spec | Relationship | +|---|---| +| [../batch-config-drift/](../batch-config-drift/) | related (queue/Batch path) | + +## Document Index + +| Document | Purpose | Status | +|---|---|---| +| [findings.md](findings.md) | Verified bottleneck inventory with file:line evidence | draft | +| [design.md](design.md) | Technical design of each fix | draft | +| [plan.md](plan.md) | Phased execution plan | Phase 0 done | +| [impact-analysis.md](impact-analysis.md) | Risk, blast radius, backward compat | draft | +| [test-plan.md](test-plan.md) | Benchmark harness & regression coverage | draft | +| [results.md](results.md) | **Phase 0 measured baseline** (603 round-trips, 20.8 s API, 40.3 s UI TTI @ 50×5) | done | +| [user-stories.md](user-stories.md) | User outcomes, acceptance criteria, and agent assignments | in-progress | +| [tools/](tools/) | Seed + benchmarks: `phase0_baseline.py`, `bench_api_http.py`, `ui_bench.cjs`; `docker/docker-compose.perf.yml` overlay | done | diff --git a/spec/features/perf-layer-loading/design.md b/spec/features/perf-layer-loading/design.md new file mode 100644 index 00000000..0146409e --- /dev/null +++ b/spec/features/perf-layer-loading/design.md @@ -0,0 +1,153 @@ +# Technical Design: Image Layer & Model Run Loading Performance + +## Overview + +Collapse `GetProjectDetails` from hundreds of sequential logical data-layer calls to +seven top-level operations. Preserve storage-key semantics for legacy records, overlap +independent I/O behind one process-wide concurrency budget, and reuse Azure SDK clients. +A bounded process-local cache deduplicates concurrent requests and serves fresh ETag +checks without storage work. Underlying Blob GET transactions still scale with the +records returned; a materialized project view would be required to make those constant. + +## Contents + +- [Architecture](#architecture) +- [API Design](#api-design) +- [Internal Interfaces](#internal-interfaces-hastegeo) +- [Caching](#caching) +- [UI Design](#ui-design-phase-4) +- [Configuration](#configuration) +- [Observability](#observability) + +## Architecture + +``` +┌──────────────┐ GetProjectDetails ┌────────────────────┐ 1 read / type ┌──────────────┐ +│ React UI │───────────────────────▶│ hastefuncapi │──────────────────▶│ Blob / Cosmos │ +│ Project.jsx │ (single-flight/ETag) │ GetProjectDetails │ keyed batch reads │ data layer │ +└──────────────┘ └─────────┬──────────┘ └──────────────┘ + ▲ smart poll (304 fast-path) │ uses + │ ┌─────────▼──────────┐ + └── active-job-only polling │ ProjectDetailsProc. │ load_map() / exact + │ + shared I/O budget │ metadata prefixes + └────────────────────┘ +``` + +## API Design + +### `GET /api/GetProjectDetails` — reworked internals (contract unchanged by default) + +The response shape remains unchanged. The only implemented query parameter is: + +| Param | Type | Default | Effect | +|---|---|---|---| +| `includeModels` | bool | `false` | Include models, artifacts, and train-label URLs. | + +`summary` and `includeArtifacts` remain deferred. Do not send them until their response +contracts are implemented and tested. + +Response headers are `ETag`, `Cache-Control: private, max-age=`, and +`X-Haste-Cache`. A matching `If-None-Match` returns `304` with an empty body. Request +`Cache-Control: no-cache` or `max-age=0` forces a storage refresh before comparison. + +### Reworked handler logic (replaces [function_app.py:534-638](../../../api/hastefuncapi/function_app.py#L534-L638)) + +```python +project = await ProjectDetailsProcessor(project_id, config).load(include_models) +payload = json.dumps(project) +etag = sha256(payload.encode()).hexdigest() +``` + +`ProjectDetailsProcessor` loads the project first as the `404` gate, loads layers, +labels, and models concurrently, then uses keyed `load_map` calls for validation and +artifacts. Keyed maps preserve the old storage-key joins even when legacy document +bodies omit optional `imageLayerId` or `modelId` fields. + +## Internal Interfaces (hastegeo) + +| Module | Change | Signature | Purpose | +|---|---|---|---| +| `core/processors/project_details.py` | **new** | `ProjectDetailsProcessor.load(include_models)` | Own storage orchestration and pure response assembly. | +| `core/processors/metadata.py` | **new** | `load_map(keys, max_workers=None)` | Prefer one native Cosmos/PostgreSQL query; otherwise use bounded keyed loads. | +| `core/processors/metadata.py` | **new** | `load_filtered(predicate)` | Explicit client-side partition scan and property filter; not a server-side optimization. | +| `core/utils/parallel.py` | **new** | `parallel_map(...)` | One process-wide worker budget shared by requests and storage operations. | +| `core/utils/async_cache.py` | **new** | `AsyncTTLCache` | Bounded TTL cache with per-key single-flight loading. | +| `core/utils/blob.py` | **fix** | `get_blob_service_client(...)` | Reuse top-level clients and connection pools by credential target. | +| Blob/local/Data Lake/Cosmos layers | **fix** | exact metadata matching | Longest known type wins, so `model` cannot consume `model_catalog`. | +| Blob artifact storage | **fix** | atomic bounded download | Validate paths and replace temporary files only after successful download. | + +## Caching + +The cache key is `(projectId, includeModels)`. It stores only successful serialized +responses, shares an in-flight load among concurrent callers, and evicts least-recently +used entries beyond the configured bound. It is process-local and therefore does not +provide cross-instance coherence. After the TTL, active-job polling refreshes storage. + +## Queue design (Phase 3) + +- `GetCreateModelRunQueueTrigger`: replace `load_all_from_partition` + Python filter + with `load_filtered({"imageLayerId": ...})` (Q2). +- Inference / image triggers: wrap independent loads in `asyncio.gather` (Q3). +- Add `MetadataProcessor.save_batch(items)` and collect intermediate writes (Q4). +- `host.json`: evaluate `batchSize` > 1 for I/O-bound triggers and + `maxDequeueCount` ≥ 3 with a real poison path (Q1); raise `visibilityTimeout` for + long steps (Q5). These are config changes gated on load testing. + +## UI design (Phase 4) + +> **Measured priority (Phase 0):** TTI is API-bound (~2 s render over the API call), so +> the backend phases carry the TTI win. Within Phase 4, the highest-value items are the +> single-flight guard (U7) and the poll guard (U1) — they roughly halve effective +> latency and stop request pile-up. Memoization/virtualization (U3/U5) help poll-time +> re-render churn, not first paint; `summary` payload mode (B6) is a minor win. + +- **Single-flight + cancellation (U7, implemented):** one keyed in-flight promise is + shared; a different project or real unmount aborts superseded work. +- **Smart poll (U1, implemented):** send `If-None-Match`; return before state updates on + `304`; do not poll while hidden, in flight, or when all known jobs are terminal. +- **Route assets (implemented):** route modules use `React.lazy`; Azure Maps control, + drawing, and swipe assets load in order only before a map-dependent route mounts. +- **Split context (U2):** move volatile fields (`isLoading`, `dialogParams`, + `bootstrapBreakpoint`) into a separate context/provider so a loading toggle doesn't + re-render layer rows. +- **Memoization (U3):** wrap `LayerRow`, `ModelRow`, `ModelRowMobile` in `React.memo`; + `useMemo` derived arrays; `useCallback` handlers passed as props. +- **Lazy expansion / virtualization (U5):** render a layer's `models[]` only when + expanded; introduce windowing (e.g. `react-window`) if list sizes warrant. +- **Parallel fetches (U4):** `Promise.all` the independent GETs in + `CreateEditImageLayerHelper.js` and `LabelingTool.jsx`. +- **Tiles (U6):** request an overview/thumbnail first; defer full-res second map. + +## Configuration + +| Config Key | Type | Default | Where | Description | +|---|---|---|---|---| +| `HASTE_BLOB_DOWNLOAD_WORKERS` | int (1–64) | 16 | App Settings | Process-wide blocking I/O thread budget. | +| `HASTE_METADATA_LOAD_WORKERS` | int (1–64) | 8 | App Settings | Per-map limit within the global I/O budget. | +| `HASTE_ARTIFACT_DOWNLOAD_WORKERS` | int (1–64) | 8 | App Settings | Per-artifact limit within the global budget. | +| `HASTE_PROJECTDETAILS_CACHE_SECONDS` | int (0–300) | 15 | App Settings | Process-local freshness and response `max-age`. | +| `HASTE_PROJECTDETAILS_CACHE_ENTRIES` | int (1–512) | 64 | App Settings | Maximum process-local project response entries. | +| queue `batchSize` | int | 1 → TBD | `host.json` | Concurrent messages per instance (load-test gated) | + +## Observability + +- Log logical data-layer call count/time and cache hit/miss in `GetProjectDetails`. + These metrics do not count Azure REST transactions; use Azure Storage metrics or SDK + pipeline instrumentation for transaction/cost analysis. +- Emit request duration to App Insights; add a synthetic 50-layer project to the perf + test to track p95 over time. +- Track queue depth and dequeue/poison counts when changing `host.json`. + +## Open Questions + +- [x] `load_filtered` is an explicit partition-scan fallback and does not reduce + transferred records. A future backend query API is required for server filtering. +- [ ] Is the double-serialization (H4) safe to fix in place, or are there existing + blobs already double-encoded that need a migration/back-compat read path? + *(Partially answered 2026-08-20: legacy double-encoded blobs must be assumed, so + Phase 2 shipped a tolerant read (`_read_blob_content` parses twice when the first + parse yields a string) and deferred the save-side fix behind a migration. New + constraint from the data-publishing merge: `save()` now stamps `_index_metadata` + blob metadata, which any save-path rewrite must preserve.)* +- [ ] Acceptable default polling interval / should we move to server-push (SignalR) + instead of polling for run-status updates? diff --git a/spec/features/perf-layer-loading/findings.md b/spec/features/perf-layer-loading/findings.md new file mode 100644 index 00000000..c9416112 --- /dev/null +++ b/spec/features/perf-layer-loading/findings.md @@ -0,0 +1,228 @@ +# Findings: Verified Performance Bottlenecks + +## Contents + +- [Cost Model](#cost-model-why-it-scales) +- [HTTP API](#backend--apihastefuncapifunction_apppy) +- [Core Library](#backend--hastelibsrchastegeocore) +- [Queue Workers](#queue--apihastefuncqueuesfunction_apppy) +- [UI](#ui--uisrc) +- [Measured Priority](#measured-priority-adjustments-phase-0) + +> **Baseline terminology:** the Phase 0 counter records logical data-layer calls, not +> Azure REST transactions. The N+1 findings and latency measurements remain valid, but +> one partition call can contain a listing plus many Blob downloads. + +> **Implementation status (2026-09-01):** B1–B5, B7, H2, and H5 are addressed on the +> cumulative branch. B6, server-side filtering, a materialized project view, and queue +> configuration remain open. UI single-flight, ETag handling, in-flight/visibility +> guards, and active-job-only polling are implemented. + +Each finding was confirmed by reading the referenced code. Severity reflects impact +on the reported symptom (slow layer/run loading that scales with layer count). + +## Cost model (why it scales) + +For a project with **L** image layers and an average of **M** models per layer, the +`GetProjectDetails?includeModels=True` request currently issues approximately: + +``` +3 (project + image_layers + all-models partition reads) ++ L × 1 (LABELS full-partition scan — once PER layer, see B1) ++ L × 1 (VALIDATION load per layer) ++ L × M × 2 (MODEL_ARTIFACTS load + TRAIN_LABELS export per model) += 3 + 2L + 2LM sequential, blocking logical data-layer calls +``` + +For L=50, M=5 that is **~603 sequential round-trips**, each an `await asyncio.to_thread(...)` +that blocks the next. None are parallelized. The target is a **small constant** (≤ ~6). + +--- + +## Backend — `api/hastefuncapi/function_app.py` + +### B1 — CRITICAL: full LABELS partition scan re-run once per layer +[function_app.py:594-599](../../../api/hastefuncapi/function_app.py#L594-L599) + +Inside `for image_layer in image_layers:` the code calls +`MetadataProcessor(LABELS, partition_key=project_id).load_all_from_partition`. The call +takes **no per-layer argument** — it returns the identical full label set every +iteration, then filters in Python by `imageLayerId`. This is a pure redundancy bug: +the download is repeated L times when it should happen **once** before the loop. +`load_all_from_partition` downloads and deserializes every label blob in the +partition, so this is L full-partition downloads. + +**Measured (Phase 0):** per-round-trip cost is **super-linear** — 8.5 → 12.5 → 22 ms +as the partition grows (small → medium → large), because each redundant scan lists + +downloads an ever-larger label set. So B1 costs *more* the bigger the project, on top +of running L times. See [results.md](results.md). + +### B2 — CRITICAL: per-model artifact + labels loads, sequential +[function_app.py:565-591](../../../api/hastefuncapi/function_app.py#L565-L591) + +For every model of every layer, two sequential awaits: `MODEL_ARTIFACTS.load(modelId)` +and `TRAIN_LABELS.export(modelId)`. That is `L × M × 2` blocking round-trips executed +one at a time. This is the single largest contributor for model-heavy projects. + +### B3 — HIGH: per-layer VALIDATION load, sequential +[function_app.py:621-632](../../../api/hastefuncapi/function_app.py#L621-L632) + +`VALIDATION.load(image_layer_id)` per layer, sequential — `L` more blocking round-trips +that could be one batched/parallel read. + +### B4 — HIGH: no parallelism anywhere in the handler +The handler uses `await asyncio.to_thread(...)` for each I/O but never +`asyncio.gather`. Even the independent initial loads (project → image_layers → +all-models) are chained. Every storage call waits for the previous one. + +### B5 — HIGH: same N+1 repeated in `GenerateProjectStats` +[function_app.py:2602-2624](../../../api/hastefuncapi/function_app.py#L2602-L2624) + +The identical per-layer `LABELS.load_all_from_partition` pattern exists in the stats +path, so dashboards/stats regeneration degrade the same way. + +### B6 — MEDIUM: unbounded response, no pagination +[function_app.py:638](../../../api/hastefuncapi/function_app.py#L638) returns the entire +nested project (`imageLayer[].models[].artifacts`) in one payload. Serialization and +transfer grow with total model count; there is no page/limit and no lightweight +"summary" shape for the initial list render. + +### B7 — MEDIUM: no HTTP caching on list/detail endpoints +`GetModelArtifact` sets `Cache-Control`/`ETag`, but `GetProjectDetails`, +`GetProjectStats`, `GetLayerModelsDetails`, `GetLayerDetailView` set none — every poll +is a full recompute + full transfer even when nothing changed. + +--- + +## Backend — `hastelib/src/hastegeo/` + +### H1 — HIGH: `load_all` scans the whole container, no prefix +[data_layer/azure_blob_storage_data_layer.py:268-323](../../../hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py#L268-L323) + +`walk_blobs()` is called with **no `name_starts_with`**, listing the entire container, +then filtering in Python. `load_all_from_partition` does pass a prefix (good), but +`load_all` does not. All matched blobs are then fully downloaded even when only counts +/ metadata are needed. + +### H2 — HIGH: sequential per-blob download in listing + artifact fetch +[artifact_storage/azure_blob_artifact_storage.py:179-187](../../../hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py#L179-L187) +and the download loops in `load_all` / `load_all_from_partition`. Each blob is +downloaded and read to completion before the next starts — `N` files ⇒ `N ×` latency. +A bounded `ThreadPoolExecutor` would collapse this to roughly one round-trip of +latency. + +### H3 — HIGH: no filtered query — everything filtered in Python +`MetadataProcessor.load_all_from_partition` has no `imageLayerId`/predicate parameter, +forcing callers (B1, and the queue in Q2) to pull the whole partition and filter +in-process. A `load_filtered(...)` method would let callers request only what they need. + +### H4 — MEDIUM: double JSON deserialization +[data_layer/azure_blob_storage_data_layer.py:254-259](../../../hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py#L254-L259) +(and 3 sibling paths) — `json.loads` result is `json.loads`-ed again for +projects/image_layers because they were double-serialized on save. CPU cost on every +read; the real fix is single-serialize on write, then drop the re-parse. + +### H5 — MEDIUM: `BlobServiceClient` created per call +[core/blob.py:79-81, 150-151](../../../hastelib/src/hastegeo/core/blob.py#L79-L81) — +`BlobServiceClient.from_connection_string(...)` on every `download_blob_to_tempfile` / +`read_blob_range` call; no module-level reuse, so credential parse + connection setup +repeat. Same theme: `MetadataProcessor`/`UnifiedDataLayer` are re-instantiated dozens +of times per request/message rather than reused. + +### H6 — MEDIUM: no caching layer anywhere except `footprints.get_latest_release` +[core/utils/footprints.py:155](../../../hastelib/src/hastegeo/core/utils/footprints.py#L155) +is the only `lru_cache` in the core library. There is no request-scoped memoization, +so `load_and_combine_sub_data_types` and similar re-fetch identical data within a +single request. + +--- + +## Queue — `api/hastefuncqueues/function_app.py` + +### Q1 — HIGH: `batchSize: 1`, `maxDequeueCount: 1` +[host.json](../../../api/hastefuncqueues/host.json) processes one message per instance +and dead-letters after a single failure (no retry). Throughput depends entirely on +instance scale-out; a transient error poisons the message immediately. + +### Q2 — HIGH: N+1 label lookup in training trigger +[function_app.py:354-367](../../../api/hastefuncqueues/function_app.py#L354-L367) — +`LABELS.load_all_from_partition` then Python `next(...)` filter; should use H3's +filtered load. + +### Q3 — MEDIUM: independent loads not parallelized +Inference trigger loads image layer then experiment config sequentially +([function_app.py:695-712](../../../api/hastefuncqueues/function_app.py#L695-L712)); +image-processing trigger does store-artifact then get-URL sequentially. `asyncio.gather` +would halve these. + +### Q4 — MEDIUM: 3–5 sequential `MetadataProcessor.save` calls per message, no batching. +Every trigger re-instantiates `MetadataProcessor` several times and awaits each save +serially. A `save_batch` would cut round-trips. + +### Q5 — LOW: `visibilityTimeout: 30s` risks re-enqueue for slow checkpoints; consider +raising or extending visibility on long steps. + +--- + +## UI — `ui/src/` + +### U1 — CRITICAL: 20s poll refetches the entire project incl. all models +[Project.jsx:149-157](../../../ui/src/Components/Project.jsx#L149-L157) → +[Project.jsx:102-147](../../../ui/src/Components/Project.jsx#L102-L147) calls +`GetProjectDetails?includeModels=True` every 20s, always `setComponentState(...)` even +when data is unchanged. This drives B1–B4 repeatedly and re-renders the whole tree on +a timer. + +**Measured (Phase 0):** the large-project response is **~21–38 s — longer than the 20 s +poll interval**, so a new poll fires before the previous one returns and overlapping +603-round-trip requests pile up on the backend. The poll must not fire while a request +is in flight (single-flight guard) and/or the interval must adapt to response time. + +### U7 — HIGH (new, measured): duplicate concurrent `GetProjectDetails` on load +Playwright observed **two concurrent** `GetProjectDetails` calls on the initial project +load, so browser-observed latency is ~2× the isolated API call (large: ~38 s vs 20.8 s). +Dev-mode React StrictMode double-invokes effects, but the root cause that survives a +production build is the **absence of in-flight de-duplication / request cancellation**: +[Project.jsx:102-147](../../../ui/src/Components/Project.jsx#L102-L147) has no +single-flight guard or `AbortController`, so overlapping mounts/effects/polls each issue +a full request. Fix: dedupe in-flight requests and abort superseded ones. + +### U2 — HIGH: monolithic `AppContext` → broad re-renders +[AppContext.jsx:266-281](../../../ui/src/AppContext.jsx#L266-L281) — a single +`appParams` object (loading flag, dialog, breakpoint, …) is consumed by ~38 +components. Any `setIsLoading` (fired by every poll) re-renders all of them. + +### U3 — HIGH: no memoization +No `React.memo` / `useMemo` / `useCallback` anywhere under `ui/src/Components/`. Every +parent render re-renders all `LayerRow` / `ModelRow` children with freshly-created prop +objects. + +### U4 — MEDIUM: sequential independent API calls +[CreateEditImageLayerHelper.js:28-37](../../../ui/src/Components/CreateEditImageLayerHelper.js#L28-L37) +and [LabelingTool.jsx:174-181](../../../ui/src/Components/LabelingTool/LabelingTool.jsx#L174-L181) +`await` two independent GETs in series; should be `Promise.all`. + +### U5 — MEDIUM: no change detection on poll; eager model-row rendering on expand; +no virtualization for large layer/model lists +([Project.jsx:193-228](../../../ui/src/Components/Project.jsx#L193-L228), +[ProjectManagement/LayerRow.jsx:359-404](../../../ui/src/Components/ProjectManagement/LayerRow.jsx#L359-L404)). + +### U6 — MEDIUM-LOW: Visualizer requests full-res pre+post tiles eagerly with no +overview/thumbnail +([Visualizer/Visualizer.jsx:315-345](../../../ui/src/Components/Visualizer/Visualizer.jsx#L315-L345)). + +## Measured priority adjustments (Phase 0) + +The browser baseline reshuffles a few priorities relative to first estimates: + +- **Backend-first is strongly validated.** UI time-to-interactive is **API-bound**: + TTI ≈ the `GetProjectDetails` duration + only ~2 s of render (large: 40.3 s TTI vs + 38.4 s call). So Phase 1/2 (backend) captures the overwhelming majority of the TTI + win; no UI change can hide a 20 s API call. +- **U7 + poll-overlap (U1) rise in priority** — they roughly *double* effective latency + and pile requests on the backend. Cheap to fix (single-flight guard), high impact. +- **Render-side items (U3 memoization, U5 virtualization) drop for TTI** — with only + ~2 s of render at 50 layers, they matter for *poll-time smoothness / re-render churn*, + not first paint. Still worth doing, but not the TTI lever. +- **B6 payload is minor** — the large default payload is only ~83 KB; `summary` mode is + a nice-to-have, not a latency driver. Round-trip count, not payload size, dominates. diff --git a/spec/features/perf-layer-loading/impact-analysis.md b/spec/features/perf-layer-loading/impact-analysis.md new file mode 100644 index 00000000..b58e70e6 --- /dev/null +++ b/spec/features/perf-layer-loading/impact-analysis.md @@ -0,0 +1,88 @@ +# Impact Analysis: Image Layer & Model Run Loading Performance + +## Contents + +- [Scope](#scope-of-change) +- [Azure Services](#azure-service-impact) +- [Dependencies](#dependency-analysis) +- [Risks](#risk-assessment) +- [Performance](#performance-impact-measured-baseline--target) +- [Security](#security-impact) +- [Rollback](#rollback-assessment) + +## Scope of Change + +| Component | Path | Type | Severity | +|---|---|---|---| +| REST API | `api/hastefuncapi/function_app.py` | modified (`GetProjectDetails`, `GenerateProjectStats`) | high | +| Core library | `hastelib/src/hastegeo/core/processors/metadata.py` | modified (new methods) | medium | +| Core library | `hastelib/src/hastegeo/core/data_layer/azure_blob_storage_data_layer.py` | modified | medium | +| Core library | `hastelib/src/hastegeo/core/artifact_storage/azure_blob_artifact_storage.py` | modified | low | +| Core library | `hastelib/src/hastegeo/core/blob.py` | modified (client reuse) | low | +| Queue workers | `api/hastefuncqueues/function_app.py` + `host.json` | modified | medium | +| React UI | `ui/src/Components/Project.jsx`, `AppContext.jsx`, `ProjectManagement/*` | modified | medium | + +## Azure Service Impact + +| Service | Change | Cost Impact | +|---|---|---| +| Blob Storage / Cosmos | Fewer repeated scans and overlapped keyed reads | Lower transaction count than baseline, but still proportional to returned Blob records | +| Azure Functions | Lower per-request CPU/wall-time; `gather` uses more threads briefly per request | Net **lower** consumption; watch thread-pool sizing under load | +| Queue Storage | `batchSize`>1 raises concurrency per instance | Neutral–lower; validate scale behavior | + +## Dependency Analysis + +### Upstream (needed by this work) +| Dependency | Status | Risk | +|---|---|---| +| `UnifiedDataLayer` predicate support | to confirm | If Blob path can't filter server-side, `load_filtered` = prefix scan + in-layer filter (still removes caller N+1, smaller win) | +| Existing blob serialization format | available | H4 fix depends on whether legacy blobs are double-encoded | + +### Downstream (affected) +| Consumer | How | Breaking? | Migration? | +|---|---|---|---| +| UI `GetProjectDetails` callers | Same default shape; new optional params | no | no | +| UI polling | Gains `304` fast-path; old client still works | no | no | +| Existing blobs (if double-encoded) | H4 read-path change | **maybe** | Add back-compat read (accept single- or double-encoded) before removing re-parse | +| Queue message format | unchanged | no | no | + +## Risk Assessment + +| Risk | Likelihood | Impact | Mitigation | +|---|---|---|---| +| H4 double-serialize fix breaks reads of legacy blobs | med | high | Keep tolerant read (try single, fall back to double) during transition; migrate lazily on next save | +| Concurrent fan-out exhausts Functions threads | low after mitigation | med | One process-wide executor caps blocking I/O at `HASTE_BLOB_DOWNLOAD_WORKERS`; UI/API single-flight removes duplicate same-key work. Multi-request load testing remains required. | +| `batchSize`>1 changes ordering/poison behavior | med | med | Gate behind load test; keep separate from Phase 1; revert via config | +| Cache headers serve stale run-status to UI | low | med | Short `max-age` (≤15s); UI still change-detects; runs move to terminal states, not backwards | +| UI context split / memo introduces render regressions | med | low | Ship incrementally; visual + interaction QA per component | + +## Performance Impact (measured baseline → target) + +Phase 0 baselines (50×5, Azurite/dev — lower bound; see [results.md](results.md)): + +- **API latency:** `GetProjectDetails` **20.8 s p50 / 21.8 s p95** baseline; + hardened clean fixture **1.85 s / 2.00 s uncached** and **9.1 / 12.2 ms cached**. +- **Logical data-layer calls:** **603 → 7**. This is not an Azure transaction count. +- **UI time-to-interactive:** **40.3 s** (large) → target **< 2 s**. TTI is API-bound + (~2 s render over the API call), so the backend fix drives most of this; the UI + single-flight/poll guards remove the ~2× amplification and request pile-up. +- **Backend load:** idle projects no longer poll; active-job polls do not overlap. + Fresh cache hits use zero data-layer calls. Active polls after cache expiry still read + storage because no materialized project version exists. +- **Queue throughput:** higher with `batchSize`>1; training/inference lose partition scans. + +## Security Impact + +- [x] No new endpoints exposed; `GetProjectDetails` auth level unchanged. +- [x] No new data classification; same imagery/metadata. +- [x] No auth/CORS changes. +- [ ] `ETag` is a payload hash — ensure it doesn't leak across tenants (it's per-project, + auth already scopes access). Confirm no cross-user cache reuse. + +## Rollback Assessment + +- **Reversibility:** fully reversible — behavioral/perf changes, no schema migration + required (H4 handled with a tolerant read path, so no data rewrite). +- **API:** contract preserved; new params are opt-in. Revert = redeploy prior build. +- **Config:** `host.json` / env knobs revert independently. +- **Estimated rollback time:** one redeploy (minutes). diff --git a/spec/features/perf-layer-loading/plan.md b/spec/features/perf-layer-loading/plan.md new file mode 100644 index 00000000..d9f36f1d --- /dev/null +++ b/spec/features/perf-layer-loading/plan.md @@ -0,0 +1,195 @@ +# Execution Plan: Image Layer & Model Run Loading Performance + +Sequenced by impact-per-risk. Phase 0 establishes a baseline so every later phase is +measured, not guessed. Backend before UI: the backend N+1 is the dominant cost and its +fixes are transparent to the UI. Each phase is independently shippable. + +## Contents + +- [Phase 0: Baseline and Instrumentation](#phase-0-baseline--instrumentation--done-2026-08-03) +- [Phase 1: Backend API Hot Path](#phase-1-backend-api-hot-path-hastefuncapi--highest-roi) +- [Phase 2: Core Library](#phase-2-core-library-hastelib--enables-phase-1-batchfilter) +- [Phase 3: Queue Workers](#phase-3-queue-workers-hastefuncqueues--throughput) +- [Phase 4: UI](#phase-4-ui-uisrc--perceived-performance) +- [Phase 5: Integration and Validation](#phase-5-integration--validation) +- [Milestones](#milestones) +- [Agent Summary](#agent-summary) +- [Open Questions](#open-questions) + +## Phase 0: Baseline & Instrumentation — DONE (2026-08-03) + +**Goal:** Make the problem measurable before changing it. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Add opt-in logical data-layer counter + timing (`hastelib/.../utils/perf.py`, wired into `MetadataProcessor` reads + `GetProjectDetails` via `HASTE_PERF`) | `backend-dev` | — | B1–B4 | **done** | +| Seed script for synthetic L×M project (`tools/seed_synthetic_project.py`) | `backend-dev` | — | cost-model | **done** | +| Capture baseline data-layer calls + payload (`tools/phase0_baseline.py` → `results.md`) | `backend-dev` | above | success-criteria | **done** | +| HTTP latency harness for running stack (`tools/bench_api_http.py`) | `backend-dev` | above | success-criteria | **done** | +| Capture real API p50/p95 latency against running stack (Docker + Azurite) | `backend-dev` | stack up + seed | success-criteria | **done** | +| Compose overlay + reproducible run (`docker/docker-compose.perf.yml`) | `backend-dev` | — | — | **done** | +| Record UI time-to-interactive + poll cost (Playwright, `tools/ui_bench.cjs`) | `ui` | stack up | U1 | **done** | + +**Exit Criteria:** +- [x] Baseline logical calls (33 / 243 / **603**) + payload captured in + [results.md](results.md) and `test-plan.md`; per-op breakdown quantifies B1/B2. +- [x] Real API latency captured (large **20.8 s p50 / 21.8 s p95** on Azurite; + storage = 13.4 s of that). Confirms the reported symptom. +- [x] Browser-side UI TTI captured (large **40.3 s**; API-bound + fired twice + concurrently; 20 s poll re-does the full call). Phase 0 complete. + +## Phase 1: Backend API hot path (`hastefuncapi`) — highest ROI + +**Goal:** Collapse `GetProjectDetails` to a constant number of storage reads. No API +contract change; UI untouched. + +Implemented on branch `prbatero/feat/performance-improvements-phase1`. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Hoist `LABELS.load_all_from_partition` out of the layer loop (load once) | `backend-dev` | P0 | B1 | **done** | +| Parallelize top-level reads (project/layers/models/labels) with `asyncio.gather` | `backend-dev` | P0 | B4 | **done** | +| Batch per-model artifacts with keyed `load_map` + `labelsUrl` with one `list_keys` | `backend-dev` | Ph2 | B2 | **done** | +| Batch per-layer VALIDATION with keyed `load_map` | `backend-dev` | Ph2 | B3 | **done** | +| Apply the same hoist to `GenerateProjectStats` | `backend-dev` | B1 | B5 | **done** | +| Add bounded process-local single-flight cache + `ETag`/`304` handling | `backend-dev` | — | B7 | **done** | +| Add optional `summary` / `includeArtifacts=false` response modes | `backend-dev` | above | B6 | deferred (payload minor per Phase 0 — see findings); if revived, build on the publishing feature's `load_page`/`_index_metadata` metadata-indexed listing rather than a new listing path | + +> **Design note:** the final implementation uses keyed `load_map` for artifacts and +> validation, preserving legacy storage-key joins. Blob performs bounded per-key GETs; +> Cosmos and PostgreSQL issue one native query per map. The seven-call count is a +> logical data-layer metric, not an Azure transaction count. + +**Exit Criteria:** +- [x] Logical data-layer calls for a 50×5 project drop from **603 → 7**. +- [ ] Uncached `GetProjectDetails` is **1.85 s p50 / 2.00 s p95** on the clean + 50×5 Azurite fixture; the `<1.5 s p95` target is not yet met. +- [x] Fresh process-local cache hits are **9.1 ms p50 / 12.2 ms p95** and perform + zero logical data-layer calls. +- [x] Existing UI still works unchanged (contract preserved; verified via UI bench — + TTI 40.3 s → 5.5 s). See [results.md](results.md). + +## Phase 2: Core library (`hastelib`) — enables Phase 1 batch/filter + +**Goal:** Give the API layer the primitives it needs and remove data-layer waste. +(Ships alongside Phase 1.) + +> **Done in Phase 1:** added `list_identifiers` (metadata-only, no-download listing) +> to the abstract/blob/local-FS/unified layers, `MetadataProcessor.list_keys` + +> `build_url`, and a `check_exists=False` fast path on `get_file_remote_path` — these +> are what let B2 drop the export N+1. `load_map`/`load_filtered`, the `load_all` +> prefix fix (H1), parallel download loops (H2), double-serialize fix (H4), and +> `BlobServiceClient` reuse (H5) remain for a dedicated Phase 2 pass. + +Implemented on branch `prbatero/feat/performance-improvements-phase2`. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Add `MetadataProcessor.load_map(keys, max_workers)` with native Cosmos/PostgreSQL queries and bounded fallback | `backend-dev` | — | H2 | **done** | +| Add `MetadataProcessor.load_filtered(predicate)` | `backend-dev` | — | H3 | **done** | +| Parallelize download loops through one process-wide bounded executor; make artifact files atomic | `backend-dev` | — | H2 | **done** | +| Reuse module-level `BlobServiceClient` keyed by connection target | `backend-dev` | — | H5 | **done** | +| Unit, contract, API, and local-storage regression tests | `backend-dev` | above | — | **done** | +| Exact metadata-type matching (`model` must not include `model_catalog`) | `backend-dev` | — | H1 | **done** | +| Pass `name_starts_with` prefix in `load_all` | `backend-dev` | — | H1 | deferred — `load_all` is cross-partition (varying `{partition}/` prefix), so no single prefix applies; `load_all_from_partition` already prefixes | +| Fix double-serialization on save; drop redundant `json.loads` | `backend-dev` | migration Q | H4 | deferred — kept tolerant read (`_read_blob_content` handles both encodings); save-side fix needs a data migration for legacy blobs **and** must preserve the `_index_metadata` blob metadata now attached on save (publishing) | + +> **Note:** `list_identifiers` / `list_keys` / `build_url` / `check_exists` were landed +> in Phase 1. H2's parallel downloads are a **production-latency** win (no measurable +> effect on Azurite/localhost where per-blob latency is ~1 ms — see results.md). +> +> **Post-rebase (2026-08-20):** main's data-publishing feature independently added a +> parallel blob loader (`_load_blob_names`, for `load_page`); consolidated it onto +> `_read_blob_content` + the shared `HASTE_BLOB_DOWNLOAD_WORKERS` policy so the +> download/deserialize logic (incl. the H4 double-parse tolerance) lives in one place. + +**Exit Criteria:** +- [x] New core utilities have 100% statement/branch coverage; cross-backend read + contracts, response parity, cache, and failure paths are covered. +- [x] Clean 50×5 HTTP fixture returns 50 layers, 250 models/artifacts, and correct + validation counts with seven logical data-layer calls. + +## Phase 3: Queue workers (`hastefuncqueues`) — throughput + +**Goal:** Remove the queue-side N+1 and unlock concurrency. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Replace label N+1 in training trigger with `load_filtered` | `backend-dev` | Ph2 H3 | Q2 | not-started | +| Parallelize independent loads (inference, image) with `asyncio.gather` | `backend-dev` | — | Q3 | not-started | +| Add `save_batch`; batch intermediate saves | `backend-dev` | — | Q4 | not-started | +| Load-test `batchSize`>1 / `maxDequeueCount`≥3 / `visibilityTimeout` in `host.json` | `backend-dev` | — | Q1,Q5 | not-started | + +**Exit Criteria:** +- [ ] Training/inference triggers issue no full-partition scans. +- [ ] `host.json` changes validated under load without poison-queue regressions. + +## Phase 4: UI (`ui/src`) — perceived performance + +**Goal:** Stop the poll from thrashing the tree; render only what's needed. + +> **Measured (Phase 0):** TTI is API-bound, so backend phases own the TTI win. Rank the +> UI work by impact: **single-flight guard (U7) + poll guard (U1) first** (halve +> effective latency, stop pile-up), then context split (U2); memoization/virtualization +> (U3/U5) and `summary` payload (B6) are lower-value follow-ups. +> +> **Post-rebase (2026-08-20):** main's `PublishedDatasets.jsx` is an in-repo precedent +> for smart polling — 5 s interval that runs *only while active items exist*, with a +> ref to avoid stale closures; consider extracting a shared hook both pages use. Main +> also reworked `AppContext.jsx`/`App.jsx` and many Phase 4 target components, so plan +> U2/U3 against current main, not the original file inventory in findings.md. + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| Single-flight guard + `AbortController` on `fetchProjectDetails` (dedupe/cancel concurrent calls) | `ui` | — | U7 | **done** | +| Smart poll: send `If-None-Match`, handle `304`, skip `setState` if unchanged | `ui` | Ph1 B7 | U1 | **done** | +| Poll guard: don't fire while a request is in flight | `ui` | — | U1 | **done** | +| Pause polling when hidden and when no active jobs; configurable/adaptive interval | `ui` | — | U1 | **partial** (guards done; interval remains 20 s) | +| Route-level code splitting; load Azure Maps SDK only on map routes | `ui` | — | U6-adjacent | **done** | +| Split volatile fields out of `AppContext` into a lighter provider | `ui` | — | U2 | not-started | +| `React.memo` / `useMemo` / `useCallback` on `LayerRow`/`ModelRow`/`ModelRowMobile` | `ui` | — | U3 | not-started | +| Lazy-render model rows on expand; evaluate list virtualization | `ui` | — | U5 | not-started | +| `Promise.all` the independent GETs in edit/labeling helpers | `ui` | — | U4 | not-started | +| Use `summary` mode for list view; fetch full models on expand | `ui` | Ph1 B6 | B6,U5 | not-started | +| Thumbnail/overview-first in Visualizer; defer full-res second map | `ui` | — | U6 | not-started | + +**Exit Criteria:** +- [ ] Project page time-to-interactive < 2s p95 on the synthetic project. Current + production-bundle observation: **2.12 s**; one initial request, 58 ms render. +- [x] Terminal-job project issued **zero polls** during a 26-second idle window. +- [x] Active-job project issued one non-overlapping conditional poll (`304`). + +## Phase 5: Integration & Validation + +| Task | Agent | Dependencies | Ref | Status | +|---|---|---|---|---| +| End-to-end run on Docker Compose stack with synthetic large project | `backend-dev` | Ph1–4 | — | not-started | +| Compare against Phase 0 baseline; record deltas | `backend-dev` | above | success-criteria | not-started | +| Update `docs/` (api-overview, architecture) with new params/caching | `backend-dev` | — | — | not-started | + +**Exit Criteria:** +- [ ] All success criteria in [README.md](README.md#success-criteria) met and recorded. +- [ ] CI passes (secret-scan, deploy-apps). + +## Milestones + +| Milestone | Deliverable | +|---|---| +| Baseline captured | Phase 0 numbers in test-plan.md | +| Backend hot path fixed | Seven logical calls, bounded Blob I/O, cache headers | +| Core lib primitives | `load_map` / `load_filtered` / prefix scans merged | +| Queue optimized | N+1 removed, config load-tested | +| UI responsive | smart poll + memoization shipped | +| Validated | End-to-end deltas meet success criteria | + +## Agent Summary + +| Agent | Phases | +|---|---| +| `backend-dev` | 0, 1, 2, 3, 5 | +| `ui` | 0, 4 | + +## Open Questions + +- [ ] Ship Phase 1 alone first (backend-only, invisible) to bank the win before UI work? +- [ ] Poll vs push (SignalR) for run-status — defer or fold into Phase 4? diff --git a/spec/features/perf-layer-loading/results.md b/spec/features/perf-layer-loading/results.md new file mode 100644 index 00000000..8bb5de3a --- /dev/null +++ b/spec/features/perf-layer-loading/results.md @@ -0,0 +1,240 @@ +# Measured Results + +## Contents + +- [Post-review Hardening](#post-review-hardening-2026-09-01) +- [Phase 2](#phase-2--core-library-hastelib) +- [Phase 1](#phase-1--measured-after-backend-hot-path) +- [Phase 0](#phase-0-baseline--measured-results) +- [Reproduction](#how-to-reproduce) +- [Targets](#targets-to-beat-from-readmemdsuccess-criteria) +- [Environment](#environment-notes) + +> **Metric correction (2026-09-01):** the `HASTE_PERF` counter measures logical +> data-layer method calls, not Azure Storage REST transactions. A partition or keyed +> Blob operation can issue one listing plus many GET requests. Historical `33/243/603` +> and `7` values below are logical calls. + +## Post-review hardening (2026-09-01) + +Clean synthetic project: 50 layers, 5 models per layer, 20 label records per layer, +10 validation labels per layer. Docker Functions + Azurite, forced-refresh requests, +10 measured iterations after 2 warmups: + +| Path | Logical calls | p50 | p95 | Payload | +|---|---:|---:|---:|---:| +| Forced refresh (`Cache-Control: no-cache`) | 7 | **1.85 s** | **2.00 s** | 166.2 KB | +| Fresh process-local cache (`--allow-cache`) | 0 | **9.1 ms** | **12.2 ms** | 166.2 KB | + +Correctness check: exactly 50 layers, 250 models, 250 non-null artifact records, and +`validationLabelCount == 10` for every layer. The `<1.5 s` uncached p95 target remains +open. Cache numbers are process-local and do not imply cross-instance coherence. + +Production UI bundle against the same API fixture: + +| Browser path | TTI | Initial calls | Post-response render | 26 s poll window | +|---|---:|---:|---:|---:| +| Terminal jobs | **2.12 s** | **1** | 58 ms | **0 calls** | +| One active job | **2.27 s** | **1** | 69 ms | **1 call**, HTTP `304` | + +The original page loaded Azure Maps control/drawing scripts and styles globally. In the +test environment those four CDN requests blocked `DOMContentLoaded` for about 14 s, +including on the non-map project page. Route-level code splitting plus on-demand Azure +Maps loading reduced the main JS chunk from 1.49 MB to about 120 KB and moved +`DOMContentLoaded` to 68 ms. A map-route browser smoke test confirmed `atlas`, drawing, +and `SwipeMap` are available after the lazy loader runs. + +The `<2 s` production TTI target remains open by about 120 ms on the terminal fixture; +the uncached API call (1.93 s in that run) is now the dominant component. + +## Phase 2 — Core library (hastelib) + +**Date:** 2026-08-03 · **Branch:** `prbatero/feat/performance-improvements-phase2` + +Phase 2 removes data-layer waste and adds batch primitives. Delivered: + +- **H2 — parallel per-blob downloads** in `load_all_from_partition` / `load_all` + (blob layer) and `fetch_artifact` through one process-wide bounded executor. +- **H5 — `BlobServiceClient` reuse**: a process-wide `lru_cache`d factory in + `utils/blob.py` (was created per call in `download_blob_to_tempfile` / + `read_blob_range`). +- **Batch primitives**: `MetadataProcessor.load_map` (parallel per-key load, perf + counter propagated into worker threads via `perf.bind`) and `load_filtered` + (partition scan + in-process predicate), backed by cross-backend regression tests. + +**Measured (50×5, Azurite/dev):** `GetProjectDetails` correctness unchanged (50 +layers, 250 models/artifacts, counts correct); logical calls remain **7**; latency +**~2.08 s p50 — flat vs Phase 1**. + +The Azurite benchmark validates the control flow, not production network performance. +Real Azure concurrency must be tuned and measured under representative request load. + +**Net:** Phase 1 removed the sequential logical-call amplification; Phase 2 bounds +aggregate concurrency and improves production I/O behavior. Underlying Blob transactions +remain proportional to the number of records downloaded. + +The earlier statement that Phase 2 necessarily makes real Azure sub-second was a +projection, not a measurement, and is superseded by the measured table above. + +## Phase 1 — Measured After (backend hot path) + +**Date:** 2026-08-03 · **Branch:** `prbatero/feat/performance-improvements-phase1` + +Phase 1 loads every metadata type **once per partition** and joins in memory +(`asyncio.gather` for the independent reads), and replaces the 250 per-model +`TRAIN_LABELS` export `exists()` round-trips with **one** blob-name listing + local +SAS URL construction. Same measurement harnesses as Phase 0. + +| Fixture | Logical calls | API p50 | UI TTI | | before → after | +|---|---|---|---|---|---| +| small (5×2) | 33 → **7** | 0.70 → **0.20 s** | 3.10 → **2.97 s** | | | +| medium (20×5) | 243 → **7** | 6.18 → **0.84 s** | 12.36 → **3.63 s** | | | +| large (50×5) | 603 → **7** | 20.77 → **2.02 s** | 40.27 → **5.50 s** | | **API 10.3× · TTI 7.3×** | + +- **Logical calls are seven** regardless of project size. Blob REST transactions still + scale with the downloaded records. The seven operations are project plus + {imagelayer, labels, validation, model, artifacts} + partition reads + 1 train-labels key listing, the last six run concurrently. +- **`X-Haste-Storage-Ms` (5.5 s) now exceeds `X-Haste-Wall-Ms` (2.0 s)** on large, + confirming the reads run in parallel (sum of per-call time > wall time). +- The large poll call dropped **36.5 s → 2.05 s**. +- Correctness verified: 50 layers, 5 models each, all 250 artifacts joined, + `labelProjectCount`/`validationLabelCount` correct, and the `labelsUrl` build path + confirmed (populates a SAS URL when a train-labels blob exists, else null). +- The after-payload is *larger* (166 KB vs 82 KB — the seed was enriched for the UI + run), so the latency win is conservative. + +**Historical Phase 1 state:** UI TTI (5.5 s) exceeded the API call because the page +fired `GetProjectDetails` twice and did not consume ETags. The post-review hardening +measurements above supersede that state. + +--- + +## Phase 0 Baseline — Measured Results + +**Date:** 2026-08-03 +**Branch:** `prbatero/feat/performance-improvements` +**Method:** `tools/phase0_baseline.py` — seeds synthetic projects into the real +`local` filesystem backend and replays the exact `GetProjectDetails` read+assemble +sequence ([function_app.py:534-638](../../../api/hastefuncapi/function_app.py#L534-L638)) +with `HASTE_PERF` instrumentation on. Models seeded **without** `labelsUrl` (worst-case +N+1 that triggers the per-model `TRAIN_LABELS` export). + +### Headline metric — logical data-layer calls per request + +| Fixture | Layers × Models | **Logical calls** | Payload | Formula `3 + L·(2M+2)` | +|---|---|---|---|---| +| small | 5 × 2 | **33** | 9.1 KB | 33 ✓ | +| medium | 20 × 5 | **243** | 82.5 KB | 243 ✓ | +| large | 50 × 5 | **603** | 205.8 KB | 603 ✓ | + +Logical-call counts match the derived formula, confirming the replay is +faithful to the handler and that cost scales as **O(layers × models)**. + +## Per-op breakdown (large, 50 × 5) + +| Op | Count | Source | +|---|---|---| +| `load` | 301 | 1 project + 250 per-model `MODEL_ARTIFACTS` (B2) + 50 per-layer `VALIDATION` (B3) | +| `load_all_from_partition` | 52 | 1 imagelayer + 1 model + **50 redundant full `LABELS` scans** (B1 — should be 1) | +| `export` | 250 | per-model `TRAIN_LABELS` export (B2) | +| **total** | **603** | | + +**B1 is quantified:** 50 of the 52 partition scans are the identical full `LABELS` +download re-run once per layer. **B2 is quantified:** 500 of 603 round-trips (83%) are +the per-model artifact + labels-export pair. + +## Latency — MEASURED against the real API (Docker + Azurite blob backend) + +Captured via `tools/bench_api_http.py` against the running stack (compose overlay +`docker/docker-compose.perf.yml`, `HASTE_PERF=true`, `blob` backend on Azurite), +after seeding each project into Azurite. End-to-end `GetProjectDetails?includeModels=True`: + +| Fixture | **latency p50** | latency p95 | storage calls | storage time p50 | payload | per-call cost | +|---|---|---|---|---|---|---| +| small (5×2) | **0.70 s** | 0.71 s | 33 | 0.28 s | 4.2 KB | ~8.5 ms/call | +| medium (20×5) | **6.18 s** | 6.71 s | 243 | 3.05 s | 33.2 KB | ~12.5 ms/call | +| large (50×5) | **20.77 s** | 21.78 s | 603 | 13.39 s | 82.8 KB | ~22 ms/call | + +**A single large-project load takes ~21 seconds** — and this is a *localhost* emulator; +real Azure Blob has higher per-op latency, so production is worse. Storage accounts for +~65% of wall time (13.4 s of 20.8 s); the rest is Python serialization, the double-JSON +deserialize (H4), and per-blob client creation inside the blob layer. + +**B1 is super-linear:** per-call cost climbs 8.5 → 12.5 → 22 ms as the partition grows, +because each of the 50 redundant full-`LABELS` partition scans lists + downloads *every* +label blob, and that set grows with the project. So the redundant scans cost more the +bigger the project gets — compounding the O(layers × models) round-trip growth. + +For reference, the local-FS replay wall (no network, relative floor only) was +6.7 / 51.0 / 356.5 ms for small / medium / large. + +## Browser-side (UI) baseline — MEASURED (Playwright vs the real React app) + +Captured via `tools/ui_bench.cjs` driving the real UI (swa-cli emulator in Docker) +with Playwright. The cheap auth/user bootstrap is mocked (crafted SWA admin cookie + +route interception of `/.auth/me`, `GetUserById`, `PutUser`); **`GetProjectDetails` +hits the real API**. TTI = navigation → first image-layer row visible in the DOM. + +| Fixture | **TTI** (open → layers) | initial `GetProjectDetails` | GPD calls on load | 20 s poll | +|---|---|---|---|---| +| small (5×2) | 3.10 s | 1.27 s | 2 | (interval not reached) | +| medium (20×5) | 12.36 s | 10.40 s | 2 | 1 call, 6.20 s | +| large (50×5) | **40.27 s** | 38.44 s | 2 | 1 call, **36.50 s** | + +**TTI is API-bound:** it tracks the `GetProjectDetails` duration almost exactly (large +40.3 s TTI vs 38.4 s call — only ~2 s of render). So the layer-loading fix is +fundamentally the backend fix; UI work reduces the *amplifiers* below. + +Two UI-specific amplifiers the browser run exposed: +- **Duplicate concurrent fetch (U1-adjacent):** the page issues `GetProjectDetails` + **twice concurrently** on load (React StrictMode in dev + no request dedup). The two + 603-round-trip requests contend, so browser-observed latency is **~2× the isolated + API call** (large: 38.4 s browser vs 20.8 s isolated `curl`). Production build drops + the StrictMode double, but the absence of dedup/caching is real. +- **Poll re-does everything (U1):** the 20 s poll fires a fresh full + `GetProjectDetails` (large poll call = 36.5 s), re-incurring all 603 round-trips and + re-rendering the whole tree (no memoization, monolithic context). Because the + response (~21–38 s) is **longer than the 20 s interval**, polls overlap and pile up. + +**Caveats (inflate absolute numbers, not the structural findings):** UI ran under Vite +**dev mode** (unminified + StrictMode double-invoke); the API image is amd64 emulated on +Apple Silicon; storage is a localhost Azurite emulator. A production build + real Azure +would shift the constants but not the O(layers × models) scaling or the amplifiers. +- To capture TTI/poll cost precisely once the stack is up: Chrome DevTools Performance + trace on the project page; `Server-Timing` (now emitted by the API) surfaces + server storage time directly in the Network panel. + +## How to reproduce + +```bash +# Headline logical-call baseline (no infra needed): +PYTHONPATH=hastelib/src python3 \ + spec/features/perf-layer-loading/tools/phase0_baseline.py + +# Real latency (requires running stack + HASTE_PERF=true on the API + seeded project): +METADATA_STORAGE_TYPE=local DATA_PATH=/tmp/haste-bench PYTHONPATH=hastelib/src \ + python3 spec/features/perf-layer-loading/tools/seed_synthetic_project.py \ + --project-id --layers 50 --models 5 +python3 spec/features/perf-layer-loading/tools/bench_api_http.py \ + --base-url http://localhost:7071/api --project-id --repeats 30 +``` + +## Targets to beat (from [README.md](README.md#success-criteria)) + +| Metric | Baseline (large, measured) | Target | +|---|---|---| +| Logical calls / request | **603** | 7 | +| API p50 / p95 latency | **20.8 s / 21.8 s** (Azurite) | < 1.5 s | +| Payload (default shape) | 82.8 KB | smaller via `summary` mode | + +## Environment notes + +- Backend: Azurite blob emulator on `localhost` (compose). Real Azure Blob per-op + latency is higher, so these numbers are a **lower bound** on production. +- Host: Docker Desktop on Apple Silicon; the amd64 API image runs emulated, inflating + the non-storage (CPU) portion somewhat. Logical-call counts are hardware-independent; + use Azure metrics for actual transaction counts. +- Reproduce: `docker compose -f docker/docker-compose.yml -f docker/docker-compose.perf.yml + up -d hastefuncapi api-proxy`, seed via `seed_synthetic_project.py` inside the + `hastefuncapi` container, then run `bench_api_http.py` from the host. diff --git a/spec/features/perf-layer-loading/test-plan.md b/spec/features/perf-layer-loading/test-plan.md new file mode 100644 index 00000000..e33e13dd --- /dev/null +++ b/spec/features/perf-layer-loading/test-plan.md @@ -0,0 +1,129 @@ +# Test Plan: Image Layer & Model Run Loading Performance + +## Contents + +- [Strategy](#strategy) +- [Benchmark Fixture](#benchmark-fixture) +- [Metrics](#metrics-captured-per-fixture-size) +- [Baseline](#baseline-capture-phase-0--measured-2026-08-03) +- [Regression Tests](#correctness--regression-tests) +- [Load Test](#load-test-phase-3-gate) +- [Exit Gate](#exit-gate) + +## Strategy + +Performance work must be **measured, not asserted**. Every phase compares against the +Phase 0 baseline on a fixed synthetic project. Correctness tests guard that the +refactors preserve behavior (same data, new speed). + +## Benchmark fixture + +A synthetic project used everywhere in this spec: + +- **Small:** 5 layers × 2 models +- **Medium:** 20 layers × 5 models +- **Large (headline):** 50 layers × ~5 models, plus labels + validation per layer + +Provide a seed script (`hastelib/tests` helper or a queue-message replay) that +populates the local Docker Compose storage emulator (Azurite) with these shapes. + +## Metrics captured (per fixture size) + +| Metric | How | Target (Large) | +|---|---|---| +| Uncached `GetProjectDetails` p50 / p95 | HTTP harness sends `Cache-Control: no-cache` | p95 < 1.5s | +| Warm process-cache p50 / p95 | HTTP harness with `--allow-cache` | tracked separately | +| Logical data-layer calls | `HASTE_PERF` headers | 7; not an Azure transaction metric | +| Response payload size | `Content-Length` | tracked; smaller in `summary` mode | +| UI time-to-interactive | DevTools performance trace | p95 < 2s | +| Idle open-project network/CPU over 60s | DevTools, no data change | no full refetch; `304` on poll | + +## Baseline capture (Phase 0 — measured 2026-08-03) + +Full detail in [results.md](results.md). Captured via `tools/phase0_baseline.py` +(real code, real seeded data, local FS backend). + +| Fixture | logical calls | API p50 | API p95 | payload | **UI TTI** | +|---|---|---|---|---|---| +| Small (5×2) | **33** | 0.70 s | 0.71 s | 4.2 KB | 3.10 s | +| Medium (20×5) | **243** | 6.18 s | 6.71 s | 33.2 KB | 12.36 s | +| Large (50×5) | **603** | **20.77 s** | **21.78 s** | 82.8 KB | **40.27 s** | + +Logical calls match `3 + L·(2M+2)` exactly. Large breakdown: 301 `load`, 52 +`load_all_from_partition` (50 redundant `LABELS` scans = B1), 250 `export` (B2). +API latency measured via `tools/bench_api_http.py`; UI TTI via `tools/ui_bench.cjs` +(Playwright vs the real app). TTI is API-bound (~2 s render over the API call) and the +UI fires the call twice concurrently on load. All numbers are a lower bound (localhost +Azurite, amd64 emulation, Vite dev mode). See [results.md](results.md). + +## Correctness / regression tests + +### Backend (`hastelib/tests/`) +- [x] `load_map`: duplicate/missing keys, worker bounds, native query and fallback paths. +- [x] `load_filtered`: non-empty predicates and missing-field semantics. +- [x] Exact metadata matching and partition isolation across file/blob/list paths. +- [x] H4 tolerant read for legacy double-encoded and current JSON. +- [x] `BlobServiceClient` reuse by connection target. +- [x] Shared executor aggregate cap, ordering, nesting, cancellation, and exceptions. +- [x] Atomic artifact download, traversal rejection, and partial-file cleanup. +- [x] Cosmos, Data Lake, PostgreSQL, Blob, and local read-contract coverage. + +### API (`hastefuncapi`) +- [x] `GetProjectDetails` response assembly is byte-for-byte checked against a complete + expected fixture; a real local-storage test covers legacy key-only related records. +- [ ] Deferred: `summary` mode omits `models[]` but keeps counts. +- [ ] Deferred: `includeArtifacts=false` skips artifact expansion. +- [x] ETag/304, weak/list matching, cache hit/miss, refresh, failure retry, and keying. +- [x] `GenerateProjectStats` loads labels once and reports zero for unlabeled layers. + +### Queue (`hastefuncqueues`) +- Training trigger with `load_filtered` selects the same label project as before. +- `save_batch` persists all items; partial-failure behavior defined and tested. +- With `batchSize`>1: N messages processed, no dropped/duplicated results; poison + path exercised at `maxDequeueCount`. + +### UI (`ui`) +- [x] **Single-flight (U7):** utility covers dedupe, supersession, abort, and retry; + browser benchmark remains the integration proof for exactly one initial request. +- [x] Poll guard checks visibility, in-flight state, and active jobs. +- [x] HTTP helper handles `304` without parsing a body; component returns before state + update. +- [x] Production browser: one initial call, zero terminal-project polls over 26 s. +- [x] Active browser: one conditional poll at 20 s, returning `304` without overlap. +- [x] Map-route smoke test: lazy loader provides `atlas`, drawing, and `SwipeMap`. +- [x] Route splitting: main JS 1.49 MB → about 120 KB; Azure Maps CDN assets no longer block + non-map routes. +- `LayerRow`/`ModelRow` memoized: unchanged props ⇒ no re-render (React Profiler). +- Expanding a layer renders its model rows; collapsed layers render none. +- Edit/labeling helpers issue independent GETs concurrently (`Promise.all`). + +## Load test (Phase 3 gate) + +- Enqueue a burst (e.g. 100 image/inference messages); compare end-to-end drain time + and poison count for `batchSize` 1 vs candidate value. +- Confirm no `visibilityTimeout` re-enqueue storms on long-running steps. + +## Exit gate + +### Current Validation (2026-09-01) + +- Focused regression matrix and final suite counts are refreshed during the final + validation pass. +- New performance core modules have 100% statement and branch coverage. +- Performance-stack API suite: **50 passed**; two server-managed-field cases are + owned by the independent security PR. Queue suite: **6 passed**. + UI suite: **121 passed**; production build passes. +- New UI utilities: 100% line, branch, and function coverage. +- Full `hastelib` suite: **581 passed**. The stale `ArtifactProcessor.zip` test was + replaced with an isolated test of the current fetch delegation contract. +- UI production build passes. Changed UI files have zero ESLint diagnostics; the + repository-wide lint command remains red from unrelated existing files. +- Clean 50×5 HTTP fixture: 50 layers, 250 models/artifacts, correct validation counts; + uncached 1.85 s p50 / 2.00 s p95, warm-cache 9.1 ms / 12.2 ms. +- Production project page: 2.12 s TTI, one initial request, 58 ms post-response render, + and zero terminal-project polls during 26 seconds. Target remains open. + +- [ ] Large-fixture targets met and recorded in the baseline table (before/after). +- [x] Feature-specific correctness/parity tests green. +- [ ] `docker compose up` runs the full stack with the synthetic project without error. +- [ ] CI (secret-scan, deploy-apps) passes. diff --git a/spec/features/perf-layer-loading/tools/bench_api_http.py b/spec/features/perf-layer-loading/tools/bench_api_http.py new file mode 100644 index 00000000..ec5a76eb --- /dev/null +++ b/spec/features/perf-layer-loading/tools/bench_api_http.py @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Benchmark GetProjectDetails against a running HASTE API (real latency). + +Use this once the full stack is up (Docker Compose or Azure) and a synthetic +project has been seeded into that backend. It captures end-to-end p50/p95 latency +plus the server-side round-trip count and storage time exposed by the Phase 0 +headers (requires HASTE_PERF=true on the API). + +Run: + python spec/features/perf-layer-loading/tools/bench_api_http.py \ + --base-url http://localhost:7071/api \ + --project-id --repeats 30 [--code ] + +Reads only the standard library so it can run anywhere. +""" +import argparse +import json +import statistics +import time +import urllib.request + + +def _pct(values, p): + if not values: + return None + s = sorted(values) + k = max(0, min(len(s) - 1, int(round((p / 100.0) * (len(s) - 1))))) + return s[k] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:7071/api") + ap.add_argument("--project-id", required=True) + ap.add_argument("--repeats", type=int, default=30) + ap.add_argument("--warmup", type=int, default=3) + ap.add_argument("--code", default=None, help="Azure Functions key, if required") + ap.add_argument( + "--allow-cache", + action="store_true", + help="Measure warm server-cache behavior instead of forcing reloads.", + ) + args = ap.parse_args() + + qs = f"?projectId={args.project_id}&includeModels=True" + if args.code: + qs += f"&code={args.code}" + url = f"{args.base_url}/GetProjectDetails{qs}" + + latencies, storage_calls, storage_ms, payloads = [], [], [], [] + for i in range(args.warmup + args.repeats): + t0 = time.perf_counter() + headers = {} if args.allow_cache else {"Cache-Control": "no-cache"} + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request) as resp: + body = resp.read() + hdrs = resp.headers + dt = (time.perf_counter() - t0) * 1000.0 + if i < args.warmup: + continue + latencies.append(dt) + payloads.append(len(body)) + calls_header = hdrs.get("X-Haste-Data-Layer-Calls") or hdrs.get( + "X-Haste-Storage-Calls" + ) + timing_header = hdrs.get("X-Haste-Data-Layer-Ms") or hdrs.get( + "X-Haste-Storage-Ms" + ) + if calls_header: + storage_calls.append(int(calls_header)) + if timing_header: + storage_ms.append(float(timing_header)) + + result = { + "project_id": args.project_id, + "repeats": args.repeats, + "latency_p50_ms": round(statistics.median(latencies), 1), + "latency_p95_ms": round(_pct(latencies, 95), 1), + "latency_max_ms": round(max(latencies), 1), + "payload_kb": round(statistics.median(payloads) / 1024, 1), + "server_data_layer_calls": storage_calls[0] if storage_calls else None, + "server_data_layer_ms_p50": ( + round(statistics.median(storage_ms), 1) if storage_ms else None + ), + } + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/spec/features/perf-layer-loading/tools/phase0_baseline.py b/spec/features/perf-layer-loading/tools/phase0_baseline.py new file mode 100644 index 00000000..f4918dc7 --- /dev/null +++ b/spec/features/perf-layer-loading/tools/phase0_baseline.py @@ -0,0 +1,157 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Phase 0 baseline: measure GetProjectDetails logical data-layer calls. + +Seeds synthetic projects (small / medium / large) into a temporary local-FS +backend and replays the *exact* read sequence of the ``GetProjectDetails`` handler +(``api/hastefuncapi/function_app.py`` lines ~534-638) with perf instrumentation on. + +Reports, per size, the number of logical data-layer calls plus a per-op breakdown +and a local-FS wall-clock (for relative comparison only; absolute latency p50/p95 +must be measured against the running Azure/Docker stack via bench_api_http.py). + +Run: + PYTHONPATH=hastelib/src \ + python spec/features/perf-layer-loading/tools/phase0_baseline.py +""" +import json +import os +import statistics +import sys +import tempfile + +# Force the local filesystem backend for a self-contained, infra-free baseline. +os.environ.setdefault("METADATA_STORAGE_TYPE", "local") + +sys.path.insert(0, os.path.join(os.path.dirname(__file__))) # for seed import +from seed_synthetic_project import seed # noqa: E402 + +from hastegeo.core.config import Config # noqa: E402 +from hastegeo.core.processors.metadata import MetadataProcessor # noqa: E402 +from hastegeo.core.utils import perf # noqa: E402 + +SIZES = [ + ("small", 5, 2), + ("medium", 20, 5), + ("large", 50, 5), +] +REPEATS = 5 + + +def _mp(data_type, project_id): + return MetadataProcessor(data_type=data_type, partition_key=project_id) + + +def replay_get_project_details(project_id, include_models=True): + """Faithful replay of the GetProjectDetails read+assemble sequence. + + Mirrors api/hastefuncapi/function_app.py:534-638. Returns the serialized + payload length so we can report response size alongside round-trips. + """ + types = Config.get_metadata_types() + + project = _mp(types.PROJECT.value, project_id).load(project_id) + image_layers = _mp(types.IMAGELAYER.value, project_id).load_all_from_partition() + models = [] + if include_models: + models = _mp(types.MODEL.value, project_id).load_all_from_partition() + + for image_layer in image_layers: + image_layer_id = image_layer["imageLayerId"] + if include_models: + match_models = [ + m for m in models if m["imageLayerId"] == image_layer_id + ] + match_models.sort(key=lambda x: x["creationDate"], reverse=True) + for model in match_models: + try: + model["artifacts"] = _mp( + types.MODEL_ARTIFACTS.value, project_id + ).load(model["modelId"]) + except FileNotFoundError: + model["artifacts"] = None + try: + if not model.get("labelsUrl"): + model["labelsUrl"] = _mp( + types.TRAIN_LABELS.value, project_id + ).export(key=model["modelId"], data_format="geojson") + except FileNotFoundError: + model["labelsUrl"] = None + image_layer["models"] = match_models + image_layer["modelCount"] = len(match_models) + + label_projects = _mp( + types.LABELS.value, project_id + ).load_all_from_partition() + match = next( + (lp for lp in label_projects + if lp["imageLayerId"] == image_layer_id), + None, + ) + if match is not None and match.get("labels") is not None: + image_layer["labelProjectCount"] = len(match["labels"]) + + try: + validation = _mp(types.VALIDATION.value, project_id).load( + image_layer_id + ) + image_layer["validationLabelCount"] = len( + validation.get("labels") or {} + ) + except FileNotFoundError: + image_layer["validationLabelCount"] = 0 + + project["imageLayer"] = image_layers + project["imageLayerCount"] = len(image_layers) + project["imageLayer"].sort(key=lambda x: x["creationDate"], reverse=True) + return len(json.dumps(project)) + + +def run(): + rows = [] + for name, layers, models_per in SIZES: + with tempfile.TemporaryDirectory(prefix=f"haste-bench-{name}-") as d: + os.environ["DATA_PATH"] = d + project_id = f"00000000-0000-4000-8000-{layers:06d}{models_per:06d}" + total_models = seed( + project_id, layers, models_per, + labels_per_layer=20, validation_per_layer=10, + with_labels_url=False, + ) + + walls, calls, payload, ops = [], None, None, None + for _ in range(REPEATS): + counter = perf.begin(True) + import time + t0 = time.perf_counter() + payload = replay_get_project_details(project_id) + walls.append((time.perf_counter() - t0) * 1000.0) + calls = counter.calls + ops = {k: v["calls"] for k, v in counter.by_op.items()} + perf.end() + + rows.append({ + "size": name, "layers": layers, "models_per": models_per, + "total_models": total_models, "data_layer_calls": calls, + "ops": ops, "payload_bytes": payload, + "wall_p50_ms": round(statistics.median(walls), 1), + "wall_p95_ms": round(max(walls), 1), + }) + + hdr = (f"{'size':7} {'layers':6} {'mdl/l':5} {'data_calls':10} " + f"{'payload_kb':10} {'localfs_p50ms':13} {'ops (by type)'}") + print(hdr) + print("-" * len(hdr)) + for r in rows: + print(f"{r['size']:7} {r['layers']:<6} {r['models_per']:<5} " + f"{r['data_layer_calls']:<10} {r['payload_bytes']/1024:<10.1f} " + f"{r['wall_p50_ms']:<13} {r['ops']}") + print() + for r in rows: + L, M = r["layers"], r["models_per"] + print(f" {r['size']}: data_layer_calls={r['data_layer_calls']} " + f"formula 3 + L*(2M+2) = {3 + L * (2 * M + 2)}") + + +if __name__ == "__main__": + run() diff --git a/spec/features/perf-layer-loading/tools/seed_synthetic_project.py b/spec/features/perf-layer-loading/tools/seed_synthetic_project.py new file mode 100644 index 00000000..7551a382 --- /dev/null +++ b/spec/features/perf-layer-loading/tools/seed_synthetic_project.py @@ -0,0 +1,161 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Seed a synthetic HASTE project for perf baselining (Phase 0). + +Writes a project with ``--layers`` image layers, each with ``--models`` models, +plus per-layer LABELS + VALIDATION records and per-model MODEL_ARTIFACTS, using the +real ``MetadataProcessor`` against whatever backend the environment is configured +for (defaults to the ``local`` filesystem layer, so no Azure/Docker is required). + +Usage (local FS, no infra): + METADATA_STORAGE_TYPE=local DATA_PATH=/tmp/haste-bench \ + PYTHONPATH=hastelib/src \ + python spec/features/perf-layer-loading/tools/seed_synthetic_project.py \ + --project-id 00000000-0000-4000-8000-000000000050 --layers 50 --models 5 + +The project id is a canonical GUID so the same seed is reusable against the real +HTTP API (which validates ``projectId`` as a GUID). +""" +import argparse +import uuid + +from hastegeo.core.config import Config +from hastegeo.core.processors.metadata import MetadataProcessor + +# Fixed base date so seeds are deterministic (no wall-clock dependency). +_BASE_DATE = "2026-01-01T00:00:00Z" + + +def _iso(seq): + # Distinct, sortable creationDate values without importing time. + return f"2026-01-{(seq % 27) + 1:02d}T00:00:00Z" + + +def _mp(data_type, project_id): + return MetadataProcessor(data_type=data_type, partition_key=project_id) + + +def seed(project_id, layers, models, labels_per_layer, validation_per_layer, + with_labels_url): + types = Config.get_metadata_types() + + _mp(types.PROJECT.value, project_id).save( + project_id, + { + "projectId": project_id, + "name": f"Perf baseline {layers}x{models}", + "description": "Synthetic project for perf-layer-loading Phase 0.", + "creationDate": _BASE_DATE, + }, + ) + + model_total = 0 + for li in range(layers): + layer_id = f"layer-{li:04d}" + _mp(types.IMAGELAYER.value, project_id).save( + layer_id, + { + "imageLayerId": layer_id, + "projectId": project_id, + "name": f"Layer {li}", + "creationDate": _iso(li), + "userId": "bench@example.com", + "status": "Processed", + "statusMessage": "", + "currentStep": 10, + "totalSteps": 10, + "progressPct": 100, + }, + ) + + # One label project per layer, with N labels. + _mp(types.LABELS.value, project_id).save( + f"labelproj-{li:04d}", + { + "labelprojectId": f"labelproj-{li:04d}", + "imageLayerId": layer_id, + "labels": [{"id": j} for j in range(labels_per_layer)], + }, + ) + + # Validation record keyed by imageLayerId. + _mp(types.VALIDATION.value, project_id).save( + layer_id, + {"imageLayerId": layer_id, + "labels": {str(j): {"id": j} for j in range(validation_per_layer)}}, + ) + + for mi in range(models): + model_id = f"{li:04d}{mi:02d}" + # Fields below mirror what the UI's ModelRow / ModelResultsButton read + # so the real React tree renders without crashing (e.g. inferenceJobs + # must be an array). This enriches the record but does NOT change the + # GetProjectDetails round-trip count. + model = { + "modelId": model_id, + "imageLayerId": layer_id, + "projectId": project_id, + "name": f"Model {li}-{mi}", + "userId": "bench@example.com", + "modelType": "segmentation", + "status": "Trained", + "statusMessage": "", + "trainDate": _iso(mi), + "creationDate": _iso(mi), + "labelsCount": 100, + "currentStep": 10, + "totalSteps": 10, + "progressPct": 100, + "inferenceJobs": [], + "inferenceStatus": None, + "inferenceStatusMessage": "", + "inferenceCurrentStep": 0, + "inferenceTotalSteps": 0, + "inferenceProgressPct": 0, + "labelsUrl": None, + "artifacts": None, + } + if with_labels_url: + model["labelsUrl"] = f"https://example/{model_id}.geojson" + _mp(types.MODEL.value, project_id).save(model_id, model) + + _mp(types.MODEL_ARTIFACTS.value, project_id).save( + model_id, + {"modelId": model_id, "metrics": {"iou": 0.5, "f1": 0.6}}, + ) + model_total += 1 + + return model_total + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--project-id", default=str(uuid.uuid4())) + ap.add_argument("--layers", type=int, default=50) + ap.add_argument("--models", type=int, default=5) + ap.add_argument("--labels-per-layer", type=int, default=20) + ap.add_argument("--validation-per-layer", type=int, default=10) + ap.add_argument( + "--with-labels-url", + action="store_true", + help="Seed models with labelsUrl set (skips the per-model TRAIN_LABELS " + "export round-trip; omit to reproduce the worst-case N+1).", + ) + args = ap.parse_args() + + total = seed( + args.project_id, + args.layers, + args.models, + args.labels_per_layer, + args.validation_per_layer, + args.with_labels_url, + ) + print( + f"Seeded project {args.project_id}: {args.layers} layers, " + f"{total} models, with_labels_url={args.with_labels_url}" + ) + + +if __name__ == "__main__": + main() diff --git a/spec/features/perf-layer-loading/tools/ui_bench.cjs b/spec/features/perf-layer-loading/tools/ui_bench.cjs new file mode 100644 index 00000000..751d2327 --- /dev/null +++ b/spec/features/perf-layer-loading/tools/ui_bench.cjs @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// Browser-side Phase 0 baseline for the project page (perf-layer-loading spec). +// +// Drives the real React UI (served by the swa-cli emulator in Docker) with +// Playwright and measures, for a seeded project: +// - time-to-interactive: navigation start -> first image-layer row visible +// - the real GetProjectDetails request duration (the expensive call) +// - the 20s background poll: whether it fires and its cost +// +// The cheap auth/user bootstrap is mocked (crafted SWA admin cookie + route +// interception of /.auth/me, GetUserById, PutUser) so the page renders without a +// real login; GetProjectDetails hits the REAL API and is what we measure. +// +// Run (playwright installed in a scratch dir): +// NODE_PATH=/tmp/haste-uibench/node_modules \ +// node spec/features/perf-layer-loading/tools/ui_bench.cjs \ +// --ui http://localhost:4280 --api http://localhost:7071 \ +// --project 00000000-0000-4000-8000-000050000005 +const { chromium } = require("playwright"); + +function arg(name, def) { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : def; +} + +const UI = arg("ui", "http://localhost:4280"); +const API = arg("api", "http://localhost:7071"); +const PROJECT = arg("project", "00000000-0000-4000-8000-000050000005"); +const POLL_WAIT_MS = parseInt(arg("pollwait", "26000"), 10); + +const principal = { + identityProvider: "aad", + userId: "benchuser", + userDetails: "bench@example.com", + userRoles: ["authenticated", "administrators", "contributors"], + claims: [], +}; +const mockUser = { + userId: "bench@example.com", + email: "bench@example.com", + name: "Bench User", + status: "Active", + userRoles: ["administrators"], + identityProvider: "aad", + settings: { itemsPerPage: 10 }, +}; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ bypassCSP: true }); + + // Crafted SWA auth cookie: base64(JSON(clientPrincipal)), no signing locally. + const cookieVal = Buffer.from(JSON.stringify(principal)).toString("base64"); + await context.addCookies([ + { name: "StaticWebAppsAuthCookie", value: cookieVal, domain: "localhost", path: "/" }, + ]); + + const page = await context.newPage(); + const consoleErrors = []; + const requestTimeline = []; + const trackedRequests = new WeakMap(); + page.on("console", (m) => { if (m.type() === "error") consoleErrors.push(m.text()); }); + + // Mock cheap bootstrap calls (not what we measure). + await context.route("**/.auth/me", (r) => + r.fulfill({ contentType: "application/json", body: JSON.stringify({ clientPrincipal: principal }) }) + ); + await context.route("**/GetUserById**", (r) => + r.fulfill({ contentType: "application/json", body: JSON.stringify(mockUser) }) + ); + await context.route("**/PutUser**", (r) => + r.fulfill({ contentType: "application/json", body: JSON.stringify(mockUser) }) + ); + + // Time every GetProjectDetails call (the real, expensive one). + const gpd = []; + const requestRecords = new WeakMap(); + page.on("request", (req) => { + if (!req.url().includes("GetProjectDetails")) return; + const record = { url: req.url(), startedAt: Date.now(), ms: null }; + requestRecords.set(req, record); + gpd.push(record); + }); + page.on("requestfinished", async (req) => { + if (!req.url().includes("GetProjectDetails")) return; + const t = req.timing(); + const record = requestRecords.get(req); + if (record) { + record.ms = Number.isFinite(t.responseEnd) + ? Math.round(t.responseEnd) + : Date.now() - record.startedAt; + } + }); + page.on("response", (response) => { + const record = requestRecords.get(response.request()); + if (!record) return; + record.status = response.status(); + record.cache = response.headers()["x-haste-cache"] ?? null; + }); + + const observedApiOrigins = new Set(); + page.on("request", (req) => { + const u = req.url(); + if (u.includes("/api/GetProjectDetails")) observedApiOrigins.add(new URL(u).origin); + }); + + const t0 = Date.now(); + page.on("request", (req) => { + const url = req.url(); + if ( + req.resourceType() === "document" || + /\.auth\/me|GetUserById|PutUser|GetPublishingProviders/.test(url) + ) { + const record = { + resourceType: req.resourceType(), + path: new URL(url).pathname, + startedMs: Date.now() - t0, + finishedMs: null, + }; + trackedRequests.set(req, record); + requestTimeline.push(record); + } + }); + page.on("requestfinished", (req) => { + const record = trackedRequests.get(req); + if (record) record.finishedMs = Date.now() - t0; + }); + await page.goto(`${UI}/project/${PROJECT}`, { waitUntil: "commit", timeout: 60000 }); + + // TTI: first image-layer row (seed names layers "Layer "). + const rowTimeout = parseInt(arg("rowtimeout", "90000"), 10); + let tti = null, rowError = null; + try { + await page.waitForFunction( + () => !!document.body && /Layer \d+/.test(document.body.innerText), + null, + { timeout: rowTimeout, polling: 50 } + ); + tti = Date.now() - t0; + } catch (e) { + rowError = String(e).split("\n")[0]; + } + + let bodyText = null; + try { + bodyText = (await page.locator("body").innerText()).replace(/\s+/g, " ").slice(0, 600); + } catch (e) { bodyText = ""; } + try { await page.screenshot({ path: arg("shot", "/tmp/haste-uibench/shot.png"), fullPage: true }); } catch (e) {} + + const interactiveAt = Date.now(); + const initialCalls = gpd.filter((call) => call.startedAt <= interactiveAt); + const initialGpdMs = initialCalls.length ? initialCalls[0].ms : null; + const initialGpdStartedMs = initialCalls.length + ? initialCalls[0].startedAt - t0 + : null; + const initialGpdFinishedMs = + initialGpdStartedMs !== null && initialGpdMs !== null + ? initialGpdStartedMs + initialGpdMs + : null; + const gpdCountAfterLoad = initialCalls.length; + + // Observe the 20s background poll. + const pollStart = Date.now(); + await page.waitForTimeout(POLL_WAIT_MS); + const pollCalls = gpd.filter((call) => call.startedAt >= pollStart); + + const result = { + project: PROJECT, + time_to_interactive_ms: tti, + initial_getprojectdetails_started_ms: initialGpdStartedMs, + initial_getprojectdetails_ms: initialGpdMs, + initial_getprojectdetails_finished_ms: initialGpdFinishedMs, + initial_getprojectdetails_status: initialCalls[0]?.status ?? null, + initial_getprojectdetails_cache: initialCalls[0]?.cache ?? null, + render_after_project_response_ms: + tti !== null && initialGpdFinishedMs !== null + ? Math.max(0, tti - initialGpdFinishedMs) + : null, + getprojectdetails_calls_during_load: gpdCountAfterLoad, + poll_window_ms: POLL_WAIT_MS, + poll_getprojectdetails_calls: pollCalls.length, + poll_getprojectdetails_ms: pollCalls.map((c) => c.ms), + poll_getprojectdetails: pollCalls.map((call) => ({ + ms: call.ms, + status: call.status ?? null, + cache: call.cache ?? null, + })), + api_origins_observed: [...observedApiOrigins], + navigation_timing: await page.evaluate(() => { + const navigation = performance.getEntriesByType("navigation")[0]; + return navigation + ? { + responseEnd: Math.round(navigation.responseEnd), + domInteractive: Math.round(navigation.domInteractive), + domContentLoadedEventEnd: Math.round( + navigation.domContentLoadedEventEnd + ), + loadEventEnd: Math.round(navigation.loadEventEnd), + } + : null; + }), + slowest_resources: await page.evaluate(() => + performance + .getEntriesByType("resource") + .sort((left, right) => right.duration - left.duration) + .slice(0, 10) + .map((entry) => ({ + path: new URL(entry.name).pathname, + initiatorType: entry.initiatorType, + startTime: Math.round(entry.startTime), + duration: Math.round(entry.duration), + transferSize: entry.transferSize, + })) + ), + bootstrap_requests: requestTimeline, + row_wait_error: rowError, + body_text_sample: bodyText, + console_errors: consoleErrors.slice(0, 4), + }; + console.log(JSON.stringify(result, null, 2)); + + await browser.close(); +})().catch((e) => { console.error("FATAL", e); process.exit(1); }); diff --git a/spec/features/perf-layer-loading/user-stories.md b/spec/features/perf-layer-loading/user-stories.md new file mode 100644 index 00000000..fc28f213 --- /dev/null +++ b/spec/features/perf-layer-loading/user-stories.md @@ -0,0 +1,131 @@ +# User Stories: Image Layer and Model Run Loading Performance + +## Contents + +- [Personas](#personas) +- [Stories](#stories) +- [Agent Assignment Map](#agent-assignment-map) +- [Story Map](#story-map) +- [Out of Scope](#out-of-scope) + +## Personas + +| Persona | Description | Key Goal | +|---|---|---| +| Disaster Analyst | Reviews imagery layers and model results during response work | Open and refresh large projects without long blocking waits | +| ML Engineer | Monitors training, embedding, and inference runs | Receive current run status without duplicate requests | +| Operator | Runs HASTE on supported metadata backends | Bound resource use and diagnose storage cost accurately | + +## Stories + +### US-001: Load Large Projects Reliably + +**As a** disaster analyst, **I want** project layers and model runs loaded without +sequential N+1 storage waits, **so that** large assessments remain usable. + +**Priority:** P0 +**Components:** `hastefuncapi`, `hastelib` + +**Acceptance Criteria:** + +```gherkin +Given a project with 50 layers and 5 models per layer +When GetProjectDetails is requested with includeModels=true +Then the response contains the same layers, models, artifacts, counts, and ordering as the legacy endpoint +And the request uses seven logical data-layer operations +``` + +```gherkin +Given legacy artifact or validation documents without embedded join IDs +When project details are assembled +Then related records are joined by their storage identifiers +``` + +### US-002: Bound Backend Concurrency + +**As an** operator, **I want** concurrent metadata downloads to share one bounded +worker budget, **so that** requests cannot multiply thread usage without limit. + +**Priority:** P0 +**Components:** `hastelib` + +**Acceptance Criteria:** + +```gherkin +Given multiple concurrent storage map operations +When they execute in one Functions worker process +Then aggregate blocking I/O never exceeds HASTE_BLOB_DOWNLOAD_WORKERS +And invalid worker settings fail fast +``` + +### US-003: Refresh Without Duplicate Work + +**As an** ML engineer, **I want** project refreshes deduplicated and conditional, +**so that** polling does not pile up while jobs run. + +**Priority:** P0 +**Components:** `hastefuncapi`, `ui/src` + +**Acceptance Criteria:** + +```gherkin +Given two concurrent requests for the same project response +When the first request is still loading +Then the API and UI each share one in-flight operation +``` + +```gherkin +Given an unchanged fresh cached response +When the UI sends its ETag +Then the API returns 304 without storage work +And the UI does not update project state +``` + +```gherkin +Given a project with no active jobs or a hidden browser tab +When the poll interval elapses +Then no project-details request is started +``` + +### US-004: Preserve Backend Compatibility + +**As an** operator, **I want** the read contract consistent across configured metadata +backends, **so that** an optimization does not silently make Blob the only working path. + +**Priority:** P1 +**Components:** `hastelib` + +**Acceptance Criteria:** + +```gherkin +Given Blob, local filesystem, Cosmos DB, Data Lake, or PostgreSQL metadata storage +When project-related read primitives are called +Then each backend accepts the shared read signatures +And unsupported remote label URLs degrade to null rather than fail the project +``` + +## Agent Assignment Map + +| Story | Implementing Agent | Validating Agent | Notes | +|---|---|---|---| +| US-001 | `backend-dev` | `backend-validation` | Golden response and local-storage regression | +| US-002 | `backend-dev` | `backend-validation` | Concurrency and failure-path tests | +| US-003 | `backend-dev`, `ui` | `backend-validation`, `ui-validation` | API cache plus browser request guards | +| US-004 | `backend-dev` | `backend-validation` | Mocked backend contract tests | + +## Story Map + +| Priority | Story | Phase | Component | +|---|---|---|---| +| P0 | US-001 | Backend hot path | `hastefuncapi`, `hastelib` | +| P0 | US-002 | Core library | `hastelib` | +| P0 | US-003 | Cache and UI polling | `hastefuncapi`, `ui/src` | +| P1 | US-004 | Backend compatibility | `hastelib` | + +## Out of Scope + +- A materialized per-project response document and write-side invalidation. +- Cross-instance distributed caching. +- Queue concurrency configuration changes without load testing. +- `summary` and `includeArtifacts` response modes. +- SignalR or another server-push transport. \ No newline at end of file diff --git a/ui/index.html b/ui/index.html index 56132907..dd92091c 100644 --- a/ui/index.html +++ b/ui/index.html @@ -6,13 +6,6 @@ - - - - - - -
diff --git a/ui/src/Components/AppBody.jsx b/ui/src/Components/AppBody.jsx index 369f250b..8f9a640a 100644 --- a/ui/src/Components/AppBody.jsx +++ b/ui/src/Components/AppBody.jsx @@ -1,29 +1,43 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import { Route, Routes } from "react-router-dom"; -import { useContext } from "react"; +import { lazy, Suspense, useContext } from "react"; import Loading from "./OtherComponents/Loading"; -import Error404 from "./Error404"; -import Project from "./Project"; -import Projects from "./Projects"; -import ImageLayer from "./ImageLayer"; -import Home from "./Home"; -import LabelingTool from "./LabelingTool/LabelingTool"; -import BuildingValidation from "./BuildingValidation/BuildingValidation"; -import InteractiveLabeler from "./InteractiveLabeler/InteractiveLabeler"; -import Visualizer from "./Visualizer/Visualizer"; -import ModelCatalog from "./ModelCatalog"; -import PublishedDatasets from "./PublishedDatasets"; - -import AdminUsers from "./AdminUsers"; -import AdminSourceTypes from "./AdminSourceTypes"; -import AdminLabelingTool from "./AdminLabelingTool"; -import CreateEditImageLayerForm from "./CreateEditImageLayerForm"; -import HelpDocs from "./HelpDocs"; import PropType from "prop-types"; import { AppContext } from "../AppContext"; +import { loadAzureMaps } from "../util/azureMapsLoader"; + +const loadMapRoute = (importRoute) => () => + loadAzureMaps().then(() => importRoute()); + +const AdminLabelingTool = lazy(() => import("./AdminLabelingTool")); +const AdminSourceTypes = lazy(() => import("./AdminSourceTypes")); +const AdminUsers = lazy(() => import("./AdminUsers")); +const BuildingValidation = lazy( + loadMapRoute(() => import("./BuildingValidation/BuildingValidation")) +); +const CreateEditImageLayerForm = lazy( + loadMapRoute(() => import("./CreateEditImageLayerForm")) +); +const Error404 = lazy(() => import("./Error404")); +const HelpDocs = lazy(() => import("./HelpDocs")); +const Home = lazy(() => import("./Home")); +const ImageLayer = lazy(() => import("./ImageLayer")); +const InteractiveLabeler = lazy( + loadMapRoute(() => import("./InteractiveLabeler/InteractiveLabeler")) +); +const LabelingTool = lazy( + loadMapRoute(() => import("./LabelingTool/LabelingTool")) +); +const ModelCatalog = lazy(() => import("./ModelCatalog")); +const Project = lazy(() => import("./Project")); +const Projects = lazy(() => import("./Projects")); +const PublishedDatasets = lazy(() => import("./PublishedDatasets")); +const Visualizer = lazy( + loadMapRoute(() => import("./Visualizer/Visualizer")) +); const AppBody = ({ setModalComponent }) => { const { appParams } = useContext(AppContext); @@ -33,7 +47,7 @@ const AppBody = ({ setModalComponent }) => { return (
{appParams.isLoading && } - {routesReady && + {routesReady && }> {appParams.userRoles !== null && appParams.publishingEnabled && ( } /> )} @@ -103,7 +117,7 @@ const AppBody = ({ setModalComponent }) => { )} } /> - } + }
); }; diff --git a/ui/src/Components/Project.jsx b/ui/src/Components/Project.jsx index b9f7ca37..203009d3 100644 --- a/ui/src/Components/Project.jsx +++ b/ui/src/Components/Project.jsx @@ -21,7 +21,7 @@ import { } from "@fluentui/react-components"; import { useParams } from "react-router-dom"; import { useState, useEffect } from "react"; -import { apiGet } from "../util/api"; +import { apiGet, apiGetResponse } from "../util/api"; import { useNavigate } from "react-router-dom"; import LayerRow from "./ProjectManagement/LayerRow"; @@ -37,7 +37,9 @@ import { updateUserSettings } from "../AppHelper"; import { collectProjectJobStates, findJobStatusTransitions, + hasActiveProjectJobs, } from "../util/jobNotifications"; +import { createSingleFlight } from "../util/singleFlight"; import PropType from "prop-types"; @@ -64,6 +66,7 @@ const GROUP_OPTIONS = [ ]; const PAGE_SIZE_OPTIONS = [5, 8, 10, 20, 50]; +const EMPTY_LAYERS = []; /** Resolve the group bucket label for an image layer given the grouping. */ function getLayerGroupLabel(item, mode) { @@ -133,13 +136,11 @@ const Project = ({ setModalComponent }) => { const defaultProjectDetailsRef = useRef(null); const projectJobStatesRef = useRef(null); + const projectLoadRef = useRef(createSingleFlight()); + const projectEtagRef = useRef(null); + const projectLifecycleRef = useRef(0); + const projectMountedRef = useRef(false); const { dispatchToast } = useToastController("job-completion-toaster"); - - - useEffect(() => { - setCurrentPage(1); - }, [appParams.userSettings.itemsPerPageLayers]); - const DEFAULT_COMPONENT_STATE = { project: null, visibleModelId: imageLayerId || "-1", @@ -148,41 +149,45 @@ const Project = ({ setModalComponent }) => { const projectCurrentTouruseRef = useRef(DEFAULT_COMPONENT_STATE.visibleModelId === "-1" ? "singleProjectGuide" : "singleProjectModelGuide"); - const [moreInfoVisibleId, setMoreInfoVisibleId] = useState(null); + const [, setMoreInfoVisibleId] = useState(null); const [componentState, setComponentState] = useState(DEFAULT_COMPONENT_STATE); const navigate = useNavigate(); useEffect(() => { - const fetchData = async () => { - await fetchProjectDetails(); - }; - fetchData(); + fetchProjectDetails(true, false); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [imageLayerId]); + }, [projectId, imageLayerId]); useEffect(() => { - const fetchData = async () => { - await fetchProjectDetails(); - - initGuidedTourState(projectCurrentTouruseRef.current, appParams.guidedTourProperties); - initCurrentTour(projectCurrentTouruseRef.current); - - setAppHeaderRightButtons([ - { - iconName: "help", - title: "Help", - id: "helpButton", - onClick: () => - setGuidedTourState(false, initCurrentTour, projectCurrentTouruseRef.current, appParams.guidedTourProperties), - }, - ]); - }; - fetchData(); + const lifecycle = ++projectLifecycleRef.current; + const projectLoad = projectLoadRef.current; + const initialTour = projectCurrentTouruseRef.current; + projectMountedRef.current = true; + initGuidedTourState(initialTour, appParams.guidedTourProperties); + initCurrentTour(initialTour); + + setAppHeaderRightButtons([ + { + iconName: "help", + title: "Help", + id: "helpButton", + onClick: () => + setGuidedTourState(false, initCurrentTour, projectCurrentTouruseRef.current, appParams.guidedTourProperties), + }, + ]); //On component dismount return () => { + projectMountedRef.current = false; + queueMicrotask(() => { + // StrictMode immediately advances this generation before the microtask. + // eslint-disable-next-line react-hooks/exhaustive-deps + if (projectLifecycleRef.current === lifecycle) { + projectLoad.abort(); + } + }); initCurrentTour(null); - initGuidedTourState(projectCurrentTouruseRef.current, appParams.guidedTourProperties); + initGuidedTourState(initialTour, appParams.guidedTourProperties); setAppHeaderRightButtons([]); setModalComponent(null); }; @@ -198,12 +203,38 @@ const Project = ({ setModalComponent }) => { } }, [componentState.visibleModelId]); - async function fetchProjectDetails(showLoading = true) { - if (showLoading) { + async function fetchProjectDetails( + showLoading = true, + forceRefresh = showLoading + ) { + if (projectEtagRef.current?.projectId !== projectId) { + projectEtagRef.current = null; + } + + const projectLoad = projectLoadRef.current; + if (forceRefresh && projectLoad.isRunning(projectId)) { + projectLoad.abort(); + } + const startsRequest = !projectLoad.isRunning(projectId); + if (showLoading && startsRequest) { setIsLoading(true); } - await apiGet("GetProjectDetails?projectId=" + projectId + "&includeModels=True") - .then((response) => { + const headers = {}; + if (forceRefresh) { + headers["Cache-Control"] = "no-cache"; + } + if (projectEtagRef.current?.etag) { + headers["If-None-Match"] = projectEtagRef.current.etag; + } + + const requestPromise = projectLoad.run(projectId, async (signal) => { + try { + const { data: response, etag, status } = await apiGetResponse( + "GetProjectDetails?projectId=" + projectId + "&includeModels=True", + { signal, headers, cache: forceRefresh ? "no-cache" : "default" } + ); + if (etag) projectEtagRef.current = { projectId, etag }; + if (status === 304) return defaultProjectDetailsRef.current; defaultProjectDetailsRef.current = response; const currentJobStates = collectProjectJobStates(response); const previousJobState = projectJobStatesRef.current; @@ -250,18 +281,33 @@ const Project = ({ setModalComponent }) => { filter: false, }, })); - }) - .catch((error) => { + return response; + } catch (error) { + if (error.name === "AbortError") return null; console.error("Error fetching projects:", error); + return null; + } + }); + if (showLoading && startsRequest) { + requestPromise.finally(() => { + if (!projectLoad.isRunning() && projectMountedRef.current) { + setIsLoading(false); + } }); - if (showLoading) { - setIsLoading(false); } + return requestPromise; } useEffect(() => { - const intervalId = setInterval(async () => { - fetchProjectDetails(false); + const intervalId = setInterval(() => { + const jobs = projectJobStatesRef.current?.jobs; + if ( + document.visibilityState === "visible" && + !projectLoadRef.current.isRunning() && + hasActiveProjectJobs(jobs) + ) { + fetchProjectDetails(false); + } }, 20000); return () => clearInterval(intervalId); @@ -307,7 +353,7 @@ const Project = ({ setModalComponent }) => { } // Filter + sort + group the image layers (memoised so pagination is cheap). - const imageLayers = componentState.project?.imageLayer || []; + const imageLayers = componentState.project?.imageLayer || EMPTY_LAYERS; const processed = useMemo(() => { const search = searchText.toLowerCase(); const filtered = imageLayers.filter( @@ -332,7 +378,6 @@ const Project = ({ setModalComponent }) => { return String(av ?? "").localeCompare(String(bv ?? "")) * dir; }); return sorted; - // eslint-disable-next-line react-hooks/exhaustive-deps }, [imageLayers, searchText, sort, effectiveGroupBy]); if (!componentState.project) { @@ -416,7 +461,7 @@ const Project = ({ setModalComponent }) => { { setModalComponent(null); - fetchProjectDetails(false); + fetchProjectDetails(false, true); }} projectId={projectId} /> diff --git a/ui/src/util/api.js b/ui/src/util/api.js index 62c10143..c5b2e759 100644 --- a/ui/src/util/api.js +++ b/ui/src/util/api.js @@ -5,11 +5,7 @@ const APIUrl = import.meta.env.VITE_API_URL; const APIMSubscriptionKey = import.meta.env.VITE_APIM_SUBSCRIPTION_KEY; import { upsertUser } from "../AppHelper.js"; import { sanitizeRedirectPath } from "./validation.js"; - -function resolveVarConcatChar(text) { - if (text === "") return ""; - return text.includes("?") ? "&" : "?"; -} +import { fetchJsonResponse } from "./http.js"; export function buildUrl(endpoint) { const base = APIUrl + endpoint; @@ -68,14 +64,15 @@ export async function apiLogout(redirectPath = "/") { } export async function apiGet(endpoint) { - try { - const response = await fetch(buildUrl(endpoint)); + const response = await apiGetResponse(endpoint); + return response.data; +} - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - return await response.json(); +export async function apiGetResponse(endpoint, options = {}) { + try { + return await fetchJsonResponse(buildUrl(endpoint), options); } catch (error) { + if (error.name === "AbortError") throw error; console.error("Error fetching.:", error); throw new Error("Error fetching."); } @@ -137,7 +134,7 @@ export async function apiPost(endpoint, data, isFormData = false) { throw new Error(message.error || `HTTP error! status: ${response.status}`); } return await response.json(); - } catch (error) { + } catch { throw new Error("Error uploading chunk."); } } @@ -151,7 +148,7 @@ export async function apiDelete(endpoint) { throw new Error(`HTTP error! status: ${response.status}`); } return response; - } catch (error) { + } catch { throw new Error("Error deleting element."); } } \ No newline at end of file diff --git a/ui/src/util/azureMapsLoader.js b/ui/src/util/azureMapsLoader.js new file mode 100644 index 00000000..99289d1e --- /dev/null +++ b/ui/src/util/azureMapsLoader.js @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +const MAP_CONTROL_CSS = + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.css"; +const MAP_CONTROL_JS = + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.js"; +const DRAWING_CSS = + "https://atlas.microsoft.com/sdk/javascript/drawing/1/atlas-drawing.min.css"; +const DRAWING_JS = + "https://atlas.microsoft.com/sdk/javascript/drawing/1/atlas-drawing.min.js"; +const SWIPE_JS = "/assets/js/azure-maps-swipe-map.min.js"; + +let loadPromise = null; + +function loadElement(documentRef, selector, createElement) { + const existing = documentRef.querySelector(selector); + if (existing?.dataset.loaded === "true") return Promise.resolve(); + + const element = existing || createElement(); + return new Promise((resolve, reject) => { + element.addEventListener( + "load", + () => { + element.dataset.loaded = "true"; + resolve(); + }, + { once: true } + ); + element.addEventListener( + "error", + () => { + element.remove(); + reject(new Error(`Unable to load Azure Maps asset: ${element.src || element.href}`)); + }, + { once: true } + ); + if (!existing) documentRef.head.appendChild(element); + }); +} + +function loadStylesheet(documentRef, href) { + return loadElement( + documentRef, + `link[data-azure-maps-href="${href}"]`, + () => { + const link = documentRef.createElement("link"); + link.rel = "stylesheet"; + link.href = href; + link.dataset.azureMapsHref = href; + return link; + } + ); +} + +function loadScript(documentRef, src) { + return loadElement( + documentRef, + `script[data-azure-maps-src="${src}"]`, + () => { + const script = documentRef.createElement("script"); + script.src = src; + script.async = true; + script.dataset.azureMapsSrc = src; + return script; + } + ); +} + +export function loadAzureMaps(documentRef = document) { + if (loadPromise) return loadPromise; + + loadPromise = Promise.all([ + loadStylesheet(documentRef, MAP_CONTROL_CSS), + loadStylesheet(documentRef, DRAWING_CSS), + ]) + .then(() => loadScript(documentRef, MAP_CONTROL_JS)) + .then(() => loadScript(documentRef, DRAWING_JS)) + .then(() => loadScript(documentRef, SWIPE_JS)) + .catch((error) => { + loadPromise = null; + throw error; + }); + return loadPromise; +} + +export function resetAzureMapsLoaderForTests() { + loadPromise = null; +} \ No newline at end of file diff --git a/ui/src/util/azureMapsLoader.test.js b/ui/src/util/azureMapsLoader.test.js new file mode 100644 index 00000000..6992d473 --- /dev/null +++ b/ui/src/util/azureMapsLoader.test.js @@ -0,0 +1,164 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + loadAzureMaps, + resetAzureMapsLoaderForTests, +} from "./azureMapsLoader.js"; + +function fakeDocument({ failOnce = null } = {}) { + const elements = []; + const attempts = new Map(); + + function asset(element) { + return element.src || element.href; + } + + return { + elements, + head: { + appendChild(element) { + elements.push(element); + const name = asset(element); + attempts.set(name, (attempts.get(name) || 0) + 1); + queueMicrotask(() => { + const event = name === failOnce && attempts.get(name) === 1 + ? "error" + : "load"; + element.listeners.get(event)?.(); + }); + }, + }, + createElement(tagName) { + return { + tagName, + dataset: {}, + listeners: new Map(), + addEventListener(name, callback) { + this.listeners.set(name, callback); + }, + remove() { + const index = elements.indexOf(this); + if (index >= 0) elements.splice(index, 1); + }, + }; + }, + querySelector(selector) { + const match = selector.match(/data-azure-maps-(?:src|href)="(.+)"/); + return elements.find((element) => asset(element) === match?.[1]) || null; + }, + }; +} + +test.beforeEach(() => resetAzureMapsLoaderForTests()); + +test("loads styles, map control, drawing tools, and swipe in order", async () => { + const documentRef = fakeDocument(); + + await loadAzureMaps(documentRef); + + assert.deepEqual( + documentRef.elements.map((element) => element.src || element.href), + [ + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.css", + "https://atlas.microsoft.com/sdk/javascript/drawing/1/atlas-drawing.min.css", + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.js", + "https://atlas.microsoft.com/sdk/javascript/drawing/1/atlas-drawing.min.js", + "/assets/js/azure-maps-swipe-map.min.js", + ] + ); +}); + +test("deduplicates concurrent and completed loads", async () => { + const documentRef = fakeDocument(); + + const first = loadAzureMaps(documentRef); + const second = loadAzureMaps(documentRef); + assert.equal(first, second); + await first; + await loadAzureMaps(documentRef); + + assert.equal(documentRef.elements.length, 5); +}); + +test("reuses an existing loaded asset", async () => { + const documentRef = fakeDocument(); + const existing = documentRef.createElement("link"); + existing.href = + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.css"; + existing.dataset.azureMapsHref = existing.href; + existing.dataset.loaded = "true"; + documentRef.elements.push(existing); + + await loadAzureMaps(documentRef); + + assert.equal( + documentRef.elements.filter((element) => element.href === existing.href) + .length, + 1 + ); +}); + +test("waits for an existing asset that is still loading", async () => { + const documentRef = fakeDocument(); + const existing = documentRef.createElement("link"); + existing.href = + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.css"; + existing.dataset.azureMapsHref = existing.href; + documentRef.elements.push(existing); + + const loading = loadAzureMaps(documentRef); + queueMicrotask(() => existing.listeners.get("load")?.()); + await loading; + + assert.equal( + documentRef.elements.filter((element) => element.href === existing.href) + .length, + 1 + ); +}); + +test("removes a failed asset and allows retry", async () => { + const failedAsset = + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.js"; + const documentRef = fakeDocument({ failOnce: failedAsset }); + + await assert.rejects(loadAzureMaps(documentRef), /Unable to load/); + assert.equal( + documentRef.elements.some((element) => element.src === failedAsset), + false + ); + + await loadAzureMaps(documentRef); + assert.equal( + documentRef.elements.filter((element) => element.src === failedAsset).length, + 1 + ); +}); + +test("reports a failed stylesheet URL", async () => { + const failedAsset = + "https://atlas.microsoft.com/sdk/javascript/mapcontrol/3/atlas.min.css"; + const documentRef = fakeDocument({ failOnce: failedAsset }); + + await assert.rejects(loadAzureMaps(documentRef), (error) => { + assert.equal( + error.message, + `Unable to load Azure Maps asset: ${failedAsset}` + ); + return true; + }); +}); + +test("uses the global document by default", async () => { + const documentRef = fakeDocument(); + globalThis.document = documentRef; + + try { + await loadAzureMaps(); + } finally { + delete globalThis.document; + } + + assert.equal(documentRef.elements.length, 5); +}); \ No newline at end of file diff --git a/ui/src/util/http.js b/ui/src/util/http.js new file mode 100644 index 00000000..5ba7c917 --- /dev/null +++ b/ui/src/util/http.js @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +export async function fetchJsonResponse(url, options = {}, fetchImpl = fetch) { + const response = await fetchImpl(url, options); + const etag = response.headers?.get?.("etag") ?? null; + + if (response.status === 304) { + return { data: null, etag, status: response.status }; + } + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const data = response.status === 204 ? null : await response.json(); + return { data, etag, status: response.status }; +} \ No newline at end of file diff --git a/ui/src/util/http.test.js b/ui/src/util/http.test.js new file mode 100644 index 00000000..27cd4056 --- /dev/null +++ b/ui/src/util/http.test.js @@ -0,0 +1,78 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { fetchJsonResponse } from "./http.js"; + +function response({ status = 200, data = null, etag = null } = {}) { + return { + status, + ok: status >= 200 && status < 300, + headers: { get: (name) => (name === "etag" ? etag : null) }, + json: async () => data, + }; +} + +test("returns parsed JSON and ETag", async () => { + const fetchImpl = async () => + response({ data: { projectId: "project-1" }, etag: '"etag"' }); + + const result = await fetchJsonResponse("/project", {}, fetchImpl); + + assert.deepEqual(result, { + data: { projectId: "project-1" }, + etag: '"etag"', + status: 200, + }); +}); + +test("returns an empty successful result for 304", async () => { + let jsonCalled = false; + const fetchImpl = async () => ({ + ...response({ status: 304, etag: '"etag"' }), + json: async () => { + jsonCalled = true; + }, + }); + + const result = await fetchJsonResponse("/project", {}, fetchImpl); + + assert.deepEqual(result, { data: null, etag: '"etag"', status: 304 }); + assert.equal(jsonCalled, false); +}); + +test("returns null for a successful empty response", async () => { + const result = await fetchJsonResponse( + "/project", + {}, + async () => response({ status: 204 }) + ); + + assert.deepEqual(result, { data: null, etag: null, status: 204 }); +}); + +test("rejects unsuccessful responses", async () => { + await assert.rejects( + fetchJsonResponse( + "/project", + {}, + async () => response({ status: 503 }) + ), + /status: 503/ + ); +}); + +test("passes request options to fetch", async () => { + const controller = new AbortController(); + const options = { + signal: controller.signal, + headers: { "If-None-Match": '"etag"' }, + }; + let receivedOptions; + + await fetchJsonResponse("/project", options, async (_url, received) => { + receivedOptions = received; + return response({ data: {} }); + }); + + assert.equal(receivedOptions, options); +}); \ No newline at end of file diff --git a/ui/src/util/jobNotifications.js b/ui/src/util/jobNotifications.js index 29050361..50e218eb 100644 --- a/ui/src/util/jobNotifications.js +++ b/ui/src/util/jobNotifications.js @@ -1,7 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -const TERMINAL_STATUSES = new Set(["Processed", "Failed", "Cancelled"]); +const TERMINAL_STATUSES = new Set([ + "Processed", + "Trained", + "Failed", + "Cancelled", +]); export function collectProjectJobStates(project) { const jobs = new Map(); @@ -54,4 +59,11 @@ export function findJobStatusTransitions(previousJobs, currentJobs) { } return transitions; +} + +export function hasActiveProjectJobs(jobs) { + if (!jobs) return false; + return [...jobs.values()].some( + (job) => job.status && !TERMINAL_STATUSES.has(job.status) + ); } \ No newline at end of file diff --git a/ui/src/util/jobNotifications.test.js b/ui/src/util/jobNotifications.test.js index 555075c9..73be59eb 100644 --- a/ui/src/util/jobNotifications.test.js +++ b/ui/src/util/jobNotifications.test.js @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { collectProjectJobStates, findJobStatusTransitions, + hasActiveProjectJobs, } from "./jobNotifications.js"; function projectWithEmbeddingStatus(status) { @@ -95,4 +96,68 @@ test("reports terminal failures but ignores intermediate updates", () => { subject: "Fire model", }, ]); +}); + +test("detects whether project jobs still need polling", () => { + assert.equal( + hasActiveProjectJobs( + new Map([ + ["done", { status: "Processed" }], + ["active", { status: "InProgress" }], + ]) + ), + true + ); + assert.equal( + hasActiveProjectJobs( + new Map([ + ["done", { status: "Processed" }], + ["trained", { status: "Trained" }], + ["failed", { status: "Failed" }], + ["unknown", { status: null }], + ]) + ), + false + ); + assert.equal(hasActiveProjectJobs(null), false); +}); + +test("collecting jobs ignores malformed records and uses fallback names", () => { + assert.deepEqual([...collectProjectJobStates(null)], []); + assert.deepEqual([...collectProjectJobStates({})], []); + + const jobs = collectProjectJobStates({ + imageLayer: [ + { name: "Missing id", models: [] }, + { imageLayerId: "layer-without-models", status: "Processed" }, + { + imageLayerId: "layer-1", + models: [ + { status: "Queued" }, + { modelId: "model-1", status: "Queued" }, + { + modelId: "embedding-1", + modelType: "embedding", + status: "Queued", + inferenceStatus: "InProgress", + }, + ], + }, + ], + }); + + assert.deepEqual(jobs.get("imagery:layer-1").subject, "Image layer"); + assert.deepEqual(jobs.get("training:model-1").subject, "Model"); + assert.deepEqual(jobs.get("embedding:embedding-1").subject, "Embedding"); + assert.deepEqual(jobs.get("inference:embedding-1").subject, "Model"); +}); + +test("does not report unchanged or nonterminal job states", () => { + const previous = new Map([ + ["same", { status: "Processed" }], + ["active", { status: "Queued" }], + ]); + const current = new Map(previous); + + assert.deepEqual(findJobStatusTransitions(previous, current), []); }); \ No newline at end of file diff --git a/ui/src/util/singleFlight.js b/ui/src/util/singleFlight.js new file mode 100644 index 00000000..25a9e0f8 --- /dev/null +++ b/ui/src/util/singleFlight.js @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +export function createSingleFlight() { + let active = null; + + return { + run(key, task) { + if (active?.key === key) return active.promise; + active?.controller.abort(); + + const controller = new AbortController(); + const entry = { controller, key, promise: null }; + entry.promise = Promise.resolve() + .then(() => task(controller.signal)) + .finally(() => { + if (active === entry) active = null; + }); + active = entry; + return entry.promise; + }, + + abort() { + active?.controller.abort(); + active = null; + }, + + isRunning(key) { + return active !== null && (key === undefined || active.key === key); + }, + }; +} \ No newline at end of file diff --git a/ui/src/util/singleFlight.test.js b/ui/src/util/singleFlight.test.js new file mode 100644 index 00000000..10fb5b6b --- /dev/null +++ b/ui/src/util/singleFlight.test.js @@ -0,0 +1,98 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { createSingleFlight } from "./singleFlight.js"; + +test("deduplicates concurrent work for the same key", async () => { + const flight = createSingleFlight(); + let calls = 0; + let release; + const pending = new Promise((resolve) => { + release = resolve; + }); + const task = async () => { + calls += 1; + await pending; + return "value"; + }; + + const first = flight.run("project-1", task); + const second = flight.run("project-1", task); + release(); + + assert.equal(first, second); + assert.equal(await first, "value"); + assert.equal(calls, 1); + assert.equal(flight.isRunning(), false); +}); + +test("starting a different key aborts the previous task", async () => { + const flight = createSingleFlight(); + let firstSignal; + const first = flight.run("project-1", async (signal) => { + firstSignal = signal; + await new Promise((resolve) => signal.addEventListener("abort", resolve)); + return "aborted"; + }); + await Promise.resolve(); + + const second = flight.run("project-2", async () => "current"); + + assert.equal(firstSignal.aborted, true); + assert.equal(await first, "aborted"); + assert.equal(await second, "current"); +}); + +test("failed work clears the flight so it can be retried", async () => { + const flight = createSingleFlight(); + + await assert.rejects( + flight.run("project-1", async () => { + throw new Error("failed"); + }), + /failed/ + ); + assert.equal(flight.isRunning("project-1"), false); + assert.equal( + await flight.run("project-1", async () => "recovered"), + "recovered" + ); +}); + +test("abort signals and clears active work", async () => { + const flight = createSingleFlight(); + let signal; + const pending = flight.run("project-1", async (currentSignal) => { + signal = currentSignal; + await new Promise((resolve) => + currentSignal.addEventListener("abort", resolve) + ); + }); + await Promise.resolve(); + + flight.abort(); + + assert.equal(signal.aborted, true); + assert.equal(flight.isRunning(), false); + await pending; +}); + +test("reports running state by key and tolerates idle abort", async () => { + const flight = createSingleFlight(); + let release; + const pending = flight.run( + "project-1", + () => new Promise((resolve) => { + release = resolve; + }) + ); + await Promise.resolve(); + + assert.equal(flight.isRunning(), true); + assert.equal(flight.isRunning("project-1"), true); + assert.equal(flight.isRunning("project-2"), false); + release(); + await pending; + flight.abort(); + assert.equal(flight.isRunning(), false); +}); \ No newline at end of file