From 9326c3b44483c63b0eb6e63d98434423964289a9 Mon Sep 17 00:00:00 2001 From: Caleb Robinson Date: Mon, 31 Aug 2026 22:14:31 +0000 Subject: [PATCH 1/2] feat: tile building footprints once per image layer Footprint geometry belongs to the image layer: every model trained on a layer draws the same buildings. The embedding job tiled them anyway, once per model, and standard layers got no tiles at all. Imagery prep now queues a tiling job as soon as a layer's footprints are cached, for both workflow types, and the archive lands on ImageLayer.footprintPmtilesUrl. The standard labeling workflow does not read it yet; this only produces the artifact. The embedding job stops tiling. tippecanoe and subprocess drop out of embed_buildings entirely and it emits the embeddings GeoJSON, the HFTR feature sidecar and a manifest. The feature sidecar is genuinely per model and is unchanged. The two archives were the same file. Both ran tippecanoe with the same layer name, the same --use-attribute-for-id=id, the same -y id -y overture_id and the same zoom range, over the same rows in the same order (embed_buildings reads the layer footprints, reset_index, and keeps every row including the ones with no valid tokens, which is what the positional prediction join already depends on). So an embedding layer was paying for a duplicate. tippecanoe ships only in the training image, and it is not in the apt repos of the imageryprep base image, so tiling runs as a queued task in the training container through the existing UnifiedRunner rather than inline in imagery prep. The queue is named footprint-tiles-queue rather than for any one consumer, since it tiles layers for every workflow. With one archive, Model.pmtilesUrl and ArtifactTypes.BUILDING_PMTILES retire, GetModelArtifact serves footprint_pmtiles as a layer-scoped kind instead of a per-model one, and the Interactive Labeler reads that. Existing layers have no archive and existing embedding models point at a per-model one that nothing writes any more. Re-tiling them is out of scope; the queued job means it can be done later by enqueuing one message per layer, without re-running imagery prep. The labeler used to refuse to start when a model had no archive. That guard goes with the field, so a failed tile load now raises a clear error rather than warning to the console and leaving an empty map with no buildings to label. --- api/hastefuncapi/function_app.py | 70 ++- api/hastefuncqueues/function_app.py | 99 +++- docker/data-init/upload_data.py | 1 + docker/docker-compose.yml | 2 + docker/training/Dockerfile | 10 +- hastelib/src/hastegeo/core/config.py | 28 +- hastelib/src/hastegeo/core/models/projects.py | 16 +- .../src/hastegeo/core/processors/embedding.py | 10 - .../core/processors/footprint_tiles.py | 517 ++++++++++++++++++ .../src/hastegeo/core/processors/imagery.py | 62 +++ .../src/hastegeo/workflows/embed_buildings.py | 77 +-- .../workflows/prepare_footprint_tiles.py | 480 ++++++++++++++++ .../core/processors/test_footprint_tiles.py | 141 +++++ .../test_imagery_footprint_tiles.py | 105 ++++ .../workflows/test_prepare_footprint_tiles.py | 148 +++++ local.settings.example.jsonc | 1 + .../InteractiveLabeler/InteractiveLabeler.jsx | 27 +- 17 files changed, 1674 insertions(+), 120 deletions(-) create mode 100644 hastelib/src/hastegeo/core/processors/footprint_tiles.py create mode 100644 hastelib/src/hastegeo/workflows/prepare_footprint_tiles.py create mode 100644 hastelib/tests/core/processors/test_footprint_tiles.py create mode 100644 hastelib/tests/core/processors/test_imagery_footprint_tiles.py create mode 100644 hastelib/tests/workflows/test_prepare_footprint_tiles.py diff --git a/api/hastefuncapi/function_app.py b/api/hastefuncapi/function_app.py index 707cd6c3..19201ef3 100644 --- a/api/hastefuncapi/function_app.py +++ b/api/hastefuncapi/function_app.py @@ -1367,16 +1367,21 @@ async def GetLayerModelsDetails(req: func.HttpRequest) -> func.HttpResponse: return func.HttpResponse("Error loading models.", status_code=500) -# Embedding-model artifacts the Interactive Labeler fetches by HTTP byte -# range, mapped to the Model field that holds each blob URL. +# Model artifacts the browser fetches by HTTP byte range, mapped to the +# Model field that holds each blob URL. _MODEL_ARTIFACT_URL_FIELDS = { - "pmtiles": "pmtilesUrl", "sidecar": "featuresSidecarUrl", "geojson": "embeddingsGeoJSONUrl", "gpkg": "gpkgUrl", } +# Artifacts owned by the ImageLayer rather than a model. Footprint +# geometry is shared by every model trained on the layer, so it lives +# here and is looked up by imageLayerId. +_LAYER_ARTIFACT_URL_FIELDS = { + "footprint_pmtiles": "footprintPmtilesUrl", +} _MODEL_ARTIFACT_CONTENT_TYPES = { - "pmtiles": "application/octet-stream", + "footprint_pmtiles": "application/vnd.pmtiles", "sidecar": "application/octet-stream", "geojson": "application/geo+json", "gpkg": "application/geopackage+sqlite3", @@ -1400,11 +1405,16 @@ async def GetModelArtifact(req: func.HttpRequest) -> func.HttpResponse: blob I/O server-side over the Azure backbone, honoring ``Range`` so pmtiles.js can do partial reads. - Supported ``kind`` values: ``pmtiles``, ``sidecar`` and ``geojson`` - (fetched/parsed in-browser), plus ``gpkg`` — the per-building - predictions GeoPackage saved by ``PutBuildingPredictions``, served as - a downloadable attachment. Example: + Supported ``kind`` values: ``sidecar`` and ``geojson`` + (fetched/parsed in-browser), ``gpkg`` — the per-building predictions + GeoPackage saved by ``PutBuildingPredictions``, served as a + downloadable attachment — and ``footprint_pmtiles``, the image + layer's shared building-footprint vector tiles. Example: ``GET /api/GetModelArtifact?projectId=&modelId=&kind=gpkg``. + + ``footprint_pmtiles`` is layer-scoped: geometry is shared by every + model trained on the layer, so it resolves against ``imageLayerId`` + (taken from the query string, else from the model's own layer). """ try: project_id = _require_guid_param(req, "projectId") @@ -1414,13 +1424,15 @@ async def GetModelArtifact(req: func.HttpRequest) -> func.HttpResponse: kind = (req.params.get("kind") or "").lower() url_field = _MODEL_ARTIFACT_URL_FIELDS.get(kind) - if url_field is None: + layer_url_field = _LAYER_ARTIFACT_URL_FIELDS.get(kind) + if url_field is None and layer_url_field is None: return _bad_request( - f"kind must be one of {sorted(_MODEL_ARTIFACT_URL_FIELDS)}" + "kind must be one of " + f"{sorted({**_MODEL_ARTIFACT_URL_FIELDS, **_LAYER_ARTIFACT_URL_FIELDS})}" ) try: - model = await asyncio.to_thread( + document = await asyncio.to_thread( MetadataProcessor( data_type=config.get_metadata_types().MODEL.value, partition_key=project_id, @@ -1436,11 +1448,39 @@ async def GetModelArtifact(req: func.HttpRequest) -> func.HttpResponse: ) return func.HttpResponse("Error loading model.", status_code=500) - blob_url = (model or {}).get(url_field) or "" + if layer_url_field is not None: + # Layer-scoped artifact: take an explicit imageLayerId when the + # caller passes one, otherwise the model's own layer — a client + # holding a modelId always means that model's footprints. + url_field = layer_url_field + model_document = document or {} + image_layer_id = req.params.get("imageLayerId") or model_document.get( + "imageLayerId" + ) + if not image_layer_id or not _GUID_RE.match(str(image_layer_id)): + return _bad_request("Invalid or missing parameter: imageLayerId") + try: + document = await asyncio.to_thread( + MetadataProcessor( + data_type=config.get_metadata_types().IMAGELAYER.value, + partition_key=project_id, + ).load, + image_layer_id, + ) + except FileNotFoundError: + return func.HttpResponse("Image layer not found.", status_code=404) + except Exception as e: + logger.error( + f"GetModelArtifact image layer load failed: {e}\n" + f"{traceback.format_exc()}" + ) + return func.HttpResponse( + "Error loading image layer.", status_code=500 + ) + + blob_url = (document or {}).get(url_field) or "" if not blob_url: - return func.HttpResponse( - "Artifact not available for this model.", status_code=404 - ) + return func.HttpResponse("Artifact not available.", status_code=404) try: offset, length, is_range = parse_byte_range(req.headers.get("Range")) diff --git a/api/hastefuncqueues/function_app.py b/api/hastefuncqueues/function_app.py index 07cef137..26c4fb17 100644 --- a/api/hastefuncqueues/function_app.py +++ b/api/hastefuncqueues/function_app.py @@ -20,6 +20,7 @@ from hastegeo.core.models.training import ExperimentConfig from hastegeo.core.processors.artifacts import ArtifactProcessor from hastegeo.core.processors.embedding import EmbeddingPostprocessor +from hastegeo.core.processors.footprint_tiles import FootprintTilesPreprocessor from hastegeo.core.processors.imagery import ImageryPostProcessor from hastegeo.core.processors.inference import ( InferencePostprocessor, @@ -601,6 +602,100 @@ async def GetRunEmbeddingQueueMessage(msg: func.QueueMessage) -> None: ) +@app.function_name(name="GetPrepareFootprintTilesQueueTrigger") +@app.queue_trigger( + arg_name="msg", + queue_name=config.get_queue_config()["footprint_tiles_queue_name"], + connection="AzureWebJobsStorage", +) +async def GetPrepareFootprintTilesQueueMessage( + msg: func.QueueMessage, +) -> None: + """Build an image layer's shared building-footprint vector tiles. + + Message schema (identifiers only):: + + {"projectId", "imageLayerId", "sourceFootprintsUrl", "force"} + + Imagery prep enqueues this as soon as a layer's footprints are cached, + so every map that draws those buildings finds the archive already + built. The authoritative state is read from metadata, so a fresh + request and the preprocessor's own poll messages take the same path. + The work runs as a task in the training docker image because + tippecanoe ships only there. + """ + logger.info( + "GetPrepareFootprintTilesQueueTrigger function processed a message: " + f'{msg.get_body().decode("utf-8")}' + ) + try: + payload = json.loads(msg.get_body().decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError("Queue message must be a JSON object") + project_id = payload.get("projectId") + image_layer_id = payload.get("imageLayerId") + if not project_id or not image_layer_id: + raise ValueError( + "Queue message requires projectId and imageLayerId, got: " + f"{sorted(payload.keys())}" + ) + + try: + layer_record = await asyncio.to_thread( + MetadataProcessor( + data_type=config.get_metadata_types().IMAGELAYER.value, + partition_key=project_id, + ).load, + image_layer_id, + ) + except FileNotFoundError: + layer_record = None + + if not layer_record: + logger.info( + f"Image layer {image_layer_id} not found, likely deleted, " + "skipping footprint tile preparation." + ) + return + + image_layer = ImageLayer(**layer_record) + if not image_layer.buildingFootprintsUrl: + logger.info( + f"Image layer {image_layer_id} has no cached building " + "footprints; nothing to tile." + ) + return + + output = await asyncio.to_thread( + FootprintTilesPreprocessor(image_layer).process + ) + + await asyncio.to_thread( + MetadataProcessor( + data_type=config.get_metadata_types().IMAGELAYER.value, + partition_key=project_id, + ).save, + image_layer_id, + output.dict(), + ) + except ValidationError as e: + logger.error( + f"GetPrepareFootprintTilesQueueTrigger: Validation error: {e}\n" + f"{traceback.format_exc()}" + ) + except ValueError as e: + logger.error( + "GetPrepareFootprintTilesQueueTrigger: Invalid queue message: " + f"{e}\n{traceback.format_exc()}" + ) + except Exception as e: + logger.error( + "GetPrepareFootprintTilesQueueTrigger: Error processing queue " + f"message: {e}\n{traceback.format_exc()}", + stack_info=True, + ) + + @app.function_name(name="GetRunInferenceQueueTrigger") @app.queue_trigger( arg_name="msg", @@ -1013,7 +1108,9 @@ async def GetPublishDatasetQueueMessage(msg: func.QueueMessage) -> None: message = PublishQueueMessage( **json.loads(msg.get_body().decode("utf-8")) ) - await asyncio.to_thread(PublishingProcessor(config=config).run_step, message) + await asyncio.to_thread( + PublishingProcessor(config=config).run_step, message + ) except Exception as error: logger.error( "PublishDatasetQueueTrigger failed with %s", diff --git a/docker/data-init/upload_data.py b/docker/data-init/upload_data.py index 76cbc26a..8d604a24 100644 --- a/docker/data-init/upload_data.py +++ b/docker/data-init/upload_data.py @@ -46,6 +46,7 @@ def create_queues(): "local-zip-queue", "local-inference-queue", "local-embedding-queue", + "local-footprint-tiles-queue", "local-image-queue-poison", "local-embedding-queue-poison" ] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 923dce16..d738b177 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -106,6 +106,7 @@ services: ZIP_QUEUE_NAME: "local-zip-queue" INFERENCE_QUEUE_NAME: "local-inference-queue" EMBEDDING_QUEUE_NAME: "local-embedding-queue" + FOOTPRINT_TILES_QUEUE_NAME: "local-footprint-tiles-queue" PUBLISH_QUEUE_NAME: "local-publish-queue" PUBLISHING_ENABLED: "true" PC_PROVIDER_ENABLED: "false" @@ -154,6 +155,7 @@ services: ZIP_QUEUE_NAME: "local-zip-queue" INFERENCE_QUEUE_NAME: "local-inference-queue" EMBEDDING_QUEUE_NAME: "local-embedding-queue" + FOOTPRINT_TILES_QUEUE_NAME: "local-footprint-tiles-queue" PUBLISH_QUEUE_NAME: "local-publish-queue" PUBLISHING_ENABLED: "true" PC_PROVIDER_ENABLED: "false" diff --git a/docker/training/Dockerfile b/docker/training/Dockerfile index fdddc408..c06d5593 100644 --- a/docker/training/Dockerfile +++ b/docker/training/Dockerfile @@ -48,10 +48,14 @@ RUN for i in 1 2 3 4 5; do \ && update-ca-certificates \ && rm -rf /var/lib/apt/lists/* -# CLI shim for the building-embedding workflow (mirrors imageryprep's -# prepare-imagery shim). `python` resolves to the conda env at runtime via PATH. +# CLI shims for the workflows that run in this image (mirroring +# imageryprep's prepare-imagery shim). `python` resolves to the conda env +# at runtime via PATH. prepare-footprint-tiles lives here because +# tippecanoe ships only in this image. RUN printf '#!/bin/bash\nexec python -m hastegeo.workflows.embed_buildings "$@"\n' > /usr/local/bin/embed-buildings \ - && chmod +x /usr/local/bin/embed-buildings + && chmod +x /usr/local/bin/embed-buildings \ + && printf '#!/bin/bash\nexec python -m hastegeo.workflows.prepare_footprint_tiles "$@"\n' > /usr/local/bin/prepare-footprint-tiles \ + && chmod +x /usr/local/bin/prepare-footprint-tiles # Prepare conda env directory owned by the target user, then extract the packed env # as that user — non-root tar uses the running user's UID/GID, so no chown -R needed. diff --git a/hastelib/src/hastegeo/core/config.py b/hastelib/src/hastegeo/core/config.py index fc4472f0..b4798a76 100644 --- a/hastelib/src/hastegeo/core/config.py +++ b/hastelib/src/hastegeo/core/config.py @@ -100,6 +100,10 @@ class ArtifactTypes(Enum): - BUILDING_FOOTPRINTS: Cached Overture Maps building footprints, scoped to the image layer's AOI. Generated during imageryprep so the inference workflow can reuse the same set across multiple model runs. + - LAYER_FOOTPRINT_PMTILES: Geometry-only vector tiles of an image + layer's cached building footprints, carrying the row-index ``id`` + and ``overture_id``. Built once per layer and shared by every + model trained on it, so nothing model-specific belongs in it. - VALID_AREA_MASK: GeoJSON FeatureCollection of the valid-data polygon derived from the post-event mosaic — i.e. the imagery's actual AOI excluding nodata. Same polygon used to bbox-filter Overture; @@ -137,6 +141,7 @@ class ArtifactTypes(Enum): BUILDING_FOOTPRINTS = Template( "building_footprints_${projectId}_${imageLayerId}" ) + LAYER_FOOTPRINT_PMTILES = Template("footprints_${imageLayerId}") VALID_AREA_MASK = Template("valid_area_mask_${projectId}_${imageLayerId}") INFERENCE_GPKG = Template("predicted_damage_${modelName}") VISUALIZER = Template( @@ -146,11 +151,12 @@ class ArtifactTypes(Enum): TRAINING_ARTIFACTS_ZIP = Template("training_artifacts_${modelName}") INFERENCE_ARTIFACTS_ZIP = Template("inference_artifacts_${modelName}") # Building labeling workflow: per-building MOSAIKS / DINOv2 embeddings - # (footprints + f_* feature columns), the matching PMTiles vector tiles - # (geometry + id only), the binary HFTR sidecar (id -> feature vector), - # and the per-building predictions written by the interactive labeler. + # (footprints + f_* feature columns), the binary HFTR sidecar + # (id -> feature vector), and the per-building predictions written by + # the interactive labeler. The footprint vector tiles are NOT here: + # they belong to the image layer (LAYER_FOOTPRINT_PMTILES), shared by + # every model trained on it. BUILDING_EMBEDDINGS = Template("building_embeddings_${modelName}") - BUILDING_PMTILES = Template("building_pmtiles_${modelName}") BUILDING_FEATURES_SIDECAR = Template("building_features_${modelName}") BUILDING_PREDICTIONS_GPKG = Template("building_predictions_${modelName}") @@ -321,18 +327,19 @@ def get_queue_config(): "publish_queue_name": os.getenv( "PUBLISH_QUEUE_NAME", "publish-queue" ), + # Layer footprint vector tiles. Runs in the training container + # because tippecanoe ships only in that image. + "footprint_tiles_queue_name": os.getenv( + "FOOTPRINT_TILES_QUEUE_NAME", "footprint-tiles-queue" + ), } @staticmethod def get_publishing_config(): """Get publishing feature and provider configuration.""" return { - "publishing_enabled": _get_bool_env( - "PUBLISHING_ENABLED", True - ), - "pc_provider_enabled": _get_bool_env( - "PC_PROVIDER_ENABLED", False - ), + "publishing_enabled": _get_bool_env("PUBLISHING_ENABLED", True), + "pc_provider_enabled": _get_bool_env("PC_PROVIDER_ENABLED", False), "max_total_bytes": _get_bounded_int_env( "PUBLISH_MAX_TOTAL_BYTES", 5 * 1024**3, 1 ), @@ -405,6 +412,7 @@ class DataTypes(Enum): EXPERIMENT_CONFIG = "experiment_config" IMAGERY_CONFIG = "imageryprep_config" EMBEDDING_CONFIG = "embedding_config" + FOOTPRINT_TILES_CONFIG = "footprint_tiles_config" PROCESSED_IMAGERY = "processed_imagery_post_event_cog" RAW_IMAGERY = "raw_imagery" PREVIEW_RAW_IMAGERY = "preview_raw_imagery" diff --git a/hastelib/src/hastegeo/core/models/projects.py b/hastelib/src/hastegeo/core/models/projects.py index afdb0ee7..6c10adc2 100644 --- a/hastelib/src/hastegeo/core/models/projects.py +++ b/hastelib/src/hastegeo/core/models/projects.py @@ -455,7 +455,6 @@ class Model(BaseModel): numFeatures: Optional[int] = Field(default=None) embeddingJob: Optional[TrainingJob] = Field(default=None) embeddingsGeoJSONUrl: Optional[str] = Field(default=None) - pmtilesUrl: Optional[str] = Field(default=None) # Binary sidecar (HFTR format) carrying per-building f_* feature # vectors keyed by row-index id. The Interactive Labeler fetches it # once at session start and looks vectors up by id; the PMTiles @@ -690,6 +689,15 @@ class ImageLayer(BaseModel): AOI, and writes the result to ``buildingFootprintsUrl``. The URL must satisfy ``validate_footprint_url`` (same allowlist as imagery URLs plus the configured local upload host). + footprintPmtilesUrl: URL to the PMTiles archive of this layer's + cached building footprints (geometry plus the row-index ``id`` + and ``overture_id``). Built once per layer and shared by every + model trained on it. Populated by the footprint-tiles job, + which imagery prep kicks off as soon as the footprints are + cached. + footprintTilesJob: Job reference for that tiling task. + footprintTilesStatus: Status of the tiling job. + footprintTilesStatusMessage: Progress/error log of the tiling job. validAreaMaskUrl: URL to a single-feature GeoJSON FeatureCollection containing the valid-data polygon (EPSG:4326) derived from the post-event mosaic, i.e. the imagery's actual AOI excluding @@ -772,6 +780,12 @@ class ImageLayer(BaseModel): # Catalog "clip to area" flow. clipBbox: Optional[list[float]] = Field(default=None) validAreaMaskUrl: Optional[str] = Field(default=None) + # Shared footprint vector tiles. Geometry belongs to the layer, not to + # any one model, so every model trained here draws the same archive. + footprintPmtilesUrl: Optional[str] = Field(default=None) + footprintTilesJob: Optional[TrainingJob] = Field(default=None) + footprintTilesStatus: Optional[str] = Field(default=None) + footprintTilesStatusMessage: Optional[str] = Field(default="") dependsOn: Optional[tuple[str, str]] = Field( default=("Project", "projectId") ) diff --git a/hastelib/src/hastegeo/core/processors/embedding.py b/hastelib/src/hastegeo/core/processors/embedding.py index 9afc8bc0..f49b6f9e 100644 --- a/hastelib/src/hastegeo/core/processors/embedding.py +++ b/hastelib/src/hastegeo/core/processors/embedding.py @@ -235,12 +235,6 @@ def _create_embedding_config(self): ) + ".geojson" ) - pmtiles_name = ( - ArtifactTypes.BUILDING_PMTILES.value.substitute( - modelName=self.model_data.modelId - ) - + ".pmtiles" - ) sidecar_name = ( ArtifactTypes.BUILDING_FEATURES_SIDECAR.value.substitute( modelName=self.model_data.modelId @@ -260,7 +254,6 @@ def _create_embedding_config(self): "imagery": imagery_fn, "footprints": footprints_fn, "embeddings": embeddings_name, - "pmtiles": pmtiles_name, "sidecar": sidecar_name, }, "pipeline": { @@ -317,9 +310,6 @@ def _update_results_from_job(self): self.model_data.embeddingsGeoJSONUrl = self._artifact_url( manifest.get("embeddings_filename", "") ) - self.model_data.pmtilesUrl = self._artifact_url( - manifest.get("pmtiles_filename", "") - ) self.model_data.featuresSidecarUrl = self._artifact_url( manifest.get("sidecar_filename", "") ) diff --git a/hastelib/src/hastegeo/core/processors/footprint_tiles.py b/hastelib/src/hastegeo/core/processors/footprint_tiles.py new file mode 100644 index 00000000..4b62f1e4 --- /dev/null +++ b/hastelib/src/hastegeo/core/processors/footprint_tiles.py @@ -0,0 +1,517 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Build and track an image layer's shared footprint vector tiles. + +Every map that draws a layer's buildings needs the same PMTiles archive of +that layer's cached footprints. Geometry belongs to the layer, so the +archive is built once — when imagery prep finishes caching the footprints +— and shared by every model trained on it. + +``tippecanoe`` ships only in the training docker image, so the work cannot +run inline in an Azure Functions handler. This module is the seam: + +* :func:`enqueue_footprint_tiles` puts an identifiers-only message on the + footprint-tiles queue. Imagery prep calls it; nothing waits on it. +* :class:`FootprintTilesPreprocessor` is driven from the queue trigger and + walks one job through submit -> poll -> finalize, recording state on the + ``ImageLayer`` itself (``footprintTilesStatus`` / ``footprintTilesJob`` + / ``footprintTilesStatusMessage``) and the resulting archive on + ``ImageLayer.footprintPmtilesUrl``. + +The queue message carries identifiers only. Authoritative state is read +from metadata, so a fresh request and the preprocessor's own poll messages +take the same path and a duplicate message is a no-op rather than a second +container job. +""" + +from __future__ import annotations + +import json +import os +from typing import Any, Dict, List, Optional, Tuple + +from ..config import ArtifactTypes, Config +from ..data_layer.unified import UnifiedDataLayer +from ..models.projects import ImageLayer, TrainingJob +from ..runners.unified_runner import UnifiedRunner +from ..utils.data import extract_from_url +from ..utils.logs import Logger +from ..utils.metadata import MetadataUtils +from ..utils.queues import AzureQueueHandler + +# Do not prefix with '$'. Replaced at runtime with the task working dir. +BATCH_JOB_WORKDIR = "AZ_BATCH_TASK_WORKING_DIR" +FOOTPRINT_TILES_PREFIX = "ftl" +MANIFEST_FILENAME = "footprint_tiles_manifest.json" +FRIENDLY_LOG_FILENAME = "footprint_tiles_friendly.log" + + +def pmtiles_artifact_name(image_layer_id: str) -> str: + """Artifact name for a layer's footprint PMTiles archive.""" + return ( + ArtifactTypes.LAYER_FOOTPRINT_PMTILES.value.substitute( + imageLayerId=image_layer_id + ) + + ".pmtiles" + ) + + +def layer_needs_footprint_tiles(image_layer: ImageLayer) -> bool: + """Report whether a tiling job is worth queueing for this layer. + + Tiles can only be built once the footprints GeoPackage is cached, and + there is no point rebuilding an archive the layer already has. + """ + return bool(image_layer.buildingFootprintsUrl) and not bool( + image_layer.footprintPmtilesUrl + ) + + +def build_tiles_message( + project_id: str, + image_layer_id: str, + source_footprints_url: Optional[str] = None, + force: bool = False, +) -> Dict[str, Any]: + """Build a footprint-tiles queue message payload. + + Carries identifiers only; the trigger reads the authoritative state + from metadata so a re-queued poll message and a fresh request take + the same code path. + """ + return { + "projectId": project_id, + "imageLayerId": image_layer_id, + "sourceFootprintsUrl": source_footprints_url or "", + "force": bool(force), + } + + +def enqueue_footprint_tiles( + project_id: str, + image_layer_id: str, + source_footprints_url: Optional[str] = None, + force: bool = False, + config: Optional[Config] = None, +) -> Dict[str, Any]: + """Put a tiling request on the footprint-tiles queue. + + Convenience seam for imagery prep, which may not run ``tippecanoe`` + inline. Returns the enqueued message. + """ + if config is None: + config = Config() + message = build_tiles_message( + project_id=project_id, + image_layer_id=image_layer_id, + source_footprints_url=source_footprints_url, + force=force, + ) + queue_client = AzureQueueHandler( + config.queue_config["queue_connection_string"], + config.queue_config["footprint_tiles_queue_name"], + config.queue_config["queue_account_url"], + ) + queue_client.put_message(json.dumps(message), visibility_timeout=0) + return message + + +def request_preparation( + image_layer: ImageLayer, + force: bool = False, + config: Optional[Config] = None, +) -> Dict[str, Any]: + """Decide whether to enqueue a tiling job, and do it if so. + + Mutates ``image_layer`` in place when a job is queued; the caller + persists it. + + Returns: + ``{"imageLayerId", "queued", "tilesReady", "status", + "statusMessage"}``. + + Raises: + ValueError: when the layer has no cached building footprints, so + there is nothing to tile. + """ + if config is None: + config = Config() + statuses = config.get_status_types() + logger = Logger.get_logger(__name__) + + if not image_layer.buildingFootprintsUrl: + raise ValueError( + f"Image layer {image_layer.imageLayerId} has no cached " + "building footprints; footprint tiles cannot be built " + "without them." + ) + + needs_tiles = layer_needs_footprint_tiles(image_layer) + in_flight = image_layer.footprintTilesStatus in ( + statuses.PENDING.value, + statuses.IN_PROGRESS.value, + ) + + def _state(queued: bool) -> Dict[str, Any]: + return { + "imageLayerId": image_layer.imageLayerId, + "queued": queued, + "tilesReady": not layer_needs_footprint_tiles(image_layer), + "status": image_layer.footprintTilesStatus, + "statusMessage": image_layer.footprintTilesStatusMessage or "", + } + + if not force and not needs_tiles: + # The archive already exists: record the transition once rather + # than paying for a redundant container job. + if image_layer.footprintTilesStatus != statuses.COMPLETED.value: + image_layer.footprintTilesStatus = statuses.COMPLETED.value + image_layer.footprintTilesStatusMessage = ( + MetadataUtils.append_status_message( + image_layer.footprintTilesStatusMessage, + "Footprint tiles already available", + ) + ) + return _state(False) + + if not force and in_flight: + # A job is already queued or running for this layer. Re-queueing + # would submit a second task for the same archive. + logger.info( + "Footprint tiles for image layer %s already %s; not re-queueing", + image_layer.imageLayerId, + image_layer.footprintTilesStatus, + ) + return _state(False) + + image_layer.footprintTilesStatus = statuses.PENDING.value + image_layer.footprintTilesStatusMessage = ( + MetadataUtils.append_status_message( + "", "Queued for footprint tile preparation" + ) + ) + enqueue_footprint_tiles( + project_id=image_layer.projectId, + image_layer_id=image_layer.imageLayerId, + source_footprints_url=image_layer.buildingFootprintsUrl, + force=force, + config=config, + ) + logger.info( + "Queued footprint tiles for image layer %s (force=%s)", + image_layer.imageLayerId, + force, + ) + return _state(True) + + +class FootprintTilesPreprocessor: + """Submit, poll and finalize one layer's footprint tiling job. + + Mirrors the other container-job preprocessors: ``process()`` advances + the state machine by exactly one step and returns the image layer, + which the caller persists. + """ + + def __init__( + self, + image_layer: ImageLayer, + config: Optional[Config] = None, + ) -> None: + if config is None: + config = Config() + if image_layer is None: + raise ValueError( + "FootprintTilesPreprocessor requires an image layer." + ) + self.config = config + self.image_layer = image_layer + self.project_id = image_layer.projectId + self.storage = UnifiedDataLayer( + storage_type=config.storage_type, + partition_key=self.project_id, + **config.storage_config, + ) + self.logger = Logger.get_logger(__name__) + self.runner = UnifiedRunner( + runner_type=config.runner_type, + config=self.config, + pool_id=self.config.get_azure_batch_config()["training_pool_id"], + candidate_pool_ids=self.config.get_azure_batch_config()[ + "training_pool_ids" + ], + ) + self.queue_client = AzureQueueHandler( + config.queue_config["queue_connection_string"], + config.queue_config["footprint_tiles_queue_name"], + config.queue_config["queue_account_url"], + ) + + @property + def layer_id(self) -> str: + return self.image_layer.imageLayerId + + def _poll_message(self) -> str: + """Message that brings this job back for another status poll.""" + return json.dumps( + build_tiles_message( + project_id=self.project_id, + image_layer_id=self.layer_id, + source_footprints_url=(self.image_layer.buildingFootprintsUrl), + ) + ) + + def process(self) -> ImageLayer: + """Advance the job state machine by one step.""" + self.logger.info( + "FootprintTilesPreprocessor.process: image layer %s status %s", + self.layer_id, + self.image_layer.footprintTilesStatus, + ) + statuses = self.config.get_status_types() + status = self.image_layer.footprintTilesStatus + + if status == statuses.PENDING.value: + self._update_progress("Submitting footprint tile job") + self._execute_job() + + elif status == statuses.IN_PROGRESS.value: + job = self.image_layer.footprintTilesJob + if job is None: + self.image_layer.footprintTilesStatus = statuses.FAILED.value + self._update_progress( + "Footprint tile job reference is missing; cannot poll " + "for completion" + ) + return self.image_layer + + task_status = self.runner.get_task_status( + job_id=job.jobId, task_id=job.taskId + ) + self.logger.info( + "Task status for footprint tiles of %s is %s", + self.layer_id, + task_status, + ) + + if task_status == statuses.COMPLETED.value: + job.status = task_status + job.completedDate = MetadataUtils.get_timestamp() + try: + self._update_results_from_job() + self.image_layer.footprintTilesStatus = task_status + except Exception as error: + self.logger.error( + "Error finalizing footprint tiles for " + f"{self.layer_id}: {error}", + stack_info=True, + ) + self.image_layer.footprintTilesStatus = ( + statuses.FAILED.value + ) + job.status = statuses.FAILED.value + self._update_progress( + f"Footprint tile job failed: {error}" + ) + self._replay_friendly_logs() + self.runner.cleanup_task(job_id=job.jobId, task_id=job.taskId) + + elif task_status == statuses.FAILED.value: + self.image_layer.footprintTilesStatus = task_status + job.status = task_status + job.completedDate = MetadataUtils.get_timestamp() + self._replay_friendly_logs() + self._update_progress("Footprint tile job failed") + self.runner.cleanup_task(job_id=job.jobId, task_id=job.taskId) + else: + self.image_layer.footprintTilesStatus = task_status + job.status = task_status + self.queue_client.put_message(self._poll_message()) + + return self.image_layer + + def _execute_job(self) -> ImageLayer: + statuses = self.config.get_status_types() + try: + input_files = self._create_job_config() + config_path = input_files["config"]["file_path"] + command = ( + f'"mkdir -p ${BATCH_JOB_WORKDIR} ' + f"&& cd ${BATCH_JOB_WORKDIR} " + "&& python -m hastegeo.workflows.prepare_footprint_tiles " + f'--config ${BATCH_JOB_WORKDIR}/{config_path}"' + ) + job_id = self.config.get_azure_batch_config()[ + "training_batch_job_id" + ][:64] + task_id = f"{FOOTPRINT_TILES_PREFIX}-{MetadataUtils.generate_id()}" + output_prefix = ( + f"{MetadataUtils.hash_string(self.project_id)}/{task_id}" + ) + job_id, task_id = self.runner.add_task( + job_id=job_id, + task_id=task_id, + output_prefix=output_prefix, + resource_files_for_upload=input_files, + file_pattern=f"${BATCH_JOB_WORKDIR}/outputs/*.*", + command=command, + image_name=self.config.get_azure_batch_config()[ + "docker_image" + ], + ) + self.image_layer.footprintTilesJob = TrainingJob( + jobId=job_id, + taskId=task_id, + modelId=None, + projectId=self.project_id, + status=statuses.IN_PROGRESS.value, + creationDate=MetadataUtils.get_timestamp(), + ) + self.image_layer.footprintTilesStatus = statuses.IN_PROGRESS.value + self._update_progress( + f"Footprint tiles submitted with task id {task_id}" + ) + self.queue_client.put_message(self._poll_message()) + except Exception as error: + self.logger.error( + f"Error submitting footprint tiles for {self.layer_id}: " + f"{error}", + stack_info=True, + ) + self.image_layer.footprintTilesStatus = statuses.FAILED.value + self._update_progress(f"Footprint tile job failed: {error}") + return self.image_layer + + def _create_job_config(self) -> Dict[str, Dict[str, str]]: + """Write the workflow config and describe the task input files.""" + filename_pattern = ( + rf"{MetadataUtils.hash_string(self.project_id)}/(.*)\?+" + ) + plain_url_pattern = r"(.*)\?+" + + footprints_url = self.image_layer.buildingFootprintsUrl + if not footprints_url: + raise ValueError("Image layer has no building footprints.") + footprints_fn = ( + f"inputs/{extract_from_url(footprints_url, filename_pattern)}" + ) + + workflow_config: Dict[str, Any] = { + "project_id": self.project_id, + "image_layer_id": self.layer_id, + "output_dir": "outputs", + # Relative to the task working dir: the command cd's into + # $AZ_BATCH_TASK_WORKING_DIR before running the workflow. + "files": { + "footprints": footprints_fn, + "pmtiles": pmtiles_artifact_name(self.layer_id), + }, + "store_artifacts": True, + } + + config_type = ( + self.config.get_metadata_types().FOOTPRINT_TILES_CONFIG.value + ) + self.storage.save( + identifier=self.layer_id, + data=workflow_config, + data_type=config_type, + data_format="json", + ) + config_filepath = self.storage.get_file_remote_path( + self.layer_id, config_type, data_format="json" + ) + config_fn = ( + f"inputs/{extract_from_url(config_filepath, filename_pattern)}" + ) + + return { + "config": { + "http_url": extract_from_url( + config_filepath, plain_url_pattern + ), + "file_path": config_fn, + }, + "footprints": { + "http_url": extract_from_url( + footprints_url, plain_url_pattern + ), + "file_path": footprints_fn, + }, + } + + def _update_results_from_job(self) -> None: + """Persist the archive URL and count from the task manifest.""" + job = self.image_layer.footprintTilesJob + content = self.runner.get_filecontent_from_task( + job_id=job.jobId, + task_id=job.taskId, + filename=MANIFEST_FILENAME, + ) + if not content: + raise FileNotFoundError( + f"Footprint tiles manifest not found for {self.layer_id}" + ) + manifest = json.loads(content) + + pmtiles_url = manifest.get("pmtiles_url") or self._artifact_url( + manifest.get("pmtiles_filename", "") + ) + if not pmtiles_url: + raise ValueError( + "Footprint tiles manifest carries no PMTiles URL for image " + f"layer {self.layer_id}" + ) + self.image_layer.footprintPmtilesUrl = pmtiles_url + self._update_progress( + "Prepared footprint tiles for " + f"{int(manifest.get('building_count', 0))} buildings" + ) + + def _artifact_url(self, filename: str) -> str: + """Resolve a task output filename to a downloadable URL.""" + if not filename: + return "" + return self.storage.get_file_remote_path( + identifier=filename, + extra_partition_keys=( + f"{self.image_layer.footprintTilesJob.taskId}" + ), + data_format=os.path.splitext(filename)[1].strip("."), + ) + + def _replay_friendly_logs(self) -> None: + for timestamp, message in self._get_friendly_logs(): + if message not in ( + self.image_layer.footprintTilesStatusMessage or "" + ): + self._update_progress(message, timestamp=timestamp) + + def _get_friendly_logs(self) -> List[Tuple[str, str]]: + job = self.image_layer.footprintTilesJob + content = self.runner.get_filecontent_from_task( + job_id=job.jobId, + task_id=job.taskId, + filename=FRIENDLY_LOG_FILENAME, + ) + logs: List[Tuple[str, str]] = [] + if content: + for record in content.splitlines(): + if not record: + continue + parts = record.split("|", 1) + if len(parts) == 2: + logs.append((parts[0], parts[1])) + return logs + + def _update_progress( + self, message: str, timestamp: Optional[str] = None + ) -> None: + self.image_layer.footprintTilesStatusMessage = ( + MetadataUtils.append_status_message( + self.image_layer.footprintTilesStatusMessage or "", + message, + timestamp=timestamp, + ) + ) diff --git a/hastelib/src/hastegeo/core/processors/imagery.py b/hastelib/src/hastegeo/core/processors/imagery.py index bc46172a..0af58eb0 100644 --- a/hastelib/src/hastegeo/core/processors/imagery.py +++ b/hastelib/src/hastegeo/core/processors/imagery.py @@ -14,6 +14,10 @@ from ..utils.logs import Logger from ..utils.metadata import MetadataUtils from ..utils.queues import AzureQueueHandler +from .footprint_tiles import ( + enqueue_footprint_tiles, + layer_needs_footprint_tiles, +) BATCH_JOB_WORKDIR = "AZ_BATCH_TASK_WORKING_DIR" IMAGERY_PREFIX = "img" @@ -248,6 +252,16 @@ def process(self): task_id=self.image_data.preprocessJob.taskId, ) + # Tile the footprints now that they are cached, so every + # map that draws this layer's buildings finds the archive + # already built. Skipped when _update_results_from_job + # flipped the layer to FAILED (no usable footprints). + if ( + self.image_data.status + == self.config.get_status_types().COMPLETED.value + ): + self._enqueue_footprint_tiles() + elif task_status == self.config.get_status_types().FAILED.value: self.image_data.preprocessJob.status = task_status self.image_data.preprocessJob.completedDate = ( @@ -570,6 +584,54 @@ def _update_results_from_job(self): imagery_type=self.config.get_artifact_types().VALID_AREA_MASK, ) + def _enqueue_footprint_tiles(self) -> None: + """Queue the layer's footprint PMTiles build (best effort). + + Geometry belongs to the layer, so one archive serves every model + trained on it. The work itself runs as a queued task in the + training image, the only one carrying ``tippecanoe``. + + Deliberately non-fatal: the tiles are derived data. If the queue + is unreachable the layer is still perfectly usable, and the job + can be re-requested later without re-running imagery prep. + """ + if not layer_needs_footprint_tiles(self.image_data): + self.logger.info( + "Not queueing footprint tiles for image layer %s " + "(footprints=%s, tiles=%s)", + self.image_data.imageLayerId, + bool(self.image_data.buildingFootprintsUrl), + bool(self.image_data.footprintPmtilesUrl), + ) + return + try: + enqueue_footprint_tiles( + project_id=self.image_data.projectId, + image_layer_id=self.image_data.imageLayerId, + source_footprints_url=self.image_data.buildingFootprintsUrl, + config=self.config, + ) + self.image_data.footprintTilesStatus = ( + self.config.get_status_types().PENDING.value + ) + self.image_data.footprintTilesStatusMessage = ( + MetadataUtils.append_status_message( + "", "Queued for footprint tile preparation" + ) + ) + self.logger.info( + "Queued footprint tiles for image layer %s", + self.image_data.imageLayerId, + ) + except Exception as e: + self.logger.warning( + "Could not queue footprint tiles for image layer %s: %s. " + "Imagery preprocessing is unaffected; the tiles can be " + "built later without re-running it.", + self.image_data.imageLayerId, + e, + ) + def _generate_imagery_url( self, filename: str, imagery_type: ArtifactTypes, validate=True ): diff --git a/hastelib/src/hastegeo/workflows/embed_buildings.py b/hastelib/src/hastegeo/workflows/embed_buildings.py index d82cb8c2..1512aad0 100644 --- a/hastelib/src/hastegeo/workflows/embed_buildings.py +++ b/hastelib/src/hastegeo/workflows/embed_buildings.py @@ -14,7 +14,12 @@ MOSAIKS (torchgeo RCF) model on the crop, mask the building polygon into the token grid, and average the token features into one vector per building. * Writes three outputs to ``outputs/``: a GeoJSON of footprints + ``f_0..f_N`` - feature columns, a PMTiles vector-tile file (via tippecanoe), and a manifest. + feature columns, a binary feature sidecar, and a manifest. + +This workflow does NOT tile the footprints. The layer's shared footprint +PMTiles archive is built once per image layer by +``workflows.prepare_footprint_tiles`` and reused by every model trained on +that layer, so tiling the same geometry per model produced a duplicate. CRITICAL — row-order invariant: ``GetValidationReport`` / ``GetAssessmentReport`` join predictions to the @@ -29,7 +34,6 @@ import json import math import os -import subprocess import sys from collections import defaultdict from datetime import datetime, timezone @@ -709,60 +713,6 @@ def embed_footprints( return out -def write_pmtiles(geojson_path: str, pmtiles_path: str) -> None: - """Convert a GeoJSON to PMTiles via tippecanoe. - - Keeps only the row-index ``id`` and ``overture_id`` attributes on each - feature (via ``-y``); the heavy ``f_*`` columns ride in a separate - binary sidecar (see :func:`write_features_sidecar`). Without ``-y`` - every tile would carry several KB of feature vectors per building, - blowing up archive size and slowing every pan. - - Lift the per-tile size cap so a dense max-zoom tile carrying every - geometry-only feature isn't trimmed; with ``f_*`` no longer in the - tiles this is a very cheap "keep all buildings" guarantee. - """ - cmd = [ - "tippecanoe", - "-o", - pmtiles_path, - "-l", - "buildings", - # Bake the row-index `id` into each tile as the MVT feature id. This - # gives every footprint a stable, native feature id so the interactive - # labeler can drive per-building coloring via map feature-state — both - # MapLibre and Azure Maps need this (Azure Maps' VectorTileSource does - # not honor client-side promoteId). - "--use-attribute-for-id=id", - # Strip every attribute except id + overture_id. The labeler does - # not need anything else in the tiles — feature vectors are looked - # up from the sidecar by id, and Overture id is needed for the - # PutInteractiveLabels round-trip. - "-y", - "id", - "-y", - "overture_id", - "--force", - # Cull world-scale levels — buildings are invisible below z=10, - # so generating tiles there only adds bytes. - "--minimum-zoom=10", - # Pin the max zoom to the labeler's working zoom + 1. Tiles at - # z>15 are rendered by the SDK via overzoom; the interactive - # labeler's queryRenderedFeatures + setFeatureState work unchanged - # on overzoomed tiles. - "--maximum-zoom=15", - # With f_* gone from the tiles even a dense urban tile is small; - # we still pass --no-tile-size-limit so the default 500 KB cap - # never bites and the build never silently drops a footprint. - "--no-tile-size-limit", - # Last-resort safety valve. - "--drop-densest-as-needed", - geojson_path, - ] - log_progress(f"Running tippecanoe -> {os.path.basename(pmtiles_path)}") - subprocess.run(cmd, check=True) - - # Binary sidecar format: # bytes 0-3 : magic "HFTR" (4 ASCII chars) # bytes 4-7 : u32 LE version (currently 1) @@ -787,8 +737,9 @@ def write_features_sidecar( The labeler fetches this file once at session start and looks vectors up by their row-index id — the in-tile attributes are kept down to just - id + overture_id (see ``write_pmtiles``), so the PMTiles archive stays - small and the labeler still has the data it needs to train + predict. + id + overture_id, so the layer's shared footprint PMTiles archive + stays small and the labeler still has what it needs to train and + predict. Returns ``(num_buildings, feat_dim)``. """ @@ -860,9 +811,6 @@ def main(): embeddings_name = files.get( "embeddings", f"building_embeddings_{config.get('model_id')}.geojson" ) - pmtiles_name = files.get( - "pmtiles", f"building_pmtiles_{config.get('model_id')}.pmtiles" - ) sidecar_name = files.get( "sidecar", f"building_features_{config.get('model_id')}.bin" ) @@ -896,18 +844,11 @@ def main(): log_progress(f"Writing {len(gdf)} buildings -> {embeddings_path}") gdf.to_file(embeddings_path, driver="GeoJSON") - # Features sidecar — must be written BEFORE the PMTiles step strips - # the f_* columns from the tiles, so the in-memory gdf is still - # the authoritative source for both. sidecar_path = os.path.join(output_dir, os.path.basename(sidecar_name)) num_buildings, feat_dim = write_features_sidecar(gdf, sidecar_path) - pmtiles_path = os.path.join(output_dir, os.path.basename(pmtiles_name)) - write_pmtiles(embeddings_path, pmtiles_path) - manifest = { "embeddings_filename": os.path.basename(embeddings_path), - "pmtiles_filename": os.path.basename(pmtiles_path), "sidecar_filename": os.path.basename(sidecar_path), "num_buildings": int(num_buildings), "num_features": int(feat_dim), diff --git a/hastelib/src/hastegeo/workflows/prepare_footprint_tiles.py b/hastelib/src/hastegeo/workflows/prepare_footprint_tiles.py new file mode 100644 index 00000000..277b290e --- /dev/null +++ b/hastelib/src/hastegeo/workflows/prepare_footprint_tiles.py @@ -0,0 +1,480 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Build an image layer's shared building-footprint vector tiles. + +Every map that draws a layer's buildings — the interactive labeler today, +the results viewer later — needs the same thing: the layer's cached +building footprints as a PMTiles archive carrying a stable per-building +id. Geometry belongs to the *layer*, not to any one model: every model +trained on a layer draws exactly the same buildings. So the archive is +built once, when the footprints are cached, and shared. + +The archive holds geometry plus two attributes: + +* ``id`` — the footprint's row index, promoted to the native MVT feature + id so a browser can drive per-building colouring through map + feature-state, and +* ``overture_id`` — the Overture string id, which round-trips through the + labeling APIs. + +Nothing model-specific goes in the tiles. Per-model values (embedding +feature vectors, damage scores) travel in their own sidecars keyed by the +same ``id``, which is what keeps a dense urban tile small. + +CRITICAL — row-order invariant: + Predictions and embeddings join to the layer's ``buildingFootprintsUrl`` + GeoPackage **by row index**, so ``id`` MUST be ``0..N-1`` in the + footprints file's native order. Rows are never dropped or reordered. + +CRITICAL — where this runs: + ``tippecanoe`` ships only in the HASTE training docker image + (``docker/training/env/env.yml``). This module must therefore run as a + queued Batch/local-runner task inside that container, never inline in + an Azure Functions HTTP handler. See + ``hastegeo.core.processors.footprint_tiles``. + +Config (written by the processor, read by ``main``):: + + { + "project_id": "...", + "image_layer_id": "...", + "output_dir": "outputs", + "files": { + "footprints": "inputs/.gpkg", + "pmtiles": "footprints_.pmtiles" + }, + "tiles": {"minimum_zoom": 10, "maximum_zoom": 15}, + "store_artifacts": true + } +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +import geopandas as gpd +from hastegeo.core.config import ArtifactTypes, Config, StorageType +from hastegeo.core.utils.gdal_security import harden_gdal +from hastegeo.core.utils.logs import Logger as HasteLogger + +WORKDIR = os.getenv("WORKDIR", ".") +LOG_DIR = os.path.join(WORKDIR, "logs") +LOG_FILE = "footprint_tiles_verbose.log" +FRIENDLY_LOG_FILE = "footprint_tiles_friendly.log" +os.makedirs(LOG_DIR, exist_ok=True) + +logger = HasteLogger.get_logger( + "prepare_footprint_tiles", log_dir=LOG_DIR, log_file=LOG_FILE +) +logger.info("Executing %s", __file__) + +# Harden GDAL/OGR before reading any user-reachable vector file (GDAL CVE +# compensating control — docs/known-vulnerabilities.md Root Cause C). +harden_gdal() + +TIPPECANOE_BIN = "tippecanoe" +TILE_LAYER_NAME = "buildings" +# Buildings are invisible below z10, and the maps that read these tiles +# work at z<=15; anything above is produced by the map SDK via overzoom, +# on which queryRenderedFeatures and setFeatureState keep working. +DEFAULT_MIN_ZOOM = 10 +DEFAULT_MAX_ZOOM = 15 +# Tiles carry only these two attributes. Per-model values ride in their +# own sidecars, which keeps a dense urban tile small. +TILE_ID_FIELD = "id" +TILE_OVERTURE_ID_FIELD = "overture_id" +TILING_CRS = "EPSG:4326" + +MANIFEST_FILENAME = "footprint_tiles_manifest.json" +TILING_GEOJSON_NAME = "footprints_4326.geojson" +DEFAULT_OUTPUT_DIR = "outputs" + + +class TippecanoeNotFoundError(RuntimeError): + """Raised when the tippecanoe binary is unavailable.""" + + +class TippecanoeError(RuntimeError): + """Raised when tippecanoe runs but exits non-zero.""" + + +def log_progress(message: str) -> None: + """Append a friendly progress line consumed by the postprocessor.""" + logger.info(message) + log_file = os.path.join(LOG_DIR, FRIENDLY_LOG_FILE) + with open(log_file, "a") as handle: + handle.write(f"{datetime.now(timezone.utc).isoformat()}|{message}\n") + + +def require_tippecanoe() -> str: + """Return the tippecanoe executable path or fail with guidance. + + Raises: + TippecanoeNotFoundError: when the binary is not on PATH. The + message names the one image that ships it, so the failure is + actionable rather than a bare ``FileNotFoundError`` traceback + out of ``subprocess``. + """ + binary = shutil.which(TIPPECANOE_BIN) + if binary: + return binary + raise TippecanoeNotFoundError( + "tippecanoe was not found on PATH, so the footprint vector tiles " + "cannot be built. tippecanoe ships only in the HASTE training " + "image (docker/training/env/env.yml); run this workflow as a " + "queued task in that container " + "(hastegeo.core.processors.footprint_tiles), never inline in the " + "Azure Functions app." + ) + + +def footprints_to_tiling_geojson( + footprints_path: str, geojson_path: str +) -> int: + """Write a tiling-ready EPSG:4326 GeoJSON of the footprints. + + Emits exactly two attributes per feature: an integer ``id`` equal to + the footprint's row index (the positional join key) and the Overture + string id as ``overture_id``. + + Args: + footprints_path: Building-footprints GeoPackage for the layer. + geojson_path: Destination GeoJSON path. + + Returns: + Number of footprints written. + + Raises: + ValueError: if the GeoPackage is empty, has no CRS, or lacks the + Overture ``id`` column. + """ + footprints = gpd.read_file(footprints_path) + if len(footprints) == 0: + raise ValueError( + f"Building footprints GeoPackage is empty: {footprints_path}" + ) + if footprints.crs is None: + raise ValueError( + "Building footprints GeoPackage has no CRS, refusing to tile " + f"unreferenced geometry: {footprints_path}" + ) + if TILE_ID_FIELD not in footprints.columns: + raise ValueError( + "Building footprints GeoPackage has no 'id' column (expected " + f"the Overture id): {footprints_path}" + ) + + # Tiles are always geographic; reproject only when needed so an + # already-4326 layer keeps its exact coordinates. + if footprints.crs.to_epsg() != 4326: + logger.info( + "Reprojecting footprints from %s to %s for tiling", + footprints.crs, + TILING_CRS, + ) + footprints = footprints.to_crs(TILING_CRS) + + tiles_gdf = gpd.GeoDataFrame( + { + # Row index -> integer feature id. MUST stay 0..N-1 in native + # footprint order to match the positional join. + TILE_ID_FIELD: range(len(footprints)), + TILE_OVERTURE_ID_FIELD: footprints[TILE_ID_FIELD].astype(str), + }, + geometry=footprints.geometry.values, + crs=TILING_CRS, + ) + tiles_gdf.to_file(geojson_path, driver="GeoJSON") + logger.info( + "Wrote %d footprints to %s for tiling", + len(tiles_gdf), + geojson_path, + ) + return len(tiles_gdf) + + +def run_tippecanoe( + geojson_path: str, + pmtiles_path: str, + minimum_zoom: int = DEFAULT_MIN_ZOOM, + maximum_zoom: int = DEFAULT_MAX_ZOOM, +) -> None: + """Convert a footprints GeoJSON to PMTiles via tippecanoe. + + ``id`` becomes the native MVT feature id + (``--use-attribute-for-id=id``) so a browser can colour buildings + through map feature-state, and only ``id`` + ``overture_id`` survive + into the tiles. + + Raises: + TippecanoeNotFoundError: when the binary is missing. + TippecanoeError: when tippecanoe exits non-zero. + """ + binary = require_tippecanoe() + cmd = [ + binary, + "-o", + pmtiles_path, + "-l", + TILE_LAYER_NAME, + # Bake the row-index id into each tile as the MVT feature id. + # Azure Maps' VectorTileSource does not honor client-side + # promoteId, so the id has to be native. + f"--use-attribute-for-id={TILE_ID_FIELD}", + # Keep only the join key and the Overture id. + "-y", + TILE_ID_FIELD, + "-y", + TILE_OVERTURE_ID_FIELD, + "--force", + f"--minimum-zoom={int(minimum_zoom)}", + f"--maximum-zoom={int(maximum_zoom)}", + # Geometry-only features are tiny; lifting the 500 KB per-tile + # cap guarantees no footprint is silently dropped. + "--no-tile-size-limit", + # Last-resort safety valve. + "--drop-densest-as-needed", + geojson_path, + ] + log_progress(f"Running tippecanoe -> {os.path.basename(pmtiles_path)}") + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as exc: + raise TippecanoeError( + f"tippecanoe failed with exit code {exc.returncode} while " + f"building {os.path.basename(pmtiles_path)}" + ) from exc + + +def build_footprint_pmtiles( + footprints_path: str, + pmtiles_path: str, + minimum_zoom: int = DEFAULT_MIN_ZOOM, + maximum_zoom: int = DEFAULT_MAX_ZOOM, + geojson_path: Optional[str] = None, +) -> int: + """Build the layer's footprint PMTiles archive. + + Returns: + Number of footprints tiled. + """ + # Fail before the (potentially slow) reprojection when the binary is + # missing, so the error the user sees is the actionable one. + require_tippecanoe() + geojson_path = geojson_path or os.path.join( + os.path.dirname(pmtiles_path) or ".", TILING_GEOJSON_NAME + ) + count = footprints_to_tiling_geojson(footprints_path, geojson_path) + run_tippecanoe( + geojson_path, + pmtiles_path, + minimum_zoom=minimum_zoom, + maximum_zoom=maximum_zoom, + ) + log_progress(f"Built footprint tiles for {count} buildings") + return count + + +def artifact_storage_available(config: Config) -> bool: + """Report whether this process can reach artifact storage. + + Azure Batch tasks are not given storage credentials by default (the + runner uploads ``outputs/`` for them), while the local docker runner + does pass them through. Probing up front lets the workflow store the + archive itself when it can and fall back to the runner's upload + otherwise, instead of dying on a misleading credential error. + """ + storage_config = config.artifact_storage_config or {} + if config.artifact_storage_type == StorageType.BLOB.value: + return bool( + storage_config.get("connection_string") + or storage_config.get("account_url") + ) + if config.artifact_storage_type == StorageType.LOCAL.value: + return bool(storage_config.get("directory")) + return False + + +def store_artifacts( + project_id: str, + artifacts: Dict[str, str], + config: Optional[Config] = None, +) -> Dict[str, str]: + """Store artifacts through the artifact-storage façade. + + Args: + project_id: Storage partition key. + artifacts: ``{artifact_name: local_path}``. + config: Optional config override. + + Returns: + ``{artifact_name: download_url}`` for everything stored. + """ + # Use the storage layer directly rather than ArtifactProcessor: that + # processor also drives zip jobs and therefore imports the queue SDK, + # which the training image does not install. This workflow only ever + # needs to put bytes in blob storage. + from hastegeo.core.artifact_storage.unified_artifact_storage import ( + UnifiedArtifactStorage, + ) + + config = config or Config() + storage = UnifiedArtifactStorage( + storage_type=config.artifact_storage_type, + partition_key=project_id, + **config.artifact_storage_config, + ) + urls: Dict[str, str] = {} + for artifact_name, local_path in artifacts.items(): + if not os.path.exists(local_path): + raise FileNotFoundError( + f"Artifact {artifact_name} not found at {local_path}" + ) + storage.store_artifact( + artifact_name=artifact_name, src_path=local_path + ) + urls[artifact_name] = storage.get_download_url( + identifier=artifact_name + ) + logger.info("Stored artifact %s", artifact_name) + return urls + + +def default_pmtiles_name(image_layer_id: str) -> str: + """Artifact filename of a layer's footprint PMTiles archive.""" + return ( + ArtifactTypes.LAYER_FOOTPRINT_PMTILES.value.substitute( + imageLayerId=image_layer_id + ) + + ".pmtiles" + ) + + +def run(config: Dict[str, Any], output_dir: str) -> Dict[str, Any]: + """Build the layer's footprint tiles and return the manifest. + + Args: + config: Parsed workflow config (see the module docstring). + output_dir: Directory the archive is written to. The runner + uploads everything in here after the task completes. + + Returns: + The manifest dict, which is also written to ``output_dir``. + + Raises: + ValueError: when the identifiers are missing. + FileNotFoundError: when the footprints file is absent. + """ + files: Dict[str, Any] = config.get("files", {}) + tiles_config: Dict[str, Any] = config.get("tiles", {}) + project_id = config.get("project_id") + image_layer_id = config.get("image_layer_id") + if not project_id or not image_layer_id: + raise ValueError("Config must set project_id and image_layer_id.") + + footprints_path = files.get("footprints") + if not footprints_path or not os.path.exists(footprints_path): + raise FileNotFoundError( + f"Building footprints not found: {footprints_path}" + ) + + pmtiles_name = os.path.basename( + files.get("pmtiles") or default_pmtiles_name(image_layer_id) + ) + + manifest: Dict[str, Any] = { + "project_id": project_id, + "image_layer_id": image_layer_id, + "pmtiles_filename": pmtiles_name, + "pmtiles_url": None, + "building_count": 0, + } + + log_progress("Building footprint vector tiles") + pmtiles_path = os.path.join(output_dir, pmtiles_name) + tiled_count = build_footprint_pmtiles( + footprints_path, + pmtiles_path, + minimum_zoom=int(tiles_config.get("minimum_zoom", DEFAULT_MIN_ZOOM)), + maximum_zoom=int(tiles_config.get("maximum_zoom", DEFAULT_MAX_ZOOM)), + geojson_path=os.path.join(output_dir, TILING_GEOJSON_NAME), + ) + manifest["building_count"] = int(tiled_count) + + if config.get("store_artifacts", True): + workflow_config = Config() + if artifact_storage_available(workflow_config): + urls = store_artifacts( + project_id, + {pmtiles_name: pmtiles_path}, + config=workflow_config, + ) + manifest["pmtiles_url"] = urls.get(pmtiles_name) + else: + log_progress( + "Artifact storage is unreachable from this task; leaving " + "the archive in outputs/ for the runner to upload." + ) + + # The GeoJSON is only tippecanoe's input; drop it so the runner does + # not upload a second full copy of every footprint. + stray_geojson = os.path.join(output_dir, TILING_GEOJSON_NAME) + if os.path.exists(stray_geojson): + os.remove(stray_geojson) + + log_progress("Finalizing outputs") + with open(os.path.join(output_dir, MANIFEST_FILENAME), "w") as handle: + json.dump(manifest, handle, indent=4) + return manifest + + +def main() -> None: + """CLI entrypoint: ``prepare-footprint-tiles --config ``.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", type=str, required=True, help="Path to config JSON file" + ) + args = parser.parse_args() + + if not os.path.exists(args.config): + logger.error("No config file found at location %s", args.config) + sys.exit(1) + + with open(args.config) as handle: + config = json.load(handle) + if not config: + logger.error("Config file is empty") + sys.exit(1) + + output_dir = os.path.join( + WORKDIR, config.get("output_dir", DEFAULT_OUTPUT_DIR) + ) + os.makedirs(output_dir, exist_ok=True) + + try: + run(config, output_dir) + logger.info("Footprint tile preparation completed successfully.") + except TippecanoeNotFoundError as exc: + logger.error("%s", exc) + log_progress(f"Footprint tile preparation failed: {exc}") + raise + except Exception as exc: + logger.error("Error during footprint tile preparation", exc_info=True) + log_progress(f"Error during footprint tile preparation: {exc}") + raise + + +if __name__ == "__main__": + try: + main() + except Exception as error: # pragma: no cover - CLI guard + logger.error(f"Error during main execution: {error}", exc_info=True) + sys.exit(1) diff --git a/hastelib/tests/core/processors/test_footprint_tiles.py b/hastelib/tests/core/processors/test_footprint_tiles.py new file mode 100644 index 00000000..71e9bc08 --- /dev/null +++ b/hastelib/tests/core/processors/test_footprint_tiles.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Unit tests for the layer footprint-tiles processor. + +Covers the decisions that keep the job cheap and idempotent: whether a +layer needs tiling at all, and whether a request should actually reach +the queue. The container job itself is not exercised here — tippecanoe +runs only in the training image. +""" + +import unittest +from unittest.mock import patch + +from hastegeo.core.config import Config +from hastegeo.core.models.projects import ImageLayer +from hastegeo.core.processors import footprint_tiles + +STATUSES = Config.get_status_types() + + +def _layer(**overrides): + data = { + "projectId": "proj-1", + "imageLayerId": "11111111-1111-1111-1111-111111111111", + "buildingFootprintsUrl": "https://acct/footprints.gpkg?sas", + } + data.update(overrides) + return ImageLayer(**data) + + +class TestLayerNeedsFootprintTiles(unittest.TestCase): + def test_needs_tiles_once_footprints_are_cached(self): + self.assertTrue(footprint_tiles.layer_needs_footprint_tiles(_layer())) + + def test_no_footprints_means_nothing_to_tile(self): + self.assertFalse( + footprint_tiles.layer_needs_footprint_tiles( + _layer(buildingFootprintsUrl=None) + ) + ) + + def test_an_existing_archive_is_not_rebuilt(self): + self.assertFalse( + footprint_tiles.layer_needs_footprint_tiles( + _layer(footprintPmtilesUrl="https://acct/footprints.pmtiles") + ) + ) + + +class TestArtifactName(unittest.TestCase): + def test_archive_is_named_for_the_layer_not_a_model(self): + # Keyed on the layer because every model on it shares the archive. + self.assertEqual( + footprint_tiles.pmtiles_artifact_name("layer-7"), + "footprints_layer-7.pmtiles", + ) + + +class TestQueueMessage(unittest.TestCase): + def test_message_carries_identifiers_only(self): + message = footprint_tiles.build_tiles_message( + project_id="p", image_layer_id="l", source_footprints_url="u" + ) + self.assertEqual( + message, + { + "projectId": "p", + "imageLayerId": "l", + "sourceFootprintsUrl": "u", + "force": False, + }, + ) + + +class TestRequestPreparation(unittest.TestCase): + """The HTTP/imagery-side seam: enqueue only when there is work.""" + + def test_queues_when_the_layer_has_no_archive(self): + layer = _layer() + with patch.object( + footprint_tiles, "enqueue_footprint_tiles" + ) as enqueue: + result = footprint_tiles.request_preparation(layer) + + enqueue.assert_called_once() + self.assertTrue(result["queued"]) + self.assertFalse(result["tilesReady"]) + self.assertEqual(layer.footprintTilesStatus, STATUSES.PENDING.value) + + def test_is_a_no_op_when_the_archive_exists(self): + layer = _layer(footprintPmtilesUrl="https://acct/f.pmtiles") + with patch.object( + footprint_tiles, "enqueue_footprint_tiles" + ) as enqueue: + result = footprint_tiles.request_preparation(layer) + + enqueue.assert_not_called() + self.assertFalse(result["queued"]) + self.assertTrue(result["tilesReady"]) + self.assertEqual(layer.footprintTilesStatus, STATUSES.COMPLETED.value) + + def test_does_not_queue_a_second_job_while_one_is_in_flight(self): + # Re-queueing would submit a duplicate container job for the same + # archive. + for status in ( + STATUSES.PENDING.value, + STATUSES.IN_PROGRESS.value, + ): + layer = _layer(footprintTilesStatus=status) + with patch.object( + footprint_tiles, "enqueue_footprint_tiles" + ) as enqueue: + result = footprint_tiles.request_preparation(layer) + + enqueue.assert_not_called() + self.assertFalse(result["queued"], status) + + def test_force_rebuilds_an_existing_archive(self): + layer = _layer(footprintPmtilesUrl="https://acct/f.pmtiles") + with patch.object( + footprint_tiles, "enqueue_footprint_tiles" + ) as enqueue: + result = footprint_tiles.request_preparation(layer, force=True) + + enqueue.assert_called_once() + self.assertTrue(result["queued"]) + + def test_without_footprints_there_is_nothing_to_tile(self): + layer = _layer(buildingFootprintsUrl=None) + with patch.object( + footprint_tiles, "enqueue_footprint_tiles" + ) as enqueue: + with self.assertRaises(ValueError): + footprint_tiles.request_preparation(layer) + + enqueue.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/hastelib/tests/core/processors/test_imagery_footprint_tiles.py b/hastelib/tests/core/processors/test_imagery_footprint_tiles.py new file mode 100644 index 00000000..a5d2cf98 --- /dev/null +++ b/hastelib/tests/core/processors/test_imagery_footprint_tiles.py @@ -0,0 +1,105 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""The imagery processor queues footprint tiling once footprints exist. + +Footprint geometry belongs to the image layer, so the archive is built +when imagery prep caches the footprints rather than per model. These +tests pin the two things that matter: it happens for both workflow types, +and a queue failure never takes the image layer down with it. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from hastegeo.core.config import Config +from hastegeo.core.models.projects import ImageLayer +from hastegeo.core.processors import imagery + +STATUSES = Config.get_status_types() + + +def _processor(image_data): + """An ImageryPostProcessor with only the fields the hook reads.""" + processor = MagicMock(spec=imagery.ImageryPostProcessor) + processor.image_data = image_data + processor.config = Config() + processor.logger = MagicMock() + # Bind the real method so the mock exercises production logic. + processor._enqueue_footprint_tiles = ( + imagery.ImageryPostProcessor._enqueue_footprint_tiles.__get__( + processor + ) + ) + return processor + + +def _layer(**overrides): + data = { + "projectId": "proj-1", + "imageLayerId": "22222222-2222-2222-2222-222222222222", + "buildingFootprintsUrl": "https://acct/footprints.gpkg?sas", + } + data.update(overrides) + return ImageLayer(**data) + + +class TestEnqueueFootprintTiles(unittest.TestCase): + def test_queues_for_the_standard_workflow(self): + layer = _layer(workflowType="standard") + processor = _processor(layer) + with patch.object(imagery, "enqueue_footprint_tiles") as enqueue: + processor._enqueue_footprint_tiles() + + enqueue.assert_called_once() + self.assertEqual( + enqueue.call_args.kwargs["image_layer_id"], layer.imageLayerId + ) + self.assertEqual(layer.footprintTilesStatus, STATUSES.PENDING.value) + + def test_queues_for_the_building_workflow(self): + # The embedding job no longer tiles, so this layer type depends on + # the same archive as every other. + layer = _layer(workflowType="building") + processor = _processor(layer) + with patch.object(imagery, "enqueue_footprint_tiles") as enqueue: + processor._enqueue_footprint_tiles() + + enqueue.assert_called_once() + self.assertEqual(layer.footprintTilesStatus, STATUSES.PENDING.value) + + def test_skips_when_the_layer_has_no_footprints(self): + layer = _layer(buildingFootprintsUrl=None) + processor = _processor(layer) + with patch.object(imagery, "enqueue_footprint_tiles") as enqueue: + processor._enqueue_footprint_tiles() + + enqueue.assert_not_called() + self.assertIsNone(layer.footprintTilesStatus) + + def test_skips_when_the_archive_already_exists(self): + layer = _layer(footprintPmtilesUrl="https://acct/f.pmtiles") + processor = _processor(layer) + with patch.object(imagery, "enqueue_footprint_tiles") as enqueue: + processor._enqueue_footprint_tiles() + + enqueue.assert_not_called() + + def test_a_queue_failure_does_not_fail_the_layer(self): + # The tiles are derived data. An unreachable queue must leave the + # imagery perfectly usable. + layer = _layer(status=STATUSES.COMPLETED.value) + processor = _processor(layer) + with patch.object( + imagery, + "enqueue_footprint_tiles", + side_effect=RuntimeError("queue unreachable"), + ): + processor._enqueue_footprint_tiles() + + self.assertEqual(layer.status, STATUSES.COMPLETED.value) + processor.logger.warning.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/hastelib/tests/workflows/test_prepare_footprint_tiles.py b/hastelib/tests/workflows/test_prepare_footprint_tiles.py new file mode 100644 index 00000000..4036f489 --- /dev/null +++ b/hastelib/tests/workflows/test_prepare_footprint_tiles.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Unit tests for the footprint tiling workflow. + +The tippecanoe step needs the binary and is exercised on the dev stack, +not here. What these tests pin is the part that silently corrupts a map +when it goes wrong: the tiling GeoJSON's id space. Predictions and +embeddings join to the footprints GeoPackage positionally, so ``id`` has +to be 0..N-1 in the file's native order, and every row has to survive. +""" + +import os +import tempfile +import unittest + +import fiona +from fiona.crs import CRS +from fiona.model import Feature, Geometry +from hastegeo.workflows import prepare_footprint_tiles as workflow + + +def _square(i, offset=0.0): + x = float(i) + offset + return Geometry( + type="Polygon", + coordinates=[ + [ + (x, 0.0), + (x + 0.0005, 0.0), + (x + 0.0005, 0.0005), + (x, 0.0005), + (x, 0.0), + ] + ], + ) + + +def _write_footprints(path, overture_ids, crs="EPSG:4326"): + schema = {"geometry": "Polygon", "properties": {"id": "str"}} + with fiona.open( + path, + "w", + driver="GPKG", + crs=CRS.from_string(crs), + schema=schema, + ) as dst: + for i, oid in enumerate(overture_ids): + dst.write(Feature(geometry=_square(i), properties={"id": oid})) + + +class TestFootprintsToTilingGeojson(unittest.TestCase): + def test_id_is_the_row_index_and_overture_id_is_preserved(self): + with tempfile.TemporaryDirectory() as tmp: + src = os.path.join(tmp, "footprints.gpkg") + dst = os.path.join(tmp, "tiles.geojson") + _write_footprints(src, ["over-a", "over-b", "over-c"]) + + count = workflow.footprints_to_tiling_geojson(src, dst) + self.assertEqual(count, 3) + + with fiona.open(dst) as read_back: + rows = [feat["properties"] for feat in read_back] + + # The positional join key: 0..N-1 in the footprints file's order. + self.assertEqual([row["id"] for row in rows], [0, 1, 2]) + self.assertEqual( + [row["overture_id"] for row in rows], + ["over-a", "over-b", "over-c"], + ) + + def test_every_footprint_survives(self): + # A dropped row would shift every later id and silently mis-colour + # buildings, so the count is a hard invariant. + with tempfile.TemporaryDirectory() as tmp: + src = os.path.join(tmp, "footprints.gpkg") + dst = os.path.join(tmp, "tiles.geojson") + _write_footprints(src, [f"over-{i}" for i in range(25)]) + + count = workflow.footprints_to_tiling_geojson(src, dst) + with fiona.open(dst) as read_back: + self.assertEqual(len(list(read_back)), 25) + + self.assertEqual(count, 25) + + def test_reprojects_to_4326(self): + with tempfile.TemporaryDirectory() as tmp: + src = os.path.join(tmp, "footprints.gpkg") + dst = os.path.join(tmp, "tiles.geojson") + _write_footprints(src, ["a", "b"], crs="EPSG:3857") + + workflow.footprints_to_tiling_geojson(src, dst) + with fiona.open(dst) as read_back: + self.assertEqual(read_back.crs.to_epsg(), 4326) + + def test_rejects_an_empty_geopackage(self): + with tempfile.TemporaryDirectory() as tmp: + src = os.path.join(tmp, "footprints.gpkg") + dst = os.path.join(tmp, "tiles.geojson") + _write_footprints(src, []) + + with self.assertRaises(ValueError): + workflow.footprints_to_tiling_geojson(src, dst) + + def test_rejects_footprints_without_an_overture_id(self): + with tempfile.TemporaryDirectory() as tmp: + src = os.path.join(tmp, "footprints.gpkg") + dst = os.path.join(tmp, "tiles.geojson") + schema = { + "geometry": "Polygon", + "properties": {"name": "str"}, + } + with fiona.open( + src, + "w", + driver="GPKG", + crs=CRS.from_epsg(4326), + schema=schema, + ) as handle: + handle.write( + Feature(geometry=_square(0), properties={"name": "no id"}) + ) + + with self.assertRaises(ValueError): + workflow.footprints_to_tiling_geojson(src, dst) + + +class TestArtifactNaming(unittest.TestCase): + def test_archive_is_keyed_on_the_layer(self): + self.assertEqual( + workflow.default_pmtiles_name("layer-3"), + "footprints_layer-3.pmtiles", + ) + + +class TestTippecanoeGuard(unittest.TestCase): + def test_missing_binary_names_the_image_that_ships_it(self): + # A bare FileNotFoundError from subprocess tells an operator + # nothing; this message has to say where tippecanoe lives. + if workflow.shutil.which(workflow.TIPPECANOE_BIN): + self.skipTest("tippecanoe is installed in this environment") + with self.assertRaises(workflow.TippecanoeNotFoundError) as ctx: + workflow.require_tippecanoe() + self.assertIn("training", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/local.settings.example.jsonc b/local.settings.example.jsonc index ba32276d..b0952ac1 100644 --- a/local.settings.example.jsonc +++ b/local.settings.example.jsonc @@ -46,6 +46,7 @@ "STATS_QUEUE_NAME": "my-local-stats-queue", "ZIP_QUEUE_NAME": "my-local-zip-queue", "EMBEDDING_QUEUE_NAME": "my-local-embedding-queue", + "FOOTPRINT_TILES_QUEUE_NAME": "my-local-footprint-tiles-queue", // ===== REQUIRED: Docker Images ===== "AZURE_BATCH_DOCKER_IMAGE": ".azurecr.io/hastetraining:", diff --git a/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx b/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx index e5a20338..5c6868a5 100644 --- a/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx +++ b/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx @@ -769,10 +769,10 @@ const InteractiveLabeler = () => { // the layer has no pre-event imagery). layerImageryRef.current = layerData?.imagery || null; - // Resolve the model's PMTiles URL. Models are returned by - // GetLayerModelsDetails; pick ours by modelId. The pmtilesUrl is - // populated by the embedding workflow's postprocessor. - let pmtilesUrl = ""; + // The features sidecar is per model, so it still comes from + // GetLayerModelsDetails. The footprint tiles do not: geometry belongs + // to the image layer and one archive is shared by every model on it, + // so those are requested by kind below. let sidecarUrl = ""; setInitialLoad({ step: 1, loaded: null, total: null }); try { @@ -782,17 +782,11 @@ const InteractiveLabeler = () => { const model = (models || []).find( (m) => String(m.modelId) === String(modelId) ); - pmtilesUrl = model?.pmtilesUrl || ""; sidecarUrl = model?.featuresSidecarUrl || ""; } catch (e) { console.warn("Could not fetch model URLs:", e); } signal.throwIfAborted(); - if (!pmtilesUrl) { - throw new Error( - "No PMTiles available for this model — the embedding workflow has not produced building tiles." - ); - } if (!sidecarUrl) { throw new Error( "No features sidecar available for this model — re-embed the layer to produce one." @@ -805,7 +799,7 @@ const InteractiveLabeler = () => { // remote/mobile labelers hit a 403. const browserPmtilesUrl = buildUrl( `GetModelArtifact?projectId=${projectId}&modelId=${modelId}` + - `&kind=pmtiles` + `&imageLayerId=${imageLayerId}&kind=footprint_pmtiles` ); const browserSidecarUrl = buildUrl( `GetModelArtifact?projectId=${projectId}&modelId=${modelId}` + @@ -836,7 +830,16 @@ const InteractiveLabeler = () => { // (otherwise the map sits at [0, 0] zoom 3 and the user sees no tiles). pmtilesHeader = await pm.getHeader(); } catch (e) { - console.warn("Failed to load PMTiles archive (continuing):", e); + // Without the footprint tiles there are no buildings to label, so an + // empty map is the one thing this must not silently become. The + // archive belongs to the image layer and is built once its footprints + // are cached, so the usual cause is that job not having finished yet. + console.error("Failed to load the footprint PMTiles archive:", e); + throw new Error( + "The building footprint tiles for this image layer are not ready " + + "yet. They are built once per layer, shortly after the layer " + + "finishes processing. Try again in a few minutes." + ); } // Fetch the binary features sidecar and parse the HFTR header. The From afac0c49fa82fc796750da15d9cb695607e983a0 Mon Sep 17 00:00:00 2001 From: Caleb Robinson Date: Mon, 31 Aug 2026 22:47:50 +0000 Subject: [PATCH 2/2] fix(api): reach a layer's footprint tiles without a model id GetModelArtifact required modelId for every kind, including the layer-scoped footprint tiles. A standard-workflow layer can have no models at all, which made its own archive unfetchable. modelId is now optional when the kind is layer-scoped and an explicit imageLayerId is given. Every model-scoped kind still requires it, and a request carrying a modelId still resolves through that model's layer, so the Interactive Labeler is unaffected. --- api/hastefuncapi/function_app.py | 57 +++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/api/hastefuncapi/function_app.py b/api/hastefuncapi/function_app.py index 19201ef3..aa3d72d8 100644 --- a/api/hastefuncapi/function_app.py +++ b/api/hastefuncapi/function_app.py @@ -1418,7 +1418,6 @@ async def GetModelArtifact(req: func.HttpRequest) -> func.HttpResponse: """ try: project_id = _require_guid_param(req, "projectId") - model_id = _require_short_int_id_param(req, "modelId") except ValueError as e: return _bad_request(str(e)) @@ -1431,30 +1430,48 @@ async def GetModelArtifact(req: func.HttpRequest) -> func.HttpResponse: f"{sorted({**_MODEL_ARTIFACT_URL_FIELDS, **_LAYER_ARTIFACT_URL_FIELDS})}" ) - try: - document = await asyncio.to_thread( - MetadataProcessor( - data_type=config.get_metadata_types().MODEL.value, - partition_key=project_id, - ).load, - model_id, - ) - except FileNotFoundError: - return func.HttpResponse("Model not found.", status_code=404) - except Exception as e: - logger.error( - f"GetModelArtifact model load failed: {e}\n" - f"{traceback.format_exc()}" - ) - return func.HttpResponse("Error loading model.", status_code=500) + # A layer-scoped artifact belongs to the image layer, so an + # imageLayerId alone is enough to reach it: a standard-workflow layer + # may have no models at all. modelId stays accepted, and is still + # required for every model-scoped kind. + explicit_layer_id = req.params.get("imageLayerId") + layer_only_request = bool( + layer_url_field is not None + and explicit_layer_id + and not req.params.get("modelId") + ) + model_id = None + if not layer_only_request: + try: + model_id = _require_short_int_id_param(req, "modelId") + except ValueError as e: + return _bad_request(str(e)) + + document = None + if model_id is not None: + try: + document = await asyncio.to_thread( + MetadataProcessor( + data_type=config.get_metadata_types().MODEL.value, + partition_key=project_id, + ).load, + model_id, + ) + except FileNotFoundError: + return func.HttpResponse("Model not found.", status_code=404) + except Exception as e: + logger.error( + f"GetModelArtifact model load failed: {e}\n" + f"{traceback.format_exc()}" + ) + return func.HttpResponse("Error loading model.", status_code=500) if layer_url_field is not None: # Layer-scoped artifact: take an explicit imageLayerId when the - # caller passes one, otherwise the model's own layer — a client - # holding a modelId always means that model's footprints. + # caller passes one, otherwise the model's own layer. url_field = layer_url_field model_document = document or {} - image_layer_id = req.params.get("imageLayerId") or model_document.get( + image_layer_id = explicit_layer_id or model_document.get( "imageLayerId" ) if not image_layer_id or not _GUID_RE.match(str(image_layer_id)):