diff --git a/src/spatialcf/adapters/ai2thor/capture.py b/src/spatialcf/adapters/ai2thor/capture.py index 28cdc24..f199512 100644 --- a/src/spatialcf/adapters/ai2thor/capture.py +++ b/src/spatialcf/adapters/ai2thor/capture.py @@ -694,6 +694,100 @@ def _stable_instance_pixel_counts( counts[obj.object_id] = int(np.count_nonzero(mask)) return counts +def _stable_instance_colors( + self, + scene: Scene, + event: Any, +) -> tuple[tuple[str, tuple[int, int, int]], ...]: + raw_objects = event.metadata.get("objects") + if not isinstance(raw_objects, list): + raise AI2ThorNativeReturnError("observation returned no object metadata") + all_native_ids = { + item.get("objectId") + for item in raw_objects + if isinstance(item, dict) and type(item.get("objectId")) is str + } + domain_objects = _domain_object_metadata(raw_objects) + native_by_name = { + str(item.get("name")): item for item in domain_objects if isinstance(item, dict) + } + if len(native_by_name) != len(domain_objects): + raise AI2ThorNativeReturnError("observation returned duplicate object names") + native_colors = getattr(event, "object_id_to_color", None) + if not isinstance(native_colors, Mapping): + raise AI2ThorNativeReturnError("observation returned invalid instance colors") + colors: dict[str, tuple[int, int, int]] = {} + for native_id, color in native_colors.items(): + if ( + type(native_id) is not str + or type(color) is not tuple + or len(color) != 3 + or any( + type(channel) is not int or not 0 <= channel <= 255 + for channel in color + ) + ): + raise AI2ThorNativeReturnError( + "observation returned invalid instance colors" + ) + colors[native_id] = color + stable_by_native: dict[str, str] = {} + for obj in scene.objects: + metadata = native_by_name.get(obj.name) + if metadata is None or type(metadata.get("objectId")) is not str: + raise AI2ThorNativeReturnError( + "observation returned invalid instance colors" + ) + stable_by_native[metadata["objectId"]] = obj.object_id + counts = self._stable_instance_pixel_counts(scene, event) + masks = event.instance_masks + if any( + native_id not in all_native_ids and np.count_nonzero(mask) > 0 + for native_id, mask in masks.items() + ): + raise AI2ThorNativeReturnError( + "observation returned unknown instance colors" + ) + stable_colors: dict[str, tuple[int, int, int]] = {} + native_by_stable = { + stable_id: native_id for native_id, stable_id in stable_by_native.items() + } + for stable_id, count in counts.items(): + native_id = native_by_stable[stable_id] + color = colors.get(native_id) + if count > 0 and color is None: + raise AI2ThorNativeReturnError( + "observation returned incomplete instance colors" + ) + if color is None: + continue + mask = masks.get(native_id) + if mask is None: + mask = np.zeros((self.height, self.width), dtype=bool) + instance = np.asarray( + getattr(event, "instance_segmentation_frame", None) + ) + png_mask = np.all( + instance == np.asarray(color, dtype=np.uint8), axis=2 + ) + if not np.array_equal(png_mask, mask): + raise AI2ThorNativeReturnError( + "observation instance colors disagree with PNG" + ) + if count > 0: + stable_colors[stable_id] = color + if set(stable_colors) != { + object_id for object_id, count in counts.items() if count > 0 + }: + raise AI2ThorNativeReturnError( + "observation returned incomplete instance colors" + ) + if len(set(stable_colors.values())) != len(stable_colors): + raise AI2ThorNativeReturnError( + "observation returned duplicate instance colors" + ) + return tuple(sorted(stable_colors.items())) + def _observation_from_event( self, scene: Scene, @@ -709,6 +803,7 @@ def _observation_from_event( pointcloud_ply=self._pointcloud_bytes(camera, depth, rgb), instance_pixel_counts=self._stable_instance_pixel_counts(scene, event), is_scene_at_rest=self._native_scene_at_rest(event), + instance_colors=self._stable_instance_colors(scene, event), ) def capture_current_observation(self, scene: Scene) -> AI2ThorObservation: @@ -862,6 +957,7 @@ class AI2ThorCaptureMixin: _png_bytes = staticmethod(_png_bytes) _npy_bytes = staticmethod(_npy_bytes) _stable_instance_pixel_counts = _stable_instance_pixel_counts + _stable_instance_colors = _stable_instance_colors _observation_from_event = _observation_from_event capture_current_observation = capture_current_observation _validated_frames = _validated_frames diff --git a/src/spatialcf/adapters/ai2thor/models.py b/src/spatialcf/adapters/ai2thor/models.py index 50b0394..a28c92f 100644 --- a/src/spatialcf/adapters/ai2thor/models.py +++ b/src/spatialcf/adapters/ai2thor/models.py @@ -23,6 +23,7 @@ AppliedCertifiedEdit, CapturedSource, CertifiedEditApplication, + InstanceEvidenceProvenance, SettledReadback, ) from spatialcf.domain.scene import ( @@ -613,6 +614,7 @@ class AI2ThorObservation: pointcloud_ply_sha256: str instance_pixel_counts: Mapping[str, int] is_scene_at_rest: bool + instance_colors: tuple[tuple[str, tuple[int, int, int]], ...] @classmethod def create( @@ -625,6 +627,7 @@ def create( pointcloud_ply: bytes, instance_pixel_counts: Mapping[str, int], is_scene_at_rest: bool, + instance_colors: tuple[tuple[str, tuple[int, int, int]], ...], ) -> AI2ThorObservation: return cls( scene=scene, @@ -640,8 +643,54 @@ def create( dict(sorted(instance_pixel_counts.items())) ), is_scene_at_rest=is_scene_at_rest, + instance_colors=tuple( + (object_id, tuple(color)) for object_id, color in instance_colors + ), ) + def __post_init__(self) -> None: + if not isinstance(self.instance_pixel_counts, Mapping) or any( + type(object_id) is not str + or type(count) is not int + or count < 0 + for object_id, count in self.instance_pixel_counts.items() + ): + raise TypeError("AI2-THOR instance pixel counts must be exact") + if set(self.instance_pixel_counts) != { + obj.object_id for obj in self.scene.objects + }: + raise ValueError( + "AI2-THOR instance pixel counts must cover every scene object" + ) + if type(self.instance_colors) is not tuple or any( + type(item) is not tuple + or len(item) != 2 + or type(item[0]) is not str + or type(item[1]) is not tuple + or len(item[1]) != 3 + or any( + type(channel) is not int or not 0 <= channel <= 255 + for channel in item[1] + ) + for item in self.instance_colors + ): + raise TypeError("AI2-THOR instance colors must be exact RGB pairs") + if ( + self.instance_colors != tuple(sorted(self.instance_colors)) + or len({item[0] for item in self.instance_colors}) + != len(self.instance_colors) + or len({item[1] for item in self.instance_colors}) + != len(self.instance_colors) + ): + raise ValueError("AI2-THOR instance colors must be sorted and unique") + positive_ids = { + object_id + for object_id, count in self.instance_pixel_counts.items() + if count > 0 + } + if {item[0] for item in self.instance_colors} != positive_ids: + raise ValueError("AI2-THOR instance colors must cover positive pixels") + @dataclass(frozen=True) class AI2ThorCameraApplication: @@ -1014,6 +1063,10 @@ def adapter_observation_from_native(value: AI2ThorObservation) -> AdapterObserva pointcloud_ply=value.pointcloud_ply, instance_pixel_counts=tuple(sorted(value.instance_pixel_counts.items())), is_settled=value.is_scene_at_rest, + instance_colors=value.instance_colors, + instance_evidence_provenance=( + InstanceEvidenceProvenance.SAME_EVENT_INSTANCE_SEGMENTATION + ), ) diff --git a/src/spatialcf/adapters/base.py b/src/spatialcf/adapters/base.py index 1c80633..7083217 100644 --- a/src/spatialcf/adapters/base.py +++ b/src/spatialcf/adapters/base.py @@ -1,6 +1,7 @@ import json import math -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field +from enum import StrEnum from hashlib import sha256 from pathlib import Path from types import TracebackType @@ -651,6 +652,14 @@ def __post_init__(self) -> None: ) +class InstanceEvidenceProvenance(StrEnum): + """Whether instance sidecars are historical-neutral or one same-event readback.""" + + LEGACY_NEUTRAL = "LEGACY_NEUTRAL" + SAME_EVENT_INSTANCE_SEGMENTATION = "SAME_EVENT_INSTANCE_SEGMENTATION" + PUBLICATION_REPLAY_COUNTS_ONLY = "PUBLICATION_REPLAY_COUNTS_ONLY" + + @dataclass(frozen=True) class AdapterObservation: scene: Scene @@ -664,6 +673,15 @@ class AdapterObservation: pointcloud_ply_sha256: str instance_pixel_counts: tuple[tuple[str, int], ...] is_settled: bool + # Derived sidecar: preserve legacy neutral/public equality identity. AI2-THOR + # capture must still provide and validate it; this is not a fallback signal. + instance_colors: tuple[tuple[str, tuple[int, int, int]], ...] = field( + default=(), compare=False + ) + instance_evidence_provenance: InstanceEvidenceProvenance = field( + default=InstanceEvidenceProvenance.LEGACY_NEUTRAL, + compare=False, + ) @classmethod def create( @@ -676,6 +694,10 @@ def create( pointcloud_ply: bytes, instance_pixel_counts: tuple[tuple[str, int], ...], is_settled: bool, + instance_colors: tuple[tuple[str, tuple[int, int, int]], ...] = (), + instance_evidence_provenance: InstanceEvidenceProvenance = ( + InstanceEvidenceProvenance.LEGACY_NEUTRAL + ), ) -> Self: return cls( scene=scene, @@ -689,6 +711,10 @@ def create( pointcloud_ply_sha256=sha256(pointcloud_ply).hexdigest(), instance_pixel_counts=instance_pixel_counts, is_settled=is_settled, + instance_colors=tuple( + (object_id, tuple(color)) for object_id, color in instance_colors + ), + instance_evidence_provenance=instance_evidence_provenance, ) def __post_init__(self) -> None: @@ -739,8 +765,77 @@ def __post_init__(self) -> None: raise TypeError( "adapter observation pixel counts must be sorted exact pairs" ) + if self.instance_pixel_counts and { + item[0] for item in self.instance_pixel_counts + } != {obj.object_id for obj in self.scene.objects}: + raise ValueError( + "adapter observation pixel counts must cover every scene object" + ) if type(self.is_settled) is not bool: raise TypeError("adapter observation settled flag must be exact") + if ( + type(self.instance_colors) is not tuple + or any( + type(item) is not tuple + or len(item) != 2 + or type(item[0]) is not str + or type(item[1]) is not tuple + or len(item[1]) != 3 + or any( + type(channel) is not int or not 0 <= channel <= 255 + for channel in item[1] + ) + for item in self.instance_colors + ) + or self.instance_colors != tuple(sorted(self.instance_colors)) + or len({item[0] for item in self.instance_colors}) + != len(self.instance_colors) + or len({item[1] for item in self.instance_colors}) + != len(self.instance_colors) + ): + raise TypeError( + "adapter observation instance colors must be sorted exact RGB pairs" + ) + positive_ids = { + object_id for object_id, count in self.instance_pixel_counts if count > 0 + } + if ( + self.instance_colors + and {item[0] for item in self.instance_colors} != positive_ids + ): + raise ValueError( + "adapter observation instance colors must cover positive pixels" + ) + if type(self.instance_evidence_provenance) is not InstanceEvidenceProvenance: + raise TypeError("adapter observation instance evidence provenance is invalid") + if ( + self.instance_evidence_provenance + is InstanceEvidenceProvenance.LEGACY_NEUTRAL + ): + if self.instance_pixel_counts or self.instance_colors: + raise ValueError( + "adapter observation legacy neutral evidence requires empty sidecars" + ) + elif ( + self.instance_evidence_provenance + is InstanceEvidenceProvenance.PUBLICATION_REPLAY_COUNTS_ONLY + ): + if ( + tuple(item[0] for item in self.instance_pixel_counts) + != tuple(sorted(obj.object_id for obj in self.scene.objects)) + or self.instance_colors + ): + raise ValueError( + "adapter observation publication replay requires full counts only" + ) + elif ( + tuple(item[0] for item in self.instance_pixel_counts) + != tuple(sorted(obj.object_id for obj in self.scene.objects)) + or {item[0] for item in self.instance_colors} != positive_ids + ): + raise ValueError( + "adapter observation same-event evidence does not close scene sidecars" + ) @dataclass(frozen=True) diff --git a/src/spatialcf/generation/capture/compiler.py b/src/spatialcf/generation/capture/compiler.py index dcbf26a..f0a108c 100644 --- a/src/spatialcf/generation/capture/compiler.py +++ b/src/spatialcf/generation/capture/compiler.py @@ -50,6 +50,7 @@ SourceSurfaceEvidence, build_competition_native_camera_pose_bank_v2_9_3, competition_native_camera_pose_bank_sha256_v2_9_3, + competition_native_roster_selection_identity_v2_9, score_competition_native_camera_scene_v2_9_3, score_competition_native_editable_camera_scene_v2_9_4, select_competition_native_camera_score_index_v2_9_3, @@ -276,7 +277,9 @@ def _candidate_identity( "reference_id": reference_id, "relation_before": relation.value, "scene_id": source.scene_id, - "source_capture_sha256": capture.source_capture_sha256, + "source_capture_sha256": competition_native_roster_selection_identity_v2_9( + capture + ), "source_id": source.source_id, "split": source.split, "subject_id": subject_id, diff --git a/src/spatialcf/generation/capture/models.py b/src/spatialcf/generation/capture/models.py index f5dfadf..79593bd 100644 --- a/src/spatialcf/generation/capture/models.py +++ b/src/spatialcf/generation/capture/models.py @@ -12,7 +12,12 @@ from hashlib import sha256 from typing import Literal, Self -from pydantic import Field, model_validator +from pydantic import ( + Field, + SerializerFunctionWrapHandler, + model_serializer, + model_validator, +) from spatialcf.adapters.base import ( AdapterCameraApplication, @@ -1549,6 +1554,8 @@ def _strict_application(value: object) -> AdapterCameraApplication: pointcloud_ply=observation.pointcloud_ply, instance_pixel_counts=observation.instance_pixel_counts, is_settled=observation.is_settled, + instance_colors=observation.instance_colors, + instance_evidence_provenance=observation.instance_evidence_provenance, ) if ( rebuilt_observation != observation @@ -2067,7 +2074,7 @@ def verify_competition_native_source_camera_evidence_v2_9_3( from pydantic import Field, model_validator -from spatialcf.domain.base import CanonicalModel +from spatialcf.domain.base import CanonicalId, CanonicalModel _PATCH_HASH_DOMAIN = "spatialcf.competition-native-receptacle-surface-patch.v2.9.2" _SUBJECT_EVIDENCE_HASH_DOMAIN = ( @@ -2542,7 +2549,7 @@ def build_competition_native_source_surface_evidence_v2_9_2( from spatialcf.domain.base import CanonicalModel from spatialcf.domain.request import Relation -from spatialcf.domain.scene import SubjectPositionRegion, Vec2 +from spatialcf.domain.scene import BBox2D, SubjectPositionRegion, Vec2 from spatialcf.domain.serialization import ( canonical_json_bytes, ) @@ -3009,6 +3016,136 @@ def build_competition_native_subject_placement_fact_v2_9( ) +_SOURCE_VIEW_FACT_HASH_DOMAIN = ( + "spatialcf.competition-native-source-view-fact.v2.9.5" +) +_SOURCE_VIEW_BINDING_HASH_DOMAIN = "spatialcf.source-view-binding.v1" +_SOURCE_VIEW_SAMPLING_POLICY_SHA256 = canonical_sha256( + { + "local_quantization_m": 1e-5, + "maximum_samples": 76_800, + "render_proxy": "weighted_sampled_splat_z_buffer", + "representative": "minimum_finite_positive_depth_then_row_column", + "rounding": "nearest_even", + "tile_height": 2, + "tile_width": 2, + "weight": "object_mask_pixel_count_in_tile", + }, + domain="spatialcf.competition-native-source-view-policy.v2.9.5", +) + + +class SourceViewObjectSamples(CanonicalModel): + object_id: CanonicalId + source_mask_bbox: BBox2D + source_mask_pixel_count: int = Field(strict=True, gt=0) + sample_rows: tuple[int, ...] + sample_columns: tuple[int, ...] + sample_weights: tuple[int, ...] + local_x_quantized: tuple[int, ...] + local_y_quantized: tuple[int, ...] + local_z_quantized: tuple[int, ...] + + @model_validator(mode="after") + def validate_samples(self) -> Self: + arrays = ( + self.sample_rows, + self.sample_columns, + self.sample_weights, + self.local_x_quantized, + self.local_y_quantized, + self.local_z_quantized, + ) + if not self.sample_rows or len({len(item) for item in arrays}) != 1: + raise ValueError("source-view sample arrays must be equal and nonempty") + if any(type(value) is not int for array in arrays for value in array): + raise TypeError("source-view samples must contain exact integers") + if any(value < 0 for value in (*self.sample_rows, *self.sample_columns)): + raise ValueError("source-view sample pixels must be in bounds") + keys = tuple( + (row // 2, column // 2) + for row, column in zip( + self.sample_rows, self.sample_columns, strict=True + ) + ) + if keys != tuple(sorted(set(keys))): + raise ValueError("source-view sample tiles must be canonical") + if any(weight < 1 or weight > 4 for weight in self.sample_weights): + raise ValueError("source-view sample weights must be in [1, 4]") + if sum(self.sample_weights) != self.source_mask_pixel_count: + raise ValueError("source-view sample weights do not close mask count") + bbox_values = ( + self.source_mask_bbox.xmin, + self.source_mask_bbox.ymin, + self.source_mask_bbox.xmax, + self.source_mask_bbox.ymax, + ) + if any( + not math.isfinite(value) or value < 0 or not float(value).is_integer() + for value in bbox_values + ): + raise ValueError("source-view mask bbox must be an integer envelope") + xmin, ymin, xmax, ymax = (int(value) for value in bbox_values) + if xmin >= xmax or ymin >= ymax: + raise ValueError("source-view mask bbox must be nonempty and half-open") + if any( + row < ymin or row >= ymax or column < xmin or column >= xmax + for row, column in zip(self.sample_rows, self.sample_columns, strict=True) + ): + raise ValueError("source-view sample pixels must lie inside the mask bbox") + if self.source_mask_pixel_count > (xmax - xmin) * (ymax - ymin): + raise ValueError("source-view mask count exceeds the mask bbox area") + for row, column, weight in zip( + self.sample_rows, + self.sample_columns, + self.sample_weights, + strict=True, + ): + tile_ymin = (row // 2) * 2 + tile_xmin = (column // 2) * 2 + tile_area = max(0, min(ymax, tile_ymin + 2) - max(ymin, tile_ymin)) * max( + 0, min(xmax, tile_xmin + 2) - max(xmin, tile_xmin) + ) + if weight > tile_area: + raise ValueError("source-view sample weight exceeds its mask tile") + return self + + +class SourceViewFact(CanonicalModel): + fact_version: Literal["competition-native-source-view-fact:2.9.5"] + source_id: CanonicalId + scene_id: CanonicalId + source_locator_sha256: Sha256Digest + runtime_identity_sha256: Sha256Digest + scene_sha256: Sha256Digest + camera_sha256: Sha256Digest + rgb_png_sha256: Sha256Digest + depth_npy_sha256: Sha256Digest + instance_png_sha256: Sha256Digest + sampling_policy_sha256: Sha256Digest + objects: tuple[SourceViewObjectSamples, ...] + source_view_fact_sha256: Sha256Digest + + @model_validator(mode="after") + def validate_fact(self) -> Self: + object_ids = tuple(item.object_id for item in self.objects) + if not object_ids or object_ids != tuple(sorted(set(object_ids))): + raise ValueError("source-view object rows must be canonical") + if sum(len(item.sample_rows) for item in self.objects) > 76_800: + raise ValueError("source-view fact exceeds the sample cap") + if self.sampling_policy_sha256 != _SOURCE_VIEW_SAMPLING_POLICY_SHA256: + raise ValueError("source-view sampling policy changed") + payload = self.model_dump( + mode="python", exclude={"source_view_fact_sha256"} + ) + expected = canonical_sha256( + payload, domain=_SOURCE_VIEW_FACT_HASH_DOMAIN + ) + if self.source_view_fact_sha256 != expected: + raise ValueError("source-view fact digest mismatch") + return self + + def _capture_payload( *, source: CompetitionNativeSourceRefV2_9, @@ -3024,8 +3161,9 @@ def _capture_payload( floor_envelope: CompetitionNativeFloorEnvelopeV2_9 | None, reachable_positions: tuple[CompetitionNativePositionV2_9, ...], placement_facts: tuple[CompetitionNativeSubjectPlacementFactV2_9, ...], + source_view_fact: SourceViewFact | None = None, ) -> dict[str, object]: - return { + payload: dict[str, object] = { "depth_npy_sha256": depth_npy_sha256, "floor_envelope": None if floor_envelope is None @@ -3046,6 +3184,9 @@ def _capture_payload( "source": source.model_dump(mode="json"), "support_facts": tuple(item.model_dump(mode="json") for item in support_facts), } + if source_view_fact is not None: + payload["source_view_fact"] = source_view_fact.model_dump(mode="json") + return payload def normalize_competition_native_source_scene_v2_9(scene: Scene) -> Scene: @@ -3286,8 +3427,19 @@ class CompetitionNativeSourceCaptureV2_9(CanonicalModel): placement_facts: tuple[CompetitionNativeSubjectPlacementFactV2_9, ...] = Field( max_length=_MAX_OBJECTS_PER_SCENE ) + source_view_fact: SourceViewFact | None = None source_capture_sha256: Sha256Digest + @model_serializer(mode="wrap") + def serialize_optional_source_view_fact( + self, + handler: SerializerFunctionWrapHandler, + ) -> dict[str, object]: + payload = handler(self) + if self.source_view_fact is None: + payload.pop("source_view_fact", None) + return payload + @model_validator(mode="after") def validate_capture(self) -> Self: if self.scene.scene_id != self.source.scene_id: @@ -3343,6 +3495,76 @@ def validate_capture(self) -> Self: ) if position_keys != tuple(sorted(set(position_keys))): raise ValueError("captured reachable positions are not canonical") + if self.source_view_fact is not None: + fact = self.source_view_fact + camera = self.scene.camera_by_id("main") + if ( + fact.source_id != self.source.source_id + or fact.scene_id != self.scene.scene_id + or fact.source_locator_sha256 != self.source.source_locator_sha256 + or fact.runtime_identity_sha256 + != canonical_sha256( + self.runtime_identity, + domain=_SOURCE_VIEW_BINDING_HASH_DOMAIN, + ) + or fact.scene_sha256 + != canonical_sha256( + self.scene, domain=_SOURCE_VIEW_BINDING_HASH_DOMAIN + ) + or fact.camera_sha256 + != canonical_sha256( + camera, domain=_SOURCE_VIEW_BINDING_HASH_DOMAIN + ) + or fact.rgb_png_sha256 != self.rgb_png_sha256 + or fact.depth_npy_sha256 != self.depth_npy_sha256 + or fact.instance_png_sha256 != self.instance_png_sha256 + ): + raise ValueError("source-view fact does not bind captured evidence") + if any( + view is not None and view.camera_id != "main" + for item in self.scene.objects + for view in (item.views.get("main"),) + ): + raise ValueError("source-view fact main view camera identity changed") + expected_object_ids = tuple( + sorted( + item.object_id + for item in self.scene.objects + if (view := item.views.get("main")) is not None + and view.visible_fraction > 0.0 + ) + ) + fact_object_ids = tuple(item.object_id for item in fact.objects) + if fact_object_ids != expected_object_ids: + raise ValueError("source-view fact object roster does not bind main views") + for samples in fact.objects: + bbox = samples.source_mask_bbox + if bbox.xmax > camera.width or bbox.ymax > camera.height: + raise ValueError("source-view fact bbox exceeds main camera bounds") + if any( + row >= camera.height or column >= camera.width + for row, column in zip( + samples.sample_rows, + samples.sample_columns, + strict=True, + ) + ): + raise ValueError("source-view fact samples exceed main camera bounds") + view = self.scene.object_by_id(samples.object_id).views["main"] + if any( + not math.isclose(actual, expected, rel_tol=0.0, abs_tol=1e-6) + for actual, expected in zip( + (bbox.xmin, bbox.ymin, bbox.xmax, bbox.ymax), + ( + view.bbox.xmin, + view.bbox.ymin, + view.bbox.xmax, + view.bbox.ymax, + ), + strict=True, + ) + ): + raise ValueError("source-view fact bbox does not bind its main view") capture_payload = _capture_payload( source=self.source, runtime_identity=self.runtime_identity, @@ -3357,6 +3579,7 @@ def validate_capture(self) -> Self: floor_envelope=self.floor_envelope, reachable_positions=self.reachable_positions, placement_facts=self.placement_facts, + source_view_fact=self.source_view_fact, ) if ( len(canonical_json_bytes(capture_payload)) @@ -3387,6 +3610,7 @@ def build_competition_native_source_capture_v2_9( floor_envelope: CompetitionNativeFloorEnvelopeV2_9 | None, reachable_positions: tuple[CompetitionNativePositionV2_9, ...], placement_facts: tuple[CompetitionNativeSubjectPlacementFactV2_9, ...], + source_view_fact: SourceViewFact | None = None, ) -> CompetitionNativeSourceCaptureV2_9: normalized_scene = normalize_competition_native_source_scene_v2_9(scene) support_facts = tuple(sorted(support_facts, key=lambda item: item.object_id)) @@ -3408,6 +3632,7 @@ def build_competition_native_source_capture_v2_9( floor_envelope=floor_envelope, reachable_positions=reachable_positions, placement_facts=placement_facts, + source_view_fact=source_view_fact, ) return CompetitionNativeSourceCaptureV2_9( source=source, @@ -3423,12 +3648,41 @@ def build_competition_native_source_capture_v2_9( floor_envelope=floor_envelope, reachable_positions=reachable_positions, placement_facts=placement_facts, + source_view_fact=source_view_fact, source_capture_sha256=canonical_sha256( payload, domain=_SOURCE_CAPTURE_HASH_DOMAIN ), ) +def competition_native_roster_selection_identity_v2_9( + capture: CompetitionNativeSourceCaptureV2_9, +) -> Sha256Digest: + """Return the source identity used only for deterministic roster selection.""" + + if type(capture) is not CompetitionNativeSourceCaptureV2_9: + raise TypeError("roster selection identity requires an exact source capture") + if capture.source_view_fact is None: + return capture.source_capture_sha256 + payload = _capture_payload( + source=capture.source, + runtime_identity=capture.runtime_identity, + scene=capture.scene, + rgb_png_sha256=capture.rgb_png_sha256, + depth_npy_sha256=capture.depth_npy_sha256, + instance_png_sha256=capture.instance_png_sha256, + pointcloud_ply_sha256=capture.pointcloud_ply_sha256, + is_scene_at_rest=capture.is_scene_at_rest, + settlement_pass_steps=capture.settlement_pass_steps, + support_facts=capture.support_facts, + floor_envelope=capture.floor_envelope, + reachable_positions=capture.reachable_positions, + placement_facts=capture.placement_facts, + source_view_fact=None, + ) + return canonical_sha256(payload, domain=_SOURCE_CAPTURE_HASH_DOMAIN) + + class CompetitionNativeSourceCaptureOutcomeV2_9(CanonicalModel): source: CompetitionNativeSourceRefV2_9 status: Literal["accepted", "rejected"] diff --git a/src/spatialcf/generation/capture/visual_evidence.py b/src/spatialcf/generation/capture/visual_evidence.py new file mode 100644 index 0000000..02ea329 --- /dev/null +++ b/src/spatialcf/generation/capture/visual_evidence.py @@ -0,0 +1,293 @@ +"""Build compact, platform-neutral source-view facts from captured raster bytes.""" + +from __future__ import annotations + +import math +from io import BytesIO + +import numpy as np +from PIL import Image + +from spatialcf.adapters.base import AdapterObservation +from spatialcf.domain.scene import BBox2D, Scene +from spatialcf.domain.serialization import canonical_sha256 +from spatialcf.generation.capture.models import ( + _SOURCE_VIEW_SAMPLING_POLICY_SHA256, + CompetitionNativeRuntimeIdentityV2_9, + CompetitionNativeSourceRefV2_9, + SourceViewFact, + SourceViewObjectSamples, +) + +SOURCE_VIEW_FACT_VERSION = "competition-native-source-view-fact:2.9.5" +SOURCE_VIEW_SAMPLE_CAP = 76_800 +SOURCE_VIEW_QUANTIZATION_M = 1e-5 +_FACT_DOMAIN = "spatialcf.competition-native-source-view-fact.v2.9.5" +SOURCE_VIEW_SAMPLING_POLICY_SHA256 = _SOURCE_VIEW_SAMPLING_POLICY_SHA256 + + +class SourceViewFactError(ValueError): + """Typed fail-closed source-view derivation error.""" + + +def _sha(value: object) -> str: + return canonical_sha256(value, domain="spatialcf.source-view-binding.v1") + + +def _decode_instance(payload: bytes, *, width: int, height: int) -> np.ndarray: + try: + with Image.open(BytesIO(payload)) as image: + image.load() + if image.mode != "RGB" or image.size != (width, height): + raise SourceViewFactError("source instance PNG dimensions or mode changed") + return np.asarray(image, dtype=np.uint8) + except SourceViewFactError: + raise + except (OSError, ValueError) as error: + raise SourceViewFactError("source instance PNG is invalid") from error + + +def _decode_depth(payload: bytes, *, width: int, height: int) -> np.ndarray: + try: + depth = np.load(BytesIO(payload), allow_pickle=False) + except (OSError, ValueError) as error: + raise SourceViewFactError("source depth NPY is invalid") from error + if type(depth) is not np.ndarray: + if isinstance(depth, np.lib.npyio.NpzFile): + depth.close() + raise SourceViewFactError( + "source depth NPY must contain a single float32 ndarray" + ) + if depth.shape != (height, width) or depth.dtype != np.float32: + raise SourceViewFactError("source depth NPY dimensions or dtype changed") + return depth + + +def _validated_instance_counts(value: object) -> dict[str, int]: + if type(value) is not tuple: + raise SourceViewFactError("source-view counts must be an exact tuple") + for item in value: + if ( + type(item) is not tuple + or len(item) != 2 + or type(item[0]) is not str + or not item[0] + or type(item[1]) is not int + or item[1] < 0 + ): + raise SourceViewFactError("source-view count is not an exact pair") + if value != tuple(sorted(value)): + raise SourceViewFactError("source-view counts are not canonical") + if len(value) != len({item[0] for item in value}): + raise SourceViewFactError("source-view counts contain duplicate objects") + return dict(value) + + +def _validated_instance_colors(value: object) -> dict[str, tuple[int, int, int]]: + if type(value) is not tuple: + raise SourceViewFactError("source-view colors must be an exact tuple") + for item in value: + if ( + type(item) is not tuple + or len(item) != 2 + or type(item[0]) is not str + or not item[0] + ): + raise SourceViewFactError("source-view color is not an exact pair") + color = item[1] + if ( + type(color) is not tuple + or len(color) != 3 + or any( + type(channel) is not int or not 0 <= channel <= 255 + for channel in color + ) + ): + raise SourceViewFactError("source-view color is not exact RGB") + if value != tuple(sorted(value)): + raise SourceViewFactError("source-view colors are not canonical") + if len(value) != len({item[0] for item in value}): + raise SourceViewFactError("source-view colors contain duplicate objects") + if len(value) != len({item[1] for item in value}): + raise SourceViewFactError("source-view colors reuse an RGB value") + return dict(value) + + +def _quantize_local_coordinate(value: float) -> int: + if not math.isfinite(value): + raise SourceViewFactError("source-view local coordinate is not finite") + return round(value / SOURCE_VIEW_QUANTIZATION_M) + + +def build_source_view_fact( + source: CompetitionNativeSourceRefV2_9, + runtime_identity: CompetitionNativeRuntimeIdentityV2_9, + scene: Scene, + observation: AdapterObservation, +) -> SourceViewFact: + """Derive one deterministic weighted 2x2-tile source-view fact.""" + if type(source) is not CompetitionNativeSourceRefV2_9: + raise SourceViewFactError("source-view source must be exact") + if type(runtime_identity) is not CompetitionNativeRuntimeIdentityV2_9: + raise SourceViewFactError("source-view runtime must be exact") + if type(scene) is not Scene or type(observation) is not AdapterObservation: + raise SourceViewFactError("source-view scene and observation must be exact") + if observation.scene != scene or scene.scene_id != source.scene_id: + raise SourceViewFactError("source-view observation does not bind the scene") + camera = scene.camera_by_id("main") + if (camera.width, camera.height) != ( + runtime_identity.width, + runtime_identity.height, + ): + raise SourceViewFactError("source-view runtime dimensions changed") + instance = _decode_instance( + observation.instance_png, width=camera.width, height=camera.height + ) + depth = _decode_depth( + observation.depth_npy, width=camera.width, height=camera.height + ) + counts = _validated_instance_counts(observation.instance_pixel_counts) + colors = _validated_instance_colors(observation.instance_colors) + required = {object_id for object_id, count in counts.items() if count > 0} + if set(colors) != required: + raise SourceViewFactError("source-view colors do not cover visible objects") + try: + camera_to_world = np.linalg.inv( + np.asarray(camera.world_to_camera, dtype=np.float64).reshape(4, 4) + ) + except np.linalg.LinAlgError as error: + raise SourceViewFactError("source-view camera matrix is singular") from error + fx, fy = camera.intrinsics[0], camera.intrinsics[4] + cx, cy = camera.intrinsics[2], camera.intrinsics[5] + if not all(math.isfinite(value) for value in (fx, fy, cx, cy)) or fx <= 0 or fy <= 0: + raise SourceViewFactError("source-view camera intrinsics are invalid") + + rows: list[SourceViewObjectSamples] = [] + total_samples = 0 + for object_id in sorted(required): + try: + obj = scene.object_by_id(object_id) + except KeyError as error: + raise SourceViewFactError( + "source-view evidence names an unknown object" + ) from error + view = obj.views.get("main") + if view is None: + raise SourceViewFactError("visible source object has no main view") + if view.camera_id != "main": + raise SourceViewFactError( + "source-view main view camera identity changed" + ) + raw_color = colors[object_id] + if ( + type(raw_color) is not tuple + or len(raw_color) != 3 + or any(type(channel) is not int or not 0 <= channel <= 255 for channel in raw_color) + ): + raise SourceViewFactError("source-view color is not exact RGB") + color = np.asarray(raw_color, dtype=np.uint8) + mask = np.all(instance == color, axis=2) + pixel_count = int(np.count_nonzero(mask)) + if pixel_count != counts[object_id]: + raise SourceViewFactError("source-view mask pixel count changed") + occupied = np.argwhere(mask) + if occupied.size == 0: + raise SourceViewFactError("source-view object mask is empty") + ymin, xmin = occupied.min(axis=0) + ymax, xmax = occupied.max(axis=0) + bbox = BBox2D( + xmin=float(xmin), + ymin=float(ymin), + xmax=float(xmax + 1), + ymax=float(ymax + 1), + ) + if any( + not math.isclose(actual, expected, rel_tol=0.0, abs_tol=1e-6) + for actual, expected in zip( + (view.bbox.xmin, view.bbox.ymin, view.bbox.xmax, view.bbox.ymax), + (bbox.xmin, bbox.ymin, bbox.xmax, bbox.ymax), + strict=True, + ) + ): + raise SourceViewFactError("source-view detection and mask bbox differ") + tile_keys = sorted({(int(row) // 2, int(column) // 2) for row, column in occupied}) + total_samples += len(tile_keys) + if total_samples > SOURCE_VIEW_SAMPLE_CAP: + raise SourceViewFactError("source-view sample cap exceeded") + sample_rows: list[int] = [] + sample_columns: list[int] = [] + sample_weights: list[int] = [] + local_x: list[int] = [] + local_y: list[int] = [] + local_z: list[int] = [] + for tile_row, tile_column in tile_keys: + tile = occupied[ + (occupied[:, 0] // 2 == tile_row) + & (occupied[:, 1] // 2 == tile_column) + ] + candidates = [ + (float(depth[row, column]), int(row), int(column)) + for row, column in tile + if math.isfinite(float(depth[row, column])) + and float(depth[row, column]) > 0.0 + ] + if not candidates: + raise SourceViewFactError("occupied source tile has no valid depth") + source_depth, row, column = min(candidates) + camera_point = np.asarray( + [ + (column - cx) * source_depth / fx, + (cy - row) * source_depth / fy, + source_depth, + 1.0, + ], + dtype=np.float64, + ) + world = camera_to_world @ camera_point + if not np.all(np.isfinite(world)) or world[3] == 0.0: + raise SourceViewFactError("source-view back-projection is invalid") + world = world[:3] / world[3] + local = ( + float(world[0]) - obj.position.x, + float(world[1]) - obj.position.y, + float(world[2]) - obj.position.z, + ) + sample_rows.append(row) + sample_columns.append(column) + sample_weights.append(len(tile)) + local_x.append(_quantize_local_coordinate(local[0])) + local_y.append(_quantize_local_coordinate(local[1])) + local_z.append(_quantize_local_coordinate(local[2])) + rows.append( + SourceViewObjectSamples( + object_id=object_id, + source_mask_bbox=bbox, + source_mask_pixel_count=pixel_count, + sample_rows=tuple(sample_rows), + sample_columns=tuple(sample_columns), + sample_weights=tuple(sample_weights), + local_x_quantized=tuple(local_x), + local_y_quantized=tuple(local_y), + local_z_quantized=tuple(local_z), + ) + ) + if not rows: + raise SourceViewFactError("source-view fact has no visible objects") + payload = { + "fact_version": SOURCE_VIEW_FACT_VERSION, + "source_id": source.source_id, + "scene_id": scene.scene_id, + "source_locator_sha256": source.source_locator_sha256, + "runtime_identity_sha256": _sha(runtime_identity), + "scene_sha256": _sha(scene), + "camera_sha256": _sha(camera), + "rgb_png_sha256": observation.rgb_png_sha256, + "depth_npy_sha256": observation.depth_npy_sha256, + "instance_png_sha256": observation.instance_png_sha256, + "sampling_policy_sha256": SOURCE_VIEW_SAMPLING_POLICY_SHA256, + "objects": tuple(rows), + } + return SourceViewFact( + **payload, + source_view_fact_sha256=canonical_sha256(payload, domain=_FACT_DOMAIN), + ) diff --git a/src/spatialcf/generation/execution/audit.py b/src/spatialcf/generation/execution/audit.py index 0aee97f..938081b 100644 --- a/src/spatialcf/generation/execution/audit.py +++ b/src/spatialcf/generation/execution/audit.py @@ -1432,6 +1432,8 @@ def _observation_with_scene( instance_png=observation.instance_png, pointcloud_ply=observation.pointcloud_ply, instance_pixel_counts=observation.instance_pixel_counts, + instance_colors=observation.instance_colors, + instance_evidence_provenance=observation.instance_evidence_provenance, is_settled=observation.is_settled, ) @@ -1445,6 +1447,8 @@ def _observation_asset_identity( observation.instance_png, observation.pointcloud_ply, observation.instance_pixel_counts, + observation.instance_colors, + observation.instance_evidence_provenance, observation.is_settled, ) diff --git a/src/spatialcf/generation/execution/campaign.py b/src/spatialcf/generation/execution/campaign.py index 03e8ffa..2193fff 100644 --- a/src/spatialcf/generation/execution/campaign.py +++ b/src/spatialcf/generation/execution/campaign.py @@ -168,8 +168,12 @@ def _batch_lineage( placement_sha256=outcome.placement_sha256, surface_evidence_sha256=outcome.surface_evidence_sha256, subject_surface_evidence_sha256=(outcome.subject_surface_evidence_sha256), + semantic_problem_sha256=outcome.semantic_problem_sha256, proxy_bundle_sha256=outcome.proxy_bundle_sha256, solve_result_sha256=outcome.solve_result_sha256, + selected_edit_sha256=outcome.selected_edit_sha256, + source_view_fact_sha256=outcome.source_view_fact_sha256, + source_view_guard=outcome.source_view_guard, runtime_collision_delegated_native_object_ids=( outcome.runtime_collision_delegated_native_object_ids ), diff --git a/src/spatialcf/generation/planning/campaign.py b/src/spatialcf/generation/planning/campaign.py index 49ba8f7..d7c37e5 100644 --- a/src/spatialcf/generation/planning/campaign.py +++ b/src/spatialcf/generation/planning/campaign.py @@ -63,6 +63,7 @@ EndpointPlan, EndpointWorkspace, ProxyBundle, + SourceViewGuard, SubjectPlacementFact, ) from spatialcf.generation.planning.problem import ( @@ -70,6 +71,7 @@ default_planning_workspace, default_solver_config, ) +from spatialcf.generation.planning.view_guard import evaluate_source_view_guard from spatialcf.verification.filesystem import ( CompetitionNativePublicationError, DirectoryIdentity, @@ -85,7 +87,7 @@ from spatialcf.verification.integrity import competition_legacy_sha256 _POLICY_DOMAIN = "spatialcf.competition-native-source-policy.v2.9.13" -_PLAN_DOMAIN = "spatialcf.competition-native-source-plan.v2.9.9" +_PLAN_DOMAIN = "spatialcf.competition-native-source-plan.v2.9.10" _TARGET_LEDGER_DOMAIN = ( "spatialcf.competition-native-source-target-reachability-ledger.v2.9.8" ) @@ -95,7 +97,7 @@ ) _RUNTIME_POSE_POLICY_DOMAIN = "spatialcf.competition-native-runtime-pose-policy.v2.9.5" _SOURCE_POLICY_VERSION = "competition-native-source-policy:2.9.13" -_SOURCE_PLAN_VERSION = "competition-native-source-plan:2.9.9" +_SOURCE_PLAN_VERSION = "competition-native-source-plan:2.9.10" _FILES = {"plan.json", "checksums.sha256"} _MAX_PLAN_BYTES = 256 * 1024 * 1024 _MAX_POLICY_BYTES = 16 * 1024 * 1024 @@ -256,6 +258,9 @@ class SourceRequestOutcome(CanonicalModel): attempted_workspace_count: int | None = Field(default=None, strict=True, gt=0) candidate_point_count: int | None = Field(default=None, strict=True, gt=0) solve_result_sha256: Sha256Digest | None + selected_edit_sha256: Sha256Digest | None = None + source_view_fact_sha256: Sha256Digest | None = None + source_view_guard: SourceViewGuard | None = None reasons: tuple[str, ...] = Field(max_length=32) surface_evidence_sha256: Sha256Digest | None = None subject_surface_evidence_sha256: Sha256Digest | None = None @@ -281,6 +286,9 @@ def validate_outcome(self) -> Self: self.attempted_workspace_count, self.candidate_point_count, self.solve_result_sha256, + self.selected_edit_sha256, + self.source_view_fact_sha256, + self.source_view_guard, ) if self.status == "planned": if any(item is None for item in metrics) or self.reasons: @@ -305,6 +313,17 @@ def validate_patch_lineage(self) -> Self: if self.status == "planned": if any(item is None for item in lineage): raise ValueError("planned patch-bound lineage is not closed") + if ( + self.source_view_guard.status != "PASSED" + or self.source_view_guard.source_view_fact_sha256 + != self.source_view_fact_sha256 + or self.source_view_guard.semantic_problem_sha256 + != self.semantic_problem_sha256 + or self.source_view_guard.solve_result_sha256 + != self.solve_result_sha256 + or self.source_view_guard.edit_sha256 != self.selected_edit_sha256 + ): + raise ValueError("planned source-view guard lineage is not closed") elif any(item is not None for item in lineage): raise ValueError("rejected patch-bound lineage must be empty") return self @@ -446,14 +465,14 @@ def _workspace_is_subset(inner: EndpointWorkspace, outer: EndpointWorkspace) -> ) -def _fresh_solve_matches( +def _fresh_solve_result( proxy: ProxyBundle, config: ContinuousYawSolverConfigV2_9, expected_solve_result_sha256: Sha256Digest, -) -> bool: +) -> ContinuousYawCertifiedSuccessResultV2_9 | None: solved = solve_minimum_cost(proxy.semantic_problem, config) result = solved.result - return bool( + if ( type(result) is ContinuousYawCertifiedSuccessResultV2_9 and result.semantic_problem_sha256 == proxy.semantic_problem.semantic_problem_sha256 @@ -461,7 +480,9 @@ def _fresh_solve_matches( and result.solver_config == config and result.solver_config.config_sha256 == config.config_sha256 and result.solve_result_sha256 == expected_solve_result_sha256 - ) + ): + return result + return None class SourcePlan(CanonicalModel): @@ -729,6 +750,9 @@ def validate_patch_bound_plan(self) -> Self: or outcome.patch_index >= len(subject_evidence.patches) or outcome.patch_sha256 != subject_evidence.patches[outcome.patch_index].patch_sha256 + or capture.source_view_fact is None + or outcome.source_view_fact_sha256 + != capture.source_view_fact.source_view_fact_sha256 ): raise ValueError("planned patch-bound evidence lineage changed") endpoint = EndpointPlan( @@ -745,8 +769,12 @@ def validate_patch_bound_plan(self) -> Self: subject_surface_evidence_sha256=( outcome.subject_surface_evidence_sha256 ), + semantic_problem_sha256=outcome.semantic_problem_sha256, proxy_bundle_sha256=outcome.proxy_bundle_sha256, solve_result_sha256=outcome.solve_result_sha256, + selected_edit_sha256=outcome.selected_edit_sha256, + source_view_fact_sha256=outcome.source_view_fact_sha256, + source_view_guard=outcome.source_view_guard, runtime_collision_delegated_native_object_ids=( outcome.runtime_collision_delegated_native_object_ids ), @@ -781,10 +809,26 @@ def validate_patch_bound_plan(self) -> Self: != proxy.binding.runtime_collision_delegated_native_object_ids ): raise ValueError("planned patch-bound proxy lineage changed") - if not _fresh_solve_matches( + fresh_result = _fresh_solve_result( proxy, self.source_policy.solver_config, outcome.solve_result_sha256 - ): + ) + if fresh_result is None: raise ValueError("planned patch-bound fresh solve lineage changed") + if ( + outcome.selected_edit_sha256 + != fresh_result.selected_witness.edit.edit_sha256 + ): + raise ValueError("planned selected edit lineage changed") + replayed_guard = evaluate_source_view_guard( + capture.scene, + intervention, + capture.source_view_fact, + fresh_result.selected_witness.edit, + semantic_problem_sha256=proxy.semantic_problem.semantic_problem_sha256, + solve_result_sha256=fresh_result.solve_result_sha256, + ) + if replayed_guard != outcome.source_view_guard: + raise ValueError("planned source-view guard replay changed") return self @model_validator(mode="after") @@ -944,6 +988,11 @@ def build_default_source_policy(compilation: RosterCompilation) -> SourcePolicy: def _endpoint_rejection_reason(reason: str) -> str: + if reason in { + "endpoint_plan:SOURCE_VIEW_MISSING", + "endpoint_plan:SOURCE_VIEW_UNCERTIFIED", + }: + return reason candidate = f"source_plan:endpoint:{reason}" if len(candidate) <= _MAX_REASON_CHARS: return candidate @@ -998,6 +1047,9 @@ def _rejected_outcome( attempted_workspace_count=None, candidate_point_count=None, solve_result_sha256=None, + selected_edit_sha256=None, + source_view_fact_sha256=None, + source_view_guard=None, reasons=tuple(sorted(set(reasons))), ) @@ -1022,6 +1074,8 @@ def _preflight_request( case_id: str, ) -> str | None: _endpoint_policy_controls(policy) + if capture.source_view_fact is None: + return "endpoint_plan:SOURCE_VIEW_MISSING" runtime = capture.runtime_identity if runtime.native_scene_name == "Procedural": return "source_plan:procedural_native_audit_unsupported" @@ -1106,13 +1160,22 @@ def _prepare_job( camera_evidence_sha256: Sha256Digest, ) -> _PlanningJob | SourceRequestOutcome: case_id = _case_id(policy, request) - verified_evidence = verify_source_surface_evidence(capture, source_surface_evidence) placements = tuple( item for item in capture.placement_facts if item.object_id == request.subject_id ) if len(placements) != 1: raise ValueError("fresh roster selected a non-unique placement fact") placement_fact = placements[0] + if capture.source_view_fact is None: + return _rejected_outcome( + request, + slot, + case_id, + "endpoint_plan:SOURCE_VIEW_MISSING", + capture=capture, + placement=placement_fact, + ) + verified_evidence = verify_source_surface_evidence(capture, source_surface_evidence) subjects = tuple( item for item in verified_evidence.subjects @@ -1185,7 +1248,7 @@ def _prepare_job( def _endpoint_worker(connection: Connection, arguments: tuple[object, ...]) -> None: try: - if len(arguments) != 11: + if len(arguments) != 12: connection.send(("error", "InvalidWorkerArguments")) return ( @@ -1195,6 +1258,7 @@ def _endpoint_worker(connection: Connection, arguments: tuple[object, ...]) -> N config, source_surface_evidence, subject_surface_evidence, + source_view_fact, placement, case_id, max_candidate_points, @@ -1209,6 +1273,7 @@ def _endpoint_worker(connection: Connection, arguments: tuple[object, ...]) -> N config, source_surface_evidence, subject_surface_evidence, + source_view_fact, placement=placement, case_id=case_id, max_candidate_points=max_candidate_points, @@ -1235,6 +1300,7 @@ def _bounded_default_endpoint_plan( policy.solver_config, job.source_surface_evidence, job.subject_surface_evidence, + job.capture.source_view_fact, job.placement, job.case_id, job.max_candidate_points, @@ -1313,6 +1379,7 @@ def _run_job( policy.solver_config, job.source_surface_evidence, job.subject_surface_evidence, + job.capture.source_view_fact, placement=job.placement, case_id=job.case_id, max_candidate_points=job.max_candidate_points, @@ -1364,6 +1431,12 @@ def _run_job( or endpoint.candidate_point_count > policy.max_endpoint_candidate_points or endpoint.attempted_workspace_count > 4 * endpoint.candidate_point_count or endpoint.source_capture_sha256 != job.capture.source_capture_sha256 + or job.capture.source_view_fact is None + or endpoint.source_view_fact_sha256 + != job.capture.source_view_fact.source_view_fact_sha256 + or endpoint.semantic_problem_sha256 + != endpoint.source_view_guard.semantic_problem_sha256 + or endpoint.selected_edit_sha256 != endpoint.source_view_guard.edit_sha256 or endpoint.placement_sha256 != job.placement_fact.placement_sha256 or endpoint.surface_evidence_sha256 != job.source_surface_evidence.surface_evidence_sha256 @@ -1431,9 +1504,10 @@ def _run_job( capture=job.capture, placement=job.placement_fact, ) - if not _fresh_solve_matches( + fresh_result = _fresh_solve_result( proxy, policy.solver_config, endpoint.solve_result_sha256 - ): + ) + if fresh_result is None: return _rejected_outcome( request, job.slot, @@ -1442,6 +1516,37 @@ def _run_job( capture=job.capture, placement=job.placement_fact, ) + if ( + endpoint.semantic_problem_sha256 + != proxy.semantic_problem.semantic_problem_sha256 + or endpoint.selected_edit_sha256 + != fresh_result.selected_witness.edit.edit_sha256 + ): + return _rejected_outcome( + request, + job.slot, + job.case_id, + "source_plan:endpoint_identity_mismatch", + capture=job.capture, + placement=job.placement_fact, + ) + replayed_guard = evaluate_source_view_guard( + job.scene, + job.intervention, + job.capture.source_view_fact, + fresh_result.selected_witness.edit, + semantic_problem_sha256=proxy.semantic_problem.semantic_problem_sha256, + solve_result_sha256=fresh_result.solve_result_sha256, + ) + if replayed_guard != endpoint.source_view_guard: + return _rejected_outcome( + request, + job.slot, + job.case_id, + "source_plan:endpoint_guard_mismatch", + capture=job.capture, + placement=job.placement_fact, + ) try: BatchRequest( request_id=request.request_id, @@ -1490,6 +1595,9 @@ def _run_job( attempted_workspace_count=endpoint.attempted_workspace_count, candidate_point_count=endpoint.candidate_point_count, solve_result_sha256=endpoint.solve_result_sha256, + selected_edit_sha256=endpoint.selected_edit_sha256, + source_view_fact_sha256=endpoint.source_view_fact_sha256, + source_view_guard=endpoint.source_view_guard, reasons=(), surface_evidence_sha256=(job.source_surface_evidence.surface_evidence_sha256), subject_surface_evidence_sha256=( diff --git a/src/spatialcf/generation/planning/endpoint.py b/src/spatialcf/generation/planning/endpoint.py index fd79027..0fd8491 100644 --- a/src/spatialcf/generation/planning/endpoint.py +++ b/src/spatialcf/generation/planning/endpoint.py @@ -31,6 +31,7 @@ from spatialcf.generation.capture.models import ( ReceptacleSurfacePatch, SourceSurfaceEvidence, + SourceViewFact, SubjectSurfaceEvidence, ) from spatialcf.generation.capture.reachability import ( @@ -47,6 +48,7 @@ _project_proxy, _project_proxy_problem, ) +from spatialcf.generation.planning.view_guard import evaluate_source_view_guard _DEFAULT_RADII_M = (0.02, 0.01, 0.005, 0.001) _NATIVE_SPAWN_RADIUS_M = 0.000001 @@ -608,6 +610,7 @@ def plan_endpoint( config: ContinuousYawSolverConfigV2_9, source_surface_evidence: SourceSurfaceEvidence, subject_surface_evidence: SubjectSurfaceEvidence, + source_view_fact: SourceViewFact | None, *, placement: SubjectPlacementFact, case_id: str, @@ -617,6 +620,11 @@ def plan_endpoint( ) -> EndpointPlan: """Return the first current solver-certified source-only endpoint.""" + if source_view_fact is None: + raise EndpointPlanRejected(("endpoint_plan:SOURCE_VIEW_MISSING",)) + if type(source_view_fact) is not SourceViewFact: + raise TypeError("source_view_fact must be exact") + if type(scene) is not Scene or type(intervention) is not InterventionSpec: raise TypeError("scene and intervention must be exact legacy values") if type(workspace) is not EndpointWorkspace: @@ -767,6 +775,17 @@ def plan_endpoint( if solved_mismatch is not None: reasons.add(solved_mismatch) continue + guard = evaluate_source_view_guard( + scene, + intervention, + source_view_fact, + solved.result.selected_witness.edit, + semantic_problem_sha256=proxy.semantic_problem.semantic_problem_sha256, + solve_result_sha256=solved.result.solve_result_sha256, + ) + if guard.status != "PASSED": + reasons.add("endpoint_plan:SOURCE_VIEW_UNCERTIFIED") + continue return EndpointPlan( planning_workspace=workspace, endpoint_workspace=endpoint_workspace, @@ -783,15 +802,21 @@ def plan_endpoint( subject_surface_evidence_sha256=( subject_surface_evidence.subject_surface_evidence_sha256 ), + semantic_problem_sha256=proxy.semantic_problem.semantic_problem_sha256, proxy_bundle_sha256=proxy.proxy_bundle_sha256, solve_result_sha256=solved.result.solve_result_sha256, + selected_edit_sha256=( + solved.result.selected_witness.edit.edit_sha256 + ), + source_view_fact_sha256=source_view_fact.source_view_fact_sha256, + source_view_guard=guard, runtime_collision_delegated_native_object_ids=( proxy.binding.runtime_collision_delegated_native_object_ids ), ) - raise EndpointPlanRejected( - tuple(reasons) or ("endpoint_plan:no_certified_single_patch_workspace",) - ) + if "endpoint_plan:SOURCE_VIEW_UNCERTIFIED" in reasons: + raise EndpointPlanRejected(("endpoint_plan:SOURCE_VIEW_UNCERTIFIED",)) + raise EndpointPlanRejected(tuple(reasons) or ("endpoint_plan:no_certified_single_patch_workspace",)) __all__ = ("EndpointPlanRejected", "plan_endpoint") diff --git a/src/spatialcf/generation/planning/models.py b/src/spatialcf/generation/planning/models.py index 8498216..4d61d41 100644 --- a/src/spatialcf/generation/planning/models.py +++ b/src/spatialcf/generation/planning/models.py @@ -7,10 +7,11 @@ from dataclasses import dataclass from typing import Literal, Self -from pydantic import model_validator +from pydantic import Field, model_validator from spatialcf.domain.base import CanonicalId, CanonicalModel, FiniteFloat, Sha256Digest from spatialcf.domain.problem import SemanticProblemV2_3 +from spatialcf.domain.request import Relation from spatialcf.domain.serialization import canonical_sha256 from spatialcf.generation.capture.models import ( CompetitionNativePlacementAvailabilityV2_9, @@ -19,10 +20,14 @@ CompetitionNativeSupportKindV2_9, ReceptacleSurfacePatch, ) +from spatialcf.relations.engine import RelationEngine _BINDING_HASH_DOMAIN = "spatialcf.competition-native-proxy-binding.v2.9.4" _BUNDLE_HASH_DOMAIN = "spatialcf.competition-native-proxy-bundle.v2.9.4" -_ENDPOINT_PLAN_HASH_DOMAIN = "spatialcf.competition-native-endpoint-plan.v2.9.4" +_ENDPOINT_PLAN_HASH_DOMAIN = "spatialcf.competition-native-endpoint-plan.v2.9.5" +_SOURCE_VIEW_GUARD_HASH_DOMAIN = ( + "spatialcf.competition-native-source-view-guard.v2.9.5" +) _NON_SUPPORT_COLLISION_MARGIN_M = 0.01 _PROXY_POLICY_SHA256 = hashlib.sha256( b"spatialcf.competition-native-scene-proxy.v2.9.4\0" @@ -43,6 +48,98 @@ def _require_sha256(value: object, label: str) -> str: return value +class SourceViewObjectProxy(CanonicalModel): + object_id: CanonicalId + bbox_center_x_lower: FiniteFloat + bbox_center_x_upper: FiniteFloat + camera_depth_lower_m: FiniteFloat + camera_depth_upper_m: FiniteFloat + image_area_fraction_lower: FiniteFloat = Field(ge=0.0, le=1.0) + visible_fraction_lower: FiniteFloat = Field(ge=0.0, le=1.0) + truncated_fraction_upper: FiniteFloat = Field(ge=0.0, le=1.0) + + @model_validator(mode="after") + def validate_intervals(self) -> Self: + if self.bbox_center_x_lower > self.bbox_center_x_upper: + raise ValueError("source-view bbox-center interval is unordered") + if self.camera_depth_lower_m > self.camera_depth_upper_m: + raise ValueError("source-view camera-depth interval is unordered") + return self + + +class SourceViewGuard(CanonicalModel): + guard_version: Literal["competition-native-source-view-guard:2.9.5"] + status: Literal["PASSED", "UNCERTIFIED"] + source_view_fact_sha256: Sha256Digest + semantic_problem_sha256: Sha256Digest + solve_result_sha256: Sha256Digest + edit_sha256: Sha256Digest + subject: SourceViewObjectProxy + reference: SourceViewObjectProxy + target_relation: Relation + target_satisfied: bool = Field(strict=True) + old_relation_satisfied: bool = Field(strict=True) + reasons: tuple[str, ...] + source_view_guard_sha256: Sha256Digest + + @model_validator(mode="after") + def validate_guard(self) -> Self: + if self.subject.object_id == self.reference.object_id: + raise ValueError("source-view guard objects must be distinct") + fallback_values = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0) + has_fallback = any( + ( + proxy.bbox_center_x_lower, + proxy.bbox_center_x_upper, + proxy.camera_depth_lower_m, + proxy.camera_depth_upper_m, + proxy.image_area_fraction_lower, + proxy.visible_fraction_lower, + proxy.truncated_fraction_upper, + ) + == fallback_values + for proxy in (self.subject, self.reference) + ) + if has_fallback and ( + self.status != "UNCERTIFIED" + or not any( + reason.startswith(("projection:", "coverage:")) + for reason in self.reasons + ) + ): + raise ValueError("source-view fallback requires a coverage reason") + if self.status == "PASSED": + visible = all( + proxy.visible_fraction_lower >= RelationEngine.MIN_VISIBLE_FRACTION + and proxy.image_area_fraction_lower + >= RelationEngine.MIN_IMAGE_AREA_FRACTION + and proxy.truncated_fraction_upper + <= RelationEngine.MAX_TRUNCATED_FRACTION + for proxy in (self.subject, self.reference) + ) + if ( + self.reasons + or not visible + or not self.target_satisfied + or self.old_relation_satisfied + ): + raise ValueError("passed source-view guard is not admissible") + elif ( + not self.reasons + or self.reasons != tuple(sorted(set(self.reasons))) + or any(type(reason) is not str or not reason for reason in self.reasons) + ): + raise ValueError("uncertified source-view reasons must be canonical") + payload = self.model_dump( + mode="python", exclude={"source_view_guard_sha256"} + ) + if self.source_view_guard_sha256 != canonical_sha256( + payload, domain=_SOURCE_VIEW_GUARD_HASH_DOMAIN + ): + raise ValueError("source-view guard digest mismatch") + return self + + class EndpointWorkspace(CanonicalModel): """Closed absolute world-XY endpoint policy for one native request.""" @@ -366,15 +463,15 @@ def proxy_bundle_sha256(self) -> Sha256Digest: class EndpointPlan(CanonicalModel): """One current solver-certified endpoint wholly owned by a source patch.""" - plan_version: Literal["competition-native-endpoint-plan:2.9.4"] = ( - "competition-native-endpoint-plan:2.9.4" + plan_version: Literal["competition-native-endpoint-plan:2.9.5"] = ( + "competition-native-endpoint-plan:2.9.5" ) candidate_strategy: Literal[ "stratified_directional_receptacle_patch_relation_ranked_" - "runtime_collision_delegated_bbox_visibility" + "runtime_collision_delegated_bbox_visibility_sampled_source_view_guard" ] = ( "stratified_directional_receptacle_patch_relation_ranked_" - "runtime_collision_delegated_bbox_visibility" + "runtime_collision_delegated_bbox_visibility_sampled_source_view_guard" ) planning_workspace: EndpointWorkspace endpoint_workspace: EndpointWorkspace @@ -387,8 +484,12 @@ class EndpointPlan(CanonicalModel): placement_sha256: str surface_evidence_sha256: str subject_surface_evidence_sha256: str + semantic_problem_sha256: Sha256Digest proxy_bundle_sha256: str solve_result_sha256: str + selected_edit_sha256: Sha256Digest + source_view_fact_sha256: Sha256Digest + source_view_guard: SourceViewGuard runtime_collision_delegated_native_object_ids: tuple[CanonicalId, ...] @model_validator(mode="after") @@ -415,10 +516,26 @@ def validate_plan(self) -> Self: ("placement", self.placement_sha256), ("surface evidence", self.surface_evidence_sha256), ("subject surface evidence", self.subject_surface_evidence_sha256), + ("semantic problem", self.semantic_problem_sha256), ("proxy bundle", self.proxy_bundle_sha256), ("solve result", self.solve_result_sha256), + ("selected edit", self.selected_edit_sha256), + ("source-view fact", self.source_view_fact_sha256), ): _require_sha256(value, f"endpoint plan {label}") + if type(self.source_view_guard) is not SourceViewGuard: + raise TypeError("endpoint plan source-view guard must be exact") + if ( + self.source_view_guard.status != "PASSED" + or self.source_view_guard.source_view_fact_sha256 + != self.source_view_fact_sha256 + or self.source_view_guard.solve_result_sha256 + != self.solve_result_sha256 + or self.source_view_guard.semantic_problem_sha256 + != self.semantic_problem_sha256 + or self.source_view_guard.edit_sha256 != self.selected_edit_sha256 + ): + raise ValueError("endpoint plan source-view guard is not closed") return self @model_validator(mode="after") diff --git a/src/spatialcf/generation/planning/view_guard.py b/src/spatialcf/generation/planning/view_guard.py new file mode 100644 index 0000000..09f0784 --- /dev/null +++ b/src/spatialcf/generation/planning/view_guard.py @@ -0,0 +1,558 @@ +"""Pure authenticated sampled-view admission guard.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + +import numpy as np + +from spatialcf.domain.edit import CanonicalEdit +from spatialcf.domain.request import InterventionSpec, Relation +from spatialcf.domain.scene import OBB, Scene, Vec3 +from spatialcf.domain.serialization import canonical_sha256 +from spatialcf.generation.capture.models import SourceViewFact +from spatialcf.generation.planning.models import ( + _SOURCE_VIEW_GUARD_HASH_DOMAIN, + SourceViewGuard, + SourceViewObjectProxy, +) +from spatialcf.relations.engine import RelationEngine, ground_gap + +_QUANTIZATION_M = 1e-5 + + +@dataclass(frozen=True, slots=True) +class _ProjectedSample: + object_id: str + row: int + column: int + depth_lower: float + depth_upper: float + weight: int + + +def _digest(value: object, label: str) -> str: + if ( + type(value) is not str + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise ValueError(f"{label} must be lowercase SHA-256") + return value + + +def _rotation(obb: OBB) -> np.ndarray: + q = obb.rotation + values = (q.x, q.y, q.z, q.w) + if not all(math.isfinite(value) for value in values): + raise ValueError("source-view OBB rotation must be finite") + norm = math.sqrt(sum(value * value for value in values)) + if norm <= 0.0: + raise ValueError("source-view OBB rotation must be nonzero") + x, y, z, w = (value / norm for value in values) + return np.asarray( + ( + (1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)), + (2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)), + (2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)), + ), + dtype=np.float64, + ) + + +def _translated_obb(obb: OBB, dx: float, dy: float) -> OBB: + return obb.model_copy( + update={ + "center": Vec3( + x=obb.center.x + dx, + y=obb.center.y + dy, + z=obb.center.z, + ) + } + ) + + +def _obb_corners(obb: OBB) -> tuple[np.ndarray, ...]: + center = np.asarray((obb.center.x, obb.center.y, obb.center.z), dtype=np.float64) + half = np.asarray((obb.extent.x, obb.extent.y, obb.extent.z), dtype=np.float64) / 2 + if not np.all(np.isfinite(center)) or not np.all(np.isfinite(half)) or np.any(half <= 0): + raise ValueError("source-view OBB must be finite and nondegenerate") + rotation = _rotation(obb) + return tuple( + center + rotation @ (half * np.asarray((sx, sy, sz), dtype=np.float64)) + for sx in (-1.0, 1.0) + for sy in (-1.0, 1.0) + for sz in (-1.0, 1.0) + ) + + +def _camera_point(matrix: np.ndarray, point: np.ndarray) -> np.ndarray: + homogeneous = matrix @ np.asarray((*point, 1.0), dtype=np.float64) + if not np.all(np.isfinite(homogeneous)) or homogeneous[3] == 0.0: + raise ArithmeticError("nonfinite_projection") + return homogeneous[:3] / homogeneous[3] + + +def _pixel(camera, camera_point: np.ndarray) -> tuple[float, float, float]: + x, y, depth = (float(value) for value in camera_point) + if not math.isfinite(depth) or depth <= 0.0: + raise ArithmeticError("behind_camera") + fx, fy, cx, cy = ( + camera.intrinsics[0], + camera.intrinsics[4], + camera.intrinsics[2], + camera.intrinsics[5], + ) + values = (fx, fy, cx, cy) + if not all(math.isfinite(value) for value in values) or fx <= 0.0 or fy <= 0.0: + raise ArithmeticError("invalid_intrinsics") + return fx * x / depth + cx, cy - fy * y / depth, depth + + +def _ray_hits_obb_before( + origin: np.ndarray, + endpoint: np.ndarray, + obb: OBB, +) -> bool: + rotation = _rotation(obb) + center = np.asarray((obb.center.x, obb.center.y, obb.center.z), dtype=np.float64) + half = np.asarray((obb.extent.x, obb.extent.y, obb.extent.z), dtype=np.float64) / 2 + local_origin = rotation.T @ (origin - center) + local_direction = rotation.T @ (endpoint - origin) + lower, upper = 0.0, 1.0 + for axis in range(3): + direction = float(local_direction[axis]) + if direction == 0.0: + if local_origin[axis] < -half[axis] or local_origin[axis] > half[axis]: + return False + continue + first = (-half[axis] - local_origin[axis]) / direction + second = (half[axis] - local_origin[axis]) / direction + lower = max(lower, min(first, second)) + upper = min(upper, max(first, second)) + if lower > upper: + return False + return lower <= upper and lower <= 1.0 + + +def _outer_projection(camera, matrix: np.ndarray, obb: OBB) -> tuple[float, float]: + pixels = tuple(_pixel(camera, _camera_point(matrix, corner)) for corner in _obb_corners(obb)) + xs = tuple(item[0] for item in pixels) + ys = tuple(item[1] for item in pixels) + xmin, xmax, ymin, ymax = min(xs), max(xs), min(ys), max(ys) + area = (xmax - xmin) * (ymax - ymin) + if not math.isfinite(area) or area <= 0.0: + raise ArithmeticError("degenerate_outer_projection") + clipped_width = max(0.0, min(xmax, camera.width) - max(xmin, 0.0)) + clipped_height = max(0.0, min(ymax, camera.height) - max(ymin, 0.0)) + return area, 1.0 - clipped_width * clipped_height / area + + +def _fallback(object_id: str) -> SourceViewObjectProxy: + return SourceViewObjectProxy( + object_id=object_id, + bbox_center_x_lower=0.0, + bbox_center_x_upper=0.0, + camera_depth_lower_m=0.0, + camera_depth_upper_m=0.0, + image_area_fraction_lower=0.0, + visible_fraction_lower=0.0, + truncated_fraction_upper=1.0, + ) + + +def _zbuffer( + projected: tuple[_ProjectedSample, ...], +) -> tuple[tuple[_ProjectedSample, ...], tuple[str, ...]]: + grouped: dict[tuple[int, int], list[_ProjectedSample]] = {} + for sample in projected: + grouped.setdefault((sample.row, sample.column), []).append(sample) + survivors: list[_ProjectedSample] = [] + reasons: list[str] = [] + for cell_samples in grouped.values(): + if len(cell_samples) == 1: + survivors.append(cell_samples[0]) + continue + winners = tuple( + candidate + for candidate in cell_samples + if all( + candidate is other + or candidate.depth_upper < other.depth_lower + for other in cell_samples + ) + ) + if len(winners) == 1: + survivors.append(winners[0]) + else: + reasons.append("coverage:depth_interval_tie_or_overlap") + return tuple(survivors), tuple(reasons) + + +def _guard( + *, + fact: SourceViewFact, + semantic_problem_sha256: str, + solve_result_sha256: str, + edit: CanonicalEdit, + subject: SourceViewObjectProxy, + reference: SourceViewObjectProxy, + target_relation: Relation, + target_satisfied: bool, + old_relation_satisfied: bool, + reasons: tuple[str, ...], +) -> SourceViewGuard: + payload = { + "guard_version": "competition-native-source-view-guard:2.9.5", + "status": "PASSED" if not reasons else "UNCERTIFIED", + "source_view_fact_sha256": fact.source_view_fact_sha256, + "semantic_problem_sha256": semantic_problem_sha256, + "solve_result_sha256": solve_result_sha256, + "edit_sha256": edit.edit_sha256, + "subject": subject, + "reference": reference, + "target_relation": target_relation, + "target_satisfied": target_satisfied, + "old_relation_satisfied": old_relation_satisfied, + "reasons": tuple(sorted(set(reasons))), + } + return SourceViewGuard( + **payload, + source_view_guard_sha256=canonical_sha256( + payload, domain=_SOURCE_VIEW_GUARD_HASH_DOMAIN + ), + ) + + +def _definite_relation( + relation: Relation, + subject: SourceViewObjectProxy, + reference: SourceViewObjectProxy, + scene: Scene, + subject_obb: OBB, + reference_obb: OBB, +) -> tuple[bool, bool]: + tolerance = RelationEngine.COMPARISON_TOLERANCE + + def at_least(value: float, threshold: float) -> bool: + return value >= threshold or math.isclose( + value, threshold, rel_tol=0.0, abs_tol=tolerance + ) + + def below(value: float, threshold: float) -> bool: + return value < threshold and not math.isclose( + value, threshold, rel_tol=0.0, abs_tol=tolerance + ) + + if relation is Relation.LEFT: + lower = reference.bbox_center_x_lower - subject.bbox_center_x_upper + upper = reference.bbox_center_x_upper - subject.bbox_center_x_lower + threshold = scene.camera_by_id("main").width * RelationEngine.LEFT_RIGHT_FRACTION + elif relation is Relation.RIGHT: + lower = subject.bbox_center_x_lower - reference.bbox_center_x_upper + upper = subject.bbox_center_x_upper - reference.bbox_center_x_lower + threshold = scene.camera_by_id("main").width * RelationEngine.LEFT_RIGHT_FRACTION + elif relation is Relation.FRONT: + lower = reference.camera_depth_lower_m - subject.camera_depth_upper_m + upper = reference.camera_depth_upper_m - subject.camera_depth_lower_m + threshold = RelationEngine.FRONT_BEHIND_METERS + elif relation is Relation.BEHIND: + lower = subject.camera_depth_lower_m - reference.camera_depth_upper_m + upper = subject.camera_depth_upper_m - reference.camera_depth_lower_m + threshold = RelationEngine.FRONT_BEHIND_METERS + else: + gap = ground_gap(subject_obb, reference_obb) + if relation is Relation.NEAR: + return ( + gap <= RelationEngine.NEAR_METERS, + gap > RelationEngine.NEAR_METERS, + ) + return ( + gap >= RelationEngine.FAR_METERS, + gap < RelationEngine.FAR_METERS, + ) + return at_least(lower, threshold), below(upper, threshold) + + +def evaluate_source_view_guard( + scene: Scene, + intervention: InterventionSpec, + fact: SourceViewFact, + edit: CanonicalEdit, + *, + semantic_problem_sha256: str, + solve_result_sha256: str, +) -> SourceViewGuard: + """Evaluate one bound endpoint without importing a platform adapter.""" + + if type(scene) is not Scene: + raise TypeError("source-view guard scene must be exact") + if type(intervention) is not InterventionSpec: + raise TypeError("source-view guard intervention must be exact") + if type(fact) is not SourceViewFact: + raise TypeError("source-view guard fact must be exact") + if type(edit) is not CanonicalEdit: + raise TypeError("source-view guard edit must be exact") + semantic_problem_sha256 = _digest( + semantic_problem_sha256, "source-view semantic problem digest" + ) + solve_result_sha256 = _digest( + solve_result_sha256, "source-view solve result digest" + ) + SourceViewFact.model_validate(fact.model_dump(mode="python"), strict=True) + if edit.semantic_problem_sha256 != semantic_problem_sha256: + raise ValueError("source-view edit does not bind semantic problem") + if edit.subject_id != intervention.subject_id: + raise ValueError("source-view edit does not bind intervention subject") + if intervention.camera_id != "main": + raise ValueError("source-view guard requires the main camera") + try: + subject_object = scene.object_by_id(intervention.subject_id) + reference_object = scene.object_by_id(intervention.reference_id) + camera = scene.camera_by_id("main") + except KeyError as error: + raise ValueError("source-view guard object/camera binding is incomplete") from error + if subject_object.object_id == reference_object.object_id: + raise ValueError("source-view guard objects must be distinct") + if fact.scene_id != scene.scene_id: + raise ValueError("source-view fact scene identity changed") + binding_domain = "spatialcf.source-view-binding.v1" + if fact.scene_sha256 != canonical_sha256(scene, domain=binding_domain): + raise ValueError("source-view fact does not bind scene") + if fact.camera_sha256 != canonical_sha256(camera, domain=binding_domain): + raise ValueError("source-view fact does not bind main camera") + rows = {row.object_id: row for row in fact.objects} + positive_ids: list[str] = [] + for item in scene.objects: + view = item.views.get("main") + if view is not None and view.camera_id != "main": + raise ValueError("source-view main view camera identity changed") + if view is not None and view.visible_fraction > 0.0: + positive_ids.append(item.object_id) + if tuple(rows) != tuple(sorted(positive_ids)): + raise ValueError("source-view fact object roster does not bind main views") + for row in fact.objects: + view = scene.object_by_id(row.object_id).views["main"] + actual_bbox = ( + row.source_mask_bbox.xmin, + row.source_mask_bbox.ymin, + row.source_mask_bbox.xmax, + row.source_mask_bbox.ymax, + ) + expected_bbox = ( + view.bbox.xmin, + view.bbox.ymin, + view.bbox.xmax, + view.bbox.ymax, + ) + if any( + not math.isclose(actual, expected, rel_tol=0.0, abs_tol=1e-6) + for actual, expected in zip(actual_bbox, expected_bbox, strict=True) + ): + raise ValueError("source-view fact bbox does not bind main view") + if ( + row.source_mask_bbox.xmin < 0.0 + or row.source_mask_bbox.ymin < 0.0 + or row.source_mask_bbox.xmax > camera.width + or row.source_mask_bbox.ymax > camera.height + or any(value < 0 or value >= camera.height for value in row.sample_rows) + or any(value < 0 or value >= camera.width for value in row.sample_columns) + ): + raise ValueError("source-view fact pixels exceed main camera bounds") + if {intervention.subject_id, intervention.reference_id} - set(rows): + raise ValueError("source-view fact lacks intervention rows") + try: + for item in scene.objects: + _obb_corners(item.obb) + except ValueError as error: + raise ValueError("source-view scene OBB is malformed") from error + + reasons: list[str] = [] + try: + matrix = np.asarray(camera.world_to_camera, dtype=np.float64).reshape(4, 4) + inverse = np.linalg.inv(matrix) + camera_origin_h = inverse @ np.asarray((0.0, 0.0, 0.0, 1.0)) + if camera_origin_h[3] == 0.0: + raise ArithmeticError("invalid camera homogeneous coordinate") + camera_origin = camera_origin_h[:3] / camera_origin_h[3] + if not np.all(np.isfinite(matrix)) or not np.all(np.isfinite(camera_origin)): + raise ArithmeticError("nonfinite_camera") + except (ValueError, np.linalg.LinAlgError, ArithmeticError, ZeroDivisionError): + return _guard( + fact=fact, + semantic_problem_sha256=semantic_problem_sha256, + solve_result_sha256=solve_result_sha256, + edit=edit, + subject=_fallback(intervention.subject_id), + reference=_fallback(intervention.reference_id), + target_relation=intervention.relation_after, + target_satisfied=False, + old_relation_satisfied=True, + reasons=("projection:camera_transform_invalid",), + ) + + dx, dy = edit.translation_xy_m.x, edit.translation_xy_m.y + moved_subject_obb = _translated_obb(subject_object.obb, dx, dy) + obbs = { + item.object_id: ( + moved_subject_obb if item.object_id == subject_object.object_id else item.obb + ) + for item in scene.objects + } + projected: list[_ProjectedSample] = [] + for row in fact.objects: + obj = scene.object_by_id(row.object_id) + translation = (dx, dy) if row.object_id == subject_object.object_id else (0.0, 0.0) + for qx, qy, qz, weight in zip( + row.local_x_quantized, + row.local_y_quantized, + row.local_z_quantized, + row.sample_weights, + strict=True, + ): + corners = tuple( + np.asarray( + ( + obj.position.x + translation[0] + (qx + sx * 0.5) * _QUANTIZATION_M, + obj.position.y + translation[1] + (qy + sy * 0.5) * _QUANTIZATION_M, + obj.position.z + (qz + sz * 0.5) * _QUANTIZATION_M, + ), + dtype=np.float64, + ) + for sx in (-1.0, 1.0) + for sy in (-1.0, 1.0) + for sz in (-1.0, 1.0) + ) + try: + pixels = tuple(_pixel(camera, _camera_point(matrix, corner)) for corner in corners) + except ArithmeticError as error: + reasons.append(f"projection:{error.args[0]}") + continue + cells = {(round(pixel[1]), round(pixel[0])) for pixel in pixels} + if len(cells) != 1: + reasons.append("projection:quantization_cell_straddle") + continue + row_px, column_px = next(iter(cells)) + if not (0 <= column_px < camera.width and 0 <= row_px < camera.height): + reasons.append("projection:sample_out_of_frame") + continue + try: + blocked = any( + _ray_hits_obb_before(camera_origin, corner, blocker) + for corner in corners + for blocker_id, blocker in obbs.items() + if blocker_id != row.object_id + ) + except ValueError: + reasons.append("coverage:blocker_invalid") + continue + if blocked: + continue + depths = tuple(pixel[2] for pixel in pixels) + projected.append( + _ProjectedSample( + object_id=row.object_id, + row=row_px, + column=column_px, + depth_lower=min(depths), + depth_upper=max(depths), + weight=weight, + ) + ) + + survivors, zbuffer_reasons = _zbuffer(tuple(projected)) + reasons.extend(zbuffer_reasons) + + proxies: dict[str, SourceViewObjectProxy] = {} + for object_id in (intervention.subject_id, intervention.reference_id): + object_samples = tuple(item for item in survivors if item.object_id == object_id) + try: + outer_area, truncation = _outer_projection(camera, matrix, obbs[object_id]) + scene_object = scene.object_by_id(object_id) + translation = (dx, dy) if object_id == subject_object.object_id else (0.0, 0.0) + anchor = _camera_point( + matrix, + np.asarray( + ( + scene_object.position.x + translation[0], + scene_object.position.y + translation[1], + scene_object.position.z, + ), + dtype=np.float64, + ), + ) + if not math.isfinite(float(anchor[2])) or anchor[2] <= 0.0: + raise ArithmeticError("behind_camera_anchor") + except (ArithmeticError, ValueError): + reasons.append(f"projection:{object_id}:outer_invalid") + proxies[object_id] = _fallback(object_id) + continue + if not object_samples: + reasons.append(f"coverage:{object_id}:empty") + proxies[object_id] = _fallback(object_id) + continue + columns = tuple(item.column for item in object_samples) + rows_px = tuple(item.row for item in object_samples) + bbox_area = (max(columns) - min(columns) + 1) * ( + max(rows_px) - min(rows_px) + 1 + ) + center = (min(columns) + max(columns) + 1) / 2.0 + proxies[object_id] = SourceViewObjectProxy( + object_id=object_id, + bbox_center_x_lower=center, + bbox_center_x_upper=center, + camera_depth_lower_m=float(anchor[2]), + camera_depth_upper_m=float(anchor[2]), + image_area_fraction_lower=min(1.0, bbox_area / (camera.width * camera.height)), + visible_fraction_lower=min( + 1.0, sum(item.weight for item in object_samples) / outer_area + ), + truncated_fraction_upper=min(1.0, max(0.0, truncation)), + ) + + subject_proxy = proxies[intervention.subject_id] + reference_proxy = proxies[intervention.reference_id] + target_true, _ = _definite_relation( + intervention.relation_after, + subject_proxy, + reference_proxy, + scene, + moved_subject_obb, + reference_object.obb, + ) + _, old_false = _definite_relation( + intervention.relation_before, + subject_proxy, + reference_proxy, + scene, + moved_subject_obb, + reference_object.obb, + ) + old_satisfied = not old_false + for label, proxy in (("subject", subject_proxy), ("reference", reference_proxy)): + if proxy.visible_fraction_lower < RelationEngine.MIN_VISIBLE_FRACTION: + reasons.append(f"visibility:{label}:visible_fraction") + if proxy.image_area_fraction_lower < RelationEngine.MIN_IMAGE_AREA_FRACTION: + reasons.append(f"visibility:{label}:image_area_fraction") + if proxy.truncated_fraction_upper > RelationEngine.MAX_TRUNCATED_FRACTION: + reasons.append(f"visibility:{label}:truncated_fraction") + if not target_true: + reasons.append("relation:target_not_definite") + if not old_false: + reasons.append("relation:old_not_definitely_false") + return _guard( + fact=fact, + semantic_problem_sha256=semantic_problem_sha256, + solve_result_sha256=solve_result_sha256, + edit=edit, + subject=subject_proxy, + reference=reference_proxy, + target_relation=intervention.relation_after, + target_satisfied=target_true, + old_relation_satisfied=old_satisfied, + reasons=tuple(reasons), + ) + + +__all__ = ("evaluate_source_view_guard",) diff --git a/src/spatialcf/generation/publication/assets.py b/src/spatialcf/generation/publication/assets.py index e468a8b..78b5686 100644 --- a/src/spatialcf/generation/publication/assets.py +++ b/src/spatialcf/generation/publication/assets.py @@ -15,7 +15,7 @@ from PIL import Image, UnidentifiedImageError from pydantic import Field, model_validator -from spatialcf.adapters.base import AdapterObservation +from spatialcf.adapters.base import AdapterObservation, InstanceEvidenceProvenance from spatialcf.domain.base import CanonicalId, CanonicalModel, Sha256Digest from spatialcf.domain.scene import Scene from spatialcf.domain.serialization import ( @@ -675,6 +675,9 @@ def _observation_from_bundle( instance_pixel_counts=tuple( (item.object_id, item.pixel_count) for item in counts ), + instance_evidence_provenance=( + InstanceEvidenceProvenance.PUBLICATION_REPLAY_COUNTS_ONLY + ), is_settled=True, ) diff --git a/src/spatialcf/generation/workflows/capture.py b/src/spatialcf/generation/workflows/capture.py index c0ef99a..524ec12 100644 --- a/src/spatialcf/generation/workflows/capture.py +++ b/src/spatialcf/generation/workflows/capture.py @@ -9,6 +9,7 @@ from spatialcf.adapters.base import ( AdapterCameraApplication, + AdapterObservation, AdapterOperationError, AdapterPose, AdapterPosition, @@ -19,6 +20,7 @@ CapturedSource, CaptureRequest, EnvironmentAdapter, + InstanceEvidenceProvenance, SourceCaptureFacts, SourceCaptureOptions, ) @@ -63,6 +65,10 @@ ) from spatialcf.generation.capture.plan import CapturePlan, CaptureSettings from spatialcf.generation.capture.storage import publish_roster +from spatialcf.generation.capture.visual_evidence import ( + SourceViewFactError, + build_source_view_fact, +) _PROCTHOR_DATASET_ID = "allenai/procthor-10k" _PROCTHOR_DATASET_NAME = "procthor-10k" @@ -350,6 +356,69 @@ def _capture_selected_camera_source( runtime_identity = CompetitionNativeRuntimeIdentityV2_9( **asdict(facts.runtime_identity) ) + raw_observation = application.observation + source_view_fact = None + if ( + raw_observation.scene != application.observed_scene + or application.observed_scene != scene + ): + return ( + _rejected_source(source, "source_capture:source_view_fact_invalid"), + (), + ) + try: + observation = AdapterObservation.create( + scene=normalized_scene, + rgb_png=raw_observation.rgb_png, + depth_npy=raw_observation.depth_npy, + instance_png=raw_observation.instance_png, + pointcloud_ply=raw_observation.pointcloud_ply, + instance_pixel_counts=raw_observation.instance_pixel_counts, + instance_colors=raw_observation.instance_colors, + instance_evidence_provenance=( + raw_observation.instance_evidence_provenance + ), + is_settled=raw_observation.is_settled, + ) + except (TypeError, ValueError): + return ( + _rejected_source(source, "source_capture:source_view_fact_invalid"), + (), + ) + if ( + observation.rgb_png_sha256 != raw_observation.rgb_png_sha256 + or observation.depth_npy_sha256 != raw_observation.depth_npy_sha256 + or observation.instance_png_sha256 != raw_observation.instance_png_sha256 + or observation.pointcloud_ply_sha256 != raw_observation.pointcloud_ply_sha256 + ): + return ( + _rejected_source(source, "source_capture:source_view_fact_invalid"), + (), + ) + if ( + observation.instance_evidence_provenance + is InstanceEvidenceProvenance.SAME_EVENT_INSTANCE_SEGMENTATION + ): + try: + source_view_fact = build_source_view_fact( + source, + runtime_identity, + normalized_scene, + observation, + ) + except (SourceViewFactError, TypeError, ValueError): + return ( + _rejected_source(source, "source_capture:source_view_fact_invalid"), + (), + ) + elif ( + observation.instance_evidence_provenance + is not InstanceEvidenceProvenance.LEGACY_NEUTRAL + ): + return ( + _rejected_source(source, "source_capture:source_view_fact_invalid"), + (), + ) floor = None floor_reason = None floor_subjects = tuple( @@ -498,11 +567,11 @@ def _capture_selected_camera_source( source=source, runtime_identity=runtime_identity, scene=normalized_scene, - rgb_png_sha256=application.observation.rgb_png_sha256, - depth_npy_sha256=application.observation.depth_npy_sha256, - instance_png_sha256=application.observation.instance_png_sha256, - pointcloud_ply_sha256=application.observation.pointcloud_ply_sha256, - is_scene_at_rest=application.observation.is_settled, + rgb_png_sha256=observation.rgb_png_sha256, + depth_npy_sha256=observation.depth_npy_sha256, + instance_png_sha256=observation.instance_png_sha256, + pointcloud_ply_sha256=observation.pointcloud_ply_sha256, + is_scene_at_rest=observation.is_settled, settlement_pass_steps=settlement_pass_steps, support_facts=support_facts, floor_envelope=floor, @@ -510,6 +579,7 @@ def _capture_selected_camera_source( placement_facts=tuple( placement_by_id[item.object_id] for item in normalized_scene.objects ), + source_view_fact=source_view_fact, ) except (TypeError, ValueError): return _rejected_source(source, "source_capture:normalized_capture_invalid"), () diff --git a/src/spatialcf/generation/workflows/dataset.py b/src/spatialcf/generation/workflows/dataset.py index ccd87b9..0a1ff4b 100644 --- a/src/spatialcf/generation/workflows/dataset.py +++ b/src/spatialcf/generation/workflows/dataset.py @@ -85,6 +85,33 @@ _MAX_ASSET_BYTES = 512 * 1024 * 1024 +class SourcePlanIncompleteError(RuntimeError): + """The persisted current source plan cannot enter native execution.""" + + +def _require_complete_source_plan(plan: planning.SourcePlan) -> None: + if type(plan) is not planning.SourcePlan: + raise TypeError("dataset source plan must be exact") + request_ids = tuple(item.request_id for item in plan.roster_manifest.requests) + outcome_ids = tuple(item.request_id for item in plan.request_outcomes) + if outcome_ids != request_ids or len(set(outcome_ids)) != len(outcome_ids): + raise SourcePlanIncompleteError( + "dataset source plan is incomplete:planned=0:" + f"rejected={len(request_ids)}:" + "reasons=source_plan:request_outcomes_not_closed" + ) + rejected = tuple( + item for item in plan.request_outcomes if item.status != "planned" + ) + if rejected: + reasons = tuple(sorted({reason for item in rejected for reason in item.reasons})) + raise SourcePlanIncompleteError( + "dataset source plan is incomplete:" + f"planned={len(plan.request_outcomes) - len(rejected)}:" + f"rejected={len(rejected)}:reasons={','.join(reasons)}" + ) + + def _config_sha256(config: GenerationConfig) -> Sha256Digest: return canonical_sha256( config.model_dump(mode="json", warnings="error"), @@ -295,6 +322,7 @@ def _run_batches( adapter_factory: Callable[..., AI2ThorAdapter], fresh_transitions: _FreshTransitions | None = None, ) -> execution.SourceExecutionSummary: + _require_complete_source_plan(plan) _ensure_batches_root(root) class _FreshAuditRunner: @@ -1531,6 +1559,7 @@ def generate_dataset( root / ".spatialcf" / "source-plan", fresh_transitions=fresh_transitions, ) + _require_complete_source_plan(source_plan) execution_summary = _run_batches( source_plan, root / ".spatialcf" / "batches", diff --git a/tests/public_smoke/test_readme.py b/tests/public_smoke/test_readme.py index 4c7663f..51dede7 100644 --- a/tests/public_smoke/test_readme.py +++ b/tests/public_smoke/test_readme.py @@ -310,9 +310,16 @@ def generate_with_fake_adapter(config: Path, output: Path): generated["accepted_request_count"] + generated["execution_rejected_request_count"] ) - assert generated["frozen_request_count"] > 0 - assert generated["planned_request_count"] > 0 - assert generated["accepted_request_count"] > 0 + assert tuple( + generated[key] + for key in ( + "frozen_request_count", + "planned_request_count", + "planning_rejected_request_count", + "accepted_request_count", + "execution_rejected_request_count", + ) + ) == (0, 0, 0, 0, 0) records = read_dataset_records(tmp_path / "dataset") assert len(records) == generated["accepted_request_count"] expected_bundle_files = {