From a867748b7b4964657bf39c91892650c3434fabab Mon Sep 17 00:00:00 2001 From: covesturtevant Date: Tue, 18 Aug 2026 12:52:45 -0600 Subject: [PATCH 1/6] add effective times to sensor positions file --- .../sensor_positions/sensor_positions_file.py | 106 ++++++------ .../sensor_specific_processors.py | 11 +- .../test_sensor_positions_file.py | 163 +++++++++++++++--- 3 files changed, 194 insertions(+), 86 deletions(-) diff --git a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py index 544e9b8da..f08185c9c 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py @@ -25,17 +25,11 @@ def write_file(out_path: Path, location_path: Path, elements: PathElements, time history into every monthly sensor_positions.csv. """ - # Effective-dates columns are emitted only when the loader-driven JSON codepath is in - # play (i.e. this pipeline is concH2oSoilSalinity today). The DB codepath doesn't yet - # compute the cfgloc-geo × ref_geolocation intersection needed to fill them, so other - # DPs keep their pre-change schema until that follow-up lands. - include_effective_dates = position_history_path is not None - filename = get_filename(elements, timestamp=timestamp, file_type='sensor_positions', extension='csv') file_path = Path(out_path, filename) with open(file_path, 'w', encoding='UTF8', newline='') as file: writer = csv.writer(file) - writer.writerow(get_column_names(include_effective_dates=include_effective_dates)) + writer.writerow(get_column_names()) file_rows = [] # Parse location file path for the datum elements. Assume we end at site (**/site/location/*/location_file.json) site = location_path.parts[-1] @@ -61,12 +55,10 @@ def write_file(out_path: Path, location_path: Path, elements: PathElements, time rows = sensor_specific_processors.create_tchain_rows( database, location, geolocation, row_hor_ver, row_location_id, row_description, - _create_base_row_data, _add_reference_position_data, - include_effective_dates=include_effective_dates) + _create_base_row_data, _add_reference_position_data) else: rows = _create_standard_rows(database, geolocation, row_hor_ver, - row_location_id, row_description, - include_effective_dates=include_effective_dates) + row_location_id, row_description) # Add rows, preventing duplicates for row in rows: @@ -184,6 +176,16 @@ def _strip_tz(dt: Optional[datetime]) -> Optional[datetime]: return dt.replace(tzinfo=None) if dt.tzinfo is not None else dt +def _intersect_dates(start_a: Optional[datetime], end_a: Optional[datetime], + start_b: Optional[datetime], end_b: Optional[datetime]) -> Tuple[Optional[datetime], Optional[datetime]]: + """Return the overlap of two date ranges; a None bound means open-ended on that side.""" + starts = [_strip_tz(s) for s in (start_a, start_b) if s is not None] + ends = [_strip_tz(e) for e in (end_a, end_b) if e is not None] + start = max(starts) if starts else None + end = min(ends) if ends else None + return start, end + + def _create_base_row_data(database: SensorPositionsDatabase, geolocation, row_hor_ver: str, row_location_id: str, row_description: str) -> Dict: """Create the common row data used by both standard and specific sensors.""" @@ -235,9 +237,17 @@ def _add_reference_position_data(database: SensorPositionsDatabase, base_data: D continue reference_position = get_position(reference_geolocation, geolocation.x_offset, geolocation.y_offset) - + + # Effective window = cfgloc-geo x ref_geolocation intersection (both already + # confirmed to overlap above), mirroring the loader-driven JSON codepath. + (effective_start_date, effective_end_date) = _intersect_dates( + geolocation.start_date, geolocation.end_date, + reference_geolocation.start_date, reference_geolocation.end_date) + complete_row_data = base_data.copy() complete_row_data.update({ + 'row_effective_start_date': format_date(effective_start_date), + 'row_effective_end_date': format_date(effective_end_date), 'row_x_azimuth': round(reference_position.x_azimuth, 2) if reference_position.x_azimuth is not None else '', 'row_y_azimuth': round(reference_position.y_azimuth, 2) if reference_position.y_azimuth is not None else '', 'row_east_offset': round(reference_position.east_offset, 2) if reference_position.east_offset is not None else '', @@ -251,14 +261,10 @@ def _add_reference_position_data(database: SensorPositionsDatabase, base_data: D def _create_standard_rows(database: SensorPositionsDatabase, geolocation, - row_hor_ver: str, row_location_id: str, row_description: str, - include_effective_dates: bool = False) -> List[List]: - """Create a standard sensor rows. - - When `include_effective_dates` is True, the row leaves two blank cells for - `effectiveStartDateTime` / `effectiveEndDateTime` so it aligns with the header the - JSON codepath emits. The DB codepath doesn't yet compute the intersection needed to - populate them; when it does, this parameter's default can flip. + row_hor_ver: str, row_location_id: str, row_description: str) -> List[List]: + """Create a standard sensor rows, including the cfgloc-geo x ref_geolocation + intersection (computed in `_add_reference_position_data`) in the + `effectiveStartDateTime` / `effectiveEndDateTime` cells. """ base_data = _create_base_row_data(database, geolocation, row_hor_ver, row_location_id, row_description) complete_rows = _add_reference_position_data(database, base_data, geolocation, geolocation.offset_name) @@ -271,9 +277,9 @@ def _create_standard_rows(database: SensorPositionsDatabase, geolocation, row_data['row_hor_ver'], row_data['row_location_id'], row_data['row_description'], + row_data.get('row_effective_start_date', ''), + row_data.get('row_effective_end_date', ''), ] - if include_effective_dates: - leading.extend(['', '']) row = leading + [ row_data['row_position_start_date'], row_data['row_position_end_date'], @@ -299,41 +305,37 @@ def _create_standard_rows(database: SensorPositionsDatabase, geolocation, return rows -def get_column_names(include_effective_dates: bool = False) -> List[str]: +def get_column_names() -> List[str]: """Return the CSV header for sensor_positions.csv. - When `include_effective_dates` is True, two extra columns - (`effectiveStartDateTime` / `effectiveEndDateTime`) are inserted between - `sensorLocationDescription` and `positionStartDateTime`. These carry the - non-overlapping timeline that combines position and reference-location date - ranges; only the JSON codepath (concH2oSoilSalinity loader) populates them - today. The DB codepath keeps the pre-change schema until it's separately - updated to compute the cfgloc-geo × ref_geolocation intersection. + `effectiveStartDateTime` / `effectiveEndDateTime` carry the cfgloc-geo x + ref_geolocation intersection, computed by both the DB codepath and the + loader-driven JSON codepath (concH2oSoilSalinity loader). """ columns = ['HOR.VER', 'sensorLocationID', - 'sensorLocationDescription'] - if include_effective_dates: - columns.extend(['effectiveStartDateTime', 'effectiveEndDateTime']) - columns.extend(['positionStartDateTime', - 'positionEndDateTime', - 'referenceLocationID', - 'referenceLocationIDDescription', - 'referenceLocationIDStartDateTime', - 'referenceLocationIDEndDateTime', - 'xOffset', - 'yOffset', - 'zOffset', - 'pitch', - 'roll', - 'azimuth', - 'locationReferenceLatitude', - 'locationReferenceLongitude', - 'locationReferenceElevation', - 'eastOffset', - 'northOffset', - 'xAzimuth', - 'yAzimuth']) + 'sensorLocationDescription', + 'effectiveStartDateTime', + 'effectiveEndDateTime', + 'positionStartDateTime', + 'positionEndDateTime', + 'referenceLocationID', + 'referenceLocationIDDescription', + 'referenceLocationIDStartDateTime', + 'referenceLocationIDEndDateTime', + 'xOffset', + 'yOffset', + 'zOffset', + 'pitch', + 'roll', + 'azimuth', + 'locationReferenceLatitude', + 'locationReferenceLongitude', + 'locationReferenceElevation', + 'eastOffset', + 'northOffset', + 'xAzimuth', + 'yAzimuth'] return columns diff --git a/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py b/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py index 74b6d2bbf..a5ca6c956 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py @@ -22,12 +22,11 @@ def get_thermistor_depths(location) -> Dict[str, Optional[float]]: def create_tchain_rows(database: SensorPositionsDatabase, location, geolocation, row_hor_ver: str, row_location_id: str, row_description: str, - create_base_row_data_func, add_reference_position_data_func, - include_effective_dates: bool = False) -> List[List]: + create_base_row_data_func, add_reference_position_data_func) -> List[List]: """Create multiple rows for tchain sensor, one for each thermistor depth. - `include_effective_dates` leaves two blank cells for effectiveStart/End to align - with the JSON codepath's header; used only when this pipeline's caller enables it. + The cfgloc-geo x ref_geolocation intersection (computed by + `add_reference_position_data_func`) fills the effectiveStart/End cells. """ base_data = create_base_row_data_func(database, geolocation, row_hor_ver, row_location_id, row_description) complete_rows = add_reference_position_data_func(database, base_data, geolocation, geolocation.offset_name) @@ -50,9 +49,9 @@ def create_tchain_rows(database: SensorPositionsDatabase, location, geolocation, modified_hor_ver, complete_row_data['row_location_id'], complete_row_data['row_description'], + complete_row_data.get('row_effective_start_date', ''), + complete_row_data.get('row_effective_end_date', ''), ] - if include_effective_dates: - leading.extend(['', '']) tchain_row = leading + [ complete_row_data['row_position_start_date'], complete_row_data['row_position_end_date'], diff --git a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py index acc0122a5..88304298e 100644 --- a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py +++ b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py @@ -2,8 +2,9 @@ import csv import json import os +from datetime import datetime from pathlib import Path -from typing import List +from typing import List, Optional from pyfakefs.fake_filesystem import FakeFilesystem from pyfakefs.fake_filesystem_unittest import TestCase @@ -16,6 +17,7 @@ from pub_files.input_files.file_metadata import PathElements from pub_files.main import get_timestamp from pub_files.output_files.sensor_positions.sensor_positions_file import SensorPositionsDatabase +from pub_files.output_files.sensor_positions.sensor_positions_file import _add_reference_position_data from pub_files.output_files.sensor_positions.sensor_positions_file import get_column_names from pub_files.output_files.sensor_positions.sensor_positions_file import write_file from pub_files.output_files.sensor_positions.sensor_specific_processors import create_tchain_rows @@ -216,11 +218,10 @@ def __init__(self, offset_name='REFLOC001'): class TchainRowShapeTest(TestCase): - """create_tchain_rows produces one row per thermistor depth. When - include_effective_dates is False (the default all pipelines hit today), the row - shape matches the pre-change 22-column schema exactly. When True (the shape the - follow-up DB-codepath effective-dates work will flip to), two blank cells appear - between description and positionStart, aligning with get_column_names(True). + """create_tchain_rows produces one row per thermistor depth. The cfgloc-geo x + ref_geolocation intersection (computed upstream and carried on the row data + dict) fills the effectiveStart/End cells between description and positionStart, + aligning with get_column_names(). """ @staticmethod @@ -244,6 +245,8 @@ def _base_row_stub(_database, _geolocation, row_hor_ver, row_location_id, row_de def _ref_position_stub(_database, base_data, _geolocation, _offset_name): row = dict(base_data) row.update({ + 'row_effective_start_date': '2020-01-01T00:00:00Z', + 'row_effective_end_date': '', 'row_x_azimuth': 0.0, 'row_y_azimuth': 0.0, 'row_east_offset': 0.0, 'row_north_offset': 0.0, 'row_reference_location_start_date': '2010-01-01T00:00:00Z', @@ -258,7 +261,7 @@ def _tchain_location(): Property(name='ThermistorDepth502', value='0.16'), ]) - def _run(self, include_effective_dates: bool): + def _run(self): return create_tchain_rows( database=None, location=self._tchain_location(), geolocation=_FakeGeolocation(), @@ -266,35 +269,139 @@ def _run(self, include_effective_dates: bool): row_description='Test Soil Temp', create_base_row_data_func=self._base_row_stub, add_reference_position_data_func=self._ref_position_stub, - include_effective_dates=include_effective_dates, ) - def test_default_flag_omits_effective_cells(self): - rows = self._run(include_effective_dates=False) - self.assertEqual(len(rows), 2) # one per thermistor depth - header = get_column_names(include_effective_dates=False) - self.assertEqual(len(header), 22) - # Thermistor ids substitute into the VER slot of HOR.VER - self.assertEqual({row[0] for row in rows}, {'000.501', '000.502'}) - for row in rows: - self.assertEqual(len(row), 22) - # Position dates land right after description; no blank cells between. - self.assertEqual(row[3], '2020-01-01T00:00:00Z') - self.assertEqual(row[4], '') - - def test_flag_true_inserts_two_blank_cells_matching_header(self): - rows = self._run(include_effective_dates=True) + def test_inserts_effective_dates_matching_header(self): + rows = self._run() self.assertEqual(len(rows), 2) - header = get_column_names(include_effective_dates=True) + header = get_column_names() self.assertEqual(len(header), 24) self.assertEqual(header[3], 'effectiveStartDateTime') self.assertEqual(header[4], 'effectiveEndDateTime') self.assertEqual({row[0] for row in rows}, {'000.501', '000.502'}) for row in rows: self.assertEqual(len(row), 24) - # Two blank cells for effective land between description and positionStart. - # DB codepath doesn't compute the intersection yet; follow-up PR fills these. - self.assertEqual(row[3], '') - self.assertEqual(row[4], '') + # Effective cells land between description and positionStart. + self.assertEqual(row[3], '2020-01-01T00:00:00Z') # effectiveStart + self.assertEqual(row[4], '') # effectiveEnd self.assertEqual(row[5], '2020-01-01T00:00:00Z') # positionStart self.assertEqual(row[6], '') # positionEnd + + +class ReferencePositionOverlapTest(TestCase): + """_add_reference_position_data must skip a reference geolocation whose validity + window doesn't overlap the sensor geolocation's window, and vice versa, so no + row is emitted for a period where the two positions were never both in effect. + """ + + @staticmethod + def _geolocation(start_date: Optional[datetime], end_date: Optional[datetime], + offset_name: str = 'REFLOC001') -> GeoLocation: + return GeoLocation(location_id=1, geometry='POINT Z (0 0 0)', + start_date=start_date, end_date=end_date, + alpha=0.0, beta=0.0, gamma=0.0, + x_offset=1.0, y_offset=1.0, z_offset=1.0, + offset_id=1, offset_name=offset_name, offset_description='ref', + properties=[]) + + @staticmethod + def _base_data() -> dict: + return { + 'row_hor_ver': '000.010', 'row_location_id': 'CFGLOC000001', + 'row_description': 'Test', + 'row_position_start_date': '', 'row_position_end_date': '', + 'row_x_offset': 1.0, 'row_y_offset': 1.0, 'row_z_offset': 1.0, + 'row_pitch': 0.0, 'row_roll': 0.0, 'row_azimuth': 0.0, + 'row_reference_location_id': 'REFLOC001', + 'row_reference_location_description': 'ref', + 'row_reference_location_latitude': 0.0, + 'row_reference_location_longitude': 0.0, + 'row_reference_location_elevation': 0.0, + } + + def _rows_for(self, geolocation: GeoLocation, reference_geolocations: List[GeoLocation]) -> List[dict]: + database = SensorPositionsDatabase(get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: None, + get_named_location=lambda _name: None) + return _add_reference_position_data(database, self._base_data(), geolocation, 'REFLOC001') + + def test_no_row_when_sensor_position_ends_before_reference_starts(self): + sensor = self._geolocation(datetime(2020, 1, 1), datetime(2020, 6, 1)) + reference = self._geolocation(datetime(2021, 1, 1), None) + self.assertEqual(self._rows_for(sensor, [reference]), []) + + def test_no_row_when_reference_ends_before_sensor_position_starts(self): + sensor = self._geolocation(datetime(2021, 1, 1), None) + reference = self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1)) + self.assertEqual(self._rows_for(sensor, [reference]), []) + + def test_row_produced_when_windows_overlap(self): + sensor = self._geolocation(datetime(2020, 1, 1), datetime(2020, 6, 1)) + reference = self._geolocation(datetime(2019, 1, 1), None) + rows = self._rows_for(sensor, [reference]) + self.assertEqual(len(rows), 1) + + +class MultipleGeolocationChangesEffectiveWindowTest(TestCase): + """When both the sensor's own geolocation history and the reference location's + history each change independently (asynchronously) over time, every overlapping + (sensor, reference) pair must produce its own row, with effective start/end equal + to the intersection of that pair's two windows. + """ + + @staticmethod + def _geolocation(start_date: Optional[datetime], end_date: Optional[datetime]) -> GeoLocation: + return GeoLocation(location_id=1, geometry='POINT Z (0 0 0)', + start_date=start_date, end_date=end_date, + alpha=0.0, beta=0.0, gamma=0.0, + x_offset=1.0, y_offset=1.0, z_offset=1.0, + offset_id=1, offset_name='REFLOC001', offset_description='ref', + properties=[]) + + @staticmethod + def _base_data() -> dict: + return { + 'row_hor_ver': '000.010', 'row_location_id': 'CFGLOC000001', + 'row_description': 'Test', + 'row_position_start_date': '', 'row_position_end_date': '', + 'row_x_offset': 1.0, 'row_y_offset': 1.0, 'row_z_offset': 1.0, + 'row_pitch': 0.0, 'row_roll': 0.0, 'row_azimuth': 0.0, + 'row_reference_location_id': 'REFLOC001', + 'row_reference_location_description': 'ref', + 'row_reference_location_latitude': 0.0, + 'row_reference_location_longitude': 0.0, + 'row_reference_location_elevation': 0.0, + } + + def test_asynchronous_changes_on_both_sides_yield_intersected_windows(self): + # Reference location moved once: an earlier fixed period, then open-ended. + reference_geolocations = [ + self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1)), + self._geolocation(datetime(2020, 1, 1), None), + ] + # Sensor also moved once, at a different (asynchronous) time than the reference: + # first period spans across the reference's own change; second starts after + # the reference's fixed period has already ended. + sensor_geolocations = [ + self._geolocation(datetime(2019, 6, 1), datetime(2020, 6, 1)), + self._geolocation(datetime(2021, 1, 1), None), + ] + + database = SensorPositionsDatabase(get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: None, + get_named_location=lambda _name: None) + + rows = [] + for sensor_geolocation in sensor_geolocations: + rows.extend(_add_reference_position_data(database, self._base_data(), + sensor_geolocation, 'REFLOC001')) + + # First sensor period overlaps both reference periods; second overlaps only the open one. + self.assertEqual(len(rows), 3) + effective_windows = {(row['row_effective_start_date'], row['row_effective_end_date']) + for row in rows} + self.assertEqual(effective_windows, { + ('2019-06-01T00:00:00Z', '2020-01-01T00:00:00Z'), + ('2020-01-01T00:00:00Z', '2020-06-01T00:00:00Z'), + ('2021-01-01T00:00:00Z', ''), + }) From c3b699757963bb88b6211d2679a3fcbbda7a0af0 Mon Sep 17 00:00:00 2001 From: covesturtevant Date: Tue, 18 Aug 2026 13:28:53 -0600 Subject: [PATCH 2/6] add sorting first by hor.ver then by effective start time --- .../output_files/sensor_positions/sensor_positions_file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py index f08185c9c..ce80e354f 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py @@ -65,6 +65,8 @@ def write_file(out_path: Path, location_path: Path, elements: PathElements, time if row not in file_rows: file_rows.append(row) + # Sort by HOR.VER column (index 0), then by effectiveStartDateTime (index 3) + file_rows.sort(key=lambda row: (row[0], row[3])) writer.writerows(file_rows) return file_path From c931d9d8f108a0ccfce381990875fdb8b9900d0e Mon Sep 17 00:00:00 2001 From: covesturtevant Date: Tue, 18 Aug 2026 15:01:54 -0600 Subject: [PATCH 3/6] fix bug that pulled only the first reference geolocation --- .../sensor_positions/sensor_positions_file.py | 21 ++++ .../test_sensor_positions_file.py | 99 ++++++++++++++++++- 2 files changed, 118 insertions(+), 2 deletions(-) diff --git a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py index ce80e354f..ee853e139 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py @@ -6,6 +6,7 @@ import common.date_formatter as date_formatter from pub_files.database.geolocation_geometry import Geometry +from pub_files.geometry import parse_coordinates from pub_files.input_files.file_metadata import PathElements from pub_files.output_files.filename_format import get_filename from pub_files.output_files.sensor_positions.sensor_position import get_position @@ -246,8 +247,15 @@ def _add_reference_position_data(database: SensorPositionsDatabase, base_data: D geolocation.start_date, geolocation.end_date, reference_geolocation.start_date, reference_geolocation.end_date) + # Coordinates must come from this reference geolocation, not the named location's + # first geolocation, so a reference that moved reports the position in effect. + (latitude, longitude, elevation) = _reference_coordinates(reference_geolocation, base_data) + complete_row_data = base_data.copy() complete_row_data.update({ + 'row_reference_location_latitude': latitude, + 'row_reference_location_longitude': longitude, + 'row_reference_location_elevation': elevation, 'row_effective_start_date': format_date(effective_start_date), 'row_effective_end_date': format_date(effective_end_date), 'row_x_azimuth': round(reference_position.x_azimuth, 2) if reference_position.x_azimuth is not None else '', @@ -262,6 +270,19 @@ def _add_reference_position_data(database: SensorPositionsDatabase, base_data: D return complete_rows +def _reference_coordinates(reference_geolocation, base_data: Dict) -> Tuple: + """Return the latitude, longitude and elevation of the given reference geolocation.""" + geometry = getattr(reference_geolocation, 'geometry', None) + if not geometry: + return (base_data.get('row_reference_location_latitude'), + base_data.get('row_reference_location_longitude'), + base_data.get('row_reference_location_elevation')) + (latitude, longitude, elevation) = parse_coordinates(geometry) + return (round(latitude, 6) if latitude is not None else None, + round(longitude, 6) if longitude is not None else None, + round(elevation, 2) if elevation is not None else None) + + def _create_standard_rows(database: SensorPositionsDatabase, geolocation, row_hor_ver: str, row_location_id: str, row_description: str) -> List[List]: """Create a standard sensor rows, including the cfgloc-geo x ref_geolocation diff --git a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py index 88304298e..866749248 100644 --- a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py +++ b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py @@ -18,6 +18,7 @@ from pub_files.main import get_timestamp from pub_files.output_files.sensor_positions.sensor_positions_file import SensorPositionsDatabase from pub_files.output_files.sensor_positions.sensor_positions_file import _add_reference_position_data +from pub_files.output_files.sensor_positions.sensor_positions_file import _create_base_row_data from pub_files.output_files.sensor_positions.sensor_positions_file import get_column_names from pub_files.output_files.sensor_positions.sensor_positions_file import write_file from pub_files.output_files.sensor_positions.sensor_specific_processors import create_tchain_rows @@ -398,10 +399,104 @@ def test_asynchronous_changes_on_both_sides_yield_intersected_windows(self): # First sensor period overlaps both reference periods; second overlaps only the open one. self.assertEqual(len(rows), 3) + + +class ReferenceGeolocationCoordinatesTest(TestCase): + """Each row must carry the coordinates of the reference geolocation in effect for + that row, not the coordinates of the reference location's first geolocation. + """ + + @staticmethod + def _geolocation(start_date: Optional[datetime], end_date: Optional[datetime], + geometry: str) -> GeoLocation: + return GeoLocation(location_id=1, geometry=geometry, + start_date=start_date, end_date=end_date, + alpha=0.0, beta=0.0, gamma=0.0, + x_offset=1.0, y_offset=1.0, z_offset=1.0, + offset_id=1, offset_name='REFLOC001', offset_description='ref', + properties=[]) + + @staticmethod + def _base_data() -> dict: + # Populated from the named location's (first) geometry by _create_base_row_data. + return { + 'row_hor_ver': '000.010', 'row_location_id': 'CFGLOC000001', + 'row_description': 'Test', + 'row_position_start_date': '', 'row_position_end_date': '', + 'row_x_offset': 1.0, 'row_y_offset': 1.0, 'row_z_offset': 1.0, + 'row_pitch': 0.0, 'row_roll': 0.0, 'row_azimuth': 0.0, + 'row_reference_location_id': 'REFLOC001', + 'row_reference_location_description': 'ref', + 'row_reference_location_latitude': 40.815536, + 'row_reference_location_longitude': -104.745591, + 'row_reference_location_elevation': 1653.92, + } + + def test_each_row_uses_its_own_reference_geolocation_coordinates(self): + reference_geolocations = [ + self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1), + 'POINT Z (-104.745591 40.815536 1653.9151)'), + self._geolocation(datetime(2020, 1, 1), None, + 'POINT Z (-104.746013 40.815892 1654.0094)'), + ] + sensor_geolocation = self._geolocation(datetime(2019, 6, 1), None, 'POINT Z (0 0 0)') + + database = SensorPositionsDatabase(get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: None, + get_named_location=lambda _name: None) + rows = _add_reference_position_data(database, self._base_data(), + sensor_geolocation, 'REFLOC001') + + self.assertEqual(len(rows), 2) + self.assertEqual((rows[0]['row_reference_location_latitude'], + rows[0]['row_reference_location_longitude'], + rows[0]['row_reference_location_elevation']), + (40.815536, -104.745591, 1653.92)) + self.assertEqual((rows[1]['row_reference_location_latitude'], + rows[1]['row_reference_location_longitude'], + rows[1]['row_reference_location_elevation']), + (40.815892, -104.746013, 1654.01)) + self.assertNotEqual(rows[0]['row_reference_location_latitude'], + rows[1]['row_reference_location_latitude']) effective_windows = {(row['row_effective_start_date'], row['row_effective_end_date']) for row in rows} self.assertEqual(effective_windows, { ('2019-06-01T00:00:00Z', '2020-01-01T00:00:00Z'), - ('2020-01-01T00:00:00Z', '2020-06-01T00:00:00Z'), - ('2021-01-01T00:00:00Z', ''), + ('2020-01-01T00:00:00Z', ''), }) + + def test_tchain_rows_use_their_own_reference_geolocation_coordinates(self): + reference_geolocations = [ + self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1), + 'POINT Z (-104.745591 40.815536 1653.9151)'), + self._geolocation(datetime(2020, 1, 1), None, + 'POINT Z (-104.746013 40.815892 1654.0094)'), + ] + sensor_geolocation = self._geolocation(datetime(2019, 6, 1), None, 'POINT Z (0 0 0)') + location = _FakeLocation(properties=[ + Property(name='ThermistorDepth501', value='0.06'), + Property(name='ThermistorDepth502', value='0.16'), + ]) + + database = SensorPositionsDatabase( + get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: build_geometry( + geometry='POINT Z (-104.745591 40.815536 1653.9151)', srid=4979), + get_named_location=lambda name: NamedLocation(location_id=1, name=name, + description='ref', properties=[])) + + rows = create_tchain_rows(database, location, sensor_geolocation, + '000.010', 'CFGLOC000001', 'Test Soil Temp', + _create_base_row_data, _add_reference_position_data) + + # Two reference geolocations x two thermistor depths. + self.assertEqual(len(rows), 4) + header = get_column_names() + latitude_index = header.index('locationReferenceLatitude') + elevation_index = header.index('locationReferenceElevation') + by_effective_start = {} + for row in rows: + by_effective_start.setdefault(row[3], set()).add( + (row[latitude_index], row[elevation_index])) + self.assertEqual(by_effective_start['2019-06-01T00:00:00Z'], {(40.815536, 1653.92)}) + self.assertEqual(by_effective_start['2020-01-01T00:00:00Z'], {(40.815892, 1654.01)}) From 1b9e0313ca15c23cb99efb76a0e73ca851248285 Mon Sep 17 00:00:00 2001 From: Cove Sturtevant Date: Tue, 18 Aug 2026 16:42:31 -0600 Subject: [PATCH 4/6] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../output_files/sensor_positions/sensor_positions_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py index ee853e139..f68fed5f4 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py @@ -67,7 +67,7 @@ def write_file(out_path: Path, location_path: Path, elements: PathElements, time file_rows.append(row) # Sort by HOR.VER column (index 0), then by effectiveStartDateTime (index 3) - file_rows.sort(key=lambda row: (row[0], row[3])) + file_rows.sort(key=lambda row: (row[0], row[3] or '')) writer.writerows(file_rows) return file_path From b26536681b313139c71c50e537d686c9cfdb8c75 Mon Sep 17 00:00:00 2001 From: covesturtevant Date: Tue, 18 Aug 2026 16:44:46 -0600 Subject: [PATCH 5/6] align logic for reference and sensor position date range overlap. Improve failure logic for malformed geometry. --- .../sensor_positions/sensor_positions_file.py | 37 +++++++++---------- .../test_sensor_positions_file.py | 19 +++++++++- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py index ee853e139..a82b18ba7 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py @@ -153,11 +153,7 @@ def _match_reference_geolocation(ref_geolocations: List, entry: Dict): entry_start = _strip_tz(_parse_iso(entry.get('position_start_date'))) entry_end = _strip_tz(_parse_iso(entry.get('position_end_date'))) for ref in ref_geolocations: - ref_start = _strip_tz(ref.start_date) - ref_end = _strip_tz(ref.end_date) - if entry_end is not None and ref_start is not None and entry_end <= ref_start: - continue - if entry_start is not None and ref_end is not None and entry_start >= ref_end: + if not _dates_overlap(entry_start, entry_end, ref.start_date, ref.end_date): continue return ref # Fall back to the first available so downstream still gets azimuth-based east/north math. @@ -179,6 +175,15 @@ def _strip_tz(dt: Optional[datetime]) -> Optional[datetime]: return dt.replace(tzinfo=None) if dt.tzinfo is not None else dt +def _dates_overlap(start_a: Optional[datetime], end_a: Optional[datetime], + start_b: Optional[datetime], end_b: Optional[datetime]) -> bool: + """Return whether two half-open date ranges overlap; None bounds are open-ended.""" + start_a, end_a = _strip_tz(start_a), _strip_tz(end_a) + start_b, end_b = _strip_tz(start_b), _strip_tz(end_b) + return not ((end_a is not None and start_b is not None and end_a <= start_b) or + (start_a is not None and end_b is not None and start_a >= end_b)) + + def _intersect_dates(start_a: Optional[datetime], end_a: Optional[datetime], start_b: Optional[datetime], end_b: Optional[datetime]) -> Tuple[Optional[datetime], Optional[datetime]]: """Return the overlap of two date ranges; a None bound means open-ended on that side.""" @@ -224,20 +229,9 @@ def _add_reference_position_data(database: SensorPositionsDatabase, base_data: D complete_rows = [] for reference_geolocation in database.get_geolocations(offset_name): - - # Determine if this reference position is applicable based on the time. - # If reference location and geolocation don't overlap, skip - if (reference_geolocation.end_date is None) and (geolocation.end_date is not None): - if geolocation.end_date <= reference_geolocation.start_date: - continue - elif (geolocation.end_date is None) and (reference_geolocation.end_date is not None): - if geolocation.start_date >= reference_geolocation.end_date: - continue - elif (geolocation.end_date is not None) and (reference_geolocation.end_date is not None): - if geolocation.end_date <= reference_geolocation.start_date: - continue - elif geolocation.start_date >= reference_geolocation.end_date: - continue + if not _dates_overlap(geolocation.start_date, geolocation.end_date, + reference_geolocation.start_date, reference_geolocation.end_date): + continue reference_position = get_position(reference_geolocation, geolocation.x_offset, geolocation.y_offset) @@ -277,7 +271,10 @@ def _reference_coordinates(reference_geolocation, base_data: Dict) -> Tuple: return (base_data.get('row_reference_location_latitude'), base_data.get('row_reference_location_longitude'), base_data.get('row_reference_location_elevation')) - (latitude, longitude, elevation) = parse_coordinates(geometry) + try: + (latitude, longitude, elevation) = parse_coordinates(geometry) + except Exception as error: + raise ValueError(f'Unable to parse reference geolocation geometry: {geometry!r}') from error return (round(latitude, 6) if latitude is not None else None, round(longitude, 6) if longitude is not None else None, round(elevation, 2) if elevation is not None else None) diff --git a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py index 866749248..bb009d548 100644 --- a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py +++ b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py @@ -2,7 +2,7 @@ import csv import json import os -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import List, Optional @@ -19,6 +19,7 @@ from pub_files.output_files.sensor_positions.sensor_positions_file import SensorPositionsDatabase from pub_files.output_files.sensor_positions.sensor_positions_file import _add_reference_position_data from pub_files.output_files.sensor_positions.sensor_positions_file import _create_base_row_data +from pub_files.output_files.sensor_positions.sensor_positions_file import _reference_coordinates from pub_files.output_files.sensor_positions.sensor_positions_file import get_column_names from pub_files.output_files.sensor_positions.sensor_positions_file import write_file from pub_files.output_files.sensor_positions.sensor_specific_processors import create_tchain_rows @@ -218,6 +219,16 @@ def __init__(self, offset_name='REFLOC001'): self.offset_name = offset_name +class ReferenceCoordinatesTest(TestCase): + + def test_invalid_geometry_includes_geometry_in_error(self): + reference_geolocation = _FakeGeolocation() + reference_geolocation.geometry = 'LINESTRING (1 2, 3 4)' + + with self.assertRaisesRegex(ValueError, "LINESTRING \\(1 2, 3 4\\)"): + _reference_coordinates(reference_geolocation, {}) + + class TchainRowShapeTest(TestCase): """create_tchain_rows produces one row per thermistor depth. The cfgloc-geo x ref_geolocation intersection (computed upstream and carried on the row data @@ -342,6 +353,12 @@ def test_row_produced_when_windows_overlap(self): rows = self._rows_for(sensor, [reference]) self.assertEqual(len(rows), 1) + def test_row_produced_for_open_start_and_mixed_timezone_windows(self): + sensor = self._geolocation(None, datetime(2020, 6, 1)) + reference = self._geolocation(datetime(2020, 1, 1, tzinfo=timezone.utc), None) + rows = self._rows_for(sensor, [reference]) + self.assertEqual(len(rows), 1) + class MultipleGeolocationChangesEffectiveWindowTest(TestCase): """When both the sensor's own geolocation history and the reference location's From f41eb1b0727e51c425efe871ab79defc50786516 Mon Sep 17 00:00:00 2001 From: covesturtevant Date: Wed, 19 Aug 2026 14:49:48 -0600 Subject: [PATCH 6/6] bump jinja2 package version per dependabot --- modules/pub_files/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pub_files/requirements.txt b/modules/pub_files/requirements.txt index 3c828b51c..fecb434d9 100644 --- a/modules/pub_files/requirements.txt +++ b/modules/pub_files/requirements.txt @@ -1,6 +1,6 @@ common==0.1.2 data_access==0.0.1a2 -Jinja2==3.1.4 +Jinja2==3.1.6 environs==11.0.0 marshmallow==3.21.3 pandas==2.1.4