diff --git a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py index 544e9b8da..779123940 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_positions_file.py @@ -6,6 +6,7 @@ import common.date_formatter as date_formatter from pub_files.database.geolocation_geometry import Geometry +from pub_files.geometry import parse_coordinates from pub_files.input_files.file_metadata import PathElements from pub_files.output_files.filename_format import get_filename from pub_files.output_files.sensor_positions.sensor_position import get_position @@ -25,17 +26,11 @@ def write_file(out_path: Path, location_path: Path, elements: PathElements, time history into every monthly sensor_positions.csv. """ - # Effective-dates columns are emitted only when the loader-driven JSON codepath is in - # play (i.e. this pipeline is concH2oSoilSalinity today). The DB codepath doesn't yet - # compute the cfgloc-geo × ref_geolocation intersection needed to fill them, so other - # DPs keep their pre-change schema until that follow-up lands. - include_effective_dates = position_history_path is not None - filename = get_filename(elements, timestamp=timestamp, file_type='sensor_positions', extension='csv') file_path = Path(out_path, filename) with open(file_path, 'w', encoding='UTF8', newline='') as file: writer = csv.writer(file) - writer.writerow(get_column_names(include_effective_dates=include_effective_dates)) + writer.writerow(get_column_names()) file_rows = [] # Parse location file path for the datum elements. Assume we end at site (**/site/location/*/location_file.json) site = location_path.parts[-1] @@ -61,18 +56,18 @@ def write_file(out_path: Path, location_path: Path, elements: PathElements, time rows = sensor_specific_processors.create_tchain_rows( database, location, geolocation, row_hor_ver, row_location_id, row_description, - _create_base_row_data, _add_reference_position_data, - include_effective_dates=include_effective_dates) + _create_base_row_data, _add_reference_position_data) else: rows = _create_standard_rows(database, geolocation, row_hor_ver, - row_location_id, row_description, - include_effective_dates=include_effective_dates) + row_location_id, row_description) # Add rows, preventing duplicates for row in rows: if row not in file_rows: file_rows.append(row) + # Sort by HOR.VER column (index 0), then by effectiveStartDateTime (index 3) + file_rows.sort(key=lambda row: (row[0], row[3] or '')) writer.writerows(file_rows) return file_path @@ -158,11 +153,7 @@ def _match_reference_geolocation(ref_geolocations: List, entry: Dict): entry_start = _strip_tz(_parse_iso(entry.get('position_start_date'))) entry_end = _strip_tz(_parse_iso(entry.get('position_end_date'))) for ref in ref_geolocations: - ref_start = _strip_tz(ref.start_date) - ref_end = _strip_tz(ref.end_date) - if entry_end is not None and ref_start is not None and entry_end <= ref_start: - continue - if entry_start is not None and ref_end is not None and entry_start >= ref_end: + if not _dates_overlap(entry_start, entry_end, ref.start_date, ref.end_date): continue return ref # Fall back to the first available so downstream still gets azimuth-based east/north math. @@ -184,6 +175,25 @@ def _strip_tz(dt: Optional[datetime]) -> Optional[datetime]: return dt.replace(tzinfo=None) if dt.tzinfo is not None else dt +def _dates_overlap(start_a: Optional[datetime], end_a: Optional[datetime], + start_b: Optional[datetime], end_b: Optional[datetime]) -> bool: + """Return whether two half-open date ranges overlap; None bounds are open-ended.""" + start_a, end_a = _strip_tz(start_a), _strip_tz(end_a) + start_b, end_b = _strip_tz(start_b), _strip_tz(end_b) + return not ((end_a is not None and start_b is not None and end_a <= start_b) or + (start_a is not None and end_b is not None and start_a >= end_b)) + + +def _intersect_dates(start_a: Optional[datetime], end_a: Optional[datetime], + start_b: Optional[datetime], end_b: Optional[datetime]) -> Tuple[Optional[datetime], Optional[datetime]]: + """Return the overlap of two date ranges; a None bound means open-ended on that side.""" + starts = [_strip_tz(s) for s in (start_a, start_b) if s is not None] + ends = [_strip_tz(e) for e in (end_a, end_b) if e is not None] + start = max(starts) if starts else None + end = min(ends) if ends else None + return start, end + + def _create_base_row_data(database: SensorPositionsDatabase, geolocation, row_hor_ver: str, row_location_id: str, row_description: str) -> Dict: """Create the common row data used by both standard and specific sensors.""" @@ -219,25 +229,29 @@ def _add_reference_position_data(database: SensorPositionsDatabase, base_data: D complete_rows = [] for reference_geolocation in database.get_geolocations(offset_name): - - # Determine if this reference position is applicable based on the time. - # If reference location and geolocation don't overlap, skip - if (reference_geolocation.end_date is None) and (geolocation.end_date is not None): - if geolocation.end_date <= reference_geolocation.start_date: - continue - elif (geolocation.end_date is None) and (reference_geolocation.end_date is not None): - if geolocation.start_date >= reference_geolocation.end_date: - continue - elif (geolocation.end_date is not None) and (reference_geolocation.end_date is not None): - if geolocation.end_date <= reference_geolocation.start_date: - continue - elif geolocation.start_date >= reference_geolocation.end_date: - continue + if not _dates_overlap(geolocation.start_date, geolocation.end_date, + reference_geolocation.start_date, reference_geolocation.end_date): + continue reference_position = get_position(reference_geolocation, geolocation.x_offset, geolocation.y_offset) - + + # Effective window = cfgloc-geo x ref_geolocation intersection (both already + # confirmed to overlap above), mirroring the loader-driven JSON codepath. + (effective_start_date, effective_end_date) = _intersect_dates( + geolocation.start_date, geolocation.end_date, + reference_geolocation.start_date, reference_geolocation.end_date) + + # Coordinates must come from this reference geolocation, not the named location's + # first geolocation, so a reference that moved reports the position in effect. + (latitude, longitude, elevation) = _reference_coordinates(reference_geolocation, base_data) + complete_row_data = base_data.copy() complete_row_data.update({ + 'row_reference_location_latitude': latitude, + 'row_reference_location_longitude': longitude, + 'row_reference_location_elevation': elevation, + 'row_effective_start_date': format_date(effective_start_date), + 'row_effective_end_date': format_date(effective_end_date), 'row_x_azimuth': round(reference_position.x_azimuth, 2) if reference_position.x_azimuth is not None else '', 'row_y_azimuth': round(reference_position.y_azimuth, 2) if reference_position.y_azimuth is not None else '', 'row_east_offset': round(reference_position.east_offset, 2) if reference_position.east_offset is not None else '', @@ -250,15 +264,27 @@ def _add_reference_position_data(database: SensorPositionsDatabase, base_data: D return complete_rows +def _reference_coordinates(reference_geolocation, base_data: Dict) -> Tuple: + """Return the latitude, longitude and elevation of the given reference geolocation.""" + geometry = getattr(reference_geolocation, 'geometry', None) + if not geometry: + return (base_data.get('row_reference_location_latitude'), + base_data.get('row_reference_location_longitude'), + base_data.get('row_reference_location_elevation')) + try: + (latitude, longitude, elevation) = parse_coordinates(geometry) + except Exception as error: + raise ValueError(f'Unable to parse reference geolocation geometry: {geometry!r}') from error + return (round(latitude, 6) if latitude is not None else None, + round(longitude, 6) if longitude is not None else None, + round(elevation, 2) if elevation is not None else None) + + def _create_standard_rows(database: SensorPositionsDatabase, geolocation, - row_hor_ver: str, row_location_id: str, row_description: str, - include_effective_dates: bool = False) -> List[List]: - """Create a standard sensor rows. - - When `include_effective_dates` is True, the row leaves two blank cells for - `effectiveStartDateTime` / `effectiveEndDateTime` so it aligns with the header the - JSON codepath emits. The DB codepath doesn't yet compute the intersection needed to - populate them; when it does, this parameter's default can flip. + row_hor_ver: str, row_location_id: str, row_description: str) -> List[List]: + """Create a standard sensor rows, including the cfgloc-geo x ref_geolocation + intersection (computed in `_add_reference_position_data`) in the + `effectiveStartDateTime` / `effectiveEndDateTime` cells. """ base_data = _create_base_row_data(database, geolocation, row_hor_ver, row_location_id, row_description) complete_rows = _add_reference_position_data(database, base_data, geolocation, geolocation.offset_name) @@ -271,9 +297,9 @@ def _create_standard_rows(database: SensorPositionsDatabase, geolocation, row_data['row_hor_ver'], row_data['row_location_id'], row_data['row_description'], + row_data.get('row_effective_start_date', ''), + row_data.get('row_effective_end_date', ''), ] - if include_effective_dates: - leading.extend(['', '']) row = leading + [ row_data['row_position_start_date'], row_data['row_position_end_date'], @@ -299,41 +325,37 @@ def _create_standard_rows(database: SensorPositionsDatabase, geolocation, return rows -def get_column_names(include_effective_dates: bool = False) -> List[str]: +def get_column_names() -> List[str]: """Return the CSV header for sensor_positions.csv. - When `include_effective_dates` is True, two extra columns - (`effectiveStartDateTime` / `effectiveEndDateTime`) are inserted between - `sensorLocationDescription` and `positionStartDateTime`. These carry the - non-overlapping timeline that combines position and reference-location date - ranges; only the JSON codepath (concH2oSoilSalinity loader) populates them - today. The DB codepath keeps the pre-change schema until it's separately - updated to compute the cfgloc-geo × ref_geolocation intersection. + `effectiveStartDateTime` / `effectiveEndDateTime` carry the cfgloc-geo x + ref_geolocation intersection, computed by both the DB codepath and the + loader-driven JSON codepath (concH2oSoilSalinity loader). """ columns = ['HOR.VER', 'sensorLocationID', - 'sensorLocationDescription'] - if include_effective_dates: - columns.extend(['effectiveStartDateTime', 'effectiveEndDateTime']) - columns.extend(['positionStartDateTime', - 'positionEndDateTime', - 'referenceLocationID', - 'referenceLocationIDDescription', - 'referenceLocationIDStartDateTime', - 'referenceLocationIDEndDateTime', - 'xOffset', - 'yOffset', - 'zOffset', - 'pitch', - 'roll', - 'azimuth', - 'locationReferenceLatitude', - 'locationReferenceLongitude', - 'locationReferenceElevation', - 'eastOffset', - 'northOffset', - 'xAzimuth', - 'yAzimuth']) + 'sensorLocationDescription', + 'effectiveStartDateTime', + 'effectiveEndDateTime', + 'positionStartDateTime', + 'positionEndDateTime', + 'referenceLocationID', + 'referenceLocationIDDescription', + 'referenceLocationIDStartDateTime', + 'referenceLocationIDEndDateTime', + 'xOffset', + 'yOffset', + 'zOffset', + 'pitch', + 'roll', + 'azimuth', + 'locationReferenceLatitude', + 'locationReferenceLongitude', + 'locationReferenceElevation', + 'eastOffset', + 'northOffset', + 'xAzimuth', + 'yAzimuth'] return columns diff --git a/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py b/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py index 74b6d2bbf..a5ca6c956 100644 --- a/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py +++ b/modules/pub_files/output_files/sensor_positions/sensor_specific_processors.py @@ -22,12 +22,11 @@ def get_thermistor_depths(location) -> Dict[str, Optional[float]]: def create_tchain_rows(database: SensorPositionsDatabase, location, geolocation, row_hor_ver: str, row_location_id: str, row_description: str, - create_base_row_data_func, add_reference_position_data_func, - include_effective_dates: bool = False) -> List[List]: + create_base_row_data_func, add_reference_position_data_func) -> List[List]: """Create multiple rows for tchain sensor, one for each thermistor depth. - `include_effective_dates` leaves two blank cells for effectiveStart/End to align - with the JSON codepath's header; used only when this pipeline's caller enables it. + The cfgloc-geo x ref_geolocation intersection (computed by + `add_reference_position_data_func`) fills the effectiveStart/End cells. """ base_data = create_base_row_data_func(database, geolocation, row_hor_ver, row_location_id, row_description) complete_rows = add_reference_position_data_func(database, base_data, geolocation, geolocation.offset_name) @@ -50,9 +49,9 @@ def create_tchain_rows(database: SensorPositionsDatabase, location, geolocation, modified_hor_ver, complete_row_data['row_location_id'], complete_row_data['row_description'], + complete_row_data.get('row_effective_start_date', ''), + complete_row_data.get('row_effective_end_date', ''), ] - if include_effective_dates: - leading.extend(['', '']) tchain_row = leading + [ complete_row_data['row_position_start_date'], complete_row_data['row_position_end_date'], diff --git a/modules/pub_files/requirements.txt b/modules/pub_files/requirements.txt index 3c828b51c..fecb434d9 100644 --- a/modules/pub_files/requirements.txt +++ b/modules/pub_files/requirements.txt @@ -1,6 +1,6 @@ common==0.1.2 data_access==0.0.1a2 -Jinja2==3.1.4 +Jinja2==3.1.6 environs==11.0.0 marshmallow==3.21.3 pandas==2.1.4 diff --git a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py index acc0122a5..bb009d548 100644 --- a/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py +++ b/modules/pub_files/tests/sensor_positions_file/test_sensor_positions_file.py @@ -2,8 +2,9 @@ import csv import json import os +from datetime import datetime, timezone from pathlib import Path -from typing import List +from typing import List, Optional from pyfakefs.fake_filesystem import FakeFilesystem from pyfakefs.fake_filesystem_unittest import TestCase @@ -16,6 +17,9 @@ from pub_files.input_files.file_metadata import PathElements from pub_files.main import get_timestamp from pub_files.output_files.sensor_positions.sensor_positions_file import SensorPositionsDatabase +from pub_files.output_files.sensor_positions.sensor_positions_file import _add_reference_position_data +from pub_files.output_files.sensor_positions.sensor_positions_file import _create_base_row_data +from pub_files.output_files.sensor_positions.sensor_positions_file import _reference_coordinates from pub_files.output_files.sensor_positions.sensor_positions_file import get_column_names from pub_files.output_files.sensor_positions.sensor_positions_file import write_file from pub_files.output_files.sensor_positions.sensor_specific_processors import create_tchain_rows @@ -215,12 +219,21 @@ def __init__(self, offset_name='REFLOC001'): self.offset_name = offset_name +class ReferenceCoordinatesTest(TestCase): + + def test_invalid_geometry_includes_geometry_in_error(self): + reference_geolocation = _FakeGeolocation() + reference_geolocation.geometry = 'LINESTRING (1 2, 3 4)' + + with self.assertRaisesRegex(ValueError, "LINESTRING \\(1 2, 3 4\\)"): + _reference_coordinates(reference_geolocation, {}) + + class TchainRowShapeTest(TestCase): - """create_tchain_rows produces one row per thermistor depth. When - include_effective_dates is False (the default all pipelines hit today), the row - shape matches the pre-change 22-column schema exactly. When True (the shape the - follow-up DB-codepath effective-dates work will flip to), two blank cells appear - between description and positionStart, aligning with get_column_names(True). + """create_tchain_rows produces one row per thermistor depth. The cfgloc-geo x + ref_geolocation intersection (computed upstream and carried on the row data + dict) fills the effectiveStart/End cells between description and positionStart, + aligning with get_column_names(). """ @staticmethod @@ -244,6 +257,8 @@ def _base_row_stub(_database, _geolocation, row_hor_ver, row_location_id, row_de def _ref_position_stub(_database, base_data, _geolocation, _offset_name): row = dict(base_data) row.update({ + 'row_effective_start_date': '2020-01-01T00:00:00Z', + 'row_effective_end_date': '', 'row_x_azimuth': 0.0, 'row_y_azimuth': 0.0, 'row_east_offset': 0.0, 'row_north_offset': 0.0, 'row_reference_location_start_date': '2010-01-01T00:00:00Z', @@ -258,7 +273,7 @@ def _tchain_location(): Property(name='ThermistorDepth502', value='0.16'), ]) - def _run(self, include_effective_dates: bool): + def _run(self): return create_tchain_rows( database=None, location=self._tchain_location(), geolocation=_FakeGeolocation(), @@ -266,35 +281,239 @@ def _run(self, include_effective_dates: bool): row_description='Test Soil Temp', create_base_row_data_func=self._base_row_stub, add_reference_position_data_func=self._ref_position_stub, - include_effective_dates=include_effective_dates, ) - def test_default_flag_omits_effective_cells(self): - rows = self._run(include_effective_dates=False) - self.assertEqual(len(rows), 2) # one per thermistor depth - header = get_column_names(include_effective_dates=False) - self.assertEqual(len(header), 22) - # Thermistor ids substitute into the VER slot of HOR.VER - self.assertEqual({row[0] for row in rows}, {'000.501', '000.502'}) - for row in rows: - self.assertEqual(len(row), 22) - # Position dates land right after description; no blank cells between. - self.assertEqual(row[3], '2020-01-01T00:00:00Z') - self.assertEqual(row[4], '') - - def test_flag_true_inserts_two_blank_cells_matching_header(self): - rows = self._run(include_effective_dates=True) + def test_inserts_effective_dates_matching_header(self): + rows = self._run() self.assertEqual(len(rows), 2) - header = get_column_names(include_effective_dates=True) + header = get_column_names() self.assertEqual(len(header), 24) self.assertEqual(header[3], 'effectiveStartDateTime') self.assertEqual(header[4], 'effectiveEndDateTime') self.assertEqual({row[0] for row in rows}, {'000.501', '000.502'}) for row in rows: self.assertEqual(len(row), 24) - # Two blank cells for effective land between description and positionStart. - # DB codepath doesn't compute the intersection yet; follow-up PR fills these. - self.assertEqual(row[3], '') - self.assertEqual(row[4], '') + # Effective cells land between description and positionStart. + self.assertEqual(row[3], '2020-01-01T00:00:00Z') # effectiveStart + self.assertEqual(row[4], '') # effectiveEnd self.assertEqual(row[5], '2020-01-01T00:00:00Z') # positionStart self.assertEqual(row[6], '') # positionEnd + + +class ReferencePositionOverlapTest(TestCase): + """_add_reference_position_data must skip a reference geolocation whose validity + window doesn't overlap the sensor geolocation's window, and vice versa, so no + row is emitted for a period where the two positions were never both in effect. + """ + + @staticmethod + def _geolocation(start_date: Optional[datetime], end_date: Optional[datetime], + offset_name: str = 'REFLOC001') -> GeoLocation: + return GeoLocation(location_id=1, geometry='POINT Z (0 0 0)', + start_date=start_date, end_date=end_date, + alpha=0.0, beta=0.0, gamma=0.0, + x_offset=1.0, y_offset=1.0, z_offset=1.0, + offset_id=1, offset_name=offset_name, offset_description='ref', + properties=[]) + + @staticmethod + def _base_data() -> dict: + return { + 'row_hor_ver': '000.010', 'row_location_id': 'CFGLOC000001', + 'row_description': 'Test', + 'row_position_start_date': '', 'row_position_end_date': '', + 'row_x_offset': 1.0, 'row_y_offset': 1.0, 'row_z_offset': 1.0, + 'row_pitch': 0.0, 'row_roll': 0.0, 'row_azimuth': 0.0, + 'row_reference_location_id': 'REFLOC001', + 'row_reference_location_description': 'ref', + 'row_reference_location_latitude': 0.0, + 'row_reference_location_longitude': 0.0, + 'row_reference_location_elevation': 0.0, + } + + def _rows_for(self, geolocation: GeoLocation, reference_geolocations: List[GeoLocation]) -> List[dict]: + database = SensorPositionsDatabase(get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: None, + get_named_location=lambda _name: None) + return _add_reference_position_data(database, self._base_data(), geolocation, 'REFLOC001') + + def test_no_row_when_sensor_position_ends_before_reference_starts(self): + sensor = self._geolocation(datetime(2020, 1, 1), datetime(2020, 6, 1)) + reference = self._geolocation(datetime(2021, 1, 1), None) + self.assertEqual(self._rows_for(sensor, [reference]), []) + + def test_no_row_when_reference_ends_before_sensor_position_starts(self): + sensor = self._geolocation(datetime(2021, 1, 1), None) + reference = self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1)) + self.assertEqual(self._rows_for(sensor, [reference]), []) + + def test_row_produced_when_windows_overlap(self): + sensor = self._geolocation(datetime(2020, 1, 1), datetime(2020, 6, 1)) + reference = self._geolocation(datetime(2019, 1, 1), None) + rows = self._rows_for(sensor, [reference]) + self.assertEqual(len(rows), 1) + + def test_row_produced_for_open_start_and_mixed_timezone_windows(self): + sensor = self._geolocation(None, datetime(2020, 6, 1)) + reference = self._geolocation(datetime(2020, 1, 1, tzinfo=timezone.utc), None) + rows = self._rows_for(sensor, [reference]) + self.assertEqual(len(rows), 1) + + +class MultipleGeolocationChangesEffectiveWindowTest(TestCase): + """When both the sensor's own geolocation history and the reference location's + history each change independently (asynchronously) over time, every overlapping + (sensor, reference) pair must produce its own row, with effective start/end equal + to the intersection of that pair's two windows. + """ + + @staticmethod + def _geolocation(start_date: Optional[datetime], end_date: Optional[datetime]) -> GeoLocation: + return GeoLocation(location_id=1, geometry='POINT Z (0 0 0)', + start_date=start_date, end_date=end_date, + alpha=0.0, beta=0.0, gamma=0.0, + x_offset=1.0, y_offset=1.0, z_offset=1.0, + offset_id=1, offset_name='REFLOC001', offset_description='ref', + properties=[]) + + @staticmethod + def _base_data() -> dict: + return { + 'row_hor_ver': '000.010', 'row_location_id': 'CFGLOC000001', + 'row_description': 'Test', + 'row_position_start_date': '', 'row_position_end_date': '', + 'row_x_offset': 1.0, 'row_y_offset': 1.0, 'row_z_offset': 1.0, + 'row_pitch': 0.0, 'row_roll': 0.0, 'row_azimuth': 0.0, + 'row_reference_location_id': 'REFLOC001', + 'row_reference_location_description': 'ref', + 'row_reference_location_latitude': 0.0, + 'row_reference_location_longitude': 0.0, + 'row_reference_location_elevation': 0.0, + } + + def test_asynchronous_changes_on_both_sides_yield_intersected_windows(self): + # Reference location moved once: an earlier fixed period, then open-ended. + reference_geolocations = [ + self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1)), + self._geolocation(datetime(2020, 1, 1), None), + ] + # Sensor also moved once, at a different (asynchronous) time than the reference: + # first period spans across the reference's own change; second starts after + # the reference's fixed period has already ended. + sensor_geolocations = [ + self._geolocation(datetime(2019, 6, 1), datetime(2020, 6, 1)), + self._geolocation(datetime(2021, 1, 1), None), + ] + + database = SensorPositionsDatabase(get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: None, + get_named_location=lambda _name: None) + + rows = [] + for sensor_geolocation in sensor_geolocations: + rows.extend(_add_reference_position_data(database, self._base_data(), + sensor_geolocation, 'REFLOC001')) + + # First sensor period overlaps both reference periods; second overlaps only the open one. + self.assertEqual(len(rows), 3) + + +class ReferenceGeolocationCoordinatesTest(TestCase): + """Each row must carry the coordinates of the reference geolocation in effect for + that row, not the coordinates of the reference location's first geolocation. + """ + + @staticmethod + def _geolocation(start_date: Optional[datetime], end_date: Optional[datetime], + geometry: str) -> GeoLocation: + return GeoLocation(location_id=1, geometry=geometry, + start_date=start_date, end_date=end_date, + alpha=0.0, beta=0.0, gamma=0.0, + x_offset=1.0, y_offset=1.0, z_offset=1.0, + offset_id=1, offset_name='REFLOC001', offset_description='ref', + properties=[]) + + @staticmethod + def _base_data() -> dict: + # Populated from the named location's (first) geometry by _create_base_row_data. + return { + 'row_hor_ver': '000.010', 'row_location_id': 'CFGLOC000001', + 'row_description': 'Test', + 'row_position_start_date': '', 'row_position_end_date': '', + 'row_x_offset': 1.0, 'row_y_offset': 1.0, 'row_z_offset': 1.0, + 'row_pitch': 0.0, 'row_roll': 0.0, 'row_azimuth': 0.0, + 'row_reference_location_id': 'REFLOC001', + 'row_reference_location_description': 'ref', + 'row_reference_location_latitude': 40.815536, + 'row_reference_location_longitude': -104.745591, + 'row_reference_location_elevation': 1653.92, + } + + def test_each_row_uses_its_own_reference_geolocation_coordinates(self): + reference_geolocations = [ + self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1), + 'POINT Z (-104.745591 40.815536 1653.9151)'), + self._geolocation(datetime(2020, 1, 1), None, + 'POINT Z (-104.746013 40.815892 1654.0094)'), + ] + sensor_geolocation = self._geolocation(datetime(2019, 6, 1), None, 'POINT Z (0 0 0)') + + database = SensorPositionsDatabase(get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: None, + get_named_location=lambda _name: None) + rows = _add_reference_position_data(database, self._base_data(), + sensor_geolocation, 'REFLOC001') + + self.assertEqual(len(rows), 2) + self.assertEqual((rows[0]['row_reference_location_latitude'], + rows[0]['row_reference_location_longitude'], + rows[0]['row_reference_location_elevation']), + (40.815536, -104.745591, 1653.92)) + self.assertEqual((rows[1]['row_reference_location_latitude'], + rows[1]['row_reference_location_longitude'], + rows[1]['row_reference_location_elevation']), + (40.815892, -104.746013, 1654.01)) + self.assertNotEqual(rows[0]['row_reference_location_latitude'], + rows[1]['row_reference_location_latitude']) + effective_windows = {(row['row_effective_start_date'], row['row_effective_end_date']) + for row in rows} + self.assertEqual(effective_windows, { + ('2019-06-01T00:00:00Z', '2020-01-01T00:00:00Z'), + ('2020-01-01T00:00:00Z', ''), + }) + + def test_tchain_rows_use_their_own_reference_geolocation_coordinates(self): + reference_geolocations = [ + self._geolocation(datetime(2019, 1, 1), datetime(2020, 1, 1), + 'POINT Z (-104.745591 40.815536 1653.9151)'), + self._geolocation(datetime(2020, 1, 1), None, + 'POINT Z (-104.746013 40.815892 1654.0094)'), + ] + sensor_geolocation = self._geolocation(datetime(2019, 6, 1), None, 'POINT Z (0 0 0)') + location = _FakeLocation(properties=[ + Property(name='ThermistorDepth501', value='0.06'), + Property(name='ThermistorDepth502', value='0.16'), + ]) + + database = SensorPositionsDatabase( + get_geolocations=lambda _name: reference_geolocations, + get_geometry=lambda _name: build_geometry( + geometry='POINT Z (-104.745591 40.815536 1653.9151)', srid=4979), + get_named_location=lambda name: NamedLocation(location_id=1, name=name, + description='ref', properties=[])) + + rows = create_tchain_rows(database, location, sensor_geolocation, + '000.010', 'CFGLOC000001', 'Test Soil Temp', + _create_base_row_data, _add_reference_position_data) + + # Two reference geolocations x two thermistor depths. + self.assertEqual(len(rows), 4) + header = get_column_names() + latitude_index = header.index('locationReferenceLatitude') + elevation_index = header.index('locationReferenceElevation') + by_effective_start = {} + for row in rows: + by_effective_start.setdefault(row[3], set()).add( + (row[latitude_index], row[elevation_index])) + self.assertEqual(by_effective_start['2019-06-01T00:00:00Z'], {(40.815536, 1653.92)}) + self.assertEqual(by_effective_start['2020-01-01T00:00:00Z'], {(40.815892, 1654.01)})