diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java index 9943a848..c7db17b9 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.ListIterator; import java.util.Map.Entry; @@ -325,6 +324,13 @@ private boolean isParserAtEndOfBuffer(int bufferLength, int currentByteIndex) { * {@code inter-block ticks / samples-per-block} is the exact per-sample period. * With fewer than two blocks in the payload the header-derived estimate the * blocks were created with is left in place. + *

+ * The measurements are accumulated per sensor across the payloads of the + * current parse run (see + * {@link UtilCsvSplitting#refineSlowSensorSamplingRateLimits(SENSORS, java.util.List)}), + * and it is the median over that history - not this payload's handful of + * inter-block gaps - that is applied to the blocks and used to (re)derive the + * CSV gap-splitting window. */ private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slowSensorId) { List slowSensorBlocks = new ArrayList(); @@ -361,50 +367,61 @@ private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slow if(perSamplePeriodsS.isEmpty()) { return; } - // Median so a dropped block (a 2x gap) can't skew the period. - Collections.sort(perSamplePeriodsS); - double medianPeriodS = perSamplePeriodsS.get(perSamplePeriodsS.size()/2); - if(!(medianPeriodS>0)) { + + // Feed this payload's measurements into the sensor's running history and take + // the median over EVERYTHING measured so far in this parse run, then seed the + // CSV gap-splitting window from it. + // + // Why accumulate rather than re-derive the window from this payload alone: + // the checker (UtilCsvSplitting.isSamplingRateOutsideOfLimits) computes + // exactly 1/period for each block boundary, i.e. the very quantities measured + // here. A payload only carries 2-3 slow-sensor blocks, so a window re-derived + // from just this payload would absorb a dropped block's 2x spacing into its + // own median and never flag it - while the healthy boundary back to the + // previous payload got flagged instead. Against a history spanning hundreds + // of payloads a single 2x outlier barely moves the median, so the real gap + // stays outside the window. + // + // Why the window is needed at all: the header-derived estimate the blocks were + // created with can sit within ~1% of a +/-10% band edge (VD6283: 10 Hz + // estimated vs ~9.09 Hz achieved), and the slow sensors' cadence is inherently + // jittery - the light's is bimodal (exposure vs exposure + dead time: ~100 vs + // ~110 ms at the default exposure) and the MLX90632's conversions can slip by + // several refresh periods and then catch up (observed +12.5% block spacing + // with no samples lost - DEV-927 validation data). + double achievedRateHz = Double.NaN; + for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) { + if(sensorClassKey!=SENSORS.CLOCK) { + double medianRateHz = UtilCsvSplitting.refineSlowSensorSamplingRateLimits(sensorClassKey, perSamplePeriodsS); + if(Double.isNaN(achievedRateHz)) { + // All of a data block's sensor class keys are fed the same + // measurements, so they all return the same median - just keep the + // first one for the block sampling rate below. + achievedRateHz = medianRateHz; + } + } + } + if(Double.isNaN(achievedRateHz)) { + // No sensor class key was accumulated against (nothing but CLOCK mapped to + // this data block id), so fall back to this payload's own median. + double medianPeriodS = UtilCsvSplitting.calculateMedian(perSamplePeriodsS); + achievedRateHz = medianPeriodS>0? 1.0/medianPeriodS:Double.NaN; + } + if(!(achievedRateHz>0)) { return; } - double achievedRateHz = 1.0/medianPeriodS; + // The blocks are given the same accumulated median the gap window is built + // from, rather than this payload's own possibly-skewed median: the rate is + // what the block start times (and hence the CSV timestamps) are back-filled + // with, so a payload that happens to contain a dropped block would otherwise + // stretch its own samples' spacing by the very artefact the window is meant to + // report. Keeping both on one estimate also stops the timestamps and the + // continuity check disagreeing about what the achieved cadence is. for(DataBlockDetails dataBlockDetails:slowSensorBlocks) { dataBlockDetails.setSamplingRate(achievedRateHz); dataBlockDetails.calculateTimestampDiffInS(); } - - // Seed the CSV gap-splitting window from the OBSERVED cadence rather than a - // single-rate +/-10% band. The header-derived estimate can sit within ~1% of - // the band edge (VD6283: 10 Hz estimated vs ~9.09 Hz achieved), and the slow - // sensors' cadence is inherently jittery: the light's is bimodal (exposure vs - // exposure + dead time: ~100 vs ~110 ms at the default exposure) and the - // MLX90632's conversions can slip by several refresh periods and then catch - // up (observed +12.5% block spacing with no samples lost - DEV-927 - // validation data). A single payload carries only 2-3 slow-sensor blocks, - // i.e. one or two inter-block gaps - no spread information - so the gap - // side of the window cannot rely on observed spread at all: it is set to - // tolerate anything up to SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO x the - // achieved median spacing, which keeps healthy jitter continuous while a - // genuinely dropped block (2x spacing) still splits. The fast side keeps - // the observed-minimum-period basis with the standard tolerance. - // The put is deliberately UNCONDITIONAL: the limits map is global across - // payloads, and a payload with fewer than two blocks of this sensor (early - // return above - e.g. the very first payload of a recording) leaves - // populateExpectedPayloadTsDiffLimitMapIfNeeded to seed a configured-rate - // +/-10% band first. A containsKey guard here would then lock that too-tight - // estimate in for the whole file (observed: 25-min DEV-927 skin-temp - // recording fragmented into 7 CSVs); the measured window must win as soon as - // it exists, and re-measuring on every payload keeps it tracking the sensor. - double minPeriodS = perSamplePeriodsS.get(0); - double[] samplingRateLimits = new double[] { - achievedRateHz/UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, - (1.0/minPeriodS)*UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; - for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) { - if(sensorClassKey!=SENSORS.CLOCK) { - UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, samplingRateLimits); - } - } } private void backfillDataBlockRwcTimestamps() { diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java index f6c23ee0..1eba6a51 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java @@ -1,5 +1,7 @@ package com.shimmerresearch.verisense.payloaddesign; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -11,26 +13,61 @@ public class UtilCsvSplitting { - public class FILE_GAP_TOLERANCE_MULTIPLIER { + public static class FILE_GAP_TOLERANCE_MULTIPLIER { // +/- 10% public static final double UPPER = 1.1; public static final double LOWER = 0.9; /** * Slow sensors only (VD6283 light / MLX90632 skin temp): the largest * inter-block gap, as a multiple of the achieved median block spacing, that - * is still treated as continuous. The MLX90632's conversions can slip by - * several refresh periods and then catch up (observed up to +12.5% block - * spacing on the DEV-927 validation recording with no samples lost), and the - * window is seeded from the first payload that carries >= 2 blocks - often a - * single inter-block gap, i.e. no spread information - so the standard - * LOWER (-10%) band is routinely violated by healthy data. A genuinely - * dropped block doubles the spacing (2x), so 1.5x keeps comfortable margin - * on both sides. + * is still treated as continuous. It sets the SLOW (gap) side of the + * sampling-rate window only - the fast side stays on the standard UPPER + * (+10%) tolerance, see {@link + * UtilCsvSplitting#calculateSlowSensorSamplingRateLimits(double)}. + *

+ * The median it is applied to is accumulated across every payload parsed so + * far in the current parse run (see + * {@link UtilCsvSplitting#refineSlowSensorSamplingRateLimits(SENSORS, List)}), + * not re-derived from the handful of inter-block gaps in the payload + * currently being judged - a payload only carries 2-3 slow-sensor blocks, so + * a per-payload estimate would absorb a dropped block into its own window and + * never report it. Against the accumulated median a single 2x outlier barely + * moves the centre, so the gap stays outside the window. + *

+ * The band still has to be wide because the slow sensors' cadence is + * inherently jittery even when no samples are lost: the light's is bimodal + * (exposure vs exposure + dead time) and the MLX90632's conversions can slip + * by several refresh periods and then catch up (observed up to +12.5% block + * spacing on the DEV-927 validation recording with no samples lost), which + * routinely violates the standard LOWER (-10%) band. A genuinely dropped + * block doubles the spacing (2x), so 1.5x sits comfortably between healthy + * jitter and a real gap. */ public static final double SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO = 1.5; } - - protected static HashMap SAMPLING_RATE_LIMITS_PER_SENSOR = new HashMap(); + + /** + * The maximum number of slow-sensor per-sample periods kept per sensor in + * {@link #SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR}. Once full, the oldest + * measurements are dropped so that the median follows any genuine long-term + * drift in the sensor's cadence while staying deep enough (hundreds of + * payloads' worth of inter-block gaps) that individual dropped blocks cannot + * shift it. + */ + protected static final int SLOW_SENSOR_PERIOD_HISTORY_MAX = 1024; + + protected static HashMap SAMPLING_RATE_LIMITS_PER_SENSOR = new HashMap(); + + /** + * Slow-sensor (VD6283 light / MLX90632 skin temp) per-sample periods, in + * seconds, as measured from the inter-block tick spacing of every payload + * parsed so far in the current parse run. Shares its lifecycle with + * {@link #SAMPLING_RATE_LIMITS_PER_SENSOR}: both are cleared together by + * {@link #clearMapOfSamplingRateLimitsPerSensor()}, which the file parser calls + * on each CSV-set boundary so that measurements never leak from one recording + * into the next. + */ + protected static HashMap> SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR = new HashMap>(); public static boolean isTsDifferenceOutsideOfLimits(double expectedPayloadTsDiffLimits[], double unixTimeInMs_1, double unixTimeInMs_2) { double differenceInMillisec = Math.abs(unixTimeInMs_1 - unixTimeInMs_2); @@ -99,8 +136,116 @@ public static double[] calculateSamplingRateLimits(double configuredSamplingRate return new double[] {configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.LOWER, configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; } + /** + * Slow-sensor (VD6283 light / MLX90632 skin temp) window either side of the + * achieved median rate. Both sides are derived from the SAME robust median so + * that neither edge can be dragged around by a single extreme inter-block + * spacing: the slow (gap) side tolerates up to + * {@link FILE_GAP_TOLERANCE_MULTIPLIER#SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO} + * times the median spacing, the fast side the standard + * {@link FILE_GAP_TOLERANCE_MULTIPLIER#UPPER} tolerance. + * + * @param medianRateHz the achieved median sampling rate, in Hz + * @return {min, max} sampling rate, in Hz, still treated as continuous + */ + public static double[] calculateSlowSensorSamplingRateLimits(double medianRateHz) { + return new double[] { + medianRateHz/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, + medianRateHz*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; + } + + /** + * Median of the supplied values. Unlike a bare {@code get(size/2)} this + * averages the two middle values for an even-sized input, and it sorts a copy + * so the caller's list ordering is left alone. + * + * @param values the values to take the median of + * @return the median, or {@link Double#NaN} if there are no values + */ + public static double calculateMedian(List values) { + if(values==null || values.isEmpty()) { + return Double.NaN; + } + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + int size = sortedValues.size(); + if(size%2==0) { + return (sortedValues.get((size/2)-1) + sortedValues.get(size/2))/2.0; + } + return sortedValues.get(size/2); + } + + /** + * Add the per-sample periods measured in the payload just parsed to this + * sensor's running history and return the median over EVERYTHING accumulated so + * far in the current parse run (not just the latest payload's values). + * + * @param sensorClassKey the sensor the periods were measured for + * @param newlyObservedPeriodsS the per-sample periods, in seconds, measured in + * the payload just parsed (may be empty/null to just read the + * current median back) + * @return the accumulated median per-sample period, in seconds, or + * {@link Double#NaN} if nothing has been measured for this sensor yet + */ + public static double accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSORS sensorClassKey, List newlyObservedPeriodsS) { + List accumulatedPeriodsS = SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.get(sensorClassKey); + if(accumulatedPeriodsS==null) { + accumulatedPeriodsS = new ArrayList(); + SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.put(sensorClassKey, accumulatedPeriodsS); + } + if(newlyObservedPeriodsS!=null) { + for(Double periodS:newlyObservedPeriodsS) { + if(periodS!=null && periodS>0) { + accumulatedPeriodsS.add(periodS); + } + } + } + // Bounded history: drop the oldest measurements rather than growing without + // limit over a multi-day recording. + int excess = accumulatedPeriodsS.size()-SLOW_SENSOR_PERIOD_HISTORY_MAX; + if(excess>0) { + accumulatedPeriodsS.subList(0, excess).clear(); + } + return calculateMedian(accumulatedPeriodsS); + } + + /** + * Accumulate the slow-sensor per-sample periods measured in the payload just + * parsed and (re)apply the resulting CSV gap-splitting window for that sensor. + *

+ * The put into {@link #SAMPLING_RATE_LIMITS_PER_SENSOR} is deliberately + * UNCONDITIONAL. A payload that carries fewer than two blocks of this sensor + * (e.g. the very first payload of a recording) leaves + * {@link #populateExpectedPayloadTsDiffLimitMapIfNeeded(VerisenseDevice, HashMap)} + * to seed a configured-rate +/-10% band first; the header-derived rates for the + * slow sensors are only estimates (the light rate isn't stored at all), so that + * band can be far too tight (observed: a 25-min DEV-927 skin-temp recording + * fragmented into 7 CSVs). A containsKey guard here would lock that estimate in + * for the whole file, so the measured window must win as soon as it exists. + * + * @param sensorClassKey the sensor the periods were measured for + * @param newlyObservedPeriodsS the per-sample periods, in seconds, measured in + * the payload just parsed + * @return the accumulated median sampling rate, in Hz, or {@link Double#NaN} if + * nothing has been measured for this sensor yet (in which case the + * limits map is left untouched) + */ + public static double refineSlowSensorSamplingRateLimits(SENSORS sensorClassKey, List newlyObservedPeriodsS) { + double medianPeriodS = accumulateSlowSensorPeriodsAndGetMedianPeriodS(sensorClassKey, newlyObservedPeriodsS); + if(!(medianPeriodS>0)) { + return Double.NaN; + } + double medianRateHz = 1.0/medianPeriodS; + SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, calculateSlowSensorSamplingRateLimits(medianRateHz)); + return medianRateHz; + } + public static void clearMapOfSamplingRateLimitsPerSensor() { SAMPLING_RATE_LIMITS_PER_SENSOR.clear(); + // Same lifecycle as the limits map itself - the accumulated slow-sensor + // measurements that the limits are derived from must not survive a CSV-set + // boundary either. + SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.clear(); } public static String isDataBlockContinuous(SENSORS sensorClassKey, DataSegmentDetails dataSegmentDetailsPrevious, DataBlockDetails nextDataBlockDetails) { diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_UtilCsvSplittingSlowSensorGapWindow.java b/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_UtilCsvSplittingSlowSensorGapWindow.java new file mode 100644 index 00000000..7a8f5dd1 --- /dev/null +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_UtilCsvSplittingSlowSensorGapWindow.java @@ -0,0 +1,275 @@ +package com.shimmerresearch.verisense.payloaddesign; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import com.shimmerresearch.driver.Configuration.COMMUNICATION_TYPE; +import com.shimmerresearch.sensors.AbstractSensor.SENSORS; +import com.shimmerresearch.verisense.VerisenseDevice; +import com.shimmerresearch.verisense.payloaddesign.DataBlockDetails.DATABLOCK_SENSOR_ID; +import com.shimmerresearch.verisense.payloaddesign.UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER; + +/** + * Unit tests for the slow-sensor (VD6283 light / MLX90632 skin temp) CSV + * gap-splitting window in {@link UtilCsvSplitting} - the accumulated-median + * estimate that + * {@code PayloadContentsDetailsV8orAbove.refineSlowSensorSamplingRateFromBlockTicks} + * feeds on every payload. + *

+ * The tests drive the window through per-sample periods, in seconds, exactly as + * the payload parser measures them ({@code inter-block ticks / samples-per-block}), + * so no binary test files or hardware recordings are needed. The boundary rate + * the CSV splitter then judges is simply {@code 1/period} - see + * {@link UtilCsvSplitting#isSamplingRateOutsideOfLimits(double[], DataBlockDetails, DataBlockDetails, SENSORS)}, + * which computes samples/second between two consecutive block end times. + *

+ * End-to-end coverage against real recordings lives in + * ASM_PC_00005_VerisenseFileParserPC (ASM_PC repository). + */ +public class API_00009_UtilCsvSplittingSlowSensorGapWindow { + + /** ~9.09 Hz - the achieved VD6283 cadence at the default exposure. */ + private static final double NOMINAL_PERIOD_S = 0.11; + private static final double NOMINAL_RATE_HZ = 1.0/NOMINAL_PERIOD_S; + /** Number of healthy payloads used to build up a history before the payload under test. */ + private static final int HEALTHY_PAYLOAD_COUNT = 20; + /** Slow-sensor blocks per payload is 2-3 in the field, i.e. 1-2 inter-block gaps. */ + private static final int GAPS_PER_PAYLOAD = 2; + + private static final SENSORS SENSOR_UNDER_TEST = SENSORS.VD6283; + + private static final double DELTA = 1e-9; + + @Before + public void resetStaticState() { + // The limits map and the period history are process-wide statics, cleared by + // the file parser at each CSV-set boundary - do the same between tests so + // that they cannot leak into one another. + UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); + } + + // ---------------------------------------------------------------- helpers + + /** Feed one payload's worth of measurements in, as the parser does per payload. */ + private double feedPayload(double... perSamplePeriodsS) { + List periodsS = new ArrayList(); + for(double periodS:perSamplePeriodsS) { + periodsS.add(periodS); + } + return UtilCsvSplitting.refineSlowSensorSamplingRateLimits(SENSOR_UNDER_TEST, periodsS); + } + + /** Build up a history of healthy payloads at the nominal cadence. */ + private void feedHealthyHistory() { + for(int i=0;i callersList = new ArrayList(Arrays.asList(4.0, 1.0, 3.0, 2.0)); + UtilCsvSplitting.calculateMedian(callersList); + assertEquals(Arrays.asList(4.0, 1.0, 3.0, 2.0), callersList); + // Nothing measured yet. + assertTrue(Double.isNaN(UtilCsvSplitting.calculateMedian(new ArrayList()))); + assertTrue(Double.isNaN(UtilCsvSplitting.calculateMedian(null))); + } + + @Test + public void testAccumulatedMedianAveragesTheTwoMiddleValuesForAnEvenCount() { + // Four measurements across two payloads -> the mean of the middle two. + feedPayload(0.10, 0.12); + double medianPeriodS = UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, Arrays.asList(0.14, 0.16)); + assertEquals(0.13, medianPeriodS, DELTA); + } + + // ------------------------------------------------------- window geometry + + @Test + public void testBothLimitsAreDerivedFromTheSameMedian() { + double[] limits = UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(NOMINAL_RATE_HZ); + assertEquals(NOMINAL_RATE_HZ/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, limits[0], DELTA); + assertEquals(NOMINAL_RATE_HZ*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER, limits[1], DELTA); + + // A single fast outlier in the history must not push the fast side out with + // it - the old limits[1] used 1/minObservedPeriod, i.e. that extremum. + feedHealthyHistory(); + double fastOutlierPeriodS = NOMINAL_PERIOD_S/2.0; + feedPayload(NOMINAL_PERIOD_S, fastOutlierPeriodS); + assertArrayEquals(UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(NOMINAL_RATE_HZ), currentLimits(), 1e-6); + assertTrue(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(fastOutlierPeriodS))); + } + + // ------------------------------------------------------ (b) healthy jitter + + @Test + public void testHealthyJitterStaysInsideTheWindow() { + feedHealthyHistory(); + + // +12.5% block spacing with no samples lost - observed on the DEV-927 + // MLX90632 validation recording. This payload contributes to the estimate. + double jitteredPeriodS = NOMINAL_PERIOD_S*1.125; + feedPayload(NOMINAL_PERIOD_S, jitteredPeriodS); + + double[] limits = currentLimits(); + assertFalse("+12.5% block spacing must still be judged continuous", + UtilCsvSplitting.isSamplingRateOutsideOfLimits(limits, boundaryRateHz(jitteredPeriodS))); + assertFalse("the nominal cadence must obviously be judged continuous", + UtilCsvSplitting.isSamplingRateOutsideOfLimits(limits, boundaryRateHz(NOMINAL_PERIOD_S))); + // A single +12.5% sample barely moves the median off the nominal cadence. + assertEquals(NOMINAL_RATE_HZ, 1.0/UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null), 1e-6); + } + + @Test + public void testHealthyJitterStaysInsideTheWindowFromTheVeryFirstPayload() { + // No history at all yet: the first payload with >= 2 blocks is all there is, + // and the window must already be wide enough for the jitter it contains. + double jitteredPeriodS = NOMINAL_PERIOD_S*1.125; + feedPayload(NOMINAL_PERIOD_S, jitteredPeriodS); + assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(jitteredPeriodS))); + assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S))); + } + + // ------------------------------------------------------- (c) dropped block + + @Test + public void testDroppedBlockIsDetectedEvenThoughItsPayloadFedTheEstimate() { + feedHealthyHistory(); + + // A dropped block doubles the spacing. This payload is fed into the estimate + // BEFORE the boundary it contains is judged - exactly the self-referential + // case that a per-payload window could not detect. + double droppedBlockPeriodS = NOMINAL_PERIOD_S*2.0; + feedPayload(NOMINAL_PERIOD_S, droppedBlockPeriodS); + + assertTrue("a 2x inter-block gap must fall outside the window", + UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(droppedBlockPeriodS))); + // The healthy boundary in the same payload must NOT be flagged instead. + assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S))); + // One 2x outlier in ~40 measurements leaves the median where it was. + assertEquals(NOMINAL_RATE_HZ, 1.0/UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null), 1e-6); + } + + @Test + public void testPerPayloadWindowWouldHaveAbsorbedTheDroppedBlock() { + // Regression guard for the finding this change addresses: a window rebuilt + // from ONLY the payload being judged (2 blocks -> 1 or 2 gaps) swallows the + // dropped block into its own centre and reports nothing. + double droppedBlockPeriodS = NOMINAL_PERIOD_S*2.0; + double payloadLocalMedianPeriodS = UtilCsvSplitting.calculateMedian(Arrays.asList(NOMINAL_PERIOD_S, droppedBlockPeriodS)); + double[] payloadLocalLimits = UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(1.0/payloadLocalMedianPeriodS); + assertFalse("baseline: a payload-local window does NOT see the dropped block", + UtilCsvSplitting.isSamplingRateOutsideOfLimits(payloadLocalLimits, boundaryRateHz(droppedBlockPeriodS))); + + // The accumulated window does. + feedHealthyHistory(); + feedPayload(NOMINAL_PERIOD_S, droppedBlockPeriodS); + assertTrue(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(droppedBlockPeriodS))); + } + + // ------------------------------------------- (d) fallback seeding interplay + + @Test + public void testMeasuredWindowWinsOverAFallbackSeededBand() { + // populateExpectedPayloadTsDiffLimitMapIfNeeded seeds a configured-rate + // +/-10% band for any sensor not yet measured - e.g. after a first payload + // that carried fewer than two blocks of this sensor. + double configuredRateHz = 10.0; + UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.put(SENSOR_UNDER_TEST, UtilCsvSplitting.calculateSamplingRateLimits(configuredRateHz)); + + // The header-derived 10 Hz is an estimate; the achieved cadence is ~9.09 Hz, + // which the +/-10% band already calls a gap on every single boundary (this + // fragmented a 25-min DEV-927 recording into 7 CSVs). + assertTrue("baseline: the fallback band is too tight for the achieved cadence", + UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S*1.125))); + + // First measurement must take over immediately - no containsKey guard. + feedPayload(NOMINAL_PERIOD_S, NOMINAL_PERIOD_S); + assertArrayEquals(UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(NOMINAL_RATE_HZ), currentLimits(), 1e-6); + assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S*1.125))); + } + + @Test + public void testFallbackSeedingDoesNotClobberAnExistingMeasuredWindow() { + feedHealthyHistory(); + double[] measuredLimits = currentLimits().clone(); + + // populateExpectedPayloadTsDiffLimitMapIfNeeded runs after the refinement on + // every payload; its containsKey guard must leave the measurement alone. + HashMap> mapOfSensorIdsPerDataBlock = new HashMap>(); + mapOfSensorIdsPerDataBlock.put(DATABLOCK_SENSOR_ID.LIGHT, Arrays.asList(SENSOR_UNDER_TEST)); + UtilCsvSplitting.populateExpectedPayloadTsDiffLimitMapIfNeeded(new VerisenseDevice(COMMUNICATION_TYPE.SD), mapOfSensorIdsPerDataBlock); + + assertArrayEquals(measuredLimits, currentLimits(), DELTA); + } + + // -------------------------------------------------------- lifecycle contract + + @Test + public void testClearingTheLimitsMapAlsoClearsTheAccumulatedPeriods() { + feedHealthyHistory(); + assertEquals(NOMINAL_PERIOD_S, UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null), DELTA); + + // The file parser calls this on each CSV-set boundary: measurements from one + // recording must not survive into the next. + UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); + + assertTrue(UtilCsvSplitting.SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.isEmpty()); + assertTrue(Double.isNaN(UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null))); + assertTrue(currentLimits()==null || currentLimits().length==0); + } + + @Test + public void testPeriodHistoryIsBounded() { + int payloadsToOverflowHistory = (UtilCsvSplitting.SLOW_SENSOR_PERIOD_HISTORY_MAX/GAPS_PER_PAYLOAD)+10; + for(int i=0;i NaN and no limits written. + assertTrue(Double.isNaN(UtilCsvSplitting.refineSlowSensorSamplingRateLimits(SENSOR_UNDER_TEST, new ArrayList()))); + assertTrue(currentLimits()==null); + + // Zero/negative periods (a same-tick or out-of-order block pair) are dropped + // rather than dragging the median to zero. + assertEquals(NOMINAL_RATE_HZ, feedPayload(0.0, -1.0, NOMINAL_PERIOD_S), 1e-6); + } +}