DEV-793 Widen + always apply the slow-sensor CSV gap window

ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java

@@ -2,7 +2,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map.Entry;
@@ -325,6 +324,13 @@ private boolean isParserAtEndOfBuffer(int bufferLength, int currentByteIndex) {
 	 * {@code inter-block ticks / samples-per-block} is the exact per-sample period.
 	 * With fewer than two blocks in the payload the header-derived estimate the
 	 * blocks were created with is left in place.
+	 * <p>
+	 * The measurements are accumulated per sensor across the payloads of the
+	 * current parse run (see
+	 * {@link UtilCsvSplitting#refineSlowSensorSamplingRateLimits(SENSORS, java.util.List)}),
+	 * and it is the median over that history - not this payload's handful of
+	 * inter-block gaps - that is applied to the blocks and used to (re)derive the
+	 * CSV gap-splitting window.
 	 */
 	private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slowSensorId) {
 		List<DataBlockDetails> slowSensorBlocks = new ArrayList<DataBlockDetails>();
@@ -361,39 +367,61 @@ private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slow
 		if(perSamplePeriodsS.isEmpty()) {
 			return;
 		}
-		// Median so a dropped block (a 2x gap) can't skew the period.
-		Collections.sort(perSamplePeriodsS);
-		double medianPeriodS = perSamplePeriodsS.get(perSamplePeriodsS.size()/2);
-		if(!(medianPeriodS>0)) {
+
+		// Feed this payload's measurements into the sensor's running history and take
+		// the median over EVERYTHING measured so far in this parse run, then seed the
+		// CSV gap-splitting window from it.
+		//
+		// Why accumulate rather than re-derive the window from this payload alone:
+		// the checker (UtilCsvSplitting.isSamplingRateOutsideOfLimits) computes
+		// exactly 1/period for each block boundary, i.e. the very quantities measured
+		// here. A payload only carries 2-3 slow-sensor blocks, so a window re-derived
+		// from just this payload would absorb a dropped block's 2x spacing into its
+		// own median and never flag it - while the healthy boundary back to the
+		// previous payload got flagged instead. Against a history spanning hundreds
+		// of payloads a single 2x outlier barely moves the median, so the real gap
+		// stays outside the window.
+		//
+		// Why the window is needed at all: the header-derived estimate the blocks were
+		// created with can sit within ~1% of a +/-10% band edge (VD6283: 10 Hz
+		// estimated vs ~9.09 Hz achieved), and the slow sensors' cadence is inherently
+		// jittery - the light's is bimodal (exposure vs exposure + dead time: ~100 vs
+		// ~110 ms at the default exposure) and the MLX90632's conversions can slip by
+		// several refresh periods and then catch up (observed +12.5% block spacing
+		// with no samples lost - DEV-927 validation data).
+		double achievedRateHz = Double.NaN;
+		for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) {
+			if(sensorClassKey!=SENSORS.CLOCK) {
+				double medianRateHz = UtilCsvSplitting.refineSlowSensorSamplingRateLimits(sensorClassKey, perSamplePeriodsS);
+				if(Double.isNaN(achievedRateHz)) {
+					// All of a data block's sensor class keys are fed the same
+					// measurements, so they all return the same median - just keep the
+					// first one for the block sampling rate below.
+					achievedRateHz = medianRateHz;
+				}
+			}
+		}
+		if(Double.isNaN(achievedRateHz)) {
+			// No sensor class key was accumulated against (nothing but CLOCK mapped to
+			// this data block id), so fall back to this payload's own median.
+			double medianPeriodS = UtilCsvSplitting.calculateMedian(perSamplePeriodsS);
+			achievedRateHz = medianPeriodS>0? 1.0/medianPeriodS:Double.NaN;
+		}
+		if(!(achievedRateHz>0)) {
 			return;
 		}
 
-		double achievedRateHz = 1.0/medianPeriodS;
+		// The blocks are given the same accumulated median the gap window is built
+		// from, rather than this payload's own possibly-skewed median: the rate is
+		// what the block start times (and hence the CSV timestamps) are back-filled
+		// with, so a payload that happens to contain a dropped block would otherwise
+		// stretch its own samples' spacing by the very artefact the window is meant to
+		// report. Keeping both on one estimate also stops the timestamps and the
+		// continuity check disagreeing about what the achieved cadence is.
 		for(DataBlockDetails dataBlockDetails:slowSensorBlocks) {
 			dataBlockDetails.setSamplingRate(achievedRateHz);
 			dataBlockDetails.calculateTimestampDiffInS();
 		}
-
-		// Seed the CSV gap-splitting window from the OBSERVED period spread rather
-		// than a single-rate +/-10% band. The header-derived estimate can sit within
-		// ~1% of the band edge (VD6283: 10 Hz estimated vs ~9.09 Hz achieved), and
-		// the slow sensors' cadence is inherently bimodal (exposure vs exposure +
-		// dead time: ~100 vs ~110 ms for the light at the default exposure), so a
-		// band centred on ANY single rate can clip real spacing and fragment the
-		// CSV. Spanning [min, max] observed period with the standard tolerance keeps
-		// everything seen continuous while a dropped block (2x period) still splits.
-		// populateExpectedPayloadTsDiffLimitMapIfNeeded runs after this and is
-		// containsKey-guarded, so this seeding wins.
-		double minPeriodS = perSamplePeriodsS.get(0);
-		double maxPeriodS = perSamplePeriodsS.get(perSamplePeriodsS.size()-1);
-		double[] samplingRateLimits = new double[] {
-				(1.0/maxPeriodS)*UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.LOWER,
-				(1.0/minPeriodS)*UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.UPPER};
-		for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) {
-			if(sensorClassKey!=SENSORS.CLOCK && !UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.containsKey(sensorClassKey)) {
-				UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, samplingRateLimits);
-			}
-		}
 	}
 
 	private void backfillDataBlockRwcTimestamps() {

ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java

@@ -1,5 +1,7 @@
 package com.shimmerresearch.verisense.payloaddesign;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 
@@ -11,13 +13,61 @@
 
 public class UtilCsvSplitting {
 
-	public class FILE_GAP_TOLERANCE_MULTIPLIER {
+	public static class FILE_GAP_TOLERANCE_MULTIPLIER {
 		// +/- 10%
 		public static final double UPPER = 1.1;
 		public static final double LOWER = 0.9;
+		/**
+		 * Slow sensors only (VD6283 light / MLX90632 skin temp): the largest
+		 * inter-block gap, as a multiple of the achieved median block spacing, that
+		 * is still treated as continuous. It sets the SLOW (gap) side of the
+		 * sampling-rate window only - the fast side stays on the standard UPPER
+		 * (+10%) tolerance, see {@link
+		 * UtilCsvSplitting#calculateSlowSensorSamplingRateLimits(double)}.
+		 * <p>
+		 * The median it is applied to is accumulated across every payload parsed so
+		 * far in the current parse run (see
+		 * {@link UtilCsvSplitting#refineSlowSensorSamplingRateLimits(SENSORS, List)}),
+		 * not re-derived from the handful of inter-block gaps in the payload
+		 * currently being judged - a payload only carries 2-3 slow-sensor blocks, so
+		 * a per-payload estimate would absorb a dropped block into its own window and
+		 * never report it. Against the accumulated median a single 2x outlier barely
+		 * moves the centre, so the gap stays outside the window.
+		 * <p>
+		 * The band still has to be wide because the slow sensors' cadence is
+		 * inherently jittery even when no samples are lost: the light's is bimodal
+		 * (exposure vs exposure + dead time) and the MLX90632's conversions can slip
+		 * by several refresh periods and then catch up (observed up to +12.5% block
+		 * spacing on the DEV-927 validation recording with no samples lost), which
+		 * routinely violates the standard LOWER (-10%) band. A genuinely dropped
+		 * block doubles the spacing (2x), so 1.5x sits comfortably between healthy
+		 * jitter and a real gap.
+		 */
+		public static final double SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO = 1.5;
 	}
-
-	protected static HashMap<SENSORS, double[]> SAMPLING_RATE_LIMITS_PER_SENSOR = new HashMap<SENSORS, double[]>(); 
+
+	/**
+	 * The maximum number of slow-sensor per-sample periods kept per sensor in
+	 * {@link #SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR}. Once full, the oldest
+	 * measurements are dropped so that the median follows any genuine long-term
+	 * drift in the sensor's cadence while staying deep enough (hundreds of
+	 * payloads' worth of inter-block gaps) that individual dropped blocks cannot
+	 * shift it.
+	 */
+	protected static final int SLOW_SENSOR_PERIOD_HISTORY_MAX = 1024;
+
+	protected static HashMap<SENSORS, double[]> SAMPLING_RATE_LIMITS_PER_SENSOR = new HashMap<SENSORS, double[]>();
+
+	/**
+	 * Slow-sensor (VD6283 light / MLX90632 skin temp) per-sample periods, in
+	 * seconds, as measured from the inter-block tick spacing of every payload
+	 * parsed so far in the current parse run. Shares its lifecycle with
+	 * {@link #SAMPLING_RATE_LIMITS_PER_SENSOR}: both are cleared together by
+	 * {@link #clearMapOfSamplingRateLimitsPerSensor()}, which the file parser calls
+	 * on each CSV-set boundary so that measurements never leak from one recording
+	 * into the next.
+	 */
+	protected static HashMap<SENSORS, List<Double>> SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR = new HashMap<SENSORS, List<Double>>();
 
 	public static boolean isTsDifferenceOutsideOfLimits(double expectedPayloadTsDiffLimits[], double unixTimeInMs_1, double unixTimeInMs_2) {
 		double differenceInMillisec = Math.abs(unixTimeInMs_1 - unixTimeInMs_2);
@@ -86,8 +136,116 @@ public static double[] calculateSamplingRateLimits(double configuredSamplingRate
 		return new double[] {configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.LOWER, configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER};
 	}
 
+	/**
+	 * Slow-sensor (VD6283 light / MLX90632 skin temp) window either side of the
+	 * achieved median rate. Both sides are derived from the SAME robust median so
+	 * that neither edge can be dragged around by a single extreme inter-block
+	 * spacing: the slow (gap) side tolerates up to
+	 * {@link FILE_GAP_TOLERANCE_MULTIPLIER#SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO}
+	 * times the median spacing, the fast side the standard
+	 * {@link FILE_GAP_TOLERANCE_MULTIPLIER#UPPER} tolerance.
+	 *
+	 * @param medianRateHz the achieved median sampling rate, in Hz
+	 * @return {min, max} sampling rate, in Hz, still treated as continuous
+	 */
+	public static double[] calculateSlowSensorSamplingRateLimits(double medianRateHz) {
+		return new double[] {
+				medianRateHz/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO,
+				medianRateHz*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER};
+	}
+
+	/**
+	 * Median of the supplied values. Unlike a bare {@code get(size/2)} this
+	 * averages the two middle values for an even-sized input, and it sorts a copy
+	 * so the caller's list ordering is left alone.
+	 *
+	 * @param values the values to take the median of
+	 * @return the median, or {@link Double#NaN} if there are no values
+	 */
+	public static double calculateMedian(List<Double> values) {
+		if(values==null || values.isEmpty()) {
+			return Double.NaN;
+		}
+		List<Double> sortedValues = new ArrayList<Double>(values);
+		Collections.sort(sortedValues);
+		int size = sortedValues.size();
+		if(size%2==0) {
+			return (sortedValues.get((size/2)-1) + sortedValues.get(size/2))/2.0;
+		}
+		return sortedValues.get(size/2);
+	}
+
+	/**
+	 * Add the per-sample periods measured in the payload just parsed to this
+	 * sensor's running history and return the median over EVERYTHING accumulated so
+	 * far in the current parse run (not just the latest payload's values).
+	 *
+	 * @param sensorClassKey the sensor the periods were measured for
+	 * @param newlyObservedPeriodsS the per-sample periods, in seconds, measured in
+	 *            the payload just parsed (may be empty/null to just read the
+	 *            current median back)
+	 * @return the accumulated median per-sample period, in seconds, or
+	 *         {@link Double#NaN} if nothing has been measured for this sensor yet
+	 */
+	public static double accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSORS sensorClassKey, List<Double> newlyObservedPeriodsS) {
+		List<Double> accumulatedPeriodsS = SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.get(sensorClassKey);
+		if(accumulatedPeriodsS==null) {
+			accumulatedPeriodsS = new ArrayList<Double>();
+			SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.put(sensorClassKey, accumulatedPeriodsS);
+		}
+		if(newlyObservedPeriodsS!=null) {
+			for(Double periodS:newlyObservedPeriodsS) {
+				if(periodS!=null && periodS>0) {
+					accumulatedPeriodsS.add(periodS);
+				}
+			}
+		}
+		// Bounded history: drop the oldest measurements rather than growing without
+		// limit over a multi-day recording.
+		int excess = accumulatedPeriodsS.size()-SLOW_SENSOR_PERIOD_HISTORY_MAX;
+		if(excess>0) {
+			accumulatedPeriodsS.subList(0, excess).clear();
+		}
+		return calculateMedian(accumulatedPeriodsS);
+	}
+
+	/**
+	 * Accumulate the slow-sensor per-sample periods measured in the payload just
+	 * parsed and (re)apply the resulting CSV gap-splitting window for that sensor.
+	 * <p>
+	 * The put into {@link #SAMPLING_RATE_LIMITS_PER_SENSOR} is deliberately
+	 * UNCONDITIONAL. A payload that carries fewer than two blocks of this sensor
+	 * (e.g. the very first payload of a recording) leaves
+	 * {@link #populateExpectedPayloadTsDiffLimitMapIfNeeded(VerisenseDevice, HashMap)}
+	 * to seed a configured-rate +/-10% band first; the header-derived rates for the
+	 * slow sensors are only estimates (the light rate isn't stored at all), so that
+	 * band can be far too tight (observed: a 25-min DEV-927 skin-temp recording
+	 * fragmented into 7 CSVs). A containsKey guard here would lock that estimate in
+	 * for the whole file, so the measured window must win as soon as it exists.
+	 *
+	 * @param sensorClassKey the sensor the periods were measured for
+	 * @param newlyObservedPeriodsS the per-sample periods, in seconds, measured in
+	 *            the payload just parsed
+	 * @return the accumulated median sampling rate, in Hz, or {@link Double#NaN} if
+	 *         nothing has been measured for this sensor yet (in which case the
+	 *         limits map is left untouched)
+	 */
+	public static double refineSlowSensorSamplingRateLimits(SENSORS sensorClassKey, List<Double> newlyObservedPeriodsS) {
+		double medianPeriodS = accumulateSlowSensorPeriodsAndGetMedianPeriodS(sensorClassKey, newlyObservedPeriodsS);
+		if(!(medianPeriodS>0)) {
+			return Double.NaN;
+		}
+		double medianRateHz = 1.0/medianPeriodS;
+		SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, calculateSlowSensorSamplingRateLimits(medianRateHz));
+		return medianRateHz;
+	}
+
 	public static void clearMapOfSamplingRateLimitsPerSensor() {
 		SAMPLING_RATE_LIMITS_PER_SENSOR.clear();
+		// Same lifecycle as the limits map itself - the accumulated slow-sensor
+		// measurements that the limits are derived from must not survive a CSV-set
+		// boundary either.
+		SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.clear();
 	}
 
 	public static String isDataBlockContinuous(SENSORS sensorClassKey, DataSegmentDetails dataSegmentDetailsPrevious, DataBlockDetails nextDataBlockDetails) {