From cbc08eac2f950af2660a7ecd8011872b20e4d9b1 Mon Sep 17 00:00:00 2001 From: Kan Fu Date: Mon, 16 Nov 2026 11:03:42 -0700 Subject: [PATCH] bug: fix sensor data handling for multi-page responses to accommodate missing and new sensors --- src/onc/modules/_MultiPage.py | 32 ++++++---- tests/scalar_data/test_scalardata_device.py | 69 +++++++++++++++++++++ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/src/onc/modules/_MultiPage.py b/src/onc/modules/_MultiPage.py index 748c70f..59a5e00 100644 --- a/src/onc/modules/_MultiPage.py +++ b/src/onc/modules/_MultiPage.py @@ -133,23 +133,33 @@ def _catenateData(self, response: object, nextResponse: object, service: str): Compatible with the row structure of different services """ if service.startswith("scalardata"): - for sensorData in response["sensorData"]: - sensorCode = sensorData["sensorCode"] - - # Sensor order may differ between pages, so match by sensor code. - nextSensor = next( - ns - for ns in nextResponse["sensorData"] - if ns["sensorCode"] == sensorCode - ) + # Index accumulated sensors by their unique code. + # sensor: {"sensorCode": "clock", "data": {"sampleTimes": [...], ...}, ...} + # sensors: {"clock": , ...} + sensors = {} + for sensor in response["sensorData"]: + sensors[sensor["sensorCode"]] = sensor + + # Sensors can appear or disappear between pages. + for nextSensor in nextResponse["sensorData"]: + # nextSensor has the same structure as sensor, from the next page. + sensorCode = nextSensor["sensorCode"] + # sensorData is the accumulated sensor dict, or None if it is new. + sensorData = sensors.get(sensorCode) + if sensorData is None: + # Preserve new sensors and make them available for later matches. + response["sensorData"].append(nextSensor) + sensors[sensorCode] = nextSensor + continue + data = sensorData["data"] nextData = nextSensor["data"] if not isinstance(data, list): - # Array format stores each field in a separate list. + # Array format: {"sampleTimes": [...], "values": [...], "qaqcFlags": [...]} for key in data: data[key] += nextData[key] else: - # Object format stores whole readings in a single list. + # Object format: [{"sampleTime": "...", "value": 0.0, "qaqcFlag": 1}, ...] data.extend(nextData) elif service.startswith("rawdata"): diff --git a/tests/scalar_data/test_scalardata_device.py b/tests/scalar_data/test_scalardata_device.py index cea7240..42c37b5 100644 --- a/tests/scalar_data/test_scalardata_device.py +++ b/tests/scalar_data/test_scalardata_device.py @@ -78,5 +78,74 @@ def test_valid_params_multiple_pages_object_format( assert data_all_pages["next"] is None, "Test should return only one page." +def test_production_multiple_pages_with_missing_sensors(requester): + if not requester.production: + pytest.skip("This regression dataset is only available in production") + + # It actually only returns 200 rows. + params = { + "deviceCode": "NANOMETRICSTITANEA000788", + "dateFrom": "2026-11-01T07:00:00.000Z", + "dateTo": "2026-11-01T07:05:00.000Z", + "rowLimit": 1000, + } + + # Confirm the two pages reproduce sensors disappearing on the second page. + first_page = requester.getScalardata(params) + assert first_page["next"] is not None, "Regression data must span multiple pages" + second_page = requester.getScalardata(first_page["next"]["parameters"]) + assert second_page["next"] is None, "Regression data should have two pages" + + first_sensors = {s["sensorCode"]: s["data"] for s in first_page["sensorData"]} + second_sensors = {s["sensorCode"]: s["data"] for s in second_page["sensorData"]} + missing_sensors = first_sensors.keys() - second_sensors.keys() + assert {"clock", "period", "amplitude"} <= missing_sensors + assert second_sensors.keys() <= first_sensors.keys() + + # All sensors and readings from both pages must be present in the merged result. + result = requester.getScalardata(params, allPages=True) + assert result["next"] is None + result_sensors = {s["sensorCode"]: s["data"] for s in result["sensorData"]} + assert result_sensors.keys() == first_sensors.keys() + for code, first_data in first_sensors.items(): + for key in first_data: + expected = first_data[key] + second_sensors.get(code, {}).get(key, []) + assert result_sensors[code][key] == expected + + +def test_production_multiple_pages_with_new_sensors(requester): + if not requester.production: + pytest.skip("This regression dataset is only available in production") + + params = { + "deviceCode": "NANOMETRICSTITANEA000788", + "dateFrom": "2026-11-01T09:00:00.000Z", + "dateTo": "2026-11-01T09:06:00.000Z", + "rowLimit": 1000, + } + + # The sensors begin reporting on the second page. + first_page = requester.getScalardata(params) + assert first_page["next"] is not None, "Regression data must span multiple pages" + second_page = requester.getScalardata(first_page["next"]["parameters"]) + assert second_page["next"] is None, "Regression data should have two pages" + + first_sensors = {s["sensorCode"]: s["data"] for s in first_page["sensorData"]} + second_sensors = {s["sensorCode"]: s["data"] for s in second_page["sensorData"]} + new_sensors = second_sensors.keys() - first_sensors.keys() + assert {"clock", "period", "amplitude"} <= new_sensors + + # The combined response must include each sensor and its readings from both pages. + result = requester.getScalardata(params, allPages=True) + assert result["next"] is None + result_sensors = {s["sensorCode"]: s["data"] for s in result["sensorData"]} + assert result_sensors.keys() == first_sensors.keys() | second_sensors.keys() + for code, data in result_sensors.items(): + for key in data: + expected = first_sensors.get(code, {}).get(key, []) + expected = expected + second_sensors.get(code, {}).get(key, []) + assert data[key] == expected + + def _get_row_num(data): return len(data["sensorData"][0]["data"]["values"])