Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 21 additions & 11 deletions src/onc/modules/_MultiPage.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,23 +133,33 @@ def _catenateData(self, response: object, nextResponse: object, service: str):
Compatible with the row structure of different services
"""
if service.startswith("scalardata"):
for sensorData in response["sensorData"]:
sensorCode = sensorData["sensorCode"]

# Sensor order may differ between pages, so match by sensor code.
nextSensor = next(
ns
for ns in nextResponse["sensorData"]
if ns["sensorCode"] == sensorCode
)
# Index accumulated sensors by their unique code.
# sensor: {"sensorCode": "clock", "data": {"sampleTimes": [...], ...}, ...}
# sensors: {"clock": <sensor>, ...}
sensors = {}
for sensor in response["sensorData"]:
sensors[sensor["sensorCode"]] = sensor

# Sensors can appear or disappear between pages.
for nextSensor in nextResponse["sensorData"]:
# nextSensor has the same structure as sensor, from the next page.
sensorCode = nextSensor["sensorCode"]
# sensorData is the accumulated sensor dict, or None if it is new.
sensorData = sensors.get(sensorCode)
if sensorData is None:
# Preserve new sensors and make them available for later matches.
response["sensorData"].append(nextSensor)
sensors[sensorCode] = nextSensor
continue

data = sensorData["data"]
nextData = nextSensor["data"]
if not isinstance(data, list):
# Array format stores each field in a separate list.
# Array format: {"sampleTimes": [...], "values": [...], "qaqcFlags": [...]}
for key in data:
data[key] += nextData[key]
else:
# Object format stores whole readings in a single list.
# Object format: [{"sampleTime": "...", "value": 0.0, "qaqcFlag": 1}, ...]
data.extend(nextData)

elif service.startswith("rawdata"):
Expand Down
69 changes: 69 additions & 0 deletions tests/scalar_data/test_scalardata_device.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,5 +78,74 @@ def test_valid_params_multiple_pages_object_format(
assert data_all_pages["next"] is None, "Test should return only one page."


def test_production_multiple_pages_with_missing_sensors(requester):
if not requester.production:
pytest.skip("This regression dataset is only available in production")

# It actually only returns 200 rows.
params = {
"deviceCode": "NANOMETRICSTITANEA000788",
"dateFrom": "2026-11-01T07:00:00.000Z",
"dateTo": "2026-11-01T07:05:00.000Z",
"rowLimit": 1000,
}

# Confirm the two pages reproduce sensors disappearing on the second page.
first_page = requester.getScalardata(params)
assert first_page["next"] is not None, "Regression data must span multiple pages"
second_page = requester.getScalardata(first_page["next"]["parameters"])
assert second_page["next"] is None, "Regression data should have two pages"

first_sensors = {s["sensorCode"]: s["data"] for s in first_page["sensorData"]}
second_sensors = {s["sensorCode"]: s["data"] for s in second_page["sensorData"]}
missing_sensors = first_sensors.keys() - second_sensors.keys()
assert {"clock", "period", "amplitude"} <= missing_sensors
assert second_sensors.keys() <= first_sensors.keys()

# All sensors and readings from both pages must be present in the merged result.
result = requester.getScalardata(params, allPages=True)
assert result["next"] is None
result_sensors = {s["sensorCode"]: s["data"] for s in result["sensorData"]}
assert result_sensors.keys() == first_sensors.keys()
for code, first_data in first_sensors.items():
for key in first_data:
expected = first_data[key] + second_sensors.get(code, {}).get(key, [])
assert result_sensors[code][key] == expected


def test_production_multiple_pages_with_new_sensors(requester):
if not requester.production:
pytest.skip("This regression dataset is only available in production")

params = {
"deviceCode": "NANOMETRICSTITANEA000788",
"dateFrom": "2026-11-01T09:00:00.000Z",
"dateTo": "2026-11-01T09:06:00.000Z",
"rowLimit": 1000,
}

# The sensors begin reporting on the second page.
first_page = requester.getScalardata(params)
assert first_page["next"] is not None, "Regression data must span multiple pages"
second_page = requester.getScalardata(first_page["next"]["parameters"])
assert second_page["next"] is None, "Regression data should have two pages"

first_sensors = {s["sensorCode"]: s["data"] for s in first_page["sensorData"]}
second_sensors = {s["sensorCode"]: s["data"] for s in second_page["sensorData"]}
new_sensors = second_sensors.keys() - first_sensors.keys()
assert {"clock", "period", "amplitude"} <= new_sensors

# The combined response must include each sensor and its readings from both pages.
result = requester.getScalardata(params, allPages=True)
assert result["next"] is None
result_sensors = {s["sensorCode"]: s["data"] for s in result["sensorData"]}
assert result_sensors.keys() == first_sensors.keys() | second_sensors.keys()
for code, data in result_sensors.items():
for key in data:
expected = first_sensors.get(code, {}).get(key, [])
expected = expected + second_sensors.get(code, {}).get(key, [])
assert data[key] == expected


def _get_row_num(data):
return len(data["sensorData"][0]["data"]["values"])
Loading