From 6ad4c42d58b19eb5547c923a9d5b08ee83eca553 Mon Sep 17 00:00:00 2001 From: Kan Fu Date: Tue, 10 Nov 2026 13:41:41 -0700 Subject: [PATCH] bug: consider object format for scalar/raw data when requesting multiple pages --- src/onc/modules/_MultiPage.py | 63 +++++++++++++++---- tests/raw_data/test_rawdata_device.py | 16 +++++ tests/raw_data/test_rawdata_location.py | 16 +++++ tests/scalar_data/test_scalardata_device.py | 16 +++++ tests/scalar_data/test_scalardata_location.py | 16 +++++ 5 files changed, 115 insertions(+), 12 deletions(-) diff --git a/src/onc/modules/_MultiPage.py b/src/onc/modules/_MultiPage.py index af458de..748c70f 100644 --- a/src/onc/modules/_MultiPage.py +++ b/src/onc/modules/_MultiPage.py @@ -133,22 +133,35 @@ def _catenateData(self, response: object, nextResponse: object, service: str): Compatible with the row structure of different services """ if service.startswith("scalardata"): - keys = response["sensorData"][0]["data"].keys() - for sensorData in response["sensorData"]: sensorCode = sensorData["sensorCode"] + # Sensor order may differ between pages, so match by sensor code. nextSensor = next( ns for ns in nextResponse["sensorData"] if ns["sensorCode"] == sensorCode ) - for key in keys: - sensorData["data"][key] += nextSensor["data"][key] + data = sensorData["data"] + nextData = nextSensor["data"] + if not isinstance(data, list): + # Array format stores each field in a separate list. + for key in data: + data[key] += nextData[key] + else: + # Object format stores whole readings in a single list. + data.extend(nextData) elif service.startswith("rawdata"): - for key in response["data"]: - response["data"][key] += nextResponse["data"][key] + data = response["data"] + nextData = nextResponse["data"] + if not isinstance(data, list): + # Array format stores each field in a separate list. + for key in data: + data[key] += nextData[key] + else: + # Object format stores whole readings in a single list. + data.extend(nextData) elif service.startswith("archivefile"): response["files"] += nextResponse["files"] @@ -190,10 +203,22 @@ def _rowCount(self, response, service: str): Returns the number of records in the response """ if service.startswith("scalardata"): - return len(response["sensorData"][0]["data"]["sampleTimes"]) + data = response["sensorData"][0]["data"] + if not isinstance(data, list): + # Default array format: {"sampleTimes": ["...", ...], "values": [...], "qaqcFlags": [...]} + return len(data["sampleTimes"]) + else: + # Object format: [{"sampleTime": "...", "value": 0.0, "qaqcFlag": 1}, ...] + return len(data) elif service.startswith("rawdata"): - return len(response["data"]["times"]) + data = response["data"] + if not isinstance(data, list): + # Default array format: {"times": ["...", ...], "readings": [...], "lineTypes": [...]} + return len(data["times"]) + else: + # Object format: [{"sampleTime": "...", "rawData": "...", "lineType": "..."}, ...] + return len(data) elif service.startswith("archivefile"): return len(response["files"]) @@ -208,12 +233,26 @@ def _responseTimespan(self, response, service: str): # grab the first and last sample times if service.startswith("scalardata"): - first = response["sensorData"][0]["data"]["sampleTimes"][0] - last = response["sensorData"][0]["data"]["sampleTimes"][-1] + data = response["sensorData"][0]["data"] + if not isinstance(data, list): + # Default array format: {"sampleTimes": ["...", ...], "values": [...], "qaqcFlags": [...]} + first = data["sampleTimes"][0] + last = data["sampleTimes"][-1] + else: + # Object format: [{"sampleTime": "...", "value": 0.0, "qaqcFlag": 1}, ...] + first = data[0]["sampleTime"] + last = data[-1]["sampleTime"] elif service.startswith("rawdata"): - first = response["data"]["times"][0] - last = response["data"]["times"][-1] + data = response["data"] + if not isinstance(data, list): + # Default array format: {"times": ["...", ...], "readings": [...], "lineTypes": [...]} + first = data["times"][0] + last = data["times"][-1] + else: + # Object format: [{"sampleTime": "...", "rawData": "...", "lineType": "..."}, ...] + first = data[0]["sampleTime"] + last = data[-1]["sampleTime"] elif service.startswith("archivefile"): row0 = response["files"][0] diff --git a/tests/raw_data/test_rawdata_device.py b/tests/raw_data/test_rawdata_device.py index 6fe6585..aca6495 100644 --- a/tests/raw_data/test_rawdata_device.py +++ b/tests/raw_data/test_rawdata_device.py @@ -72,5 +72,21 @@ def test_valid_params_multi_pages(requester, params_multiple_pages): assert data["next"] is not None, "Test should return multiple pages." +def test_valid_params_multiple_pages_object_format( + requester, params, params_multiple_pages +): + object_format = {"outputFormat": "object"} + data = requester.getRawdata(params | object_format) + data_all_pages = requester.getRawdata( + params_multiple_pages | object_format, allPages=True + ) + + assert isinstance(data_all_pages["data"], list) + assert ( + data_all_pages["data"] == data["data"] + ), "Test should concatenate rows for all pages." + assert data_all_pages["next"] is None, "Test should return only one page." + + def _get_row_num(data): return len(data["data"]["readings"]) diff --git a/tests/raw_data/test_rawdata_location.py b/tests/raw_data/test_rawdata_location.py index 9889c45..415c489 100644 --- a/tests/raw_data/test_rawdata_location.py +++ b/tests/raw_data/test_rawdata_location.py @@ -66,5 +66,21 @@ def test_valid_params_multiple_pages(requester, params_multiple_pages): assert data["next"] is not None, "Test should return multiple pages." +def test_valid_params_multiple_pages_object_format( + requester, params, params_multiple_pages +): + object_format = {"outputFormat": "object"} + data = requester.getRawdata(params | object_format) + data_all_pages = requester.getRawdata( + params_multiple_pages | object_format, allPages=True + ) + + assert isinstance(data_all_pages["data"], list) + assert ( + data_all_pages["data"] == data["data"] + ), "Test should concatenate rows for all pages." + assert data_all_pages["next"] is None, "Test should return only one page." + + def _get_row_num(data): return len(data["data"]["readings"]) diff --git a/tests/scalar_data/test_scalardata_device.py b/tests/scalar_data/test_scalardata_device.py index c85d46a..cea7240 100644 --- a/tests/scalar_data/test_scalardata_device.py +++ b/tests/scalar_data/test_scalardata_device.py @@ -62,5 +62,21 @@ def test_valid_params_multiple_pages(requester, params_multiple_pages): assert data["next"] is not None, "Test should return multiple pages." +def test_valid_params_multiple_pages_object_format( + requester, params_device, params_multiple_pages +): + object_format = {"outputFormat": "object"} + data = requester.getScalardata(params_device | object_format) + data_all_pages = requester.getScalardata( + params_multiple_pages | object_format, allPages=True + ) + + assert isinstance(data_all_pages["sensorData"][0]["data"], list) + assert ( + data_all_pages["sensorData"][0]["data"] == data["sensorData"][0]["data"] + ), "Test should concatenate rows for all pages." + assert data_all_pages["next"] is None, "Test should return only one page." + + def _get_row_num(data): return len(data["sensorData"][0]["data"]["values"]) diff --git a/tests/scalar_data/test_scalardata_location.py b/tests/scalar_data/test_scalardata_location.py index bb38752..4c42ed7 100644 --- a/tests/scalar_data/test_scalardata_location.py +++ b/tests/scalar_data/test_scalardata_location.py @@ -56,5 +56,21 @@ def test_valid_params_multiple_pages(requester, params_multiple_pages): assert data["next"] is not None, "Test should return multiple pages." +def test_valid_params_multiple_pages_object_format( + requester, params_location, params_multiple_pages +): + object_format = {"outputFormat": "object"} + data = requester.getScalardata(params_location | object_format) + data_all_pages = requester.getScalardata( + params_multiple_pages | object_format, allPages=True + ) + + assert isinstance(data_all_pages["sensorData"][0]["data"], list) + assert ( + data_all_pages["sensorData"][0]["data"] == data["sensorData"][0]["data"] + ), "Test should concatenate rows for all pages." + assert data_all_pages["next"] is None, "Test should return only one page." + + def _get_row_num(data): return len(data["sensorData"][0]["data"]["values"])