From 500384b23405bdcebd373d5d22615f0673af954a Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:38:27 -0800 Subject: [PATCH] fix(query): harden query_arrow follow and non-Arrow 200 - Enforce max_auto_bytes when a large result is read from the Results API, on the size of the Arrow data; the row guard alone does not bound memory. - A 200 that is not Arrow raises ApiException with the response body kept, instead of a decode error, so the run id of a query that ran is not lost. --- CHANGELOG.md | 4 ++++ README.md | 2 +- hotdata/query.py | 47 ++++++++++++++++++++++++++++++++++++--- tests/test_query_arrow.py | 27 ++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41f85dd..4aee9a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- fix(query): `query_arrow` enforces `max_auto_bytes` when it reads a large result from the Results API, and keeps the response body when a 200 is not Arrow + ## [0.15.0] - 2026-11-19 ### Added diff --git a/README.md b/README.md index a1de6ea..1e1b80b 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ with ApiClient(Configuration(api_key="...", workspace_id="...")) as client: out.query_run_id, out.result_id, out.total_row_count, out.execution_time_ms, out.warning ``` -HTTP 429 is retried as in `query`. A result too large to send inline is read from the Results API instead (`out.followed` is `True`), up to `max_auto_rows`; pass `auto_follow=False` to get a `ResultNotInlineError` carrying the `result_id` and stream it yourself. +HTTP 429 is retried as in `query`. A result too large to send inline is read from the Results API instead (`out.followed` is `True`), up to `max_auto_rows` and `max_auto_bytes` (64 MiB of Arrow data by default; past a guard it raises `ResultTooLargeError`); pass `auto_follow=False` to get a `ResultNotInlineError` carrying the `result_id` and stream it yourself. ## File uploads diff --git a/hotdata/query.py b/hotdata/query.py index 4ebc5b6..28cffd9 100644 --- a/hotdata/query.py +++ b/hotdata/query.py @@ -441,6 +441,7 @@ def query_arrow( *, auto_follow: Optional[bool] = None, max_auto_rows: Any = _UNSET, + max_auto_bytes: Any = _UNSET, ) -> ArrowQueryResult: """Execute a query and get the rows as a :class:`pyarrow.Table`. @@ -454,7 +455,8 @@ def query_arrow( A result too large to send inline is not sent as Arrow. With ``auto_follow`` on, this waits for the saved result and reads all of it from the Results API (``followed`` is then ``True``); ``max_auto_rows`` - guards how much is pulled into memory. + and ``max_auto_bytes`` (the size of the Arrow data) guard how much is + pulled into memory. Async submission (``async=true``) is not supported here: use :meth:`query`, then :meth:`hotdata.arrow.ResultsApi.get_result_arrow`. @@ -462,11 +464,15 @@ def query_arrow( :param auto_follow: Override the instance default for this call. :param max_auto_rows: Override the instance row guard for this call. ``None`` opts into unbounded materialization. + :param max_auto_bytes: Override the instance byte guard for this call. + ``None`` opts into unbounded materialization. :raises ValueError: the request sets ``async``. :raises ServerOverloadedError: 429 retries exhausted. :raises ResultNotInlineError: the result was not sent inline and ``auto_follow`` is off. - :raises ResultTooLargeError: the followed result exceeds ``max_auto_rows``. + :raises ResultTooLargeError: the followed result exceeds a guard (rows/bytes). + :raises ApiException: an HTTP error; also a 200 that is not Arrow, with + the response body kept (the query ran). :raises ResultFailedError: the followed result failed. :raises ResultTimeoutError: the followed result never became ready. :raises ResultUnavailableError: not sent inline, with no ``result_id``. @@ -478,6 +484,9 @@ def query_arrow( ) follow = self.auto_follow if auto_follow is None else auto_follow row_guard = self.max_auto_rows if max_auto_rows is _UNSET else max_auto_rows + byte_guard = ( + self.max_auto_bytes if max_auto_bytes is _UNSET else max_auto_bytes + ) outcome = self._retry_overloaded( lambda: self._send_arrow_query( @@ -506,7 +515,12 @@ def query_arrow( result_id=result_id, query_run_id=ticket.query_run_id ) return self._follow_arrow( - ticket.query_run_id, result_id, effective_db, row_guard, _request_timeout + ticket.query_run_id, + result_id, + effective_db, + row_guard, + byte_guard, + _request_timeout, ) def wait_for_result( @@ -690,6 +704,22 @@ def _send_arrow_query( headers = response_data.headers if response_data.status == 200: + # A 200 that is not Arrow (a server that ignored `Accept`) still ran + # the query. Raise with its body rather than a decode error, so the + # run id is not lost and the caller is not led to re-run a write. + content_type = (_header(headers, "Content-Type") or "").split(";")[0] + if content_type.strip().lower() != ARROW_STREAM_MEDIA_TYPE: + try: + response_data.read() + finally: + response_data.response.release_conn() + raise ApiException( + reason=( + "expected an Arrow stream from POST /v1/query, got " + f"{content_type or 'no content type'}; the query ran" + ), + http_resp=response_data, + ) try: table = ipc.open_stream(response_data.response).read_all() finally: @@ -731,6 +761,7 @@ def _follow_arrow( result_id: str, x_database_id: str, max_auto_rows: Optional[int], + max_auto_bytes: Optional[int], _request_timeout: Any, ) -> ArrowQueryResult: self.wait_for_result(result_id, x_database_id) @@ -759,6 +790,7 @@ def _follow_arrow( batches = [] fetched = 0 + fetched_bytes = 0 # Ask for one row past the guard at most: enough to detect the overflow, # without downloading an oversized result only to discard it. limit = max_auto_rows + 1 if max_auto_rows is not None else None @@ -774,6 +806,15 @@ def _follow_arrow( observed=fetched, limit=max_auto_rows, ) + # The row guard alone does not bound memory: rows can be wide. + fetched_bytes += batch.nbytes + if max_auto_bytes is not None and fetched_bytes > max_auto_bytes: + raise ResultTooLargeError( + result_id=result_id, + kind="bytes", + observed=fetched_bytes, + limit=max_auto_bytes, + ) batches.append(batch) table = pyarrow.Table.from_batches(batches, schema=reader.schema) return ArrowQueryResult( diff --git a/tests/test_query_arrow.py b/tests/test_query_arrow.py index 2f37545..5f8ffcc 100644 --- a/tests/test_query_arrow.py +++ b/tests/test_query_arrow.py @@ -324,3 +324,30 @@ def test_too_large_without_result_id(calls: Any) -> None: _api().query_arrow(REQ, x_database_id="db1", auto_follow=False) assert "qrun1" in str(ei.value) assert "preview" not in str(ei.value) + + +def test_follow_byte_guard(calls: Any) -> None: + calls.responses = [ + _too_large(), + _result_status("ready"), + _query_run(row_count=3), + _arrow_response(_table()), + ] + # Three int64 rows are 24 bytes: within the row guard, past this one. + with pytest.raises(ResultTooLargeError) as ei: + _api().query_arrow(REQ, x_database_id="db1", max_auto_bytes=8) + assert ei.value.kind == "bytes" + assert ei.value.observed == 24 + + +def test_non_arrow_200_keeps_the_body(calls: Any) -> None: + # A server that ignored Accept and answered in JSON: the query ran. + fake = _json_response(200, {"query_run_id": "qrun1", "rows": [[1]]}) + calls.responses = [fake] + with pytest.raises(ApiException) as ei: + _api().query_arrow(REQ) + assert ei.value.status == 200 + assert "qrun1" in (ei.value.body or "") + assert "the query ran" in ei.value.reason + assert len(calls) == 1 # not retried: a write must not run twice + assert fake.release_conn_called