Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Fixed

- fix(query): `query_arrow` enforces `max_auto_bytes` when it reads a large result from the Results API, and keeps the response body when a 200 is not Arrow

## [0.15.0] - 2026-11-19

### Added
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ with ApiClient(Configuration(api_key="...", workspace_id="...")) as client:
out.query_run_id, out.result_id, out.total_row_count, out.execution_time_ms, out.warning
```

HTTP 429 is retried as in `query`. A result too large to send inline is read from the Results API instead (`out.followed` is `True`), up to `max_auto_rows`; pass `auto_follow=False` to get a `ResultNotInlineError` carrying the `result_id` and stream it yourself.
HTTP 429 is retried as in `query`. A result too large to send inline is read from the Results API instead (`out.followed` is `True`), up to `max_auto_rows` and `max_auto_bytes` (64 MiB of Arrow data by default; past a guard it raises `ResultTooLargeError`); pass `auto_follow=False` to get a `ResultNotInlineError` carrying the `result_id` and stream it yourself.

## File uploads

Expand Down
47 changes: 44 additions & 3 deletions hotdata/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,7 @@ def query_arrow(
*,
auto_follow: Optional[bool] = None,
max_auto_rows: Any = _UNSET,
max_auto_bytes: Any = _UNSET,
) -> ArrowQueryResult:
"""Execute a query and get the rows as a :class:`pyarrow.Table`.

Expand All @@ -454,19 +455,24 @@ def query_arrow(
A result too large to send inline is not sent as Arrow. With
``auto_follow`` on, this waits for the saved result and reads all of it
from the Results API (``followed`` is then ``True``); ``max_auto_rows``
guards how much is pulled into memory.
and ``max_auto_bytes`` (the size of the Arrow data) guard how much is
pulled into memory.

Async submission (``async=true``) is not supported here: use
:meth:`query`, then :meth:`hotdata.arrow.ResultsApi.get_result_arrow`.

:param auto_follow: Override the instance default for this call.
:param max_auto_rows: Override the instance row guard for this call.
``None`` opts into unbounded materialization.
:param max_auto_bytes: Override the instance byte guard for this call.
``None`` opts into unbounded materialization.
:raises ValueError: the request sets ``async``.
:raises ServerOverloadedError: 429 retries exhausted.
:raises ResultNotInlineError: the result was not sent inline and
``auto_follow`` is off.
:raises ResultTooLargeError: the followed result exceeds ``max_auto_rows``.
:raises ResultTooLargeError: the followed result exceeds a guard (rows/bytes).
:raises ApiException: an HTTP error; also a 200 that is not Arrow, with
the response body kept (the query ran).
:raises ResultFailedError: the followed result failed.
:raises ResultTimeoutError: the followed result never became ready.
:raises ResultUnavailableError: not sent inline, with no ``result_id``.
Expand All @@ -478,6 +484,9 @@ def query_arrow(
)
follow = self.auto_follow if auto_follow is None else auto_follow
row_guard = self.max_auto_rows if max_auto_rows is _UNSET else max_auto_rows
byte_guard = (
self.max_auto_bytes if max_auto_bytes is _UNSET else max_auto_bytes
)

outcome = self._retry_overloaded(
lambda: self._send_arrow_query(
Expand Down Expand Up @@ -506,7 +515,12 @@ def query_arrow(
result_id=result_id, query_run_id=ticket.query_run_id
)
return self._follow_arrow(
ticket.query_run_id, result_id, effective_db, row_guard, _request_timeout
ticket.query_run_id,
result_id,
effective_db,
row_guard,
byte_guard,
_request_timeout,
)

def wait_for_result(
Expand Down Expand Up @@ -690,6 +704,22 @@ def _send_arrow_query(
headers = response_data.headers

if response_data.status == 200:
# A 200 that is not Arrow (a server that ignored `Accept`) still ran
# the query. Raise with its body rather than a decode error, so the
# run id is not lost and the caller is not led to re-run a write.
content_type = (_header(headers, "Content-Type") or "").split(";")[0]
if content_type.strip().lower() != ARROW_STREAM_MEDIA_TYPE:
try:
response_data.read()
finally:
response_data.response.release_conn()
raise ApiException(
reason=(
"expected an Arrow stream from POST /v1/query, got "
f"{content_type or 'no content type'}; the query ran"
),
http_resp=response_data,
)
try:
table = ipc.open_stream(response_data.response).read_all()
finally:
Expand Down Expand Up @@ -731,6 +761,7 @@ def _follow_arrow(
result_id: str,
x_database_id: str,
max_auto_rows: Optional[int],
max_auto_bytes: Optional[int],
_request_timeout: Any,
) -> ArrowQueryResult:
self.wait_for_result(result_id, x_database_id)
Expand Down Expand Up @@ -759,6 +790,7 @@ def _follow_arrow(

batches = []
fetched = 0
fetched_bytes = 0
# Ask for one row past the guard at most: enough to detect the overflow,
# without downloading an oversized result only to discard it.
limit = max_auto_rows + 1 if max_auto_rows is not None else None
Expand All @@ -774,6 +806,15 @@ def _follow_arrow(
observed=fetched,
limit=max_auto_rows,
)
# The row guard alone does not bound memory: rows can be wide.
fetched_bytes += batch.nbytes
if max_auto_bytes is not None and fetched_bytes > max_auto_bytes:
raise ResultTooLargeError(
result_id=result_id,
kind="bytes",
observed=fetched_bytes,
limit=max_auto_bytes,
)
batches.append(batch)
table = pyarrow.Table.from_batches(batches, schema=reader.schema)
return ArrowQueryResult(
Expand Down
27 changes: 27 additions & 0 deletions tests/test_query_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,3 +324,30 @@ def test_too_large_without_result_id(calls: Any) -> None:
_api().query_arrow(REQ, x_database_id="db1", auto_follow=False)
assert "qrun1" in str(ei.value)
assert "preview" not in str(ei.value)


def test_follow_byte_guard(calls: Any) -> None:
calls.responses = [
_too_large(),
_result_status("ready"),
_query_run(row_count=3),
_arrow_response(_table()),
]
# Three int64 rows are 24 bytes: within the row guard, past this one.
with pytest.raises(ResultTooLargeError) as ei:
_api().query_arrow(REQ, x_database_id="db1", max_auto_bytes=8)
assert ei.value.kind == "bytes"
assert ei.value.observed == 24


def test_non_arrow_200_keeps_the_body(calls: Any) -> None:
# A server that ignored Accept and answered in JSON: the query ran.
fake = _json_response(200, {"query_run_id": "qrun1", "rows": [[1]]})
calls.responses = [fake]
with pytest.raises(ApiException) as ei:
_api().query_arrow(REQ)
assert ei.value.status == 200
assert "qrun1" in (ei.value.body or "")
assert "the query ran" in ei.value.reason
assert len(calls) == 1 # not retried: a write must not run twice
assert fake.release_conn_called