From a04ff3c61600c9c45d857fd02f21f3eb9fd2c200 Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:05:30 -0800 Subject: [PATCH 1/2] feat(query): add query_arrow for Arrow results in one call POST /v1/query can answer with an Arrow IPC stream and carries the run id, result id, row count, timing, warning and rollup outcome as response headers. The generated client cannot decode that body or surface the headers, so add QueryApi.query_arrow, which returns an ArrowQueryResult. 429 is retried as in query(). A result too large to send inline (202 with X-Result-Id) is read from the Results API, under the max_auto_rows guard. --- CHANGELOG.md | 4 + README.md | 20 ++ hotdata/query.py | 268 +++++++++++++++++++++++- tests/integration/test_query_arrow.py | 52 +++++ tests/test_query_arrow.py | 289 ++++++++++++++++++++++++++ 5 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_query_arrow.py create mode 100644 tests/test_query_arrow.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fc82b76..c1cea0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- feat(query): add `QueryApi.query_arrow`, which returns a query's rows as Arrow in one call, with the rollup outcome and other response-header values + ## [0.14.0] - 2026-11-19 ### Changed diff --git a/README.md b/README.md index 362539e..a1de6ea 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,26 @@ with ApiClient(Configuration(api_key="...", workspace_id="...")) as client: Both methods accept `offset` and `limit` for pagination. They raise `hotdata.arrow.ResultNotReadyError` if the result is still pending or processing — poll `results.get_result(result_id, database_id)` until `status == "ready"` first. +### Arrow from the query call + +`QueryApi.query_arrow` runs a query and returns the rows as Arrow in one request, without fetching the stored result afterwards. The values the JSON response would carry come back on the result: + +```python +from hotdata import ApiClient, Configuration, QueryApi, QueryRequest + +with ApiClient(Configuration(api_key="...", workspace_id="...")) as client: + out = QueryApi(client).query_arrow( + QueryRequest(sql="SELECT toStartOfHour(ts) AS h, count() FROM events GROUP BY h", + dialect="clickhouse"), + x_database_id="your_database_id", + ) + out.table # pyarrow.Table, with the server's column types + out.rollup # RollupUsage.USED / NOT_MATCHED / NOT_READY / UNAVAILABLE, or None + out.query_run_id, out.result_id, out.total_row_count, out.execution_time_ms, out.warning +``` + +HTTP 429 is retried as in `query`. A result too large to send inline is read from the Results API instead (`out.followed` is `True`), up to `max_auto_rows`; pass `auto_follow=False` to get a `ResultNotInlineError` carrying the `result_id` and stream it yourself. + ## File uploads `hotdata.uploads.UploadsApi` (also the default `hotdata.UploadsApi`) adds diff --git a/hotdata/query.py b/hotdata/query.py index e14a7c0..2ea0563 100644 --- a/hotdata/query.py +++ b/hotdata/query.py @@ -21,6 +21,12 @@ into client memory — past the guard it raises :class:`ResultTooLargeError`, pointing callers at the streaming Arrow API (:mod:`hotdata.arrow`). +* **Arrow in one call.** :meth:`QueryApi.query_arrow` asks ``POST /v1/query`` + for an Arrow IPC stream, so the rows arrive as a :class:`pyarrow.Table` with + their column types and no second request. The values the JSON body would + carry (run id, result id, row count, timing, warning, rollup outcome) come + back as response headers and are returned on :class:`ArrowQueryResult`. + Use this ``QueryApi`` in place of :class:`hotdata.api.query_api.QueryApi`; every other method on the base class works unchanged. Auto-follow and retry are both configurable per-instance (constructor) and per-call (keyword arguments), and @@ -33,15 +39,22 @@ import random import time from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union from hotdata.api.query_api import QueryApi as _GeneratedQueryApi from hotdata.api.query_runs_api import QueryRunsApi as _QueryRunsApi from hotdata.api.results_api import ResultsApi as _ResultsApi +from hotdata.arrow import ARROW_STREAM_MEDIA_TYPE, _import_pyarrow, _release_conn +from hotdata.arrow import ResultsApi as _ArrowResultsApi from hotdata.exceptions import ApiException +from hotdata.models.async_query_response import AsyncQueryResponse from hotdata.models.query_request import QueryRequest from hotdata.models.query_response import QueryResponse from hotdata.models.results_format_query import ResultsFormatQuery +from hotdata.models.rollup_usage import RollupUsage + +if TYPE_CHECKING: # pragma: no cover - import-time only for type checkers + import pyarrow as pa # type: ignore[import-untyped] # HTTP 429: too many concurrent queries (admission shedding). The server tags # the body with error code OVERLOADED; we key retry off the status code since @@ -228,6 +241,23 @@ def __init__(self, *, warning: Optional[str]) -> None: ) +class ResultNotInlineError(ResultError): + """Raised by :meth:`QueryApi.query_arrow` when the result is too large to + send inline and ``auto_follow`` is off. The query still ran: read the saved + result with :meth:`hotdata.arrow.ResultsApi.stream_result_arrow` once it is + ready (:meth:`QueryApi.wait_for_result`). + """ + + def __init__(self, *, result_id: str, query_run_id: str) -> None: + self.result_id = result_id + self.query_run_id = query_run_id + super().__init__( + f"Result {result_id} is too large to send inline. Wait for it with " + "wait_for_result, then stream it with " + "hotdata.arrow.ResultsApi.stream_result_arrow, or set auto_follow=True." + ) + + def _parse_retry_after(headers: Any) -> Optional[float]: """Return the ``Retry-After`` delay in seconds, or ``None``. @@ -252,6 +282,59 @@ def _parse_retry_after(headers: Any) -> Optional[float]: return seconds if seconds >= 0 else None +@dataclass(frozen=True) +class ArrowQueryResult: + """A query result as Arrow, with the values the JSON body would carry. + + :param table: The full result. + :param query_run_id: ID of the query run. + :param result_id: ID of the saved result, when one was saved. + :param total_row_count: Rows in the full result. + :param execution_time_ms: Query execution time, when the server reported it. + :param warning: A problem that did not fail the query, if any. + :param rollup: Whether a rollup answered the query. ``None`` when no table + the query read has a rollup, or rollups were turned off for the query. + :param followed: ``True`` when the result was too large to send inline and + was read from the Results API instead (extra round trips). + """ + + table: "pa.Table" + query_run_id: Optional[str] = None + result_id: Optional[str] = None + total_row_count: Optional[int] = None + execution_time_ms: Optional[int] = None + warning: Optional[str] = None + rollup: Optional[RollupUsage] = None + followed: bool = False + + +def _header(headers: Any, name: str) -> Optional[str]: + """One response header, or ``None``. urllib3's HTTPHeaderDict is + case-insensitive; a plain dict may not be, so try lowercase too.""" + getter = getattr(headers, "get", None) + if not callable(getter): + return None + return headers.get(name) or headers.get(name.lower()) + + +def _int_header(headers: Any, name: str) -> Optional[int]: + value = _header(headers, name) + try: + return int(value) if value is not None else None + except ValueError: + return None + + +def _rollup_header(headers: Any) -> Optional[RollupUsage]: + value = _header(headers, "X-Rollup") + try: + return RollupUsage(value) if value else None + except ValueError: + # A value this client does not know yet: the answer is the same either + # way, so report nothing rather than fail the query. + return None + + class QueryApi(_GeneratedQueryApi): """Drop-in replacement for :class:`hotdata.api.query_api.QueryApi` that adds 429 retry and transparent truncation auto-follow. All base-class methods @@ -347,6 +430,74 @@ def query( response, effective_db, row_guard, byte_guard ) + def query_arrow( + self, + query_request: QueryRequest, + x_database_id: Optional[str] = None, + _request_timeout: Union[None, float, Tuple[float, float]] = None, + *, + auto_follow: Optional[bool] = None, + max_auto_rows: Any = _UNSET, + ) -> ArrowQueryResult: + """Execute a query and get the rows as a :class:`pyarrow.Table`. + + One request: ``POST /v1/query`` answers with an Arrow IPC stream, and + the run id, result id, row count, timing, warning and rollup outcome + come back as response headers. Requires ``pip install 'hotdata[arrow]'``. + + HTTP 429 (``OVERLOADED``) is retried per the instance + :class:`RetryPolicy`, as in :meth:`query`. + + A result too large to send inline is not sent as Arrow. With + ``auto_follow`` on, this waits for the saved result and reads all of it + from the Results API (``followed`` is then ``True``); ``max_auto_rows`` + guards how much is pulled into memory. + + Async submission (``async=true``) is not supported here: use + :meth:`query`, then :meth:`hotdata.arrow.ResultsApi.get_result_arrow`. + + :param auto_follow: Override the instance default for this call. + :param max_auto_rows: Override the instance row guard for this call. + ``None`` opts into unbounded materialization. + :raises ValueError: the request sets ``async``. + :raises ServerOverloadedError: 429 retries exhausted. + :raises ResultNotInlineError: the result was not sent inline and + ``auto_follow`` is off. + :raises ResultTooLargeError: the followed result exceeds ``max_auto_rows``. + :raises ResultFailedError: the followed result failed. + :raises ResultTimeoutError: the followed result never became ready. + :raises ResultUnavailableError: not sent inline, with no ``result_id``. + """ + if query_request.var_async: + raise ValueError( + "query_arrow does not support async submission; use query() and " + "then hotdata.arrow.ResultsApi.get_result_arrow" + ) + follow = self.auto_follow if auto_follow is None else auto_follow + row_guard = self.max_auto_rows if max_auto_rows is _UNSET else max_auto_rows + + outcome = self._retry_overloaded( + lambda: self._send_arrow_query( + query_request, x_database_id, _request_timeout + ) + ) + if isinstance(outcome, ArrowQueryResult): + return outcome + + # 202: the result is larger than an inline response holds. The server + # names the saved result in X-Result-Id instead of sending partial rows. + ticket, result_id = outcome + if result_id is None: + raise ResultUnavailableError(warning=ticket.reason) + effective_db = x_database_id or query_request.database_id or "" + if not follow: + raise ResultNotInlineError( + result_id=result_id, query_run_id=ticket.query_run_id + ) + return self._follow_arrow( + ticket.query_run_id, result_id, effective_db, row_guard, _request_timeout + ) + def wait_for_result( self, result_id: str, @@ -463,13 +614,18 @@ def fetch_ready_result( # -- internals --------------------------------------------------------- def _query_with_retry(self, *args: Any, **kwargs: Any) -> QueryResponse: + return self._retry_overloaded( + lambda: super(QueryApi, self).query(*args, **kwargs) + ) + + def _retry_overloaded(self, send: Callable[[], Any]) -> Any: policy = self.retry deadline = time.monotonic() + policy.deadline_s attempts = 0 while True: attempts += 1 try: - return super().query(*args, **kwargs) + return send() except ApiException as exc: if exc.status != _HTTP_TOO_MANY_REQUESTS: raise @@ -494,6 +650,112 @@ def _query_with_retry(self, *args: Any, **kwargs: Any) -> QueryResponse: ) from exc time.sleep(delay) + def _send_arrow_query( + self, + query_request: QueryRequest, + x_database_id: Optional[str], + _request_timeout: Any, + ) -> Union[ArrowQueryResult, Tuple[AsyncQueryResponse, Optional[str]]]: + """One ``POST /v1/query`` asking for Arrow. Returns the result on 200, + or the run ticket and ``X-Result-Id`` on 202; raises on an error.""" + ipc = _import_pyarrow() + # Same request the generated op builds, with two headers overridden. + # `Accept-Encoding: identity`: see hotdata.arrow.ResultsApi._call_arrow. + params = self._query_serialize( + query_request=query_request, + x_database_id=x_database_id, + _request_auth=None, + _content_type=None, + _headers={ + "Accept": ARROW_STREAM_MEDIA_TYPE, + "Accept-Encoding": "identity", + }, + _host_index=0, + ) + response_data = self.api_client.call_api( + *params, + _request_timeout=_request_timeout, + ) + headers = response_data.headers + + if response_data.status == 200: + try: + table = ipc.open_stream(response_data.response).read_all() + finally: + _release_conn(response_data.response) + return ArrowQueryResult( + table=table, + query_run_id=_header(headers, "X-Query-Run-Id"), + result_id=_header(headers, "X-Result-Id"), + total_row_count=_int_header(headers, "X-Total-Row-Count"), + execution_time_ms=_int_header(headers, "X-Execution-Time-Ms"), + warning=_header(headers, "X-Query-Warning"), + rollup=_rollup_header(headers), + ) + + # Non-200 is JSON. response_deserialize raises ApiException for status + # >= 400 itself (headers attached, so 429 retry sees Retry-After). + try: + response_data.read() + ticket = self.api_client.response_deserialize( + response_data=response_data, + response_types_map={ + "202": "AsyncQueryResponse", + "400": "ApiErrorResponse", + "403": "ApiErrorResponse", + "404": "ApiErrorResponse", + "408": "ApiErrorResponse", + "429": "ApiErrorResponse", + "500": "ApiErrorResponse", + "503": "ApiErrorResponse", + }, + ).data + finally: + response_data.response.release_conn() + return ticket, _header(headers, "X-Result-Id") + + def _follow_arrow( + self, + query_run_id: str, + result_id: str, + x_database_id: str, + max_auto_rows: Optional[int], + _request_timeout: Any, + ) -> ArrowQueryResult: + self.wait_for_result(result_id, x_database_id) + # The run record carries what the inline headers would have: the total, + # timing, warning and rollup outcome. Best effort: the rows matter more. + try: + run = _QueryRunsApi(self.api_client).get_query_run( + query_run_id, x_database_id=x_database_id + ) + except ApiException: + run = None + total = getattr(run, "row_count", None) + _log.info( + "following result %s (%s rows) as Arrow for query run %s", + result_id, + total if total is not None else "unknown", + query_run_id, + ) + if max_auto_rows is not None and total is not None and total > max_auto_rows: + raise ResultTooLargeError( + result_id=result_id, kind="rows", observed=total, limit=max_auto_rows + ) + table = _ArrowResultsApi(self.api_client).get_result_arrow( + result_id, x_database_id, _request_timeout=_request_timeout + ) + return ArrowQueryResult( + table=table, + query_run_id=query_run_id, + result_id=result_id, + total_row_count=total if total is not None else table.num_rows, + execution_time_ms=getattr(run, "execution_time_ms", None), + warning=getattr(run, "warning_message", None), + rollup=getattr(run, "rollup", None), + followed=True, + ) + def _backoff_delay(self, exc: ApiException, attempt: int) -> float: """Delay before the next retry: honor ``Retry-After`` when present, else exponential backoff. Jitter is always added on top so retries do @@ -645,6 +907,7 @@ def _fetch_all_rows( __all__ = [ + "ArrowQueryResult", "DEFAULT_MAX_AUTO_BYTES", "DEFAULT_MAX_AUTO_ROWS", "OVERLOADED_ERROR_CODE", @@ -653,6 +916,7 @@ def _fetch_all_rows( "ResultError", "ResultFailedError", "ResultIncompleteError", + "ResultNotInlineError", "ResultTimeoutError", "ResultTooLargeError", "ResultUnavailableError", diff --git a/tests/integration/test_query_arrow.py b/tests/integration/test_query_arrow.py new file mode 100644 index 0000000..a0bb0b7 --- /dev/null +++ b/tests/integration/test_query_arrow.py @@ -0,0 +1,52 @@ +"""Scenario: query_arrow. + +Run a small query with hotdata.query.QueryApi.query_arrow and check that the +rows come back as a pyarrow.Table in one call, with the run and result ids the +server sends as response headers, and that a non-default dialect is accepted. + +pyarrow is a hard test dependency (see test-requirements.txt), so this imports +it directly rather than via importorskip. +""" + +from __future__ import annotations + +import pyarrow as pa +import pytest + +from hotdata.models.query_request import QueryRequest +from hotdata.query import QueryApi + + +@pytest.fixture +def query_api(api_client) -> QueryApi: + return QueryApi(api_client) + + +def test_query_arrow(query_api: QueryApi, database_id: str) -> None: + out = query_api.query_arrow( + QueryRequest( + sql=( + "SELECT x, msg FROM (" + "SELECT 1 AS x, 'hello' AS msg " + "UNION ALL SELECT 2, 'world'" + ") ORDER BY x" + ), + ), + x_database_id=database_id, + ) + assert isinstance(out.table, pa.Table) + assert out.table.column("x").to_pylist() == [1, 2] + assert out.table.column("msg").to_pylist() == ["hello", "world"] + assert out.query_run_id + assert out.total_row_count == 2 + assert out.followed is False + # No table read, so no rollup outcome to report. + assert out.rollup is None + + +def test_query_arrow_clickhouse_dialect(query_api: QueryApi, database_id: str) -> None: + out = query_api.query_arrow( + QueryRequest(sql="SELECT toString(1) AS s", dialect="clickhouse"), + x_database_id=database_id, + ) + assert out.table.column("s").to_pylist() == ["1"] diff --git a/tests/test_query_arrow.py b/tests/test_query_arrow.py new file mode 100644 index 0000000..48d0959 --- /dev/null +++ b/tests/test_query_arrow.py @@ -0,0 +1,289 @@ +"""Unit tests for hotdata.query.QueryApi.query_arrow. + +These stub the urllib3 transport with a queue of canned responses, so no server +is needed, and feed real Arrow IPC bytes through the SDK plumbing. They cover: + +* the request: ``Accept`` asks for Arrow only, and ``dialect`` rides in the body, +* 200: the table plus the ``X-*`` response headers on the returned result, +* 429 retry, and errors (408) raised with their status, +* 202 (result too large to inline): follow to the Results API, the row guard, + and the ``auto_follow=False`` opt-out, +* async submission is refused before any request is sent. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional +from unittest.mock import patch + +import pytest + +pa = pytest.importorskip("pyarrow") + +from hotdata import ApiClient, Configuration, rest +from hotdata.arrow import ARROW_STREAM_MEDIA_TYPE +from hotdata.exceptions import ApiException +from hotdata.models.query_request import QueryRequest +from hotdata.models.rollup_usage import RollupUsage +from hotdata.query import ( + ArrowQueryResult, + QueryApi, + ResultNotInlineError, + ResultTooLargeError, + RetryPolicy, + ServerOverloadedError, +) +from tests.test_arrow import _FakeUrllib3Response, _arrow_bytes + +JSON = {"content-type": "application/json"} + + +def _table() -> Any: + return pa.table({"n": pa.array([1, 2, 3], type=pa.int64())}) + + +def _arrow_response(table: Any, **headers: str) -> _FakeUrllib3Response: + return _FakeUrllib3Response( + status=200, + body=_arrow_bytes(table), + headers={"content-type": ARROW_STREAM_MEDIA_TYPE, **headers}, + ) + + +def _json_response( + status: int, body: Dict[str, Any], **headers: str +) -> _FakeUrllib3Response: + return _FakeUrllib3Response( + status=status, body=json.dumps(body).encode(), headers={**JSON, **headers} + ) + + +def _too_large(result_id: Optional[str] = "rslt1") -> _FakeUrllib3Response: + headers = {"X-Result-Id": result_id} if result_id else {} + return _json_response( + 202, + { + "query_run_id": "qrun1", + "status": "running", + "status_url": "/v1/query-runs/qrun1", + "reason": "the result is larger than an inline response can hold", + }, + **headers, + ) + + +def _result_status(status: str) -> _FakeUrllib3Response: + return _json_response( + 200 if status == "ready" else 202, {"result_id": "rslt1", "status": status} + ) + + +def _query_run(row_count: int) -> _FakeUrllib3Response: + return _json_response( + 200, + { + "id": "qrun1", + "created_at": "2026-01-01T00:00:00Z", + "snapshot_id": "snap1", + "sql_hash": "h", + "sql_text": "SELECT 1", + "status": "succeeded", + "result_id": "rslt1", + "row_count": row_count, + "execution_time_ms": 42, + "rollup": "used", + }, + ) + + +@pytest.fixture +def calls(monkeypatch: pytest.MonkeyPatch) -> List[Dict[str, Any]]: + """Requests sent so far. Queue responses on ``calls.responses``.""" + + class Calls(list): # type: ignore[type-arg] + responses: List[_FakeUrllib3Response] = [] + + sent = Calls() + sent.responses = [] + + def fake_request( + self: Any, + method: str, + url: str, + headers: Optional[Dict[str, str]] = None, + body: Any = None, + post_params: Any = None, + _request_timeout: Any = None, + ) -> rest.RESTResponse: + sent.append( + {"method": method, "url": url, "headers": dict(headers or {}), "body": body} + ) + return rest.RESTResponse(sent.responses.pop(0)) + + monkeypatch.setattr(rest.RESTClientObject, "request", fake_request) + return sent + + +def _api(**kwargs: Any) -> QueryApi: + config = Configuration( + host="https://api.hotdata.test", api_key="test-key", workspace_id="ws_test" + ) + return QueryApi(ApiClient(config), **kwargs) + + +REQ = QueryRequest(sql="SELECT n FROM t", dialect="clickhouse") + + +# --- 200 ---------------------------------------------------------------------- + + +def test_returns_table_and_header_values(calls: Any) -> None: + fake = _arrow_response( + _table(), + **{ + "X-Query-Run-Id": "qrun1", + "X-Result-Id": "rslt1", + "X-Total-Row-Count": "3", + "X-Execution-Time-Ms": "7", + "X-Query-Warning": "careful", + "X-Rollup": "not_matched", + }, + ) + calls.responses = [fake] + + out = _api().query_arrow(REQ, x_database_id="db1") + + assert isinstance(out, ArrowQueryResult) + assert out.table.equals(_table()) + assert out.query_run_id == "qrun1" + assert out.result_id == "rslt1" + assert out.total_row_count == 3 + assert out.execution_time_ms == 7 + assert out.warning == "careful" + assert out.rollup is RollupUsage.NOT_MATCHED + assert out.followed is False + assert fake.drain_conn_called and fake.release_conn_called + + # One round trip, asking for Arrow only, in the caller's dialect. + assert len(calls) == 1 + call = calls[0] + assert call["method"] == "POST" + assert call["url"].endswith("/v1/query") + assert call["headers"]["Accept"] == ARROW_STREAM_MEDIA_TYPE + assert call["headers"]["Accept-Encoding"] == "identity" + assert call["headers"]["X-Database-Id"] == "db1" + assert call["body"]["dialect"] == "clickhouse" + + +def test_absent_headers_are_none(calls: Any) -> None: + calls.responses = [_arrow_response(_table())] + out = _api().query_arrow(REQ) + assert out.rollup is None + assert out.warning is None + assert out.result_id is None + + +def test_unknown_rollup_value_is_ignored(calls: Any) -> None: + calls.responses = [_arrow_response(_table(), **{"X-Rollup": "some_new_value"})] + assert _api().query_arrow(REQ).rollup is None + + +# --- errors ------------------------------------------------------------------- + + +def test_429_is_retried(calls: Any) -> None: + overloaded = {"error": {"code": "OVERLOADED", "message": "busy"}} + calls.responses = [ + _json_response(429, overloaded, **{"Retry-After": "2"}), + _arrow_response(_table()), + ] + with patch("hotdata.query.time.sleep") as sleep, \ + patch("hotdata.query.random.random", return_value=0.0): + out = _api().query_arrow(REQ) + assert out.table.num_rows == 3 + assert len(calls) == 2 + sleep.assert_called_once_with(2.0) + + +def test_429_exhausted_raises_overloaded(calls: Any) -> None: + overloaded = {"error": {"code": "OVERLOADED", "message": "busy"}} + calls.responses = [_json_response(429, overloaded) for _ in range(2)] + with patch("hotdata.query.time.sleep"), \ + patch("hotdata.query.random.random", return_value=0.0): + with pytest.raises(ServerOverloadedError): + _api(retry=RetryPolicy(max_retries=1)).query_arrow(REQ) + + +def test_timeout_raises_with_status(calls: Any) -> None: + fake = _json_response(408, {"error": {"code": "TIMEOUT", "message": "too slow"}}) + calls.responses = [fake] + with pytest.raises(ApiException) as ei: + _api().query_arrow(REQ) + assert ei.value.status == 408 + assert "too slow" in (ei.value.body or "") + assert fake.release_conn_called + + +def test_async_request_is_refused(calls: Any) -> None: + with pytest.raises(ValueError, match="async"): + _api().query_arrow(QueryRequest(sql="SELECT 1", var_async=True)) + assert calls == [] + + +# --- 202: too large to inline ------------------------------------------------- + + +def test_too_large_follows_to_results_api(calls: Any) -> None: + calls.responses = [ + _too_large(), + _result_status("processing"), + _result_status("ready"), + _query_run(row_count=3), + _arrow_response(_table()), + ] + with patch("hotdata.query.time.sleep"): + out = _api().query_arrow(REQ, x_database_id="db1") + + assert out.table.equals(_table()) + assert out.followed is True + assert out.query_run_id == "qrun1" + assert out.result_id == "rslt1" + assert out.total_row_count == 3 + assert out.execution_time_ms == 42 + assert out.rollup is RollupUsage.USED + + urls = [c["url"] for c in calls] + assert "/v1/results/rslt1" in urls[1] and "limit=0" in urls[1] + assert "/v1/query-runs/qrun1" in urls[3] + assert "/v1/results/rslt1" in urls[4] and "format=arrow" in urls[4] + # Every follow-up is scoped to the query's database. + assert all(c["headers"]["X-Database-Id"] == "db1" for c in calls) + + +def test_follow_uses_body_database_id(calls: Any) -> None: + calls.responses = [ + _too_large(), + _result_status("ready"), + _query_run(row_count=3), + _arrow_response(_table()), + ] + _api().query_arrow(QueryRequest(sql="SELECT 1", database_id="db_body")) + assert all(c["headers"]["X-Database-Id"] == "db_body" for c in calls[1:]) + + +def test_follow_row_guard(calls: Any) -> None: + calls.responses = [_too_large(), _result_status("ready"), _query_run(row_count=10)] + with pytest.raises(ResultTooLargeError) as ei: + _api().query_arrow(REQ, x_database_id="db1", max_auto_rows=5) + assert ei.value.observed == 10 + assert len(calls) == 3 # the rows were never fetched + + +def test_too_large_without_auto_follow(calls: Any) -> None: + calls.responses = [_too_large()] + with pytest.raises(ResultNotInlineError) as ei: + _api().query_arrow(REQ, x_database_id="db1", auto_follow=False) + assert ei.value.result_id == "rslt1" + assert ei.value.query_run_id == "qrun1" + assert len(calls) == 1 From 2929b46b4050f80753a8a0bd43a656639e89f437 Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:10:31 -0800 Subject: [PATCH 2/2] fix(query): keep the arrow follow row guard without a total Count rows while reading the followed result, so max_auto_rows holds when the query-run lookup fails. Give the no-result-id case a message that fits query_arrow. --- hotdata/query.py | 38 +++++++++++++++++++++++++++++++++----- tests/test_query_arrow.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/hotdata/query.py b/hotdata/query.py index 2ea0563..317ddc2 100644 --- a/hotdata/query.py +++ b/hotdata/query.py @@ -231,10 +231,13 @@ class ResultUnavailableError(ResultError): follow (persistence failed — see the response ``warning`` field). """ - def __init__(self, *, warning: Optional[str]) -> None: + def __init__( + self, *, warning: Optional[str], message: Optional[str] = None + ) -> None: self.warning = warning super().__init__( - "Query result is truncated but no result_id is available to fetch " + message + or "Query result is truncated but no result_id is available to fetch " "the full result" + (f": {warning}" if warning else "") + ". Re-run with auto_follow=False to use the preview." @@ -488,7 +491,15 @@ def query_arrow( # names the saved result in X-Result-Id instead of sending partial rows. ticket, result_id = outcome if result_id is None: - raise ResultUnavailableError(warning=ticket.reason) + raise ResultUnavailableError( + warning=ticket.reason, + message=( + f"Query run {ticket.query_run_id} was not answered inline and " + "names no result_id to read the result from" + + (f": {ticket.reason}" if ticket.reason else "") + + ". Check the run with QueryRunsApi.get_query_run." + ), + ) effective_db = x_database_id or query_request.database_id or "" if not follow: raise ResultNotInlineError( @@ -742,9 +753,26 @@ def _follow_arrow( raise ResultTooLargeError( result_id=result_id, kind="rows", observed=total, limit=max_auto_rows ) - table = _ArrowResultsApi(self.api_client).get_result_arrow( + # Count rows while reading, so the guard also holds when the run lookup + # failed and the total is unknown. + import pyarrow # type: ignore[import-untyped] + + batches = [] + fetched = 0 + with _ArrowResultsApi(self.api_client).stream_result_arrow( result_id, x_database_id, _request_timeout=_request_timeout - ) + ) as reader: + for batch in reader: + fetched += batch.num_rows + if max_auto_rows is not None and fetched > max_auto_rows: + raise ResultTooLargeError( + result_id=result_id, + kind="rows", + observed=fetched, + limit=max_auto_rows, + ) + batches.append(batch) + table = pyarrow.Table.from_batches(batches, schema=reader.schema) return ArrowQueryResult( table=table, query_run_id=query_run_id, diff --git a/tests/test_query_arrow.py b/tests/test_query_arrow.py index 48d0959..717878e 100644 --- a/tests/test_query_arrow.py +++ b/tests/test_query_arrow.py @@ -31,6 +31,7 @@ QueryApi, ResultNotInlineError, ResultTooLargeError, + ResultUnavailableError, RetryPolicy, ServerOverloadedError, ) @@ -287,3 +288,37 @@ def test_too_large_without_auto_follow(calls: Any) -> None: assert ei.value.result_id == "rslt1" assert ei.value.query_run_id == "qrun1" assert len(calls) == 1 + + +def test_follow_row_guard_holds_when_run_lookup_fails(calls: Any) -> None: + # No total from the run record: the guard must still stop the read. + calls.responses = [ + _too_large(), + _result_status("ready"), + _json_response(500, {"error": {"code": "INTERNAL", "message": "boom"}}), + _arrow_response(_table()), + ] + with pytest.raises(ResultTooLargeError) as ei: + _api().query_arrow(REQ, x_database_id="db1", max_auto_rows=2) + assert ei.value.observed == 3 + + +def test_follow_survives_failed_run_lookup(calls: Any) -> None: + calls.responses = [ + _too_large(), + _result_status("ready"), + _json_response(500, {"error": {"code": "INTERNAL", "message": "boom"}}), + _arrow_response(_table()), + ] + out = _api().query_arrow(REQ, x_database_id="db1") + assert out.table.equals(_table()) + assert out.total_row_count == 3 + assert out.rollup is None + + +def test_too_large_without_result_id(calls: Any) -> None: + calls.responses = [_too_large(result_id=None)] + with pytest.raises(ResultUnavailableError) as ei: + _api().query_arrow(REQ, x_database_id="db1", auto_follow=False) + assert "qrun1" in str(ei.value) + assert "preview" not in str(ei.value)