diff --git a/CHANGELOG.md b/CHANGELOG.md index fc82b76..c1cea0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- feat(query): add `QueryApi.query_arrow`, which returns a query's rows as Arrow in one call, with the rollup outcome and other response-header values + ## [0.14.0] - 2026-11-19 ### Changed diff --git a/README.md b/README.md index 362539e..a1de6ea 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,26 @@ with ApiClient(Configuration(api_key="...", workspace_id="...")) as client: Both methods accept `offset` and `limit` for pagination. They raise `hotdata.arrow.ResultNotReadyError` if the result is still pending or processing — poll `results.get_result(result_id, database_id)` until `status == "ready"` first. +### Arrow from the query call + +`QueryApi.query_arrow` runs a query and returns the rows as Arrow in one request, without fetching the stored result afterwards. The values the JSON response would carry come back on the result: + +```python +from hotdata import ApiClient, Configuration, QueryApi, QueryRequest + +with ApiClient(Configuration(api_key="...", workspace_id="...")) as client: + out = QueryApi(client).query_arrow( + QueryRequest(sql="SELECT toStartOfHour(ts) AS h, count() FROM events GROUP BY h", + dialect="clickhouse"), + x_database_id="your_database_id", + ) + out.table # pyarrow.Table, with the server's column types + out.rollup # RollupUsage.USED / NOT_MATCHED / NOT_READY / UNAVAILABLE, or None + out.query_run_id, out.result_id, out.total_row_count, out.execution_time_ms, out.warning +``` + +HTTP 429 is retried as in `query`. A result too large to send inline is read from the Results API instead (`out.followed` is `True`), up to `max_auto_rows`; pass `auto_follow=False` to get a `ResultNotInlineError` carrying the `result_id` and stream it yourself. + ## File uploads `hotdata.uploads.UploadsApi` (also the default `hotdata.UploadsApi`) adds diff --git a/hotdata/query.py b/hotdata/query.py index e14a7c0..317ddc2 100644 --- a/hotdata/query.py +++ b/hotdata/query.py @@ -21,6 +21,12 @@ into client memory — past the guard it raises :class:`ResultTooLargeError`, pointing callers at the streaming Arrow API (:mod:`hotdata.arrow`). +* **Arrow in one call.** :meth:`QueryApi.query_arrow` asks ``POST /v1/query`` + for an Arrow IPC stream, so the rows arrive as a :class:`pyarrow.Table` with + their column types and no second request. The values the JSON body would + carry (run id, result id, row count, timing, warning, rollup outcome) come + back as response headers and are returned on :class:`ArrowQueryResult`. + Use this ``QueryApi`` in place of :class:`hotdata.api.query_api.QueryApi`; every other method on the base class works unchanged. Auto-follow and retry are both configurable per-instance (constructor) and per-call (keyword arguments), and @@ -33,15 +39,22 @@ import random import time from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union from hotdata.api.query_api import QueryApi as _GeneratedQueryApi from hotdata.api.query_runs_api import QueryRunsApi as _QueryRunsApi from hotdata.api.results_api import ResultsApi as _ResultsApi +from hotdata.arrow import ARROW_STREAM_MEDIA_TYPE, _import_pyarrow, _release_conn +from hotdata.arrow import ResultsApi as _ArrowResultsApi from hotdata.exceptions import ApiException +from hotdata.models.async_query_response import AsyncQueryResponse from hotdata.models.query_request import QueryRequest from hotdata.models.query_response import QueryResponse from hotdata.models.results_format_query import ResultsFormatQuery +from hotdata.models.rollup_usage import RollupUsage + +if TYPE_CHECKING: # pragma: no cover - import-time only for type checkers + import pyarrow as pa # type: ignore[import-untyped] # HTTP 429: too many concurrent queries (admission shedding). The server tags # the body with error code OVERLOADED; we key retry off the status code since @@ -218,16 +231,36 @@ class ResultUnavailableError(ResultError): follow (persistence failed — see the response ``warning`` field). """ - def __init__(self, *, warning: Optional[str]) -> None: + def __init__( + self, *, warning: Optional[str], message: Optional[str] = None + ) -> None: self.warning = warning super().__init__( - "Query result is truncated but no result_id is available to fetch " + message + or "Query result is truncated but no result_id is available to fetch " "the full result" + (f": {warning}" if warning else "") + ". Re-run with auto_follow=False to use the preview." ) +class ResultNotInlineError(ResultError): + """Raised by :meth:`QueryApi.query_arrow` when the result is too large to + send inline and ``auto_follow`` is off. The query still ran: read the saved + result with :meth:`hotdata.arrow.ResultsApi.stream_result_arrow` once it is + ready (:meth:`QueryApi.wait_for_result`). + """ + + def __init__(self, *, result_id: str, query_run_id: str) -> None: + self.result_id = result_id + self.query_run_id = query_run_id + super().__init__( + f"Result {result_id} is too large to send inline. Wait for it with " + "wait_for_result, then stream it with " + "hotdata.arrow.ResultsApi.stream_result_arrow, or set auto_follow=True." + ) + + def _parse_retry_after(headers: Any) -> Optional[float]: """Return the ``Retry-After`` delay in seconds, or ``None``. @@ -252,6 +285,59 @@ def _parse_retry_after(headers: Any) -> Optional[float]: return seconds if seconds >= 0 else None +@dataclass(frozen=True) +class ArrowQueryResult: + """A query result as Arrow, with the values the JSON body would carry. + + :param table: The full result. + :param query_run_id: ID of the query run. + :param result_id: ID of the saved result, when one was saved. + :param total_row_count: Rows in the full result. + :param execution_time_ms: Query execution time, when the server reported it. + :param warning: A problem that did not fail the query, if any. + :param rollup: Whether a rollup answered the query. ``None`` when no table + the query read has a rollup, or rollups were turned off for the query. + :param followed: ``True`` when the result was too large to send inline and + was read from the Results API instead (extra round trips). + """ + + table: "pa.Table" + query_run_id: Optional[str] = None + result_id: Optional[str] = None + total_row_count: Optional[int] = None + execution_time_ms: Optional[int] = None + warning: Optional[str] = None + rollup: Optional[RollupUsage] = None + followed: bool = False + + +def _header(headers: Any, name: str) -> Optional[str]: + """One response header, or ``None``. urllib3's HTTPHeaderDict is + case-insensitive; a plain dict may not be, so try lowercase too.""" + getter = getattr(headers, "get", None) + if not callable(getter): + return None + return headers.get(name) or headers.get(name.lower()) + + +def _int_header(headers: Any, name: str) -> Optional[int]: + value = _header(headers, name) + try: + return int(value) if value is not None else None + except ValueError: + return None + + +def _rollup_header(headers: Any) -> Optional[RollupUsage]: + value = _header(headers, "X-Rollup") + try: + return RollupUsage(value) if value else None + except ValueError: + # A value this client does not know yet: the answer is the same either + # way, so report nothing rather than fail the query. + return None + + class QueryApi(_GeneratedQueryApi): """Drop-in replacement for :class:`hotdata.api.query_api.QueryApi` that adds 429 retry and transparent truncation auto-follow. All base-class methods @@ -347,6 +433,82 @@ def query( response, effective_db, row_guard, byte_guard ) + def query_arrow( + self, + query_request: QueryRequest, + x_database_id: Optional[str] = None, + _request_timeout: Union[None, float, Tuple[float, float]] = None, + *, + auto_follow: Optional[bool] = None, + max_auto_rows: Any = _UNSET, + ) -> ArrowQueryResult: + """Execute a query and get the rows as a :class:`pyarrow.Table`. + + One request: ``POST /v1/query`` answers with an Arrow IPC stream, and + the run id, result id, row count, timing, warning and rollup outcome + come back as response headers. Requires ``pip install 'hotdata[arrow]'``. + + HTTP 429 (``OVERLOADED``) is retried per the instance + :class:`RetryPolicy`, as in :meth:`query`. + + A result too large to send inline is not sent as Arrow. With + ``auto_follow`` on, this waits for the saved result and reads all of it + from the Results API (``followed`` is then ``True``); ``max_auto_rows`` + guards how much is pulled into memory. + + Async submission (``async=true``) is not supported here: use + :meth:`query`, then :meth:`hotdata.arrow.ResultsApi.get_result_arrow`. + + :param auto_follow: Override the instance default for this call. + :param max_auto_rows: Override the instance row guard for this call. + ``None`` opts into unbounded materialization. + :raises ValueError: the request sets ``async``. + :raises ServerOverloadedError: 429 retries exhausted. + :raises ResultNotInlineError: the result was not sent inline and + ``auto_follow`` is off. + :raises ResultTooLargeError: the followed result exceeds ``max_auto_rows``. + :raises ResultFailedError: the followed result failed. + :raises ResultTimeoutError: the followed result never became ready. + :raises ResultUnavailableError: not sent inline, with no ``result_id``. + """ + if query_request.var_async: + raise ValueError( + "query_arrow does not support async submission; use query() and " + "then hotdata.arrow.ResultsApi.get_result_arrow" + ) + follow = self.auto_follow if auto_follow is None else auto_follow + row_guard = self.max_auto_rows if max_auto_rows is _UNSET else max_auto_rows + + outcome = self._retry_overloaded( + lambda: self._send_arrow_query( + query_request, x_database_id, _request_timeout + ) + ) + if isinstance(outcome, ArrowQueryResult): + return outcome + + # 202: the result is larger than an inline response holds. The server + # names the saved result in X-Result-Id instead of sending partial rows. + ticket, result_id = outcome + if result_id is None: + raise ResultUnavailableError( + warning=ticket.reason, + message=( + f"Query run {ticket.query_run_id} was not answered inline and " + "names no result_id to read the result from" + + (f": {ticket.reason}" if ticket.reason else "") + + ". Check the run with QueryRunsApi.get_query_run." + ), + ) + effective_db = x_database_id or query_request.database_id or "" + if not follow: + raise ResultNotInlineError( + result_id=result_id, query_run_id=ticket.query_run_id + ) + return self._follow_arrow( + ticket.query_run_id, result_id, effective_db, row_guard, _request_timeout + ) + def wait_for_result( self, result_id: str, @@ -463,13 +625,18 @@ def fetch_ready_result( # -- internals --------------------------------------------------------- def _query_with_retry(self, *args: Any, **kwargs: Any) -> QueryResponse: + return self._retry_overloaded( + lambda: super(QueryApi, self).query(*args, **kwargs) + ) + + def _retry_overloaded(self, send: Callable[[], Any]) -> Any: policy = self.retry deadline = time.monotonic() + policy.deadline_s attempts = 0 while True: attempts += 1 try: - return super().query(*args, **kwargs) + return send() except ApiException as exc: if exc.status != _HTTP_TOO_MANY_REQUESTS: raise @@ -494,6 +661,129 @@ def _query_with_retry(self, *args: Any, **kwargs: Any) -> QueryResponse: ) from exc time.sleep(delay) + def _send_arrow_query( + self, + query_request: QueryRequest, + x_database_id: Optional[str], + _request_timeout: Any, + ) -> Union[ArrowQueryResult, Tuple[AsyncQueryResponse, Optional[str]]]: + """One ``POST /v1/query`` asking for Arrow. Returns the result on 200, + or the run ticket and ``X-Result-Id`` on 202; raises on an error.""" + ipc = _import_pyarrow() + # Same request the generated op builds, with two headers overridden. + # `Accept-Encoding: identity`: see hotdata.arrow.ResultsApi._call_arrow. + params = self._query_serialize( + query_request=query_request, + x_database_id=x_database_id, + _request_auth=None, + _content_type=None, + _headers={ + "Accept": ARROW_STREAM_MEDIA_TYPE, + "Accept-Encoding": "identity", + }, + _host_index=0, + ) + response_data = self.api_client.call_api( + *params, + _request_timeout=_request_timeout, + ) + headers = response_data.headers + + if response_data.status == 200: + try: + table = ipc.open_stream(response_data.response).read_all() + finally: + _release_conn(response_data.response) + return ArrowQueryResult( + table=table, + query_run_id=_header(headers, "X-Query-Run-Id"), + result_id=_header(headers, "X-Result-Id"), + total_row_count=_int_header(headers, "X-Total-Row-Count"), + execution_time_ms=_int_header(headers, "X-Execution-Time-Ms"), + warning=_header(headers, "X-Query-Warning"), + rollup=_rollup_header(headers), + ) + + # Non-200 is JSON. response_deserialize raises ApiException for status + # >= 400 itself (headers attached, so 429 retry sees Retry-After). + try: + response_data.read() + ticket = self.api_client.response_deserialize( + response_data=response_data, + response_types_map={ + "202": "AsyncQueryResponse", + "400": "ApiErrorResponse", + "403": "ApiErrorResponse", + "404": "ApiErrorResponse", + "408": "ApiErrorResponse", + "429": "ApiErrorResponse", + "500": "ApiErrorResponse", + "503": "ApiErrorResponse", + }, + ).data + finally: + response_data.response.release_conn() + return ticket, _header(headers, "X-Result-Id") + + def _follow_arrow( + self, + query_run_id: str, + result_id: str, + x_database_id: str, + max_auto_rows: Optional[int], + _request_timeout: Any, + ) -> ArrowQueryResult: + self.wait_for_result(result_id, x_database_id) + # The run record carries what the inline headers would have: the total, + # timing, warning and rollup outcome. Best effort: the rows matter more. + try: + run = _QueryRunsApi(self.api_client).get_query_run( + query_run_id, x_database_id=x_database_id + ) + except ApiException: + run = None + total = getattr(run, "row_count", None) + _log.info( + "following result %s (%s rows) as Arrow for query run %s", + result_id, + total if total is not None else "unknown", + query_run_id, + ) + if max_auto_rows is not None and total is not None and total > max_auto_rows: + raise ResultTooLargeError( + result_id=result_id, kind="rows", observed=total, limit=max_auto_rows + ) + # Count rows while reading, so the guard also holds when the run lookup + # failed and the total is unknown. + import pyarrow # type: ignore[import-untyped] + + batches = [] + fetched = 0 + with _ArrowResultsApi(self.api_client).stream_result_arrow( + result_id, x_database_id, _request_timeout=_request_timeout + ) as reader: + for batch in reader: + fetched += batch.num_rows + if max_auto_rows is not None and fetched > max_auto_rows: + raise ResultTooLargeError( + result_id=result_id, + kind="rows", + observed=fetched, + limit=max_auto_rows, + ) + batches.append(batch) + table = pyarrow.Table.from_batches(batches, schema=reader.schema) + return ArrowQueryResult( + table=table, + query_run_id=query_run_id, + result_id=result_id, + total_row_count=total if total is not None else table.num_rows, + execution_time_ms=getattr(run, "execution_time_ms", None), + warning=getattr(run, "warning_message", None), + rollup=getattr(run, "rollup", None), + followed=True, + ) + def _backoff_delay(self, exc: ApiException, attempt: int) -> float: """Delay before the next retry: honor ``Retry-After`` when present, else exponential backoff. Jitter is always added on top so retries do @@ -645,6 +935,7 @@ def _fetch_all_rows( __all__ = [ + "ArrowQueryResult", "DEFAULT_MAX_AUTO_BYTES", "DEFAULT_MAX_AUTO_ROWS", "OVERLOADED_ERROR_CODE", @@ -653,6 +944,7 @@ def _fetch_all_rows( "ResultError", "ResultFailedError", "ResultIncompleteError", + "ResultNotInlineError", "ResultTimeoutError", "ResultTooLargeError", "ResultUnavailableError", diff --git a/tests/integration/test_query_arrow.py b/tests/integration/test_query_arrow.py new file mode 100644 index 0000000..a0bb0b7 --- /dev/null +++ b/tests/integration/test_query_arrow.py @@ -0,0 +1,52 @@ +"""Scenario: query_arrow. + +Run a small query with hotdata.query.QueryApi.query_arrow and check that the +rows come back as a pyarrow.Table in one call, with the run and result ids the +server sends as response headers, and that a non-default dialect is accepted. + +pyarrow is a hard test dependency (see test-requirements.txt), so this imports +it directly rather than via importorskip. +""" + +from __future__ import annotations + +import pyarrow as pa +import pytest + +from hotdata.models.query_request import QueryRequest +from hotdata.query import QueryApi + + +@pytest.fixture +def query_api(api_client) -> QueryApi: + return QueryApi(api_client) + + +def test_query_arrow(query_api: QueryApi, database_id: str) -> None: + out = query_api.query_arrow( + QueryRequest( + sql=( + "SELECT x, msg FROM (" + "SELECT 1 AS x, 'hello' AS msg " + "UNION ALL SELECT 2, 'world'" + ") ORDER BY x" + ), + ), + x_database_id=database_id, + ) + assert isinstance(out.table, pa.Table) + assert out.table.column("x").to_pylist() == [1, 2] + assert out.table.column("msg").to_pylist() == ["hello", "world"] + assert out.query_run_id + assert out.total_row_count == 2 + assert out.followed is False + # No table read, so no rollup outcome to report. + assert out.rollup is None + + +def test_query_arrow_clickhouse_dialect(query_api: QueryApi, database_id: str) -> None: + out = query_api.query_arrow( + QueryRequest(sql="SELECT toString(1) AS s", dialect="clickhouse"), + x_database_id=database_id, + ) + assert out.table.column("s").to_pylist() == ["1"] diff --git a/tests/test_query_arrow.py b/tests/test_query_arrow.py new file mode 100644 index 0000000..717878e --- /dev/null +++ b/tests/test_query_arrow.py @@ -0,0 +1,324 @@ +"""Unit tests for hotdata.query.QueryApi.query_arrow. + +These stub the urllib3 transport with a queue of canned responses, so no server +is needed, and feed real Arrow IPC bytes through the SDK plumbing. They cover: + +* the request: ``Accept`` asks for Arrow only, and ``dialect`` rides in the body, +* 200: the table plus the ``X-*`` response headers on the returned result, +* 429 retry, and errors (408) raised with their status, +* 202 (result too large to inline): follow to the Results API, the row guard, + and the ``auto_follow=False`` opt-out, +* async submission is refused before any request is sent. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional +from unittest.mock import patch + +import pytest + +pa = pytest.importorskip("pyarrow") + +from hotdata import ApiClient, Configuration, rest +from hotdata.arrow import ARROW_STREAM_MEDIA_TYPE +from hotdata.exceptions import ApiException +from hotdata.models.query_request import QueryRequest +from hotdata.models.rollup_usage import RollupUsage +from hotdata.query import ( + ArrowQueryResult, + QueryApi, + ResultNotInlineError, + ResultTooLargeError, + ResultUnavailableError, + RetryPolicy, + ServerOverloadedError, +) +from tests.test_arrow import _FakeUrllib3Response, _arrow_bytes + +JSON = {"content-type": "application/json"} + + +def _table() -> Any: + return pa.table({"n": pa.array([1, 2, 3], type=pa.int64())}) + + +def _arrow_response(table: Any, **headers: str) -> _FakeUrllib3Response: + return _FakeUrllib3Response( + status=200, + body=_arrow_bytes(table), + headers={"content-type": ARROW_STREAM_MEDIA_TYPE, **headers}, + ) + + +def _json_response( + status: int, body: Dict[str, Any], **headers: str +) -> _FakeUrllib3Response: + return _FakeUrllib3Response( + status=status, body=json.dumps(body).encode(), headers={**JSON, **headers} + ) + + +def _too_large(result_id: Optional[str] = "rslt1") -> _FakeUrllib3Response: + headers = {"X-Result-Id": result_id} if result_id else {} + return _json_response( + 202, + { + "query_run_id": "qrun1", + "status": "running", + "status_url": "/v1/query-runs/qrun1", + "reason": "the result is larger than an inline response can hold", + }, + **headers, + ) + + +def _result_status(status: str) -> _FakeUrllib3Response: + return _json_response( + 200 if status == "ready" else 202, {"result_id": "rslt1", "status": status} + ) + + +def _query_run(row_count: int) -> _FakeUrllib3Response: + return _json_response( + 200, + { + "id": "qrun1", + "created_at": "2026-01-01T00:00:00Z", + "snapshot_id": "snap1", + "sql_hash": "h", + "sql_text": "SELECT 1", + "status": "succeeded", + "result_id": "rslt1", + "row_count": row_count, + "execution_time_ms": 42, + "rollup": "used", + }, + ) + + +@pytest.fixture +def calls(monkeypatch: pytest.MonkeyPatch) -> List[Dict[str, Any]]: + """Requests sent so far. Queue responses on ``calls.responses``.""" + + class Calls(list): # type: ignore[type-arg] + responses: List[_FakeUrllib3Response] = [] + + sent = Calls() + sent.responses = [] + + def fake_request( + self: Any, + method: str, + url: str, + headers: Optional[Dict[str, str]] = None, + body: Any = None, + post_params: Any = None, + _request_timeout: Any = None, + ) -> rest.RESTResponse: + sent.append( + {"method": method, "url": url, "headers": dict(headers or {}), "body": body} + ) + return rest.RESTResponse(sent.responses.pop(0)) + + monkeypatch.setattr(rest.RESTClientObject, "request", fake_request) + return sent + + +def _api(**kwargs: Any) -> QueryApi: + config = Configuration( + host="https://api.hotdata.test", api_key="test-key", workspace_id="ws_test" + ) + return QueryApi(ApiClient(config), **kwargs) + + +REQ = QueryRequest(sql="SELECT n FROM t", dialect="clickhouse") + + +# --- 200 ---------------------------------------------------------------------- + + +def test_returns_table_and_header_values(calls: Any) -> None: + fake = _arrow_response( + _table(), + **{ + "X-Query-Run-Id": "qrun1", + "X-Result-Id": "rslt1", + "X-Total-Row-Count": "3", + "X-Execution-Time-Ms": "7", + "X-Query-Warning": "careful", + "X-Rollup": "not_matched", + }, + ) + calls.responses = [fake] + + out = _api().query_arrow(REQ, x_database_id="db1") + + assert isinstance(out, ArrowQueryResult) + assert out.table.equals(_table()) + assert out.query_run_id == "qrun1" + assert out.result_id == "rslt1" + assert out.total_row_count == 3 + assert out.execution_time_ms == 7 + assert out.warning == "careful" + assert out.rollup is RollupUsage.NOT_MATCHED + assert out.followed is False + assert fake.drain_conn_called and fake.release_conn_called + + # One round trip, asking for Arrow only, in the caller's dialect. + assert len(calls) == 1 + call = calls[0] + assert call["method"] == "POST" + assert call["url"].endswith("/v1/query") + assert call["headers"]["Accept"] == ARROW_STREAM_MEDIA_TYPE + assert call["headers"]["Accept-Encoding"] == "identity" + assert call["headers"]["X-Database-Id"] == "db1" + assert call["body"]["dialect"] == "clickhouse" + + +def test_absent_headers_are_none(calls: Any) -> None: + calls.responses = [_arrow_response(_table())] + out = _api().query_arrow(REQ) + assert out.rollup is None + assert out.warning is None + assert out.result_id is None + + +def test_unknown_rollup_value_is_ignored(calls: Any) -> None: + calls.responses = [_arrow_response(_table(), **{"X-Rollup": "some_new_value"})] + assert _api().query_arrow(REQ).rollup is None + + +# --- errors ------------------------------------------------------------------- + + +def test_429_is_retried(calls: Any) -> None: + overloaded = {"error": {"code": "OVERLOADED", "message": "busy"}} + calls.responses = [ + _json_response(429, overloaded, **{"Retry-After": "2"}), + _arrow_response(_table()), + ] + with patch("hotdata.query.time.sleep") as sleep, \ + patch("hotdata.query.random.random", return_value=0.0): + out = _api().query_arrow(REQ) + assert out.table.num_rows == 3 + assert len(calls) == 2 + sleep.assert_called_once_with(2.0) + + +def test_429_exhausted_raises_overloaded(calls: Any) -> None: + overloaded = {"error": {"code": "OVERLOADED", "message": "busy"}} + calls.responses = [_json_response(429, overloaded) for _ in range(2)] + with patch("hotdata.query.time.sleep"), \ + patch("hotdata.query.random.random", return_value=0.0): + with pytest.raises(ServerOverloadedError): + _api(retry=RetryPolicy(max_retries=1)).query_arrow(REQ) + + +def test_timeout_raises_with_status(calls: Any) -> None: + fake = _json_response(408, {"error": {"code": "TIMEOUT", "message": "too slow"}}) + calls.responses = [fake] + with pytest.raises(ApiException) as ei: + _api().query_arrow(REQ) + assert ei.value.status == 408 + assert "too slow" in (ei.value.body or "") + assert fake.release_conn_called + + +def test_async_request_is_refused(calls: Any) -> None: + with pytest.raises(ValueError, match="async"): + _api().query_arrow(QueryRequest(sql="SELECT 1", var_async=True)) + assert calls == [] + + +# --- 202: too large to inline ------------------------------------------------- + + +def test_too_large_follows_to_results_api(calls: Any) -> None: + calls.responses = [ + _too_large(), + _result_status("processing"), + _result_status("ready"), + _query_run(row_count=3), + _arrow_response(_table()), + ] + with patch("hotdata.query.time.sleep"): + out = _api().query_arrow(REQ, x_database_id="db1") + + assert out.table.equals(_table()) + assert out.followed is True + assert out.query_run_id == "qrun1" + assert out.result_id == "rslt1" + assert out.total_row_count == 3 + assert out.execution_time_ms == 42 + assert out.rollup is RollupUsage.USED + + urls = [c["url"] for c in calls] + assert "/v1/results/rslt1" in urls[1] and "limit=0" in urls[1] + assert "/v1/query-runs/qrun1" in urls[3] + assert "/v1/results/rslt1" in urls[4] and "format=arrow" in urls[4] + # Every follow-up is scoped to the query's database. + assert all(c["headers"]["X-Database-Id"] == "db1" for c in calls) + + +def test_follow_uses_body_database_id(calls: Any) -> None: + calls.responses = [ + _too_large(), + _result_status("ready"), + _query_run(row_count=3), + _arrow_response(_table()), + ] + _api().query_arrow(QueryRequest(sql="SELECT 1", database_id="db_body")) + assert all(c["headers"]["X-Database-Id"] == "db_body" for c in calls[1:]) + + +def test_follow_row_guard(calls: Any) -> None: + calls.responses = [_too_large(), _result_status("ready"), _query_run(row_count=10)] + with pytest.raises(ResultTooLargeError) as ei: + _api().query_arrow(REQ, x_database_id="db1", max_auto_rows=5) + assert ei.value.observed == 10 + assert len(calls) == 3 # the rows were never fetched + + +def test_too_large_without_auto_follow(calls: Any) -> None: + calls.responses = [_too_large()] + with pytest.raises(ResultNotInlineError) as ei: + _api().query_arrow(REQ, x_database_id="db1", auto_follow=False) + assert ei.value.result_id == "rslt1" + assert ei.value.query_run_id == "qrun1" + assert len(calls) == 1 + + +def test_follow_row_guard_holds_when_run_lookup_fails(calls: Any) -> None: + # No total from the run record: the guard must still stop the read. + calls.responses = [ + _too_large(), + _result_status("ready"), + _json_response(500, {"error": {"code": "INTERNAL", "message": "boom"}}), + _arrow_response(_table()), + ] + with pytest.raises(ResultTooLargeError) as ei: + _api().query_arrow(REQ, x_database_id="db1", max_auto_rows=2) + assert ei.value.observed == 3 + + +def test_follow_survives_failed_run_lookup(calls: Any) -> None: + calls.responses = [ + _too_large(), + _result_status("ready"), + _json_response(500, {"error": {"code": "INTERNAL", "message": "boom"}}), + _arrow_response(_table()), + ] + out = _api().query_arrow(REQ, x_database_id="db1") + assert out.table.equals(_table()) + assert out.total_row_count == 3 + assert out.rollup is None + + +def test_too_large_without_result_id(calls: Any) -> None: + calls.responses = [_too_large(result_id=None)] + with pytest.raises(ResultUnavailableError) as ei: + _api().query_arrow(REQ, x_database_id="db1", auto_follow=False) + assert "qrun1" in str(ei.value) + assert "preview" not in str(ei.value)