diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 5cc23cc3e..ff446a41a 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -6,6 +6,7 @@ from __future__ import annotations import copy +import functools from dataclasses import dataclass import os import re @@ -120,6 +121,13 @@ r"(?:\+(?P[a-zA-Z\d\-.]*))?" ) +# Subclasses of 'requests.exceptions.ConnectionError' that won't be fixed +# by sending the request again +_NOT_RETRIABLE_CONNECTION_ERRORS = ( + requests.exceptions.SSLError, + requests.exceptions.ProxyError, +) + def _get_requests_ca_bundle() -> str | None: """CA bundle used by 'requests' for ssl verification. @@ -427,15 +435,6 @@ def __init__( # token validation, see '_get_validation_session' self._validation_session: requests.Session | None = None - self._base_functions_mapping = { - RequestTypes.get: requests.get, - RequestTypes.post: requests.post, - RequestTypes.put: requests.put, - RequestTypes.patch: requests.patch, - RequestTypes.delete: requests.delete - } - self._session_functions_mapping = {} - # Attributes cache self._attributes_schema = None self._entity_type_attributes_cache = {} @@ -1002,18 +1001,8 @@ def create_session( session = self._validation_session self._validation_session = None if session is None: - session = requests.Session() - session.cert = self._cert - session.verify = self._ssl_verify + session = self._create_requests_session() session.headers.update(self.get_headers()) - - self._session_functions_mapping = { - RequestTypes.get: session.get, - RequestTypes.post: session.post, - RequestTypes.put: session.put, - RequestTypes.patch: session.patch, - RequestTypes.delete: session.delete - } self._session = session def close_session(self) -> None: @@ -1026,9 +1015,44 @@ def close_session(self) -> None: session = self._session self._session = None - self._session_functions_mapping = {} session.close() + def _create_requests_session(self) -> requests.Session: + """Create requests session with connection ssl settings. + + Returns: + requests.Session: Session without authorization headers. + + """ + session = requests.Session() + session.cert = self._cert + session.verify = self._ssl_verify + return session + + @contextmanager + def _request_session(self) -> Generator[requests.Session, None, None]: + """Session to use for a request. + + Yields the connection session if it was created. Otherwise yields + a temporary session, which is closed on exit. The temporary + session has the ssl settings of the connection but does not + have authorization headers. + + Yields: + requests.Session: Session to use for request. + + """ + session = self._session + if session is not None: + yield session + return + + session = self._create_requests_session() + try: + yield session + finally: + session.close() + def _get_validation_session(self) -> requests.Session: """Session for requests made before authorized session is created. @@ -1042,10 +1066,7 @@ def _get_validation_session(self) -> requests.Session: """ if self._validation_session is None: - session = requests.Session() - session.cert = self._cert - session.verify = self._ssl_verify - self._validation_session = session + self._validation_session = self._create_requests_session() return self._validation_session def create_websocket( @@ -1836,12 +1857,25 @@ def _do_rest_request( if "headers" not in kwargs: kwargs["headers"] = self.get_headers() + with self._request_session() as session: if isinstance(function, RequestType): - function = self._base_functions_mapping[function] - - elif isinstance(function, RequestType): - function = self._session_functions_mapping[function] + function = functools.partial(session.request, function.name) + return self._send_rest_request( + function, + url, + max_retries, + handle_invalid_token, + kwargs, + ) + def _send_rest_request( + self, + function: Any, + url: str, + max_retries: int, + handle_invalid_token: bool, + kwargs: dict[str, Any], + ) -> RestApiResponse: response = None new_response = None for retry_idx in reversed(range(max_retries)): @@ -1864,6 +1898,23 @@ def _do_rest_request( continue break + except _NOT_RETRIABLE_CONNECTION_ERRORS as exc: + # Sending the request again won't help + self.log.warning( + "AYON api error: Connection error happened.", + exc_info=True, + ) + new_response = RestApiResponse( + None, + { + "detail": ( + "AYON api error: Unable to connect the server." + f" {exc.__class__.__name__}: {exc}" + ) + } + ) + break + except ConnectionRefusedError: if retry_idx == 0: self.log.warning( @@ -1938,75 +1989,81 @@ def _download_file_to_stream( "stream": True, "headers": headers, } - if self._session is None: - get_func = self._base_functions_mapping[RequestTypes.get] - else: - get_func = self._session_functions_mapping[RequestTypes.get] url = self._endpoint_to_url(endpoint, use_rest=False) progress.set_source_url(url) retries = max(self.max_retries, 1) api_prepended = False - for attempt in range(retries): - # Continue in download - offset = progress.get_transferred_size() - if offset > 0: - headers["Range"] = f"bytes={offset}-" - - try: - with get_func(url, **kwargs) as response: - # Auto-fix missing 'api/' - if ( - response.status_code in (404, 405) - and not api_prepended - ): - api_prepended = True + with self._request_session() as session: + for attempt in range(retries): + # Continue in download + offset = progress.get_transferred_size() + if offset > 0: + headers["Range"] = f"bytes={offset}-" + + try: + with session.get(url, **kwargs) as response: + # Auto-fix missing 'api/' if ( - not endpoint.startswith(self._base_url) - and not endpoint.startswith("api/") + response.status_code in (404, 405) + and not api_prepended ): - url = self._endpoint_to_url( - endpoint, use_rest=True + api_prepended = True + if ( + not endpoint.startswith(self._base_url) + and not endpoint.startswith("api/") + ): + url = self._endpoint_to_url( + endpoint, use_rest=True + ) + progress.set_destination_url(url) + continue + response.raise_for_status() + if offset > 0 and response.status_code != 206: + # Server ignored 'Range' and sends whole file + # again, already downloaded content must be + # discarded + stream.seek(0) + stream.truncate() + progress.reset_transferred() + headers.pop("Range", None) + + if progress.get_content_size() is None: + progress.set_content_size( + response.headers["Content-length"] ) - progress.set_destination_url(url) - continue - response.raise_for_status() - if offset > 0 and response.status_code != 206: - # Server ignored 'Range' and sends whole file again, - # already downloaded content must be discarded - stream.seek(0) - stream.truncate() - progress.reset_transferred() - headers.pop("Range", None) - - if progress.get_content_size() is None: - progress.set_content_size( - response.headers["Content-length"] - ) - for chunk in response.iter_content(chunk_size=chunk_size): - stream.write(chunk) - progress.add_transferred_chunk(len(chunk)) - - content_size = progress.get_content_size() - transferred = progress.get_transferred_size() - if content_size is not None and transferred != content_size: - # Connection was closed before all content was received - raise requests.exceptions.ConnectionError( - f"Downloaded {transferred} out of {content_size}" - f" bytes from '{url}'." - ) - break + for chunk in response.iter_content( + chunk_size=chunk_size + ): + stream.write(chunk) + progress.add_transferred_chunk(len(chunk)) - except ( - requests.exceptions.Timeout, - requests.exceptions.ConnectionError, - requests.exceptions.ChunkedEncodingError, - ): - if attempt == retries - 1: + content_size = progress.get_content_size() + transferred = progress.get_transferred_size() + if ( + content_size is not None + and transferred != content_size + ): + # Connection was closed before all content was received + raise requests.exceptions.ConnectionError( + f"Downloaded {transferred} out of {content_size}" + f" bytes from '{url}'." + ) + break + + except _NOT_RETRIABLE_CONNECTION_ERRORS: raise - progress.next_attempt() + + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + requests.exceptions.ChunkedEncodingError, + ): + if attempt == retries - 1: + raise + progress.next_attempt() if api_prepended: self.log.warning( @@ -2377,14 +2434,10 @@ def _upload_file( headers_keys_by_low_key = {key.lower(): key for key in headers} if self._session is None: for key, value in self.get_headers().items(): - orig_key = headers_keys_by_low_key.get(key) + orig_key = headers_keys_by_low_key.get(key.lower()) if not orig_key: headers[key] = value - post_func = self._base_functions_mapping[request_type] - else: - post_func = self._session_functions_mapping[request_type] - if not chunk_size: chunk_size = self.default_upload_chunk_size @@ -2411,13 +2464,15 @@ def _upload_file( api_prepended = False for attempt in range(retries): try: - response = post_func( - url, - data=self._upload_chunks_iter( - stream, progress, chunk_size - ), - **kwargs - ) + with self._request_session() as session: + response = session.request( + request_type.name, + url, + data=self._upload_chunks_iter( + stream, progress, chunk_size + ), + **kwargs + ) # Auto-fix missing 'api/' if response.status_code in (404, 405) and not api_prepended: api_prepended = True @@ -2430,6 +2485,9 @@ def _upload_file( continue break + except _NOT_RETRIABLE_CONNECTION_ERRORS: + raise + except ( requests.exceptions.Timeout, requests.exceptions.ConnectionError, diff --git a/tests/test_download_resume.py b/tests/test_download_resume.py index 6c4bc2e15..34db8f20d 100644 --- a/tests/test_download_resume.py +++ b/tests/test_download_resume.py @@ -2,9 +2,9 @@ import io import pytest +import requests from ayon_api.server_api import ServerAPI -from ayon_api.utils import RequestTypes from .fake_transfer import FakeResponse @@ -18,14 +18,18 @@ def con(monkeypatch): return ServerAPI("http://localhost:0", create_session=False, max_retries=3) -def _download(con, get_func): - con._base_functions_mapping[RequestTypes.get] = get_func +def _download(con, monkeypatch, get_func): + monkeypatch.setattr( + requests.Session, + "get", + lambda _session, url, **kwargs: get_func(url, **kwargs), + ) stream = io.BytesIO() progress = con.download_file_to_stream("api/file", stream) return stream.getvalue(), progress -def test_incomplete_download_is_continued(con): +def test_incomplete_download_is_continued(con, monkeypatch): def get_func(url, **kwargs): if "Range" not in kwargs["headers"]: # Connection closed after 4 bytes without an exception @@ -33,12 +37,14 @@ def get_func(url, **kwargs): assert kwargs["headers"]["Range"] == "bytes=4-" return FakeResponse(206, CONTENT[4:], {"Content-Length": "6"}) - content, progress = _download(con, get_func) + content, progress = _download(con, monkeypatch, get_func) assert content == CONTENT assert progress.transferred_size == len(CONTENT) -def test_resume_without_range_support_does_not_duplicate(con): +def test_resume_without_range_support_does_not_duplicate( + con, monkeypatch +): calls = [] def get_func(url, **kwargs): @@ -48,6 +54,6 @@ def get_func(url, **kwargs): # Server ignores 'Range' header and sends whole file again return FakeResponse(200, CONTENT, SIZE_HEADER) - content, progress = _download(con, get_func) + content, progress = _download(con, monkeypatch, get_func) assert content == CONTENT assert progress.transferred_size == len(CONTENT) diff --git a/tests/test_request_retries.py b/tests/test_request_retries.py index 22708c78c..19a5eaa75 100644 --- a/tests/test_request_retries.py +++ b/tests/test_request_retries.py @@ -3,6 +3,7 @@ import requests from ayon_api.server_api import ServerAPI +from ayon_api.utils import RequestTypes from .fake_transfer import FakeResponse @@ -33,3 +34,45 @@ def request_func(url, **kwargs): ) assert response.status_code == 200 assert response.data == {"ok": True} + + +@pytest.mark.parametrize( + "error", + [ + requests.exceptions.SSLError("certificate verify failed"), + requests.exceptions.ProxyError("proxy refused"), + ], +) +def test_ssl_and_proxy_errors_are_not_retried(con, error): + calls = [] + + def request_func(url, **kwargs): + calls.append(url) + raise error + + response = con._do_rest_request( + request_func, "http://localhost:0/api/x", handle_invalid_token=False + ) + assert len(calls) == 1 + assert response.status_code == 500 + assert str(error) in response.data["detail"] + + +def test_request_without_session_uses_ssl_settings(con, monkeypatch): + con.set_ssl_verify("/path/to/ca.pem") + con.set_cert("/path/to/cert.pem") + used_sessions = [] + + def fake_request(session, method, url, **kwargs): + used_sessions.append((session.verify, session.cert, method)) + return FakeResponse(200, json_data={"ok": True}) + + monkeypatch.setattr(requests.Session, "request", fake_request) + + response = con._do_rest_request( + RequestTypes.get, + "http://localhost:0/api/x", + handle_invalid_token=False, + ) + assert response.status_code == 200 + assert used_sessions == [("/path/to/ca.pem", "/path/to/cert.pem", "GET")]