From 77ec8629337a913253fafb1365273063bfb5e919 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:25:32 +0200 Subject: [PATCH 01/11] added helper method to create new session --- ayon_api/server_api.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 87f939e45..d0ac365f3 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -1434,6 +1434,28 @@ def _do_rest_request(self, function, url, **kwargs): self.log.debug(f"Response {str(new_response)}") return new_response + def _create_new_session(self, max_retries=None, headers=None): + if max_retries is None: + max_retries = self.max_retries + if max_retries < 0: + max_retries = 0 + + if headers is None: + headers = self.get_headers() + + session = requests.Session() + session.cert = self._cert + session.verify = self._ssl_verify + session.headers.update(headers) + handlers = { + "http://": HTTPAdapter(max_retries=max_retries), + "https://": HTTPAdapter(max_retries=max_retries), + } + for prefix, adapter in handlers.items(): + session.mount(prefix, adapter) + + return session, handlers + def raw_post(self, entrypoint: str, **kwargs): url = self._endpoint_to_url(entrypoint) self.log.debug(f"Executing [POST] {url}") From 9a4634e8a70ace1d0ea5ff8b25a2b639e29b816a Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:56:17 +0200 Subject: [PATCH 02/11] don't handle max retries in do request --- ayon_api/server_api.py | 98 +++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index d0ac365f3..430ddeb95 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -1340,12 +1340,12 @@ def logout(self, soft: bool = False): def _logout(self): logout_from_server(self._base_url, self._access_token) - def _do_rest_request(self, function, url, **kwargs): + def _do_rest_request(self, request_type, url, **kwargs): kwargs.setdefault("timeout", self.timeout) - max_retries = kwargs.get("max_retries", self.max_retries) - if max_retries < 1: - max_retries = 1 - if self._session is None: + + close_session = False + session = self._session + if session is None: # Validate token if was not yet validated # - ignore validation if we're in middle of # validation @@ -1355,62 +1355,60 @@ def _do_rest_request(self, function, url, **kwargs): ): self.validate_token() - if "headers" not in kwargs: - kwargs["headers"] = self.get_headers() - - if isinstance(function, RequestType): - function = self._base_functions_mapping[function] - - elif isinstance(function, RequestType): - function = self._session_functions_mapping[function] + max_retries = kwargs.get("max_retries") + headers = kwargs.get("headers") + close_session = True + session, _ = self._create_new_session( + max_retries=max_retries, headers=headers + ) response = None new_response = None - for retry_idx in reversed(range(max_retries)): - try: - response = function(url, **kwargs) - break + try: + response = session.request(request_type.name, url, **kwargs) - except ConnectionRefusedError: - if retry_idx == 0: - self.log.warning( - "Connection error happened.", exc_info=True + except ConnectionRefusedError: + if retry_idx == 0: + self.log.warning( + "Connection error happened.", exc_info=True + ) + + # Server may be restarting + new_response = RestApiResponse( + None, + { + "detail": ( + "Unable to connect the server. Connection refused" ) + } + ) - # Server may be restarting - new_response = RestApiResponse( - None, - { - "detail": ( - "Unable to connect the server. Connection refused" - ) - } - ) + except requests.exceptions.Timeout: + # Connection timed out + new_response = RestApiResponse( + None, + {"detail": "Connection timed out."} + ) - except requests.exceptions.Timeout: - # Connection timed out - new_response = RestApiResponse( - None, - {"detail": "Connection timed out."} + except requests.exceptions.ConnectionError: + # Log warning only on last attempt + if retry_idx == 0: + self.log.warning( + "Connection error happened.", exc_info=True ) - except requests.exceptions.ConnectionError: - # Log warning only on last attempt - if retry_idx == 0: - self.log.warning( - "Connection error happened.", exc_info=True + new_response = RestApiResponse( + None, + { + "detail": ( + "Unable to connect the server. Connection error" ) + } + ) - new_response = RestApiResponse( - None, - { - "detail": ( - "Unable to connect the server. Connection error" - ) - } - ) - - time.sleep(0.1) + finally: + if close_session: + session.close() if new_response is not None: return new_response From fac4b0ec2ac748dff50d53f5c6fd1b841d8be087 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:57:22 +0200 Subject: [PATCH 03/11] use create new session method --- ayon_api/server_api.py | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 430ddeb95..5e5e403fb 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -476,6 +476,9 @@ def __init__( if not base_url: raise ValueError(f"Invalid server URL {str(base_url)}") + self._session = None + self._session_handlers = {} + base_url = base_url.rstrip("/") self._base_url: str = base_url self._rest_url: str = f"{base_url}/api" @@ -522,17 +525,6 @@ def __init__( self._graphql_allows_data_in_query = None - self._session = None - - self._base_functions_mapping = { - RequestTypes.get: requests.get, - RequestTypes.post: requests.post, - RequestTypes.put: requests.put, - RequestTypes.patch: requests.patch, - RequestTypes.delete: requests.delete - } - self._session_functions_mapping = {} - # Attributes cache self._attributes_schema = None self._entity_type_attributes_cache = {} @@ -996,19 +988,10 @@ def create_session( # Validate token before session creation self.validate_token() - session = requests.Session() - session.cert = self._cert - session.verify = self._ssl_verify - session.headers.update(self.get_headers()) - - self._session_functions_mapping = { - RequestTypes.get: session.get, - RequestTypes.post: session.post, - RequestTypes.put: session.put, - RequestTypes.patch: session.patch, - RequestTypes.delete: session.delete - } + session, handlers = self._create_new_session() + self._session = session + self._session_handlers = handlers def close_session(self): if self._session is None: @@ -1016,7 +999,7 @@ def close_session(self): session = self._session self._session = None - self._session_functions_mapping = {} + self._session_handlers = {} session.close() def _update_session_headers(self): From d6a062b0525ffdb4d9b105f86b8260bf5981d32f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:57:47 +0200 Subject: [PATCH 04/11] use create new session at download and upload methods --- ayon_api/server_api.py | 56 ++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 5e5e403fb..4b70468a6 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -2145,18 +2145,22 @@ def _download_file_to_stream( self, url: str, stream, chunk_size, progress ): kwargs = {"stream": True} - if self._session is None: - kwargs["headers"] = self.get_headers() - get_func = self._base_functions_mapping[RequestTypes.get] - else: - get_func = self._session_functions_mapping[RequestTypes.get] + session = self._session + close_session = False + if session is None: + close_session = True + session, _ = self._create_new_session() - with get_func(url, **kwargs) as response: - response.raise_for_status() - progress.set_content_size(response.headers["Content-length"]) - for chunk in response.iter_content(chunk_size=chunk_size): - stream.write(chunk) - progress.add_transferred_chunk(len(chunk)) + try: + with session.request("GET", url, **kwargs) as response: + response.raise_for_status() + progress.set_content_size(response.headers["Content-length"]) + for chunk in response.iter_content(chunk_size=chunk_size): + stream.write(chunk) + progress.add_transferred_chunk(len(chunk)) + finally: + if close_session: + session.close() def download_file_to_stream( self, @@ -2320,25 +2324,29 @@ def _upload_file( """ if request_type is None: - request_type = RequestTypes.put + request_type = "PUT" + elif isinstance(request_type, RequestType): + request_type = request_type.name + session = self._session + close_session = False if self._session is None: - headers = kwargs.setdefault("headers", {}) - for key, value in self.get_headers().items(): - if key not in headers: - headers[key] = value - post_func = self._base_functions_mapping[request_type] - else: - post_func = self._session_functions_mapping[request_type] + close_session = True + session, _ = self._create_new_session() if not chunk_size: chunk_size = self.default_upload_chunk_size - response = post_func( - url, - data=self._upload_chunks_iter(stream, progress, chunk_size), - **kwargs - ) + try: + response = session.request( + request_type, + url, + data=self._upload_chunks_iter(stream, progress, chunk_size), + **kwargs + ) + finally: + if close_session: + session.close() response.raise_for_status() return response From bfca7151eac2aa73e853fde61eb17f9ddb825e4b Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:57:56 +0200 Subject: [PATCH 05/11] added missing imports --- ayon_api/server_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 4b70468a6..008205d07 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -25,6 +25,7 @@ HTTPStatus = None import requests +from requests.adapters import HTTPAdapter, Retry try: # This should be used if 'requests' have it available from requests.exceptions import JSONDecodeError as RequestsJSONDecodeError From dc827c38d0095db98987a4e93e59081bea5329ce Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:58:06 +0200 Subject: [PATCH 06/11] change how max retries is set --- ayon_api/server_api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 008205d07..6f21bcf24 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -667,7 +667,14 @@ def set_max_retries(self, max_retries: Optional[int]): """ if max_retries is None: max_retries = self.get_default_max_retries() - self._max_retries = int(max_retries) + max_retries = int(max_retries) + if max_retries < 0: + max_retries = 0 + if max_retries == self._max_retries: + return + self._max_retries = max_retries + for handler in self._session_handlers.values(): + handler.max_retries = Retry.from_int(max_retries) timeout = property(get_timeout, set_timeout) max_retries = property(get_max_retries, set_max_retries) From 90474c838a6b05c405b728337e1520b63cb3a5f4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:02:30 +0200 Subject: [PATCH 07/11] fix automated api creation --- automated_api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/automated_api.py b/automated_api.py index 329c6c858..a1d0da9e9 100644 --- a/automated_api.py +++ b/automated_api.py @@ -22,11 +22,20 @@ # Fake modules to avoid import errors requests = type(sys)("requests") +requests_adapters = type(sys)("requests.adapters") requests.__dict__["Response"] = type( "Response", (), {"__module__": "requests"} ) +requests.__dict__["adapters"] = requests_adapters +requests_adapters.__dict__["HTTPAdapter"] = type( + "HTTPAdapter", (), {"__module__": "requests.adapters"} +) +requests_adapters.__dict__["Retry"] = type( + "Retry", (), {"__module__": "requests.adapters"} +) sys.modules["requests"] = requests +sys.modules["requests.adapters"] = requests_adapters sys.modules["unidecode"] = type(sys)("unidecode") import ayon_api # noqa: E402 From 5e177f661d7df7d545b3686499525f45df44d477 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:23:57 +0200 Subject: [PATCH 08/11] try again only on connection errors --- ayon_api/server_api.py | 71 ++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 6f21bcf24..71a12ac8d 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -1336,6 +1336,10 @@ def _do_rest_request(self, request_type, url, **kwargs): close_session = False session = self._session + max_retries = kwargs.get("max_retries") + if max_retries is None: + max_retries = self.max_retries + if session is None: # Validate token if was not yet validated # - ignore validation if we're in middle of @@ -1346,7 +1350,6 @@ def _do_rest_request(self, request_type, url, **kwargs): ): self.validate_token() - max_retries = kwargs.get("max_retries") headers = kwargs.get("headers") close_session = True session, _ = self._create_new_session( @@ -1355,55 +1358,41 @@ def _do_rest_request(self, request_type, url, **kwargs): response = None new_response = None - try: - response = session.request(request_type.name, url, **kwargs) - - except ConnectionRefusedError: - if retry_idx == 0: - self.log.warning( - "Connection error happened.", exc_info=True - ) + if max_retries < 1: + max_retries = 1 - # Server may be restarting - new_response = RestApiResponse( - None, - { - "detail": ( - "Unable to connect the server. Connection refused" + try: + for retry_idx in reversed(range(max_retries)): + try: + response = session.request( + request_type.name, url, **kwargs ) - } - ) - except requests.exceptions.Timeout: - # Connection timed out - new_response = RestApiResponse( - None, - {"detail": "Connection timed out."} - ) + except ( + # These are 'ConnectionError' but it doesn't make sense + # to retry + requests.exceptions.ProxyError, + requests.exceptions.SSLError, + ): + raise + + except ( + ConnectionRefusedError, + requests.exceptions.ConnectionError + ): + # Log warning only on last attempt + if retry_idx == 0: + self.log.warning( + "Connection error happened.", exc_info=True + ) + raise - except requests.exceptions.ConnectionError: - # Log warning only on last attempt - if retry_idx == 0: - self.log.warning( - "Connection error happened.", exc_info=True - ) - - new_response = RestApiResponse( - None, - { - "detail": ( - "Unable to connect the server. Connection error" - ) - } - ) + time.sleep(0.1) finally: if close_session: session.close() - if new_response is not None: - return new_response - content_type = response.headers.get("Content-Type") if content_type == "application/json": try: From 7d13fab3c7bed56c92631a16ca232673d9505355 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:26:59 +0200 Subject: [PATCH 09/11] break loop on success --- ayon_api/server_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 71a12ac8d..00846cd3f 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -1367,6 +1367,7 @@ def _do_rest_request(self, request_type, url, **kwargs): response = session.request( request_type.name, url, **kwargs ) + break except ( # These are 'ConnectionError' but it doesn't make sense From 2382108fe7b0bb318ed0cdfe694b9ef3e7258353 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:28:30 +0200 Subject: [PATCH 10/11] added docstring --- ayon_api/server_api.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 00846cd3f..95c9e35b6 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -1332,6 +1332,25 @@ def _logout(self): logout_from_server(self._base_url, self._access_token) def _do_rest_request(self, request_type, url, **kwargs): + """ + + Args: + request_type (RequestType): Request type. + url (str): Request url. + max_retries (int): Does affect only connection issues or + when session is not created. + **kwargs: + + Returns: + RestApiResponse: Response. + + Raises: + ConnectionRefusedError: When connection is refused. + requests.exceptions.Timeout: When connection timed out. + requests.exceptions.ConnectionError: When connection error + happens. + + """ kwargs.setdefault("timeout", self.timeout) close_session = False From eb6ed39312f219a7ee36500d1b55e9ddab11d2c8 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:38:04 +0100 Subject: [PATCH 11/11] Use sessions with ssl settings for all requests Requests without a connection session used 'requests.get' etc., which ignored 'ssl_verify' (including 'AYON_CA_FILE') and 'cert'. All requests now go through 'requests.Session' created by '_create_requests_session', and a temporary one is used when the connection has no session. SSL and proxy errors are no longer retried, as sending the request again won't help. Their message is kept in the response detail. Co-Authored-By: Claude Opus 5.5 --- ayon_api/server_api.py | 254 +++++++++++++++++++++------------- tests/test_download_resume.py | 20 ++- tests/test_request_retries.py | 43 ++++++ 3 files changed, 212 insertions(+), 105 deletions(-) diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 5cc23cc3e..ff446a41a 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -6,6 +6,7 @@ from __future__ import annotations import copy +import functools from dataclasses import dataclass import os import re @@ -120,6 +121,13 @@ r"(?:\+(?P[a-zA-Z\d\-.]*))?" ) +# Subclasses of 'requests.exceptions.ConnectionError' that won't be fixed +# by sending the request again +_NOT_RETRIABLE_CONNECTION_ERRORS = ( + requests.exceptions.SSLError, + requests.exceptions.ProxyError, +) + def _get_requests_ca_bundle() -> str | None: """CA bundle used by 'requests' for ssl verification. @@ -427,15 +435,6 @@ def __init__( # token validation, see '_get_validation_session' self._validation_session: requests.Session | None = None - self._base_functions_mapping = { - RequestTypes.get: requests.get, - RequestTypes.post: requests.post, - RequestTypes.put: requests.put, - RequestTypes.patch: requests.patch, - RequestTypes.delete: requests.delete - } - self._session_functions_mapping = {} - # Attributes cache self._attributes_schema = None self._entity_type_attributes_cache = {} @@ -1002,18 +1001,8 @@ def create_session( session = self._validation_session self._validation_session = None if session is None: - session = requests.Session() - session.cert = self._cert - session.verify = self._ssl_verify + session = self._create_requests_session() session.headers.update(self.get_headers()) - - self._session_functions_mapping = { - RequestTypes.get: session.get, - RequestTypes.post: session.post, - RequestTypes.put: session.put, - RequestTypes.patch: session.patch, - RequestTypes.delete: session.delete - } self._session = session def close_session(self) -> None: @@ -1026,9 +1015,44 @@ def close_session(self) -> None: session = self._session self._session = None - self._session_functions_mapping = {} session.close() + def _create_requests_session(self) -> requests.Session: + """Create requests session with connection ssl settings. + + Returns: + requests.Session: Session without authorization headers. + + """ + session = requests.Session() + session.cert = self._cert + session.verify = self._ssl_verify + return session + + @contextmanager + def _request_session(self) -> Generator[requests.Session, None, None]: + """Session to use for a request. + + Yields the connection session if it was created. Otherwise yields + a temporary session, which is closed on exit. The temporary + session has the ssl settings of the connection but does not + have authorization headers. + + Yields: + requests.Session: Session to use for request. + + """ + session = self._session + if session is not None: + yield session + return + + session = self._create_requests_session() + try: + yield session + finally: + session.close() + def _get_validation_session(self) -> requests.Session: """Session for requests made before authorized session is created. @@ -1042,10 +1066,7 @@ def _get_validation_session(self) -> requests.Session: """ if self._validation_session is None: - session = requests.Session() - session.cert = self._cert - session.verify = self._ssl_verify - self._validation_session = session + self._validation_session = self._create_requests_session() return self._validation_session def create_websocket( @@ -1836,12 +1857,25 @@ def _do_rest_request( if "headers" not in kwargs: kwargs["headers"] = self.get_headers() + with self._request_session() as session: if isinstance(function, RequestType): - function = self._base_functions_mapping[function] - - elif isinstance(function, RequestType): - function = self._session_functions_mapping[function] + function = functools.partial(session.request, function.name) + return self._send_rest_request( + function, + url, + max_retries, + handle_invalid_token, + kwargs, + ) + def _send_rest_request( + self, + function: Any, + url: str, + max_retries: int, + handle_invalid_token: bool, + kwargs: dict[str, Any], + ) -> RestApiResponse: response = None new_response = None for retry_idx in reversed(range(max_retries)): @@ -1864,6 +1898,23 @@ def _do_rest_request( continue break + except _NOT_RETRIABLE_CONNECTION_ERRORS as exc: + # Sending the request again won't help + self.log.warning( + "AYON api error: Connection error happened.", + exc_info=True, + ) + new_response = RestApiResponse( + None, + { + "detail": ( + "AYON api error: Unable to connect the server." + f" {exc.__class__.__name__}: {exc}" + ) + } + ) + break + except ConnectionRefusedError: if retry_idx == 0: self.log.warning( @@ -1938,75 +1989,81 @@ def _download_file_to_stream( "stream": True, "headers": headers, } - if self._session is None: - get_func = self._base_functions_mapping[RequestTypes.get] - else: - get_func = self._session_functions_mapping[RequestTypes.get] url = self._endpoint_to_url(endpoint, use_rest=False) progress.set_source_url(url) retries = max(self.max_retries, 1) api_prepended = False - for attempt in range(retries): - # Continue in download - offset = progress.get_transferred_size() - if offset > 0: - headers["Range"] = f"bytes={offset}-" - - try: - with get_func(url, **kwargs) as response: - # Auto-fix missing 'api/' - if ( - response.status_code in (404, 405) - and not api_prepended - ): - api_prepended = True + with self._request_session() as session: + for attempt in range(retries): + # Continue in download + offset = progress.get_transferred_size() + if offset > 0: + headers["Range"] = f"bytes={offset}-" + + try: + with session.get(url, **kwargs) as response: + # Auto-fix missing 'api/' if ( - not endpoint.startswith(self._base_url) - and not endpoint.startswith("api/") + response.status_code in (404, 405) + and not api_prepended ): - url = self._endpoint_to_url( - endpoint, use_rest=True + api_prepended = True + if ( + not endpoint.startswith(self._base_url) + and not endpoint.startswith("api/") + ): + url = self._endpoint_to_url( + endpoint, use_rest=True + ) + progress.set_destination_url(url) + continue + response.raise_for_status() + if offset > 0 and response.status_code != 206: + # Server ignored 'Range' and sends whole file + # again, already downloaded content must be + # discarded + stream.seek(0) + stream.truncate() + progress.reset_transferred() + headers.pop("Range", None) + + if progress.get_content_size() is None: + progress.set_content_size( + response.headers["Content-length"] ) - progress.set_destination_url(url) - continue - response.raise_for_status() - if offset > 0 and response.status_code != 206: - # Server ignored 'Range' and sends whole file again, - # already downloaded content must be discarded - stream.seek(0) - stream.truncate() - progress.reset_transferred() - headers.pop("Range", None) - - if progress.get_content_size() is None: - progress.set_content_size( - response.headers["Content-length"] - ) - for chunk in response.iter_content(chunk_size=chunk_size): - stream.write(chunk) - progress.add_transferred_chunk(len(chunk)) - - content_size = progress.get_content_size() - transferred = progress.get_transferred_size() - if content_size is not None and transferred != content_size: - # Connection was closed before all content was received - raise requests.exceptions.ConnectionError( - f"Downloaded {transferred} out of {content_size}" - f" bytes from '{url}'." - ) - break + for chunk in response.iter_content( + chunk_size=chunk_size + ): + stream.write(chunk) + progress.add_transferred_chunk(len(chunk)) - except ( - requests.exceptions.Timeout, - requests.exceptions.ConnectionError, - requests.exceptions.ChunkedEncodingError, - ): - if attempt == retries - 1: + content_size = progress.get_content_size() + transferred = progress.get_transferred_size() + if ( + content_size is not None + and transferred != content_size + ): + # Connection was closed before all content was received + raise requests.exceptions.ConnectionError( + f"Downloaded {transferred} out of {content_size}" + f" bytes from '{url}'." + ) + break + + except _NOT_RETRIABLE_CONNECTION_ERRORS: raise - progress.next_attempt() + + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + requests.exceptions.ChunkedEncodingError, + ): + if attempt == retries - 1: + raise + progress.next_attempt() if api_prepended: self.log.warning( @@ -2377,14 +2434,10 @@ def _upload_file( headers_keys_by_low_key = {key.lower(): key for key in headers} if self._session is None: for key, value in self.get_headers().items(): - orig_key = headers_keys_by_low_key.get(key) + orig_key = headers_keys_by_low_key.get(key.lower()) if not orig_key: headers[key] = value - post_func = self._base_functions_mapping[request_type] - else: - post_func = self._session_functions_mapping[request_type] - if not chunk_size: chunk_size = self.default_upload_chunk_size @@ -2411,13 +2464,15 @@ def _upload_file( api_prepended = False for attempt in range(retries): try: - response = post_func( - url, - data=self._upload_chunks_iter( - stream, progress, chunk_size - ), - **kwargs - ) + with self._request_session() as session: + response = session.request( + request_type.name, + url, + data=self._upload_chunks_iter( + stream, progress, chunk_size + ), + **kwargs + ) # Auto-fix missing 'api/' if response.status_code in (404, 405) and not api_prepended: api_prepended = True @@ -2430,6 +2485,9 @@ def _upload_file( continue break + except _NOT_RETRIABLE_CONNECTION_ERRORS: + raise + except ( requests.exceptions.Timeout, requests.exceptions.ConnectionError, diff --git a/tests/test_download_resume.py b/tests/test_download_resume.py index 6c4bc2e15..34db8f20d 100644 --- a/tests/test_download_resume.py +++ b/tests/test_download_resume.py @@ -2,9 +2,9 @@ import io import pytest +import requests from ayon_api.server_api import ServerAPI -from ayon_api.utils import RequestTypes from .fake_transfer import FakeResponse @@ -18,14 +18,18 @@ def con(monkeypatch): return ServerAPI("http://localhost:0", create_session=False, max_retries=3) -def _download(con, get_func): - con._base_functions_mapping[RequestTypes.get] = get_func +def _download(con, monkeypatch, get_func): + monkeypatch.setattr( + requests.Session, + "get", + lambda _session, url, **kwargs: get_func(url, **kwargs), + ) stream = io.BytesIO() progress = con.download_file_to_stream("api/file", stream) return stream.getvalue(), progress -def test_incomplete_download_is_continued(con): +def test_incomplete_download_is_continued(con, monkeypatch): def get_func(url, **kwargs): if "Range" not in kwargs["headers"]: # Connection closed after 4 bytes without an exception @@ -33,12 +37,14 @@ def get_func(url, **kwargs): assert kwargs["headers"]["Range"] == "bytes=4-" return FakeResponse(206, CONTENT[4:], {"Content-Length": "6"}) - content, progress = _download(con, get_func) + content, progress = _download(con, monkeypatch, get_func) assert content == CONTENT assert progress.transferred_size == len(CONTENT) -def test_resume_without_range_support_does_not_duplicate(con): +def test_resume_without_range_support_does_not_duplicate( + con, monkeypatch +): calls = [] def get_func(url, **kwargs): @@ -48,6 +54,6 @@ def get_func(url, **kwargs): # Server ignores 'Range' header and sends whole file again return FakeResponse(200, CONTENT, SIZE_HEADER) - content, progress = _download(con, get_func) + content, progress = _download(con, monkeypatch, get_func) assert content == CONTENT assert progress.transferred_size == len(CONTENT) diff --git a/tests/test_request_retries.py b/tests/test_request_retries.py index 22708c78c..19a5eaa75 100644 --- a/tests/test_request_retries.py +++ b/tests/test_request_retries.py @@ -3,6 +3,7 @@ import requests from ayon_api.server_api import ServerAPI +from ayon_api.utils import RequestTypes from .fake_transfer import FakeResponse @@ -33,3 +34,45 @@ def request_func(url, **kwargs): ) assert response.status_code == 200 assert response.data == {"ok": True} + + +@pytest.mark.parametrize( + "error", + [ + requests.exceptions.SSLError("certificate verify failed"), + requests.exceptions.ProxyError("proxy refused"), + ], +) +def test_ssl_and_proxy_errors_are_not_retried(con, error): + calls = [] + + def request_func(url, **kwargs): + calls.append(url) + raise error + + response = con._do_rest_request( + request_func, "http://localhost:0/api/x", handle_invalid_token=False + ) + assert len(calls) == 1 + assert response.status_code == 500 + assert str(error) in response.data["detail"] + + +def test_request_without_session_uses_ssl_settings(con, monkeypatch): + con.set_ssl_verify("/path/to/ca.pem") + con.set_cert("/path/to/cert.pem") + used_sessions = [] + + def fake_request(session, method, url, **kwargs): + used_sessions.append((session.verify, session.cert, method)) + return FakeResponse(200, json_data={"ok": True}) + + monkeypatch.setattr(requests.Session, "request", fake_request) + + response = con._do_rest_request( + RequestTypes.get, + "http://localhost:0/api/x", + handle_invalid_token=False, + ) + assert response.status_code == 200 + assert used_sessions == [("/path/to/ca.pem", "/path/to/cert.pem", "GET")]