Skip to content
Draft
254 changes: 156 additions & 98 deletions ayon_api/server_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from __future__ import annotations

import copy
import functools
from dataclasses import dataclass
import os
import re
Expand Down Expand Up @@ -120,6 +121,13 @@
r"(?:\+(?P<buildmetadata>[a-zA-Z\d\-.]*))?"
)

# Subclasses of 'requests.exceptions.ConnectionError' that won't be fixed
# by sending the request again
_NOT_RETRIABLE_CONNECTION_ERRORS = (
requests.exceptions.SSLError,
requests.exceptions.ProxyError,
)


def _get_requests_ca_bundle() -> str | None:
"""CA bundle used by 'requests' for ssl verification.
Expand Down Expand Up @@ -427,15 +435,6 @@ def __init__(
# token validation, see '_get_validation_session'
self._validation_session: requests.Session | None = None

self._base_functions_mapping = {
RequestTypes.get: requests.get,
RequestTypes.post: requests.post,
RequestTypes.put: requests.put,
RequestTypes.patch: requests.patch,
RequestTypes.delete: requests.delete
}
self._session_functions_mapping = {}

# Attributes cache
self._attributes_schema = None
self._entity_type_attributes_cache = {}
Expand Down Expand Up @@ -1002,18 +1001,8 @@ def create_session(
session = self._validation_session
self._validation_session = None
if session is None:
session = requests.Session()
session.cert = self._cert
session.verify = self._ssl_verify
session = self._create_requests_session()
session.headers.update(self.get_headers())

self._session_functions_mapping = {
RequestTypes.get: session.get,
RequestTypes.post: session.post,
RequestTypes.put: session.put,
RequestTypes.patch: session.patch,
RequestTypes.delete: session.delete
}
self._session = session

def close_session(self) -> None:
Expand All @@ -1026,9 +1015,44 @@ def close_session(self) -> None:

session = self._session
self._session = None
self._session_functions_mapping = {}
session.close()

def _create_requests_session(self) -> requests.Session:
"""Create requests session with connection ssl settings.

Returns:
requests.Session: Session without authorization headers.

"""
session = requests.Session()
session.cert = self._cert
session.verify = self._ssl_verify
return session

@contextmanager
def _request_session(self) -> Generator[requests.Session, None, None]:
"""Session to use for a request.

Yields the connection session if it was created. Otherwise yields
a temporary session, which is closed on exit. The temporary
session has the ssl settings of the connection but does not
have authorization headers.

Yields:
requests.Session: Session to use for request.

"""
session = self._session
if session is not None:
yield session
return

session = self._create_requests_session()
try:
yield session
finally:
session.close()

def _get_validation_session(self) -> requests.Session:
"""Session for requests made before authorized session is created.

Expand All @@ -1042,10 +1066,7 @@ def _get_validation_session(self) -> requests.Session:

"""
if self._validation_session is None:
session = requests.Session()
session.cert = self._cert
session.verify = self._ssl_verify
self._validation_session = session
self._validation_session = self._create_requests_session()
return self._validation_session

def create_websocket(
Expand Down Expand Up @@ -1836,12 +1857,25 @@ def _do_rest_request(
if "headers" not in kwargs:
kwargs["headers"] = self.get_headers()

with self._request_session() as session:
if isinstance(function, RequestType):
function = self._base_functions_mapping[function]

elif isinstance(function, RequestType):
function = self._session_functions_mapping[function]
function = functools.partial(session.request, function.name)
return self._send_rest_request(
function,
url,
max_retries,
handle_invalid_token,
kwargs,
)

def _send_rest_request(
self,
function: Any,
url: str,
max_retries: int,
handle_invalid_token: bool,
kwargs: dict[str, Any],
) -> RestApiResponse:
response = None
new_response = None
for retry_idx in reversed(range(max_retries)):
Expand All @@ -1864,6 +1898,23 @@ def _do_rest_request(
continue
break

except _NOT_RETRIABLE_CONNECTION_ERRORS as exc:
# Sending the request again won't help
self.log.warning(
"AYON api error: Connection error happened.",
exc_info=True,
)
new_response = RestApiResponse(
None,
{
"detail": (
"AYON api error: Unable to connect the server."
f" {exc.__class__.__name__}: {exc}"
)
}
)
break

except ConnectionRefusedError:
if retry_idx == 0:
self.log.warning(
Expand Down Expand Up @@ -1938,75 +1989,81 @@ def _download_file_to_stream(
"stream": True,
"headers": headers,
}
if self._session is None:
get_func = self._base_functions_mapping[RequestTypes.get]
else:
get_func = self._session_functions_mapping[RequestTypes.get]

url = self._endpoint_to_url(endpoint, use_rest=False)
progress.set_source_url(url)

retries = max(self.max_retries, 1)
api_prepended = False
for attempt in range(retries):
# Continue in download
offset = progress.get_transferred_size()
if offset > 0:
headers["Range"] = f"bytes={offset}-"

try:
with get_func(url, **kwargs) as response:
# Auto-fix missing 'api/'
if (
response.status_code in (404, 405)
and not api_prepended
):
api_prepended = True
with self._request_session() as session:
for attempt in range(retries):
# Continue in download
offset = progress.get_transferred_size()
if offset > 0:
headers["Range"] = f"bytes={offset}-"

try:
with session.get(url, **kwargs) as response:
# Auto-fix missing 'api/'
if (
not endpoint.startswith(self._base_url)
and not endpoint.startswith("api/")
response.status_code in (404, 405)
and not api_prepended
):
url = self._endpoint_to_url(
endpoint, use_rest=True
api_prepended = True
if (
not endpoint.startswith(self._base_url)
and not endpoint.startswith("api/")
):
url = self._endpoint_to_url(
endpoint, use_rest=True
)
progress.set_destination_url(url)
continue
response.raise_for_status()
if offset > 0 and response.status_code != 206:
# Server ignored 'Range' and sends whole file
# again, already downloaded content must be
# discarded
stream.seek(0)
stream.truncate()
progress.reset_transferred()
headers.pop("Range", None)

if progress.get_content_size() is None:
progress.set_content_size(
response.headers["Content-length"]
)
progress.set_destination_url(url)
continue
response.raise_for_status()
if offset > 0 and response.status_code != 206:
# Server ignored 'Range' and sends whole file again,
# already downloaded content must be discarded
stream.seek(0)
stream.truncate()
progress.reset_transferred()
headers.pop("Range", None)

if progress.get_content_size() is None:
progress.set_content_size(
response.headers["Content-length"]
)

for chunk in response.iter_content(chunk_size=chunk_size):
stream.write(chunk)
progress.add_transferred_chunk(len(chunk))

content_size = progress.get_content_size()
transferred = progress.get_transferred_size()
if content_size is not None and transferred != content_size:
# Connection was closed before all content was received
raise requests.exceptions.ConnectionError(
f"Downloaded {transferred} out of {content_size}"
f" bytes from '{url}'."
)
break
for chunk in response.iter_content(
chunk_size=chunk_size
):
stream.write(chunk)
progress.add_transferred_chunk(len(chunk))

except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
requests.exceptions.ChunkedEncodingError,
):
if attempt == retries - 1:
content_size = progress.get_content_size()
transferred = progress.get_transferred_size()
if (
content_size is not None
and transferred != content_size
):
# Connection was closed before all content was received
raise requests.exceptions.ConnectionError(
f"Downloaded {transferred} out of {content_size}"
f" bytes from '{url}'."
)
break

except _NOT_RETRIABLE_CONNECTION_ERRORS:
raise
progress.next_attempt()

except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
requests.exceptions.ChunkedEncodingError,
):
if attempt == retries - 1:
raise
progress.next_attempt()

if api_prepended:
self.log.warning(
Expand Down Expand Up @@ -2377,14 +2434,10 @@ def _upload_file(
headers_keys_by_low_key = {key.lower(): key for key in headers}
if self._session is None:
for key, value in self.get_headers().items():
orig_key = headers_keys_by_low_key.get(key)
orig_key = headers_keys_by_low_key.get(key.lower())
if not orig_key:
headers[key] = value

post_func = self._base_functions_mapping[request_type]
else:
post_func = self._session_functions_mapping[request_type]

if not chunk_size:
chunk_size = self.default_upload_chunk_size

Expand All @@ -2411,13 +2464,15 @@ def _upload_file(
api_prepended = False
for attempt in range(retries):
try:
response = post_func(
url,
data=self._upload_chunks_iter(
stream, progress, chunk_size
),
**kwargs
)
with self._request_session() as session:
response = session.request(
request_type.name,
url,
data=self._upload_chunks_iter(
stream, progress, chunk_size
),
**kwargs
)
# Auto-fix missing 'api/'
if response.status_code in (404, 405) and not api_prepended:
api_prepended = True
Expand All @@ -2430,6 +2485,9 @@ def _upload_file(
continue
break

except _NOT_RETRIABLE_CONNECTION_ERRORS:
raise

except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
Expand Down
Loading
Loading