Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 45 additions & 5 deletions ayon_api/server_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -1934,6 +1934,9 @@ def _download_file_to_stream(
progress: TransferProgress,
) -> None:
headers = self.get_headers()
# 'Content-Length' and 'Range' of compressed response are related
# to compressed content, but 'iter_content' returns decoded content
headers["Accept-Encoding"] = "identity"
kwargs = {
"stream": True,
"headers": headers,
Expand All @@ -1948,9 +1951,17 @@ def _download_file_to_stream(

retries = max(self.max_retries, 1)
api_prepended = False
can_resume = True
for attempt in range(retries):
# Continue in download
offset = progress.get_transferred_size()
if offset > 0 and not can_resume:
stream.seek(0)
stream.truncate()
progress.reset_transferred()
offset = 0

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in c4a8ddb. Range is removed at the start of every attempt and set again only when resuming. Covered by test_download_restart_does_not_keep_range (resumed response comes back compressed and is cut, the next request has no Range).


headers.pop("Range", None)
if offset > 0:
headers["Range"] = f"bytes={offset}-"

Expand Down Expand Up @@ -1978,17 +1989,46 @@ def _download_file_to_stream(
stream.seek(0)
stream.truncate()
progress.reset_transferred()
headers.pop("Range", None)

if progress.get_content_size() is None:
progress.set_content_size(
response.headers["Content-length"]
)
# Chunked response does not have the size
content_length = response.headers.get("Content-Length")
content_encoding = response.headers.get(
"Content-Encoding", ""
).strip().lower()
is_encoded = content_encoding not in ("", "identity")
if is_encoded:
# Server compressed content anyway. Size of decoded
# content is unknown and 'Range' can't be used.
can_resume = False

elif (
content_length
and progress.get_content_size() is None
and response.status_code != 206
):
progress.set_content_size(content_length)

for chunk in response.iter_content(chunk_size=chunk_size):
stream.write(chunk)
progress.add_transferred_chunk(len(chunk))

# 'urllib3' 1.x does not fail when response ends before
# 'Content-Length', compare received bytes. Size of
# whole content may be unknown (compressed, or
# resumed download of chunked response).
received = getattr(
getattr(response, "raw", None), "tell", None
)
if (
content_length
and received is not None
and received() < int(content_length)
):
raise requests.exceptions.ConnectionError(
f"Received {received()} out of"
f" {content_length} bytes from '{url}'."
)

content_size = progress.get_content_size()
transferred = progress.get_transferred_size()
if content_size is not None and transferred != content_size:
Expand Down
256 changes: 256 additions & 0 deletions tests/test_download_content_encoding.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
"""Download of compressed responses and responses without size."""
import gzip
import http.server
import json
import threading

import pytest

from ayon_api import ServerAPI

CONTENT = bytes(range(256)) * 64
API_KEY = "s" * 32


def _range_offset(headers):
value = headers.get("Range")
if not value:
return 0
return int(value[len("bytes="):].split("-")[0])


class _Handler(http.server.BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"

def _send(self, code, body=b"", headers=None):
self.send_response(code)
for key, value in (headers or {}).items():
self.send_header(key, value)
if "Transfer-Encoding" not in (headers or {}):
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)

def do_GET(self):
path = self.path.split("?")[0]
self.server.requests.append((path, dict(self.headers)))
accepts_gzip = "gzip" in self.headers.get("Accept-Encoding", "")
if path == "/gzip-if-accepted":
# Compresses content when client accepts it
if accepts_gzip:
self._send(200, gzip.compress(CONTENT), {
"Content-Encoding": "gzip",
})
else:
self._send(200, CONTENT)
return

if path == "/gzip-always":
# Compresses content even if client asks for 'identity'
self._send(200, gzip.compress(CONTENT), {
"Content-Encoding": "gzip",
})
return

if path == "/gzip-drop-once":
# Compressed content, first response is cut in half
body = gzip.compress(CONTENT)
self.server.drop_count += 1
if self.server.drop_count == 1:
self.send_response(200)
self.send_header("Content-Encoding", "gzip")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body[:len(body) // 2])
self.close_connection = True
return
self._send(200, body, {"Content-Encoding": "gzip"})
return

if path == "/chunked-resume-short":
# 1. chunked response without size is cut
# 2. resumed response ends before its 'Content-Length'
# 3. resumed response is complete
self.server.drop_count += 1
offset = _range_offset(self.headers)
if self.server.drop_count == 1:
self.send_response(200)
self.send_header("Transfer-Encoding", "chunked")
self.end_headers()
chunk = CONTENT[:4000]
self.wfile.write(f"{len(chunk):X}\r\n".encode())
self.wfile.write(chunk + b"\r\n")
self.close_connection = True
return
body = CONTENT[offset:]
headers = {
"Content-Range": (
f"bytes {offset}-{len(CONTENT) - 1}/{len(CONTENT)}"
),
}
if self.server.drop_count == 2:
self.send_response(206)
for key, value in headers.items():
self.send_header(key, value)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body[:len(body) // 2])
self.close_connection = True
return
self._send(206, body, headers)
return

if path == "/resume-then-gzip":
# 1. uncompressed response is cut
# 2. resumed response is compressed and cut
# 3. must download whole content again without 'Range'
self.server.drop_count += 1
offset = _range_offset(self.headers)
if self.server.drop_count == 1:
self.send_response(200)
self.send_header("Content-Length", str(len(CONTENT)))
self.end_headers()
self.wfile.write(CONTENT[:4000])
self.close_connection = True
return
if self.server.drop_count == 2:
body = gzip.compress(CONTENT[offset:])
self.send_response(206)
self.send_header("Content-Encoding", "gzip")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body[:len(body) // 2])
self.close_connection = True
return
if offset:
self._send(416)
return
self._send(200, CONTENT)
return

if path == "/chunked":
# No 'Content-Length' header
self.send_response(200)
self.send_header("Transfer-Encoding", "chunked")
self.end_headers()
for idx in range(0, len(CONTENT), 1000):
chunk = CONTENT[idx:idx + 1000]
self.wfile.write(f"{len(chunk):X}\r\n".encode())
self.wfile.write(chunk + b"\r\n")
self.wfile.write(b"0\r\n\r\n")
return

if path == "/api/users/me":
self._send(
200,
json.dumps({"name": "service"}).encode(),
{"Content-Type": "application/json"},
)
return

self._send(
200,
json.dumps({"version": "1.17.3", "uptime": 1}).encode(),
{"Content-Type": "application/json"},
)

def log_message(self, *args):
pass


@pytest.fixture
def server():
httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
httpd.requests = []
httpd.drop_count = 0
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
thread.start()
try:
yield httpd
finally:
httpd.shutdown()
httpd.server_close()


@pytest.fixture
def api(server, monkeypatch):
for key in ("AYON_CA_FILE", "AYON_CERT_FILE"):
monkeypatch.delenv(key, raising=False)
monkeypatch.setattr("time.sleep", lambda *args, **kwargs: None)
port = server.server_address[1]
con = ServerAPI(f"http://127.0.0.1:{port}", token=API_KEY)
yield con
con.close_session()


def _download(server, api, tmp_path, endpoint):
server.requests.clear()
filepath = tmp_path / "file.bin"
# Small chunks, so content received before connection drop is written
progress = api.download_file(endpoint, str(filepath), chunk_size=1000)
downloads = [
headers for path, headers in server.requests if path == endpoint
]
return filepath.read_bytes(), progress, downloads


def test_download_asks_for_uncompressed_content(server, api, tmp_path):
content, progress, downloads = _download(
server, api, tmp_path, "/gzip-if-accepted"
)
assert content == CONTENT
assert progress.get_content_size() == len(CONTENT)
assert progress.get_transferred_size() == len(CONTENT)
assert len(downloads) == 1
assert "Range" not in downloads[0]


def test_download_compressed_content(server, api, tmp_path):
content, progress, downloads = _download(
server, api, tmp_path, "/gzip-always"
)
assert content == CONTENT
assert progress.get_transferred_size() == len(CONTENT)
assert len(downloads) == 1
assert "Range" not in downloads[0]


def test_download_compressed_content_restarts(server, api, tmp_path):
content, progress, downloads = _download(
server, api, tmp_path, "/gzip-drop-once"
)
assert content == CONTENT
assert progress.get_transferred_size() == len(CONTENT)
# 'Range' of compressed content can't be used to continue
assert len(downloads) == 2
assert all("Range" not in headers for headers in downloads)


def test_download_resumed_response_is_complete(server, api, tmp_path):
content, progress, downloads = _download(
server, api, tmp_path, "/chunked-resume-short"
)
assert content == CONTENT
assert len(downloads) == 3
assert "Range" not in downloads[0]
assert downloads[1]["Range"] == "bytes=4000-"


def test_download_restart_does_not_keep_range(server, api, tmp_path):
content, progress, downloads = _download(
server, api, tmp_path, "/resume-then-gzip"
)
assert content == CONTENT
assert len(downloads) == 3
assert downloads[1]["Range"] == "bytes=4000-"
assert "Range" not in downloads[2]


def test_download_without_content_length(server, api, tmp_path):
content, progress, downloads = _download(
server, api, tmp_path, "/chunked"
)
assert content == CONTENT
assert progress.get_content_size() is None
assert progress.get_transferred_size() == len(CONTENT)
assert len(downloads) == 1
Loading