Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions b2sdk/_internal/transfer/inbound/downloaded_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ def __init__(
self.download_strategy = None
self.write_buffer_size = write_buffer_size
self.check_hash = check_hash
self._consumed = False

def _validate_download(self, bytes_read, actual_sha1):
if (
Expand Down Expand Up @@ -201,7 +202,10 @@ def save(self, file: BinaryIO, allow_seeking: bool | None = None) -> None:
:param file: a file-like object
:param allow_seeking: if False, download strategies that rely on seeking to write data
(parallel strategies) will be discarded.
:raises RuntimeError: if a download has already been attempted on this object
"""
self._check_not_consumed()

if allow_seeking is None:
allow_seeking = file.seekable()
elif allow_seeking and not file.seekable():
Expand Down Expand Up @@ -230,6 +234,7 @@ def save(self, file: BinaryIO, allow_seeking: bool | None = None) -> None:
else:
raise ValueError('no strategy suitable for download was found!')
self.download_strategy = strategy
self._consumed = True
bytes_read, actual_sha1 = strategy.download(
file,
response=self.response,
Expand All @@ -239,6 +244,12 @@ def save(self, file: BinaryIO, allow_seeking: bool | None = None) -> None:
)
self._validate_download(bytes_read, actual_sha1)

def _check_not_consumed(self) -> None:
if self._consumed:
raise RuntimeError(
'This download response has already been consumed. Request a new download to save again.'
)

def save_to(
self,
path_: str | pathlib.Path,
Expand All @@ -252,7 +263,9 @@ def save_to(
:param mode: mode in which the file should be opened
:param allow_seeking: if False, download strategies that rely on seeking to write data
(parallel strategies) will be discarded.
:raises RuntimeError: if a download has already been attempted on this object
"""
self._check_not_consumed()
path_ = pathlib.Path(path_)
is_stdout = points_to_stdout(path_)
if is_stdout or points_to_fifo(path_):
Expand Down
1 change: 1 addition & 0 deletions changelog.d/+downloaded-file-double-save.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Raise a clear error when saving a consumed download again, before modifying the destination.
31 changes: 31 additions & 0 deletions test/unit/bucket/test_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@
UnsatisfiableRange,
WrongEncryptionSettingForFileWrite,
)
from requests import Response
from urllib3.response import HTTPResponse

from test.helpers import NonSeekableIO, assert_dict_equal_ignore_extra

Expand Down Expand Up @@ -2956,6 +2958,35 @@ def test_download_by_name_unverified_checksum(self):

@pytest.mark.apiver(from_ver=2)
class TestChooseStrategy(TestCaseWithBucket):
def test_save_twice(self):
for data in (b'', b'hello world'):
for save_method in ('save', 'save_to'):
with self.subTest(data=data, save_method=save_method):
file_version = self.bucket.upload_bytes(data, 'file1')
downloaded_file = self.bucket.download_file_by_id(file_version.id_)
# Use a streaming response: the simulator response can be replayed.
response = Response()
response.request = downloaded_file.response.request
response.headers.update(downloaded_file.response.headers)
response.raw = HTTPResponse(body=io.BytesIO(data), preload_content=False)
downloaded_file.response = response
first_output = io.BytesIO()
downloaded_file.save(first_output)
assert first_output.getvalue() == data

second_output = io.BytesIO(b'existing data')
with tempfile.TemporaryDirectory() as directory:
destination = pathlib.Path(directory) / 'destination'
destination.write_bytes(b'existing data')
with pytest.raises(RuntimeError, match='already been consumed'):
if save_method == 'save':
downloaded_file.save(second_output)
else:
downloaded_file.save_to(destination)

assert second_output.getvalue() == b'existing data'
assert destination.read_bytes() == b'existing data'

def test_choose_strategy(self):
file_version = self.bucket.upload_bytes(b'hello world' * 8, 'file1')
download_manager = self.bucket.api.services.download_manager
Expand Down
15 changes: 4 additions & 11 deletions test/unit/bucket/test_regression_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -445,15 +445,8 @@ def recording_submit(*args, **kwargs):

class TestRegressionDownloadedFileContract:
@responses.activate
@known_defect(
'C56',
'DownloadedFile.save() called twice re-reads a used response',
raises=DefectStillPresent,
)
def test_c56_downloaded_file_can_be_saved_twice_from_a_real_response(self):
"""C56: save() called twice on a real streamed HTTP response must give the full data again;
the simulator's in-memory response hides this. Today the second call raises requests'
StreamConsumedError."""
def test_c56_downloaded_file_rejects_second_save_from_a_real_response(self):
"""A consumed HTTP response must be rejected with a clear error on re-save."""
import hashlib
import io
from unittest import mock
Expand All @@ -478,9 +471,9 @@ def test_c56_downloaded_file_can_be_saved_twice_from_a_real_response(self):
first, second = io.BytesIO(), io.BytesIO()
downloaded.save(first)
assert first.getvalue() == data, 'first save() returned wrong data'
with defect_raises(requests.exceptions.StreamConsumedError):
with pytest.raises(RuntimeError, match='already been consumed'):
downloaded.save(second)
assert second.getvalue() == data, 'second save() returned wrong data'
assert second.getvalue() == b'', 'rejected save() modified the destination'


class TestRegressionGuards(TestCaseWithBucket):
Expand Down
Loading