From 14489b8acf3747f0b269f3014c959267790232c5 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Thu, 28 Feb 2019 18:35:29 +0300 Subject: [PATCH 01/75] rst files renamed according to package name --- ...ccount_info.rst => b2sdk.account_info.rst} | 17 +++--- doc/source/{b2.rst => b2sdk.rst} | 57 ++++++++----------- doc/source/{b2.sync.rst => b2sdk.sync.rst} | 31 +++++----- 3 files changed, 48 insertions(+), 57 deletions(-) rename doc/source/{b2.account_info.rst => b2sdk.account_info.rst} (78%) rename doc/source/{b2.rst => b2sdk.rst} (88%) rename doc/source/{b2.sync.rst => b2sdk.sync.rst} (78%) diff --git a/doc/source/b2.account_info.rst b/doc/source/b2sdk.account_info.rst similarity index 78% rename from doc/source/b2.account_info.rst rename to doc/source/b2sdk.account_info.rst index b78795173..9a6137ca4 100644 --- a/doc/source/b2.account_info.rst +++ b/doc/source/b2sdk.account_info.rst @@ -1,11 +1,8 @@ b2sdk\.account\_info package -========================= - -Submodules ----------- +============================ b2sdk\.account\_info\.abstract module ----------------------------------- +------------------------------------- .. automodule:: b2sdk.account_info.abstract :members: @@ -13,7 +10,7 @@ b2sdk\.account\_info\.abstract module :show-inheritance: b2sdk\.account\_info\.exception module ------------------------------------ +-------------------------------------- .. automodule:: b2sdk.account_info.exception :members: @@ -21,7 +18,7 @@ b2sdk\.account\_info\.exception module :show-inheritance: b2sdk\.account\_info\.in\_memory module ------------------------------------- +--------------------------------------- .. automodule:: b2sdk.account_info.in_memory :members: @@ -29,7 +26,7 @@ b2sdk\.account\_info\.in\_memory module :show-inheritance: b2sdk\.account\_info\.sqlite\_account\_info module ------------------------------------------------ +-------------------------------------------------- .. automodule:: b2sdk.account_info.sqlite_account_info :members: @@ -37,7 +34,7 @@ b2sdk\.account\_info\.sqlite\_account\_info module :show-inheritance: b2sdk\.account\_info\.test\_upload\_url\_concurrency module --------------------------------------------------------- +----------------------------------------------------------- .. automodule:: b2sdk.account_info.test_upload_url_concurrency :members: @@ -45,7 +42,7 @@ b2sdk\.account\_info\.test\_upload\_url\_concurrency module :show-inheritance: b2sdk\.account\_info\.upload\_url\_pool module -------------------------------------------- +---------------------------------------------- .. automodule:: b2sdk.account_info.upload_url_pool :members: diff --git a/doc/source/b2.rst b/doc/source/b2sdk.rst similarity index 88% rename from doc/source/b2.rst rename to doc/source/b2sdk.rst index 15672ca7c..9e310beda 100644 --- a/doc/source/b2.rst +++ b/doc/source/b2sdk.rst @@ -1,19 +1,8 @@ b2sdk package ============== -Subpackages ------------ - -.. toctree:: - - b2sdk.account_info - b2sdk.sync - -Submodules ----------- - b2sdk\.api module ------------------- +----------------- .. automodule:: b2sdk.api :members: @@ -21,39 +10,39 @@ b2sdk\.api module :show-inheritance: b2sdk\.b2http module ---------------------- +-------------------- .. automodule:: b2sdk.b2http :members: :undoc-members: :show-inheritance: -b2sdk\.bucket module ---------------------- +b2sdk\.bounded\_queue\_executor module +-------------------------------------- -.. automodule:: b2sdk.bucket +.. automodule:: b2sdk.bounded_queue_executor :members: :undoc-members: :show-inheritance: -b2sdk\.cache module +b2sdk\.bucket module -------------------- -.. automodule:: b2sdk.cache +.. automodule:: b2sdk.bucket :members: :undoc-members: :show-inheritance: -b2sdk\.console\_tool module ----------------------------- +b2sdk\.cache module +------------------- -.. automodule:: b2sdk.console_tool +.. automodule:: b2sdk.cache :members: :undoc-members: :show-inheritance: b2sdk\.download\_dest module ------------------------------ +---------------------------- .. automodule:: b2sdk.download_dest :members: @@ -61,7 +50,7 @@ b2sdk\.download\_dest module :show-inheritance: b2sdk\.exception module ------------------------- +----------------------- .. automodule:: b2sdk.exception :members: @@ -69,7 +58,7 @@ b2sdk\.exception module :show-inheritance: b2sdk\.file\_version module ----------------------------- +--------------------------- .. automodule:: b2sdk.file_version :members: @@ -77,7 +66,7 @@ b2sdk\.file\_version module :show-inheritance: b2sdk\.parse\_args module --------------------------- +------------------------- .. automodule:: b2sdk.parse_args :members: @@ -85,7 +74,7 @@ b2sdk\.parse\_args module :show-inheritance: b2sdk\.part module -------------------- +------------------ .. automodule:: b2sdk.part :members: @@ -93,7 +82,7 @@ b2sdk\.part module :show-inheritance: b2sdk\.progress module ------------------------ +---------------------- .. automodule:: b2sdk.progress :members: @@ -101,7 +90,7 @@ b2sdk\.progress module :show-inheritance: b2sdk\.raw\_api module ------------------------ +---------------------- .. automodule:: b2sdk.raw_api :members: @@ -109,7 +98,7 @@ b2sdk\.raw\_api module :show-inheritance: b2sdk\.raw\_simulator module ------------------------------ +---------------------------- .. automodule:: b2sdk.raw_simulator :members: @@ -117,7 +106,7 @@ b2sdk\.raw\_simulator module :show-inheritance: b2sdk\.session module ----------------------- +--------------------- .. automodule:: b2sdk.session :members: @@ -125,7 +114,7 @@ b2sdk\.session module :show-inheritance: b2sdk\.unfinished\_large\_file module --------------------------------------- +------------------------------------- .. automodule:: b2sdk.unfinished_large_file :members: @@ -133,7 +122,7 @@ b2sdk\.unfinished\_large\_file module :show-inheritance: b2sdk\.upload\_source module ------------------------------ +---------------------------- .. automodule:: b2sdk.upload_source :members: @@ -141,7 +130,7 @@ b2sdk\.upload\_source module :show-inheritance: b2sdk\.utils module --------------------- +------------------- .. automodule:: b2sdk.utils :members: @@ -149,7 +138,7 @@ b2sdk\.utils module :show-inheritance: b2sdk\.version module ----------------------- +--------------------- .. automodule:: b2sdk.version :members: diff --git a/doc/source/b2.sync.rst b/doc/source/b2sdk.sync.rst similarity index 78% rename from doc/source/b2.sync.rst rename to doc/source/b2sdk.sync.rst index 566caf4ce..434fbc33c 100644 --- a/doc/source/b2.sync.rst +++ b/doc/source/b2sdk.sync.rst @@ -1,11 +1,8 @@ b2sdk\.sync package -==================== - -Submodules ----------- +=================== b2sdk\.sync\.action module ---------------------------- +-------------------------- .. automodule:: b2sdk.sync.action :members: @@ -13,7 +10,7 @@ b2sdk\.sync\.action module :show-inheritance: b2sdk\.sync\.exception module ------------------------------- +----------------------------- .. automodule:: b2sdk.sync.exception :members: @@ -21,7 +18,7 @@ b2sdk\.sync\.exception module :show-inheritance: b2sdk\.sync\.file module -------------------------- +------------------------ .. automodule:: b2sdk.sync.file :members: @@ -29,7 +26,7 @@ b2sdk\.sync\.file module :show-inheritance: b2sdk\.sync\.folder module ---------------------------- +-------------------------- .. automodule:: b2sdk.sync.folder :members: @@ -37,7 +34,7 @@ b2sdk\.sync\.folder module :show-inheritance: b2sdk\.sync\.folder\_parser module ------------------------------------ +---------------------------------- .. automodule:: b2sdk.sync.folder_parser :members: @@ -45,7 +42,7 @@ b2sdk\.sync\.folder\_parser module :show-inheritance: b2sdk\.sync\.policy module ---------------------------- +-------------------------- .. automodule:: b2sdk.sync.policy :members: @@ -53,7 +50,7 @@ b2sdk\.sync\.policy module :show-inheritance: b2sdk\.sync\.policy\_manager module ------------------------------------- +----------------------------------- .. automodule:: b2sdk.sync.policy_manager :members: @@ -61,15 +58,23 @@ b2sdk\.sync\.policy\_manager module :show-inheritance: b2sdk\.sync\.report module ---------------------------- +-------------------------- .. automodule:: b2sdk.sync.report :members: :undoc-members: :show-inheritance: +b2sdk\.sync\.scan\_policies +--------------------------- + +.. automodule:: b2sdk.sync.scan_policies + :members: + :undoc-members: + :show-inheritance: + b2sdk\.sync\.sync module -------------------------- +------------------------ .. automodule:: b2sdk.sync.sync :members: From 663760f10ec54c5d7410d32fd26a7b87d2d9efc1 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Thu, 28 Feb 2019 18:35:47 +0300 Subject: [PATCH 02/75] Added docs for b2sdk.transferer --- doc/source/b2sdk.transferer.rst | 59 +++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 doc/source/b2sdk.transferer.rst diff --git a/doc/source/b2sdk.transferer.rst b/doc/source/b2sdk.transferer.rst new file mode 100644 index 000000000..5175a5a62 --- /dev/null +++ b/doc/source/b2sdk.transferer.rst @@ -0,0 +1,59 @@ +b2sdk\.transferer package +========================= + +b2sdk\.transferer\.abstract module +---------------------------------- + +.. automodule:: b2sdk.transferer.abstract + :members: + :undoc-members: + :show-inheritance: + +b2sdk\.transferer\.file\_metadata module +---------------------------------------- + +.. automodule:: b2sdk.transferer.file_metadata + :members: + :undoc-members: + :show-inheritance: + +b2sdk\.transferer\.parallel module +---------------------------------- + +.. automodule:: b2sdk.transferer.parallel + :members: + :undoc-members: + :show-inheritance: + +b2sdk\.transferer\.range module +------------------------------- + +.. automodule:: b2sdk.transferer.range + :members: + :undoc-members: + :show-inheritance: + +b2sdk\.transferer\.simple module +-------------------------------- + +.. automodule:: b2sdk.transferer.simple + :members: + :undoc-members: + :show-inheritance: + +b2sdk\.transferer\.transferer module +------------------------------------ + +.. automodule:: b2sdk.transferer.transferer + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: b2sdk.transferer + :members: + :undoc-members: + :show-inheritance: From 2e370ead52392c4c34d332b5dc80d756a064a8e2 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Thu, 28 Feb 2019 18:36:23 +0300 Subject: [PATCH 03/75] Unneeded file removed --- doc/source/modules.rst | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 doc/source/modules.rst diff --git a/doc/source/modules.rst b/doc/source/modules.rst deleted file mode 100644 index 733e1746a..000000000 --- a/doc/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -b2sdk -====== - -.. toctree:: - :maxdepth: 4 - - b2sdk From d76520b2cd4690bae426979fa93a703676fc62fb Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Thu, 28 Feb 2019 18:36:45 +0300 Subject: [PATCH 04/75] Table of contents updated --- doc/source/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 284f35325..32d21e00b 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -11,6 +11,9 @@ Welcome to B2_Command_Line_Tool's documentation! :caption: Contents: b2sdk + b2sdk.account_info + b2sdk.sync + b2sdk.transferer Indices and tables ================== From 26ecabbfa5d4de933e447163eda0f8c55fa1f34e Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Thu, 28 Feb 2019 18:38:30 +0300 Subject: [PATCH 05/75] Docstrings fixed in order to eliminate warnings during docs build --- b2sdk/api.py | 8 +++--- b2sdk/bucket.py | 25 ++++++++++-------- b2sdk/file_version.py | 49 +++++++++++++++++++++--------------- b2sdk/parse_args.py | 9 ++++--- b2sdk/raw_api.py | 6 +++-- b2sdk/sync/file.py | 11 ++++---- b2sdk/transferer/abstract.py | 2 +- b2sdk/utils.py | 2 +- doc/source/conf.py | 12 +++++---- 9 files changed, 73 insertions(+), 51 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index 06e566b80..702ff5f7a 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -45,10 +45,10 @@ class B2Api(object): While B2RawApi provides direct access to the B2 web APIs, this class handles several things that simplify the task of uploading and downloading files: - - re-acquires authorization tokens when they expire - - retrying uploads when an upload URL is busy - - breaking large files into parts - - emulating a directory structure (B2 buckets are flat) + - re-acquires authorization tokens when they expire + - retrying uploads when an upload URL is busy + - breaking large files into parts + - emulating a directory structure (B2 buckets are flat) Adds an object-oriented layer on top of the raw API, so that buckets and files returned are Python objects with accessor diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 9bad4a954..0e74ae3e9 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -646,16 +646,21 @@ def from_api_response(cls, api, response): @classmethod def from_api_bucket_dict(cls, api, bucket_dict): """ - turns this: - { - "bucketType": "allPrivate", - "bucketId": "a4ba6a39d8b6b5fd561f0010", - "bucketName": "zsdfrtsazsdfafr", - "accountId": "4aa9865d6f00", - "bucketInfo": {}, - "revision": 1 - } - into a Bucket object + turns this: + + .. code-block:: python + + { + "bucketType": "allPrivate", + "bucketId": "a4ba6a39d8b6b5fd561f0010", + "bucketName": "zsdfrtsazsdfafr", + "accountId": "4aa9865d6f00", + "bucketInfo": {}, + "revision": 1 + } + + into a Bucket object + """ bucket_name = bucket_dict['bucketName'] bucket_id = bucket_dict['bucketId'] diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index e16302d22..fabfa2749 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -63,26 +63,35 @@ class FileVersionInfoFactory(object): @classmethod def from_api_response(cls, file_info_dict, force_action=None): """ - turns this: - { - "action": "hide", - "fileId": "4_zBucketName_f103b7ca31313c69c_d20151230_m030117_c001_v0001015_t0000", - "fileName": "randomdata", - "size": 0, - "uploadTimestamp": 1451444477000 - } - or this: - { - "accountId": "4aa9865d6f00", - "bucketId": "547a2a395826655d561f0010", - "contentLength": 1350, - "contentSha1": "753ca1c2d0f3e8748320b38f5da057767029a036", - "contentType": "application/octet-stream", - "fileId": "4_z547a2a395826655d561f0010_f106d4ca95f8b5b78_d20160104_m003906_c001_v0001013_t0005", - "fileInfo": {}, - "fileName": "randomdata" - } - into a FileVersionInfo object + turns this: + + .. code-block:: python + + { + "action": "hide", + "fileId": "4_zBucketName_f103b7ca31313c69c_d20151230_m030117_c001_v0001015_t0000", + "fileName": "randomdata", + "size": 0, + "uploadTimestamp": 1451444477000 + } + + or this: + + .. code-block:: python + + { + "accountId": "4aa9865d6f00", + "bucketId": "547a2a395826655d561f0010", + "contentLength": 1350, + "contentSha1": "753ca1c2d0f3e8748320b38f5da057767029a036", + "contentType": "application/octet-stream", + "fileId": "4_z547a2a395826655d561f0010_f106d4ca95f8b5b78_d20160104_m003906_c001_v0001013_t0005", + "fileInfo": {}, + "fileName": "randomdata" + } + + into a FileVersionInfo object + """ assert file_info_dict.get('action') is None or force_action is None, \ 'action was provided by both info_dict and function argument' diff --git a/b2sdk/parse_args.py b/b2sdk/parse_args.py index b50605f4d..66c583830 100644 --- a/b2sdk/parse_args.py +++ b/b2sdk/parse_args.py @@ -38,11 +38,14 @@ def check_for_duplicate_args(args_dict): This args_dict has a problem because 'required' and 'optional' both contain 'a': + .. code-block:: python + { - 'option_args': ['b', 'c'], - 'required': ['a', 'd'] - 'optional': ['a', 'e'] + 'option_args': ['b', 'c'], + 'required': ['a', 'd'], + 'optional': ['a', 'e'] } + """ categories = sorted(six.iterkeys(args_dict)) for index_a, category_a in enumerate(categories): diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 22aafccb1..821265543 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -424,10 +424,12 @@ def update_bucket( ) def unprintable_to_hex(self, string): - """Replace unprintable chars in string with a hex representation. + """ + Replace unprintable chars in string with a hex representation. :param string: An arbitrary string, possibly with unprintable characters. - :return The string, with unprintable characters changed to hex (e.g., "\x07") + :return: The string, with unprintable characters changed to hex (e.g., "\x07") + """ unprintables_pattern = re.compile(r'[\x00-\x1f]') diff --git a/b2sdk/sync/file.py b/b2sdk/sync/file.py index 7108379b0..a16c96b33 100644 --- a/b2sdk/sync/file.py +++ b/b2sdk/sync/file.py @@ -34,14 +34,15 @@ def __repr__(self): class FileVersion(object): """ Holds information about one version of a file: - - id - The B2 file id, or the local full path name - mod_time - modification time, in milliseconds, to avoid rounding issues - with millisecond times from B2 - action - "hide" or "upload" (never "start") """ def __init__(self, id_, file_name, mod_time, action, size): + """ + :param id_: the B2 file id, or the local full path name + :param mod_time: modification time, in milliseconds, to avoid rounding issues + with millisecond times from B2 + :param action: "hide" or "upload" (never "start") + """ self.id_ = id_ self.name = file_name self.mod_time = mod_time diff --git a/b2sdk/transferer/abstract.py b/b2sdk/transferer/abstract.py index 925c4ac5e..972cf7968 100644 --- a/b2sdk/transferer/abstract.py +++ b/b2sdk/transferer/abstract.py @@ -55,7 +55,7 @@ def _get_remote_range(cls, response, metadata): def is_suitable(self, metadata, progress_listener): """ analyzes metadata (possibly against options passed earlier to constructor - to find out whether the given download request should be handled by this downloader + to find out whether the given download request should be handled by this downloader) """ pass diff --git a/b2sdk/utils.py b/b2sdk/utils.py index ecc6b1445..da2c0ac73 100644 --- a/b2sdk/utils.py +++ b/b2sdk/utils.py @@ -9,7 +9,7 @@ ###################################################################### from __future__ import division, print_function - +import time import hashlib import os import platform diff --git a/doc/source/conf.py b/doc/source/conf.py index 9d0b55c7c..9eb43fe1f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -29,7 +29,9 @@ import os import sys -sys.path.insert(0, os.path.abspath('../b2sdk')) +sys.path.append(os.path.abspath('..')) + +from b2sdk.version import VERSION # -- General configuration ------------------------------------------------ @@ -58,7 +60,7 @@ # General information about the project. project = u'B2_Command_Line_Tool' -copyright = u'2017, Backblaze' +copyright = u'2019, Backblaze' author = u'Backblaze' # The version info for the project you're documenting, acts as replacement for @@ -66,9 +68,9 @@ # built documents. # # The short X.Y version. -version = u'0.7' +version = VERSION.rsplit('.', 1)[0] # The full version, including alpha/beta/rc tags. -release = u'0.7' +release = VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -104,7 +106,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. From bffcad1797c7b05a0b72a386afe4e5ec1ff99670 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Fri, 1 Mar 2019 11:22:39 +0300 Subject: [PATCH 06/75] Docstrings for b2sdk.api module updated --- b2sdk/api.py | 122 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 111 insertions(+), 11 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index 702ff5f7a..fc1302f4b 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -61,10 +61,16 @@ class handles several things that simplify the task of uploading def __init__(self, account_info=None, cache=None, raw_api=None, max_upload_workers=10): """ Initializes the API using the given account info. - :param account_info: - :param cache: - :param raw_api: - :return: + :param account_info: object, an instance of :class:`~b2sdk.account_info.upload_url_pool.UrlPoolAccountInfo`, + or any custom class derived from :class:`~b2sdk.account_info.abstract.AbstractAccountInfo` + :param cache: object, an instance of the one of the following classes: + :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, + :class:`~b2sdk.cache.AuthInfoCache`, + or any custom class derived from :class:`~b2sdk.cache.AbstractCache` + :param raw_api: object, an instance of one of the following classes: + :class:`~b2sdk.raw_api.B2RawApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, + or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` + :param max_upload_workers: int, a number of upload threads, default is 10 """ self.raw_api = raw_api or B2RawApi(B2Http()) if account_info is None: @@ -86,6 +92,8 @@ def set_thread_pool_size(self, max_workers): Must be called before any work starts, or the thread pool will get the default size of 1. + + :param max_workers: int, maximum allowed number of workers in a pool """ if self.upload_executor is not None: raise Exception('thread pool already created') @@ -100,6 +108,10 @@ def get_thread_pool(self): return self.upload_executor def authorize_automatically(self): + """ + Perform automatic account authorization, retrieving all account data + from account info object passed during initialization + """ try: self.authorize_account( self.account_info.get_realm(), @@ -112,6 +124,13 @@ def authorize_automatically(self): @limit_trace_arguments(only=('self', 'realm')) def authorize_account(self, realm, account_id_or_key_id, application_key): + """ + Perform account authorization + + :param realm: str, a realm to authiroze account in + :param account_id_or_key_id: str, account or key ID + :param application_key: str, user's application key + """ # Clean up any previous account info if it was for a different account. try: old_account_id = self.account_info.get_account_id() @@ -140,6 +159,11 @@ def authorize_account(self, realm, account_id_or_key_id, application_key): ) def get_account_id(self): + """ + Return account ID + + :return: str, account ID + """ return self.account_info.get_account_id() # buckets @@ -147,6 +171,16 @@ def get_account_id(self): def create_bucket( self, name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None ): + """ + Create a bucket + + :param name: str, bucket name + :param bucket_type: str, a bucket type, could be one of the following values: "allPublic", "allPrivate" + :param bucket_info: dict, additional bucket info to store with the bucket + :param cors_rules: dict, bucket CORS rules to store with the bucket + :param lifecycle_rules: dict, bucket lifecycle rules to store with the bucket + :return: an instance of :class:`~b2sdk.bucket.Bucket` + """ account_id = self.account_info.get_account_id() response = self.session.create_bucket( @@ -168,6 +202,23 @@ def create_bucket( return bucket def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None): + """ + Download a file with a given ID + + :param file_id: str, a file ID + :param download_dest: str, a local file path + :param progress_listener: object, an instance of the one of the following classes: \ + :class:`~b2sdk.bucket.PartProgressReporter`,\ + :class:`~b2sdk.progress.TqdmProgressListener`,\ + :class:`~b2sdk.progress.SimpleProgressListener`,\ + :class:`~b2sdk.progress.DoNothingProgressListener`,\ + :class:`~b2sdk.progress.ProgressListenerForTest`,\ + :class:`~b2sdk.report.SyncFileReporter`,\ + or any sub class of :class:`~b2sdk.progress.AbstractProgressListener` + :param range_: list, a list of two integers, the first one is a start\ + position, and the second oe is the end position in the file + :return: context manager that returns an object that supports iter_content() + """ url = self.session.get_download_url_by_id( file_id, url_factory=self.account_info.get_download_url, @@ -175,14 +226,20 @@ def download_file_by_id(self, file_id, download_dest, progress_listener=None, ra return self.transferer.download_file_from_url(url, download_dest, progress_listener, range_) def get_bucket_by_id(self, bucket_id): + """ + Return bucket object with a given ID + + :param bucket_id: str, a bucket ID + :return: an instance of :class:`~b2sdk.bucket.Bucket` + """ return Bucket(self, bucket_id) def get_bucket_by_name(self, bucket_name): """ Returns the Bucket for the given bucket_name. - :param bucket_name: The name of the bucket to return. - :return: a Bucket object + :param bucket_name: str, The name of the bucket to return. + :return: an instance of :class:`~b2sdk.bucket.Bucket` :raises NonExistentBucket: if the bucket does not exist in the account """ # Give a useful warning if the current application key does not @@ -208,6 +265,9 @@ def delete_bucket(self, bucket): For legacy reasons it returns whatever server sends in response, but API user should not rely on the response: if it doesn't raise an exception, it means that the operation was a success + + :param bucket: an instance of :class:`~b2sdk.bucket.Bucket` + :return: """ account_id = self.account_info.get_account_id() return self.session.delete_bucket(account_id, bucket.id_) @@ -222,8 +282,8 @@ def list_buckets(self, bucket_name=None): one bucket, you must specify the bucket name, or the request will be unauthorized. - :param bucket_name: Optional: the name of the one bucket to return. - :return: A list of Bucket objects. + :param bucket_name: str, the name of the one bucket to return. + :return: A list of instances of :class:`~b2sdk.bucket.Bucket`. """ # Give a useful warning if the current application key does not # allow access to the named bucket. @@ -250,9 +310,9 @@ def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Generator that yields a Part for each of the parts that have been uploaded. - :param file_id: the ID of the large file that is not finished - :param start_part_number: the first part number to return. defaults to the first part. - :param batch_size: the number of parts to fetch at a time from the server + :param file_id: str, the ID of the large file that is not finished + :param start_part_number: int, the first part number to return. defaults to the first part. + :param batch_size: int, the number of parts to fetch at a time from the server """ batch_size = batch_size or 100 while True: @@ -265,10 +325,21 @@ def list_parts(self, file_id, start_part_number=None, batch_size=None): # delete/cancel def cancel_large_file(self, file_id): + """ + Cancel a large file upload + + :param file_id: str, a file ID + """ response = self.session.cancel_large_file(file_id) return FileVersionInfoFactory.from_cancel_large_file_response(response) def delete_file_version(self, file_id, file_name): + """ + Permanently and irrevocably delete one version of a file + + :param file_id: str, a file ID + :param file_name: str, a file name + """ # filename argument is not first, because one day it may become optional response = self.session.delete_file_version(file_id, file_name) assert response['fileId'] == file_id @@ -277,12 +348,20 @@ def delete_file_version(self, file_id, file_name): # download def get_download_url_for_fileid(self, file_id): + """ + Returns a URL to download the given file by ID. + + :param file_id: str, a file ID + """ url = url_for_api(self.account_info, 'b2_download_file_by_id') return '%s?fileId=%s' % (url, file_id) def get_download_url_for_file_name(self, bucket_name, file_name): """ Returns a URL to download the given file by name. + + :param bucket_name: str, a bucket name + :param file_name: str, a file name """ self.check_bucket_restrictions(bucket_name) return '%s/file/%s/%s' % ( @@ -293,6 +372,15 @@ def get_download_url_for_file_name(self, bucket_name, file_name): def create_key( self, capabilities, key_name, valid_duration_seconds=None, bucket_id=None, name_prefix=None ): + """ + Create a new application key + + :param capabilities: list, a list of capabilities + :param key_name: str, a name of a key + :param valid_duration_seconds: int, key duration in seconds + :param bucket_id: str, a bucket ID + :param name_prefix: str, a name prefix + """ account_id = self.account_info.get_account_id() response = self.session.create_key( @@ -310,11 +398,21 @@ def create_key( return response def delete_key(self, application_key_id): + """ + Delete application key with a given ID + + :param application_key_id: str, an application key ID + """ response = self.session.delete_key(application_key_id=application_key_id) return response def list_keys(self, start_application_key_id=None): + """ + List application keys + + :param start_application_key_id: str, an application key ID to start from + """ account_id = self.account_info.get_account_id() return self.session.list_keys( @@ -333,6 +431,8 @@ def check_bucket_restrictions(self, bucket_name): If it does, does the bucket_name for a given api call match that. If not it raises a RestrictedBucket error. + + :param bucket_name: str, a bucket name """ allowed = self.account_info.get_allowed() allowed_bucket_name = allowed['bucketName'] From 89167bb3527fc7880b6975cbcb8ba4b5199fa030 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Fri, 1 Mar 2019 16:24:03 +0300 Subject: [PATCH 07/75] Correctly set types in docstrings --- b2sdk/api.py | 128 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 83 insertions(+), 45 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index fc1302f4b..5869906cf 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -61,16 +61,17 @@ class handles several things that simplify the task of uploading def __init__(self, account_info=None, cache=None, raw_api=None, max_upload_workers=10): """ Initializes the API using the given account info. - :param account_info: object, an instance of :class:`~b2sdk.account_info.upload_url_pool.UrlPoolAccountInfo`, + :param account_info: an instance of :class:`~b2sdk.account_info.upload_url_pool.UrlPoolAccountInfo`, or any custom class derived from :class:`~b2sdk.account_info.abstract.AbstractAccountInfo` - :param cache: object, an instance of the one of the following classes: + :param cache: an instance of the one of the following classes: :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, :class:`~b2sdk.cache.AuthInfoCache`, or any custom class derived from :class:`~b2sdk.cache.AbstractCache` - :param raw_api: object, an instance of one of the following classes: + :param raw_api: an instance of one of the following classes: :class:`~b2sdk.raw_api.B2RawApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` - :param max_upload_workers: int, a number of upload threads, default is 10 + :param max_upload_workers: a number of upload threads, default is 10 + :type max_upload_workers: int """ self.raw_api = raw_api or B2RawApi(B2Http()) if account_info is None: @@ -93,7 +94,8 @@ def set_thread_pool_size(self, max_workers): Must be called before any work starts, or the thread pool will get the default size of 1. - :param max_workers: int, maximum allowed number of workers in a pool + :param max_workers: maximum allowed number of workers in a pool + :type max_workers: int """ if self.upload_executor is not None: raise Exception('thread pool already created') @@ -127,9 +129,12 @@ def authorize_account(self, realm, account_id_or_key_id, application_key): """ Perform account authorization - :param realm: str, a realm to authiroze account in - :param account_id_or_key_id: str, account or key ID - :param application_key: str, user's application key + :param realm: a realm to authiroze account in + :type realm: str + :param account_id_or_key_id: account or key ID + :type account_id_or_key_id: str + :param application_key: user's application key + :type application_key: str """ # Clean up any previous account info if it was for a different account. try: @@ -162,7 +167,8 @@ def get_account_id(self): """ Return account ID - :return: str, account ID + :return: account ID + :rtype: str """ return self.account_info.get_account_id() @@ -174,12 +180,18 @@ def create_bucket( """ Create a bucket - :param name: str, bucket name - :param bucket_type: str, a bucket type, could be one of the following values: "allPublic", "allPrivate" - :param bucket_info: dict, additional bucket info to store with the bucket - :param cors_rules: dict, bucket CORS rules to store with the bucket - :param lifecycle_rules: dict, bucket lifecycle rules to store with the bucket - :return: an instance of :class:`~b2sdk.bucket.Bucket` + :param name: bucket name + :type name: str + :param bucket_type: a bucket type, could be one of the following values: "allPublic", "allPrivate" + :type bucket_type: str + :param bucket_info: additional bucket info to store with the bucket + :type bucket_info: dict + :param cors_rules: bucket CORS rules to store with the bucket + :type cors_rules: dict + :param lifecycle_rules: bucket lifecycle rules to store with the bucket + :type lifecycle_rules: dict + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket """ account_id = self.account_info.get_account_id() @@ -205,9 +217,11 @@ def download_file_by_id(self, file_id, download_dest, progress_listener=None, ra """ Download a file with a given ID - :param file_id: str, a file ID - :param download_dest: str, a local file path - :param progress_listener: object, an instance of the one of the following classes: \ + :param file_id: a file ID + :type file_id: str + :param download_dest: a local file path + :type download_dest: str + :param progress_listener: an instance of the one of the following classes: \ :class:`~b2sdk.bucket.PartProgressReporter`,\ :class:`~b2sdk.progress.TqdmProgressListener`,\ :class:`~b2sdk.progress.SimpleProgressListener`,\ @@ -215,8 +229,9 @@ def download_file_by_id(self, file_id, download_dest, progress_listener=None, ra :class:`~b2sdk.progress.ProgressListenerForTest`,\ :class:`~b2sdk.report.SyncFileReporter`,\ or any sub class of :class:`~b2sdk.progress.AbstractProgressListener` - :param range_: list, a list of two integers, the first one is a start\ + :param range_: a list of two integers, the first one is a start\ position, and the second oe is the end position in the file + :type range_: list :return: context manager that returns an object that supports iter_content() """ url = self.session.get_download_url_by_id( @@ -229,8 +244,10 @@ def get_bucket_by_id(self, bucket_id): """ Return bucket object with a given ID - :param bucket_id: str, a bucket ID - :return: an instance of :class:`~b2sdk.bucket.Bucket` + :param bucket_id: a bucket ID + :type bucket_id: str + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket """ return Bucket(self, bucket_id) @@ -238,9 +255,11 @@ def get_bucket_by_name(self, bucket_name): """ Returns the Bucket for the given bucket_name. - :param bucket_name: str, The name of the bucket to return. - :return: an instance of :class:`~b2sdk.bucket.Bucket` - :raises NonExistentBucket: if the bucket does not exist in the account + :param bucket_name: The name of the bucket to return. + :type bucket_name: str + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket + :raises b2sdk.exception.NonExistentBucket: if the bucket does not exist in the account """ # Give a useful warning if the current application key does not # allow access to the named bucket. @@ -266,8 +285,8 @@ def delete_bucket(self, bucket): but API user should not rely on the response: if it doesn't raise an exception, it means that the operation was a success - :param bucket: an instance of :class:`~b2sdk.bucket.Bucket` - :return: + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket """ account_id = self.account_info.get_account_id() return self.session.delete_bucket(account_id, bucket.id_) @@ -282,8 +301,10 @@ def list_buckets(self, bucket_name=None): one bucket, you must specify the bucket name, or the request will be unauthorized. - :param bucket_name: str, the name of the one bucket to return. - :return: A list of instances of :class:`~b2sdk.bucket.Bucket`. + :param bucket_name: the name of the one bucket to return. + :type bucket_name: str + :return: A list of instances of b2sdk.bucket.Bucket. + :rtype: list """ # Give a useful warning if the current application key does not # allow access to the named bucket. @@ -310,9 +331,12 @@ def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Generator that yields a Part for each of the parts that have been uploaded. - :param file_id: str, the ID of the large file that is not finished - :param start_part_number: int, the first part number to return. defaults to the first part. - :param batch_size: int, the number of parts to fetch at a time from the server + :param file_id: the ID of the large file that is not finished + :type file_id: str + :param start_part_number: the first part number to return. defaults to the first part. + :type start_part_number: int + :param batch_size: the number of parts to fetch at a time from the server + :type batch_size: int """ batch_size = batch_size or 100 while True: @@ -328,7 +352,8 @@ def cancel_large_file(self, file_id): """ Cancel a large file upload - :param file_id: str, a file ID + :param file_id: a file ID + :type file_id: str """ response = self.session.cancel_large_file(file_id) return FileVersionInfoFactory.from_cancel_large_file_response(response) @@ -337,8 +362,10 @@ def delete_file_version(self, file_id, file_name): """ Permanently and irrevocably delete one version of a file - :param file_id: str, a file ID - :param file_name: str, a file name + :param file_id: a file ID + :type: file_id: str + :param file_name: a file name + :type file_name: str """ # filename argument is not first, because one day it may become optional response = self.session.delete_file_version(file_id, file_name) @@ -351,7 +378,8 @@ def get_download_url_for_fileid(self, file_id): """ Returns a URL to download the given file by ID. - :param file_id: str, a file ID + :param file_id: a file ID + :type file_id: str """ url = url_for_api(self.account_info, 'b2_download_file_by_id') return '%s?fileId=%s' % (url, file_id) @@ -360,8 +388,10 @@ def get_download_url_for_file_name(self, bucket_name, file_name): """ Returns a URL to download the given file by name. - :param bucket_name: str, a bucket name - :param file_name: str, a file name + :param bucket_name: a bucket name + :type bucket_name: str + :param file_name: a file name + :type file_name: str """ self.check_bucket_restrictions(bucket_name) return '%s/file/%s/%s' % ( @@ -375,11 +405,16 @@ def create_key( """ Create a new application key - :param capabilities: list, a list of capabilities - :param key_name: str, a name of a key - :param valid_duration_seconds: int, key duration in seconds - :param bucket_id: str, a bucket ID - :param name_prefix: str, a name prefix + :param capabilities: a list of capabilities + :type capabilities: list + :param key_name: a name of a key + :type key_name: str + :param valid_duration_seconds: key duration in seconds + :type valid_duration_seconds: int + :param bucket_id: a bucket ID + :type bucket_id: str + :param name_prefix: a name prefix + :type name_prefix: str """ account_id = self.account_info.get_account_id() @@ -401,7 +436,8 @@ def delete_key(self, application_key_id): """ Delete application key with a given ID - :param application_key_id: str, an application key ID + :param application_key_id: an application key ID + :type application_key_id: """ response = self.session.delete_key(application_key_id=application_key_id) @@ -411,7 +447,8 @@ def list_keys(self, start_application_key_id=None): """ List application keys - :param start_application_key_id: str, an application key ID to start from + :param start_application_key_id: an application key ID to start from + :type start_application_key_id: str """ account_id = self.account_info.get_account_id() @@ -432,7 +469,8 @@ def check_bucket_restrictions(self, bucket_name): If it does, does the bucket_name for a given api call match that. If not it raises a RestrictedBucket error. - :param bucket_name: str, a bucket name + :param bucket_name: a bucket name + :type bucket_name: str """ allowed = self.account_info.get_allowed() allowed_bucket_name = allowed['bucketName'] From aa1d92084ce3861f8673867cf63a43efebf49b5d Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Fri, 1 Mar 2019 17:31:37 +0300 Subject: [PATCH 08/75] Docstrings for b2sdk.b2hhtp module updated --- b2sdk/b2http.py | 112 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 32 deletions(-) diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index f92029c6d..d8ddbabae 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -32,6 +32,9 @@ def _print_exception(e, indent=''): """ Used for debugging to print out nested exception structures. + + :param indent: message prefix + :type indent: str """ print(indent + 'EXCEPTION', repr(e)) print(indent + 'CLASS', type(e)) @@ -45,6 +48,9 @@ def _translate_errors(fcn, post_params=None): """ Calls the given function, turning any exception raised into the right kind of B2Error. + + :param post_params: request parameters + :type post_params: dict """ try: response = fcn() @@ -103,6 +109,11 @@ def _translate_and_retry(fcn, try_count, post_params=None): """ Try calling fcn try_count times, retrying only if the exception is a retryable B2Error. + + :param try_count: a number of retries + :type try_count: int + :param post_params: request parameters + :type post_params: dict """ # For all but the last try, catch the exception. wait_time = 1.0 @@ -147,9 +158,12 @@ def pre_request(self, method, url, headers): Raises an exception if this request should not be processed. The exception raised must inherit from B2HttpCallbackPreRequestException. - :param method: One of: 'POST', 'GET', etc. + :param method: str, One of: 'POST', 'GET', etc. + :type method: str :param url: The URL that will be used. + :type url: str :param headers: The header sent with the request. + :type headers: dict """ @@ -161,10 +175,13 @@ def post_request(self, method, url, headers, response): Raises an exception if this request should be treated as failing. The exception raised must inherit from B2HttpCallbackPostRequestException. - :param method: One of: 'POST', 'GET', etc. - :param url: The URL that will be used. - :param headers: The header sent with the request. - :param response: A response object from the requests library. + :param method: one of: 'POST', 'GET', etc. + :type method: str + :param url: the URL that will be used. + :type url: str + :param headers: the header sent with the request. + :type headers: dict + :param response: a response object from the requests library. """ @@ -175,6 +192,14 @@ def post_request(self, method, url, headers, response): clock on the local host. The Date header contains a string that looks like: "Fri, 16 Dec 2016 20:52:30 GMT". + + :param method: one of: 'POST', 'GET', etc. + :type method: str + :param url: the URL that will be used. + :type url: str + :param headers: the header sent with the request. + :type headers: dict + :param response: a response object from the requests library. """ # Make a string that uses month numbers instead of month names server_date_str = response.headers['Date'] @@ -216,11 +241,14 @@ class B2Http(object): The methods that return JSON either return a Python dict or raise a subclass of B2Error. They can be used like this: - try: - response_dict = b2_http.post_json_return_json(url, headers, params) - ... - except B2Error as e: - ... + .. code-block:: python + + try: + response_dict = b2_http.post_json_return_json(url, headers, params) + ... + except B2Error as e: + ... + """ def __init__(self, requests_module=None, install_clock_skew_hook=True): @@ -228,8 +256,9 @@ def __init__(self, requests_module=None, install_clock_skew_hook=True): Initialize with a reference to the requests module, which makes it easy to mock for testing. - The optional after_request_hook is called on the Response - object after every request that doesn't throw an exception. + :param requests_module: a reference to requests module + :param install_clock_skew_hook: if True, install a clock skew hook + :type install_clock_skew_hook: bool """ requests_to_use = requests_module or requests self.session = requests_to_use.Session() @@ -240,6 +269,9 @@ def __init__(self, requests_module=None, install_clock_skew_hook=True): def add_callback(self, callback): """ Adds a callback that inherits from HttpCallback. + + :param callback: a callback to be added to a chain + :type callback: callable """ self.callbacks.append(callback) @@ -247,16 +279,21 @@ def post_content_return_json(self, url, headers, data, try_count=1, post_params= """ Use like this: - try: - response_dict = b2_http.post_content_return_json(url, headers, data) - ... - except B2Error as e: - ... + .. code-block:: python + + try: + response_dict = b2_http.post_content_return_json(url, headers, data) + ... + except B2Error as e: + ... :param url: URL to call - :param headers: Headers to send. + :type url: str + :param headers: headers to send. + :type headers: dict :param data: bytes (Python 3) or str (Python 2), or a file-like object, to send :return: a dict that is the decoded JSON + :rtype: dict """ # Make the headers we'll send by adding User-Agent to what # the caller provided. Make a copy before modifying. @@ -287,15 +324,20 @@ def post_json_return_json(self, url, headers, params, try_count=1): """ Use like this: - try: - response_dict = b2_http.post_json_return_json(url, headers, params) - ... - except B2Error as e: - ... + .. code-block:: python + + try: + response_dict = b2_http.post_json_return_json(url, headers, params) + ... + except B2Error as e: + ... :param url: URL to call - :param headers: Headers to send. - :param params: A dict that will be converted to JSON + :type url: str + :param headers: headers to send. + :type headers: dict + :param params: a dict that will be converted to JSON + :type params: dict :return: a dict that is the decoded JSON """ data = six.BytesIO(six.b(json.dumps(params))) @@ -307,19 +349,25 @@ def get_content(self, url, headers, try_count=1): Use like this: - try: - with b2_http.get_content(url, headers) as response: - for byte_data in response.iter_content(chunk_size=1024): - ... - except B2Error as e: - ... + .. code-block:: python + + try: + with b2_http.get_content(url, headers) as response: + for byte_data in response.iter_content(chunk_size=1024): + ... + except B2Error as e: + ... The response object is only guarantee to have: - headers - iter_content() :param url: URL to call - :param headers: Headers to send + :type url: str + :param headers: headers to send + :type headers: dict + :param try_count: a number or retries + :type try_count: int :return: Context manager that returns an object that supports iter_content() """ # Make the headers we'll send by adding User-Agent to what From 8ac16fd8023e95fcf12e8e5055e58b10fbab7211 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Fri, 1 Mar 2019 17:47:07 +0300 Subject: [PATCH 09/75] Docstrings for b2sdk.bounded_queue_executor were updated --- b2sdk/bounded_queue_executor.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/b2sdk/bounded_queue_executor.py b/b2sdk/bounded_queue_executor.py index 8becd3546..4a73f3506 100644 --- a/b2sdk/bounded_queue_executor.py +++ b/b2sdk/bounded_queue_executor.py @@ -13,7 +13,7 @@ class BoundedQueueExecutor(object): """ - Wraps a futures.Executor and limits the number of requests that + Wraps a concurrent.futures.Executor and limits the number of requests that can be queued at once. Requests to submit() tasks block until there is room in the queue. @@ -26,11 +26,25 @@ class BoundedQueueExecutor(object): """ def __init__(self, executor, queue_limit): + """ + :param executor: an executor to be wrapped + :type executor: concurrent.futures.Executor + :param queue_limit: a queue limit + :type queue_limit: int + """ self.executor = executor self.semaphore = threading.Semaphore(queue_limit) self.num_exceptions = 0 def submit(self, fcn, *args, **kwargs): + """ + Start execution of a callable with the given optional and positional arguments + + :param fcn: a callable object + :type fcn: callable + :return: a future object + :rtype: concurrent.futures.Future + """ # Wait until there is room in the queue. self.semaphore.acquire() @@ -49,7 +63,15 @@ def run_it(): return self.executor.submit(run_it) def shutdown(self): + """ + Shutdown an executor + """ self.executor.shutdown() def get_num_exceptions(self): + """ + Return a number of exceptions + + :rtype: int + """ return self.num_exceptions From ae455c561070b886923106ec2bbb09dfef8cecde Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 14:43:43 +0300 Subject: [PATCH 10/75] Docstrings for objects in b2sdk.bucket module added --- b2sdk/bucket.py | 276 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 264 insertions(+), 12 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 0e74ae3e9..7ad0f852d 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -38,6 +38,10 @@ class LargeFileUploadState(object): """ def __init__(self, file_progress_listener): + """ + :param file_progress_listener: a progress listener object + :type file_progress_listener: b2sdk.progress.AbstractProgressListener + """ self.lock = threading.RLock() self.error_message = None self.file_progress_listener = file_progress_listener @@ -45,19 +49,42 @@ def __init__(self, file_progress_listener): self.bytes_completed = 0 def set_error(self, message): + """ + Set error message + + :param message: an error message + :type message: str + """ with self.lock: self.error_message = message def has_error(self): + """ + Check whether an error occured + + :rtype: bool + """ with self.lock: return self.error_message is not None def get_error_message(self): + """ + Get an error message + + :return: an error message + :rtype: str + """ with self.lock: assert self.has_error() return self.error_message def update_part_bytes(self, bytes_delta): + """ + Update listener progress info + + :param bytes_delta: number of bytes to increase a progress for + :type bytes_delta: int + """ with self.lock: self.bytes_completed += bytes_delta self.file_progress_listener.bytes_completed(self.bytes_completed) @@ -75,18 +102,37 @@ class PartProgressReporter(AbstractProgressListener): """ def __init__(self, large_file_upload_state, *args, **kwargs): + """ + :param large_file_upload_state: LargeFileUploadState object + :type large_file_upload_state: b2sdk.bucket.LargeFileUploadState + """ super(PartProgressReporter, self).__init__(*args, **kwargs) self.large_file_upload_state = large_file_upload_state self.prev_byte_count = 0 def bytes_completed(self, byte_count): + """ + Update bytes completed value + + :param bytes_count: a value to increase progress to + :type bytes_count: int + """ self.large_file_upload_state.update_part_bytes(byte_count - self.prev_byte_count) self.prev_byte_count = byte_count def close(self): + """ + Perform cleanup operations + """ pass def set_total_bytes(self, total_byte_count): + """ + Set total bytes value + + :param total_bytes_count: + :type total_bytes_count: int + """ pass @@ -112,6 +158,26 @@ def __init__( revision=None, bucket_dict=None ): + """ + :param api: an API object + :type api: b2sdk.api.B2Api + :param id_: a bucket id + :type id_: str + :param name: a bucket name + :type name: str + :param type_: a bucket type + :type type_: str + :param bucket_info: an info to store with a bucket + :type bucket_info: dict + :param cors_rules: CORS rules to store with a bucket + :type cors_rules: dict + :param lifecycle_rules: lifecycle rules to store with a bucket + :type lifecycle_rules: dict + :param revision: a bucket revision number + :type revision: int + :param bucket_dict: a dictionary which contains bucket parameters + :type bucket_dict: dict + """ self.api = api self.id_ = id_ self.name = name @@ -123,12 +189,33 @@ def __init__( self.bucket_dict = bucket_dict or {} def get_id(self): + """ + Return bucket ID + + :type: str + """ return self.id_ def set_info(self, new_bucket_info, if_revision_is=None): + """ + Update bucket info + + :param new_bucket_info: new bucket info dictionary + :type new_bucket_info: dict + :param if_revision_is: revision number, + update the info if revision equals to + if_revision_is' value + :type if_revision_is: int + """ return self.update(bucket_info=new_bucket_info, if_revision_is=if_revision_is) def set_type(self, bucket_type): + """ + Update bucket type + + :param bucket_type: a bucket type + :type bucket_type: str + """ return self.update(bucket_type=bucket_type) def update( @@ -139,6 +226,22 @@ def update( lifecycle_rules=None, if_revision_is=None ): + """ + Update various bucket parameters + + :param bucket_type: a bucket type + :type bucket_type: str + :param bucket_info: an info to store with a bucket + :type bucket_info: dict + :param cors_rules: CORS rules to store with a bucket + :type cors_rules: dict + :param lifecycle_rules: lifecycle rules to store with a bucket + :type lifecycle_rules: dict + :param if_revision_is: revision number, + update the info if revision equals to + if_revision_is' value + :type if_revision_is: int + """ account_id = self.api.account_info.get_account_id() return self.api.session.update_bucket( account_id, @@ -151,15 +254,45 @@ def update( ) def cancel_large_file(self, file_id): + """ + Cancel large file transfer + + :param file_id: a file ID + :type file_id: str + """ return self.api.cancel_large_file(file_id) def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None): - # download_file_by_id actually belongs in B2Api, not in Bucket, we just provide a convenient redirect here + """ + Download a file by ID, download_file_by_id actually belongs in B2Api, + not in Bucket, we just provide a convenient redirect here + + :param file_id: a file ID + :type file_id: str + :param download_dest: a local file path + :type download_dest: str + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener + :param range_: a list of two integer values, start and end offsets + :type range_: list + """ return self.api.download_file_by_id( file_id, download_dest, progress_listener, range_=range_ ) def download_file_by_name(self, file_name, download_dest, progress_listener=None, range_=None): + """ + Download a file by name + + :param file_id: a file ID + :type file_id: str + :param download_dest: a local file path + :type download_dest: str + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener + :param range_: a list of two integer values, start and end offsets + :type range_: list + """ url = self.api.session.get_download_url_by_name( self.name, file_name, @@ -170,23 +303,42 @@ def download_file_by_name(self, file_name, download_dest, progress_listener=None ) def get_download_authorization(self, file_name_prefix, valid_duration_in_seconds): + """ + Return an authorization token that is valid only for downloading + files from the given bucket + + :param file_name_prefix: a file name prefix, only files that match it could be downloaded + :type file_name_prefix: str + :param valid_duration_in_seconds: a token is valid only during this amount of seconds + :type valid_duration_in_seconds: int + """ response = self.api.session.get_download_authorization( self.id_, file_name_prefix, valid_duration_in_seconds ) return response['authorizationToken'] def list_parts(self, file_id, start_part_number=None, batch_size=None): + """ + Get a list of all parts that have been uploaded for a given file + + :param file_id: a file ID + :type file_id: str + :param start_part_number: the first part number to return. defaults to the first part. + :type start_part_number: int + :param batch_size: the number of parts to fetch at a time from the server + :type batch_size: int + """ return self.api.list_parts(file_id, start_part_number, batch_size) def ls( self, folder_to_list='', show_versions=False, - max_entries=None, recursive=False, - fetch_count=100 + fetch_count=None ): - """Pretends that folders exist, and yields the information about the files in a folder. + """ + Pretends that folders exist, and yields the information about the files in a folder. B2 has a flat namespace for the files in a bucket, but there is a convention of using "/" as if there were folders. This method searches through the @@ -196,14 +348,18 @@ def ls( When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. - :param folder: The name of the folder to list. Must not start with "/". - Empty string means top-level folder. - :param show_versions: When true returns info about all versions of a file, + :param folder_to_list: the name of the folder to list. Must not start with "/". + Empty string means top-level folder. + :type folder_to_list: str + :param show_versions: when true returns info about all versions of a file, when false, just returns info about the most recent versions. - :param max_entries: How many entries to return. 1 - 1000 - :param recursive: - :return: + :type show_versions: bool + :param recursive: if True, list folders recursively + :type recursive: bool + :param fetch_count: how many entries to return. 1 - 1000 + :type fetch_count: int + :rtype: generator """ # Every file returned must have a name that starts with the # folder name and a "/". @@ -287,6 +443,12 @@ def list_unfinished_large_files(self, start_file_id=None, batch_size=None): A generator that yields an UnfinishedLargeFile for each unfinished large file in the bucket, starting at the given file. + + :param start_file_id: a file ID to start with + :type start_file_id: str + :param batch_size: max file count + :type batch_size: int + :rtype: generator """ batch_size = batch_size or 100 while True: @@ -300,6 +462,16 @@ def list_unfinished_large_files(self, start_file_id=None, batch_size=None): break def start_large_file(self, file_name, content_type=None, file_info=None): + """ + Start large file transfer + + :param file_name: a file name + :type file_name: str + :param content_type: request content type + :type content_type: str + :param file_info: a file info to store with the file + :type file_info: dict + """ return UnfinishedLargeFile( self.api.session.start_large_file(self.id_, file_name, content_type, file_info) ) @@ -310,6 +482,17 @@ def upload_bytes( ): """ Upload bytes in memory to a B2 file + + :param data_bytes: a byte array to upload + :type data_bytes: bytes + :param file_name: a file name to upload bytes to + :type file_name: str + :param content_type: request content type + :type content_type: str + :param file_infos: a file info to store with the file + :type file_infos: dict + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener """ upload_source = UploadSourceBytes(data_bytes) return self.upload( @@ -332,6 +515,21 @@ def upload_local_file( ): """ Uploads a file on local disk to a B2 file. + + :param local_file: a path to a file on local disk + :type local_file: str + :param file_name: a file name of the new B2 file + :type file_name: str + :param content_type: request content type + :type content_type: str + :param file_infos: a file info to store with the file + :type file_infos: dict + :param sha1_sum: file SHA1 hash + :type sha1_sum: str + :param min_part_size: a minimum size of a part + :type min_part_size: int + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener """ upload_source = UploadSourceLocalFile(local_path=local_file, content_sha1=sha1_sum) return self.upload( @@ -360,12 +558,17 @@ def upload( file whose read() method returns bytes. :param upload_source: an UploadSource object that opens the source of the upload + :type upload_source: b2sdk.upload_source.UploadSource :param file_name: the file name of the new B2 file + :type file_name: str :param content_type: the MIME type, or None to accept the default based on file extension of the B2 file name + :type content_type: str, None :param file_infos: custom file info to be stored with the file + :type file_infos: dict :param min_part_size: the smallest part size to use + :type min_part_size: int :param progress_listener: object to notify as data is transferred - :return: + :type progress_listener: b2sdk.progress.AbstractProgressListener The function `opener` should return a file-like object, and it must be possible to call it more than once in case the upload @@ -608,6 +811,14 @@ def _get_upload_part_data(self, file_id): return (response['uploadUrl'], response['authorizationToken']) def get_download_url(self, filename): + """ + Get file download URL + + :param filename: a file name + :type filename: str + :return: file download URL + :rtype: str + """ return "%s/file/%s/%s" % ( self.api.account_info.get_download_url(), b2_url_encode(self.name), @@ -615,15 +826,36 @@ def get_download_url(self, filename): ) def hide_file(self, file_name): + """ + Hide a file + + :param file_name: a file name + :type file_name: str + :return: file version info + :rtype: b2sdk.file_version.FileVersionInfoFactory + """ response = self.api.session.hide_file(self.id_, file_name) return FileVersionInfoFactory.from_api_response(response) def delete_file_version(self, file_id, file_name): + """ + Delete file version + + :param file_id: a file ID + :type file_id: str + :param file_name: a file name + :type file_name: str + """ # filename argument is not first, because one day it may become optional return self.api.delete_file_version(file_id, file_name) @disable_trace def as_dict(self): # TODO: refactor with other as_dict() + """ + Return bucket representation as a dictionary + + :rtype: dict + """ result = { 'accountId': self.api.account_info.get_account_id(), 'bucketId': self.id_, @@ -639,14 +871,27 @@ def __repr__(self): class BucketFactory(object): + """ + This is a factory for creating bucket objects from another kind of objects + """ @classmethod def from_api_response(cls, api, response): + """ + Create a Bucket object from API response + + :param api: API object + :type api: b2sdk.api.B2Api + :param response: response object + :type response: object + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket + """ return [cls.from_api_bucket_dict(api, bucket_dict) for bucket_dict in response['buckets']] @classmethod def from_api_bucket_dict(cls, api, bucket_dict): """ - turns this: + Turns a dictionary, like this: .. code-block:: python @@ -661,6 +906,13 @@ def from_api_bucket_dict(cls, api, bucket_dict): into a Bucket object + :param api: API object + :type api: b2sdk.api.B2Api + :param bucket_dict: a dictionary with bucket properties + :type bucket_dict: dict + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket + """ bucket_name = bucket_dict['bucketName'] bucket_id = bucket_dict['bucketId'] From 65bd9c81375bb79c274d6774b0e6b7af3711482f Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 14:45:58 +0300 Subject: [PATCH 11/75] Formatting fixed --- b2sdk/bucket.py | 9 ++------- b2sdk/utils.py | 2 -- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 7ad0f852d..b6178181f 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -330,13 +330,7 @@ def list_parts(self, file_id, start_part_number=None, batch_size=None): """ return self.api.list_parts(file_id, start_part_number, batch_size) - def ls( - self, - folder_to_list='', - show_versions=False, - recursive=False, - fetch_count=None - ): + def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_count=None): """ Pretends that folders exist, and yields the information about the files in a folder. @@ -874,6 +868,7 @@ class BucketFactory(object): """ This is a factory for creating bucket objects from another kind of objects """ + @classmethod def from_api_response(cls, api, response): """ diff --git a/b2sdk/utils.py b/b2sdk/utils.py index da2c0ac73..b0910b0db 100644 --- a/b2sdk/utils.py +++ b/b2sdk/utils.py @@ -9,7 +9,6 @@ ###################################################################### from __future__ import division, print_function -import time import hashlib import os import platform @@ -26,7 +25,6 @@ except ImportError: import futures - # Global variable that says whether the app is shutting down _shutting_down = False From 31d67aab67cc56c97904dee99d10fe71884b75c4 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 16:06:11 +0300 Subject: [PATCH 12/75] Docstrings added for objects from b2sdk.progress module --- b2sdk/progress.py | 165 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 1 deletion(-) diff --git a/b2sdk/progress.py b/b2sdk/progress.py index 4fc320c45..afecd0f77 100644 --- a/b2sdk/progress.py +++ b/b2sdk/progress.py @@ -47,6 +47,9 @@ def set_total_bytes(self, total_byte_count): Always called before __enter__ to set the expected total number of bytes. May be called more than once if an upload is retried. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int """ @abstractmethod @@ -55,6 +58,9 @@ def bytes_completed(self, byte_count): Reports that the given number of bytes have been transferred so far. This is not a delta, it is the total number of bytes transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int """ @abstractmethod @@ -81,6 +87,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): class TqdmProgressListener(AbstractProgressListener): + """ + Progress listener based on tqdm library + """ + def __init__(self, description, *args, **kwargs): self.description = description self.tqdm = None # set in set_total_bytes() @@ -88,6 +98,12 @@ def __init__(self, description, *args, **kwargs): super(TqdmProgressListener, self).__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count): + """ + Set the expected total number of bytes. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int + """ raise_if_shutting_down() if self.tqdm is None: self.tqdm = tqdm( @@ -100,6 +116,14 @@ def set_total_bytes(self, total_byte_count): ) def bytes_completed(self, byte_count): + """ + Reports that the given number of bytes have been transferred + so far. This is not a delta, it is the total number of bytes + transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int + """ # tqdm doesn't support running the progress bar backwards, # so on an upload retry, it just won't move until it gets # past the point where it failed. @@ -109,12 +133,19 @@ def bytes_completed(self, byte_count): self.prev_value = byte_count def close(self): + """ + Perform clean up operations + """ if self.tqdm is not None: self.tqdm.close() super(TqdmProgressListener, self).close() class SimpleProgressListener(AbstractProgressListener): + """ + Just a simple progress listener which prints info on a console + """ + def __init__(self, description, *args, **kwargs): self.desc = description self.complete = 0 @@ -123,10 +154,24 @@ def __init__(self, description, *args, **kwargs): super(SimpleProgressListener, self).__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count): + """ + Set the expected total number of bytes. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int + """ raise_if_shutting_down() self.total = total_byte_count def bytes_completed(self, byte_count): + """ + Reports that the given number of bytes have been transferred + so far. This is not a delta, it is the total number of bytes + transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int + """ raise_if_shutting_down() now = time.time() elapsed = now - self.last_time @@ -138,6 +183,9 @@ def bytes_completed(self, byte_count): self.any_printed = True def close(self): + """ + Perform clean up operations + """ raise_if_shutting_down() if self.any_printed: print(' DONE.') @@ -145,13 +193,34 @@ def close(self): class DoNothingProgressListener(AbstractProgressListener): + """ + This listener performs no any output + """ + def set_total_bytes(self, total_byte_count): + """ + Set the expected total number of bytes. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int + """ raise_if_shutting_down() def bytes_completed(self, byte_count): + """ + Reports that the given number of bytes have been transferred + so far. This is not a delta, it is the total number of bytes + transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int + """ raise_if_shutting_down() def close(self): + """ + Perform clean up operations + """ super(DoNothingProgressListener, self).close() @@ -179,6 +248,15 @@ def get_calls(self): def make_progress_listener(description, quiet): + """ + Returns a progress listener object depending on some conditions + + :param description: listener description + :type description: str + :param quiet: if True, do not output anything + :type quiet: bool + :return: a listener object + """ if quiet: return DoNothingProgressListener() elif tqdm is not None: @@ -194,6 +272,13 @@ class RangeOfInputStream(object): """ def __init__(self, stream, offset, length): + """ + :param stream: a seekable stream + :param offset: offset in the stream + :type offset: int + :param length: max number of bytes to read + :type length: int + """ self.stream = stream self.offset = offset self.remaining = length @@ -206,9 +291,22 @@ def __exit__(self, exc_type, exc_val, exc_tb): return self.stream.__exit__(exc_type, exc_val, exc_tb) def seek(self, pos): + """ + Seek to a given position in the stream + + :param pos: position in the stream + :type pos: int + """ self.stream.seek(self.offset + pos) def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ if size is None: to_read = self.remaining else: @@ -231,8 +329,9 @@ def __init__(self, stream, progress_listener, offset=0): :param stream: the stream to read from or write to :param progress_listener: the listener that we tell about progress + :type progress_listener: b2sdk.progress.AbstractProgressListener :param offset: the starting byte offset in the file - :return: None + :type offset: int """ assert progress_listener is not None self.stream = stream @@ -247,15 +346,36 @@ def __exit__(self, exc_type, exc_val, exc_tb): return self.stream.__exit__(exc_type, exc_val, exc_tb) def seek(self, pos): + """ + Seek to a given position in the stream + + :param pos: position in the stream + :type pos: int + """ return self.stream.seek(pos) def tell(self): + """ + Return current stream position + + :rtype: int + """ return self.stream.tell() def flush(self): + """ + Flush the stream + """ self.stream.flush() def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ if size is None: data = self.stream.read() else: @@ -263,6 +383,11 @@ def read(self, size=None): return data def write(self, data): + """ + Write data to the stream + + :param data: a data to write to the stream + """ self.stream.write(data) def _update(self, delta): @@ -271,14 +396,34 @@ def _update(self, delta): class ReadingStreamWithProgress(AbstractStreamWithProgress): + """ + Wraps a file-like object, updates progress while reading + """ + def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ data = super(ReadingStreamWithProgress, self).read(size) self._update(len(data)) return data class WritingStreamWithProgress(AbstractStreamWithProgress): + """ + Wraps a file-like object, updates progress while writing + """ + def write(self, data): + """ + Write data to the stream + + :param data: a data to write to the stream + """ self._update(len(data)) super(WritingStreamWithProgress, self).write(data) @@ -306,6 +451,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): return self.stream.__exit__(exc_type, exc_val, exc_tb) def seek(self, pos): + """ + Seek to a given position in the stream + + :param pos: position in the stream + :type pos: int + """ assert pos == 0 self.stream.seek(0) self.digest = hashlib.sha1() @@ -313,6 +464,13 @@ def seek(self, pos): self.hash_read = 0 def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ data = b'' if self.hash is None: # Read some bytes from stream @@ -339,4 +497,9 @@ def read(self, size=None): return data def hash_size(self): + """ + Calculate size of a hash string + + :rtype: int + """ return self.digest.digest_size * 2 From 55d5d9064fe015214cf2f9671f301e8ab15009fb Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 17:12:41 +0300 Subject: [PATCH 13/75] Docstrings added for objects in b2sdk.sync.action module --- b2sdk/sync/action.py | 183 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index b914e1240..ee15fd2f1 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -36,6 +36,16 @@ class AbstractAction(object): """ def run(self, bucket, reporter, dry_run=False): + """ + Main action routine + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + :param dry_run: if True, perform a dry run + :type dry_run: bool + """ raise_if_shutting_down() try: if not dry_run: @@ -50,23 +60,51 @@ def run(self, bucket, reporter, dry_run=False): def get_bytes(self): """ Returns the number of bytes to transfer for this action. + + :rtype: int """ @abstractmethod def do_action(self, bucket, reporter): """ Performs the action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport """ @abstractmethod def do_report(self, bucket, reporter): """ Report the action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport """ class B2UploadAction(AbstractAction): + """ + File uploading action + """ + def __init__(self, local_full_path, relative_name, b2_file_name, mod_time_millis, size): + """ + :param local_full_path: a local file path + :type local_full_path: str + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a new remote file + :type b2_file_name: str + :param mod_time_millis: file modification time in milliseconds + :type mod_time_millis: int + :param size: a file size + :type size: int + """ self.local_full_path = local_full_path self.relative_name = relative_name self.b2_file_name = b2_file_name @@ -74,9 +112,22 @@ def __init__(self, local_full_path, relative_name, b2_file_name, mod_time_millis self.size = size def get_bytes(self): + """ + Return file size + + :rtype: int + """ return self.size def do_action(self, bucket, reporter): + """ + Performs the uploading action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ bucket.upload( UploadSourceLocalFile(self.local_full_path), self.b2_file_name, @@ -85,6 +136,14 @@ def do_action(self, bucket, reporter): ) def do_report(self, bucket, reporter): + """ + Report the uploading action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ reporter.print_completion('upload ' + self.relative_name) def __str__(self): @@ -95,16 +154,44 @@ def __str__(self): class B2HideAction(AbstractAction): def __init__(self, relative_name, b2_file_name): + """ + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a remote file + :type b2_file_name: str + """ self.relative_name = relative_name self.b2_file_name = b2_file_name def get_bytes(self): + """ + Return file size + + :return: always zero + :rtype: int + """ return 0 def do_action(self, bucket, reporter): + """ + Performs the hiding action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ bucket.hide_file(self.b2_file_name) def do_report(self, bucket, reporter): + """ + Report the hiding action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ reporter.update_transfer(1, 0) reporter.print_completion('hide ' + self.relative_name) @@ -116,6 +203,20 @@ class B2DownloadAction(AbstractAction): def __init__( self, relative_name, b2_file_name, file_id, local_full_path, mod_time_millis, file_size ): + """ + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a remote file + :type b2_file_name: str + :param file_id: a file ID + :type file_id: str + :param local_full_path: a local file path + :type local_full_path: str + :param mod_time_millis: file modification time in milliseconds + :type mod_time_millis: int + :param file_size: a file size + :type file_size: int + """ self.relative_name = relative_name self.b2_file_name = b2_file_name self.file_id = file_id @@ -124,9 +225,22 @@ def __init__( self.file_size = file_size def get_bytes(self): + """ + Return file size + + :rtype: int + """ return self.file_size def do_action(self, bucket, reporter): + """ + Performs the downloading action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ # Make sure the directory exists parent_dir = os.path.dirname(self.local_full_path) if not os.path.isdir(parent_dir): @@ -150,6 +264,14 @@ def do_action(self, bucket, reporter): os.rename(download_path, self.local_full_path) def do_report(self, bucket, reporter): + """ + Report the downloading action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ reporter.print_completion('dnload ' + self.relative_name) def __str__(self): @@ -161,18 +283,50 @@ def __str__(self): class B2DeleteAction(AbstractAction): def __init__(self, relative_name, b2_file_name, file_id, note): + """ + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a remote file + :type b2_file_name: str + :param file_id: a file ID + :type file_id: str + :param note: a deletion note + :type note: str + """ self.relative_name = relative_name self.b2_file_name = b2_file_name self.file_id = file_id self.note = note def get_bytes(self): + """ + Return file size + + :return: always zero + :rtype: int + """ return 0 def do_action(self, bucket, reporter): + """ + Performs the deleting action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ bucket.api.delete_file_version(self.file_id, self.b2_file_name) def do_report(self, bucket, reporter): + """ + Report the deleting action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name + ' ' + self.note) @@ -182,16 +336,45 @@ def __str__(self): class LocalDeleteAction(AbstractAction): def __init__(self, relative_name, full_path): + """ + :param relative_name: a relative file name + :type relative_name: str + :param full_path: a full local path + :type: str + """ self.relative_name = relative_name self.full_path = full_path def get_bytes(self): + """ + Return file size + + :return: always zero + :rtype: int + """ return 0 def do_action(self, bucket, reporter): + """ + Performs the deleting of a local file action, + returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ os.unlink(self.full_path) def do_report(self, bucket, reporter): + """ + Report the deleting of a local file action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a SyncReport object + :type reporter: b2sdk.report.SyhncReport + """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name) From c8d3a8bbbd570210d1021c88c28a3f8cdcb114af Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 17:17:58 +0300 Subject: [PATCH 14/75] Docstrings added for objects in b2sdk.sync.exception module --- b2sdk/sync/exception.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/b2sdk/sync/exception.py b/b2sdk/sync/exception.py index 3067020c8..d8dda8b0e 100644 --- a/b2sdk/sync/exception.py +++ b/b2sdk/sync/exception.py @@ -12,7 +12,17 @@ class EnvironmentEncodingError(B2Error): + """ + Raised when a file name can not be decoded with system encoding + """ + def __init__(self, filename, encoding): + """ + :param filename: an encoded file name + :type filename: str, bytes + :param encoding: file name encoding + :type encoding: str + """ super(EnvironmentEncodingError, self).__init__() self.filename = filename self.encoding = encoding From f60c82a115d2b21a6119ad91ea3310b8bc9e4ca9 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 17:24:35 +0300 Subject: [PATCH 15/75] Docstrings added for objects in b2sdk.sync.file module --- b2sdk/sync/file.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/b2sdk/sync/file.py b/b2sdk/sync/file.py index a16c96b33..3b96cf20c 100644 --- a/b2sdk/sync/file.py +++ b/b2sdk/sync/file.py @@ -21,10 +21,19 @@ class File(object): """ def __init__(self, name, versions): + """ + :param name: a relative file name + :type name: str + :param versions: a list of file versions + :type versions: list + """ self.name = name self.versions = versions def latest_version(self): + """ + Return the latest file version + """ return self.versions[0] def __repr__(self): @@ -39,9 +48,16 @@ class FileVersion(object): def __init__(self, id_, file_name, mod_time, action, size): """ :param id_: the B2 file id, or the local full path name + :type id_: str + :param file_name: a relative file name + :type file_name: str :param mod_time: modification time, in milliseconds, to avoid rounding issues with millisecond times from B2 + :type mod_time: int :param action: "hide" or "upload" (never "start") + :type action: str + :param size: a file size + :type size: int """ self.id_ = id_ self.name = file_name From 6f03ac1fb42fab5595c651068abba0dc6c07b466 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 17:27:18 +0300 Subject: [PATCH 16/75] Docstrings added for objects in b2sdk.sync.folder_parser module --- b2sdk/sync/folder_parser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/b2sdk/sync/folder_parser.py b/b2sdk/sync/folder_parser.py index d3b9c545a..d183c3663 100644 --- a/b2sdk/sync/folder_parser.py +++ b/b2sdk/sync/folder_parser.py @@ -21,6 +21,11 @@ def parse_sync_folder(folder_name, api): because the previous sync command didn't use it. Anything else is treated like a local folder. + + :param folder_name: a name of the folder, either local or remote + :type folder_name: str + :param api: an API object + :type api: b2sdk.api.B2Api """ if folder_name.startswith('b2://'): return _parse_bucket_and_folder(folder_name[5:], api) From be569e93b235bbf3903a868b802c26c17b5fccab Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 17:36:39 +0300 Subject: [PATCH 17/75] Typos fixed --- b2sdk/sync/action.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index ee15fd2f1..9627f86e1 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -42,7 +42,7 @@ def run(self, bucket, reporter, dry_run=False): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport :param dry_run: if True, perform a dry run :type dry_run: bool """ @@ -72,7 +72,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ @abstractmethod @@ -83,7 +83,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ @@ -126,7 +126,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ bucket.upload( UploadSourceLocalFile(self.local_full_path), @@ -142,7 +142,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ reporter.print_completion('upload ' + self.relative_name) @@ -179,7 +179,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ bucket.hide_file(self.b2_file_name) @@ -190,7 +190,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ reporter.update_transfer(1, 0) reporter.print_completion('hide ' + self.relative_name) @@ -239,7 +239,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ # Make sure the directory exists parent_dir = os.path.dirname(self.local_full_path) @@ -270,7 +270,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ reporter.print_completion('dnload ' + self.relative_name) @@ -314,7 +314,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ bucket.api.delete_file_version(self.file_id, self.b2_file_name) @@ -325,7 +325,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name + ' ' + self.note) @@ -362,7 +362,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ os.unlink(self.full_path) @@ -373,7 +373,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a SyncReport object - :type reporter: b2sdk.report.SyhncReport + :type reporter: b2sdk.sync.report.SyncReport """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name) From 9c42cb1c95c6148bf4e2deb85b9be7e6e5ef3e2d Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 17:54:07 +0300 Subject: [PATCH 18/75] Docstrings for reporter objects updated --- b2sdk/sync/action.py | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index 9627f86e1..2d8a91242 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -41,8 +41,7 @@ def run(self, bucket, reporter, dry_run=False): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors :param dry_run: if True, perform a dry run :type dry_run: bool """ @@ -71,8 +70,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ @abstractmethod @@ -82,8 +80,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ @@ -125,8 +122,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ bucket.upload( UploadSourceLocalFile(self.local_full_path), @@ -141,8 +137,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ reporter.print_completion('upload ' + self.relative_name) @@ -178,8 +173,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ bucket.hide_file(self.b2_file_name) @@ -189,8 +183,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion('hide ' + self.relative_name) @@ -238,8 +231,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ # Make sure the directory exists parent_dir = os.path.dirname(self.local_full_path) @@ -269,8 +261,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ reporter.print_completion('dnload ' + self.relative_name) @@ -313,8 +304,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ bucket.api.delete_file_version(self.file_id, self.b2_file_name) @@ -324,8 +314,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name + ' ' + self.note) @@ -361,8 +350,7 @@ def do_action(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ os.unlink(self.full_path) @@ -372,8 +360,7 @@ def do_report(self, bucket, reporter): :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket - :param reporter: a SyncReport object - :type reporter: b2sdk.sync.report.SyncReport + :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name) From a49394975438589aef8865497ad6e19bcc3eecb2 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 2 Mar 2019 18:05:20 +0300 Subject: [PATCH 19/75] Docstrings added for objects in module b2sdk.sync.folder --- b2sdk/sync/folder.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/b2sdk/sync/folder.py b/b2sdk/sync/folder.py index 2d3b51424..df607024d 100644 --- a/b2sdk/sync/folder.py +++ b/b2sdk/sync/folder.py @@ -49,24 +49,37 @@ def all_files(self, reporter, policies_manager): If a file is found, but does not exist (for example due to a broken symlink or a race), reporter will be informed about each such problem. + + :param reporter: a place to report errors + :param policies_manager: a policies manager object """ @abstractmethod def folder_type(self): """ Returns one of: 'b2', 'local' + + :rtype: str """ @abstractmethod def make_full_path(self, file_name): """ Only for local folders, returns the full path to the file. + + :param file_name: a file name + :type file_name: str """ def join_b2_path(b2_dir, b2_name): """ Like os.path.join, but for B2 file names where the root directory is called ''. + + :param b2_dir: a directory path + :type b2_dir: str + :param b2_name: a file name + :type b2_name: str """ if b2_dir == '': return b2_name @@ -84,19 +97,37 @@ def __init__(self, root): Initializes a new folder. :param root: Path to the root of the local folder. Must be unicode. + :type root: str """ if not isinstance(root, six.text_type): raise ValueError('folder path should be unicode: %s' % repr(root)) self.root = fix_windows_path_limit(os.path.abspath(root)) def folder_type(self): + """ + Return folder type + + :rtype: str + """ return 'local' def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + """ + Yield all files + + :param reporter: a place to report errors + :param policies_manager: a policy manager object, default is DEFAULT_SCAN_MANAGER + """ for file_object in self._walk_relative_paths(self.root, '', reporter, policies_manager): yield file_object def make_full_path(self, file_name): + """ + Convert a file name into an absolute path + + :param file_name: a file name + :type file_name: str + """ return os.path.join(self.root, file_name.replace('/', os.path.sep)) def ensure_present(self): @@ -229,12 +260,26 @@ class B2Folder(AbstractFolder): """ def __init__(self, bucket_name, folder_name, api): + """ + :param bucket_name: a name of the bucket + :type bucket_name: str + :param folder_name: a folder name + :type folder_name: str + :param api: an API object + :type api: b2sdk.api.B2Api + """ self.bucket_name = bucket_name self.folder_name = folder_name self.bucket = api.get_bucket_by_name(bucket_name) self.prefix = '' if self.folder_name == '' else self.folder_name + '/' def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + """ + Yield all files + + :param reporter: a place to report errors + :param policies_manager: a policies manager object, default is DEFAULT_SCAN_MANAGER + """ current_name = None current_versions = [] for (file_version_info, folder_name) in self.bucket.ls( @@ -267,9 +312,20 @@ def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): yield File(current_name, current_versions) def folder_type(self): + """ + Return folder type + + :rtype: str + """ return 'b2' def make_full_path(self, file_name): + """ + Make an absolute path from a file name + + :param file_name: a file name + :type file_name: str + """ if self.folder_name == '': return file_name else: From 7f2db66d691fe8885e57607fc3a1c6c1d9d15afa Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 10:27:09 +0300 Subject: [PATCH 20/75] Docstrings added for objects in policy_manager module --- b2sdk/sync/policy_manager.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/b2sdk/sync/policy_manager.py b/b2sdk/sync/policy_manager.py index d3b600ec1..9000803d7 100644 --- a/b2sdk/sync/policy_manager.py +++ b/b2sdk/sync/policy_manager.py @@ -13,16 +13,46 @@ class SyncPolicyManager(object): + """ + Policy manager, implements a logic to get a correct policy class + and create a policy object based on various parameters + """ def __init__(self): self.policies = {} # dict<,> def get_policy( self, sync_type, source_file, source_folder, dest_file, dest_folder, now_millis, args ): + """ + Return policy object + + :param sync_type: synchronization type + :type sync_type: str + :param source_file: source file name + :type source_file: str + :param source_folder: a source folder path + :type source_folder: str + :param dest_file: destination file name + :type dest_file: str + :param dest_folder: a destination folder path + :type dest_folder: str + :param now_millis: current time in milliseconds + :type now_millis: int + :param args: an object which holds command line arguments + :return: a policy object + """ policy_class = self.get_policy_class(sync_type, args) return policy_class(source_file, source_folder, dest_file, dest_folder, now_millis, args) def get_policy_class(self, sync_type, args): + """ + Get policy class by a given sync type + + :param sync_type: synchronization type + :type sync_type: str + :param args: an object which holds command line arguments + :return: a policy class + """ if sync_type == 'local-to-b2': if args.delete: return UpAndDeletePolicy From 32cd955489d856acfe75768b678822a8496d934b Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 11:00:17 +0300 Subject: [PATCH 21/75] Docstrings adde for objects in b2sdk.sync.policy module --- b2sdk/sync/policy.py | 72 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 7941f9514..d792be5f2 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -23,10 +23,26 @@ @six.add_metaclass(ABCMeta) class AbstractFileSyncPolicy(object): + """ + Abstract policy class + """ DESTINATION_PREFIX = NotImplemented SOURCE_PREFIX = NotImplemented def __init__(self, source_file, source_folder, dest_file, dest_folder, now_millis, args): + """ + :param source_file: source file name + :type source_file: str + :param source_folder: source folder name + :type source_folder: str + :param dest_file: destination file name + :type dest_file: str + :param dest_folder: destination folder name + :type dest_folder: str + :param now_millis: current time in milliseconds + :type now_millis: int + :param args: an object which holds command line arguments + """ self._source_file = source_file self._source_folder = source_folder self._dest_file = dest_file @@ -56,6 +72,13 @@ def files_are_different(cls, source_file, dest_file, args): """ Compare two files and determine if the the destination file should be replaced by the source file. + + :param source_file: source file name + :type source_file: str + :param dest_file: destination file name + :type dest_file: str + :param args: an object which holds command line arguments + :rtype: bool """ # Compare using modification time by default @@ -120,6 +143,9 @@ def files_are_different(cls, source_file, dest_file, args): raise CommandError('Invalid option for --compareVersions') def get_all_actions(self): + """ + Yield file actions. + """ if self._should_transfer(): yield self._make_transfer_action() self._transferred = True @@ -145,7 +171,7 @@ def _make_transfer_action(self): class DownPolicy(AbstractFileSyncPolicy): """ - file is synced down (from the cloud to disk) + File is synced down (from the cloud to disk) """ DESTINATION_PREFIX = 'local://' SOURCE_PREFIX = 'b2://' @@ -163,7 +189,7 @@ def _make_transfer_action(self): class UpPolicy(AbstractFileSyncPolicy): """ - file is synced up (from disk the cloud) + File is synced up (from disk the cloud) """ DESTINATION_PREFIX = 'b2://' SOURCE_PREFIX = 'local://' @@ -180,7 +206,7 @@ def _make_transfer_action(self): class UpAndDeletePolicy(UpPolicy): """ - file is synced up (from disk to the cloud) and the delete flag is SET + File is synced up (from disk to the cloud) and the delete flag is SET """ def _get_hide_delete_actions(self): @@ -197,7 +223,7 @@ def _get_hide_delete_actions(self): class UpAndKeepDaysPolicy(UpPolicy): """ - file is synced up (from disk to the cloud) and the keepDays flag is SET + File is synced up (from disk to the cloud) and the keepDays flag is SET """ def _get_hide_delete_actions(self): @@ -212,7 +238,7 @@ def _get_hide_delete_actions(self): class DownAndDeletePolicy(DownPolicy): """ - file is synced down (from the cloud to disk) and the delete flag is SET + File is synced down (from the cloud to disk) and the delete flag is SET """ def _get_hide_delete_actions(self): @@ -226,12 +252,22 @@ def _get_hide_delete_actions(self): class DownAndKeepDaysPolicy(DownPolicy): """ - file is synced down (from the cloud to disk) and the keepDays flag is SET + File is synced down (from the cloud to disk) and the keepDays flag is SET """ pass def make_b2_delete_note(version, index, transferred): + """ + Create a note message for delete action + + :param version: an object which contains file version info + :param index: file version index + :type index: int + :param transferred: if True, file has been transferred, + False otherwise + :type transferred: bool + """ note = '' if version.action == 'hide': note = '(hide marker)' @@ -243,6 +279,16 @@ def make_b2_delete_note(version, index, transferred): def make_b2_delete_actions(source_file, dest_file, dest_folder, transferred): """ Creates the actions to delete files stored on B2, which are not present locally. + + :param source_file: source file name + :type source_file: str + :param dest_file: destination file name + :type dest_file: str + :param dest_folder: destination folder + :type dest_folder: str + :param transferred: if True, file has been transferred, + False otherwise + :type transferred: bool """ if dest_file is None: # B2 does not really store folders, so there is no need to hide @@ -271,6 +317,20 @@ def make_b2_keep_days_actions( days ago, 15 days ago, and 25 days ago, and the keepDays is 10, only the 25-day old version can be deleted. The 15 day-old version was visible 10 days ago. + + :param source_file: source file name + :type source_file: str + :param dest_file: destination file name + :type dest_file: str + :param dest_folder: destination folder + :type dest_folder: str + :param transferred: if True, file has been transferred, + False otherwise + :type transferred: bool + :param keep_days: how many days to keep a file + :type keep_days: int + :param now_millis: current time in milliseconds + :type now_millis: int """ deleting = False if dest_file is None: From fca7a6626307f6bd1687687686acdeefd137c8ba Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 11:30:38 +0300 Subject: [PATCH 22/75] Docstrings added for objects in b2sdk.sync.report module --- b2sdk/sync/report.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/b2sdk/sync/report.py b/b2sdk/sync/report.py index d6463fd6a..380f32b55 100644 --- a/b2sdk/sync/report.py +++ b/b2sdk/sync/report.py @@ -39,6 +39,11 @@ class SyncReport(object): UPDATE_INTERVAL = 0.1 def __init__(self, stdout, no_progress): + """ + :param stdout: standard output file object + :param no_progress: if True, do not show progress + :type no_progress: bool + """ self.stdout = stdout self.no_progress = no_progress self.start_time = time.time() @@ -59,6 +64,9 @@ def __init__(self, stdout, no_progress): self.warnings = [] def close(self): + """ + Perform clean up + """ with self.lock: if not self.no_progress: self._print_line('', False) @@ -73,12 +81,21 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close() def error(self, message): + """ + Print an error, gracefully interleaving it with a progress bar + + :param message: an error message + :type message: str + """ self.print_completion(message) def print_completion(self, message): """ Removes the progress bar, prints a message, and puts the progress bar back. + + :param message: an error message + :type message: str """ with self.lock: if not self.closed: @@ -87,6 +104,9 @@ def print_completion(self, message): self._update_progress() def _update_progress(self): + """ + Update progress output + """ raise_if_shutting_down() if not self.closed and not self.no_progress: now = time.time() @@ -127,7 +147,9 @@ def _print_line(self, line, newline): Prints a line to stdout. :param line: A string without a \r or \n in it. + :type line: str :param newline: True if the output should move to a new line after this one. + :type newline: bool """ if len(line) < len(self.current_line): line += ' ' * (len(self.current_line) - len(line)) @@ -156,6 +178,9 @@ def _print_line(self, line, newline): def update_local(self, delta): """ Reports that more local files have been found. + + :param delta: number of files found since the last check + :type delta: int """ with self.lock: self.local_file_count += delta @@ -172,12 +197,23 @@ def end_local(self): def update_compare(self, delta): """ Reports that more files have been compared. + + :param delta: number of files compared + :type delta: int """ with self.lock: self.compare_count += delta self._update_progress() def end_compare(self, total_transfer_files, total_transfer_bytes): + """ + Report that the comparison has been finished + + :param total_transfer_files: total number of transferred files + :type total_transfer_files: int + :param total_transfer_bytes: total number of transferred bytes + :type total_transfer_bytes: int + """ with self.lock: self.compare_done = True self.total_transfer_files = total_transfer_files @@ -185,15 +221,35 @@ def end_compare(self, total_transfer_files, total_transfer_bytes): self._update_progress() def update_transfer(self, file_delta, byte_delta): + """ + Update transfer info + + :param file_delta: number of files transferred + :type file_delta: int + :param byte_delta: number of bytes transferred + :type byte_delta: int + """ with self.lock: self.transfer_files += file_delta self.transfer_bytes += byte_delta self._update_progress() def local_access_error(self, path): + """ + Add a file access error message to the list of warnings + + :param path: file path + :type path: str + """ self.warnings.append('WARNING: %s could not be accessed (broken symlink?)' % (path,)) def local_permission_error(self, path): + """ + Add a permission error message to the list of warnings + + :param path: file path + :type path: str + """ self.warnings.append( 'WARNING: %s could not be accessed (no permissions to read?)' % (path,) ) @@ -208,23 +264,44 @@ class SyncFileReporter(AbstractProgressListener): """ def __init__(self, reporter, *args, **kwargs): + """ + :param reporter: a reporter object + """ super(SyncFileReporter, self).__init__(*args, **kwargs) self.bytes_so_far = 0 self.reporter = reporter def close(self): + """ + Perform clean up + """ # no more bytes are done, but the file is done self.reporter.update_transfer(1, 0) def set_total_bytes(self, total_byte_count): + """ + Set total bytes count + + :param total_byte_count: total byte count + :type total_byte_count: int + """ pass def bytes_completed(self, byte_count): + """ + Set bytes completed count + + :param byte_count: total byte count + :type byte_count: int + """ self.reporter.update_transfer(0, byte_count - self.bytes_so_far) self.bytes_so_far = byte_count def sample_sync_report_run(): + """ + Generate sample report + """ import sys sync_report = SyncReport(sys.stdout, False) From c79216ec82dc650945d1fcd79030364663ab5dd6 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 13:23:56 +0300 Subject: [PATCH 23/75] Docstrings added for objects in b2sdk.sync.scan_policies module --- b2sdk/sync/scan_policies.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/b2sdk/sync/scan_policies.py b/b2sdk/sync/scan_policies.py index 607477696..885716d2a 100644 --- a/b2sdk/sync/scan_policies.py +++ b/b2sdk/sync/scan_policies.py @@ -21,9 +21,19 @@ class RegexSet(object): """ def __init__(self, regex_iterable): + """ + :param regex_iterable: an interable which yields regexes + """ self._compiled_list = [re.compile(r) for r in regex_iterable] def matches(self, s): + """ + Check whether a string matches any of regular expressions + + :param s: a string to check + :type s: str + :rtype: bool + """ return any(c.match(s) is not None for c in self._compiled_list) @@ -47,6 +57,9 @@ def convert_dir_regex_to_dir_prefix_regex(dir_regex): If the original regex is valid, there are only two cases to consider: either the regex ends in '$' or does not. + + :param dir_regex: a regular expression string or literal + :type dir_regex: str """ if dir_regex.endswith('$'): return dir_regex[:-1] + r'/' @@ -75,6 +88,16 @@ def __init__( include_file_regexes=tuple(), exclude_all_symlinks=False, ): + """ + :param exclude_dir_regexes: a tuple of regexes to exclude directories + :type exclude_dir_regexes: tuple + :param exclude_file_regexes: a tuple of regexes to exclude files + :type exclude_file_regexes: tuple + :param include_file_regexes: a tuple of regexes to include files + :type include_file_regexes: tuple + :param exclude_all_symlinks: if True, exclude all symlinks + :type exclude_all_symlinks: bool + """ self._exclude_dir_set = RegexSet(exclude_dir_regexes) self._exclude_file_because_of_dir_set = RegexSet( map(convert_dir_regex_to_dir_prefix_regex, exclude_dir_regexes) @@ -89,7 +112,9 @@ def should_exclude_file(self, file_path): :param file_path: The path of the file, relative to the root directory being scanned. - :return: True iff excluded. + :type: str + :return: True if excluded. + :rtype: bool """ exclude_because_of_dir = self._exclude_file_because_of_dir_set.matches(file_path) exclude_because_of_file = ( @@ -105,7 +130,8 @@ def should_exclude_directory(self, dir_path): :param dir_path: The path of the directory, relative to the root directory being scanned. The path will never end in '/'. - :return: True iff excluded. + :type dir_path: str + :return: True if excluded. """ return self._exclude_dir_set.matches(dir_path) From f35009e587ece7bdb97fcf957014983e662219a2 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 14:12:31 +0300 Subject: [PATCH 24/75] Docstrings updated, strings -> objects --- b2sdk/sync/policy.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index d792be5f2..ea42c8311 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -31,14 +31,10 @@ class AbstractFileSyncPolicy(object): def __init__(self, source_file, source_folder, dest_file, dest_folder, now_millis, args): """ - :param source_file: source file name - :type source_file: str - :param source_folder: source folder name - :type source_folder: str - :param dest_file: destination file name - :type dest_file: str - :param dest_folder: destination folder name - :type dest_folder: str + :param source_file: source file object + :param source_folder: source folder object + :param dest_file: destination file object + :param dest_folder: destination folder object :param now_millis: current time in milliseconds :type now_millis: int :param args: an object which holds command line arguments @@ -73,10 +69,8 @@ def files_are_different(cls, source_file, dest_file, args): Compare two files and determine if the the destination file should be replaced by the source file. - :param source_file: source file name - :type source_file: str - :param dest_file: destination file name - :type dest_file: str + :param source_file: source file object + :param dest_file: destination file object :param args: an object which holds command line arguments :rtype: bool """ From ed6ccc72481947590a395e234e7043cd1de755dc Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 14:25:26 +0300 Subject: [PATCH 25/75] flie and folder objects' types updated in docstrings --- b2sdk/sync/policy.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index ea42c8311..e6a5f40b9 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -32,9 +32,13 @@ class AbstractFileSyncPolicy(object): def __init__(self, source_file, source_folder, dest_file, dest_folder, now_millis, args): """ :param source_file: source file object + :type source_file: b2sdk.sync.file.File :param source_folder: source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder :param now_millis: current time in milliseconds :type now_millis: int :param args: an object which holds command line arguments @@ -70,7 +74,9 @@ def files_are_different(cls, source_file, dest_file, args): should be replaced by the source file. :param source_file: source file object + :type source_file: b2sdk.sync.file.File :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File :param args: an object which holds command line arguments :rtype: bool """ @@ -274,12 +280,12 @@ def make_b2_delete_actions(source_file, dest_file, dest_folder, transferred): """ Creates the actions to delete files stored on B2, which are not present locally. - :param source_file: source file name - :type source_file: str - :param dest_file: destination file name - :type dest_file: str + :param source_file: source file object + :type source_file: b2sdk.sync.file.File + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File :param dest_folder: destination folder - :type dest_folder: str + :type dest_folder: b2sdk.sync.folder.AbstractFolder :param transferred: if True, file has been transferred, False otherwise :type transferred: bool @@ -312,12 +318,12 @@ def make_b2_keep_days_actions( only the 25-day old version can be deleted. The 15 day-old version was visible 10 days ago. - :param source_file: source file name - :type source_file: str - :param dest_file: destination file name - :type dest_file: str - :param dest_folder: destination folder - :type dest_folder: str + :param source_file: source file object + :type source_file: b2sdk.sync.file.File + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File + :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder :param transferred: if True, file has been transferred, False otherwise :type transferred: bool From aabd7540cfc2105b55f178d4a4b26266b4333697 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 14:41:52 +0300 Subject: [PATCH 26/75] Docstrings added for objects in b2sdk.sync.sync module --- b2sdk/sync/sync.py | 56 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index 0068963a4..58ba5a316 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -46,8 +46,14 @@ def zip_folders(folder_a, folder_b, reporter, policies_manager=DEFAULT_SCAN_MANA Each item is a pair (file_a, file_b) with the corresponding file in both folders. Either file (but not both) will be None if the file is in only one folder. - :param folder_a: A Folder object. - :param folder_b: A Folder object. + + :param folder_a: first folder object. + :type folder_a: b2sdk.sync.folder.AbstractFolder + :param folder_b: second folder object. + :type folder_b: b2sdk.sync.folder.AbstractFolder + :param reporter: reporter object + :param policies_manager: policies manager object + :return: yields two element tuples """ iter_a = folder_a.all_files(reporter, policies_manager) @@ -81,6 +87,20 @@ def make_file_sync_actions( ): """ Yields the sequence of actions needed to sync the two files + + :param sync_type: synchronization type + :type sync_type: str + :param source_file: source file object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File + :param source_folder: a source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_folder: a destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param args: an object which holds command line arguments + :param now_millis: current time in milliseconds + :type now_millis: int """ policy = POLICY_MANAGER.get_policy( @@ -96,6 +116,16 @@ def make_folder_sync_actions( """ Yields a sequence of actions that will sync the destination folder to the source folder. + + :param source_folder: source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param args: an object which holds command line arguments + :param now_millis: current time in milliseconds + :type now_millis: int + :param reporter: reporter object + :param policies_manager: policies manager object """ if args.skipNewer and args.replaceNewer: raise CommandError('--skipNewer and --replaceNewer are incompatible') @@ -137,6 +167,10 @@ def make_folder_sync_actions( def count_files(local_folder, reporter): """ Counts all of the files in a local folder. + + :param local_folder: a folder object. + :type local_folder: b2sdk.sync.folder.AbstractFolder + :param reporter: reporter object """ # Don't pass in a reporter to all_files. Broken symlinks will be reported # during the next pass when the source and dest files are compared. @@ -162,6 +196,24 @@ def sync_folders( Syncs two folders. Always ensures that every file in the source is also in the destination. Deletes any file versions in the destination older than history_days. + + :param source_folder: source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param args: an object which holds command line arguments + :param now_millis: current time in milliseconds + :type now_millis: int + :param stdout: standard output file object + :param no_progress: if True, do not show progress + :type no_progress: bool + :param max_workers: max number of workers + :type max_workers: int + :param policies_manager: policies manager object + :param dry_run: + :type dry_run: bool + :param allow_empty_source: if True, do not check whether source folder is empty + :type allow_empty_source: bool """ # For downloads, make sure that the target directory is there. From ad765cea70a050a7c93eae77dfb13b375f250cf4 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 16:35:03 +0300 Subject: [PATCH 27/75] Module content entries removed from ToC as uneeded --- doc/source/b2sdk.account_info.rst | 9 --------- doc/source/b2sdk.rst | 9 --------- doc/source/b2sdk.sync.rst | 9 --------- doc/source/b2sdk.transferer.rst | 9 --------- 4 files changed, 36 deletions(-) diff --git a/doc/source/b2sdk.account_info.rst b/doc/source/b2sdk.account_info.rst index 9a6137ca4..4c5c437aa 100644 --- a/doc/source/b2sdk.account_info.rst +++ b/doc/source/b2sdk.account_info.rst @@ -48,12 +48,3 @@ b2sdk\.account\_info\.upload\_url\_pool module :members: :undoc-members: :show-inheritance: - - -Module contents ---------------- - -.. automodule:: b2sdk.account_info - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk.rst b/doc/source/b2sdk.rst index 9e310beda..919339b34 100644 --- a/doc/source/b2sdk.rst +++ b/doc/source/b2sdk.rst @@ -144,12 +144,3 @@ b2sdk\.version module :members: :undoc-members: :show-inheritance: - - -Module contents ---------------- - -.. automodule:: b2sdk - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk.sync.rst b/doc/source/b2sdk.sync.rst index 434fbc33c..21690c49f 100644 --- a/doc/source/b2sdk.sync.rst +++ b/doc/source/b2sdk.sync.rst @@ -80,12 +80,3 @@ b2sdk\.sync\.sync module :members: :undoc-members: :show-inheritance: - - -Module contents ---------------- - -.. automodule:: b2sdk.sync - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk.transferer.rst b/doc/source/b2sdk.transferer.rst index 5175a5a62..221d855fd 100644 --- a/doc/source/b2sdk.transferer.rst +++ b/doc/source/b2sdk.transferer.rst @@ -48,12 +48,3 @@ b2sdk\.transferer\.transferer module :members: :undoc-members: :show-inheritance: - - -Module contents ---------------- - -.. automodule:: b2sdk.transferer - :members: - :undoc-members: - :show-inheritance: From e965775ccb2acfc89e9e05f7e6f75634a19a9684 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 4 Mar 2019 16:35:26 +0300 Subject: [PATCH 28/75] Rename B2_Command_Line_Tool to B2 Python SDK --- doc/source/conf.py | 14 +++++++------- doc/source/index.rst | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 9eb43fe1f..2c9c8c930 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -10,7 +10,7 @@ # -*- coding: utf-8 -*- # -# B2_Command_Line_Tool documentation build configuration file, created by +# B2 Python SDK documentation build configuration file, created by # sphinx-quickstart on Fri Oct 20 18:27:20 2017. # # This file is execfile()d with the current directory set to its @@ -59,7 +59,7 @@ master_doc = 'index' # General information about the project. -project = u'B2_Command_Line_Tool' +project = u'B2 Python SDK' copyright = u'2019, Backblaze' author = u'Backblaze' @@ -127,7 +127,7 @@ # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'B2_Command_Line_Tooldoc' +htmlhelp_basename = 'B2_Python_SDKdoc' # -- Options for LaTeX output --------------------------------------------- @@ -154,7 +154,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ ( - master_doc, 'B2_Command_Line_Tool.tex', u'B2\\_Command\\_Line\\_Tool Documentation', + master_doc, 'B2_Python_SDK.tex', u'B2\\_Python\\_SDK', u'Backblaze', 'manual' ), ] @@ -164,7 +164,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'b2_command_line_tool', u'B2_Command_Line_Tool Documentation', [author], 1) + (master_doc, 'b2_python_sdk', u'B2 Python SDK Documentation', [author], 1) ] # -- Options for Texinfo output ------------------------------------------- @@ -174,8 +174,8 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, 'B2_Command_Line_Tool', u'B2_Command_Line_Tool Documentation', author, - 'B2_Command_Line_Tool', 'One line description of project.', 'Miscellaneous' + master_doc, 'B2_Python_SDK', u'B2 Python SDK Documentation', author, + 'B2_Python_SDK', 'Backblaze Python SDK', 'Miscellaneous' ), ] diff --git a/doc/source/index.rst b/doc/source/index.rst index 32d21e00b..2abc4451e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,10 +1,10 @@ -.. B2_Command_Line_Tool documentation master file, created by +.. B2 Python SDK documentation master file, created by sphinx-quickstart on Fri Oct 20 18:27:20 2017. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to B2_Command_Line_Tool's documentation! -================================================ +Welcome to B2 Python SDK's documentation! +========================================= .. toctree:: :maxdepth: 2 From b82ff7e9d74f666b9fa289ecf27b2e6f8dfa7e1d Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 10:22:41 +0300 Subject: [PATCH 29/75] Sphinx theme changed to sphinx_rtd_theme --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 2c9c8c930..74f9fafe3 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -95,7 +95,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From d2e7a6e9f20bc4b8fafc8b6cba0ff6a9e134447c Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 10:37:13 +0300 Subject: [PATCH 30/75] Docstrings added for objects in module b2sdk.account_info.abstract --- b2sdk/account_info/abstract.py | 122 +++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 7 deletions(-) diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 41a81ab35..81cff5a9e 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -45,6 +45,11 @@ class AbstractAccountInfo(object): @classmethod def all_capabilities(cls): + """ + Return a list of all possible capabilities + + :rtype: list + """ return cls.ALL_CAPABILITIES @abstractmethod @@ -58,39 +63,64 @@ def clear(self): def refresh_entire_bucket_name_cache(self, name_id_iterable): """ Removes all previous name-to-id mappings and stores new ones. + + :param name_id_iterable: a list of tuples of the form (name, id) + :type name_id_iterable: list """ @abstractmethod def remove_bucket_name(self, bucket_name): """ Removes one entry from the bucket name cache. + + :param bucket_name: a bucket name + :type bucket_name: str """ @abstractmethod def save_bucket(self, bucket): """ Remembers the ID for a bucket name. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket """ @abstractmethod def get_bucket_id_or_none_from_bucket_name(self, bucket_name): """ Looks up the bucket ID for a given bucket name. + + :param bucket_name: a bucket name + :type bucket_name: str + :return bucket ID or None: + :rtype: str, None """ @abstractmethod def clear_bucket_upload_data(self, bucket_id): """ Removes all upload URLs for the given bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str """ @abstractmethod def get_account_id(self): - """ returns account_id or raises MissingAccountData exception """ + """ + Returns account_id or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_account_id_or_app_key_id(self): - """ returns the account id or key id used to authenticate """ + """ + Returns the account id or key id used to authenticate + + :rtype: str + """ @abstractmethod def get_account_auth_token(self): @@ -98,24 +128,43 @@ def get_account_auth_token(self): @abstractmethod def get_api_url(self): - """ returns api_url or raises MissingAccountData exception """ + """ + Returns api_url or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_application_key(self): - """ returns application_key or raises MissingAccountData exception """ + """ + Returns application_key or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_download_url(self): - """ returns download_url or raises MissingAccountData exception """ + """ + Returns download_url or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_realm(self): - """ returns realm or raises MissingAccountData exception """ + """ + Returns realm or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_minimum_part_size(self): """ - :return: returns the minimum number of bytes in a part of a large file + Return the minimum number of bytes in a part of a large file + + :return: number of bytes + :rtype: int """ @abstractmethod @@ -124,6 +173,8 @@ def get_allowed(self): An 'allowed' dict, as returned by b2_authorize_account. Never None; for account info that was saved before 'allowed' existed, returns DEFAULT_ALLOWED. + + :rtype: dict """ @limit_trace_arguments(only=['self', 'api_url', 'download_url', 'minimum_part_size', 'realm']) @@ -149,6 +200,25 @@ def set_auth_data( a bucketName field. For keys with bucket restrictions, the name of the bucket is looked up and stored, too. The console_tool does everything by bucket name, so it's convenient to have the restricted bucket name handy. + + :param account_id: user account ID + :type account_id: str + :param auth_token: user authentication token + :type auth_token: str + :param api_url: an API URL + :type api_url: str + :param download_url: path download URL + :type download_url: str + :param minimum_part_size: minimum size of the file part + :type minimum_part_size: int + :param application_key: application key + :type application_key: str + :param realm: a realm to authiroze account in + :type realm: str + :param allowed: the structure to use for old account info that was saved without 'allowed' + :type allowed: dict + :param account_id_or_app_key_id: account ID or application key ID + :type account_id_or_app_key_id: str """ if allowed is None: allowed = self.DEFAULT_ALLOWED @@ -173,6 +243,10 @@ def allowed_is_valid(cls, allowed): If the bucketId is for a bucket that no longer exists, or the capabilities do not allow listBuckets, then we won't have a bucketName. + + :param allowed: the structure to use for old account info that was saved without 'allowed' + :type allowed: dict + :rtype: bool """ return ( ('bucketId' in allowed) and ('bucketName' in allowed) and @@ -203,6 +277,10 @@ def take_bucket_upload_url(self, bucket_id): Returns a pair (upload_url, upload_auth_token) that has been removed from the pool for this bucket, or (None, None) if there are no more left. + + :param bucket_id: a bucket ID + :type bucket_id: str + :rtype: tuple """ @abstractmethod @@ -211,17 +289,47 @@ def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): """ Add an (upload_url, upload_auth_token) pair to the pool available for the bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + :rtype: tuple """ @abstractmethod @limit_trace_arguments(only=['self']) def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): + """ + Put large file upload URL into a pool + + :param file_id: a file ID + :type file_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + """ pass @abstractmethod def take_large_file_upload_url(self, file_id): + """ + Take large file upload URL from a pool + + :param file_id: a file ID + :type file_id: str + """ pass @abstractmethod def clear_large_file_upload_urls(self, file_id): + """ + Clear a pool of URLs for a given file ID + + :param file_id: a file ID + :type file_id: str + """ pass From b434ce4c01e0d9f203647d07054aaf4fbfdf3255 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 13:20:53 +0300 Subject: [PATCH 31/75] Docstrings added for objects in b2sdk.account_info.exception module --- b2sdk/account_info/exception.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/b2sdk/account_info/exception.py b/b2sdk/account_info/exception.py index 6a4b869aa..dd9ff536a 100644 --- a/b2sdk/account_info/exception.py +++ b/b2sdk/account_info/exception.py @@ -17,11 +17,21 @@ @six.add_metaclass(ABCMeta) class AccountInfoError(B2Error): + """ + Base class for all account info errors + """ pass class CorruptAccountInfo(AccountInfoError): + """ + Raised when an account info file is corrupted + """ def __init__(self, file_name): + """ + :param file_name: an account info file name + :type file_name: str + """ super(CorruptAccountInfo, self).__init__() self.file_name = file_name @@ -32,7 +42,14 @@ def __str__(self): class MissingAccountData(AccountInfoError): + """ + Raised when there is no account info data available + """ def __init__(self, key): + """ + :param key: a key for getting account data + :type key: str + """ super(MissingAccountData, self).__init__() self.key = key From dd1e124ac54b79ac4f47afbf8d3fa31828430b3e Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 13:24:02 +0300 Subject: [PATCH 32/75] Docstrings added for objects in b2sdk.account_info.sqlite_account_info --- b2sdk/account_info/sqlite_account_info.py | 88 +++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/b2sdk/account_info/sqlite_account_info.py b/b2sdk/account_info/sqlite_account_info.py index f344cb585..3064e0812 100644 --- a/b2sdk/account_info/sqlite_account_info.py +++ b/b2sdk/account_info/sqlite_account_info.py @@ -40,7 +40,9 @@ class SqliteAccountInfo(UrlPoolAccountInfo): def __init__(self, file_name=None, last_upgrade_to_run=None): """ :param file_name: The sqlite file to use; overrides the default. + :type file_name: str :param last_upgrade_to_run: For testing only, override the auto-update on the db. + :type last_upgrade_to_run: int """ self.thread_local = threading.local() user_account_info_path = file_name or os.environ.get( @@ -207,6 +209,9 @@ def _ensure_update(self, update_number, update_command): ) def clear(self): + """ + Remove all info about accounts and buckets + """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') @@ -259,6 +264,24 @@ def set_auth_data_with_schema_0_for_test( application_key, realm, ): + """ + Set authentication data for tests + + :param account_id: an account ID + :type account_id: str + :param auth_token: an authentication token + :type auth_token: str + :param api_url: an API URL + :type api_url: str + :param download_url: a download URL + :type download_url: str + :param minimum_part_size: a minimum part size + :type minimum_part_size: int + :param application_key: an application key + :type application_key: str + :param realm: a realm to authorize account in + :type realm: str + """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') @@ -282,14 +305,27 @@ def set_auth_data_with_schema_0_for_test( ) def get_application_key(self): + """ + Return application key + + :rtype: str + """ return self._get_account_info_or_raise('application_key') def get_account_id(self): + """ + Return account ID + + :rtype: str + """ return self._get_account_info_or_raise('account_id') def get_account_id_or_app_key_id(self): """ + Return account ID or appliction key ID. The 'account_id_or_app_key_id' column was not in the original schema, so it may be NULL. + + :rtype: str """ result = self._get_account_info_or_raise('account_id_or_app_key_id') if result is None: @@ -298,23 +334,51 @@ def get_account_id_or_app_key_id(self): return result def get_api_url(self): + """ + Return API URL + + :rtype: str + """ return self._get_account_info_or_raise('api_url') def get_account_auth_token(self): + """ + Return account authentication token + + :rtype: str + """ return self._get_account_info_or_raise('account_auth_token') def get_download_url(self): + """ + Return download URL + + :rtype: str + """ return self._get_account_info_or_raise('download_url') def get_realm(self): + """ + Return realm name + + :rtype: str + """ return self._get_account_info_or_raise('realm') def get_minimum_part_size(self): + """ + Return minimum part size + + :rtype: int + """ return self._get_account_info_or_raise('minimum_part_size') def get_allowed(self): """ + Return 'allowed' dictionary info. The 'allowed" column was not in the original schema, so it may be NULL. + + :rtype: dict """ allowed_json = self._get_account_info_or_raise('allowed') if allowed_json is None: @@ -336,6 +400,11 @@ def _get_account_info_or_raise(self, column_name): raise MissingAccountData(str(e)) def refresh_entire_bucket_name_cache(self, name_id_iterable): + """ + Refresh names and IDs of buckets + + :param name_id_iterable: an iterable which yields buakcet name and ID + """ with self._get_connection() as conn: conn.execute('DELETE FROM bucket;') for (bucket_name, bucket_id) in name_id_iterable: @@ -345,6 +414,12 @@ def refresh_entire_bucket_name_cache(self, name_id_iterable): ) def save_bucket(self, bucket): + """ + Save bucket info + + :param bucket: a Bucket object + :type: b2sdk.bucket.Bucket + """ with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_id = ?;', (bucket.id_,)) conn.execute( @@ -353,10 +428,23 @@ def save_bucket(self, bucket): ) def remove_bucket_name(self, bucket_name): + """ + Remove bucket info by a given name + + :param bucket_name: a bucket name + :type bucket_name: str + """ with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_name = ?;', (bucket_name,)) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): + """ + Return bucket ID or None by a given name + + :param bucket_name: a bucket name + :type bucket_name: str + :rtype: str, None + """ try: with self._get_connection() as conn: cursor = conn.execute( From 2d584395ce9acb8dfee6ed3ddadb2e883bf64e3e Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 13:26:19 +0300 Subject: [PATCH 33/75] Docstrings added for objects in b2sdk.account_info.upload_url_pool --- b2sdk/account_info/upload_url_pool.py | 78 +++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/b2sdk/account_info/upload_url_pool.py b/b2sdk/account_info/upload_url_pool.py index 62adf4424..d701f07cb 100644 --- a/b2sdk/account_info/upload_url_pool.py +++ b/b2sdk/account_info/upload_url_pool.py @@ -29,6 +29,13 @@ def __init__(self): def put(self, key, url, auth_token): """ Adds the url and auth token to the pool for the given key. + + :param key: bucket ID or large file ID + :type param: str + :param url: bucket or file URL + :type url: str + :param auth_token: authentication token + :type auth_token: str """ with self._lock: pair = (url, auth_token) @@ -37,6 +44,10 @@ def put(self, key, url, auth_token): def take(self, key): """ Returns (url, auth_token) if one is available, or (None, None) if not. + + :param key: bucket ID or large file ID + :type param: str + :rtype: tuple """ with self._lock: pair_list = self._pool[key] @@ -46,18 +57,36 @@ def take(self, key): return (None, None) def clear_for_key(self, key): + """ + Remove an intem from the pool by key + + :param key: bucket ID or large file ID + :type param: str + """ with self._lock: if key in self._pool: del self._pool[key] class UrlPoolAccountInfo(AbstractAccountInfo): + """ + Holder for all account-related information that needs to be kept + between API calls, and between invocations of the command-line + tool. This includes: account id, application key, auth tokens, + API URL, download URL, and uploads URLs. + + This concrete implementation uses an instance of UploadUrlPool + as an underlying storage + """ def __init__(self): super(UrlPoolAccountInfo, self).__init__() self._reset_upload_pools() @abstractmethod def clear(self): + """ + Remove all stored information + """ self._reset_upload_pools() return super(UrlPoolAccountInfo, self).clear() @@ -67,20 +96,69 @@ def _reset_upload_pools(self): # bucket upload url def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): + """ + Add an (upload_url, upload_auth_token) pair to the pool available for + the bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + :rtype: tuple + """ self._bucket_uploads.put(bucket_id, upload_url, upload_auth_token) def clear_bucket_upload_data(self, bucket_id): + """ + Removes all upload URLs for the given bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str + """ self._bucket_uploads.clear_for_key(bucket_id) def take_bucket_upload_url(self, bucket_id): + """ + Returns a pair (upload_url, upload_auth_token) that has been removed + from the pool for this bucket, or (None, None) if there are no more + left. + + :param bucket_id: a bucket ID + :type bucket_id: str + :rtype: tuple + """ return self._bucket_uploads.take(bucket_id) # large file upload url def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): + """ + Put large file upload URL into a pool + + :param file_id: a file ID + :type file_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + """ self._large_file_uploads.put(file_id, upload_url, upload_auth_token) def take_large_file_upload_url(self, file_id): + """ + Take large file upload URL from a pool + + :param file_id: a file ID + :type file_id: str + """ return self._large_file_uploads.take(file_id) def clear_large_file_upload_urls(self, file_id): + """ + Clear a pool of URLs for a given file ID + + :param file_id: a file ID + :type file_id: str + """ self._large_file_uploads.clear_for_key(file_id) From a175b77654a2b01b05903c01962968426342d30f Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 15:16:13 +0300 Subject: [PATCH 34/75] docstring fixed --- b2sdk/account_info/abstract.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 81cff5a9e..b7ca7eb1d 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -124,7 +124,11 @@ def get_account_id_or_app_key_id(self): @abstractmethod def get_account_auth_token(self): - """ returns account_auth_token or raises MissingAccountData exception """ + """ + Returns account_auth_token or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_api_url(self): From 4b9d2a1a00b10783cb230fc6d4442b134d542ef5 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 15:16:34 +0300 Subject: [PATCH 35/75] Docstrings added for objects in b2sdk.account_info.in_memory --- b2sdk/account_info/in_memory.py | 86 +++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/b2sdk/account_info/in_memory.py b/b2sdk/account_info/in_memory.py index 4c30cbc01..a620f3b82 100644 --- a/b2sdk/account_info/in_memory.py +++ b/b2sdk/account_info/in_memory.py @@ -15,6 +15,9 @@ def _raise_missing_if_result_is_none(function): + """ + Raise MissingAccountData if function's result is None + """ @wraps(function) def inner(*args, **kwargs): assert function.__name__.startswith('get_') @@ -28,11 +31,20 @@ def inner(*args, **kwargs): class InMemoryAccountInfo(UrlPoolAccountInfo): + """ + Holder for all account-related information that needs to be kept + between API calls, and between invocations of the command-line + tool. This includes: account id, application key, auth tokens, + API URL, download URL, and uploads URLs. + """ def __init__(self, *args, **kwargs): super(InMemoryAccountInfo, self).__init__(*args, **kwargs) self._clear_in_memory_account_fields() def clear(self): + """ + Remove all stored information + """ self._clear_in_memory_account_fields() return super(InMemoryAccountInfo, self).clear() @@ -71,50 +83,124 @@ def _set_auth_data( self._allowed = allowed def refresh_entire_bucket_name_cache(self, name_id_iterable): + """ + Removes all previous name-to-id mappings and stores new ones. + + :param name_id_iterable: a list of tuples of the form (name, id) + :type name_id_iterable: list + """ self._buckets = dict(name_id_iterable) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): + """ + Looks up the bucket ID for a given bucket name. + + :param bucket_name: a bucket name + :type bucket_name: str + :return bucket ID or None: + :rtype: str, None + """ return self._buckets.get(bucket_name) def save_bucket(self, bucket): + """ + Remembers the ID for a bucket name. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + """ self._buckets[bucket.name] = bucket.id_ def remove_bucket_name(self, bucket_name): + """ + Removes one entry from the bucket name cache. + + :param bucket_name: a bucket name + :type bucket_name: str + """ if bucket_name in self._buckets: del self._buckets[bucket_name] @_raise_missing_if_result_is_none def get_account_id(self): + """ + Returns account_id or raises MissingAccountData exception + + :rtype: str + """ return self._account_id @_raise_missing_if_result_is_none def get_account_id_or_app_key_id(self): + """ + Returns the account id or key id used to authenticate + + :rtype: str + """ return self._account_id_or_app_key_id @_raise_missing_if_result_is_none def get_account_auth_token(self): + """ + Return account_auth_token or raises MissingAccountData exception + + :rtype: str + """ return self._auth_token @_raise_missing_if_result_is_none def get_api_url(self): + """ + Returns api_url or raises MissingAccountData exception + + :rtype: str + """ return self._api_url @_raise_missing_if_result_is_none def get_application_key(self): + """ + Returns application_key or raises MissingAccountData exception + + :rtype: str + """ return self._application_key @_raise_missing_if_result_is_none def get_download_url(self): + """ + Returns download_url or raises MissingAccountData exception + + :rtype: str + """ return self._download_url @_raise_missing_if_result_is_none def get_minimum_part_size(self): + """ + Return the minimum number of bytes in a part of a large file + + :return: number of bytes + :rtype: int + """ return self._minimum_part_size @_raise_missing_if_result_is_none def get_realm(self): + """ + Returns realm or raises MissingAccountData exception + + :rtype: str + """ return self._realm @_raise_missing_if_result_is_none def get_allowed(self): + """ + An 'allowed' dict, as returned by b2_authorize_account. + Never None; for account info that was saved before 'allowed' existed, + returns DEFAULT_ALLOWED. + + :rtype: dict + """ return self._allowed From 41e5a0fb8a465257c8d791bfd7209a27fab9c9cd Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 16:30:02 +0300 Subject: [PATCH 36/75] indentation fixed in docstrings --- b2sdk/api.py | 14 ++++++++------ b2sdk/sync/file.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index 5869906cf..e29cb0140 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -61,15 +61,17 @@ class handles several things that simplify the task of uploading def __init__(self, account_info=None, cache=None, raw_api=None, max_upload_workers=10): """ Initializes the API using the given account info. + :param account_info: an instance of :class:`~b2sdk.account_info.upload_url_pool.UrlPoolAccountInfo`, - or any custom class derived from :class:`~b2sdk.account_info.abstract.AbstractAccountInfo` + or any custom class derived from + :class:`~b2sdk.account_info.abstract.AbstractAccountInfo` :param cache: an instance of the one of the following classes: - :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, - :class:`~b2sdk.cache.AuthInfoCache`, - or any custom class derived from :class:`~b2sdk.cache.AbstractCache` + :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, + :class:`~b2sdk.cache.AuthInfoCache`, + or any custom class derived from :class:`~b2sdk.cache.AbstractCache` :param raw_api: an instance of one of the following classes: - :class:`~b2sdk.raw_api.B2RawApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, - or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` + :class:`~b2sdk.raw_api.B2RawApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, + or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` :param max_upload_workers: a number of upload threads, default is 10 :type max_upload_workers: int """ diff --git a/b2sdk/sync/file.py b/b2sdk/sync/file.py index 3b96cf20c..823873df6 100644 --- a/b2sdk/sync/file.py +++ b/b2sdk/sync/file.py @@ -52,7 +52,7 @@ def __init__(self, id_, file_name, mod_time, action, size): :param file_name: a relative file name :type file_name: str :param mod_time: modification time, in milliseconds, to avoid rounding issues - with millisecond times from B2 + with millisecond times from B2 :type mod_time: int :param action: "hide" or "upload" (never "start") :type action: str From 628464d5da56a4262e2c496f7384b34fb0a8e3d4 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 16:30:59 +0300 Subject: [PATCH 37/75] Docs structure updated --- doc/source/b2sdk.account_info.rst | 50 ------ doc/source/b2sdk.rst | 146 ------------------ doc/source/b2sdk.sync.rst | 82 ---------- doc/source/b2sdk.transferer.rst | 50 ------ doc/source/b2sdk/account_info/abstract.rst | 8 + doc/source/b2sdk/account_info/exception.rst | 8 + .../account_info/sqlite_account_info.rst | 8 + .../b2sdk/account_info/upload_url_pool.rst | 8 + doc/source/b2sdk/api.rst | 8 + doc/source/b2sdk/bucket.rst | 8 + doc/source/b2sdk/cache.rst | 8 + doc/source/b2sdk/download_dest.rst | 8 + doc/source/b2sdk/exception.rst | 8 + doc/source/b2sdk/file_version.rst | 8 + doc/source/b2sdk/progress.rst | 8 + doc/source/b2sdk/sync/action.rst | 8 + doc/source/b2sdk/sync/exception.rst | 8 + doc/source/b2sdk/sync/file.rst | 8 + doc/source/b2sdk/sync/folder.rst | 8 + doc/source/b2sdk/sync/folder_parser.rst | 8 + doc/source/b2sdk/sync/policy.rst | 8 + doc/source/b2sdk/sync/policy_manager.rst | 8 + doc/source/b2sdk/sync/scan_policies.rst | 8 + doc/source/b2sdk/sync/sync.rst | 8 + doc/source/b2sdk/upload_source.rst | 8 + doc/source/b2sdk/utils.rst | 8 + doc/source/index.rst | 26 +++- 27 files changed, 198 insertions(+), 332 deletions(-) delete mode 100644 doc/source/b2sdk.account_info.rst delete mode 100644 doc/source/b2sdk.rst delete mode 100644 doc/source/b2sdk.sync.rst delete mode 100644 doc/source/b2sdk.transferer.rst create mode 100644 doc/source/b2sdk/account_info/abstract.rst create mode 100644 doc/source/b2sdk/account_info/exception.rst create mode 100644 doc/source/b2sdk/account_info/sqlite_account_info.rst create mode 100644 doc/source/b2sdk/account_info/upload_url_pool.rst create mode 100644 doc/source/b2sdk/api.rst create mode 100644 doc/source/b2sdk/bucket.rst create mode 100644 doc/source/b2sdk/cache.rst create mode 100644 doc/source/b2sdk/download_dest.rst create mode 100644 doc/source/b2sdk/exception.rst create mode 100644 doc/source/b2sdk/file_version.rst create mode 100644 doc/source/b2sdk/progress.rst create mode 100644 doc/source/b2sdk/sync/action.rst create mode 100644 doc/source/b2sdk/sync/exception.rst create mode 100644 doc/source/b2sdk/sync/file.rst create mode 100644 doc/source/b2sdk/sync/folder.rst create mode 100644 doc/source/b2sdk/sync/folder_parser.rst create mode 100644 doc/source/b2sdk/sync/policy.rst create mode 100644 doc/source/b2sdk/sync/policy_manager.rst create mode 100644 doc/source/b2sdk/sync/scan_policies.rst create mode 100644 doc/source/b2sdk/sync/sync.rst create mode 100644 doc/source/b2sdk/upload_source.rst create mode 100644 doc/source/b2sdk/utils.rst diff --git a/doc/source/b2sdk.account_info.rst b/doc/source/b2sdk.account_info.rst deleted file mode 100644 index 4c5c437aa..000000000 --- a/doc/source/b2sdk.account_info.rst +++ /dev/null @@ -1,50 +0,0 @@ -b2sdk\.account\_info package -============================ - -b2sdk\.account\_info\.abstract module -------------------------------------- - -.. automodule:: b2sdk.account_info.abstract - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.exception module --------------------------------------- - -.. automodule:: b2sdk.account_info.exception - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.in\_memory module ---------------------------------------- - -.. automodule:: b2sdk.account_info.in_memory - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.sqlite\_account\_info module --------------------------------------------------- - -.. automodule:: b2sdk.account_info.sqlite_account_info - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.test\_upload\_url\_concurrency module ------------------------------------------------------------ - -.. automodule:: b2sdk.account_info.test_upload_url_concurrency - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.upload\_url\_pool module ----------------------------------------------- - -.. automodule:: b2sdk.account_info.upload_url_pool - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk.rst b/doc/source/b2sdk.rst deleted file mode 100644 index 919339b34..000000000 --- a/doc/source/b2sdk.rst +++ /dev/null @@ -1,146 +0,0 @@ -b2sdk package -============== - -b2sdk\.api module ------------------ - -.. automodule:: b2sdk.api - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.b2http module --------------------- - -.. automodule:: b2sdk.b2http - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.bounded\_queue\_executor module --------------------------------------- - -.. automodule:: b2sdk.bounded_queue_executor - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.bucket module --------------------- - -.. automodule:: b2sdk.bucket - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.cache module -------------------- - -.. automodule:: b2sdk.cache - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.download\_dest module ----------------------------- - -.. automodule:: b2sdk.download_dest - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.exception module ------------------------ - -.. automodule:: b2sdk.exception - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.file\_version module ---------------------------- - -.. automodule:: b2sdk.file_version - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.parse\_args module -------------------------- - -.. automodule:: b2sdk.parse_args - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.part module ------------------- - -.. automodule:: b2sdk.part - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.progress module ----------------------- - -.. automodule:: b2sdk.progress - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.raw\_api module ----------------------- - -.. automodule:: b2sdk.raw_api - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.raw\_simulator module ----------------------------- - -.. automodule:: b2sdk.raw_simulator - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.session module ---------------------- - -.. automodule:: b2sdk.session - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.unfinished\_large\_file module -------------------------------------- - -.. automodule:: b2sdk.unfinished_large_file - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.upload\_source module ----------------------------- - -.. automodule:: b2sdk.upload_source - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.utils module -------------------- - -.. automodule:: b2sdk.utils - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.version module ---------------------- - -.. automodule:: b2sdk.version - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk.sync.rst b/doc/source/b2sdk.sync.rst deleted file mode 100644 index 21690c49f..000000000 --- a/doc/source/b2sdk.sync.rst +++ /dev/null @@ -1,82 +0,0 @@ -b2sdk\.sync package -=================== - -b2sdk\.sync\.action module --------------------------- - -.. automodule:: b2sdk.sync.action - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.exception module ------------------------------ - -.. automodule:: b2sdk.sync.exception - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.file module ------------------------- - -.. automodule:: b2sdk.sync.file - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.folder module --------------------------- - -.. automodule:: b2sdk.sync.folder - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.folder\_parser module ----------------------------------- - -.. automodule:: b2sdk.sync.folder_parser - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.policy module --------------------------- - -.. automodule:: b2sdk.sync.policy - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.policy\_manager module ------------------------------------ - -.. automodule:: b2sdk.sync.policy_manager - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.report module --------------------------- - -.. automodule:: b2sdk.sync.report - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.scan\_policies ---------------------------- - -.. automodule:: b2sdk.sync.scan_policies - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.sync module ------------------------- - -.. automodule:: b2sdk.sync.sync - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk.transferer.rst b/doc/source/b2sdk.transferer.rst deleted file mode 100644 index 221d855fd..000000000 --- a/doc/source/b2sdk.transferer.rst +++ /dev/null @@ -1,50 +0,0 @@ -b2sdk\.transferer package -========================= - -b2sdk\.transferer\.abstract module ----------------------------------- - -.. automodule:: b2sdk.transferer.abstract - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.transferer\.file\_metadata module ----------------------------------------- - -.. automodule:: b2sdk.transferer.file_metadata - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.transferer\.parallel module ----------------------------------- - -.. automodule:: b2sdk.transferer.parallel - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.transferer\.range module -------------------------------- - -.. automodule:: b2sdk.transferer.range - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.transferer\.simple module --------------------------------- - -.. automodule:: b2sdk.transferer.simple - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.transferer\.transferer module ------------------------------------- - -.. automodule:: b2sdk.transferer.transferer - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk/account_info/abstract.rst b/doc/source/b2sdk/account_info/abstract.rst new file mode 100644 index 000000000..6902d51bd --- /dev/null +++ b/doc/source/b2sdk/account_info/abstract.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.account_info.abstract` -- Account info abstract +=========================================================== + +.. automodule:: b2sdk.account_info.abstract + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/account_info/exception.rst b/doc/source/b2sdk/account_info/exception.rst new file mode 100644 index 000000000..e9051aafc --- /dev/null +++ b/doc/source/b2sdk/account_info/exception.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.account_info.exception` -- Account info exceptions +============================================================== + +.. automodule:: b2sdk.account_info.exception + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/account_info/sqlite_account_info.rst b/doc/source/b2sdk/account_info/sqlite_account_info.rst new file mode 100644 index 000000000..c8c1d1804 --- /dev/null +++ b/doc/source/b2sdk/account_info/sqlite_account_info.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.account_info.sqlite_account_info` -- SQLite account info +==================================================================== + +.. automodule:: b2sdk.account_info.sqlite_account_info + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/account_info/upload_url_pool.rst b/doc/source/b2sdk/account_info/upload_url_pool.rst new file mode 100644 index 000000000..126814113 --- /dev/null +++ b/doc/source/b2sdk/account_info/upload_url_pool.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.account_info.upload_url_pool` -- Upload URL pool +============================================================ + +.. automodule:: b2sdk.account_info.upload_url_pool + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/api.rst b/doc/source/b2sdk/api.rst new file mode 100644 index 000000000..c40d65434 --- /dev/null +++ b/doc/source/b2sdk/api.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.api` -- API client +============================== + +.. automodule:: b2sdk.api + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/bucket.rst b/doc/source/b2sdk/bucket.rst new file mode 100644 index 000000000..f702a26e9 --- /dev/null +++ b/doc/source/b2sdk/bucket.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.bucket` -- Bucket +============================= + +.. automodule:: b2sdk.bucket + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/cache.rst b/doc/source/b2sdk/cache.rst new file mode 100644 index 000000000..3efa9cebc --- /dev/null +++ b/doc/source/b2sdk/cache.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.cache` -- Cache +=========================== + +.. automodule:: b2sdk.cache + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/download_dest.rst b/doc/source/b2sdk/download_dest.rst new file mode 100644 index 000000000..7c9f663a0 --- /dev/null +++ b/doc/source/b2sdk/download_dest.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.download_dest` -- Download destination +================================================== + +.. automodule:: b2sdk.download_dest + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/exception.rst b/doc/source/b2sdk/exception.rst new file mode 100644 index 000000000..a852e718e --- /dev/null +++ b/doc/source/b2sdk/exception.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.exception` -- Exceptions +==================================== + +.. automodule:: b2sdk.exception + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/file_version.rst b/doc/source/b2sdk/file_version.rst new file mode 100644 index 000000000..204f74813 --- /dev/null +++ b/doc/source/b2sdk/file_version.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.file_version` -- File version +========================================= + +.. automodule:: b2sdk.file_version + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/progress.rst b/doc/source/b2sdk/progress.rst new file mode 100644 index 000000000..a9a0b3d85 --- /dev/null +++ b/doc/source/b2sdk/progress.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.progress` -- Progress +================================= + +.. automodule:: b2sdk.progress + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/action.rst b/doc/source/b2sdk/sync/action.rst new file mode 100644 index 000000000..9028bedc6 --- /dev/null +++ b/doc/source/b2sdk/sync/action.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.action` -- Sync action +======================================= + +.. automodule:: b2sdk.sync.action + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/exception.rst b/doc/source/b2sdk/sync/exception.rst new file mode 100644 index 000000000..297ea2fdd --- /dev/null +++ b/doc/source/b2sdk/sync/exception.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.exception` -- Sync exceptions +============================================== + +.. automodule:: b2sdk.sync.exception + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/file.rst b/doc/source/b2sdk/sync/file.rst new file mode 100644 index 000000000..a454a0efd --- /dev/null +++ b/doc/source/b2sdk/sync/file.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.file` -- File +============================== + +.. automodule:: b2sdk.sync.file + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/folder.rst b/doc/source/b2sdk/sync/folder.rst new file mode 100644 index 000000000..53d435e15 --- /dev/null +++ b/doc/source/b2sdk/sync/folder.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.folder` -- Folder +================================== + +.. automodule:: b2sdk.sync.folder + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/folder_parser.rst b/doc/source/b2sdk/sync/folder_parser.rst new file mode 100644 index 000000000..846b3840b --- /dev/null +++ b/doc/source/b2sdk/sync/folder_parser.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.folder_parser` -- Folder parser +================================================ + +.. automodule:: b2sdk.sync.folder_parser + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/policy.rst b/doc/source/b2sdk/sync/policy.rst new file mode 100644 index 000000000..4a5ca0fb4 --- /dev/null +++ b/doc/source/b2sdk/sync/policy.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.policy` -- Policy +================================== + +.. automodule:: b2sdk.sync.policy + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/policy_manager.rst b/doc/source/b2sdk/sync/policy_manager.rst new file mode 100644 index 000000000..17409d569 --- /dev/null +++ b/doc/source/b2sdk/sync/policy_manager.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.policy_manager` -- Policy manager +================================================== + +.. automodule:: b2sdk.sync.policy_manager + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/scan_policies.rst b/doc/source/b2sdk/sync/scan_policies.rst new file mode 100644 index 000000000..4f5ce31e0 --- /dev/null +++ b/doc/source/b2sdk/sync/scan_policies.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.scan_policies` -- Scan policies +================================================ + +.. automodule:: b2sdk.sync.scan_policies + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/sync.rst b/doc/source/b2sdk/sync/sync.rst new file mode 100644 index 000000000..8948b3ac1 --- /dev/null +++ b/doc/source/b2sdk/sync/sync.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.sync` -- Sync +============================== + +.. automodule:: b2sdk.sync.sync + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/upload_source.rst b/doc/source/b2sdk/upload_source.rst new file mode 100644 index 000000000..bf050d823 --- /dev/null +++ b/doc/source/b2sdk/upload_source.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.upload_source` -- Upload source +=========================================== + +.. automodule:: b2sdk.upload_source + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/utils.rst b/doc/source/b2sdk/utils.rst new file mode 100644 index 000000000..7bc7e4304 --- /dev/null +++ b/doc/source/b2sdk/utils.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.utils` -- Utils +=========================== + +.. automodule:: b2sdk.utils + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/index.rst b/doc/source/index.rst index 2abc4451e..2dc7f5eab 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -10,10 +10,28 @@ Welcome to B2 Python SDK's documentation! :maxdepth: 2 :caption: Contents: - b2sdk - b2sdk.account_info - b2sdk.sync - b2sdk.transferer + b2sdk/api + b2sdk/bucket + b2sdk/cache + b2sdk/download_dest + b2sdk/exception + b2sdk/file_version + b2sdk/progress + b2sdk/upload_source + b2sdk/utils + b2sdk/account_info/abstract + b2sdk/account_info/exception + b2sdk/account_info/sqlite_account_info + b2sdk/account_info/upload_url_pool + b2sdk/sync/action + b2sdk/sync/exception + b2sdk/sync/file + b2sdk/sync/folder_parser + b2sdk/sync/folder + b2sdk/sync/policy_manager + b2sdk/sync/policy + b2sdk/sync/scan_policies + b2sdk/sync/sync Indices and tables ================== From cbddaaec5dbba8256f61d633963aa48d51963f27 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 17:34:02 +0300 Subject: [PATCH 38/75] Docstrings added for objects in b2sdk.utils module --- b2sdk/utils.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 7 deletions(-) diff --git a/b2sdk/utils.py b/b2sdk/utils.py index b0910b0db..9efb84dc7 100644 --- a/b2sdk/utils.py +++ b/b2sdk/utils.py @@ -45,6 +45,9 @@ def interruptible_get_result(future): by a KeyboardInterrupt. This is not necessary in Python 3, but is needed for Python 2. + + :param future: a future to get result of + :type future: Future """ while True: try: @@ -54,15 +57,25 @@ def interruptible_get_result(future): def b2_url_encode(s): - """URL-encodes a unicode string to be sent to B2 in an HTTP header. + """ + URL-encodes a unicode string to be sent to B2 in an HTTP header. + + :param s: a unicode string to encode + :type s: str + :return: URL-encoded string + :rtype: str """ return six.moves.urllib.parse.quote(s.encode('utf-8')) def b2_url_decode(s): - """Decodes a Unicode string returned from B2 in an HTTP header. + """ + Decodes a Unicode string returned from B2 in an HTTP header. - Returns a Python unicode string. + :param s: a unicode string to decode + :type s: str + :return: a Python unicode string. + :rtype: str """ result = six.moves.urllib.parse.unquote_plus(s) if six.PY2: @@ -75,6 +88,12 @@ def b2_url_decode(s): def choose_part_ranges(content_length, minimum_part_size): """ Returns a list of (offset, length) for the parts of a large file. + + :param content_length: content length value + :type content_length: int + :param minimum_part_size: a minimum file part size + :type minimum_part_size: int + :rtype: list """ # If the file is at least twice the minimum part size, we are guaranteed @@ -107,6 +126,11 @@ def hex_sha1_of_stream(input_stream, content_length): """ Returns the 40-character hex SHA1 checksum of the first content_length bytes in the input stream. + + :param input_stream: stream object, which exposes read() method + :param content_length: expected length of the stream + :type content_length: int + :rtype: str """ remaining = content_length block_size = 1024 * 1024 @@ -125,8 +149,11 @@ def hex_sha1_of_stream(input_stream, content_length): def hex_sha1_of_bytes(data): """ - Returns the 40-character hex SHA1 checksum of the first content_length - bytes in the input stream. + Returns the 40-character hex SHA1 checksum of the data + + :param data: an array of bytes + :type data: bytes + :rtype: str """ return hashlib.sha1(data).hexdigest() @@ -135,8 +162,8 @@ def validate_b2_file_name(name): """ Raises a ValueError if the name is not a valid B2 file name. - :param name: a string - :return: None + :param name: a string to check + :type name: str """ if not isinstance(name, six.string_types): raise ValueError('file name must be a string, not bytes') @@ -160,6 +187,14 @@ def validate_b2_file_name(name): def is_file_readable(local_path, reporter=None): + """ + Check if the local file has read permissions + + :param local_path: a file path + :type local_path: str + :param reporter: reporter object to put errors on + :rtype: bool + """ if not os.path.exists(local_path): if reporter is not None: reporter.local_access_error(local_path) @@ -175,6 +210,11 @@ def fix_windows_path_limit(path): """ Prefix paths when running on Windows to overcome 260 character path length limit See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath + + :param path: a path to prefix + :type path: str + :return: a prefixed path + :rtype: str """ if platform.system() == 'Windows': if path.startswith('\\\\'): @@ -197,6 +237,9 @@ class BytesIoContextManager(object): """ def __init__(self, byte_data): + """ + :param bytes_data: a byte stream + """ self.byte_data = byte_data def __enter__(self): @@ -241,6 +284,13 @@ def _pick_scale_and_suffix(x): def format_and_scale_number(x, unit): """ Picks a good scale for representing a number and formats it. + + :param x: a number + :type x: int + :param unit: an arbitrary unit name + :type unit: str + :return: scaled and formatted number + :rtype: str """ # simple case for small numbers @@ -266,6 +316,15 @@ def format_and_scale_number(x, unit): def format_and_scale_fraction(numerator, denominator, unit): """ Picks a good scale for representing a fraction, and formats it. + + :param numerator: a numerator of a fraction + :type numerator: int + :param denominator: a denominator of a fraction + :type denominator: int + :param unit: an arbitrary unit name + :type unit: str + :return: scaled and formatted fraction + :rtype: str """ # simple case for small numbers @@ -293,10 +352,27 @@ def format_and_scale_fraction(numerator, denominator, unit): def camelcase_to_underscore(input_): + """ + Convert camel cased string to string with underscores + + :param input_: an input string + :type input_: str + :return: string with underscores + :rtype: str + """ return _CAMELCASE_TO_UNDERSCORE_RE.sub(r'_\1', input_).lower() def repr_dict_deterministically(dict_): + """ + Represent a dictionary in a deterministic way, i.e. with + the same order of keys + + :param dict_: a dictionary + :type dict_: dict + :return: a string representation of a dictionary + :rtype: str + """ # a simple version had a disadvantage of outputting dictionary keys in random order. # It was hard to read. Therefore we sort items by key. fields = ', '.join('%s: %s' % (repr(k), repr(v)) for k, v in sorted(six.iteritems(dict_))) @@ -304,10 +380,17 @@ def repr_dict_deterministically(dict_): class B2TraceMeta(DefaultTraceMeta): + """ + Traces all public method calls, except for ones with names that begin with 'get_' + """ pass class B2TraceMetaAbstract(DefaultTraceAbstractMeta): + """ + Default class for tracers, to be set as + a metaclass for abstract base classes + """ pass From ad7664c056cbc989cf32f785ad8ec59003175f1a Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 17:36:04 +0300 Subject: [PATCH 39/75] Fixed quoting in docstring --- b2sdk/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2sdk/utils.py b/b2sdk/utils.py index 9efb84dc7..15aa749ef 100644 --- a/b2sdk/utils.py +++ b/b2sdk/utils.py @@ -381,7 +381,7 @@ def repr_dict_deterministically(dict_): class B2TraceMeta(DefaultTraceMeta): """ - Traces all public method calls, except for ones with names that begin with 'get_' + Traces all public method calls, except for ones with names that begin with `get_` """ pass From 8104a779757ae210816852abd489afeae4494917 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Tue, 5 Mar 2019 18:03:47 +0300 Subject: [PATCH 40/75] docstring added to url_for_api function --- b2sdk/api.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/b2sdk/api.py b/b2sdk/api.py index e29cb0140..6fae414e5 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -30,6 +30,14 @@ def url_for_api(info, api_name): + """ + Return URL for an API endpoint + + :param info: account info + :param api_name: + :type api_name: str + :rtype: str + """ if api_name in ['b2_download_file_by_id']: base = info.get_download_url() else: From 91958f934b64c2490a4f20136271f25f9a4118dc Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 9 Mar 2019 11:02:52 +0300 Subject: [PATCH 41/75] Exclude set_auth_data_with_schema_0_for_test from a set of documented methods --- doc/source/b2sdk/account_info/sqlite_account_info.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/b2sdk/account_info/sqlite_account_info.rst b/doc/source/b2sdk/account_info/sqlite_account_info.rst index c8c1d1804..b44627e2d 100644 --- a/doc/source/b2sdk/account_info/sqlite_account_info.rst +++ b/doc/source/b2sdk/account_info/sqlite_account_info.rst @@ -6,3 +6,4 @@ :undoc-members: :show-inheritance: :special-members: __init__ + :exclude-members: set_auth_data_with_schema_0_for_test From d787c89ff1411b7d03aeb7628d99d0f3044fbb39 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 9 Mar 2019 12:25:40 +0300 Subject: [PATCH 42/75] Fix formatting issues --- b2sdk/account_info/exception.py | 2 ++ b2sdk/account_info/in_memory.py | 2 ++ b2sdk/account_info/upload_url_pool.py | 1 + b2sdk/sync/policy_manager.py | 1 + doc/source/conf.py | 13 ++++--------- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/b2sdk/account_info/exception.py b/b2sdk/account_info/exception.py index dd9ff536a..715773509 100644 --- a/b2sdk/account_info/exception.py +++ b/b2sdk/account_info/exception.py @@ -27,6 +27,7 @@ class CorruptAccountInfo(AccountInfoError): """ Raised when an account info file is corrupted """ + def __init__(self, file_name): """ :param file_name: an account info file name @@ -45,6 +46,7 @@ class MissingAccountData(AccountInfoError): """ Raised when there is no account info data available """ + def __init__(self, key): """ :param key: a key for getting account data diff --git a/b2sdk/account_info/in_memory.py b/b2sdk/account_info/in_memory.py index a620f3b82..606920c1b 100644 --- a/b2sdk/account_info/in_memory.py +++ b/b2sdk/account_info/in_memory.py @@ -18,6 +18,7 @@ def _raise_missing_if_result_is_none(function): """ Raise MissingAccountData if function's result is None """ + @wraps(function) def inner(*args, **kwargs): assert function.__name__.startswith('get_') @@ -37,6 +38,7 @@ class InMemoryAccountInfo(UrlPoolAccountInfo): tool. This includes: account id, application key, auth tokens, API URL, download URL, and uploads URLs. """ + def __init__(self, *args, **kwargs): super(InMemoryAccountInfo, self).__init__(*args, **kwargs) self._clear_in_memory_account_fields() diff --git a/b2sdk/account_info/upload_url_pool.py b/b2sdk/account_info/upload_url_pool.py index d701f07cb..c7788ffef 100644 --- a/b2sdk/account_info/upload_url_pool.py +++ b/b2sdk/account_info/upload_url_pool.py @@ -78,6 +78,7 @@ class UrlPoolAccountInfo(AbstractAccountInfo): This concrete implementation uses an instance of UploadUrlPool as an underlying storage """ + def __init__(self): super(UrlPoolAccountInfo, self).__init__() self._reset_upload_pools() diff --git a/b2sdk/sync/policy_manager.py b/b2sdk/sync/policy_manager.py index 9000803d7..935236106 100644 --- a/b2sdk/sync/policy_manager.py +++ b/b2sdk/sync/policy_manager.py @@ -17,6 +17,7 @@ class SyncPolicyManager(object): Policy manager, implements a logic to get a correct policy class and create a policy object based on various parameters """ + def __init__(self): self.policies = {} # dict<,> diff --git a/doc/source/conf.py b/doc/source/conf.py index 74f9fafe3..655bb5ee1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -153,19 +153,14 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ( - master_doc, 'B2_Python_SDK.tex', u'B2\\_Python\\_SDK', - u'Backblaze', 'manual' - ), + (master_doc, 'B2_Python_SDK.tex', u'B2\\_Python\\_SDK', u'Backblaze', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'b2_python_sdk', u'B2 Python SDK Documentation', [author], 1) -] +man_pages = [(master_doc, 'b2_python_sdk', u'B2 Python SDK Documentation', [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -174,8 +169,8 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, 'B2_Python_SDK', u'B2 Python SDK Documentation', author, - 'B2_Python_SDK', 'Backblaze Python SDK', 'Miscellaneous' + master_doc, 'B2_Python_SDK', u'B2 Python SDK Documentation', author, 'B2_Python_SDK', + 'Backblaze Python SDK', 'Miscellaneous' ), ] From 09d37aafa8cef972bf4768c8b8a78a97fe839fd6 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 9 Mar 2019 13:13:22 +0300 Subject: [PATCH 43/75] Delete parse_args module and related unit tests --- b2sdk/parse_args.py | 189 ---------------------------------------- test/test_parse_args.py | 99 --------------------- 2 files changed, 288 deletions(-) delete mode 100644 b2sdk/parse_args.py delete mode 100644 test/test_parse_args.py diff --git a/b2sdk/parse_args.py b/b2sdk/parse_args.py deleted file mode 100644 index 66c583830..000000000 --- a/b2sdk/parse_args.py +++ /dev/null @@ -1,189 +0,0 @@ -###################################################################### -# -# File: b2/parse_args.py -# -# Copyright 2019 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -import logging - -import six - -from .utils import repr_dict_deterministically - -logger = logging.getLogger(__name__) - - -class Arguments(object): - """ - An object to stick attributes on. - """ - - def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - repr_dict_deterministically(self.__dict__), - ) - - -def check_for_duplicate_args(args_dict): - """ - Checks that no argument name is listed in multiple places. - - Raises a ValueError if there is a problem. - - This args_dict has a problem because 'required' and 'optional' - both contain 'a': - - .. code-block:: python - - { - 'option_args': ['b', 'c'], - 'required': ['a', 'd'], - 'optional': ['a', 'e'] - } - - """ - categories = sorted(six.iterkeys(args_dict)) - for index_a, category_a in enumerate(categories): - for category_b in categories[index_a + 1:]: - names_a = args_dict[category_a] - names_b = args_dict[category_b] - for common_name in set(names_a) & set(names_b): - raise ValueError( - "argument '%s' is in both '%s' an '%s'" % (common_name, category_a, category_b) - ) - - -def parse_arg_list( - arg_list, option_flags, option_args, list_args, optional_before, required, optional, arg_parser -): - """ - Converts a list of string arguments to an Arguments object, with - one attribute per parameter. - - The value of every parameter is set in the returned Arguments object, - even for parameters not specified on the command line. - - Option Flags set boolean values that default to False. When the - option is present on the command line, the value is set to True. - - Option Args have values provided on the command line. The default - if not present is None. - - List Args act like Option Args, but can be specified more than - once, and their values are collected into a list. Default is []. - - Required positional parameters must be present, and do not have - a double-dash name preceding them. - - Optional positional parameters are just like required parameters, - but don't have to be there and default to None. - - Arg Parser is a dict that maps from a parameter name to a function - tha converts the string argument into the value needed by the - program. These parameters can be Option Args, List Args, Required, - or Optional. - - :param arg_list sys.argv[1:], or equivalent - :param option_flags: Names of options that are boolean flags. - :param option_args: Names of options that have values. - :param list_args: Names of options whose values are collected into a list. - :param optional_before: Names of option positional params that come before the required ones. - :param required: Names of positional params that must be there. - :param optional: Names of optional params. - :param arg_parser: Map from param name to parser for values. - :return: An Argument object, or None if there was any error parsing. - """ - - # Sanity check the inputs. - check_for_duplicate_args( - { - 'option_flags': option_flags, - 'option_args': option_args, - 'optional_before': optional_before, - 'required': required, - 'optional': optional - } - ) - - # Create an object to hold the arguments. - result = Arguments() - - # Set the default value for everything that has a default value. - for name in option_flags: - setattr(result, name, False) - for name in option_args: - setattr(result, name, None) - for name in list_args: - setattr(result, name, []) - for name in optional: - setattr(result, name, None) - - # Make a function for parsing argument values - def parse_arg(name, arg_list): - value = arg_list.pop(0) - if name in arg_parser: - value = arg_parser[name](value) - return value - - # Parse the '--' options - while len(arg_list) != 0 and arg_list[0].startswith('--'): - option = arg_list.pop(0)[2:] - if option in option_flags: - logger.debug('option %s is properly recognized as OPTION_FLAGS', option) - setattr(result, option, True) - elif option in option_args: - if len(arg_list) == 0: - logger.debug( - 'option %s is recognized as OPTION_ARGS and there are no more arguments on arg_list to parse', - option - ) - return None - else: - logger.debug('option %s is properly recognized as OPTION_ARGS', option) - setattr(result, option, parse_arg(option, arg_list)) - elif option in list_args: - if len(arg_list) == 0: - logger.debug( - 'option %s is recognized as LIST_ARGS and there are no more arguments on arg_list to parse', - option - ) - return None - else: - logger.debug('option %s is properly recognized as LIST_ARGS', option) - getattr(result, option).append(parse_arg(option, arg_list)) - else: - logger.error('option %s is of unknown type!', option) - return None - - # Handle optional positional parameters that come first. - # We assume that if there are optional parameters, the - # ones that come before take precedence over the ones - # that come after the required arguments. - for arg_name in optional_before: - if len(required) < len(arg_list): - setattr(result, arg_name, parse_arg(arg_name, arg_list)) - else: - setattr(result, arg_name, None) - - # Parse the positional parameters - for arg_name in required: - if len(arg_list) == 0: - logger.debug('lack of required positional argument: %s', arg_name) - return None - setattr(result, arg_name, parse_arg(arg_name, arg_list)) - for arg_name in optional: - if len(arg_list) != 0: - setattr(result, arg_name, parse_arg(arg_name, arg_list)) - - # Anything left is a problem - if len(arg_list) != 0: - logger.debug('option parser failed to consume this: %s', arg_list) - return None - - # Return the Arguments object - return result diff --git a/test/test_parse_args.py b/test/test_parse_args.py deleted file mode 100644 index e86144bd6..000000000 --- a/test/test_parse_args.py +++ /dev/null @@ -1,99 +0,0 @@ -###################################################################### -# -# File: test/test_parse_args.py -# -# Copyright 2019 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -from .test_base import TestBase -from b2sdk.parse_args import parse_arg_list - - -class TestParseArgs(TestBase): - - NO_ARGS = { - 'option_flags': [], - 'option_args': [], - 'list_args': [], - 'optional_before': [], - 'required': [], - 'optional': [], - 'arg_parser': {} - } - - EVERYTHING = { - 'option_flags': ['optionFlag'], - 'option_args': ['optionArg'], - 'list_args': ['list'], - 'optional_before': [], - 'required': ['required'], - 'optional': ['optional'], - 'arg_parser': { - 'optionArg': int - } - } - - BEFORE_AND_AFTER = { - 'option_flags': [], - 'option_args': [], - 'list_args': [], - 'optional_before': ['optionalBefore'], - 'required': ['required'], - 'optional': ['optional'], - 'arg_parser': {} - } - - def test_no_args(self): - args = parse_arg_list([], **self.NO_ARGS) - self.assertTrue(args is not None) - - def test_unexpected_flag(self): - args = parse_arg_list(['--badFlag'], **self.NO_ARGS) - self.assertTrue(args is None) - - def test_unexpected_arg(self): - args = parse_arg_list(['badArg'], **self.NO_ARGS) - self.assertTrue(args is None) - - def test_option_defaults(self): - args = parse_arg_list(['req-value'], **self.EVERYTHING) - self.assertFalse(args.optionFlag) - self.assertTrue(args.optionArg is None) - self.assertEqual([], args.list) - self.assertEqual('req-value', args.required) - self.assertTrue(args.optional is None) - - def test_all_there(self): - args = parse_arg_list( - ['--optionFlag', '--optionArg', '99', '--list', '1', '--list', '2', 'b', 'c'], - **self.EVERYTHING - ) # yapf disable - self.assertTrue(args.optionFlag) - self.assertEqual(99, args.optionArg) - self.assertEqual('b', args.required) - self.assertEqual(['1', '2'], args.list) - self.assertEqual('c', args.optional) - - def test_optional_arg_missing_value(self): - args = parse_arg_list(['--optionArg'], **self.EVERYTHING) - self.assertTrue(args is None) - - def test_no_optional(self): - args = parse_arg_list(['a'], **self.BEFORE_AND_AFTER) - self.assertEqual((None, 'a', None), (args.optionalBefore, args.required, args.optional)) - - def test_optional_before(self): - args = parse_arg_list(['a', 'b'], **self.BEFORE_AND_AFTER) - self.assertEqual(('a', 'b', None), (args.optionalBefore, args.required, args.optional)) - - def test_same_arg_in_two_places(self): - arg_spec = dict(self.NO_ARGS) - arg_spec['optional_before'] = ['a'] - arg_spec['required'] = ['a'] - with self.assertRaisesRegexp( - ValueError, "argument 'a' is in both 'optional_before' an 'required'" - ): - parse_arg_list([], **arg_spec) From 66f2a4a11da5788a6827849b8fd088e04d85d5a7 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 9 Mar 2019 13:15:02 +0300 Subject: [PATCH 44/75] Remove repr_dict_deterministically function --- b2sdk/utils.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/b2sdk/utils.py b/b2sdk/utils.py index e32ab4524..072828511 100644 --- a/b2sdk/utils.py +++ b/b2sdk/utils.py @@ -350,22 +350,6 @@ def camelcase_to_underscore(input_): return _CAMELCASE_TO_UNDERSCORE_RE.sub(r'_\1', input_).lower() -def repr_dict_deterministically(dict_): - """ - Represent a dictionary in a deterministic way, i.e. with - the same order of keys - - :param dict_: a dictionary - :type dict_: dict - :return: a string representation of a dictionary - :rtype: str - """ - # a simple version had a disadvantage of outputting dictionary keys in random order. - # It was hard to read. Therefore we sort items by key. - fields = ', '.join('%s: %s' % (repr(k), repr(v)) for k, v in sorted(six.iteritems(dict_))) - return '{%s}' % (fields,) - - class B2TraceMeta(DefaultTraceMeta): """ Traces all public method calls, except for ones with names that begin with `get_` From d7f9ed74c88aa493b82180224410d41b20cff3b6 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sun, 10 Mar 2019 09:15:57 +0300 Subject: [PATCH 45/75] Remove trailing whitespaces from docstrings --- b2sdk/account_info/abstract.py | 14 +++++++------- b2sdk/account_info/in_memory.py | 14 +++++++------- b2sdk/account_info/upload_url_pool.py | 4 ++-- b2sdk/api.py | 6 +++--- b2sdk/b2http.py | 2 +- b2sdk/bucket.py | 4 ++-- b2sdk/file_version.py | 2 +- b2sdk/sync/action.py | 2 +- b2sdk/sync/file.py | 2 +- b2sdk/sync/policy.py | 2 +- b2sdk/sync/policy_manager.py | 2 +- 11 files changed, 27 insertions(+), 27 deletions(-) diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index b7ca7eb1d..0185f6bc5 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -109,7 +109,7 @@ def clear_bucket_upload_data(self, bucket_id): @abstractmethod def get_account_id(self): """ - Returns account_id or raises MissingAccountData exception + Returns account_id or raises MissingAccountData exception :rtype: str """ @@ -117,7 +117,7 @@ def get_account_id(self): @abstractmethod def get_account_id_or_app_key_id(self): """ - Returns the account id or key id used to authenticate + Returns the account id or key id used to authenticate :rtype: str """ @@ -133,7 +133,7 @@ def get_account_auth_token(self): @abstractmethod def get_api_url(self): """ - Returns api_url or raises MissingAccountData exception + Returns api_url or raises MissingAccountData exception :rtype: str """ @@ -141,7 +141,7 @@ def get_api_url(self): @abstractmethod def get_application_key(self): """ - Returns application_key or raises MissingAccountData exception + Returns application_key or raises MissingAccountData exception :rtype: str """ @@ -149,15 +149,15 @@ def get_application_key(self): @abstractmethod def get_download_url(self): """ - Returns download_url or raises MissingAccountData exception + Returns download_url or raises MissingAccountData exception :rtype: str """ @abstractmethod def get_realm(self): - """ - Returns realm or raises MissingAccountData exception + """ + Returns realm or raises MissingAccountData exception :rtype: str """ diff --git a/b2sdk/account_info/in_memory.py b/b2sdk/account_info/in_memory.py index 606920c1b..945edd733 100644 --- a/b2sdk/account_info/in_memory.py +++ b/b2sdk/account_info/in_memory.py @@ -126,7 +126,7 @@ def remove_bucket_name(self, bucket_name): @_raise_missing_if_result_is_none def get_account_id(self): """ - Returns account_id or raises MissingAccountData exception + Returns account_id or raises MissingAccountData exception :rtype: str """ @@ -135,7 +135,7 @@ def get_account_id(self): @_raise_missing_if_result_is_none def get_account_id_or_app_key_id(self): """ - Returns the account id or key id used to authenticate + Returns the account id or key id used to authenticate :rtype: str """ @@ -153,7 +153,7 @@ def get_account_auth_token(self): @_raise_missing_if_result_is_none def get_api_url(self): """ - Returns api_url or raises MissingAccountData exception + Returns api_url or raises MissingAccountData exception :rtype: str """ @@ -162,7 +162,7 @@ def get_api_url(self): @_raise_missing_if_result_is_none def get_application_key(self): """ - Returns application_key or raises MissingAccountData exception + Returns application_key or raises MissingAccountData exception :rtype: str """ @@ -171,7 +171,7 @@ def get_application_key(self): @_raise_missing_if_result_is_none def get_download_url(self): """ - Returns download_url or raises MissingAccountData exception + Returns download_url or raises MissingAccountData exception :rtype: str """ @@ -189,8 +189,8 @@ def get_minimum_part_size(self): @_raise_missing_if_result_is_none def get_realm(self): - """ - Returns realm or raises MissingAccountData exception + """ + Returns realm or raises MissingAccountData exception :rtype: str """ diff --git a/b2sdk/account_info/upload_url_pool.py b/b2sdk/account_info/upload_url_pool.py index c7788ffef..25ce2a1b2 100644 --- a/b2sdk/account_info/upload_url_pool.py +++ b/b2sdk/account_info/upload_url_pool.py @@ -59,7 +59,7 @@ def take(self, key): def clear_for_key(self, key): """ Remove an intem from the pool by key - + :param key: bucket ID or large file ID :type param: str """ @@ -75,7 +75,7 @@ class UrlPoolAccountInfo(AbstractAccountInfo): tool. This includes: account id, application key, auth tokens, API URL, download URL, and uploads URLs. - This concrete implementation uses an instance of UploadUrlPool + This concrete implementation uses an instance of UploadUrlPool as an underlying storage """ diff --git a/b2sdk/api.py b/b2sdk/api.py index 6fae414e5..638c696f9 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -70,8 +70,8 @@ def __init__(self, account_info=None, cache=None, raw_api=None, max_upload_worke """ Initializes the API using the given account info. - :param account_info: an instance of :class:`~b2sdk.account_info.upload_url_pool.UrlPoolAccountInfo`, - or any custom class derived from + :param account_info: an instance of :class:`~b2sdk.account_info.upload_url_pool.UrlPoolAccountInfo`, + or any custom class derived from :class:`~b2sdk.account_info.abstract.AbstractAccountInfo` :param cache: an instance of the one of the following classes: :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, @@ -121,7 +121,7 @@ def get_thread_pool(self): def authorize_automatically(self): """ - Perform automatic account authorization, retrieving all account data + Perform automatic account authorization, retrieving all account data from account info object passed during initialization """ try: diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index d8ddbabae..79ef1c363 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -350,7 +350,7 @@ def get_content(self, url, headers, try_count=1): Use like this: .. code-block:: python - + try: with b2_http.get_content(url, headers) as response: for byte_data in response.iter_content(chunk_size=1024): diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index b6178181f..74ce43f3e 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -237,8 +237,8 @@ def update( :type cors_rules: dict :param lifecycle_rules: lifecycle rules to store with a bucket :type lifecycle_rules: dict - :param if_revision_is: revision number, - update the info if revision equals to + :param if_revision_is: revision number, + update the info if revision equals to if_revision_is' value :type if_revision_is: int """ diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index fabfa2749..70031f321 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -78,7 +78,7 @@ def from_api_response(cls, file_info_dict, force_action=None): or this: .. code-block:: python - + { "accountId": "4aa9865d6f00", "bucketId": "547a2a395826655d561f0010", diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index 7086c5b2b..d6b2c5658 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -343,7 +343,7 @@ def get_bytes(self): def do_action(self, bucket, reporter): """ - Performs the deleting of a local file action, + Performs the deleting of a local file action, returning only after the action is completed. :param bucket: a Bucket object diff --git a/b2sdk/sync/file.py b/b2sdk/sync/file.py index 823873df6..ce67de746 100644 --- a/b2sdk/sync/file.py +++ b/b2sdk/sync/file.py @@ -51,7 +51,7 @@ def __init__(self, id_, file_name, mod_time, action, size): :type id_: str :param file_name: a relative file name :type file_name: str - :param mod_time: modification time, in milliseconds, to avoid rounding issues + :param mod_time: modification time, in milliseconds, to avoid rounding issues with millisecond times from B2 :type mod_time: int :param action: "hide" or "upload" (never "start") diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index e6a5f40b9..328d9e164 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -324,7 +324,7 @@ def make_b2_keep_days_actions( :type dest_file: b2sdk.sync.file.File :param dest_folder: destination folder object :type dest_folder: b2sdk.sync.folder.AbstractFolder - :param transferred: if True, file has been transferred, + :param transferred: if True, file has been transferred, False otherwise :type transferred: bool :param keep_days: how many days to keep a file diff --git a/b2sdk/sync/policy_manager.py b/b2sdk/sync/policy_manager.py index 935236106..530e7b019 100644 --- a/b2sdk/sync/policy_manager.py +++ b/b2sdk/sync/policy_manager.py @@ -14,7 +14,7 @@ class SyncPolicyManager(object): """ - Policy manager, implements a logic to get a correct policy class + Policy manager, implements a logic to get a correct policy class and create a policy object based on various parameters """ From 5fe94b6a1e8894b233a8fdcf7054e448c282950f Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Mon, 11 Mar 2019 09:44:41 +0300 Subject: [PATCH 46/75] Remove extra spaces in docstrings --- b2sdk/account_info/abstract.py | 2 +- b2sdk/account_info/in_memory.py | 2 +- b2sdk/bucket.py | 4 ++-- b2sdk/sync/policy.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 0185f6bc5..81e07086e 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -148,7 +148,7 @@ def get_application_key(self): @abstractmethod def get_download_url(self): - """ + """ Returns download_url or raises MissingAccountData exception :rtype: str diff --git a/b2sdk/account_info/in_memory.py b/b2sdk/account_info/in_memory.py index 945edd733..b654f4f68 100644 --- a/b2sdk/account_info/in_memory.py +++ b/b2sdk/account_info/in_memory.py @@ -170,7 +170,7 @@ def get_application_key(self): @_raise_missing_if_result_is_none def get_download_url(self): - """ + """ Returns download_url or raises MissingAccountData exception :rtype: str diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 74ce43f3e..f90c05fe1 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -202,8 +202,8 @@ def set_info(self, new_bucket_info, if_revision_is=None): :param new_bucket_info: new bucket info dictionary :type new_bucket_info: dict - :param if_revision_is: revision number, - update the info if revision equals to + :param if_revision_is: revision number, + update the info if revision equals to if_revision_is' value :type if_revision_is: int """ diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 328d9e164..91a746b68 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -286,7 +286,7 @@ def make_b2_delete_actions(source_file, dest_file, dest_folder, transferred): :type dest_file: b2sdk.sync.file.File :param dest_folder: destination folder :type dest_folder: b2sdk.sync.folder.AbstractFolder - :param transferred: if True, file has been transferred, + :param transferred: if True, file has been transferred, False otherwise :type transferred: bool """ From db77fd855972964f99732f8da08123b3166d3247 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Wed, 13 Mar 2019 14:09:42 +0300 Subject: [PATCH 47/75] Add installation guide docs template --- doc/source/intro/install.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/source/intro/install.rst diff --git a/doc/source/intro/install.rst b/doc/source/intro/install.rst new file mode 100644 index 000000000..9e52b736c --- /dev/null +++ b/doc/source/intro/install.rst @@ -0,0 +1,4 @@ +Installation Guide +================== + +TODO From 81dedf540457c231f2d6b359cf836a7c58a23a49 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Wed, 13 Mar 2019 14:09:54 +0300 Subject: [PATCH 48/75] Add tutorial docs template --- doc/source/intro/tutorial.rst | 74 +++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 doc/source/intro/tutorial.rst diff --git a/doc/source/intro/tutorial.rst b/doc/source/intro/tutorial.rst new file mode 100644 index 000000000..17c6d6922 --- /dev/null +++ b/doc/source/intro/tutorial.rst @@ -0,0 +1,74 @@ +======== +Tutorial +======== + +Account authorization +===================== + +TODO + +Bucket actions +============== + +Create a bucket +--------------- + +TODO + +Remove a bucket +--------------- + +TODO + +List buckets +------------- + +TODO + +Update bucket info +------------------ + +TODO + +File actions +============ + +Upload file +----------- + +TODO + +Download file +------------- + +TODO + +List files +---------- + +TODO + +Get file meta information +------------------------- + +TODO + +Delete file +----------- + +TODO + +Cancel file operations +---------------------- + +TODO + +Synchronization +=============== + +TODO + +Account information +=================== + +TODO From 70ee7558cc3f819ac8fcf12fcacda9257ca21cdb Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Wed, 13 Mar 2019 14:10:12 +0300 Subject: [PATCH 49/75] Update table of contents --- doc/source/index.rst | 73 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 2dc7f5eab..604f31cb5 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,14 +1,29 @@ -.. B2 Python SDK documentation master file, created by - sphinx-quickstart on Fri Oct 20 18:27:20 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - +========================================= Welcome to B2 Python SDK's documentation! ========================================= +First steps +=========== + +.. toctree:: + :caption: First Steps + :hidden: + + intro/install + intro/tutorial + +:doc:`intro/install` + Get B2 SDK installed on your computer + +:doc:`intro/tutorial` + A quick tutorial on how to get started with B2 SDK + +API Reference +============= + .. toctree:: - :maxdepth: 2 - :caption: Contents: + :caption: API Reference + :hidden: b2sdk/api b2sdk/bucket @@ -33,6 +48,50 @@ Welcome to B2 Python SDK's documentation! b2sdk/sync/scan_policies b2sdk/sync/sync +:doc:`b2sdk/api` + +:doc:`b2sdk/bucket` + +:doc:`b2sdk/cache` + +:doc:`b2sdk/download_dest` + +:doc:`b2sdk/exception` + +:doc:`b2sdk/file_version` + +:doc:`b2sdk/progress` + +:doc:`b2sdk/upload_source` + +:doc:`b2sdk/utils` + +:doc:`b2sdk/account_info/abstract` + +:doc:`b2sdk/account_info/exception` + +:doc:`b2sdk/account_info/sqlite_account_info` + +:doc:`b2sdk/account_info/upload_url_pool` + +:doc:`b2sdk/sync/action` + +:doc:`b2sdk/sync/exception` + +:doc:`b2sdk/sync/file` + +:doc:`b2sdk/sync/folder_parser` + +:doc:`b2sdk/sync/folder` + +:doc:`b2sdk/sync/policy_manager` + +:doc:`b2sdk/sync/policy` + +:doc:`b2sdk/sync/scan_policies` + +:doc:`b2sdk/sync/sync` + Indices and tables ================== From 83d8b154d3432bd0bc58b6ce0fc25e47e615207d Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Wed, 13 Mar 2019 15:02:28 +0300 Subject: [PATCH 50/75] Add overview section to the docs --- doc/source/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 604f31cb5..7a2715289 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -2,6 +2,15 @@ Welcome to B2 Python SDK's documentation! ========================================= +Overview +======== + +B2 SDK is a client library for easy access to all of the capabilities of B2 Cloud Storage. + +B2 command-line tool (https://github.com/Backblaze/B2_Command_Line_Tool) is an example of how it can be used +to provide command-line access to the B2 service, but there are many possible applications +(including FUSE filesystems, storage backend drivers for backup applications etc). + First steps =========== From 188e1c5a681679047cdef0db23f2c655f151dc91 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Wed, 13 Mar 2019 15:02:59 +0300 Subject: [PATCH 51/75] Add installation instructions --- doc/source/intro/install.rst | 56 +++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/doc/source/intro/install.rst b/doc/source/intro/install.rst index 9e52b736c..798b4b56c 100644 --- a/doc/source/intro/install.rst +++ b/doc/source/intro/install.rst @@ -1,4 +1,58 @@ +================== Installation Guide ================== -TODO +Installing B2 SDK +================= + +B2 SDK runs on Python 2.7+ under CPython and PyPy. + +To install B2 SDK run:: + + pip install b2sdk + +If you see a message saying that the `six` library cannot be installed, which +happens if you're installing with the system python on OS X El Capitan, try +this:: + + pip install --ignore-installed b2sdk + +Developer Info +============== + +We encourage outside contributors to perform changes on our codebase. Many such changes have been merged already. In order to make it easier to contribute, core developers of this project: + +* provide guidance (through the issue reporting system) +* provide tool assisted code review (through the Pull Request system) +* maintain a set of integration tests (run with a production cloud) +* maintain a set of (well over a hundred) unit tests +* automatically run unit tests on 14 versions of python (including osx, Jython and pypy) +* format the code automatically using [yapf](https://github.com/google/yapf) +* use static code analysis to find subtle/potential issues with maintainability +* maintain other Continous Integration tools (coverage tracker) + +You'll need to some Python packages installed. To get all the latest things:: + + pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements-test.txt -r requirements-setup.txt + +There is a `Makefile` with a rule to run the unit tests using the currently active Python:: + + make setup + make test + +will install the required packages, then run the unit tests. + +To test in multiple python virtual environments, set the enviroment variable `PYTHON_VIRTUAL_ENVS` +to be a space-separated list of their root directories. When set, the makefile will run the +unit tests in each of the environments. + +Before checking in, use the `pre-commit.sh` script to check code formatting, run +unit tests, run integration tests etc. + +The integration tests need a file in your home directory called `.b2_auth` +that contains two lines with nothing on them but your account ID and application key:: + + accountId + applicationKey + +We marked the places in the code which are significantly less intuitive than others in a special way. To find them occurrences, use `git grep '*magic*'`. From 0b30b6bd45d2fb78f76f8688caf3d32539eabca3 Mon Sep 17 00:00:00 2001 From: Dmitry Paramonov Date: Sat, 16 Mar 2019 09:10:41 +0300 Subject: [PATCH 52/75] Remove extra spaces in docstrings --- b2sdk/account_info/abstract.py | 2 +- b2sdk/account_info/in_memory.py | 2 +- b2sdk/sync/policy.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 81e07086e..2b67e7d29 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -140,7 +140,7 @@ def get_api_url(self): @abstractmethod def get_application_key(self): - """ + """ Returns application_key or raises MissingAccountData exception :rtype: str diff --git a/b2sdk/account_info/in_memory.py b/b2sdk/account_info/in_memory.py index b654f4f68..c809cebfa 100644 --- a/b2sdk/account_info/in_memory.py +++ b/b2sdk/account_info/in_memory.py @@ -161,7 +161,7 @@ def get_api_url(self): @_raise_missing_if_result_is_none def get_application_key(self): - """ + """ Returns application_key or raises MissingAccountData exception :rtype: str diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 91a746b68..ee86a3191 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -264,7 +264,7 @@ def make_b2_delete_note(version, index, transferred): :param version: an object which contains file version info :param index: file version index :type index: int - :param transferred: if True, file has been transferred, + :param transferred: if True, file has been transferred, False otherwise :type transferred: bool """ From ad68ea06fca19e85433292f68d17bd73b0463f52 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 27 Mar 2019 17:21:49 +0100 Subject: [PATCH 53/75] Add building docs to CI --- .travis.yml | 13 +++++++++++++ requirements-doc.txt | 2 ++ 2 files changed, 15 insertions(+) create mode 100644 requirements-doc.txt diff --git a/.travis.yml b/.travis.yml index 483e95b1e..9b1753736 100644 --- a/.travis.yml +++ b/.travis.yml @@ -222,6 +222,10 @@ install: - pip install -r requirements.txt - pip install -r requirements-setup.txt - pip install -r requirements-test.txt + - | + if [ "$MODE" == 'doc' ]; then + pip install -r requirements-doc.txt + fi - | # coverage has additional requirements @@ -299,6 +303,15 @@ script: python setup.py install fi fi + + # build docs + - | + # needs package to be installed already + if [ "$MODE" == 'doc' ]; then + cd doc + make html + fi + # custom things specific to b2 SDK - if [ -n "$TEST_ACCOUNT_ID" -a "$MODE" == "TEST_RAW_API" ]; then python -m b2sdk.__main__ test_raw_api; else echo skip; fi diff --git a/requirements-doc.txt b/requirements-doc.txt new file mode 100644 index 000000000..82133027c --- /dev/null +++ b/requirements-doc.txt @@ -0,0 +1,2 @@ +sphinx +sphinx_rtd_theme From 70af4c4a4aec8cb9787b71b37e130481997cc520 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 28 Mar 2019 14:25:52 +0100 Subject: [PATCH 54/75] Add docs coverage checker to CI --- .travis.yml | 12 ++++++++++++ doc/source/conf.py | 6 +++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9b1753736..dade4b88a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -312,6 +312,18 @@ script: make html fi + # check docs coverage + - | + # needs package to be installed already + if [ "$MODE" == 'doc' ]; then + cd doc + make coverage + cat build/coverage/python.txt + if [ "$(wc -l < build/coverage/python.txt)" != '3' ]; then + exit 1 + fi + fi + # custom things specific to b2 SDK - if [ -n "$TEST_ACCOUNT_ID" -a "$MODE" == "TEST_RAW_API" ]; then python -m b2sdk.__main__ test_raw_api; else echo skip; fi diff --git a/doc/source/conf.py b/doc/source/conf.py index 655bb5ee1..8f5e87faf 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -43,7 +43,11 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode' + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.coverage', ] # Add any paths that contain templates here, relative to this directory. From 5aa6a1ca997b8e5f6bf7153bf13addb371d0348d Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 28 Mar 2019 14:26:27 +0100 Subject: [PATCH 55/75] Fail on syntax issues in docs --- doc/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index f7474ac43..ab8e3caaa 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -T -W --keep-going -E SPHINXBUILD = python -msphinx SPHINXPROJ = B2_Command_Line_Tool SOURCEDIR = source @@ -10,11 +10,11 @@ BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) From 26b5bc85f90102f29fba85627d1647b3165fd636 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 11:22:18 +0100 Subject: [PATCH 56/75] Add documentation rebuilding helper script --- doc/regenerate.sh | 1 + 1 file changed, 1 insertion(+) create mode 100755 doc/regenerate.sh diff --git a/doc/regenerate.sh b/doc/regenerate.sh new file mode 100755 index 000000000..d210288c6 --- /dev/null +++ b/doc/regenerate.sh @@ -0,0 +1 @@ +make html From 157c3b9e40e06e4097b1d22e861faf056bc46033 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 11:22:39 +0100 Subject: [PATCH 57/75] Add requirements-doc.txt installation to release instructions --- README.release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.release.md b/README.release.md index 30af90944..d07a8d5e3 100644 --- a/README.release.md +++ b/README.release.md @@ -1,7 +1,7 @@ # Release Process - Get the latest versions of dependencies - - `pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements-test.txt -r requirements-setup.txt` + - `pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements-test.txt -r requirements-setup.txt -r requirements-doc.txt` - Bump the version number to an even number. - version number is in: `b2/version.py`, `README.md`, and `setup.py`. - Update the release history in README.md. From 9307db494215b74d278a0da46f8823c06a43b771 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 12:30:10 +0100 Subject: [PATCH 58/75] Change "View source" button to "Edit on GitHub" --- doc/Makefile | 2 +- doc/source/conf.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index ab8e3caaa..33d605169 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -4,7 +4,7 @@ # You can set these variables from the command line. SPHINXOPTS = -T -W --keep-going -E SPHINXBUILD = python -msphinx -SPHINXPROJ = B2_Command_Line_Tool +SPHINXPROJ = b2sdk SOURCEDIR = source BUILDDIR = build diff --git a/doc/source/conf.py b/doc/source/conf.py index 8f5e87faf..e7d45cfa0 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -63,7 +63,7 @@ master_doc = 'index' # General information about the project. -project = u'B2 Python SDK' +project = u'b2-sdk-python' copyright = u'2019, Backblaze' author = u'Backblaze' @@ -96,6 +96,15 @@ # -- Options for HTML output ---------------------------------------------- +html_context = { + "display_github": True, # Add 'Edit on Github' link instead of 'View page source' + "github_user": "reef-technologies", # TODO + "github_repo": project, + "github_version": "shipping_split_fixes", # TODO + "conf_py_path": "/doc/source/", + "source_suffix": source_suffix, +} + # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # From 6705a341eee62441518c66feefdee5c844052fb7 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 12:52:30 +0100 Subject: [PATCH 59/75] Change version to 0.1.0 to indicate that the API will change soon --- README.md | 4 ++-- b2sdk/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ae173b5a9..d32e175f4 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,13 @@ b2sdk follows [Semantic Versioning](https://semver.org/) policy, so in essence t Therefore when setting up b2sdk as a dependency, please make sure to match the version appropriately, for example you could put this in your `requirements.txt` to make sure your code is compatible with the `b2sdk` version your user will get from pypi: ``` -b2sdk>=1.0.0,<2.0.0 +b2sdk>=0.0.0,<1.0.0 ``` # Release History -## 1.0.0 (February ????, 2019) +## 0.1.0 (2019-04-??) Initial official release of SDK as a separate package (until now it was a part of B2 CLI) diff --git a/b2sdk/version.py b/b2sdk/version.py index c98500d0a..86a0ec7f8 100644 --- a/b2sdk/version.py +++ b/b2sdk/version.py @@ -13,7 +13,7 @@ # To avoid confusion between official Backblaze releases of this tool and # the versions on Github, we use the convention that the third number is # odd for Github, and even for Backblaze releases. -VERSION = '1.0.0' +VERSION = '0.1.0' PYTHON_VERSION = '.'.join(map(str, sys.version_info[:3])) # something like: 2.7.11 From 5a2bbc0b6383a907da66d5959643afdf7bb7b46a Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 13:03:59 +0100 Subject: [PATCH 60/75] Re-enable incremental documentation building --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 33d605169..6c840fad8 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = -T -W --keep-going -E +SPHINXOPTS = -T -W --keep-going SPHINXBUILD = python -msphinx SPHINXPROJ = b2sdk SOURCEDIR = source From b10edb52c9d54df710349be083ada965ce5a8702 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 13:34:46 +0100 Subject: [PATCH 61/75] Fix spacing in config (thanks yapf!) --- doc/source/conf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index e7d45cfa0..247240b35 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -97,12 +97,12 @@ # -- Options for HTML output ---------------------------------------------- html_context = { - "display_github": True, # Add 'Edit on Github' link instead of 'View page source' - "github_user": "reef-technologies", # TODO - "github_repo": project, - "github_version": "shipping_split_fixes", # TODO - "conf_py_path": "/doc/source/", - "source_suffix": source_suffix, + "display_github": True, # Add 'Edit on Github' link instead of 'View page source' + "github_user": "reef-technologies", # TODO + "github_repo": project, + "github_version": "shipping_split_fixes", # TODO + "conf_py_path": "/doc/source/", + "source_suffix": source_suffix, } # The theme to use for HTML and HTML Help pages. See the documentation for From cf7b3782dc255507c855b8a59a233c9ff4ce420e Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 14:24:12 +0100 Subject: [PATCH 62/75] Add requirements-doc.txt --- requirements-doc.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-doc.txt b/requirements-doc.txt index 82133027c..57f1b90cb 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,2 +1,3 @@ sphinx sphinx_rtd_theme +sphinxcontrib-plantuml From 5e2f1427a262d623c89c9b9aa31bdf1f67ed7478 Mon Sep 17 00:00:00 2001 From: Michal Zukowski Date: Fri, 29 Mar 2019 15:58:06 +0100 Subject: [PATCH 63/75] SqliteAccountInfo database schema in documentation --- doc/render_sqlite_account_info_schema.sh | 1 + .../account_info/sqlite_account_info.rst | 2 + doc/source/conf.py | 1 + doc/source/dot/sqlite_account_info_schema.dot | 129 ++++++++++++++++++ requirements-doc.txt | 1 + sqlite_account_info_schema.py | 29 ++++ 6 files changed, 163 insertions(+) create mode 100755 doc/render_sqlite_account_info_schema.sh create mode 100644 doc/source/dot/sqlite_account_info_schema.dot create mode 100644 sqlite_account_info_schema.py diff --git a/doc/render_sqlite_account_info_schema.sh b/doc/render_sqlite_account_info_schema.sh new file mode 100755 index 000000000..e0f9806ca --- /dev/null +++ b/doc/render_sqlite_account_info_schema.sh @@ -0,0 +1 @@ +python ../sqlite_account_info_schema.py > source/dot/sqlite_account_info_schema.dot diff --git a/doc/source/b2sdk/account_info/sqlite_account_info.rst b/doc/source/b2sdk/account_info/sqlite_account_info.rst index b44627e2d..50c683a26 100644 --- a/doc/source/b2sdk/account_info/sqlite_account_info.rst +++ b/doc/source/b2sdk/account_info/sqlite_account_info.rst @@ -7,3 +7,5 @@ :show-inheritance: :special-members: __init__ :exclude-members: set_auth_data_with_schema_0_for_test + + .. graphviz:: /dot/sqlite_account_info_schema.dot diff --git a/doc/source/conf.py b/doc/source/conf.py index 247240b35..45b5a38a7 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -48,6 +48,7 @@ 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.coverage', + 'sphinx.ext.graphviz', ] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/source/dot/sqlite_account_info_schema.dot b/doc/source/dot/sqlite_account_info_schema.dot new file mode 100644 index 000000000..b5f5b8562 --- /dev/null +++ b/doc/source/dot/sqlite_account_info_schema.dot @@ -0,0 +1,129 @@ + + digraph G { + label = "generated by sadisplay v0.4.9"; + fontname = "Bitstream Vera Sans" + fontsize = 8 + + node [ + fontname = "Bitstream Vera Sans" + fontsize = 8 + shape = "plaintext" + ] + + edge [ + fontname = "Bitstream Vera Sans" + fontsize = 8 + ] + + + account [label=< + + +
account
⚪ account_auth_tokenTEXT
⚪ account_idTEXT
⚪ account_id_or_app_key_idTEXT
⚪ allowedTEXT
⚪ api_urlTEXT
⚪ application_keyTEXT
⚪ download_urlTEXT
⚪ minimum_part_sizeINTEGER
⚪ realmTEXT
+ >] + + + bucket [label=< + + +
bucket
⚪ bucket_idTEXT
⚪ bucket_nameTEXT
+ >] + + + bucket_upload_url [label=< + + +
bucket_upload_url
⚪ bucket_idTEXT
⚪ upload_auth_tokenTEXT
⚪ upload_urlTEXT
+ >] + + + update_done [label=< + + +
update_done
⚪ update_numberINTEGER
+ >] + + edge [ + arrowhead = empty + ] + edge [ + arrowhead = ediamond + arrowtail = open + ] +} diff --git a/requirements-doc.txt b/requirements-doc.txt index 57f1b90cb..7a6e0f44b 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,3 +1,4 @@ sphinx sphinx_rtd_theme sphinxcontrib-plantuml +sadisplay diff --git a/sqlite_account_info_schema.py b/sqlite_account_info_schema.py new file mode 100644 index 000000000..7bbf5555d --- /dev/null +++ b/sqlite_account_info_schema.py @@ -0,0 +1,29 @@ +import tempfile + +import operator + +from sadisplay import describe, render +from sqlalchemy import create_engine, MetaData + +from b2sdk.account_info.sqlite_account_info import SqliteAccountInfo + + +def main(): + with tempfile.NamedTemporaryFile() as fp: + sqlite_db_name = fp.name + SqliteAccountInfo(sqlite_db_name) + engine = create_engine('sqlite:///' + sqlite_db_name) + + meta = MetaData() + + meta.reflect(bind=engine) + + tables = set(meta.tables.keys()) + + desc = describe( + map(lambda x: operator.getitem(meta.tables, x), sorted(tables))) + print(getattr(render, 'dot')(desc).encode('utf-8')) + + +if __name__ == '__main__': + main() From 2fe18ab25352a6210f4aa51d66ffeb85f6018de1 Mon Sep 17 00:00:00 2001 From: Michal Zukowski Date: Fri, 29 Mar 2019 16:51:30 +0100 Subject: [PATCH 64/75] Version in setup.py from b2sdk.version --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ebabb2301..27f403a04 100644 --- a/setup.py +++ b/setup.py @@ -51,13 +51,15 @@ with open('requirements-setup.txt', encoding='utf-8') as f: requirements_setup = f.read().splitlines() +version = __import__('b2sdk.version').version.VERSION + setup( name='b2sdk', # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='1.0.0', + version=version, description='Backblaze B2 SDK', long_description=long_description, long_description_content_type='text/markdown', From a883050e60ee7843f012d3bc4b65e2ca87de7712 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 18:01:19 +0100 Subject: [PATCH 65/75] Sort out sqlite_account_info_schema.py --- doc/source/conf.py | 3 +++ .../sqlite_account_info_schema.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) rename sqlite_account_info_schema.py => doc/sqlite_account_info_schema.py (56%) mode change 100644 => 100755 diff --git a/doc/source/conf.py b/doc/source/conf.py index 45b5a38a7..fddc3f3b1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -49,6 +49,9 @@ 'sphinx.ext.viewcode', 'sphinx.ext.coverage', 'sphinx.ext.graphviz', + 'sphinx.ext.autosummary', + #'sphinxcontrib.fulltoc', # 2019-03-29: unfortunately this doesn't work with sphinx_rtd_theme + 'sphinxcontrib.plantuml', ] # Add any paths that contain templates here, relative to this directory. diff --git a/sqlite_account_info_schema.py b/doc/sqlite_account_info_schema.py old mode 100644 new mode 100755 similarity index 56% rename from sqlite_account_info_schema.py rename to doc/sqlite_account_info_schema.py index 7bbf5555d..bbf54604a --- a/sqlite_account_info_schema.py +++ b/doc/sqlite_account_info_schema.py @@ -1,5 +1,15 @@ -import tempfile +###################################################################### +# +# File: sqlite_account_info_schema.py +# +# Copyright 2019 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +""" generates a dot file with SqliteAccountInfo database structure """ +import tempfile import operator from sadisplay import describe, render @@ -20,8 +30,7 @@ def main(): tables = set(meta.tables.keys()) - desc = describe( - map(lambda x: operator.getitem(meta.tables, x), sorted(tables))) + desc = describe(map(lambda x: operator.getitem(meta.tables, x), sorted(tables))) print(getattr(render, 'dot')(desc).encode('utf-8')) From 42845f83d2e27698bbbcf72a2eb20372f84270e1 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 18:04:52 +0100 Subject: [PATCH 66/75] Add plantuml to documentation generator, make docs in CI --- .travis.yml | 9 +++++++++ doc/source/conf.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dade4b88a..7ce34f7bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,6 +82,10 @@ matrix: - python: 3.5 env: MODE=coverage + # doc + - python: 3.6 + env: MODE=doc + cache: - pip - directories: @@ -97,6 +101,7 @@ cache: - $HOME/.virtualenvs/osx-python3 #- /usr/local/Cellar/python # see comment in the install script #- /usr/local/Cellar/python3 + - $HOME/plantuml.jar before_install: - uname -a - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then lsb_release -a; fi @@ -225,6 +230,10 @@ install: - | if [ "$MODE" == 'doc' ]; then pip install -r requirements-doc.txt + + # install plantuml without sudo + travis_retry wget http://sourceforge.net/projects/plantuml/files/plantuml.1.2019.3.jar/downloadA -O ~/plantuml.jar + alias plantuml="java -jar ~/plantuml.jar" fi - | diff --git a/doc/source/conf.py b/doc/source/conf.py index fddc3f3b1..2a2c0def2 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -101,7 +101,7 @@ # -- Options for HTML output ---------------------------------------------- html_context = { - "display_github": True, # Add 'Edit on Github' link instead of 'View page source' + "display_github": True, # Add 'Edit on Github' link instead of 'View page source' "github_user": "reef-technologies", # TODO "github_repo": project, "github_version": "shipping_split_fixes", # TODO From 8f7acceb0a22cac9fe59b885f311a9876717544d Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 18:35:03 +0100 Subject: [PATCH 67/75] Try to generate documentation in pre-commit --- pre-commit.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pre-commit.sh b/pre-commit.sh index de7a97b30..fddd23fcb 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -109,6 +109,18 @@ do fi done +header docs + +cd doc +if ./regenerate.sh +then + echo "docs passed on $d" +else + echo "docs FAILED on $d" + exit 1 +fi +cd .. + header test_raw_api if TEST_ACCOUNT_ID="$(head -n 1 ~/.b2_auth)" TEST_APPLICATION_KEY="$(tail -n 1 ~/.b2_auth)" python -m b2sdk.raw_api @@ -118,11 +130,3 @@ else echo "raw API test FAILED" exit 1 fi - -if [[ $# -ne 0 && "${1:-}" == quick ]] -then - header QUICK - echo Skipping integration tests in quick mode. - echo - exit 0 -fi From a3aeb89c7c0ea966b0d32642cf1efe8d896fc491 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Mar 2019 23:39:50 +0100 Subject: [PATCH 68/75] Fix travis doc build --- .travis.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7ce34f7bd..c5c8eae92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,6 +85,7 @@ matrix: # doc - python: 3.6 env: MODE=doc + sudo: true cache: - pip @@ -229,6 +230,7 @@ install: - pip install -r requirements-test.txt - | if [ "$MODE" == 'doc' ]; then + sudo apt-get install graphviz pip install -r requirements-doc.txt # install plantuml without sudo @@ -319,6 +321,7 @@ script: if [ "$MODE" == 'doc' ]; then cd doc make html + cd .. fi # check docs coverage @@ -327,9 +330,10 @@ script: if [ "$MODE" == 'doc' ]; then cd doc make coverage - cat build/coverage/python.txt - if [ "$(wc -l < build/coverage/python.txt)" != '3' ]; then - exit 1 + cd .. + if [ "$(wc -l < doc/build/coverage/python.txt)" != '3' ]; then + cat doc/build/coverage/python.txt >&2 + return 1 fi fi From 5324e48eee1c0796d69100f5b1c938b71d88d0fe Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 30 Mar 2019 00:25:34 +0100 Subject: [PATCH 69/75] Try to make doc builder work --- .travis.yml | 2 +- doc/source/b2sdk/account_info/sqlite_account_info.rst | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5c8eae92..ef5195d07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -331,7 +331,7 @@ script: cd doc make coverage cd .. - if [ "$(wc -l < doc/build/coverage/python.txt)" != '3' ]; then + if [ "$(wc -l < doc/build/coverage/python.txt)" != '2' ]; then cat doc/build/coverage/python.txt >&2 return 1 fi diff --git a/doc/source/b2sdk/account_info/sqlite_account_info.rst b/doc/source/b2sdk/account_info/sqlite_account_info.rst index 50c683a26..e587cad3b 100644 --- a/doc/source/b2sdk/account_info/sqlite_account_info.rst +++ b/doc/source/b2sdk/account_info/sqlite_account_info.rst @@ -6,6 +6,5 @@ :undoc-members: :show-inheritance: :special-members: __init__ - :exclude-members: set_auth_data_with_schema_0_for_test .. graphviz:: /dot/sqlite_account_info_schema.dot From b1d229f0806b4867256d5538ba17b3bc067829bb Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sun, 31 Mar 2019 18:37:56 +0200 Subject: [PATCH 70/75] Improve sqlite_account_info docs --- doc/source/b2sdk/account_info/sqlite_account_info.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/source/b2sdk/account_info/sqlite_account_info.rst b/doc/source/b2sdk/account_info/sqlite_account_info.rst index e587cad3b..acaa9fcd2 100644 --- a/doc/source/b2sdk/account_info/sqlite_account_info.rst +++ b/doc/source/b2sdk/account_info/sqlite_account_info.rst @@ -1,10 +1,14 @@ :mod:`b2sdk.account_info.sqlite_account_info` -- SQLite account info ==================================================================== +AccountInfo class which uses a `SQLite database `_ as backend. + +The database has a following schema: + +.. graphviz:: /dot/sqlite_account_info_schema.dot + .. automodule:: b2sdk.account_info.sqlite_account_info :members: :undoc-members: :show-inheritance: :special-members: __init__ - - .. graphviz:: /dot/sqlite_account_info_schema.dot From 5687bd025e289ac9dfeca0697d4bc9485f9deaf6 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sun, 31 Mar 2019 18:38:49 +0200 Subject: [PATCH 71/75] Add .readthedocs.yml --- .readthedocs.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000..82c7a61ff --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,24 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: doc/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - requirements: + - requirements.txt + - requirements-docs.txt + - method: setuptools + path: . + system_packages: false From 61efe954e85093e96d50aec7023c6c2ec5882784 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sun, 31 Mar 2019 19:00:26 +0200 Subject: [PATCH 72/75] Fix python nightly builds in Travis --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef5195d07..161af8704 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ python: - 3.4 - 3.5 - 3.6 - - nightly # stock versions of pypy provided by Travis CI don't work with cryptography 1.0 #- pypy #- pypy3 @@ -28,7 +27,6 @@ branches: matrix: allow_failures: - python: pypy3 # https://github.com/pyca/cryptography/issues/2880 - - python: nightly # on 2018-02-14 it was found failing installation with ValueError: bad marshal data (unknown type code) include: # Source Code Analysis is slow, so we run it just once on Python 2 and once on Python 3 - python: 2.7 @@ -41,6 +39,11 @@ matrix: dist: xenial sudo: true + # Enable nightly without globally enabling sudo and xenial for other build jobs + - python: nightly + dist: xenial + sudo: true + # Travis CI doesn't support OSX python builds officially, # but that won't stop us... # ... or maybe it will. It's now getting this error: From f02a824cbc37f196e827348550d96f5b31a26f0b Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sun, 31 Mar 2019 19:23:42 +0200 Subject: [PATCH 73/75] Fix building docs at readthedocs --- .readthedocs.yml | 8 ++++---- setup.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 82c7a61ff..670d14f05 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,9 +16,9 @@ formats: all python: version: 3.7 install: - - requirements: - - requirements.txt - - requirements-docs.txt - - method: setuptools + - requirements: requirements.txt + - method: pip path: . + extra_requirements: + - doc system_packages: false diff --git a/setup.py b/setup.py index 27f403a04..ad22a0f19 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,9 @@ with open('requirements-setup.txt', encoding='utf-8') as f: requirements_setup = f.read().splitlines() +with open('requirements-doc.txt', encoding='utf-8') as f: + requirements_doc = f.read().splitlines() + version = __import__('b2sdk.version').version.VERSION setup( @@ -130,8 +133,7 @@ # for example: # $ pip install -e .[dev,test] extras_require={ - 'dev': [], - 'test': [], + 'doc': requirements_doc, }, # If there are data files included in your packages that need to be From ca6e35319cac501adb2ed947cdc43e3c24afa064 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 1 Apr 2019 02:49:55 +0200 Subject: [PATCH 74/75] Improve documentation --- README.md | 2 +- doc/Makefile | 4 + doc/source/api_reference.rst | 15 +++ doc/source/authentication.rst | 10 ++ .../{intro/tutorial.rst => examples.rst} | 3 +- doc/source/index.rst | 102 ++-------------- doc/source/{intro => }/install.rst | 36 ++++-- doc/source/source_dest.rst | 8 ++ doc/source/sync.rst | 15 +++ doc/source/user_guide.rst | 112 ++++++++++++++++++ 10 files changed, 199 insertions(+), 108 deletions(-) create mode 100644 doc/source/api_reference.rst create mode 100644 doc/source/authentication.rst rename doc/source/{intro/tutorial.rst => examples.rst} (98%) rename doc/source/{intro => }/install.rst (68%) create mode 100644 doc/source/source_dest.rst create mode 100644 doc/source/sync.rst create mode 100644 doc/source/user_guide.rst diff --git a/README.md b/README.md index d32e175f4..d4f938c6d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository contains a client library and a few handy utilities for easy access to all of the capabilities of B2 Cloud Storage. -[B2 command-line tool](https://github.com/Backblaze/B2_Command_Line_Tool) is an exampl of how it can be used to provide command-line access to the B2 service, but there are many possible applications (including FUSE filesystems, storage backend drivers for backup applications etc). +[B2 command-line tool](https://github.com/Backblaze/B2_Command_Line_Tool) is an example of how it can be used to provide command-line access to the B2 service, but there are many possible applications (including FUSE filesystems, storage backend drivers for backup applications etc). # Installation diff --git a/doc/Makefile b/doc/Makefile index 6c840fad8..45078ea16 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -14,6 +14,10 @@ help: .PHONY: help Makefile +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile diff --git a/doc/source/api_reference.rst b/doc/source/api_reference.rst new file mode 100644 index 000000000..2ae3c737a --- /dev/null +++ b/doc/source/api_reference.rst @@ -0,0 +1,15 @@ +######################## +API Reference +######################## + +.. toctree:: + authentication + source_dest + b2sdk/api + b2sdk/bucket + b2sdk/cache + b2sdk/exception + b2sdk/file_version + b2sdk/progress + b2sdk/utils + sync diff --git a/doc/source/authentication.rst b/doc/source/authentication.rst new file mode 100644 index 000000000..b3fc7cba7 --- /dev/null +++ b/doc/source/authentication.rst @@ -0,0 +1,10 @@ +######################## +Authentication +######################## + +.. toctree:: + + b2sdk/account_info/abstract + b2sdk/account_info/exception + b2sdk/account_info/sqlite_account_info + b2sdk/account_info/upload_url_pool diff --git a/doc/source/intro/tutorial.rst b/doc/source/examples.rst similarity index 98% rename from doc/source/intro/tutorial.rst rename to doc/source/examples.rst index 17c6d6922..1ff3064db 100644 --- a/doc/source/intro/tutorial.rst +++ b/doc/source/examples.rst @@ -1,5 +1,5 @@ ======== -Tutorial +Examples ======== Account authorization @@ -72,3 +72,4 @@ Account information =================== TODO + diff --git a/doc/source/index.rst b/doc/source/index.rst index 7a2715289..3bfa8a996 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,108 +1,22 @@ -========================================= -Welcome to B2 Python SDK's documentation! -========================================= - +######################################### Overview -======== +######################################### B2 SDK is a client library for easy access to all of the capabilities of B2 Cloud Storage. -B2 command-line tool (https://github.com/Backblaze/B2_Command_Line_Tool) is an example of how it can be used +`B2 command-line tool `_ is an example of how it can be used to provide command-line access to the B2 service, but there are many possible applications (including FUSE filesystems, storage backend drivers for backup applications etc). -First steps -=========== - -.. toctree:: - :caption: First Steps - :hidden: - - intro/install - intro/tutorial - -:doc:`intro/install` - Get B2 SDK installed on your computer - -:doc:`intro/tutorial` - A quick tutorial on how to get started with B2 SDK - -API Reference -============= - .. toctree:: - :caption: API Reference - :hidden: - - b2sdk/api - b2sdk/bucket - b2sdk/cache - b2sdk/download_dest - b2sdk/exception - b2sdk/file_version - b2sdk/progress - b2sdk/upload_source - b2sdk/utils - b2sdk/account_info/abstract - b2sdk/account_info/exception - b2sdk/account_info/sqlite_account_info - b2sdk/account_info/upload_url_pool - b2sdk/sync/action - b2sdk/sync/exception - b2sdk/sync/file - b2sdk/sync/folder_parser - b2sdk/sync/folder - b2sdk/sync/policy_manager - b2sdk/sync/policy - b2sdk/sync/scan_policies - b2sdk/sync/sync - -:doc:`b2sdk/api` - -:doc:`b2sdk/bucket` - -:doc:`b2sdk/cache` - -:doc:`b2sdk/download_dest` - -:doc:`b2sdk/exception` - -:doc:`b2sdk/file_version` - -:doc:`b2sdk/progress` - -:doc:`b2sdk/upload_source` - -:doc:`b2sdk/utils` - -:doc:`b2sdk/account_info/abstract` - -:doc:`b2sdk/account_info/exception` - -:doc:`b2sdk/account_info/sqlite_account_info` - -:doc:`b2sdk/account_info/upload_url_pool` - -:doc:`b2sdk/sync/action` - -:doc:`b2sdk/sync/exception` - -:doc:`b2sdk/sync/file` - -:doc:`b2sdk/sync/folder_parser` - -:doc:`b2sdk/sync/folder` - -:doc:`b2sdk/sync/policy_manager` - -:doc:`b2sdk/sync/policy` - -:doc:`b2sdk/sync/scan_policies` -:doc:`b2sdk/sync/sync` + install + user_guide + api_reference +######################################### Indices and tables -================== +######################################### * :ref:`genindex` * :ref:`modindex` diff --git a/doc/source/intro/install.rst b/doc/source/install.rst similarity index 68% rename from doc/source/intro/install.rst rename to doc/source/install.rst index 798b4b56c..818f3d0a4 100644 --- a/doc/source/intro/install.rst +++ b/doc/source/install.rst @@ -1,21 +1,33 @@ -================== +######################## Installation Guide -================== +######################## Installing B2 SDK ================= +Safety first +~~~~~~~~~~~~~~~ + +.. caution:: Correct version pinning is necessary to make sure your code remains compatible with upcoming b2sdk releases. + + See :ref:`this ` chapter for details. + +Installing for development +~~~~~~~~~~~~~~~~~~~~~~~~~~ + B2 SDK runs on Python 2.7+ under CPython and PyPy. -To install B2 SDK run:: +To install B2 SDK, simply run:: pip install b2sdk -If you see a message saying that the `six` library cannot be installed, which -happens if you're installing with the system python on OS X El Capitan, try -this:: +In your python environment. + +.. note:: If you see a message saying that the ``six`` library cannot be installed, which + happens if you're installing with the system python on OS X El Capitan, try this:: + + pip install --ignore-installed b2sdk - pip install --ignore-installed b2sdk Developer Info ============== @@ -27,7 +39,7 @@ We encourage outside contributors to perform changes on our codebase. Many such * maintain a set of integration tests (run with a production cloud) * maintain a set of (well over a hundred) unit tests * automatically run unit tests on 14 versions of python (including osx, Jython and pypy) -* format the code automatically using [yapf](https://github.com/google/yapf) +* format the code automatically using `yapf `_ * use static code analysis to find subtle/potential issues with maintainability * maintain other Continous Integration tools (coverage tracker) @@ -42,17 +54,17 @@ There is a `Makefile` with a rule to run the unit tests using the currently acti will install the required packages, then run the unit tests. -To test in multiple python virtual environments, set the enviroment variable `PYTHON_VIRTUAL_ENVS` +To test in multiple python virtual environments, set the enviroment variable ``PYTHON_VIRTUAL_ENVS`` to be a space-separated list of their root directories. When set, the makefile will run the unit tests in each of the environments. -Before checking in, use the `pre-commit.sh` script to check code formatting, run +Before checking in, use the ``pre-commit.sh`` script to check code formatting, run unit tests, run integration tests etc. -The integration tests need a file in your home directory called `.b2_auth` +The integration tests need a file in your home directory called ``.b2_auth`` that contains two lines with nothing on them but your account ID and application key:: accountId applicationKey -We marked the places in the code which are significantly less intuitive than others in a special way. To find them occurrences, use `git grep '*magic*'`. +We marked the places in the code which are significantly less intuitive than others in a special way. To find them occurrences, use ``git grep '*magic*'``. diff --git a/doc/source/source_dest.rst b/doc/source/source_dest.rst new file mode 100644 index 000000000..e219a42a9 --- /dev/null +++ b/doc/source/source_dest.rst @@ -0,0 +1,8 @@ +######################## +Sources and Destinations +######################## + +.. toctree:: + + b2sdk/download_dest + b2sdk/upload_source diff --git a/doc/source/sync.rst b/doc/source/sync.rst new file mode 100644 index 000000000..912a316b3 --- /dev/null +++ b/doc/source/sync.rst @@ -0,0 +1,15 @@ +######################## +Sync +######################## + +.. toctree:: + + b2sdk/sync/action + b2sdk/sync/exception + b2sdk/sync/file + b2sdk/sync/folder_parser + b2sdk/sync/folder + b2sdk/sync/policy_manager + b2sdk/sync/policy + b2sdk/sync/scan_policies + b2sdk/sync/sync diff --git a/doc/source/user_guide.rst b/doc/source/user_guide.rst new file mode 100644 index 000000000..d49280653 --- /dev/null +++ b/doc/source/user_guide.rst @@ -0,0 +1,112 @@ +########## +User guide +########## + +.. _semver: + +*************************************** +Version pinning +*************************************** + +b2sdk is divided into three parts. Please pay attention to which group you use, as the stability of your application depends on correct pinning of versions. + +++++++++++ +Interfaces +++++++++++ + +Public +====== + +Public interface consists of *public* members of the following modules: + +.. autosummary:: + :nosignatures: + + b2sdk.api.B2Api + b2sdk.bucket.Bucket + b2sdk.exception + b2sdk.sync + b2sdk.sync.exception + b2sdk.account_info.abstract + b2sdk.account_info.exception + b2sdk.account_info.sqlite_account_info + b2sdk.account_info.upload_url_pool + b2sdk.transferer + b2sdk.utils + +Those will not change in a backwards-incompatible way between non-major versions. In other words, if you pin your dependencies to `>=x.0.0;=4.5.6;<5.0.0 + +.. note:: b2sdk.*._something and b2sdk.*.*._something, having a name which begins with an underscore, are NOT considred public interface. + + +Protected +========= + +Things which sometimes might be necssary to use that are NOT considered public interface (and may change in a non-major version): + +.. autosummary:: + :nosignatures: + + b2sdk.session.B2Session + b2sdk.raw_api.B2RawApi + b2sdk.b2http.B2Http + +.. note:: it is ok for you to use those (better that, than copying our sources), however if you do, please pin your dependencies to middle version. + +.. hint:: If the current version of b2sdk is 4.5.6 and you use the public and protected interfaces, put this in your requirements.txt:: + + >=4.5.6;<4.6.0 + + +Private +======= + +If you need to use some of our private interfaces, pin your dependencies strictly. + +.. hint:: If the current version of b2sdk is 4.5.6 and you use the private interface, put this in your requirements.txt:: + + ==4.5.6 + +************* +Authorization +************* + +Before you can use b2sdk, you need to prove who you are to the server. For that you will need to pass `account id` and `api token` to one of the authorization classes. + +In case you are storing that information in a database or something, you can implement your own class by inheriting from AbstractAuthorization. Otherwise, use one of the classes included in b2sdk package: + + +InMemoryAccountInfo: + +This is probably what your application should be using and also what we use in our tests. + + +SqliteAccountInfo: + +this is what B2 CLI uses to authorize the user. Stores information in a local file. + + +B2Api +~~~~~ + +The "main" object that abstracts the communication with B2 cloud is B2Api. It lets you manage buckets and download files by id. + +example + + +Bucket +~~~~~~ + +Bucket class abstracts the B2 bucket, which is essentially a namespace for objects. + +The best way to transfer your files into a bucket and back, is to use *sync*. + +If for some reason you cannot use sync, it is also possible to upload and download files directly into/from the bucket, using Bucket.upload_file and Bucket.download_by_name. + +The Bucket object also contains a few methods to list the contents of the bucket and the metadata associated with the objects contained in it. + From 4f1f9fb069899d1557d8328b78e928cf5aea1045 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 3 Apr 2019 00:58:40 +0200 Subject: [PATCH 75/75] Remove CLI things --- contrib/bash_completion/b2 | 70 -------------------------------------- contrib/debug_logs.ini | 2 +- 2 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 contrib/bash_completion/b2 diff --git a/contrib/bash_completion/b2 b/contrib/bash_completion/b2 deleted file mode 100644 index eecfa83c5..000000000 --- a/contrib/bash_completion/b2 +++ /dev/null @@ -1,70 +0,0 @@ -_have b2 && -_b2 () { - local _b2_subcommands=( - 'authorize_account' - 'clear_account' - 'create_bucket' - 'delete_bucket' - 'delete_file_version' - 'download_file_by_id' - 'download_file_by_name' - 'get_file_info' - 'hide_file' - 'list_buckets' - 'list_file_names' - 'list_file_versions' - 'ls' - 'make_url' - 'update_bucket' - 'upload_file' - 'version' - ) - local _b2_lsopts=(--long --version) - local _b2_bucket_types=(allPrivate allPublic) - _b2_buckets() { - b2 list_buckets | cut -f1 -d' '; - } - _b2_files() { - b2 ls "$1" - } - - local wants_bucket="@(delete_bucket|hide_file|list_file_names|list_file_versions|update_bucket|upload_file|hide_file|download_file_by_name)" - local wants_bucket_type="@(create_bucket|update_bucket)" - local wants_file="@(download_file_by_name|hide_file|list_file_names|list_file_versions)" - - local cur prev words subcommand - _get_comp_words_by_ref cur prev words - - subcommand="${words[1]}" - - if [[ $COMP_CWORD -eq 1 ]]; then - COMPREPLY=($(compgen -W '${_b2_subcommands[@]}' -- "$cur")) - elif [[ $subcommand == ls ]]; then - if [[ $cur =~ ^- && $COMP_CWORD -eq 2 ]]; then - COMPREPLY=($(compgen -W '${_b2_lsopts[@]}' -- "$cur")) - elif [[ ( $prev =~ ^- && $COMP_CWORD -eq 3 ) || $COMP_CWORD -eq 2 ]]; then - COMPREPLY=($(compgen -W '$(_b2_buckets)' -- "$cur")) - fi - elif [[ $subcommand == create_bucket && $COMP_CWORD -eq 3 ]]; then - COMPREPLY=($(compgen -W '${_b2_bucket_types[@]}' -- "$cur")) - elif [[ $subcommand == $wants_bucket && $COMP_CWORD -eq 2 ]]; then - COMPREPLY=($(compgen -W '$(_b2_buckets)' -- "$cur")) - elif [[ $subcommand == $wants_bucket_type && $COMP_CWORD -eq 3 ]]; then - COMPREPLY=($(compgen -W '${_b2_bucket_types[@]}' -- "$cur")) - elif [[ $subcommand == upload_file && $COMP_CWORD -eq 3 ]]; then - _filedir - elif [[ $subcommand == $wants_file && $COMP_CWORD -eq 3 ]]; then - COMPREPLY=($(compgen -W '$(_b2_files $prev)' -- "$cur")) - fi - - # It's not clear how to complete fileName/fileId not having the bucket - # b2 delete_file_version - # b2 download_file_by_id - # b2 get_file_info - # b2 make_url - - unset -f _b2_buckets - unset -f _b2_files - return 0 -} && -complete -F _b2 b2 diff --git a/contrib/debug_logs.ini b/contrib/debug_logs.ini index 4c5c4d5f8..0c5ffce83 100644 --- a/contrib/debug_logs.ini +++ b/contrib/debug_logs.ini @@ -22,7 +22,7 @@ keys=fileHandler class=logging.handlers.TimedRotatingFileHandler level=DEBUG formatter=simpleFormatter -args=('b2_cli.log', 'midnight') +args=('b2sdk.log', 'midnight')