diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 41a81ab35..2b67e7d29 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -45,6 +45,11 @@ class AbstractAccountInfo(object): @classmethod def all_capabilities(cls): + """ + Return a list of all possible capabilities + + :rtype: list + """ return cls.ALL_CAPABILITIES @abstractmethod @@ -58,64 +63,112 @@ def clear(self): def refresh_entire_bucket_name_cache(self, name_id_iterable): """ Removes all previous name-to-id mappings and stores new ones. + + :param name_id_iterable: a list of tuples of the form (name, id) + :type name_id_iterable: list """ @abstractmethod def remove_bucket_name(self, bucket_name): """ Removes one entry from the bucket name cache. + + :param bucket_name: a bucket name + :type bucket_name: str """ @abstractmethod def save_bucket(self, bucket): """ Remembers the ID for a bucket name. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket """ @abstractmethod def get_bucket_id_or_none_from_bucket_name(self, bucket_name): """ Looks up the bucket ID for a given bucket name. + + :param bucket_name: a bucket name + :type bucket_name: str + :return bucket ID or None: + :rtype: str, None """ @abstractmethod def clear_bucket_upload_data(self, bucket_id): """ Removes all upload URLs for the given bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str """ @abstractmethod def get_account_id(self): - """ returns account_id or raises MissingAccountData exception """ + """ + Returns account_id or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_account_id_or_app_key_id(self): - """ returns the account id or key id used to authenticate """ + """ + Returns the account id or key id used to authenticate + + :rtype: str + """ @abstractmethod def get_account_auth_token(self): - """ returns account_auth_token or raises MissingAccountData exception """ + """ + Returns account_auth_token or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_api_url(self): - """ returns api_url or raises MissingAccountData exception """ + """ + Returns api_url or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_application_key(self): - """ returns application_key or raises MissingAccountData exception """ + """ + Returns application_key or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_download_url(self): - """ returns download_url or raises MissingAccountData exception """ + """ + Returns download_url or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_realm(self): - """ returns realm or raises MissingAccountData exception """ + """ + Returns realm or raises MissingAccountData exception + + :rtype: str + """ @abstractmethod def get_minimum_part_size(self): """ - :return: returns the minimum number of bytes in a part of a large file + Return the minimum number of bytes in a part of a large file + + :return: number of bytes + :rtype: int """ @abstractmethod @@ -124,6 +177,8 @@ def get_allowed(self): An 'allowed' dict, as returned by b2_authorize_account. Never None; for account info that was saved before 'allowed' existed, returns DEFAULT_ALLOWED. + + :rtype: dict """ @limit_trace_arguments(only=['self', 'api_url', 'download_url', 'minimum_part_size', 'realm']) @@ -149,6 +204,25 @@ def set_auth_data( a bucketName field. For keys with bucket restrictions, the name of the bucket is looked up and stored, too. The console_tool does everything by bucket name, so it's convenient to have the restricted bucket name handy. + + :param account_id: user account ID + :type account_id: str + :param auth_token: user authentication token + :type auth_token: str + :param api_url: an API URL + :type api_url: str + :param download_url: path download URL + :type download_url: str + :param minimum_part_size: minimum size of the file part + :type minimum_part_size: int + :param application_key: application key + :type application_key: str + :param realm: a realm to authiroze account in + :type realm: str + :param allowed: the structure to use for old account info that was saved without 'allowed' + :type allowed: dict + :param account_id_or_app_key_id: account ID or application key ID + :type account_id_or_app_key_id: str """ if allowed is None: allowed = self.DEFAULT_ALLOWED @@ -173,6 +247,10 @@ def allowed_is_valid(cls, allowed): If the bucketId is for a bucket that no longer exists, or the capabilities do not allow listBuckets, then we won't have a bucketName. + + :param allowed: the structure to use for old account info that was saved without 'allowed' + :type allowed: dict + :rtype: bool """ return ( ('bucketId' in allowed) and ('bucketName' in allowed) and @@ -203,6 +281,10 @@ def take_bucket_upload_url(self, bucket_id): Returns a pair (upload_url, upload_auth_token) that has been removed from the pool for this bucket, or (None, None) if there are no more left. + + :param bucket_id: a bucket ID + :type bucket_id: str + :rtype: tuple """ @abstractmethod @@ -211,17 +293,47 @@ def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): """ Add an (upload_url, upload_auth_token) pair to the pool available for the bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + :rtype: tuple """ @abstractmethod @limit_trace_arguments(only=['self']) def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): + """ + Put large file upload URL into a pool + + :param file_id: a file ID + :type file_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + """ pass @abstractmethod def take_large_file_upload_url(self, file_id): + """ + Take large file upload URL from a pool + + :param file_id: a file ID + :type file_id: str + """ pass @abstractmethod def clear_large_file_upload_urls(self, file_id): + """ + Clear a pool of URLs for a given file ID + + :param file_id: a file ID + :type file_id: str + """ pass diff --git a/b2sdk/account_info/exception.py b/b2sdk/account_info/exception.py index 6a4b869aa..715773509 100644 --- a/b2sdk/account_info/exception.py +++ b/b2sdk/account_info/exception.py @@ -17,11 +17,22 @@ @six.add_metaclass(ABCMeta) class AccountInfoError(B2Error): + """ + Base class for all account info errors + """ pass class CorruptAccountInfo(AccountInfoError): + """ + Raised when an account info file is corrupted + """ + def __init__(self, file_name): + """ + :param file_name: an account info file name + :type file_name: str + """ super(CorruptAccountInfo, self).__init__() self.file_name = file_name @@ -32,7 +43,15 @@ def __str__(self): class MissingAccountData(AccountInfoError): + """ + Raised when there is no account info data available + """ + def __init__(self, key): + """ + :param key: a key for getting account data + :type key: str + """ super(MissingAccountData, self).__init__() self.key = key diff --git a/b2sdk/account_info/in_memory.py b/b2sdk/account_info/in_memory.py index 4c30cbc01..c809cebfa 100644 --- a/b2sdk/account_info/in_memory.py +++ b/b2sdk/account_info/in_memory.py @@ -15,6 +15,10 @@ def _raise_missing_if_result_is_none(function): + """ + Raise MissingAccountData if function's result is None + """ + @wraps(function) def inner(*args, **kwargs): assert function.__name__.startswith('get_') @@ -28,11 +32,21 @@ def inner(*args, **kwargs): class InMemoryAccountInfo(UrlPoolAccountInfo): + """ + Holder for all account-related information that needs to be kept + between API calls, and between invocations of the command-line + tool. This includes: account id, application key, auth tokens, + API URL, download URL, and uploads URLs. + """ + def __init__(self, *args, **kwargs): super(InMemoryAccountInfo, self).__init__(*args, **kwargs) self._clear_in_memory_account_fields() def clear(self): + """ + Remove all stored information + """ self._clear_in_memory_account_fields() return super(InMemoryAccountInfo, self).clear() @@ -71,50 +85,124 @@ def _set_auth_data( self._allowed = allowed def refresh_entire_bucket_name_cache(self, name_id_iterable): + """ + Removes all previous name-to-id mappings and stores new ones. + + :param name_id_iterable: a list of tuples of the form (name, id) + :type name_id_iterable: list + """ self._buckets = dict(name_id_iterable) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): + """ + Looks up the bucket ID for a given bucket name. + + :param bucket_name: a bucket name + :type bucket_name: str + :return bucket ID or None: + :rtype: str, None + """ return self._buckets.get(bucket_name) def save_bucket(self, bucket): + """ + Remembers the ID for a bucket name. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + """ self._buckets[bucket.name] = bucket.id_ def remove_bucket_name(self, bucket_name): + """ + Removes one entry from the bucket name cache. + + :param bucket_name: a bucket name + :type bucket_name: str + """ if bucket_name in self._buckets: del self._buckets[bucket_name] @_raise_missing_if_result_is_none def get_account_id(self): + """ + Returns account_id or raises MissingAccountData exception + + :rtype: str + """ return self._account_id @_raise_missing_if_result_is_none def get_account_id_or_app_key_id(self): + """ + Returns the account id or key id used to authenticate + + :rtype: str + """ return self._account_id_or_app_key_id @_raise_missing_if_result_is_none def get_account_auth_token(self): + """ + Return account_auth_token or raises MissingAccountData exception + + :rtype: str + """ return self._auth_token @_raise_missing_if_result_is_none def get_api_url(self): + """ + Returns api_url or raises MissingAccountData exception + + :rtype: str + """ return self._api_url @_raise_missing_if_result_is_none def get_application_key(self): + """ + Returns application_key or raises MissingAccountData exception + + :rtype: str + """ return self._application_key @_raise_missing_if_result_is_none def get_download_url(self): + """ + Returns download_url or raises MissingAccountData exception + + :rtype: str + """ return self._download_url @_raise_missing_if_result_is_none def get_minimum_part_size(self): + """ + Return the minimum number of bytes in a part of a large file + + :return: number of bytes + :rtype: int + """ return self._minimum_part_size @_raise_missing_if_result_is_none def get_realm(self): + """ + Returns realm or raises MissingAccountData exception + + :rtype: str + """ return self._realm @_raise_missing_if_result_is_none def get_allowed(self): + """ + An 'allowed' dict, as returned by b2_authorize_account. + Never None; for account info that was saved before 'allowed' existed, + returns DEFAULT_ALLOWED. + + :rtype: dict + """ return self._allowed diff --git a/b2sdk/account_info/sqlite_account_info.py b/b2sdk/account_info/sqlite_account_info.py index f344cb585..3064e0812 100644 --- a/b2sdk/account_info/sqlite_account_info.py +++ b/b2sdk/account_info/sqlite_account_info.py @@ -40,7 +40,9 @@ class SqliteAccountInfo(UrlPoolAccountInfo): def __init__(self, file_name=None, last_upgrade_to_run=None): """ :param file_name: The sqlite file to use; overrides the default. + :type file_name: str :param last_upgrade_to_run: For testing only, override the auto-update on the db. + :type last_upgrade_to_run: int """ self.thread_local = threading.local() user_account_info_path = file_name or os.environ.get( @@ -207,6 +209,9 @@ def _ensure_update(self, update_number, update_command): ) def clear(self): + """ + Remove all info about accounts and buckets + """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') @@ -259,6 +264,24 @@ def set_auth_data_with_schema_0_for_test( application_key, realm, ): + """ + Set authentication data for tests + + :param account_id: an account ID + :type account_id: str + :param auth_token: an authentication token + :type auth_token: str + :param api_url: an API URL + :type api_url: str + :param download_url: a download URL + :type download_url: str + :param minimum_part_size: a minimum part size + :type minimum_part_size: int + :param application_key: an application key + :type application_key: str + :param realm: a realm to authorize account in + :type realm: str + """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') @@ -282,14 +305,27 @@ def set_auth_data_with_schema_0_for_test( ) def get_application_key(self): + """ + Return application key + + :rtype: str + """ return self._get_account_info_or_raise('application_key') def get_account_id(self): + """ + Return account ID + + :rtype: str + """ return self._get_account_info_or_raise('account_id') def get_account_id_or_app_key_id(self): """ + Return account ID or appliction key ID. The 'account_id_or_app_key_id' column was not in the original schema, so it may be NULL. + + :rtype: str """ result = self._get_account_info_or_raise('account_id_or_app_key_id') if result is None: @@ -298,23 +334,51 @@ def get_account_id_or_app_key_id(self): return result def get_api_url(self): + """ + Return API URL + + :rtype: str + """ return self._get_account_info_or_raise('api_url') def get_account_auth_token(self): + """ + Return account authentication token + + :rtype: str + """ return self._get_account_info_or_raise('account_auth_token') def get_download_url(self): + """ + Return download URL + + :rtype: str + """ return self._get_account_info_or_raise('download_url') def get_realm(self): + """ + Return realm name + + :rtype: str + """ return self._get_account_info_or_raise('realm') def get_minimum_part_size(self): + """ + Return minimum part size + + :rtype: int + """ return self._get_account_info_or_raise('minimum_part_size') def get_allowed(self): """ + Return 'allowed' dictionary info. The 'allowed" column was not in the original schema, so it may be NULL. + + :rtype: dict """ allowed_json = self._get_account_info_or_raise('allowed') if allowed_json is None: @@ -336,6 +400,11 @@ def _get_account_info_or_raise(self, column_name): raise MissingAccountData(str(e)) def refresh_entire_bucket_name_cache(self, name_id_iterable): + """ + Refresh names and IDs of buckets + + :param name_id_iterable: an iterable which yields buakcet name and ID + """ with self._get_connection() as conn: conn.execute('DELETE FROM bucket;') for (bucket_name, bucket_id) in name_id_iterable: @@ -345,6 +414,12 @@ def refresh_entire_bucket_name_cache(self, name_id_iterable): ) def save_bucket(self, bucket): + """ + Save bucket info + + :param bucket: a Bucket object + :type: b2sdk.bucket.Bucket + """ with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_id = ?;', (bucket.id_,)) conn.execute( @@ -353,10 +428,23 @@ def save_bucket(self, bucket): ) def remove_bucket_name(self, bucket_name): + """ + Remove bucket info by a given name + + :param bucket_name: a bucket name + :type bucket_name: str + """ with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_name = ?;', (bucket_name,)) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): + """ + Return bucket ID or None by a given name + + :param bucket_name: a bucket name + :type bucket_name: str + :rtype: str, None + """ try: with self._get_connection() as conn: cursor = conn.execute( diff --git a/b2sdk/account_info/upload_url_pool.py b/b2sdk/account_info/upload_url_pool.py index 62adf4424..25ce2a1b2 100644 --- a/b2sdk/account_info/upload_url_pool.py +++ b/b2sdk/account_info/upload_url_pool.py @@ -29,6 +29,13 @@ def __init__(self): def put(self, key, url, auth_token): """ Adds the url and auth token to the pool for the given key. + + :param key: bucket ID or large file ID + :type param: str + :param url: bucket or file URL + :type url: str + :param auth_token: authentication token + :type auth_token: str """ with self._lock: pair = (url, auth_token) @@ -37,6 +44,10 @@ def put(self, key, url, auth_token): def take(self, key): """ Returns (url, auth_token) if one is available, or (None, None) if not. + + :param key: bucket ID or large file ID + :type param: str + :rtype: tuple """ with self._lock: pair_list = self._pool[key] @@ -46,18 +57,37 @@ def take(self, key): return (None, None) def clear_for_key(self, key): + """ + Remove an intem from the pool by key + + :param key: bucket ID or large file ID + :type param: str + """ with self._lock: if key in self._pool: del self._pool[key] class UrlPoolAccountInfo(AbstractAccountInfo): + """ + Holder for all account-related information that needs to be kept + between API calls, and between invocations of the command-line + tool. This includes: account id, application key, auth tokens, + API URL, download URL, and uploads URLs. + + This concrete implementation uses an instance of UploadUrlPool + as an underlying storage + """ + def __init__(self): super(UrlPoolAccountInfo, self).__init__() self._reset_upload_pools() @abstractmethod def clear(self): + """ + Remove all stored information + """ self._reset_upload_pools() return super(UrlPoolAccountInfo, self).clear() @@ -67,20 +97,69 @@ def _reset_upload_pools(self): # bucket upload url def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): + """ + Add an (upload_url, upload_auth_token) pair to the pool available for + the bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + :rtype: tuple + """ self._bucket_uploads.put(bucket_id, upload_url, upload_auth_token) def clear_bucket_upload_data(self, bucket_id): + """ + Removes all upload URLs for the given bucket. + + :param bucket_id: a bucket ID + :type bucket_id: str + """ self._bucket_uploads.clear_for_key(bucket_id) def take_bucket_upload_url(self, bucket_id): + """ + Returns a pair (upload_url, upload_auth_token) that has been removed + from the pool for this bucket, or (None, None) if there are no more + left. + + :param bucket_id: a bucket ID + :type bucket_id: str + :rtype: tuple + """ return self._bucket_uploads.take(bucket_id) # large file upload url def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): + """ + Put large file upload URL into a pool + + :param file_id: a file ID + :type file_id: str + :param upload_url: an upload URL + :type upload_url: str + :param upload_auth_token: an upload authentication token + :type upload_auth_token: str + """ self._large_file_uploads.put(file_id, upload_url, upload_auth_token) def take_large_file_upload_url(self, file_id): + """ + Take large file upload URL from a pool + + :param file_id: a file ID + :type file_id: str + """ return self._large_file_uploads.take(file_id) def clear_large_file_upload_urls(self, file_id): + """ + Clear a pool of URLs for a given file ID + + :param file_id: a file ID + :type file_id: str + """ self._large_file_uploads.clear_for_key(file_id) diff --git a/b2sdk/api.py b/b2sdk/api.py index 06e566b80..638c696f9 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -30,6 +30,14 @@ def url_for_api(info, api_name): + """ + Return URL for an API endpoint + + :param info: account info + :param api_name: + :type api_name: str + :rtype: str + """ if api_name in ['b2_download_file_by_id']: base = info.get_download_url() else: @@ -45,10 +53,10 @@ class B2Api(object): While B2RawApi provides direct access to the B2 web APIs, this class handles several things that simplify the task of uploading and downloading files: - - re-acquires authorization tokens when they expire - - retrying uploads when an upload URL is busy - - breaking large files into parts - - emulating a directory structure (B2 buckets are flat) + - re-acquires authorization tokens when they expire + - retrying uploads when an upload URL is busy + - breaking large files into parts + - emulating a directory structure (B2 buckets are flat) Adds an object-oriented layer on top of the raw API, so that buckets and files returned are Python objects with accessor @@ -61,10 +69,19 @@ class handles several things that simplify the task of uploading def __init__(self, account_info=None, cache=None, raw_api=None, max_upload_workers=10): """ Initializes the API using the given account info. - :param account_info: - :param cache: - :param raw_api: - :return: + + :param account_info: an instance of :class:`~b2sdk.account_info.upload_url_pool.UrlPoolAccountInfo`, + or any custom class derived from + :class:`~b2sdk.account_info.abstract.AbstractAccountInfo` + :param cache: an instance of the one of the following classes: + :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, + :class:`~b2sdk.cache.AuthInfoCache`, + or any custom class derived from :class:`~b2sdk.cache.AbstractCache` + :param raw_api: an instance of one of the following classes: + :class:`~b2sdk.raw_api.B2RawApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, + or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` + :param max_upload_workers: a number of upload threads, default is 10 + :type max_upload_workers: int """ self.raw_api = raw_api or B2RawApi(B2Http()) if account_info is None: @@ -86,6 +103,9 @@ def set_thread_pool_size(self, max_workers): Must be called before any work starts, or the thread pool will get the default size of 1. + + :param max_workers: maximum allowed number of workers in a pool + :type max_workers: int """ if self.upload_executor is not None: raise Exception('thread pool already created') @@ -100,6 +120,10 @@ def get_thread_pool(self): return self.upload_executor def authorize_automatically(self): + """ + Perform automatic account authorization, retrieving all account data + from account info object passed during initialization + """ try: self.authorize_account( self.account_info.get_realm(), @@ -112,6 +136,16 @@ def authorize_automatically(self): @limit_trace_arguments(only=('self', 'realm')) def authorize_account(self, realm, account_id_or_key_id, application_key): + """ + Perform account authorization + + :param realm: a realm to authiroze account in + :type realm: str + :param account_id_or_key_id: account or key ID + :type account_id_or_key_id: str + :param application_key: user's application key + :type application_key: str + """ # Clean up any previous account info if it was for a different account. try: old_account_id = self.account_info.get_account_id() @@ -140,6 +174,12 @@ def authorize_account(self, realm, account_id_or_key_id, application_key): ) def get_account_id(self): + """ + Return account ID + + :return: account ID + :rtype: str + """ return self.account_info.get_account_id() # buckets @@ -147,6 +187,22 @@ def get_account_id(self): def create_bucket( self, name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None ): + """ + Create a bucket + + :param name: bucket name + :type name: str + :param bucket_type: a bucket type, could be one of the following values: "allPublic", "allPrivate" + :type bucket_type: str + :param bucket_info: additional bucket info to store with the bucket + :type bucket_info: dict + :param cors_rules: bucket CORS rules to store with the bucket + :type cors_rules: dict + :param lifecycle_rules: bucket lifecycle rules to store with the bucket + :type lifecycle_rules: dict + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket + """ account_id = self.account_info.get_account_id() response = self.session.create_bucket( @@ -168,6 +224,26 @@ def create_bucket( return bucket def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None): + """ + Download a file with a given ID + + :param file_id: a file ID + :type file_id: str + :param download_dest: a local file path + :type download_dest: str + :param progress_listener: an instance of the one of the following classes: \ + :class:`~b2sdk.bucket.PartProgressReporter`,\ + :class:`~b2sdk.progress.TqdmProgressListener`,\ + :class:`~b2sdk.progress.SimpleProgressListener`,\ + :class:`~b2sdk.progress.DoNothingProgressListener`,\ + :class:`~b2sdk.progress.ProgressListenerForTest`,\ + :class:`~b2sdk.report.SyncFileReporter`,\ + or any sub class of :class:`~b2sdk.progress.AbstractProgressListener` + :param range_: a list of two integers, the first one is a start\ + position, and the second oe is the end position in the file + :type range_: list + :return: context manager that returns an object that supports iter_content() + """ url = self.session.get_download_url_by_id( file_id, url_factory=self.account_info.get_download_url, @@ -175,6 +251,14 @@ def download_file_by_id(self, file_id, download_dest, progress_listener=None, ra return self.transferer.download_file_from_url(url, download_dest, progress_listener, range_) def get_bucket_by_id(self, bucket_id): + """ + Return bucket object with a given ID + + :param bucket_id: a bucket ID + :type bucket_id: str + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket + """ return Bucket(self, bucket_id) def get_bucket_by_name(self, bucket_name): @@ -182,8 +266,10 @@ def get_bucket_by_name(self, bucket_name): Returns the Bucket for the given bucket_name. :param bucket_name: The name of the bucket to return. + :type bucket_name: str :return: a Bucket object - :raises NonExistentBucket: if the bucket does not exist in the account + :rtype: b2sdk.bucket.Bucket + :raises b2sdk.exception.NonExistentBucket: if the bucket does not exist in the account """ # Give a useful warning if the current application key does not # allow access to the named bucket. @@ -208,6 +294,9 @@ def delete_bucket(self, bucket): For legacy reasons it returns whatever server sends in response, but API user should not rely on the response: if it doesn't raise an exception, it means that the operation was a success + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket """ account_id = self.account_info.get_account_id() return self.session.delete_bucket(account_id, bucket.id_) @@ -222,8 +311,10 @@ def list_buckets(self, bucket_name=None): one bucket, you must specify the bucket name, or the request will be unauthorized. - :param bucket_name: Optional: the name of the one bucket to return. - :return: A list of Bucket objects. + :param bucket_name: the name of the one bucket to return. + :type bucket_name: str + :return: A list of instances of b2sdk.bucket.Bucket. + :rtype: list """ # Give a useful warning if the current application key does not # allow access to the named bucket. @@ -251,8 +342,11 @@ def list_parts(self, file_id, start_part_number=None, batch_size=None): Generator that yields a Part for each of the parts that have been uploaded. :param file_id: the ID of the large file that is not finished + :type file_id: str :param start_part_number: the first part number to return. defaults to the first part. + :type start_part_number: int :param batch_size: the number of parts to fetch at a time from the server + :type batch_size: int """ batch_size = batch_size or 100 while True: @@ -265,10 +359,24 @@ def list_parts(self, file_id, start_part_number=None, batch_size=None): # delete/cancel def cancel_large_file(self, file_id): + """ + Cancel a large file upload + + :param file_id: a file ID + :type file_id: str + """ response = self.session.cancel_large_file(file_id) return FileVersionInfoFactory.from_cancel_large_file_response(response) def delete_file_version(self, file_id, file_name): + """ + Permanently and irrevocably delete one version of a file + + :param file_id: a file ID + :type: file_id: str + :param file_name: a file name + :type file_name: str + """ # filename argument is not first, because one day it may become optional response = self.session.delete_file_version(file_id, file_name) assert response['fileId'] == file_id @@ -277,12 +385,23 @@ def delete_file_version(self, file_id, file_name): # download def get_download_url_for_fileid(self, file_id): + """ + Returns a URL to download the given file by ID. + + :param file_id: a file ID + :type file_id: str + """ url = url_for_api(self.account_info, 'b2_download_file_by_id') return '%s?fileId=%s' % (url, file_id) def get_download_url_for_file_name(self, bucket_name, file_name): """ Returns a URL to download the given file by name. + + :param bucket_name: a bucket name + :type bucket_name: str + :param file_name: a file name + :type file_name: str """ self.check_bucket_restrictions(bucket_name) return '%s/file/%s/%s' % ( @@ -293,6 +412,20 @@ def get_download_url_for_file_name(self, bucket_name, file_name): def create_key( self, capabilities, key_name, valid_duration_seconds=None, bucket_id=None, name_prefix=None ): + """ + Create a new application key + + :param capabilities: a list of capabilities + :type capabilities: list + :param key_name: a name of a key + :type key_name: str + :param valid_duration_seconds: key duration in seconds + :type valid_duration_seconds: int + :param bucket_id: a bucket ID + :type bucket_id: str + :param name_prefix: a name prefix + :type name_prefix: str + """ account_id = self.account_info.get_account_id() response = self.session.create_key( @@ -310,11 +443,23 @@ def create_key( return response def delete_key(self, application_key_id): + """ + Delete application key with a given ID + + :param application_key_id: an application key ID + :type application_key_id: + """ response = self.session.delete_key(application_key_id=application_key_id) return response def list_keys(self, start_application_key_id=None): + """ + List application keys + + :param start_application_key_id: an application key ID to start from + :type start_application_key_id: str + """ account_id = self.account_info.get_account_id() return self.session.list_keys( @@ -333,6 +478,9 @@ def check_bucket_restrictions(self, bucket_name): If it does, does the bucket_name for a given api call match that. If not it raises a RestrictedBucket error. + + :param bucket_name: a bucket name + :type bucket_name: str """ allowed = self.account_info.get_allowed() allowed_bucket_name = allowed['bucketName'] diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index f92029c6d..79ef1c363 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -32,6 +32,9 @@ def _print_exception(e, indent=''): """ Used for debugging to print out nested exception structures. + + :param indent: message prefix + :type indent: str """ print(indent + 'EXCEPTION', repr(e)) print(indent + 'CLASS', type(e)) @@ -45,6 +48,9 @@ def _translate_errors(fcn, post_params=None): """ Calls the given function, turning any exception raised into the right kind of B2Error. + + :param post_params: request parameters + :type post_params: dict """ try: response = fcn() @@ -103,6 +109,11 @@ def _translate_and_retry(fcn, try_count, post_params=None): """ Try calling fcn try_count times, retrying only if the exception is a retryable B2Error. + + :param try_count: a number of retries + :type try_count: int + :param post_params: request parameters + :type post_params: dict """ # For all but the last try, catch the exception. wait_time = 1.0 @@ -147,9 +158,12 @@ def pre_request(self, method, url, headers): Raises an exception if this request should not be processed. The exception raised must inherit from B2HttpCallbackPreRequestException. - :param method: One of: 'POST', 'GET', etc. + :param method: str, One of: 'POST', 'GET', etc. + :type method: str :param url: The URL that will be used. + :type url: str :param headers: The header sent with the request. + :type headers: dict """ @@ -161,10 +175,13 @@ def post_request(self, method, url, headers, response): Raises an exception if this request should be treated as failing. The exception raised must inherit from B2HttpCallbackPostRequestException. - :param method: One of: 'POST', 'GET', etc. - :param url: The URL that will be used. - :param headers: The header sent with the request. - :param response: A response object from the requests library. + :param method: one of: 'POST', 'GET', etc. + :type method: str + :param url: the URL that will be used. + :type url: str + :param headers: the header sent with the request. + :type headers: dict + :param response: a response object from the requests library. """ @@ -175,6 +192,14 @@ def post_request(self, method, url, headers, response): clock on the local host. The Date header contains a string that looks like: "Fri, 16 Dec 2016 20:52:30 GMT". + + :param method: one of: 'POST', 'GET', etc. + :type method: str + :param url: the URL that will be used. + :type url: str + :param headers: the header sent with the request. + :type headers: dict + :param response: a response object from the requests library. """ # Make a string that uses month numbers instead of month names server_date_str = response.headers['Date'] @@ -216,11 +241,14 @@ class B2Http(object): The methods that return JSON either return a Python dict or raise a subclass of B2Error. They can be used like this: - try: - response_dict = b2_http.post_json_return_json(url, headers, params) - ... - except B2Error as e: - ... + .. code-block:: python + + try: + response_dict = b2_http.post_json_return_json(url, headers, params) + ... + except B2Error as e: + ... + """ def __init__(self, requests_module=None, install_clock_skew_hook=True): @@ -228,8 +256,9 @@ def __init__(self, requests_module=None, install_clock_skew_hook=True): Initialize with a reference to the requests module, which makes it easy to mock for testing. - The optional after_request_hook is called on the Response - object after every request that doesn't throw an exception. + :param requests_module: a reference to requests module + :param install_clock_skew_hook: if True, install a clock skew hook + :type install_clock_skew_hook: bool """ requests_to_use = requests_module or requests self.session = requests_to_use.Session() @@ -240,6 +269,9 @@ def __init__(self, requests_module=None, install_clock_skew_hook=True): def add_callback(self, callback): """ Adds a callback that inherits from HttpCallback. + + :param callback: a callback to be added to a chain + :type callback: callable """ self.callbacks.append(callback) @@ -247,16 +279,21 @@ def post_content_return_json(self, url, headers, data, try_count=1, post_params= """ Use like this: - try: - response_dict = b2_http.post_content_return_json(url, headers, data) - ... - except B2Error as e: - ... + .. code-block:: python + + try: + response_dict = b2_http.post_content_return_json(url, headers, data) + ... + except B2Error as e: + ... :param url: URL to call - :param headers: Headers to send. + :type url: str + :param headers: headers to send. + :type headers: dict :param data: bytes (Python 3) or str (Python 2), or a file-like object, to send :return: a dict that is the decoded JSON + :rtype: dict """ # Make the headers we'll send by adding User-Agent to what # the caller provided. Make a copy before modifying. @@ -287,15 +324,20 @@ def post_json_return_json(self, url, headers, params, try_count=1): """ Use like this: - try: - response_dict = b2_http.post_json_return_json(url, headers, params) - ... - except B2Error as e: - ... + .. code-block:: python + + try: + response_dict = b2_http.post_json_return_json(url, headers, params) + ... + except B2Error as e: + ... :param url: URL to call - :param headers: Headers to send. - :param params: A dict that will be converted to JSON + :type url: str + :param headers: headers to send. + :type headers: dict + :param params: a dict that will be converted to JSON + :type params: dict :return: a dict that is the decoded JSON """ data = six.BytesIO(six.b(json.dumps(params))) @@ -307,19 +349,25 @@ def get_content(self, url, headers, try_count=1): Use like this: - try: - with b2_http.get_content(url, headers) as response: - for byte_data in response.iter_content(chunk_size=1024): - ... - except B2Error as e: - ... + .. code-block:: python + + try: + with b2_http.get_content(url, headers) as response: + for byte_data in response.iter_content(chunk_size=1024): + ... + except B2Error as e: + ... The response object is only guarantee to have: - headers - iter_content() :param url: URL to call - :param headers: Headers to send + :type url: str + :param headers: headers to send + :type headers: dict + :param try_count: a number or retries + :type try_count: int :return: Context manager that returns an object that supports iter_content() """ # Make the headers we'll send by adding User-Agent to what diff --git a/b2sdk/bounded_queue_executor.py b/b2sdk/bounded_queue_executor.py index 8becd3546..4a73f3506 100644 --- a/b2sdk/bounded_queue_executor.py +++ b/b2sdk/bounded_queue_executor.py @@ -13,7 +13,7 @@ class BoundedQueueExecutor(object): """ - Wraps a futures.Executor and limits the number of requests that + Wraps a concurrent.futures.Executor and limits the number of requests that can be queued at once. Requests to submit() tasks block until there is room in the queue. @@ -26,11 +26,25 @@ class BoundedQueueExecutor(object): """ def __init__(self, executor, queue_limit): + """ + :param executor: an executor to be wrapped + :type executor: concurrent.futures.Executor + :param queue_limit: a queue limit + :type queue_limit: int + """ self.executor = executor self.semaphore = threading.Semaphore(queue_limit) self.num_exceptions = 0 def submit(self, fcn, *args, **kwargs): + """ + Start execution of a callable with the given optional and positional arguments + + :param fcn: a callable object + :type fcn: callable + :return: a future object + :rtype: concurrent.futures.Future + """ # Wait until there is room in the queue. self.semaphore.acquire() @@ -49,7 +63,15 @@ def run_it(): return self.executor.submit(run_it) def shutdown(self): + """ + Shutdown an executor + """ self.executor.shutdown() def get_num_exceptions(self): + """ + Return a number of exceptions + + :rtype: int + """ return self.num_exceptions diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 9bad4a954..f90c05fe1 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -38,6 +38,10 @@ class LargeFileUploadState(object): """ def __init__(self, file_progress_listener): + """ + :param file_progress_listener: a progress listener object + :type file_progress_listener: b2sdk.progress.AbstractProgressListener + """ self.lock = threading.RLock() self.error_message = None self.file_progress_listener = file_progress_listener @@ -45,19 +49,42 @@ def __init__(self, file_progress_listener): self.bytes_completed = 0 def set_error(self, message): + """ + Set error message + + :param message: an error message + :type message: str + """ with self.lock: self.error_message = message def has_error(self): + """ + Check whether an error occured + + :rtype: bool + """ with self.lock: return self.error_message is not None def get_error_message(self): + """ + Get an error message + + :return: an error message + :rtype: str + """ with self.lock: assert self.has_error() return self.error_message def update_part_bytes(self, bytes_delta): + """ + Update listener progress info + + :param bytes_delta: number of bytes to increase a progress for + :type bytes_delta: int + """ with self.lock: self.bytes_completed += bytes_delta self.file_progress_listener.bytes_completed(self.bytes_completed) @@ -75,18 +102,37 @@ class PartProgressReporter(AbstractProgressListener): """ def __init__(self, large_file_upload_state, *args, **kwargs): + """ + :param large_file_upload_state: LargeFileUploadState object + :type large_file_upload_state: b2sdk.bucket.LargeFileUploadState + """ super(PartProgressReporter, self).__init__(*args, **kwargs) self.large_file_upload_state = large_file_upload_state self.prev_byte_count = 0 def bytes_completed(self, byte_count): + """ + Update bytes completed value + + :param bytes_count: a value to increase progress to + :type bytes_count: int + """ self.large_file_upload_state.update_part_bytes(byte_count - self.prev_byte_count) self.prev_byte_count = byte_count def close(self): + """ + Perform cleanup operations + """ pass def set_total_bytes(self, total_byte_count): + """ + Set total bytes value + + :param total_bytes_count: + :type total_bytes_count: int + """ pass @@ -112,6 +158,26 @@ def __init__( revision=None, bucket_dict=None ): + """ + :param api: an API object + :type api: b2sdk.api.B2Api + :param id_: a bucket id + :type id_: str + :param name: a bucket name + :type name: str + :param type_: a bucket type + :type type_: str + :param bucket_info: an info to store with a bucket + :type bucket_info: dict + :param cors_rules: CORS rules to store with a bucket + :type cors_rules: dict + :param lifecycle_rules: lifecycle rules to store with a bucket + :type lifecycle_rules: dict + :param revision: a bucket revision number + :type revision: int + :param bucket_dict: a dictionary which contains bucket parameters + :type bucket_dict: dict + """ self.api = api self.id_ = id_ self.name = name @@ -123,12 +189,33 @@ def __init__( self.bucket_dict = bucket_dict or {} def get_id(self): + """ + Return bucket ID + + :type: str + """ return self.id_ def set_info(self, new_bucket_info, if_revision_is=None): + """ + Update bucket info + + :param new_bucket_info: new bucket info dictionary + :type new_bucket_info: dict + :param if_revision_is: revision number, + update the info if revision equals to + if_revision_is' value + :type if_revision_is: int + """ return self.update(bucket_info=new_bucket_info, if_revision_is=if_revision_is) def set_type(self, bucket_type): + """ + Update bucket type + + :param bucket_type: a bucket type + :type bucket_type: str + """ return self.update(bucket_type=bucket_type) def update( @@ -139,6 +226,22 @@ def update( lifecycle_rules=None, if_revision_is=None ): + """ + Update various bucket parameters + + :param bucket_type: a bucket type + :type bucket_type: str + :param bucket_info: an info to store with a bucket + :type bucket_info: dict + :param cors_rules: CORS rules to store with a bucket + :type cors_rules: dict + :param lifecycle_rules: lifecycle rules to store with a bucket + :type lifecycle_rules: dict + :param if_revision_is: revision number, + update the info if revision equals to + if_revision_is' value + :type if_revision_is: int + """ account_id = self.api.account_info.get_account_id() return self.api.session.update_bucket( account_id, @@ -151,15 +254,45 @@ def update( ) def cancel_large_file(self, file_id): + """ + Cancel large file transfer + + :param file_id: a file ID + :type file_id: str + """ return self.api.cancel_large_file(file_id) def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None): - # download_file_by_id actually belongs in B2Api, not in Bucket, we just provide a convenient redirect here + """ + Download a file by ID, download_file_by_id actually belongs in B2Api, + not in Bucket, we just provide a convenient redirect here + + :param file_id: a file ID + :type file_id: str + :param download_dest: a local file path + :type download_dest: str + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener + :param range_: a list of two integer values, start and end offsets + :type range_: list + """ return self.api.download_file_by_id( file_id, download_dest, progress_listener, range_=range_ ) def download_file_by_name(self, file_name, download_dest, progress_listener=None, range_=None): + """ + Download a file by name + + :param file_id: a file ID + :type file_id: str + :param download_dest: a local file path + :type download_dest: str + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener + :param range_: a list of two integer values, start and end offsets + :type range_: list + """ url = self.api.session.get_download_url_by_name( self.name, file_name, @@ -170,23 +303,36 @@ def download_file_by_name(self, file_name, download_dest, progress_listener=None ) def get_download_authorization(self, file_name_prefix, valid_duration_in_seconds): + """ + Return an authorization token that is valid only for downloading + files from the given bucket + + :param file_name_prefix: a file name prefix, only files that match it could be downloaded + :type file_name_prefix: str + :param valid_duration_in_seconds: a token is valid only during this amount of seconds + :type valid_duration_in_seconds: int + """ response = self.api.session.get_download_authorization( self.id_, file_name_prefix, valid_duration_in_seconds ) return response['authorizationToken'] def list_parts(self, file_id, start_part_number=None, batch_size=None): + """ + Get a list of all parts that have been uploaded for a given file + + :param file_id: a file ID + :type file_id: str + :param start_part_number: the first part number to return. defaults to the first part. + :type start_part_number: int + :param batch_size: the number of parts to fetch at a time from the server + :type batch_size: int + """ return self.api.list_parts(file_id, start_part_number, batch_size) - def ls( - self, - folder_to_list='', - show_versions=False, - max_entries=None, - recursive=False, - fetch_count=100 - ): - """Pretends that folders exist, and yields the information about the files in a folder. + def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_count=None): + """ + Pretends that folders exist, and yields the information about the files in a folder. B2 has a flat namespace for the files in a bucket, but there is a convention of using "/" as if there were folders. This method searches through the @@ -196,14 +342,18 @@ def ls( When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. - :param folder: The name of the folder to list. Must not start with "/". - Empty string means top-level folder. - :param show_versions: When true returns info about all versions of a file, + :param folder_to_list: the name of the folder to list. Must not start with "/". + Empty string means top-level folder. + :type folder_to_list: str + :param show_versions: when true returns info about all versions of a file, when false, just returns info about the most recent versions. - :param max_entries: How many entries to return. 1 - 1000 - :param recursive: - :return: + :type show_versions: bool + :param recursive: if True, list folders recursively + :type recursive: bool + :param fetch_count: how many entries to return. 1 - 1000 + :type fetch_count: int + :rtype: generator """ # Every file returned must have a name that starts with the # folder name and a "/". @@ -287,6 +437,12 @@ def list_unfinished_large_files(self, start_file_id=None, batch_size=None): A generator that yields an UnfinishedLargeFile for each unfinished large file in the bucket, starting at the given file. + + :param start_file_id: a file ID to start with + :type start_file_id: str + :param batch_size: max file count + :type batch_size: int + :rtype: generator """ batch_size = batch_size or 100 while True: @@ -300,6 +456,16 @@ def list_unfinished_large_files(self, start_file_id=None, batch_size=None): break def start_large_file(self, file_name, content_type=None, file_info=None): + """ + Start large file transfer + + :param file_name: a file name + :type file_name: str + :param content_type: request content type + :type content_type: str + :param file_info: a file info to store with the file + :type file_info: dict + """ return UnfinishedLargeFile( self.api.session.start_large_file(self.id_, file_name, content_type, file_info) ) @@ -310,6 +476,17 @@ def upload_bytes( ): """ Upload bytes in memory to a B2 file + + :param data_bytes: a byte array to upload + :type data_bytes: bytes + :param file_name: a file name to upload bytes to + :type file_name: str + :param content_type: request content type + :type content_type: str + :param file_infos: a file info to store with the file + :type file_infos: dict + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener """ upload_source = UploadSourceBytes(data_bytes) return self.upload( @@ -332,6 +509,21 @@ def upload_local_file( ): """ Uploads a file on local disk to a B2 file. + + :param local_file: a path to a file on local disk + :type local_file: str + :param file_name: a file name of the new B2 file + :type file_name: str + :param content_type: request content type + :type content_type: str + :param file_infos: a file info to store with the file + :type file_infos: dict + :param sha1_sum: file SHA1 hash + :type sha1_sum: str + :param min_part_size: a minimum size of a part + :type min_part_size: int + :param progress_listener: a progress listener object + :type progress_listener: b2sdk.progress.AbstractProgressListener """ upload_source = UploadSourceLocalFile(local_path=local_file, content_sha1=sha1_sum) return self.upload( @@ -360,12 +552,17 @@ def upload( file whose read() method returns bytes. :param upload_source: an UploadSource object that opens the source of the upload + :type upload_source: b2sdk.upload_source.UploadSource :param file_name: the file name of the new B2 file + :type file_name: str :param content_type: the MIME type, or None to accept the default based on file extension of the B2 file name + :type content_type: str, None :param file_infos: custom file info to be stored with the file + :type file_infos: dict :param min_part_size: the smallest part size to use + :type min_part_size: int :param progress_listener: object to notify as data is transferred - :return: + :type progress_listener: b2sdk.progress.AbstractProgressListener The function `opener` should return a file-like object, and it must be possible to call it more than once in case the upload @@ -608,6 +805,14 @@ def _get_upload_part_data(self, file_id): return (response['uploadUrl'], response['authorizationToken']) def get_download_url(self, filename): + """ + Get file download URL + + :param filename: a file name + :type filename: str + :return: file download URL + :rtype: str + """ return "%s/file/%s/%s" % ( self.api.account_info.get_download_url(), b2_url_encode(self.name), @@ -615,15 +820,36 @@ def get_download_url(self, filename): ) def hide_file(self, file_name): + """ + Hide a file + + :param file_name: a file name + :type file_name: str + :return: file version info + :rtype: b2sdk.file_version.FileVersionInfoFactory + """ response = self.api.session.hide_file(self.id_, file_name) return FileVersionInfoFactory.from_api_response(response) def delete_file_version(self, file_id, file_name): + """ + Delete file version + + :param file_id: a file ID + :type file_id: str + :param file_name: a file name + :type file_name: str + """ # filename argument is not first, because one day it may become optional return self.api.delete_file_version(file_id, file_name) @disable_trace def as_dict(self): # TODO: refactor with other as_dict() + """ + Return bucket representation as a dictionary + + :rtype: dict + """ result = { 'accountId': self.api.account_info.get_account_id(), 'bucketId': self.id_, @@ -639,23 +865,49 @@ def __repr__(self): class BucketFactory(object): + """ + This is a factory for creating bucket objects from another kind of objects + """ + @classmethod def from_api_response(cls, api, response): + """ + Create a Bucket object from API response + + :param api: API object + :type api: b2sdk.api.B2Api + :param response: response object + :type response: object + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket + """ return [cls.from_api_bucket_dict(api, bucket_dict) for bucket_dict in response['buckets']] @classmethod def from_api_bucket_dict(cls, api, bucket_dict): """ - turns this: - { - "bucketType": "allPrivate", - "bucketId": "a4ba6a39d8b6b5fd561f0010", - "bucketName": "zsdfrtsazsdfafr", - "accountId": "4aa9865d6f00", - "bucketInfo": {}, - "revision": 1 - } - into a Bucket object + Turns a dictionary, like this: + + .. code-block:: python + + { + "bucketType": "allPrivate", + "bucketId": "a4ba6a39d8b6b5fd561f0010", + "bucketName": "zsdfrtsazsdfafr", + "accountId": "4aa9865d6f00", + "bucketInfo": {}, + "revision": 1 + } + + into a Bucket object + + :param api: API object + :type api: b2sdk.api.B2Api + :param bucket_dict: a dictionary with bucket properties + :type bucket_dict: dict + :return: a Bucket object + :rtype: b2sdk.bucket.Bucket + """ bucket_name = bucket_dict['bucketName'] bucket_id = bucket_dict['bucketId'] diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index e16302d22..70031f321 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -63,26 +63,35 @@ class FileVersionInfoFactory(object): @classmethod def from_api_response(cls, file_info_dict, force_action=None): """ - turns this: - { - "action": "hide", - "fileId": "4_zBucketName_f103b7ca31313c69c_d20151230_m030117_c001_v0001015_t0000", - "fileName": "randomdata", - "size": 0, - "uploadTimestamp": 1451444477000 - } - or this: - { - "accountId": "4aa9865d6f00", - "bucketId": "547a2a395826655d561f0010", - "contentLength": 1350, - "contentSha1": "753ca1c2d0f3e8748320b38f5da057767029a036", - "contentType": "application/octet-stream", - "fileId": "4_z547a2a395826655d561f0010_f106d4ca95f8b5b78_d20160104_m003906_c001_v0001013_t0005", - "fileInfo": {}, - "fileName": "randomdata" - } - into a FileVersionInfo object + turns this: + + .. code-block:: python + + { + "action": "hide", + "fileId": "4_zBucketName_f103b7ca31313c69c_d20151230_m030117_c001_v0001015_t0000", + "fileName": "randomdata", + "size": 0, + "uploadTimestamp": 1451444477000 + } + + or this: + + .. code-block:: python + + { + "accountId": "4aa9865d6f00", + "bucketId": "547a2a395826655d561f0010", + "contentLength": 1350, + "contentSha1": "753ca1c2d0f3e8748320b38f5da057767029a036", + "contentType": "application/octet-stream", + "fileId": "4_z547a2a395826655d561f0010_f106d4ca95f8b5b78_d20160104_m003906_c001_v0001013_t0005", + "fileInfo": {}, + "fileName": "randomdata" + } + + into a FileVersionInfo object + """ assert file_info_dict.get('action') is None or force_action is None, \ 'action was provided by both info_dict and function argument' diff --git a/b2sdk/parse_args.py b/b2sdk/parse_args.py deleted file mode 100644 index b50605f4d..000000000 --- a/b2sdk/parse_args.py +++ /dev/null @@ -1,186 +0,0 @@ -###################################################################### -# -# File: b2/parse_args.py -# -# Copyright 2019 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -import logging - -import six - -from .utils import repr_dict_deterministically - -logger = logging.getLogger(__name__) - - -class Arguments(object): - """ - An object to stick attributes on. - """ - - def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - repr_dict_deterministically(self.__dict__), - ) - - -def check_for_duplicate_args(args_dict): - """ - Checks that no argument name is listed in multiple places. - - Raises a ValueError if there is a problem. - - This args_dict has a problem because 'required' and 'optional' - both contain 'a': - - { - 'option_args': ['b', 'c'], - 'required': ['a', 'd'] - 'optional': ['a', 'e'] - } - """ - categories = sorted(six.iterkeys(args_dict)) - for index_a, category_a in enumerate(categories): - for category_b in categories[index_a + 1:]: - names_a = args_dict[category_a] - names_b = args_dict[category_b] - for common_name in set(names_a) & set(names_b): - raise ValueError( - "argument '%s' is in both '%s' an '%s'" % (common_name, category_a, category_b) - ) - - -def parse_arg_list( - arg_list, option_flags, option_args, list_args, optional_before, required, optional, arg_parser -): - """ - Converts a list of string arguments to an Arguments object, with - one attribute per parameter. - - The value of every parameter is set in the returned Arguments object, - even for parameters not specified on the command line. - - Option Flags set boolean values that default to False. When the - option is present on the command line, the value is set to True. - - Option Args have values provided on the command line. The default - if not present is None. - - List Args act like Option Args, but can be specified more than - once, and their values are collected into a list. Default is []. - - Required positional parameters must be present, and do not have - a double-dash name preceding them. - - Optional positional parameters are just like required parameters, - but don't have to be there and default to None. - - Arg Parser is a dict that maps from a parameter name to a function - tha converts the string argument into the value needed by the - program. These parameters can be Option Args, List Args, Required, - or Optional. - - :param arg_list sys.argv[1:], or equivalent - :param option_flags: Names of options that are boolean flags. - :param option_args: Names of options that have values. - :param list_args: Names of options whose values are collected into a list. - :param optional_before: Names of option positional params that come before the required ones. - :param required: Names of positional params that must be there. - :param optional: Names of optional params. - :param arg_parser: Map from param name to parser for values. - :return: An Argument object, or None if there was any error parsing. - """ - - # Sanity check the inputs. - check_for_duplicate_args( - { - 'option_flags': option_flags, - 'option_args': option_args, - 'optional_before': optional_before, - 'required': required, - 'optional': optional - } - ) - - # Create an object to hold the arguments. - result = Arguments() - - # Set the default value for everything that has a default value. - for name in option_flags: - setattr(result, name, False) - for name in option_args: - setattr(result, name, None) - for name in list_args: - setattr(result, name, []) - for name in optional: - setattr(result, name, None) - - # Make a function for parsing argument values - def parse_arg(name, arg_list): - value = arg_list.pop(0) - if name in arg_parser: - value = arg_parser[name](value) - return value - - # Parse the '--' options - while len(arg_list) != 0 and arg_list[0].startswith('--'): - option = arg_list.pop(0)[2:] - if option in option_flags: - logger.debug('option %s is properly recognized as OPTION_FLAGS', option) - setattr(result, option, True) - elif option in option_args: - if len(arg_list) == 0: - logger.debug( - 'option %s is recognized as OPTION_ARGS and there are no more arguments on arg_list to parse', - option - ) - return None - else: - logger.debug('option %s is properly recognized as OPTION_ARGS', option) - setattr(result, option, parse_arg(option, arg_list)) - elif option in list_args: - if len(arg_list) == 0: - logger.debug( - 'option %s is recognized as LIST_ARGS and there are no more arguments on arg_list to parse', - option - ) - return None - else: - logger.debug('option %s is properly recognized as LIST_ARGS', option) - getattr(result, option).append(parse_arg(option, arg_list)) - else: - logger.error('option %s is of unknown type!', option) - return None - - # Handle optional positional parameters that come first. - # We assume that if there are optional parameters, the - # ones that come before take precedence over the ones - # that come after the required arguments. - for arg_name in optional_before: - if len(required) < len(arg_list): - setattr(result, arg_name, parse_arg(arg_name, arg_list)) - else: - setattr(result, arg_name, None) - - # Parse the positional parameters - for arg_name in required: - if len(arg_list) == 0: - logger.debug('lack of required positional argument: %s', arg_name) - return None - setattr(result, arg_name, parse_arg(arg_name, arg_list)) - for arg_name in optional: - if len(arg_list) != 0: - setattr(result, arg_name, parse_arg(arg_name, arg_list)) - - # Anything left is a problem - if len(arg_list) != 0: - logger.debug('option parser failed to consume this: %s', arg_list) - return None - - # Return the Arguments object - return result diff --git a/b2sdk/progress.py b/b2sdk/progress.py index 4fc320c45..704b66827 100644 --- a/b2sdk/progress.py +++ b/b2sdk/progress.py @@ -14,8 +14,6 @@ import time import hashlib -from .utils import raise_if_shutting_down - # tqdm doesn't work on 2.6 with at least some encodings # on sys.stderr. See: https://github.com/Backblaze/B2_Command_Line_Tool/issues/272 if sys.version_info < (2, 7): @@ -47,6 +45,9 @@ def set_total_bytes(self, total_byte_count): Always called before __enter__ to set the expected total number of bytes. May be called more than once if an upload is retried. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int """ @abstractmethod @@ -55,6 +56,9 @@ def bytes_completed(self, byte_count): Reports that the given number of bytes have been transferred so far. This is not a delta, it is the total number of bytes transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int """ @abstractmethod @@ -81,6 +85,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): class TqdmProgressListener(AbstractProgressListener): + """ + Progress listener based on tqdm library + """ + def __init__(self, description, *args, **kwargs): self.description = description self.tqdm = None # set in set_total_bytes() @@ -88,7 +96,12 @@ def __init__(self, description, *args, **kwargs): super(TqdmProgressListener, self).__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count): - raise_if_shutting_down() + """ + Set the expected total number of bytes. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int + """ if self.tqdm is None: self.tqdm = tqdm( desc=self.description, @@ -100,21 +113,35 @@ def set_total_bytes(self, total_byte_count): ) def bytes_completed(self, byte_count): + """ + Reports that the given number of bytes have been transferred + so far. This is not a delta, it is the total number of bytes + transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int + """ # tqdm doesn't support running the progress bar backwards, # so on an upload retry, it just won't move until it gets # past the point where it failed. - raise_if_shutting_down() if self.prev_value < byte_count: self.tqdm.update(byte_count - self.prev_value) self.prev_value = byte_count def close(self): + """ + Perform clean up operations + """ if self.tqdm is not None: self.tqdm.close() super(TqdmProgressListener, self).close() class SimpleProgressListener(AbstractProgressListener): + """ + Just a simple progress listener which prints info on a console + """ + def __init__(self, description, *args, **kwargs): self.desc = description self.complete = 0 @@ -123,11 +150,23 @@ def __init__(self, description, *args, **kwargs): super(SimpleProgressListener, self).__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count): - raise_if_shutting_down() + """ + Set the expected total number of bytes. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int + """ self.total = total_byte_count def bytes_completed(self, byte_count): - raise_if_shutting_down() + """ + Reports that the given number of bytes have been transferred + so far. This is not a delta, it is the total number of bytes + transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int + """ now = time.time() elapsed = now - self.last_time if 3 <= elapsed and self.total != 0: @@ -138,20 +177,43 @@ def bytes_completed(self, byte_count): self.any_printed = True def close(self): - raise_if_shutting_down() + """ + Perform clean up operations + """ if self.any_printed: print(' DONE.') super(SimpleProgressListener, self).close() class DoNothingProgressListener(AbstractProgressListener): + """ + This listener performs no any output + """ + def set_total_bytes(self, total_byte_count): - raise_if_shutting_down() + """ + Set the expected total number of bytes. + + :param total_byte_count: expected total number of bytes + :type total_byte_count: int + """ + pass def bytes_completed(self, byte_count): - raise_if_shutting_down() + """ + Reports that the given number of bytes have been transferred + so far. This is not a delta, it is the total number of bytes + transferred so far. + + :param byte_count: number of bytes have been transferred + :type byte_count: int + """ + pass def close(self): + """ + Perform clean up operations + """ super(DoNothingProgressListener, self).close() @@ -179,6 +241,15 @@ def get_calls(self): def make_progress_listener(description, quiet): + """ + Returns a progress listener object depending on some conditions + + :param description: listener description + :type description: str + :param quiet: if True, do not output anything + :type quiet: bool + :return: a listener object + """ if quiet: return DoNothingProgressListener() elif tqdm is not None: @@ -194,6 +265,13 @@ class RangeOfInputStream(object): """ def __init__(self, stream, offset, length): + """ + :param stream: a seekable stream + :param offset: offset in the stream + :type offset: int + :param length: max number of bytes to read + :type length: int + """ self.stream = stream self.offset = offset self.remaining = length @@ -206,9 +284,22 @@ def __exit__(self, exc_type, exc_val, exc_tb): return self.stream.__exit__(exc_type, exc_val, exc_tb) def seek(self, pos): + """ + Seek to a given position in the stream + + :param pos: position in the stream + :type pos: int + """ self.stream.seek(self.offset + pos) def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ if size is None: to_read = self.remaining else: @@ -231,8 +322,9 @@ def __init__(self, stream, progress_listener, offset=0): :param stream: the stream to read from or write to :param progress_listener: the listener that we tell about progress + :type progress_listener: b2sdk.progress.AbstractProgressListener :param offset: the starting byte offset in the file - :return: None + :type offset: int """ assert progress_listener is not None self.stream = stream @@ -247,15 +339,36 @@ def __exit__(self, exc_type, exc_val, exc_tb): return self.stream.__exit__(exc_type, exc_val, exc_tb) def seek(self, pos): + """ + Seek to a given position in the stream + + :param pos: position in the stream + :type pos: int + """ return self.stream.seek(pos) def tell(self): + """ + Return current stream position + + :rtype: int + """ return self.stream.tell() def flush(self): + """ + Flush the stream + """ self.stream.flush() def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ if size is None: data = self.stream.read() else: @@ -263,6 +376,11 @@ def read(self, size=None): return data def write(self, data): + """ + Write data to the stream + + :param data: a data to write to the stream + """ self.stream.write(data) def _update(self, delta): @@ -271,14 +389,34 @@ def _update(self, delta): class ReadingStreamWithProgress(AbstractStreamWithProgress): + """ + Wraps a file-like object, updates progress while reading + """ + def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ data = super(ReadingStreamWithProgress, self).read(size) self._update(len(data)) return data class WritingStreamWithProgress(AbstractStreamWithProgress): + """ + Wraps a file-like object, updates progress while writing + """ + def write(self, data): + """ + Write data to the stream + + :param data: a data to write to the stream + """ self._update(len(data)) super(WritingStreamWithProgress, self).write(data) @@ -306,6 +444,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): return self.stream.__exit__(exc_type, exc_val, exc_tb) def seek(self, pos): + """ + Seek to a given position in the stream + + :param pos: position in the stream + :type pos: int + """ assert pos == 0 self.stream.seek(0) self.digest = hashlib.sha1() @@ -313,6 +457,13 @@ def seek(self, pos): self.hash_read = 0 def read(self, size=None): + """ + Read data from the stream + + :param size: number of bytes to read + :type size: int + :return: data read from the stream + """ data = b'' if self.hash is None: # Read some bytes from stream @@ -339,4 +490,9 @@ def read(self, size=None): return data def hash_size(self): + """ + Calculate size of a hash string + + :rtype: int + """ return self.digest.digest_size * 2 diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 22aafccb1..821265543 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -424,10 +424,12 @@ def update_bucket( ) def unprintable_to_hex(self, string): - """Replace unprintable chars in string with a hex representation. + """ + Replace unprintable chars in string with a hex representation. :param string: An arbitrary string, possibly with unprintable characters. - :return The string, with unprintable characters changed to hex (e.g., "\x07") + :return: The string, with unprintable characters changed to hex (e.g., "\x07") + """ unprintables_pattern = re.compile(r'[\x00-\x1f]') diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index b914e1240..d6b2c5658 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -16,7 +16,6 @@ from ..download_dest import DownloadDestLocalFile from ..upload_source import UploadSourceLocalFile -from ..utils import raise_if_shutting_down from ..raw_api import SRC_LAST_MODIFIED_MILLIS from .report import SyncFileReporter @@ -36,7 +35,15 @@ class AbstractAction(object): """ def run(self, bucket, reporter, dry_run=False): - raise_if_shutting_down() + """ + Main action routine + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + :param dry_run: if True, perform a dry run + :type dry_run: bool + """ try: if not dry_run: self.do_action(bucket, reporter) @@ -50,23 +57,49 @@ def run(self, bucket, reporter, dry_run=False): def get_bytes(self): """ Returns the number of bytes to transfer for this action. + + :rtype: int """ @abstractmethod def do_action(self, bucket, reporter): """ Performs the action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors """ @abstractmethod def do_report(self, bucket, reporter): """ Report the action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors """ class B2UploadAction(AbstractAction): + """ + File uploading action + """ + def __init__(self, local_full_path, relative_name, b2_file_name, mod_time_millis, size): + """ + :param local_full_path: a local file path + :type local_full_path: str + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a new remote file + :type b2_file_name: str + :param mod_time_millis: file modification time in milliseconds + :type mod_time_millis: int + :param size: a file size + :type size: int + """ self.local_full_path = local_full_path self.relative_name = relative_name self.b2_file_name = b2_file_name @@ -74,9 +107,21 @@ def __init__(self, local_full_path, relative_name, b2_file_name, mod_time_millis self.size = size def get_bytes(self): + """ + Return file size + + :rtype: int + """ return self.size def do_action(self, bucket, reporter): + """ + Performs the uploading action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ bucket.upload( UploadSourceLocalFile(self.local_full_path), self.b2_file_name, @@ -85,6 +130,13 @@ def do_action(self, bucket, reporter): ) def do_report(self, bucket, reporter): + """ + Report the uploading action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ reporter.print_completion('upload ' + self.relative_name) def __str__(self): @@ -95,16 +147,42 @@ def __str__(self): class B2HideAction(AbstractAction): def __init__(self, relative_name, b2_file_name): + """ + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a remote file + :type b2_file_name: str + """ self.relative_name = relative_name self.b2_file_name = b2_file_name def get_bytes(self): + """ + Return file size + + :return: always zero + :rtype: int + """ return 0 def do_action(self, bucket, reporter): + """ + Performs the hiding action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ bucket.hide_file(self.b2_file_name) def do_report(self, bucket, reporter): + """ + Report the hiding action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ reporter.update_transfer(1, 0) reporter.print_completion('hide ' + self.relative_name) @@ -116,6 +194,20 @@ class B2DownloadAction(AbstractAction): def __init__( self, relative_name, b2_file_name, file_id, local_full_path, mod_time_millis, file_size ): + """ + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a remote file + :type b2_file_name: str + :param file_id: a file ID + :type file_id: str + :param local_full_path: a local file path + :type local_full_path: str + :param mod_time_millis: file modification time in milliseconds + :type mod_time_millis: int + :param file_size: a file size + :type file_size: int + """ self.relative_name = relative_name self.b2_file_name = b2_file_name self.file_id = file_id @@ -124,9 +216,21 @@ def __init__( self.file_size = file_size def get_bytes(self): + """ + Return file size + + :rtype: int + """ return self.file_size def do_action(self, bucket, reporter): + """ + Performs the downloading action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ # Make sure the directory exists parent_dir = os.path.dirname(self.local_full_path) if not os.path.isdir(parent_dir): @@ -150,6 +254,13 @@ def do_action(self, bucket, reporter): os.rename(download_path, self.local_full_path) def do_report(self, bucket, reporter): + """ + Report the downloading action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ reporter.print_completion('dnload ' + self.relative_name) def __str__(self): @@ -161,18 +272,48 @@ def __str__(self): class B2DeleteAction(AbstractAction): def __init__(self, relative_name, b2_file_name, file_id, note): + """ + :param relative_name: a relative file name + :type relative_name: str + :param b2_file_name: a name of a remote file + :type b2_file_name: str + :param file_id: a file ID + :type file_id: str + :param note: a deletion note + :type note: str + """ self.relative_name = relative_name self.b2_file_name = b2_file_name self.file_id = file_id self.note = note def get_bytes(self): + """ + Return file size + + :return: always zero + :rtype: int + """ return 0 def do_action(self, bucket, reporter): + """ + Performs the deleting action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ bucket.api.delete_file_version(self.file_id, self.b2_file_name) def do_report(self, bucket, reporter): + """ + Report the deleting action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name + ' ' + self.note) @@ -182,16 +323,43 @@ def __str__(self): class LocalDeleteAction(AbstractAction): def __init__(self, relative_name, full_path): + """ + :param relative_name: a relative file name + :type relative_name: str + :param full_path: a full local path + :type: str + """ self.relative_name = relative_name self.full_path = full_path def get_bytes(self): + """ + Return file size + + :return: always zero + :rtype: int + """ return 0 def do_action(self, bucket, reporter): + """ + Performs the deleting of a local file action, + returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ os.unlink(self.full_path) def do_report(self, bucket, reporter): + """ + Report the deleting of a local file action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name) diff --git a/b2sdk/sync/exception.py b/b2sdk/sync/exception.py index 3067020c8..d8dda8b0e 100644 --- a/b2sdk/sync/exception.py +++ b/b2sdk/sync/exception.py @@ -12,7 +12,17 @@ class EnvironmentEncodingError(B2Error): + """ + Raised when a file name can not be decoded with system encoding + """ + def __init__(self, filename, encoding): + """ + :param filename: an encoded file name + :type filename: str, bytes + :param encoding: file name encoding + :type encoding: str + """ super(EnvironmentEncodingError, self).__init__() self.filename = filename self.encoding = encoding diff --git a/b2sdk/sync/file.py b/b2sdk/sync/file.py index 7108379b0..ce67de746 100644 --- a/b2sdk/sync/file.py +++ b/b2sdk/sync/file.py @@ -21,10 +21,19 @@ class File(object): """ def __init__(self, name, versions): + """ + :param name: a relative file name + :type name: str + :param versions: a list of file versions + :type versions: list + """ self.name = name self.versions = versions def latest_version(self): + """ + Return the latest file version + """ return self.versions[0] def __repr__(self): @@ -34,14 +43,22 @@ def __repr__(self): class FileVersion(object): """ Holds information about one version of a file: - - id - The B2 file id, or the local full path name - mod_time - modification time, in milliseconds, to avoid rounding issues - with millisecond times from B2 - action - "hide" or "upload" (never "start") """ def __init__(self, id_, file_name, mod_time, action, size): + """ + :param id_: the B2 file id, or the local full path name + :type id_: str + :param file_name: a relative file name + :type file_name: str + :param mod_time: modification time, in milliseconds, to avoid rounding issues + with millisecond times from B2 + :type mod_time: int + :param action: "hide" or "upload" (never "start") + :type action: str + :param size: a file size + :type size: int + """ self.id_ = id_ self.name = file_name self.mod_time = mod_time diff --git a/b2sdk/sync/folder.py b/b2sdk/sync/folder.py index 2d3b51424..df607024d 100644 --- a/b2sdk/sync/folder.py +++ b/b2sdk/sync/folder.py @@ -49,24 +49,37 @@ def all_files(self, reporter, policies_manager): If a file is found, but does not exist (for example due to a broken symlink or a race), reporter will be informed about each such problem. + + :param reporter: a place to report errors + :param policies_manager: a policies manager object """ @abstractmethod def folder_type(self): """ Returns one of: 'b2', 'local' + + :rtype: str """ @abstractmethod def make_full_path(self, file_name): """ Only for local folders, returns the full path to the file. + + :param file_name: a file name + :type file_name: str """ def join_b2_path(b2_dir, b2_name): """ Like os.path.join, but for B2 file names where the root directory is called ''. + + :param b2_dir: a directory path + :type b2_dir: str + :param b2_name: a file name + :type b2_name: str """ if b2_dir == '': return b2_name @@ -84,19 +97,37 @@ def __init__(self, root): Initializes a new folder. :param root: Path to the root of the local folder. Must be unicode. + :type root: str """ if not isinstance(root, six.text_type): raise ValueError('folder path should be unicode: %s' % repr(root)) self.root = fix_windows_path_limit(os.path.abspath(root)) def folder_type(self): + """ + Return folder type + + :rtype: str + """ return 'local' def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + """ + Yield all files + + :param reporter: a place to report errors + :param policies_manager: a policy manager object, default is DEFAULT_SCAN_MANAGER + """ for file_object in self._walk_relative_paths(self.root, '', reporter, policies_manager): yield file_object def make_full_path(self, file_name): + """ + Convert a file name into an absolute path + + :param file_name: a file name + :type file_name: str + """ return os.path.join(self.root, file_name.replace('/', os.path.sep)) def ensure_present(self): @@ -229,12 +260,26 @@ class B2Folder(AbstractFolder): """ def __init__(self, bucket_name, folder_name, api): + """ + :param bucket_name: a name of the bucket + :type bucket_name: str + :param folder_name: a folder name + :type folder_name: str + :param api: an API object + :type api: b2sdk.api.B2Api + """ self.bucket_name = bucket_name self.folder_name = folder_name self.bucket = api.get_bucket_by_name(bucket_name) self.prefix = '' if self.folder_name == '' else self.folder_name + '/' def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + """ + Yield all files + + :param reporter: a place to report errors + :param policies_manager: a policies manager object, default is DEFAULT_SCAN_MANAGER + """ current_name = None current_versions = [] for (file_version_info, folder_name) in self.bucket.ls( @@ -267,9 +312,20 @@ def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): yield File(current_name, current_versions) def folder_type(self): + """ + Return folder type + + :rtype: str + """ return 'b2' def make_full_path(self, file_name): + """ + Make an absolute path from a file name + + :param file_name: a file name + :type file_name: str + """ if self.folder_name == '': return file_name else: diff --git a/b2sdk/sync/folder_parser.py b/b2sdk/sync/folder_parser.py index d3b9c545a..d183c3663 100644 --- a/b2sdk/sync/folder_parser.py +++ b/b2sdk/sync/folder_parser.py @@ -21,6 +21,11 @@ def parse_sync_folder(folder_name, api): because the previous sync command didn't use it. Anything else is treated like a local folder. + + :param folder_name: a name of the folder, either local or remote + :type folder_name: str + :param api: an API object + :type api: b2sdk.api.B2Api """ if folder_name.startswith('b2://'): return _parse_bucket_and_folder(folder_name[5:], api) diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 7941f9514..ee86a3191 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -23,10 +23,26 @@ @six.add_metaclass(ABCMeta) class AbstractFileSyncPolicy(object): + """ + Abstract policy class + """ DESTINATION_PREFIX = NotImplemented SOURCE_PREFIX = NotImplemented def __init__(self, source_file, source_folder, dest_file, dest_folder, now_millis, args): + """ + :param source_file: source file object + :type source_file: b2sdk.sync.file.File + :param source_folder: source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File + :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param now_millis: current time in milliseconds + :type now_millis: int + :param args: an object which holds command line arguments + """ self._source_file = source_file self._source_folder = source_folder self._dest_file = dest_file @@ -56,6 +72,13 @@ def files_are_different(cls, source_file, dest_file, args): """ Compare two files and determine if the the destination file should be replaced by the source file. + + :param source_file: source file object + :type source_file: b2sdk.sync.file.File + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File + :param args: an object which holds command line arguments + :rtype: bool """ # Compare using modification time by default @@ -120,6 +143,9 @@ def files_are_different(cls, source_file, dest_file, args): raise CommandError('Invalid option for --compareVersions') def get_all_actions(self): + """ + Yield file actions. + """ if self._should_transfer(): yield self._make_transfer_action() self._transferred = True @@ -145,7 +171,7 @@ def _make_transfer_action(self): class DownPolicy(AbstractFileSyncPolicy): """ - file is synced down (from the cloud to disk) + File is synced down (from the cloud to disk) """ DESTINATION_PREFIX = 'local://' SOURCE_PREFIX = 'b2://' @@ -163,7 +189,7 @@ def _make_transfer_action(self): class UpPolicy(AbstractFileSyncPolicy): """ - file is synced up (from disk the cloud) + File is synced up (from disk the cloud) """ DESTINATION_PREFIX = 'b2://' SOURCE_PREFIX = 'local://' @@ -180,7 +206,7 @@ def _make_transfer_action(self): class UpAndDeletePolicy(UpPolicy): """ - file is synced up (from disk to the cloud) and the delete flag is SET + File is synced up (from disk to the cloud) and the delete flag is SET """ def _get_hide_delete_actions(self): @@ -197,7 +223,7 @@ def _get_hide_delete_actions(self): class UpAndKeepDaysPolicy(UpPolicy): """ - file is synced up (from disk to the cloud) and the keepDays flag is SET + File is synced up (from disk to the cloud) and the keepDays flag is SET """ def _get_hide_delete_actions(self): @@ -212,7 +238,7 @@ def _get_hide_delete_actions(self): class DownAndDeletePolicy(DownPolicy): """ - file is synced down (from the cloud to disk) and the delete flag is SET + File is synced down (from the cloud to disk) and the delete flag is SET """ def _get_hide_delete_actions(self): @@ -226,12 +252,22 @@ def _get_hide_delete_actions(self): class DownAndKeepDaysPolicy(DownPolicy): """ - file is synced down (from the cloud to disk) and the keepDays flag is SET + File is synced down (from the cloud to disk) and the keepDays flag is SET """ pass def make_b2_delete_note(version, index, transferred): + """ + Create a note message for delete action + + :param version: an object which contains file version info + :param index: file version index + :type index: int + :param transferred: if True, file has been transferred, + False otherwise + :type transferred: bool + """ note = '' if version.action == 'hide': note = '(hide marker)' @@ -243,6 +279,16 @@ def make_b2_delete_note(version, index, transferred): def make_b2_delete_actions(source_file, dest_file, dest_folder, transferred): """ Creates the actions to delete files stored on B2, which are not present locally. + + :param source_file: source file object + :type source_file: b2sdk.sync.file.File + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File + :param dest_folder: destination folder + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param transferred: if True, file has been transferred, + False otherwise + :type transferred: bool """ if dest_file is None: # B2 does not really store folders, so there is no need to hide @@ -271,6 +317,20 @@ def make_b2_keep_days_actions( days ago, 15 days ago, and 25 days ago, and the keepDays is 10, only the 25-day old version can be deleted. The 15 day-old version was visible 10 days ago. + + :param source_file: source file object + :type source_file: b2sdk.sync.file.File + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File + :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param transferred: if True, file has been transferred, + False otherwise + :type transferred: bool + :param keep_days: how many days to keep a file + :type keep_days: int + :param now_millis: current time in milliseconds + :type now_millis: int """ deleting = False if dest_file is None: diff --git a/b2sdk/sync/policy_manager.py b/b2sdk/sync/policy_manager.py index d3b600ec1..530e7b019 100644 --- a/b2sdk/sync/policy_manager.py +++ b/b2sdk/sync/policy_manager.py @@ -13,16 +13,47 @@ class SyncPolicyManager(object): + """ + Policy manager, implements a logic to get a correct policy class + and create a policy object based on various parameters + """ + def __init__(self): self.policies = {} # dict<,> def get_policy( self, sync_type, source_file, source_folder, dest_file, dest_folder, now_millis, args ): + """ + Return policy object + + :param sync_type: synchronization type + :type sync_type: str + :param source_file: source file name + :type source_file: str + :param source_folder: a source folder path + :type source_folder: str + :param dest_file: destination file name + :type dest_file: str + :param dest_folder: a destination folder path + :type dest_folder: str + :param now_millis: current time in milliseconds + :type now_millis: int + :param args: an object which holds command line arguments + :return: a policy object + """ policy_class = self.get_policy_class(sync_type, args) return policy_class(source_file, source_folder, dest_file, dest_folder, now_millis, args) def get_policy_class(self, sync_type, args): + """ + Get policy class by a given sync type + + :param sync_type: synchronization type + :type sync_type: str + :param args: an object which holds command line arguments + :return: a policy class + """ if sync_type == 'local-to-b2': if args.delete: return UpAndDeletePolicy diff --git a/b2sdk/sync/report.py b/b2sdk/sync/report.py index d6463fd6a..6a40e0324 100644 --- a/b2sdk/sync/report.py +++ b/b2sdk/sync/report.py @@ -15,7 +15,7 @@ import six from ..progress import AbstractProgressListener -from ..utils import format_and_scale_number, format_and_scale_fraction, raise_if_shutting_down +from ..utils import format_and_scale_number, format_and_scale_fraction logger = logging.getLogger(__name__) @@ -39,6 +39,11 @@ class SyncReport(object): UPDATE_INTERVAL = 0.1 def __init__(self, stdout, no_progress): + """ + :param stdout: standard output file object + :param no_progress: if True, do not show progress + :type no_progress: bool + """ self.stdout = stdout self.no_progress = no_progress self.start_time = time.time() @@ -59,6 +64,9 @@ def __init__(self, stdout, no_progress): self.warnings = [] def close(self): + """ + Perform clean up + """ with self.lock: if not self.no_progress: self._print_line('', False) @@ -73,12 +81,21 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close() def error(self, message): + """ + Print an error, gracefully interleaving it with a progress bar + + :param message: an error message + :type message: str + """ self.print_completion(message) def print_completion(self, message): """ Removes the progress bar, prints a message, and puts the progress bar back. + + :param message: an error message + :type message: str """ with self.lock: if not self.closed: @@ -87,7 +104,6 @@ def print_completion(self, message): self._update_progress() def _update_progress(self): - raise_if_shutting_down() if not self.closed and not self.no_progress: now = time.time() interval = now - self._last_update_time @@ -127,7 +143,9 @@ def _print_line(self, line, newline): Prints a line to stdout. :param line: A string without a \r or \n in it. + :type line: str :param newline: True if the output should move to a new line after this one. + :type newline: bool """ if len(line) < len(self.current_line): line += ' ' * (len(self.current_line) - len(line)) @@ -156,6 +174,9 @@ def _print_line(self, line, newline): def update_local(self, delta): """ Reports that more local files have been found. + + :param delta: number of files found since the last check + :type delta: int """ with self.lock: self.local_file_count += delta @@ -172,12 +193,23 @@ def end_local(self): def update_compare(self, delta): """ Reports that more files have been compared. + + :param delta: number of files compared + :type delta: int """ with self.lock: self.compare_count += delta self._update_progress() def end_compare(self, total_transfer_files, total_transfer_bytes): + """ + Report that the comparison has been finished + + :param total_transfer_files: total number of transferred files + :type total_transfer_files: int + :param total_transfer_bytes: total number of transferred bytes + :type total_transfer_bytes: int + """ with self.lock: self.compare_done = True self.total_transfer_files = total_transfer_files @@ -185,15 +217,35 @@ def end_compare(self, total_transfer_files, total_transfer_bytes): self._update_progress() def update_transfer(self, file_delta, byte_delta): + """ + Update transfer info + + :param file_delta: number of files transferred + :type file_delta: int + :param byte_delta: number of bytes transferred + :type byte_delta: int + """ with self.lock: self.transfer_files += file_delta self.transfer_bytes += byte_delta self._update_progress() def local_access_error(self, path): + """ + Add a file access error message to the list of warnings + + :param path: file path + :type path: str + """ self.warnings.append('WARNING: %s could not be accessed (broken symlink?)' % (path,)) def local_permission_error(self, path): + """ + Add a permission error message to the list of warnings + + :param path: file path + :type path: str + """ self.warnings.append( 'WARNING: %s could not be accessed (no permissions to read?)' % (path,) ) @@ -208,23 +260,44 @@ class SyncFileReporter(AbstractProgressListener): """ def __init__(self, reporter, *args, **kwargs): + """ + :param reporter: a reporter object + """ super(SyncFileReporter, self).__init__(*args, **kwargs) self.bytes_so_far = 0 self.reporter = reporter def close(self): + """ + Perform clean up + """ # no more bytes are done, but the file is done self.reporter.update_transfer(1, 0) def set_total_bytes(self, total_byte_count): + """ + Set total bytes count + + :param total_byte_count: total byte count + :type total_byte_count: int + """ pass def bytes_completed(self, byte_count): + """ + Set bytes completed count + + :param byte_count: total byte count + :type byte_count: int + """ self.reporter.update_transfer(0, byte_count - self.bytes_so_far) self.bytes_so_far = byte_count def sample_sync_report_run(): + """ + Generate sample report + """ import sys sync_report = SyncReport(sys.stdout, False) diff --git a/b2sdk/sync/scan_policies.py b/b2sdk/sync/scan_policies.py index 607477696..885716d2a 100644 --- a/b2sdk/sync/scan_policies.py +++ b/b2sdk/sync/scan_policies.py @@ -21,9 +21,19 @@ class RegexSet(object): """ def __init__(self, regex_iterable): + """ + :param regex_iterable: an interable which yields regexes + """ self._compiled_list = [re.compile(r) for r in regex_iterable] def matches(self, s): + """ + Check whether a string matches any of regular expressions + + :param s: a string to check + :type s: str + :rtype: bool + """ return any(c.match(s) is not None for c in self._compiled_list) @@ -47,6 +57,9 @@ def convert_dir_regex_to_dir_prefix_regex(dir_regex): If the original regex is valid, there are only two cases to consider: either the regex ends in '$' or does not. + + :param dir_regex: a regular expression string or literal + :type dir_regex: str """ if dir_regex.endswith('$'): return dir_regex[:-1] + r'/' @@ -75,6 +88,16 @@ def __init__( include_file_regexes=tuple(), exclude_all_symlinks=False, ): + """ + :param exclude_dir_regexes: a tuple of regexes to exclude directories + :type exclude_dir_regexes: tuple + :param exclude_file_regexes: a tuple of regexes to exclude files + :type exclude_file_regexes: tuple + :param include_file_regexes: a tuple of regexes to include files + :type include_file_regexes: tuple + :param exclude_all_symlinks: if True, exclude all symlinks + :type exclude_all_symlinks: bool + """ self._exclude_dir_set = RegexSet(exclude_dir_regexes) self._exclude_file_because_of_dir_set = RegexSet( map(convert_dir_regex_to_dir_prefix_regex, exclude_dir_regexes) @@ -89,7 +112,9 @@ def should_exclude_file(self, file_path): :param file_path: The path of the file, relative to the root directory being scanned. - :return: True iff excluded. + :type: str + :return: True if excluded. + :rtype: bool """ exclude_because_of_dir = self._exclude_file_because_of_dir_set.matches(file_path) exclude_because_of_file = ( @@ -105,7 +130,8 @@ def should_exclude_directory(self, dir_path): :param dir_path: The path of the directory, relative to the root directory being scanned. The path will never end in '/'. - :return: True iff excluded. + :type dir_path: str + :return: True if excluded. """ return self._exclude_dir_set.matches(dir_path) diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index 0068963a4..58ba5a316 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -46,8 +46,14 @@ def zip_folders(folder_a, folder_b, reporter, policies_manager=DEFAULT_SCAN_MANA Each item is a pair (file_a, file_b) with the corresponding file in both folders. Either file (but not both) will be None if the file is in only one folder. - :param folder_a: A Folder object. - :param folder_b: A Folder object. + + :param folder_a: first folder object. + :type folder_a: b2sdk.sync.folder.AbstractFolder + :param folder_b: second folder object. + :type folder_b: b2sdk.sync.folder.AbstractFolder + :param reporter: reporter object + :param policies_manager: policies manager object + :return: yields two element tuples """ iter_a = folder_a.all_files(reporter, policies_manager) @@ -81,6 +87,20 @@ def make_file_sync_actions( ): """ Yields the sequence of actions needed to sync the two files + + :param sync_type: synchronization type + :type sync_type: str + :param source_file: source file object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_file: destination file object + :type dest_file: b2sdk.sync.file.File + :param source_folder: a source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_folder: a destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param args: an object which holds command line arguments + :param now_millis: current time in milliseconds + :type now_millis: int """ policy = POLICY_MANAGER.get_policy( @@ -96,6 +116,16 @@ def make_folder_sync_actions( """ Yields a sequence of actions that will sync the destination folder to the source folder. + + :param source_folder: source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param args: an object which holds command line arguments + :param now_millis: current time in milliseconds + :type now_millis: int + :param reporter: reporter object + :param policies_manager: policies manager object """ if args.skipNewer and args.replaceNewer: raise CommandError('--skipNewer and --replaceNewer are incompatible') @@ -137,6 +167,10 @@ def make_folder_sync_actions( def count_files(local_folder, reporter): """ Counts all of the files in a local folder. + + :param local_folder: a folder object. + :type local_folder: b2sdk.sync.folder.AbstractFolder + :param reporter: reporter object """ # Don't pass in a reporter to all_files. Broken symlinks will be reported # during the next pass when the source and dest files are compared. @@ -162,6 +196,24 @@ def sync_folders( Syncs two folders. Always ensures that every file in the source is also in the destination. Deletes any file versions in the destination older than history_days. + + :param source_folder: source folder object + :type source_folder: b2sdk.sync.folder.AbstractFolder + :param dest_folder: destination folder object + :type dest_folder: b2sdk.sync.folder.AbstractFolder + :param args: an object which holds command line arguments + :param now_millis: current time in milliseconds + :type now_millis: int + :param stdout: standard output file object + :param no_progress: if True, do not show progress + :type no_progress: bool + :param max_workers: max number of workers + :type max_workers: int + :param policies_manager: policies manager object + :param dry_run: + :type dry_run: bool + :param allow_empty_source: if True, do not check whether source folder is empty + :type allow_empty_source: bool """ # For downloads, make sure that the target directory is there. diff --git a/b2sdk/transferer/abstract.py b/b2sdk/transferer/abstract.py index 925c4ac5e..972cf7968 100644 --- a/b2sdk/transferer/abstract.py +++ b/b2sdk/transferer/abstract.py @@ -55,7 +55,7 @@ def _get_remote_range(cls, response, metadata): def is_suitable(self, metadata, progress_listener): """ analyzes metadata (possibly against options passed earlier to constructor - to find out whether the given download request should be handled by this downloader + to find out whether the given download request should be handled by this downloader) """ pass diff --git a/b2sdk/utils.py b/b2sdk/utils.py index a91195b89..072828511 100644 --- a/b2sdk/utils.py +++ b/b2sdk/utils.py @@ -9,7 +9,6 @@ ###################################################################### from __future__ import division, print_function - import hashlib import os import platform @@ -26,14 +25,6 @@ except ImportError: import futures -# Global variable that says whether the app is shutting down -_shutting_down = False - - -def raise_if_shutting_down(): - if _shutting_down: - raise KeyboardInterrupt() - def interruptible_get_result(future): """ @@ -41,6 +32,9 @@ def interruptible_get_result(future): by a KeyboardInterrupt. This is not necessary in Python 3, but is needed for Python 2. + + :param future: a future to get result of + :type future: Future """ while True: try: @@ -50,15 +44,25 @@ def interruptible_get_result(future): def b2_url_encode(s): - """URL-encodes a unicode string to be sent to B2 in an HTTP header. + """ + URL-encodes a unicode string to be sent to B2 in an HTTP header. + + :param s: a unicode string to encode + :type s: str + :return: URL-encoded string + :rtype: str """ return six.moves.urllib.parse.quote(s.encode('utf-8')) def b2_url_decode(s): - """Decodes a Unicode string returned from B2 in an HTTP header. + """ + Decodes a Unicode string returned from B2 in an HTTP header. - Returns a Python unicode string. + :param s: a unicode string to decode + :type s: str + :return: a Python unicode string. + :rtype: str """ result = six.moves.urllib.parse.unquote_plus(s) if six.PY2: @@ -71,6 +75,12 @@ def b2_url_decode(s): def choose_part_ranges(content_length, minimum_part_size): """ Returns a list of (offset, length) for the parts of a large file. + + :param content_length: content length value + :type content_length: int + :param minimum_part_size: a minimum file part size + :type minimum_part_size: int + :rtype: list """ # If the file is at least twice the minimum part size, we are guaranteed @@ -103,6 +113,11 @@ def hex_sha1_of_stream(input_stream, content_length): """ Returns the 40-character hex SHA1 checksum of the first content_length bytes in the input stream. + + :param input_stream: stream object, which exposes read() method + :param content_length: expected length of the stream + :type content_length: int + :rtype: str """ remaining = content_length block_size = 1024 * 1024 @@ -121,8 +136,11 @@ def hex_sha1_of_stream(input_stream, content_length): def hex_sha1_of_bytes(data): """ - Returns the 40-character hex SHA1 checksum of the first content_length - bytes in the input stream. + Returns the 40-character hex SHA1 checksum of the data + + :param data: an array of bytes + :type data: bytes + :rtype: str """ return hashlib.sha1(data).hexdigest() @@ -131,8 +149,8 @@ def validate_b2_file_name(name): """ Raises a ValueError if the name is not a valid B2 file name. - :param name: a string - :return: None + :param name: a string to check + :type name: str """ if not isinstance(name, six.string_types): raise ValueError('file name must be a string, not bytes') @@ -156,6 +174,14 @@ def validate_b2_file_name(name): def is_file_readable(local_path, reporter=None): + """ + Check if the local file has read permissions + + :param local_path: a file path + :type local_path: str + :param reporter: reporter object to put errors on + :rtype: bool + """ if not os.path.exists(local_path): if reporter is not None: reporter.local_access_error(local_path) @@ -171,6 +197,11 @@ def fix_windows_path_limit(path): """ Prefix paths when running on Windows to overcome 260 character path length limit See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath + + :param path: a path to prefix + :type path: str + :return: a prefixed path + :rtype: str """ if platform.system() == 'Windows': if path.startswith('\\\\'): @@ -193,6 +224,9 @@ class BytesIoContextManager(object): """ def __init__(self, byte_data): + """ + :param bytes_data: a byte stream + """ self.byte_data = byte_data def __enter__(self): @@ -237,6 +271,13 @@ def _pick_scale_and_suffix(x): def format_and_scale_number(x, unit): """ Picks a good scale for representing a number and formats it. + + :param x: a number + :type x: int + :param unit: an arbitrary unit name + :type unit: str + :return: scaled and formatted number + :rtype: str """ # simple case for small numbers @@ -262,6 +303,15 @@ def format_and_scale_number(x, unit): def format_and_scale_fraction(numerator, denominator, unit): """ Picks a good scale for representing a fraction, and formats it. + + :param numerator: a numerator of a fraction + :type numerator: int + :param denominator: a denominator of a fraction + :type denominator: int + :param unit: an arbitrary unit name + :type unit: str + :return: scaled and formatted fraction + :rtype: str """ # simple case for small numbers @@ -289,21 +339,29 @@ def format_and_scale_fraction(numerator, denominator, unit): def camelcase_to_underscore(input_): - return _CAMELCASE_TO_UNDERSCORE_RE.sub(r'_\1', input_).lower() - + """ + Convert camel cased string to string with underscores -def repr_dict_deterministically(dict_): - # a simple version had a disadvantage of outputting dictionary keys in random order. - # It was hard to read. Therefore we sort items by key. - fields = ', '.join('%s: %s' % (repr(k), repr(v)) for k, v in sorted(six.iteritems(dict_))) - return '{%s}' % (fields,) + :param input_: an input string + :type input_: str + :return: string with underscores + :rtype: str + """ + return _CAMELCASE_TO_UNDERSCORE_RE.sub(r'_\1', input_).lower() class B2TraceMeta(DefaultTraceMeta): + """ + Traces all public method calls, except for ones with names that begin with `get_` + """ pass class B2TraceMetaAbstract(DefaultTraceAbstractMeta): + """ + Default class for tracers, to be set as + a metaclass for abstract base classes + """ pass diff --git a/doc/source/b2.account_info.rst b/doc/source/b2.account_info.rst deleted file mode 100644 index b78795173..000000000 --- a/doc/source/b2.account_info.rst +++ /dev/null @@ -1,62 +0,0 @@ -b2sdk\.account\_info package -========================= - -Submodules ----------- - -b2sdk\.account\_info\.abstract module ----------------------------------- - -.. automodule:: b2sdk.account_info.abstract - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.exception module ------------------------------------ - -.. automodule:: b2sdk.account_info.exception - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.in\_memory module ------------------------------------- - -.. automodule:: b2sdk.account_info.in_memory - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.sqlite\_account\_info module ------------------------------------------------ - -.. automodule:: b2sdk.account_info.sqlite_account_info - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.test\_upload\_url\_concurrency module --------------------------------------------------------- - -.. automodule:: b2sdk.account_info.test_upload_url_concurrency - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.account\_info\.upload\_url\_pool module -------------------------------------------- - -.. automodule:: b2sdk.account_info.upload_url_pool - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: b2sdk.account_info - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2.rst b/doc/source/b2.rst deleted file mode 100644 index 15672ca7c..000000000 --- a/doc/source/b2.rst +++ /dev/null @@ -1,166 +0,0 @@ -b2sdk package -============== - -Subpackages ------------ - -.. toctree:: - - b2sdk.account_info - b2sdk.sync - -Submodules ----------- - -b2sdk\.api module ------------------- - -.. automodule:: b2sdk.api - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.b2http module ---------------------- - -.. automodule:: b2sdk.b2http - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.bucket module ---------------------- - -.. automodule:: b2sdk.bucket - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.cache module --------------------- - -.. automodule:: b2sdk.cache - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.console\_tool module ----------------------------- - -.. automodule:: b2sdk.console_tool - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.download\_dest module ------------------------------ - -.. automodule:: b2sdk.download_dest - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.exception module ------------------------- - -.. automodule:: b2sdk.exception - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.file\_version module ----------------------------- - -.. automodule:: b2sdk.file_version - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.parse\_args module --------------------------- - -.. automodule:: b2sdk.parse_args - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.part module -------------------- - -.. automodule:: b2sdk.part - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.progress module ------------------------ - -.. automodule:: b2sdk.progress - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.raw\_api module ------------------------ - -.. automodule:: b2sdk.raw_api - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.raw\_simulator module ------------------------------ - -.. automodule:: b2sdk.raw_simulator - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.session module ----------------------- - -.. automodule:: b2sdk.session - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.unfinished\_large\_file module --------------------------------------- - -.. automodule:: b2sdk.unfinished_large_file - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.upload\_source module ------------------------------ - -.. automodule:: b2sdk.upload_source - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.utils module --------------------- - -.. automodule:: b2sdk.utils - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.version module ----------------------- - -.. automodule:: b2sdk.version - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: b2sdk - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2.sync.rst b/doc/source/b2.sync.rst deleted file mode 100644 index 566caf4ce..000000000 --- a/doc/source/b2.sync.rst +++ /dev/null @@ -1,86 +0,0 @@ -b2sdk\.sync package -==================== - -Submodules ----------- - -b2sdk\.sync\.action module ---------------------------- - -.. automodule:: b2sdk.sync.action - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.exception module ------------------------------- - -.. automodule:: b2sdk.sync.exception - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.file module -------------------------- - -.. automodule:: b2sdk.sync.file - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.folder module ---------------------------- - -.. automodule:: b2sdk.sync.folder - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.folder\_parser module ------------------------------------ - -.. automodule:: b2sdk.sync.folder_parser - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.policy module ---------------------------- - -.. automodule:: b2sdk.sync.policy - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.policy\_manager module ------------------------------------- - -.. automodule:: b2sdk.sync.policy_manager - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.report module ---------------------------- - -.. automodule:: b2sdk.sync.report - :members: - :undoc-members: - :show-inheritance: - -b2sdk\.sync\.sync module -------------------------- - -.. automodule:: b2sdk.sync.sync - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: b2sdk.sync - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/b2sdk/account_info/abstract.rst b/doc/source/b2sdk/account_info/abstract.rst new file mode 100644 index 000000000..6902d51bd --- /dev/null +++ b/doc/source/b2sdk/account_info/abstract.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.account_info.abstract` -- Account info abstract +=========================================================== + +.. automodule:: b2sdk.account_info.abstract + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/account_info/exception.rst b/doc/source/b2sdk/account_info/exception.rst new file mode 100644 index 000000000..e9051aafc --- /dev/null +++ b/doc/source/b2sdk/account_info/exception.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.account_info.exception` -- Account info exceptions +============================================================== + +.. automodule:: b2sdk.account_info.exception + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/account_info/sqlite_account_info.rst b/doc/source/b2sdk/account_info/sqlite_account_info.rst new file mode 100644 index 000000000..b44627e2d --- /dev/null +++ b/doc/source/b2sdk/account_info/sqlite_account_info.rst @@ -0,0 +1,9 @@ +:mod:`b2sdk.account_info.sqlite_account_info` -- SQLite account info +==================================================================== + +.. automodule:: b2sdk.account_info.sqlite_account_info + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ + :exclude-members: set_auth_data_with_schema_0_for_test diff --git a/doc/source/b2sdk/account_info/upload_url_pool.rst b/doc/source/b2sdk/account_info/upload_url_pool.rst new file mode 100644 index 000000000..126814113 --- /dev/null +++ b/doc/source/b2sdk/account_info/upload_url_pool.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.account_info.upload_url_pool` -- Upload URL pool +============================================================ + +.. automodule:: b2sdk.account_info.upload_url_pool + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/api.rst b/doc/source/b2sdk/api.rst new file mode 100644 index 000000000..c40d65434 --- /dev/null +++ b/doc/source/b2sdk/api.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.api` -- API client +============================== + +.. automodule:: b2sdk.api + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/bucket.rst b/doc/source/b2sdk/bucket.rst new file mode 100644 index 000000000..f702a26e9 --- /dev/null +++ b/doc/source/b2sdk/bucket.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.bucket` -- Bucket +============================= + +.. automodule:: b2sdk.bucket + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/cache.rst b/doc/source/b2sdk/cache.rst new file mode 100644 index 000000000..3efa9cebc --- /dev/null +++ b/doc/source/b2sdk/cache.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.cache` -- Cache +=========================== + +.. automodule:: b2sdk.cache + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/download_dest.rst b/doc/source/b2sdk/download_dest.rst new file mode 100644 index 000000000..7c9f663a0 --- /dev/null +++ b/doc/source/b2sdk/download_dest.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.download_dest` -- Download destination +================================================== + +.. automodule:: b2sdk.download_dest + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/exception.rst b/doc/source/b2sdk/exception.rst new file mode 100644 index 000000000..a852e718e --- /dev/null +++ b/doc/source/b2sdk/exception.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.exception` -- Exceptions +==================================== + +.. automodule:: b2sdk.exception + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/file_version.rst b/doc/source/b2sdk/file_version.rst new file mode 100644 index 000000000..204f74813 --- /dev/null +++ b/doc/source/b2sdk/file_version.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.file_version` -- File version +========================================= + +.. automodule:: b2sdk.file_version + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/progress.rst b/doc/source/b2sdk/progress.rst new file mode 100644 index 000000000..a9a0b3d85 --- /dev/null +++ b/doc/source/b2sdk/progress.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.progress` -- Progress +================================= + +.. automodule:: b2sdk.progress + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/action.rst b/doc/source/b2sdk/sync/action.rst new file mode 100644 index 000000000..9028bedc6 --- /dev/null +++ b/doc/source/b2sdk/sync/action.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.action` -- Sync action +======================================= + +.. automodule:: b2sdk.sync.action + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/exception.rst b/doc/source/b2sdk/sync/exception.rst new file mode 100644 index 000000000..297ea2fdd --- /dev/null +++ b/doc/source/b2sdk/sync/exception.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.exception` -- Sync exceptions +============================================== + +.. automodule:: b2sdk.sync.exception + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/file.rst b/doc/source/b2sdk/sync/file.rst new file mode 100644 index 000000000..a454a0efd --- /dev/null +++ b/doc/source/b2sdk/sync/file.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.file` -- File +============================== + +.. automodule:: b2sdk.sync.file + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/folder.rst b/doc/source/b2sdk/sync/folder.rst new file mode 100644 index 000000000..53d435e15 --- /dev/null +++ b/doc/source/b2sdk/sync/folder.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.folder` -- Folder +================================== + +.. automodule:: b2sdk.sync.folder + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/folder_parser.rst b/doc/source/b2sdk/sync/folder_parser.rst new file mode 100644 index 000000000..846b3840b --- /dev/null +++ b/doc/source/b2sdk/sync/folder_parser.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.folder_parser` -- Folder parser +================================================ + +.. automodule:: b2sdk.sync.folder_parser + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/policy.rst b/doc/source/b2sdk/sync/policy.rst new file mode 100644 index 000000000..4a5ca0fb4 --- /dev/null +++ b/doc/source/b2sdk/sync/policy.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.policy` -- Policy +================================== + +.. automodule:: b2sdk.sync.policy + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/policy_manager.rst b/doc/source/b2sdk/sync/policy_manager.rst new file mode 100644 index 000000000..17409d569 --- /dev/null +++ b/doc/source/b2sdk/sync/policy_manager.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.policy_manager` -- Policy manager +================================================== + +.. automodule:: b2sdk.sync.policy_manager + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/scan_policies.rst b/doc/source/b2sdk/sync/scan_policies.rst new file mode 100644 index 000000000..4f5ce31e0 --- /dev/null +++ b/doc/source/b2sdk/sync/scan_policies.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.scan_policies` -- Scan policies +================================================ + +.. automodule:: b2sdk.sync.scan_policies + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/sync/sync.rst b/doc/source/b2sdk/sync/sync.rst new file mode 100644 index 000000000..8948b3ac1 --- /dev/null +++ b/doc/source/b2sdk/sync/sync.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.sync.sync` -- Sync +============================== + +.. automodule:: b2sdk.sync.sync + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/upload_source.rst b/doc/source/b2sdk/upload_source.rst new file mode 100644 index 000000000..bf050d823 --- /dev/null +++ b/doc/source/b2sdk/upload_source.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.upload_source` -- Upload source +=========================================== + +.. automodule:: b2sdk.upload_source + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/b2sdk/utils.rst b/doc/source/b2sdk/utils.rst new file mode 100644 index 000000000..7bc7e4304 --- /dev/null +++ b/doc/source/b2sdk/utils.rst @@ -0,0 +1,8 @@ +:mod:`b2sdk.utils` -- Utils +=========================== + +.. automodule:: b2sdk.utils + :members: + :undoc-members: + :show-inheritance: + :special-members: __init__ diff --git a/doc/source/conf.py b/doc/source/conf.py index 9d0b55c7c..655bb5ee1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -10,7 +10,7 @@ # -*- coding: utf-8 -*- # -# B2_Command_Line_Tool documentation build configuration file, created by +# B2 Python SDK documentation build configuration file, created by # sphinx-quickstart on Fri Oct 20 18:27:20 2017. # # This file is execfile()d with the current directory set to its @@ -29,7 +29,9 @@ import os import sys -sys.path.insert(0, os.path.abspath('../b2sdk')) +sys.path.append(os.path.abspath('..')) + +from b2sdk.version import VERSION # -- General configuration ------------------------------------------------ @@ -57,8 +59,8 @@ master_doc = 'index' # General information about the project. -project = u'B2_Command_Line_Tool' -copyright = u'2017, Backblaze' +project = u'B2 Python SDK' +copyright = u'2019, Backblaze' author = u'Backblaze' # The version info for the project you're documenting, acts as replacement for @@ -66,9 +68,9 @@ # built documents. # # The short X.Y version. -version = u'0.7' +version = VERSION.rsplit('.', 1)[0] # The full version, including alpha/beta/rc tags. -release = u'0.7' +release = VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -93,7 +95,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -104,7 +106,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -125,7 +127,7 @@ # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'B2_Command_Line_Tooldoc' +htmlhelp_basename = 'B2_Python_SDKdoc' # -- Options for LaTeX output --------------------------------------------- @@ -151,19 +153,14 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ( - master_doc, 'B2_Command_Line_Tool.tex', u'B2\\_Command\\_Line\\_Tool Documentation', - u'Backblaze', 'manual' - ), + (master_doc, 'B2_Python_SDK.tex', u'B2\\_Python\\_SDK', u'Backblaze', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'b2_command_line_tool', u'B2_Command_Line_Tool Documentation', [author], 1) -] +man_pages = [(master_doc, 'b2_python_sdk', u'B2 Python SDK Documentation', [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -172,8 +169,8 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, 'B2_Command_Line_Tool', u'B2_Command_Line_Tool Documentation', author, - 'B2_Command_Line_Tool', 'One line description of project.', 'Miscellaneous' + master_doc, 'B2_Python_SDK', u'B2 Python SDK Documentation', author, 'B2_Python_SDK', + 'Backblaze Python SDK', 'Miscellaneous' ), ] diff --git a/doc/source/index.rst b/doc/source/index.rst index 284f35325..7a2715289 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,16 +1,105 @@ -.. B2_Command_Line_Tool documentation master file, created by - sphinx-quickstart on Fri Oct 20 18:27:20 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +========================================= +Welcome to B2 Python SDK's documentation! +========================================= -Welcome to B2_Command_Line_Tool's documentation! -================================================ +Overview +======== + +B2 SDK is a client library for easy access to all of the capabilities of B2 Cloud Storage. + +B2 command-line tool (https://github.com/Backblaze/B2_Command_Line_Tool) is an example of how it can be used +to provide command-line access to the B2 service, but there are many possible applications +(including FUSE filesystems, storage backend drivers for backup applications etc). + +First steps +=========== + +.. toctree:: + :caption: First Steps + :hidden: + + intro/install + intro/tutorial + +:doc:`intro/install` + Get B2 SDK installed on your computer + +:doc:`intro/tutorial` + A quick tutorial on how to get started with B2 SDK + +API Reference +============= .. toctree:: - :maxdepth: 2 - :caption: Contents: + :caption: API Reference + :hidden: + + b2sdk/api + b2sdk/bucket + b2sdk/cache + b2sdk/download_dest + b2sdk/exception + b2sdk/file_version + b2sdk/progress + b2sdk/upload_source + b2sdk/utils + b2sdk/account_info/abstract + b2sdk/account_info/exception + b2sdk/account_info/sqlite_account_info + b2sdk/account_info/upload_url_pool + b2sdk/sync/action + b2sdk/sync/exception + b2sdk/sync/file + b2sdk/sync/folder_parser + b2sdk/sync/folder + b2sdk/sync/policy_manager + b2sdk/sync/policy + b2sdk/sync/scan_policies + b2sdk/sync/sync + +:doc:`b2sdk/api` + +:doc:`b2sdk/bucket` + +:doc:`b2sdk/cache` + +:doc:`b2sdk/download_dest` + +:doc:`b2sdk/exception` + +:doc:`b2sdk/file_version` + +:doc:`b2sdk/progress` + +:doc:`b2sdk/upload_source` + +:doc:`b2sdk/utils` + +:doc:`b2sdk/account_info/abstract` + +:doc:`b2sdk/account_info/exception` + +:doc:`b2sdk/account_info/sqlite_account_info` + +:doc:`b2sdk/account_info/upload_url_pool` + +:doc:`b2sdk/sync/action` + +:doc:`b2sdk/sync/exception` + +:doc:`b2sdk/sync/file` + +:doc:`b2sdk/sync/folder_parser` + +:doc:`b2sdk/sync/folder` + +:doc:`b2sdk/sync/policy_manager` + +:doc:`b2sdk/sync/policy` + +:doc:`b2sdk/sync/scan_policies` - b2sdk +:doc:`b2sdk/sync/sync` Indices and tables ================== diff --git a/doc/source/intro/install.rst b/doc/source/intro/install.rst new file mode 100644 index 000000000..798b4b56c --- /dev/null +++ b/doc/source/intro/install.rst @@ -0,0 +1,58 @@ +================== +Installation Guide +================== + +Installing B2 SDK +================= + +B2 SDK runs on Python 2.7+ under CPython and PyPy. + +To install B2 SDK run:: + + pip install b2sdk + +If you see a message saying that the `six` library cannot be installed, which +happens if you're installing with the system python on OS X El Capitan, try +this:: + + pip install --ignore-installed b2sdk + +Developer Info +============== + +We encourage outside contributors to perform changes on our codebase. Many such changes have been merged already. In order to make it easier to contribute, core developers of this project: + +* provide guidance (through the issue reporting system) +* provide tool assisted code review (through the Pull Request system) +* maintain a set of integration tests (run with a production cloud) +* maintain a set of (well over a hundred) unit tests +* automatically run unit tests on 14 versions of python (including osx, Jython and pypy) +* format the code automatically using [yapf](https://github.com/google/yapf) +* use static code analysis to find subtle/potential issues with maintainability +* maintain other Continous Integration tools (coverage tracker) + +You'll need to some Python packages installed. To get all the latest things:: + + pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements-test.txt -r requirements-setup.txt + +There is a `Makefile` with a rule to run the unit tests using the currently active Python:: + + make setup + make test + +will install the required packages, then run the unit tests. + +To test in multiple python virtual environments, set the enviroment variable `PYTHON_VIRTUAL_ENVS` +to be a space-separated list of their root directories. When set, the makefile will run the +unit tests in each of the environments. + +Before checking in, use the `pre-commit.sh` script to check code formatting, run +unit tests, run integration tests etc. + +The integration tests need a file in your home directory called `.b2_auth` +that contains two lines with nothing on them but your account ID and application key:: + + accountId + applicationKey + +We marked the places in the code which are significantly less intuitive than others in a special way. To find them occurrences, use `git grep '*magic*'`. diff --git a/doc/source/intro/tutorial.rst b/doc/source/intro/tutorial.rst new file mode 100644 index 000000000..17c6d6922 --- /dev/null +++ b/doc/source/intro/tutorial.rst @@ -0,0 +1,74 @@ +======== +Tutorial +======== + +Account authorization +===================== + +TODO + +Bucket actions +============== + +Create a bucket +--------------- + +TODO + +Remove a bucket +--------------- + +TODO + +List buckets +------------- + +TODO + +Update bucket info +------------------ + +TODO + +File actions +============ + +Upload file +----------- + +TODO + +Download file +------------- + +TODO + +List files +---------- + +TODO + +Get file meta information +------------------------- + +TODO + +Delete file +----------- + +TODO + +Cancel file operations +---------------------- + +TODO + +Synchronization +=============== + +TODO + +Account information +=================== + +TODO diff --git a/doc/source/modules.rst b/doc/source/modules.rst deleted file mode 100644 index 733e1746a..000000000 --- a/doc/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -b2sdk -====== - -.. toctree:: - :maxdepth: 4 - - b2sdk diff --git a/test/test_parse_args.py b/test/test_parse_args.py deleted file mode 100644 index e86144bd6..000000000 --- a/test/test_parse_args.py +++ /dev/null @@ -1,99 +0,0 @@ -###################################################################### -# -# File: test/test_parse_args.py -# -# Copyright 2019 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -from .test_base import TestBase -from b2sdk.parse_args import parse_arg_list - - -class TestParseArgs(TestBase): - - NO_ARGS = { - 'option_flags': [], - 'option_args': [], - 'list_args': [], - 'optional_before': [], - 'required': [], - 'optional': [], - 'arg_parser': {} - } - - EVERYTHING = { - 'option_flags': ['optionFlag'], - 'option_args': ['optionArg'], - 'list_args': ['list'], - 'optional_before': [], - 'required': ['required'], - 'optional': ['optional'], - 'arg_parser': { - 'optionArg': int - } - } - - BEFORE_AND_AFTER = { - 'option_flags': [], - 'option_args': [], - 'list_args': [], - 'optional_before': ['optionalBefore'], - 'required': ['required'], - 'optional': ['optional'], - 'arg_parser': {} - } - - def test_no_args(self): - args = parse_arg_list([], **self.NO_ARGS) - self.assertTrue(args is not None) - - def test_unexpected_flag(self): - args = parse_arg_list(['--badFlag'], **self.NO_ARGS) - self.assertTrue(args is None) - - def test_unexpected_arg(self): - args = parse_arg_list(['badArg'], **self.NO_ARGS) - self.assertTrue(args is None) - - def test_option_defaults(self): - args = parse_arg_list(['req-value'], **self.EVERYTHING) - self.assertFalse(args.optionFlag) - self.assertTrue(args.optionArg is None) - self.assertEqual([], args.list) - self.assertEqual('req-value', args.required) - self.assertTrue(args.optional is None) - - def test_all_there(self): - args = parse_arg_list( - ['--optionFlag', '--optionArg', '99', '--list', '1', '--list', '2', 'b', 'c'], - **self.EVERYTHING - ) # yapf disable - self.assertTrue(args.optionFlag) - self.assertEqual(99, args.optionArg) - self.assertEqual('b', args.required) - self.assertEqual(['1', '2'], args.list) - self.assertEqual('c', args.optional) - - def test_optional_arg_missing_value(self): - args = parse_arg_list(['--optionArg'], **self.EVERYTHING) - self.assertTrue(args is None) - - def test_no_optional(self): - args = parse_arg_list(['a'], **self.BEFORE_AND_AFTER) - self.assertEqual((None, 'a', None), (args.optionalBefore, args.required, args.optional)) - - def test_optional_before(self): - args = parse_arg_list(['a', 'b'], **self.BEFORE_AND_AFTER) - self.assertEqual(('a', 'b', None), (args.optionalBefore, args.required, args.optional)) - - def test_same_arg_in_two_places(self): - arg_spec = dict(self.NO_ARGS) - arg_spec['optional_before'] = ['a'] - arg_spec['required'] = ['a'] - with self.assertRaisesRegexp( - ValueError, "argument 'a' is in both 'optional_before' an 'required'" - ): - parse_arg_list([], **arg_spec)