diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..81eba8cc --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..7fa4e241 --- /dev/null +++ b/.flake8 @@ -0,0 +1,20 @@ +[flake8] +max_line_length = 150 +# Google docstring convention + D204 & D401 +# TODO enable D100 & D401 +docstring-convention=all +ignore= + D100 + D104 + D203 + D213 + D215 + D400 + D404 + D406 + D407 + D408 + D409 + D413 +per-file-ignores = + tests/*: D diff --git a/.github/workflows/lint_and_test.yaml b/.github/workflows/lint_and_test.yaml new file mode 100644 index 00000000..c30a4e39 --- /dev/null +++ b/.github/workflows/lint_and_test.yaml @@ -0,0 +1,32 @@ +name: Lint and test + +on: push + +jobs: + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + + - name: Lint + run: ./lint_and_test.sh lint + + - name: Type check + run: ./lint_and_test.sh types + + - name: Unit tests + run: ./lint_and_test.sh tests diff --git a/.gitignore b/.gitignore index 1c46ab7d..734c88f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,12 @@ -build -dist -apify_client.egg-info +.venv +.direnv +.envrc + +.mypy_cache +.pytest_cache + +*.egg-info/ +*.egg + +.vscode .idea diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..7021a016 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[isort] +include_trailing_comma = True +line_length = 150 +use_parentheses = True +multi_line_output = 3 +sections = FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,LOCALFOLDER diff --git a/LICENSE b/LICENSE index 8effb33b..896170f6 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018 Apify Technologies s.r.o. + Copyright 2020 Apify Technologies s.r.o. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 4551edd7..0e334ce8 100644 --- a/README.md +++ b/README.md @@ -3,4 +3,41 @@ This will be an official client for [Apify API](https://www.apify.com/docs/api/v2). It's still work in progress, so please don't use it yet! +## Installation +Requires Python 3.7+ + +Right now the client is not available on PyPI yet, so you can install it only from the git repo. +To do that, run `pip install git+https://github.com/apify/apify-client-python.git` + +## Development + +### Environment + +For local development, it is required to have Python 3.7 installed. + +It is recommended to set up a virtual environment while developing this package to isolate your development environment, +however, due to the many varied ways Python can be installed and virtual environments can be set up, +this is left up to the developers to do themselves. + +One recommended way is with the builtin `venv` module: + +```bash +python3 -m venv .venv +source .venv/bin/activate +``` + +To improve on the experience, you can use [pyenv](https://github.com/pyenv/pyenv) to have an environment with a pinned Python version, +and [direnv](https://github.com/direnv/direnv) to automatically activate/deactivate the environment when you enter/exit the project folder. + +### Dependencies + +To install this package and its development dependencies, run `pip install -e '.[dev]'` + +### Formatting + +We use `autopep8` and `isort` to automatically format the code to a common format. To run the formatting, just run `./format.sh`. + +### Linting and Testing + +We use `flake8` for linting, `mypy` for type checking and `pytest` for unit testing. To run these tools, just run `./lint_and_test.sh`. diff --git a/apify_client/__init__.py b/apify_client/__init__.py deleted file mode 100644 index 13b507bc..00000000 --- a/apify_client/__init__.py +++ /dev/null @@ -1 +0,0 @@ -name = "apify_client" diff --git a/format.sh b/format.sh new file mode 100755 index 00000000..264324e7 --- /dev/null +++ b/format.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env /bin/bash + +set -o errexit + +echo "🔤 isort formatting..." +python3 -m isort src tests + +echo "🎨 autopep8 formatting..." +python3 -m autopep8 --in-place --recursive src tests + +echo "🎉 formatted 🎉" diff --git a/lint_and_test.sh b/lint_and_test.sh new file mode 100755 index 00000000..2b9fbaa2 --- /dev/null +++ b/lint_and_test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -u + +checks_to_run=${1:-"everything"} +shift || true + + +lint() { + echo "⛄️ flake8 check..." + python3 -m flake8 src tests +} + +type_check() { + echo "📝 checking types" + python3 -m mypy src +} + +unit_tests() { + echo "👮‍♀️ running unit tests" + python3 -m pytest -rA tests +} + + +if [ "$checks_to_run" = "lint" ] ; then + lint +elif [ "$checks_to_run" = "types" ] ; then + type_check +elif [ "$checks_to_run" = "tests" ] ; then + unit_tests +elif [ "$checks_to_run" = "everything" ] ; then + lint + type_check + unit_tests +else + echo "Invalid type of test ($checks_to_run) requested. Use lint / types / tests or leave empty to run all tests" + exit 1 +fi + +echo "🎉 success 🎉" diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..99d7e074 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +check_untyped_defs = True +disallow_incomplete_defs = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_untyped_defs = True +no_implicit_optional = True +warn_redundant_casts = True +warn_return_any = True +warn_unreachable = True +warn_unused_ignores = True diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..2bed0f3a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --doctest-modules \ No newline at end of file diff --git a/setup.py b/setup.py index d800b7fc..857afbc4 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,52 @@ -import setuptools +import pathlib -with open("README.md", "r") as fh: - long_description = fh.read() +from setuptools import find_packages, setup + +here = pathlib.Path(__file__).parent.resolve() + +long_description = (here / 'README.md').read_text(encoding='utf-8') + +setup( + name='apify_client', + version='0.0.1', -setuptools.setup( - name="apify_client", - version="0.0.1", author="Apify Technologies s.r.o.", author_email="support@apify.com", - description="Work in progress: Apify API client for Python", + url="https://github.com/apify/apify-client-python", + project_urls={ + 'Apify Homepage': 'https://apify.com', + }, + license='Apache Software License', + license_file='LICENSE', + + description='Apify API client for Python', long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/apifytech/apify-client-python", - packages=setuptools.find_packages(), + long_description_content_type='text/markdown', + classifiers=[ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Libraries', ], -) + keywords='apify, api, client, scraping, automation', + package_dir={'': 'src'}, + packages=find_packages(where='src'), + python_requires='>=3.7', + install_requires=['requests ~=2.25.1'], + extras_require={ + 'dev': [ + 'autopep8 ~= 1.5.4', + 'flake8 ~= 3.8.4', + 'flake8-commas ~= 2.0.0', + 'flake8-docstrings ~= 1.5.0', + 'flake8-isort ~= 4.0.0', + 'isort ~= 5.7.0', + 'mypy ~= 0.790', + 'pep8-naming ~= 0.11.1', + 'pytest ~= 6.2.1', + ], + }, +) diff --git a/src/apify_client/__init__.py b/src/apify_client/__init__.py new file mode 100644 index 00000000..9ad9417d --- /dev/null +++ b/src/apify_client/__init__.py @@ -0,0 +1,3 @@ +from .client import ApifyClient + +__all__ = ['ApifyClient'] diff --git a/src/apify_client/_errors.py b/src/apify_client/_errors.py new file mode 100644 index 00000000..a717c94b --- /dev/null +++ b/src/apify_client/_errors.py @@ -0,0 +1,69 @@ +from typing import Optional + +import requests + + +class ApifyClientError(Exception): + """Base class for errors specific to the Apify API Client.""" + + pass + + +class ApifyApiError(ApifyClientError): + """Error specific to requests to the Apify API. + + An `ApifyApiError` is thrown for successful HTTP requests that reach the API, + but the API responds with an error response. Typically, those are rate limit + errors and internal errors, which are automatically retried, or validation + errors, which are thrown immediately, because a correction by the user is needed. + """ + + def __init__(self, response: requests.models.Response, attempt: int) -> None: + """Create the ApifyApiError instance. + + Args: + response: The response to the failed API call + attempt: Which retry was the request that failed + """ + self.message: Optional[str] = None + self.type: Optional[str] = None + + response_data = response.json() + if 'error' in response_data: + self.message = response_data['error']['message'] + self.type = response_data['error']['type'] + else: + self.message = f'Unexpected error: {response.text}' + + super().__init__(self.message) + + self.name = 'ApifyApiError' + self.status_code = response.status_code + self.attempt = attempt + self.http_method = response.request.method + + # TODO self.client_method + # TODO self.original_stack + # TODO self.path + # TODO self.stack + + +class InvalidResponseBodyError(ApifyClientError): + """Error caused by the response body failing to be parsed. + + This error exists for the quite common situation, where only a partial JSON response is received and + an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the + request. We do that by identifying this error in the _HTTPClient. + """ + + def __init__(self, response: requests.models.Response) -> None: + """Create the InvalidResponseBodyError instance. + + Args: + response: The response which failed to be parsed + """ + super().__init__('Response body could not be parsed') + + self.name = 'InvalidResponseBodyError' + self.code = 'invalid-response-body' + self.response = response diff --git a/src/apify_client/_http_client.py b/src/apify_client/_http_client.py new file mode 100644 index 00000000..99b8b29b --- /dev/null +++ b/src/apify_client/_http_client.py @@ -0,0 +1,118 @@ +import os +import sys +from http import HTTPStatus +from typing import Any, Callable, Dict, Optional + +import requests +from requests.exceptions import ConnectionError, Timeout + +from ._errors import ApifyApiError, InvalidResponseBodyError +from ._types import JSONSerializable +from ._utils import _is_content_type_json, _is_content_type_text, _is_content_type_xml, _retry_with_exp_backoff + +DEFAULT_BACKOFF_EXPONENTIAL_FACTOR = 2 +DEFAULT_BACKOFF_RANDOM_FACTOR = 1 + + +class _HTTPClient: + def __init__(self, max_retries: int = 8, min_delay_between_retries_millis: int = 500) -> None: + self.max_retries = max_retries + self.min_delay_between_retries_millis = min_delay_between_retries_millis + self.requests_session = requests.Session() + + # TODO add client version + is_at_home = ('APIFY_IS_AT_HOME' in os.environ) + python_version = '.'.join([str(x) for x in sys.version_info[:3]]) + + user_agent = f'ApifyClient ({sys.platform}; Python/{python_version}); isAtHome/{is_at_home}' + self.requests_session.headers.update({'User-Agent': user_agent}) + + def call( + self, + *, + method: str, + url: str, + headers: Optional[Dict] = None, + params: Optional[Dict] = None, + data: Optional[str] = None, + json: Optional[JSONSerializable] = None, + stream: Optional[bool] = None, + parse_response: Optional[bool] = True, + ) -> requests.models.Response: + request_params = self._parse_params(params) + requests_session = self.requests_session + + def _make_request(bail: Callable, attempt: int) -> requests.models.Response: # type: ignore[return] + try: + response = requests_session.request( + method, + url, + headers=headers, + params=request_params, + data=data, + json=json, + stream=stream, + ) + if response.status_code < 300: + if parse_response: + _maybe_parsed_body = self._maybe_parse_response(response) + elif stream: + response.raw.decode_content = True + _maybe_parsed_body = response.raw + else: + _maybe_parsed_body = response.content + setattr(response, '_maybe_parsed_body', _maybe_parsed_body) + return response + + except (ConnectionError, Timeout, InvalidResponseBodyError) as e: + raise e + except Exception as e: + bail(e) + + api_error = ApifyApiError(response, attempt) + if response.status_code == HTTPStatus.TOO_MANY_REQUESTS or response.status_code >= 500: + raise api_error + else: + bail(api_error) + + return _retry_with_exp_backoff( + _make_request, + max_retries=self.max_retries, + backoff_base_millis=self.min_delay_between_retries_millis, + backoff_factor=DEFAULT_BACKOFF_EXPONENTIAL_FACTOR, + random_factor=DEFAULT_BACKOFF_RANDOM_FACTOR, + ) + + @staticmethod + def _maybe_parse_response(response: requests.models.Response) -> Any: + if response.status_code == HTTPStatus.NO_CONTENT: + return None + + content_type = '' + if 'content-type' in response.headers: + content_type = response.headers['content-type'].split(';')[0].strip() + + try: + if _is_content_type_json(content_type): + return response.json() + elif _is_content_type_xml(content_type) or _is_content_type_text(content_type): + return response.text + else: + return response.content + except ValueError as err: + raise InvalidResponseBodyError(response) from err + + @staticmethod + def _parse_params(params: Optional[Dict]) -> Optional[Dict]: + if params is None: + return None + + parsed_params = {} + for key, value in params.items(): + # Our API needs to have boolean parameters passed as 0 or 1, therefore we have to replace them + if isinstance(value, bool): + parsed_params[key] = int(value) + else: + parsed_params[key] = value + + return parsed_params diff --git a/src/apify_client/_types.py b/src/apify_client/_types.py new file mode 100644 index 00000000..66a6a0b9 --- /dev/null +++ b/src/apify_client/_types.py @@ -0,0 +1,6 @@ +from typing import Any, Dict, List, Union + +# Type for representing json-serializable values +# It's close enough to the real thing supported by json.parse, and the best we can do until mypy supports recursive types +# It was suggested in a discussion with (and approved by) Guido van Rossum, so I'd consider it correct enough +JSONSerializable = Union[str, int, float, bool, None, Dict[str, Any], List[Any]] diff --git a/src/apify_client/_utils.py b/src/apify_client/_utils.py new file mode 100644 index 00000000..166008ec --- /dev/null +++ b/src/apify_client/_utils.py @@ -0,0 +1,124 @@ +import io +import random +import re +import time +from datetime import datetime, timezone +from http import HTTPStatus +from typing import Any, Callable, Dict, TypeVar, cast + +from ._errors import ApifyApiError + +PARSE_DATE_FIELDS_MAX_DEPTH = 3 +PARSE_DATE_FIELDS_KEY_SUFFIX = 'At' + +NOT_FOUND_TYPE = 'record-not-found' +NOT_FOUND_ON_S3 = 'NoSuchKey' + + +def _to_safe_id(id: str) -> str: + # Identificators of resources in the API are either in the format `resource_id` or `username/resource_id`. + # Since the `/` character has a special meaning in URL paths, + # we replace it with `~` for proper route parsing on the API, where after parsing the URL it's replaced back to `/`. + return id.replace('/', '~') + + +def _parse_date_fields(data: Dict) -> Dict: + return cast(Dict, _parse_date_fields_internal(data)) + + +def _parse_date_fields_internal(data: object, max_depth: int = PARSE_DATE_FIELDS_MAX_DEPTH) -> object: + if max_depth < 0: + return data + + if isinstance(data, list): + return [_parse_date_fields_internal(item, max_depth - 1) for item in data] + + if isinstance(data, dict): + def parse(key: str, value: object) -> object: + parsed_value = value + if key.endswith(PARSE_DATE_FIELDS_KEY_SUFFIX) and isinstance(value, str): + try: + parsed_value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc) + except ValueError: + pass + else: + parsed_value = _parse_date_fields_internal(value, max_depth - 1) + return parsed_value + + return {key: parse(key, value) for (key, value) in data.items()} + + return data + + +def _pluck_data(parsed_response: Any) -> Dict: + if isinstance(parsed_response, dict) and 'data' in parsed_response: + return cast(Dict, parsed_response['data']) + + raise ValueError('The "data" property is missing in the response.') + + +def _is_content_type_json(content_type: str) -> bool: + return bool(re.search(r'^application/json', content_type, flags=re.IGNORECASE)) + + +def _is_content_type_xml(content_type: str) -> bool: + return bool(re.search(r'^application/.*xml$', content_type, flags=re.IGNORECASE)) + + +def _is_content_type_text(content_type: str) -> bool: + return bool(re.search(r'^text/', content_type, flags=re.IGNORECASE)) + + +def _is_file_or_bytes(value: Any) -> bool: + # The check for IOBase is not ideal, it would be better to use duck typing, + # but then the check would be super complex, judging from how the 'requests' library does it. + # This way should be good enough for the vast majority of use cases, if it causes issues, we can improve it later. + return isinstance(value, (bytes, bytearray, io.IOBase)) + + +T = TypeVar('T') +BailType = Callable[[Exception], None] + + +def _retry_with_exp_backoff( + func: Callable[[BailType, int], T], + *, + max_retries: int = 8, + backoff_base_millis: int = 500, + backoff_factor: float = 2, + random_factor: float = 1, +) -> T: + + random_factor = min(max(0, random_factor), 1) + backoff_factor = min(max(1, backoff_factor), 10) + swallow = True + + def bail(exception: Exception) -> None: + nonlocal swallow + swallow = False + raise exception + + for attempt in range(1, max_retries): + try: + return func(bail, attempt) + except Exception as e: + if not swallow: + raise e + + random_sleep_factor = random.uniform(1, 1 + random_factor) + backoff_base_secs = backoff_base_millis / 1000 + backoff_exp_factor = backoff_factor ** (attempt - 1) + + sleep_time_secs = random_sleep_factor * backoff_base_secs * backoff_exp_factor + time.sleep(sleep_time_secs) + + return func(bail, max_retries) + + +def _catch_not_found_or_throw(exc: ApifyApiError) -> None: + is_not_found_status = (exc.status_code == HTTPStatus.NOT_FOUND) + is_not_found_message = (exc.type == NOT_FOUND_TYPE) or (isinstance(exc.message, str) and NOT_FOUND_ON_S3 in exc.message) + if not (is_not_found_status and is_not_found_message): + raise exc + + return None diff --git a/src/apify_client/client.py b/src/apify_client/client.py new file mode 100644 index 00000000..9fdfbab7 --- /dev/null +++ b/src/apify_client/client.py @@ -0,0 +1,91 @@ +from typing import Dict, Optional + +from ._http_client import _HTTPClient +from .clients.resource_clients.dataset import DatasetClient +from .clients.resource_clients.dataset_collection import DatasetCollectionClient +from .clients.resource_clients.key_value_store import KeyValueStoreClient +from .clients.resource_clients.key_value_store_collection import KeyValueStoreCollectionClient +from .clients.resource_clients.log import LogClient +from .clients.resource_clients.request_queue import RequestQueueClient +from .clients.resource_clients.request_queue_collection import RequestQueueCollectionClient + +DEFAULT_BASE_API_URL = 'https://api.apify.com/v2' + + +class ApifyClient: + """The Apify API client.""" + + def __init__( + self, + token: Optional[str] = None, + *, + base_url: str = DEFAULT_BASE_API_URL, + max_retries: int = 8, + min_delay_between_retries_millis: int = 500, + ): + """Initialize the Apify API Client. + + Args: + token: The Apify API token + base_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com/v2 + max_retries: How many times to retry a failed request at most + min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value) + """ + self.token = token + self.base_url = base_url + self.max_retries = max_retries + self.min_delay_between_retries_millis = min_delay_between_retries_millis + + self.http_client = _HTTPClient() + # TODO statistics + # TODO logger + + def _options(self) -> Dict: + return { + 'base_url': self.base_url, + 'http_client': self.http_client, + 'params': { + 'token': self.token, + }, + } + + def dataset(self, dataset_id: str) -> DatasetClient: + """Retrieve the sub-client for manipulating a single dataset. + + Args: + dataset_id (str): ID of the dataset to be manipulated + """ + return DatasetClient(resource_id=dataset_id, **self._options()) + + def datasets(self) -> DatasetCollectionClient: + """Retrieve the sub-client for manipulating datasets.""" + return DatasetCollectionClient(**self._options()) + + def key_value_store(self, key_value_store_id: str) -> KeyValueStoreClient: + """Retrieve the sub-client for manipulating a single key-value store. + + Args: + key_value_store_id (str): ID of the key-value store to be manipulated + """ + return KeyValueStoreClient(resource_id=key_value_store_id, **self._options()) + + def key_value_stores(self) -> KeyValueStoreCollectionClient: + """Retrieve the sub-client for manipulating key-value stores.""" + return KeyValueStoreCollectionClient(**self._options()) + + def request_queue(self, request_queue_id: str, *, client_key: Optional[str] = None) -> RequestQueueClient: + """Retrieve the sub-client for manipulating a single request queue. + + Args: + request_queue_id (str) : ID of the request queue to be manipulated + client_key (str): A unique identifier of the client accessing the request queue + """ + return RequestQueueClient(resource_id=request_queue_id, client_key=client_key, **self._options()) + + def request_queues(self) -> RequestQueueCollectionClient: + """Retrieve the sub-client for manipulating request queues.""" + return RequestQueueCollectionClient(**self._options()) + + def log(self, build_or_run_id: str) -> LogClient: + """Retrieve the sub-client for retrieving logs.""" + return LogClient(resource_id=build_or_run_id, **self._options()) diff --git a/src/apify_client/clients/__init__.py b/src/apify_client/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apify_client/clients/base/__init__.py b/src/apify_client/clients/base/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apify_client/clients/base/actor_job_base_client.py b/src/apify_client/clients/base/actor_job_base_client.py new file mode 100644 index 00000000..afdf313d --- /dev/null +++ b/src/apify_client/clients/base/actor_job_base_client.py @@ -0,0 +1,13 @@ +from typing import Optional + +from .resource_client import ResourceClient + +# from ..._utils import _parse_date_fields, _pluck_data + + +class ActorJobBaseClient(ResourceClient): + """Base sub-client class for actor runs and actor builds.""" + + def _wait_for_finish(self, wait_secs: Optional[int] = None) -> None: + # TODO implement when implementing run and build endpoints + pass diff --git a/src/apify_client/clients/base/base_client.py b/src/apify_client/clients/base/base_client.py new file mode 100644 index 00000000..3fcafcc2 --- /dev/null +++ b/src/apify_client/clients/base/base_client.py @@ -0,0 +1,49 @@ +from typing import Any, Dict, Optional + +from ..._http_client import _HTTPClient +from ..._utils import _to_safe_id + + +class BaseClient: + """Base class for sub-clients.""" + + def __init__( + self, + *, + base_url: str, + http_client: _HTTPClient, + resource_id: Optional[str] = None, + resource_path: str, + params: Optional[Dict] = None, + ) -> None: + """Initialize the sub-client. + + Args: + base_url (str): Base URL of the API server + http_client (_HTTPClient): The _HTTPClient instance to be used in this client + resource_id (str): ID of the manipulated resource, in case of a single-resource client + resource_path (str): Path to the resource's endpoint on the API server + params (dict): Parameters to include in all requests from this client + """ + if resource_path.endswith('/'): + raise ValueError('resource_path must not end with "/"') + + self.base_url = base_url + self.http_client = http_client + self.params = params or {} + self.resource_path = resource_path + self.resource_id = resource_id + self.url = f'{self.base_url}/{self.resource_path}' + if self.resource_id is not None: + self.safe_id = _to_safe_id(self.resource_id) + self.url = f'{self.url}/{self.safe_id}' + + def _url(self, path: Optional[str] = None) -> str: + if path is not None: + return f'{self.url}/{path}' + return self.url + + def _params(self, **kwargs: Any) -> Dict: + merged_params = self.params.copy() + merged_params.update(kwargs) + return merged_params diff --git a/src/apify_client/clients/base/resource_client.py b/src/apify_client/clients/base/resource_client.py new file mode 100644 index 00000000..25d2a641 --- /dev/null +++ b/src/apify_client/clients/base/resource_client.py @@ -0,0 +1,45 @@ +from typing import Dict, Optional + +from ..._errors import ApifyApiError +from ..._utils import _catch_not_found_or_throw, _parse_date_fields, _pluck_data +from .base_client import BaseClient + + +class ResourceClient(BaseClient): + """Base class for sub-clients manipulating a single resource.""" + + def _get(self) -> Optional[Dict]: + try: + response = self.http_client.call( + url=self.url, + method='GET', + params=self._params(), + ) + + return _parse_date_fields(_pluck_data(response.json())) + + except ApifyApiError as exc: + _catch_not_found_or_throw(exc) + + return None + + def _update(self, new_fields: Dict) -> Dict: + response = self.http_client.call( + url=self._url(), + method='PUT', + params=self._params(), + json=new_fields, + ) + + return _parse_date_fields(_pluck_data(response.json())) + + def _delete(self) -> None: + try: + self.http_client.call( + url=self._url(), + method='DELETE', + params=self._params(), + ) + + except ApifyApiError as exc: + _catch_not_found_or_throw(exc) diff --git a/src/apify_client/clients/base/resource_collection_client.py b/src/apify_client/clients/base/resource_collection_client.py new file mode 100644 index 00000000..11115d56 --- /dev/null +++ b/src/apify_client/clients/base/resource_collection_client.py @@ -0,0 +1,36 @@ +from typing import Any, Dict + +from ..._utils import _parse_date_fields, _pluck_data +from .base_client import BaseClient + + +class ResourceCollectionClient(BaseClient): + """Base class for sub-clients manipulating a resource collection.""" + + def _list(self, **kwargs: Any) -> Dict: + response = self.http_client.call( + url=self._url(), + method='GET', + params=self._params(**kwargs), + ) + + return _parse_date_fields(_pluck_data(response.json())) + + def _create(self, resource: Dict) -> Dict: + response = self.http_client.call( + url=self._url(), + method='POST', + params=self._params(), + json=resource, + ) + + return _parse_date_fields(_pluck_data(response.json())) + + def _get_or_create(self, name: str = '') -> Dict: + response = self.http_client.call( + url=self._url(), + method='POST', + params=self._params(name=name), + ) + + return _parse_date_fields(_pluck_data(response.json())) diff --git a/src/apify_client/clients/resource_clients/__init__.py b/src/apify_client/clients/resource_clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apify_client/clients/resource_clients/dataset.py b/src/apify_client/clients/resource_clients/dataset.py new file mode 100644 index 00000000..beedbeb9 --- /dev/null +++ b/src/apify_client/clients/resource_clients/dataset.py @@ -0,0 +1,386 @@ +import io +from typing import Any, Dict, Generator, List, Optional, cast + +from ..._types import JSONSerializable +from ..base.resource_client import ResourceClient + + +class DatasetClient(ResourceClient): + """Sub-client for manipulating a single dataset.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Initialize the DatasetClient.""" + super().__init__(*args, resource_path='datasets', **kwargs) + + def get(self) -> Optional[Dict]: + """Retrieve the dataset. + + https://docs.apify.com/api/v2#/reference/datasets/dataset/get-dataset + + Returns: + The retrieved dataset + """ + return self._get() + + def update(self, new_fields: Dict) -> Dict: + """Update the dataset with specified fields. + + https://docs.apify.com/api/v2#/reference/datasets/dataset/update-dataset + + Args: + new_fields (dict): The fields of the dataset to update + + Returns: + The updated dataset + """ + return self._update(new_fields) + + def delete(self) -> None: + """Delete the dataset. + + https://docs.apify.com/api/v2#/reference/datasets/dataset/delete-dataset + """ + return self._delete() + + def list_items( + self, + *, + offset: Optional[int] = None, + limit: Optional[int] = None, + clean: Optional[bool] = None, + desc: Optional[bool] = None, + fields: Optional[List[str]] = None, + omit: Optional[List[str]] = None, + unwind: Optional[str] = None, + skip_empty: Optional[bool] = None, + skip_hidden: Optional[bool] = None, + ) -> Dict: + """List the items of the dataset. + + https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items + + Args: + offset: (int, optional): Number of items that should be skipped at the start. The default value is 0 + limit: (int, optional): Maximum number of items to return. By default there is no limit. + desc (bool, optional): By default, results are returned in the same order as they were stored. + To reverse the order, set this parameter to True. + clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character). + The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters. + Note that since some objects might be skipped from the output, that the result might contain less items than the limit value. + fields (list[str], optional): A list of fields which should be picked from the items, + only these fields will remain in the resulting record objects. + Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter. + You can use this feature to effectively fix the output format. + omit: (list[str], optional): A list of fields which should be omitted from the items. + unwind: (str, optional): Name of a field which should be unwound. + If the field is an array then every element of the array will become a separate record and merged with parent object. + If the unwound field is an object then it is merged with the parent object. + If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object, + then the item gets preserved as it is. Note that the unwound items ignore the desc parameter. + skip_empty (bool, optional): If True, then empty items are skipped from the output. + Note that if used, the results might contain less items than the limit value. + skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character. + + Returns: + The dataset items + """ + request_params = self._params( + offset=offset, + limit=limit, + desc=desc, + clean=clean, + fields=fields, + omit=omit, + unwind=unwind, + skipEmpty=skip_empty, + skipHidden=skip_hidden, + ) + + response = self.http_client.call( + url=self._url('items'), + method='GET', + params=request_params, + ) + + data = response.json() + + return { + 'items': data, + 'total': int(response.headers['x-apify-pagination-total']), + 'offset': int(response.headers['x-apify-pagination-offset']), + 'count': len(data), # because x-apify-pagination-count returns invalid values when hidden/empty items are skipped + 'limit': int(response.headers['x-apify-pagination-limit']), # API returns 999999999999 when no limit is used + } + + def iterate_items( + self, + offset: int = 0, + limit: Optional[int] = None, + clean: Optional[bool] = None, + desc: Optional[bool] = None, + fields: Optional[List[str]] = None, + omit: Optional[List[str]] = None, + unwind: Optional[str] = None, + skip_empty: Optional[bool] = None, + skip_hidden: Optional[bool] = None, + ) -> Generator: + """Iterate over the items in the dataset. + + https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items + + Args: + offset: (int, optional): Number of items that should be skipped at the start. The default value is 0 + limit: (int, optional): Maximum number of items to return. By default there is no limit. + desc (bool, optional): By default, results are returned in the same order as they were stored. + To reverse the order, set this parameter to True. + clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character). + The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters. + Note that since some objects might be skipped from the output, that the result might contain less items than the limit value. + fields (list[str], optional): A list of fields which should be picked from the items, + only these fields will remain in the resulting record objects. + Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter. + You can use this feature to effectively fix the output format. + omit: (list[str], optional): A list of fields which should be omitted from the items. + unwind: (str, optional): Name of a field which should be unwound. + If the field is an array then every element of the array will become a separate record and merged with parent object. + If the unwound field is an object then it is merged with the parent object. + If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object, + then the item gets preserved as it is. Note that the unwound items ignore the desc parameter. + skip_empty (bool, optional): If True, then empty items are skipped from the output. + Note that if used, the results might contain less items than the limit value. + skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character. + + Returns: + A generator yielding the requested dataset items + """ + cache_size = 1000 + first_item = offset + + # If there is no limit, set last_item to None until we get the total from the first API response + if limit is None: + last_item = None + else: + last_item = offset + limit + + current_offset = first_item + while last_item is None or current_offset < last_item: + if last_item is None: + current_limit = cache_size + else: + current_limit = min(cache_size, last_item - current_offset) + + current_items_pagination = self.list_items( + offset=current_offset, + limit=current_limit, + clean=clean, + desc=desc, + fields=fields, + omit=omit, + unwind=unwind, + skip_empty=skip_empty, + skip_hidden=skip_hidden, + ) + + current_offset += current_items_pagination['count'] + if last_item is None or current_items_pagination['total'] < last_item: + last_item = current_items_pagination['total'] + + yield from current_items_pagination['items'] + + def download_items( + self, + item_format: str = 'json', + *, + offset: Optional[int] = None, + limit: Optional[int] = None, + desc: Optional[bool] = None, + clean: Optional[bool] = None, + bom: Optional[bool] = None, + delimiter: Optional[str] = None, + fields: Optional[List[str]] = None, + omit: Optional[List[str]] = None, + unwind: Optional[str] = None, + skip_empty: Optional[bool] = None, + skip_header_row: Optional[bool] = None, + skip_hidden: Optional[bool] = None, + xml_root: Optional[str] = None, + xml_row: Optional[str] = None, + ) -> bytes: + """Download the items in the dataset as raw bytes. + + https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items + + Args: + item_format(str): Format of the results, possible values are: json, jsonl, csv, html, xlsx, xml and rss. The default value is json. + offset: (int, optional): Number of items that should be skipped at the start. The default value is 0 + limit: (int, optional): Maximum number of items to return. By default there is no limit. + desc (bool, optional): By default, results are returned in the same order as they were stored. + To reverse the order, set this parameter to True. + clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character). + The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters. + Note that since some objects might be skipped from the output, that the result might contain less items than the limit value. + bom (bool, optional): All text responses are encoded in UTF-8 encoding. + By default, csv files are prefixed with the UTF-8 Byte Order Mark (BOM), + while json, jsonl, xml, html and rss files are not. If you want to override this default behavior, + specify bom=True query parameter to include the BOM or bom=False to skip it. + delimiter (str, optional): A delimiter character for CSV files. The default delimiter is a simple comma (,). + fields (list[str], optional): A list of fields which should be picked from the items, + only these fields will remain in the resulting record objects. + Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter. + You can use this feature to effectively fix the output format. + omit: (list[str], optional): A list of fields which should be omitted from the items. + unwind: (str, optional): Name of a field which should be unwound. + If the field is an array then every element of the array will become a separate record and merged with parent object. + If the unwound field is an object then it is merged with the parent object. + If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object, + then the item gets preserved as it is. Note that the unwound items ignore the desc parameter. + skip_empty (bool, optional): If True, then empty items are skipped from the output. + Note that if used, the results might contain less items than the limit value. + skip_header_row (bool, optional): If True, then header row in the csv format is skipped. + skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character. + xml_root (str, optional): Overrides default root element name of xml output. By default the root element is items. + xml_row (str, optional): Overrides default element name that wraps each page or page function result object in xml output. + By default the element name is item. + + Returns: + The dataset items in the specified format, either as raw bytes or a file-like object + """ + request_params = self._params( + format=item_format, + offset=offset, + limit=limit, + desc=desc, + clean=clean, + bom=bom, + delimiter=delimiter, + fields=fields, + omit=omit, + unwind=unwind, + skipEmpty=skip_empty, + skipHeaderRow=skip_header_row, + skipHidden=skip_hidden, + xmlRoot=xml_root, + xmlRow=xml_row, + ) + + response = self.http_client.call( + url=self._url('items'), + method='GET', + params=request_params, + parse_response=False, + ) + + return response.content + + def stream_items( + self, + item_format: str = 'json', + *, + offset: Optional[int] = None, + limit: Optional[int] = None, + desc: Optional[bool] = None, + clean: Optional[bool] = None, + bom: Optional[bool] = None, + delimiter: Optional[str] = None, + fields: Optional[List[str]] = None, + omit: Optional[List[str]] = None, + unwind: Optional[str] = None, + skip_empty: Optional[bool] = None, + skip_header_row: Optional[bool] = None, + skip_hidden: Optional[bool] = None, + xml_root: Optional[str] = None, + xml_row: Optional[str] = None, + ) -> io.IOBase: + """Retrieve the items in the dataset as a file-like object. + + https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items + + Args: + item_format(str): Format of the results, possible values are: json, jsonl, csv, html, xlsx, xml and rss. The default value is json. + offset: (int, optional): Number of items that should be skipped at the start. The default value is 0 + limit: (int, optional): Maximum number of items to return. By default there is no limit. + desc (bool, optional): By default, results are returned in the same order as they were stored. + To reverse the order, set this parameter to True. + clean (bool, optional): If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character). + The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters. + Note that since some objects might be skipped from the output, that the result might contain less items than the limit value. + bom (bool, optional): All text responses are encoded in UTF-8 encoding. + By default, csv files are prefixed with the UTF-8 Byte Order Mark (BOM), + while json, jsonl, xml, html and rss files are not. If you want to override this default behavior, + specify bom=True query parameter to include the BOM or bom=False to skip it. + delimiter (str, optional): A delimiter character for CSV files. The default delimiter is a simple comma (,). + fields (list[str], optional): A list of fields which should be picked from the items, + only these fields will remain in the resulting record objects. + Note that the fields in the outputted items are sorted the same way as they are specified in the fields parameter. + You can use this feature to effectively fix the output format. + omit: (list[str], optional): A list of fields which should be omitted from the items. + unwind: (str, optional): Name of a field which should be unwound. + If the field is an array then every element of the array will become a separate record and merged with parent object. + If the unwound field is an object then it is merged with the parent object. + If the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object, + then the item gets preserved as it is. Note that the unwound items ignore the desc parameter. + skip_empty (bool, optional): If True, then empty items are skipped from the output. + Note that if used, the results might contain less items than the limit value. + skip_header_row (bool, optional): If True, then header row in the csv format is skipped. + skip_hidden (bool, optional): If True, then hidden fields are skipped from the output, i.e. fields starting with the # character. + xml_root (str, optional): Overrides default root element name of xml output. By default the root element is items. + xml_row (str, optional): Overrides default element name that wraps each page or page function result object in xml output. + By default the element name is item. + + Returns: + The dataset items in the specified format, either as raw bytes or a file-like object + """ + request_params = self._params( + format=item_format, + offset=offset, + limit=limit, + desc=desc, + clean=clean, + bom=bom, + delimiter=delimiter, + fields=fields, + omit=omit, + unwind=unwind, + skipEmpty=skip_empty, + skipHeaderRow=skip_header_row, + skipHidden=skip_hidden, + xmlRoot=xml_root, + xmlRow=xml_row, + ) + + response = self.http_client.call( + url=self._url('items'), + method='GET', + params=request_params, + stream=True, + parse_response=False, + ) + + response.raw.decode_content = True + # response.raw is the raw urllib3 response, which subclasses IOBase + return cast(io.IOBase, response.raw) + + def push_items(self, items: JSONSerializable) -> None: + """Push items to the dataset. + + https://docs.apify.com/api/v2#/reference/datasets/item-collection/put-items + + Args: + items: The items which to push in the dataset. Either a stringified JSON, a dictionary, or a list of strings or dictionaries. + """ + data = None + json = None + + if isinstance(items, str): + data = items + else: + json = items + + self.http_client.call( + url=self._url('items'), + method='POST', + headers={'content-type': 'application/json; charset=utf-8'}, + params=self._params(), + data=data, + json=json, + ) diff --git a/src/apify_client/clients/resource_clients/dataset_collection.py b/src/apify_client/clients/resource_clients/dataset_collection.py new file mode 100644 index 00000000..ad5dfb34 --- /dev/null +++ b/src/apify_client/clients/resource_clients/dataset_collection.py @@ -0,0 +1,40 @@ +from typing import Any, Dict, Optional + +from ..base.resource_collection_client import ResourceCollectionClient + + +class DatasetCollectionClient(ResourceCollectionClient): + """Sub-client for manipulating datasets.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Initialize the DatasetCollectionClient with the passed arguments.""" + super().__init__(*args, resource_path='datasets', **kwargs) + + def list(self, *, unnamed: Optional[bool] = None, limit: Optional[int] = None, offset: Optional[int] = None, desc: Optional[bool] = None) -> Dict: + """List the available datasets. + + https://docs.apify.com/api/v2#/reference/datasets/dataset-collection/get-list-of-datasets + + Args: + unnamed (bool, optional): Whether to include unnamed datasets in the list + limit (int, optional): How many datasets to retrieve + offset (int, optional): What dataset to include as first when retrieving the list + desc (bool, optional): Whether to sort the datasets in descending order based on their modification date + + Returns: + The list of available datasets matching the specified filters. + """ + return self._list(unnamed=unnamed, limit=limit, offset=offset, desc=desc) + + def get_or_create(self, *, name: str = '') -> Dict: + """Retrieve a named dataset, or create a new one when it doesn't exist. + + https://docs.apify.com/api/v2#/reference/datasets/dataset-collection/create-dataset + + Args: + name (str, optional): The name of the dataset to retrieve or create. + + Returns: + The retrieved dataset. + """ + return self._get_or_create(name=name) diff --git a/src/apify_client/clients/resource_clients/key_value_store.py b/src/apify_client/clients/resource_clients/key_value_store.py new file mode 100644 index 00000000..667e36e2 --- /dev/null +++ b/src/apify_client/clients/resource_clients/key_value_store.py @@ -0,0 +1,153 @@ +import json +from typing import Any, Dict, Optional + +from ..._errors import ApifyApiError +from ..._utils import _catch_not_found_or_throw, _is_file_or_bytes, _parse_date_fields, _pluck_data +from ..base.resource_client import ResourceClient + + +class KeyValueStoreClient(ResourceClient): + """Sub-client for manipulating a single key-value store.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Initialize the KeyValueStoreClient.""" + super().__init__(*args, resource_path='key-value-stores', **kwargs) + + def get(self) -> Optional[Dict]: + """Retrieve the key-value store. + + https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/get-store + + Returns: + The retrieved key-value store + """ + return self._get() + + def update(self, new_fields: Dict) -> Optional[Dict]: + """Update the key-value store with specified fields. + + https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/update-store + + Args: + new_fields (dict): The fields of the key-value store to update + + Returns: + The updated key-value store + """ + return self._update(new_fields) + + def delete(self) -> None: + """Delete the key-value store. + + https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/delete-store + """ + return self._delete() + + def list_keys(self, *, limit: Optional[int] = None, exclusive_start_key: Optional[str] = None) -> Dict: + """List the keys in the key-value store. + + https://docs.apify.com/api/v2#/reference/key-value-stores/key-collection/get-list-of-keys + + Args: + limit (int, optional): Number of keys to be returned. Maximum value is 1000 + exclusive_start_key (str, optional): All keys up to this one (including) are skipped from the result + """ + request_params = self._params( + limit=limit, + exclusiveStartKey=exclusive_start_key, + ) + + response = self.http_client.call( + url=self._url('keys'), + method='GET', + params=request_params, + ) + + return _parse_date_fields(_pluck_data(response.json())) + + def get_record(self, key: str, *, as_bytes: bool = False, as_file: bool = False) -> Optional[Dict]: + """Retrieve the given record from the key-value store. + + https://docs.apify.com/api/v2#/reference/key-value-stores/record/get-record + + Args: + key (str): Key of the record to retrieve + as_bytes (bool, optional): Whether to retrieve the record as unparsed bytes, default False + as_file (bool, optional): Whether to retrieve the record as a file-like object, default False + """ + try: + # TODO revisit the as_bytes and as_file parameters when we decide how to rewrite the record-getting functions + if as_bytes and as_file: + raise ValueError('You cannot have both as_bytes and as_file set.') + + response = self.http_client.call( + url=self._url(f'records/{key}'), + method='GET', + params=self._params(), + stream=as_file, + parse_response=(not as_bytes and not as_file), + ) + + return { + 'key': key, + 'value': response._maybe_parsed_body, # type: ignore + 'content_type': response.headers['content-type'], + } + + except ApifyApiError as exc: + _catch_not_found_or_throw(exc) + + return None + + def set_record(self, key: str, value: Any, content_type: Optional[str] = None) -> None: + """Set a value to the given record in the key-value store. + + https://docs.apify.com/api/v2#/reference/key-value-stores/record/put-record + + Args: + key (str): The key of the record to save the value to + value (Any): The value to save into the record + content_type (str, optional): The content type of the saved value + """ + # TODO revisit this when it's decided whether we keep using the signed URL route or not + + headers = None + + if not content_type: + if _is_file_or_bytes(value): + content_type = 'application/octet-stream' + elif isinstance(value, str): + content_type = 'text/plain; charset=utf-8' + else: + content_type = 'application/json; charset=utf-8' + + # TODO encode to utf-8 + + if 'application/json' in content_type and not _is_file_or_bytes(value) and not isinstance(value, str): + # TODO decide if we should keep indenting the JSON or not, it could increase the record size considerably, + # but if we gzip the requests it should not matter too much with transfer size + value = json.dumps(value, indent=2) + + headers = {'content-type': content_type} + + self.http_client.call( + url=self._url(f'records/{key}'), + method='PUT', + params=self._params(), + data=value, + headers=headers, + ) + + def delete_record(self, key: str) -> None: + """Delete the specified record from the key-value store. + + https://docs.apify.com/api/v2#/reference/key-value-stores/record/delete-record + + Args: + key (str): The key of the record which to delete + """ + self.http_client.call( + url=self._url(f'records/{key}'), + method='DELETE', + params=self._params(), + ) diff --git a/src/apify_client/clients/resource_clients/key_value_store_collection.py b/src/apify_client/clients/resource_clients/key_value_store_collection.py new file mode 100644 index 00000000..95456a4a --- /dev/null +++ b/src/apify_client/clients/resource_clients/key_value_store_collection.py @@ -0,0 +1,40 @@ +from typing import Any, Dict, Optional + +from ..base.resource_collection_client import ResourceCollectionClient + + +class KeyValueStoreCollectionClient(ResourceCollectionClient): + """Sub-client for manipulating key-value stores.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Initialize the KeyValueStoreCollectionClient with the passed arguments.""" + super().__init__(*args, resource_path='key-value-stores', **kwargs) + + def list(self, *, unnamed: Optional[bool] = None, limit: Optional[int] = None, offset: Optional[int] = None, desc: Optional[bool] = None) -> Dict: + """List the available key-value stores. + + https://docs.apify.com/api/v2#/reference/key-value-stores/store-collection/get-list-of-key-value-stores + + Args: + unnamed: Whether to include unnamed key-value stores in the list + limit: How many key-value stores to retrieve + offset: What key-value store to include as first when retrieving the list + desc: Whether to sort the key-value stores in descending order based on their modification date + + Returns: + The list of available key-value stores matching the specified filters. + """ + return self._list(unnamed=unnamed, limit=limit, offset=offset, desc=desc) + + def get_or_create(self, *, name: str = '') -> Dict: + """Retrieve a named key-value store, or create a new one when it doesn't exist. + + https://docs.apify.com/api/v2#/reference/key-value-stores/store-collection/create-key-value-store + + Args: + name: The name of the key-value store to retrieve or create. + + Returns: + The retrieved key-value store. + """ + return self._get_or_create(name=name) diff --git a/src/apify_client/clients/resource_clients/log.py b/src/apify_client/clients/resource_clients/log.py new file mode 100644 index 00000000..b5582b58 --- /dev/null +++ b/src/apify_client/clients/resource_clients/log.py @@ -0,0 +1,63 @@ +import io +from typing import Any, Optional, cast + +from ..._errors import ApifyApiError +from ..._utils import _catch_not_found_or_throw +from ..base.resource_client import ResourceClient + + +class LogClient(ResourceClient): + """Sub-client for manipulating logs.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Initialize the LogClient.""" + super().__init__(*args, resource_path='logs', **kwargs) + + def get(self) -> Optional[str]: + """Retrieve the log as text. + + https://docs.apify.com/api/v2#/reference/logs/log/get-log + + Returns: + The retrieved log + """ + try: + response = self.http_client.call( + url=self.url, + method='GET', + params=self._params(), + ) + + return response.text + + except ApifyApiError as exc: + _catch_not_found_or_throw(exc) + + return None + + def stream(self) -> Optional[io.IOBase]: + """Retrieve the log as a file-like object. + + https://docs.apify.com/api/v2#/reference/logs/log/get-log + + Returns: + The retrieved log + """ + try: + response = self.http_client.call( + url=self.url, + method='GET', + params=self._params(), + stream=True, + parse_response=False, + ) + + response.raw.decode_content = True + # TODO explain response.raw.close() + # response.raw is the raw urllib3 response, which subclasses IOBase + return cast(io.IOBase, response.raw) + + except ApifyApiError as exc: + _catch_not_found_or_throw(exc) + + return None diff --git a/src/apify_client/clients/resource_clients/request_queue.py b/src/apify_client/clients/resource_clients/request_queue.py new file mode 100644 index 00000000..7c405451 --- /dev/null +++ b/src/apify_client/clients/resource_clients/request_queue.py @@ -0,0 +1,153 @@ +from typing import Any, Dict, Optional + +from ..._errors import ApifyApiError +from ..._utils import _catch_not_found_or_throw, _parse_date_fields, _pluck_data +from ..base.resource_client import ResourceClient + + +class RequestQueueClient(ResourceClient): + """Sub-client for manipulating a single request queue.""" + + def __init__(self, *args: Any, client_key: Optional[str] = None, **kwargs: Any) -> None: + """Initialize the RequestQueueClient. + + Args: + client_key (str): A unique identifier of the client accessing the request queue + """ + super().__init__(*args, resource_path='request-queues', **kwargs) + self.client_key = client_key + + def get(self) -> Optional[Dict]: + """Retrieve the request queue. + + https://docs.apify.com/api/v2#/reference/request-queues/queue/get-request-queue + + Returns: + The retrieved request queue + """ + return self._get() + + def update(self, new_fields: Dict) -> Optional[Dict]: + """Update the request queue with specified fields. + + https://docs.apify.com/api/v2#/reference/request-queues/queue/update-request-queue + + Args: + new_fields (dict): The fields of the request queue to update + + Returns: + The updated request queue + """ + return self._update(new_fields) + + def delete(self) -> None: + """Delete the request queue. + + https://docs.apify.com/api/v2#/reference/request-queues/queue/delete-request-queue + """ + return self._delete() + + def list_head(self, *, limit: Optional[int] = None) -> Dict: + """Retrieve a given number of requests from the beginning of the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/queue-head/get-head + + Args: + limit (int): How many requests to retrieve + """ + request_params = self._params(limit=limit, clientKey=self.client_key) + + response = self.http_client.call( + url=self._url('head'), + method='GET', + params=request_params, + ) + + return _parse_date_fields(_pluck_data(response.json())) + + def add_request(self, request: Dict, *, forefront: Optional[bool] = None) -> Dict: + """Add a request to the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request + + Args: + request (dict): The request to add to the queue + forefront (bool): Whether to add the request to the head or the end of the queue + """ + request_params = self._params( + forefront=forefront, + clientKey=self.client_key, + ) + + response = self.http_client.call( + url=self._url('requests'), + method='POST', + json=request, + params=request_params, + ) + + return _parse_date_fields(_pluck_data(response.json())) + + def get_request(self, request_id: str) -> Optional[Dict]: + """Retrieve a request from the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request/get-request + + Args: + request_id (str): ID of the request to retrieve + """ + try: + response = self.http_client.call( + url=self._url(f'requests/{request_id}'), + method='GET', + params=self._params(), + ) + return _parse_date_fields(_pluck_data(response.json())) + + except ApifyApiError as exc: + _catch_not_found_or_throw(exc) + + return None + + def update_request(self, request: Dict, *, forefront: Optional[bool] = None) -> Optional[Dict]: + """Update a request in the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request/update-request + + Args: + request (dict): The updated request + forefront (bool): Whether to put the updated request in the beginning or the end of the queue + """ + request_id = request['id'] + + request_params = self._params( + forefront=forefront, + clientKey=self.client_key, + ) + + response = self.http_client.call( + url=self._url(f'requests/{request_id}'), + method='PUT', + json=request, + params=request_params, + ) + + return _parse_date_fields(_pluck_data(response.json())) + + def delete_request(self, request_id: str) -> None: + """Delete a request from the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request/delete-request + + Args: + request_id (str): ID of the request to delete. + """ + request_params = self._params( + clientKey=self.client_key, + ) + + self.http_client.call( + url=self._url(f'requests/{request_id}'), + method='DELETE', + params=request_params, + ) diff --git a/src/apify_client/clients/resource_clients/request_queue_collection.py b/src/apify_client/clients/resource_clients/request_queue_collection.py new file mode 100644 index 00000000..fdf25fc9 --- /dev/null +++ b/src/apify_client/clients/resource_clients/request_queue_collection.py @@ -0,0 +1,40 @@ +from typing import Any, Dict, Optional + +from ..base.resource_collection_client import ResourceCollectionClient + + +class RequestQueueCollectionClient(ResourceCollectionClient): + """Sub-client for manipulating request queues.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Initialize the RequestQueueCollectionClient with the passed arguments.""" + super().__init__(*args, resource_path='request-queues', **kwargs) + + def list(self, *, unnamed: Optional[bool] = None, limit: Optional[int] = None, offset: Optional[int] = None, desc: Optional[bool] = None) -> Dict: + """List the available request queues. + + https://docs.apify.com/api/v2#/reference/request-queues/queue-collection/get-list-of-request-queues + + Args: + unnamed (bool): Whether to include unnamed request queues in the list + limit (int): How many request queues to retrieve + offset (int): What request queue to include as first when retrieving the list + desc (bool): Whether to sort therequest queues in descending order based on their modification date + + Returns: + The list of available request queues matching the specified filters. + """ + return self._list(unnamed=unnamed, limit=limit, offset=offset, desc=desc) + + def get_or_create(self, *, name: str = '') -> Dict: + """Retrieve a named request queue, or create a new one when it doesn't exist. + + https://docs.apify.com/api/v2#/reference/request-queues/queue-collection/create-request-queue + + Args: + name (str): The name of the request queue to retrieve or create. + + Returns: + The retrieved request queue. + """ + return self._get_or_create(name=name) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 00000000..a5c5be03 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,150 @@ +import io +import time +import unittest +from datetime import datetime, timezone +from typing import Any, Callable + +from apify_client._utils import ( + _is_content_type_json, + _is_content_type_text, + _is_content_type_xml, + _is_file_or_bytes, + _parse_date_fields, + _pluck_data, + _retry_with_exp_backoff, + _to_safe_id, +) + + +class UtilsTest(unittest.TestCase): + def test__to_safe_id(self) -> None: + self.assertEqual(_to_safe_id('abc'), 'abc') + self.assertEqual(_to_safe_id('abc/def'), 'abc~def') + self.assertEqual(_to_safe_id('abc~def'), 'abc~def') + + def test__parse_date_fields(self) -> None: + # works correctly on empty dicts + self.assertEqual(_parse_date_fields({}), {}) + + # correctly parses dates on fields ending with -At + expected_datetime = datetime(2016, 11, 14, 11, 10, 52, 425000, timezone.utc) + self.assertEqual(_parse_date_fields({'createdAt': '2016-11-14T11:10:52.425Z'}), {'createdAt': expected_datetime}) + + # doesn't parse dates on fields not ending with -At + self.assertEqual(_parse_date_fields({'saveUntil': '2016-11-14T11:10:52.425Z'}), {'saveUntil': '2016-11-14T11:10:52.425Z'}) + + # parses dates in dicts in lists + expected_datetime = datetime(2016, 11, 14, 11, 10, 52, 425000, timezone.utc) + self.assertEqual(_parse_date_fields([{'createdAt': '2016-11-14T11:10:52.425Z'}]), [{'createdAt': expected_datetime}]) + + # parses nested dates + expected_datetime = datetime(2020, 2, 29, 10, 9, 8, 100000, timezone.utc) + self.assertEqual( + _parse_date_fields({'a': {'b': {'c': {'createdAt': '2020-02-29T10:09:08.100Z'}}}}), + {'a': {'b': {'c': {'createdAt': expected_datetime}}}}, + ) + + # doesn't parse dates nested too deep + expected_datetime = datetime(2020, 2, 29, 10, 9, 8, 100000, timezone.utc) + self.assertEqual( + _parse_date_fields({'a': {'b': {'c': {'d': {'createdAt': '2020-02-29T10:09:08.100Z'}}}}}), + {'a': {'b': {'c': {'d': {'createdAt': '2020-02-29T10:09:08.100Z'}}}}}, + ) + + # doesn't die when the date can't be parsed + self.assertEqual(_parse_date_fields({'createdAt': 'NOT_A_DATE'}), {'createdAt': 'NOT_A_DATE'}) + + def test__pluck_data(self) -> None: + # works correctly when data is present + self.assertEqual(_pluck_data({'data': {}}), {}) + self.assertEqual(_pluck_data({'a': 'b', 'data': {'b': 'c'}}), {'b': 'c'}) + + # throws the right error when it is not + with self.assertRaises(ValueError): + _pluck_data({'a': 'b'}) + with self.assertRaises(ValueError): + _pluck_data(None) + with self.assertRaises(ValueError): + _pluck_data('{"a": "b"}') + + def test__is_content_type_json(self) -> None: + # returns True for the right content types + self.assertEqual(_is_content_type_json('application/json'), True) + self.assertEqual(_is_content_type_json('application/jsonc'), True) + # returns False for bad content types + self.assertEqual(_is_content_type_json('application/xml'), False) + self.assertEqual(_is_content_type_json('application/ld+json'), False) + + def test__is_content_type_xml(self) -> None: + # returns True for the right content types + self.assertEqual(_is_content_type_xml('application/xml'), True) + self.assertEqual(_is_content_type_xml('application/xhtml+xml'), True) + # returns False for bad content types + self.assertEqual(_is_content_type_xml('application/json'), False) + self.assertEqual(_is_content_type_xml('text/html'), False) + + def test__is_content_type_text(self) -> None: + # returns True for the right content types + self.assertEqual(_is_content_type_text('text/html'), True) + self.assertEqual(_is_content_type_text('text/plain'), True) + # returns False for bad content types + self.assertEqual(_is_content_type_text('application/json'), False) + self.assertEqual(_is_content_type_text('application/text'), False) + + def test__is_file_or_bytes(self) -> None: + # returns True for the right value types + self.assertEqual(_is_file_or_bytes(b'abc'), True) + self.assertEqual(_is_file_or_bytes(bytearray.fromhex('F0F1F2')), True) + self.assertEqual(_is_file_or_bytes(io.BytesIO(b'\x00\x01\x02')), True) + + # returns False for bad value types + self.assertEqual(_is_file_or_bytes('abc'), False) + self.assertEqual(_is_file_or_bytes(['a', 'b', 'c']), False) + self.assertEqual(_is_file_or_bytes({'a': 'b'}), False) + self.assertEqual(_is_file_or_bytes(None), False) + + def test__retry_with_exp_backoff(self) -> None: + attempt_counter = 0 + + class RetryableException(Exception): + pass + + class BailException(Exception): + pass + + def returns_on_fifth_attempt(bail: Callable, attempt: int) -> Any: + nonlocal attempt_counter + attempt_counter += 1 + + if attempt == 5: + return 'SUCCESS' + raise RetryableException() + + def bails_on_third_attempt(bail: Callable, attempt: int) -> Any: + nonlocal attempt_counter + attempt_counter += 1 + + if attempt == 3: + bail(BailException()) + raise RetryableException() + + # Returns the correct result after the correct time (should take 100 + 200 + 400 + 800 = 1500 ms) + start = time.time() + result = _retry_with_exp_backoff(returns_on_fifth_attempt, backoff_base_millis=100, backoff_factor=2, random_factor=0) + elapsed_time_seconds = time.time() - start + self.assertEqual(result, 'SUCCESS') + self.assertEqual(attempt_counter, 5) + self.assertGreater(elapsed_time_seconds, 1.4) + self.assertLess(elapsed_time_seconds, 2.0) + + # Stops retrying when failed for max_retries times + attempt_counter = 0 + with self.assertRaises(RetryableException): + _retry_with_exp_backoff(returns_on_fifth_attempt, max_retries=4, backoff_base_millis=1) + self.assertEqual(attempt_counter, 4) + + # Bails when the bail function is called + attempt_counter = 0 + with self.assertRaises(BailException): + _retry_with_exp_backoff(bails_on_third_attempt, backoff_base_millis=1) + self.assertEqual(attempt_counter, 3)