From 3860fb259a68f76672f7c5044d2ffaf19b3f9d0c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 20 Aug 2019 15:34:19 +0100 Subject: [PATCH 1/3] Move Dispatcher, AsyncDispatcher into dispatch/base.py --- httpx/__init__.py | 3 +- httpx/client.py | 3 +- httpx/dispatch/asgi.py | 2 +- httpx/dispatch/base.py | 112 ++++++++++++++++++++++++++++++ httpx/dispatch/connection.py | 3 +- httpx/dispatch/connection_pool.py | 3 +- httpx/dispatch/threaded.py | 3 +- httpx/dispatch/wsgi.py | 2 +- httpx/interfaces.py | 109 +---------------------------- 9 files changed, 124 insertions(+), 116 deletions(-) create mode 100644 httpx/dispatch/base.py diff --git a/httpx/__init__.py b/httpx/__init__.py index 69759f4d75..ec9fa5273e 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -13,6 +13,7 @@ TimeoutTypes, VerifyTypes, ) +from .dispatch.base import AsyncDispatcher, Dispatcher from .dispatch.connection import HTTPConnection from .dispatch.connection_pool import ConnectionPool from .exceptions import ( @@ -33,13 +34,11 @@ WriteTimeout, ) from .interfaces import ( - AsyncDispatcher, BaseBackgroundManager, BasePoolSemaphore, BaseReader, BaseWriter, ConcurrencyBackend, - Dispatcher, ) from .models import ( URL, diff --git a/httpx/client.py b/httpx/client.py index 04dbfea538..f2e39a0d40 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -17,6 +17,7 @@ VerifyTypes, ) from .dispatch.asgi import ASGIDispatch +from .dispatch.base import AsyncDispatcher, Dispatcher from .dispatch.connection_pool import ConnectionPool from .dispatch.threaded import ThreadedDispatcher from .dispatch.wsgi import WSGIDispatch @@ -27,7 +28,7 @@ RedirectLoop, TooManyRedirects, ) -from .interfaces import AsyncDispatcher, ConcurrencyBackend, Dispatcher +from .interfaces import ConcurrencyBackend from .models import ( URL, AsyncRequest, diff --git a/httpx/dispatch/asgi.py b/httpx/dispatch/asgi.py index 01313fedf4..22f93811cf 100644 --- a/httpx/dispatch/asgi.py +++ b/httpx/dispatch/asgi.py @@ -1,8 +1,8 @@ import asyncio import typing +from .base import AsyncDispatcher from ..config import CertTypes, TimeoutTypes, VerifyTypes -from ..interfaces import AsyncDispatcher from ..models import AsyncRequest, AsyncResponse diff --git a/httpx/dispatch/base.py b/httpx/dispatch/base.py new file mode 100644 index 0000000000..b3d3c1bdc0 --- /dev/null +++ b/httpx/dispatch/base.py @@ -0,0 +1,112 @@ +import ssl +import typing +from types import TracebackType + +from ..config import CertTypes, PoolLimits, TimeoutConfig, TimeoutTypes, VerifyTypes +from ..models import ( + AsyncRequest, + AsyncRequestData, + AsyncResponse, + HeaderTypes, + QueryParamTypes, + Request, + RequestData, + Response, + URLTypes, +) + + +class AsyncDispatcher: + """ + Base class for async dispatcher classes, that handle sending the request. + + Stubs out the interface, as well as providing a `.request()` convenience + implementation, to make it easy to use or test stand-alone dispatchers, + without requiring a complete `Client` instance. + """ + + async def request( + self, + method: str, + url: URLTypes, + *, + data: AsyncRequestData = b"", + params: QueryParamTypes = None, + headers: HeaderTypes = None, + verify: VerifyTypes = None, + cert: CertTypes = None, + timeout: TimeoutTypes = None, + ) -> AsyncResponse: + request = AsyncRequest(method, url, data=data, params=params, headers=headers) + return await self.send(request, verify=verify, cert=cert, timeout=timeout) + + async def send( + self, + request: AsyncRequest, + verify: VerifyTypes = None, + cert: CertTypes = None, + timeout: TimeoutTypes = None, + ) -> AsyncResponse: + raise NotImplementedError() # pragma: nocover + + async def close(self) -> None: + pass # pragma: nocover + + async def __aenter__(self) -> "AsyncDispatcher": + return self + + async def __aexit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + await self.close() + + +class Dispatcher: + """ + Base class for synchronous dispatcher classes, that handle sending the request. + + Stubs out the interface, as well as providing a `.request()` convenience + implementation, to make it easy to use or test stand-alone dispatchers, + without requiring a complete `Client` instance. + """ + + def request( + self, + method: str, + url: URLTypes, + *, + data: RequestData = b"", + params: QueryParamTypes = None, + headers: HeaderTypes = None, + verify: VerifyTypes = None, + cert: CertTypes = None, + timeout: TimeoutTypes = None, + ) -> Response: + request = Request(method, url, data=data, params=params, headers=headers) + return self.send(request, verify=verify, cert=cert, timeout=timeout) + + def send( + self, + request: Request, + verify: VerifyTypes = None, + cert: CertTypes = None, + timeout: TimeoutTypes = None, + ) -> Response: + raise NotImplementedError() # pragma: nocover + + def close(self) -> None: + pass # pragma: nocover + + def __enter__(self) -> "Dispatcher": + return self + + def __exit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + self.close() diff --git a/httpx/dispatch/connection.py b/httpx/dispatch/connection.py index d82ea5db21..1aa79c60e4 100644 --- a/httpx/dispatch/connection.py +++ b/httpx/dispatch/connection.py @@ -2,6 +2,7 @@ import ssl import typing +from .base import AsyncDispatcher from ..concurrency.asyncio import AsyncioBackend from ..config import ( DEFAULT_TIMEOUT_CONFIG, @@ -13,7 +14,7 @@ TimeoutTypes, VerifyTypes, ) -from ..interfaces import AsyncDispatcher, ConcurrencyBackend +from ..interfaces import ConcurrencyBackend from ..models import AsyncRequest, AsyncResponse, Origin from .http2 import HTTP2Connection from .http11 import HTTP11Connection diff --git a/httpx/dispatch/connection_pool.py b/httpx/dispatch/connection_pool.py index b88d142c4c..d6ea027eef 100644 --- a/httpx/dispatch/connection_pool.py +++ b/httpx/dispatch/connection_pool.py @@ -1,5 +1,6 @@ import typing +from .base import AsyncDispatcher from ..concurrency.asyncio import AsyncioBackend from ..config import ( DEFAULT_POOL_LIMITS, @@ -10,7 +11,7 @@ TimeoutTypes, VerifyTypes, ) -from ..interfaces import AsyncDispatcher, ConcurrencyBackend +from ..interfaces import ConcurrencyBackend from ..models import AsyncRequest, AsyncResponse, Origin from .connection import HTTPConnection diff --git a/httpx/dispatch/threaded.py b/httpx/dispatch/threaded.py index 602001092d..0381cccde2 100644 --- a/httpx/dispatch/threaded.py +++ b/httpx/dispatch/threaded.py @@ -1,5 +1,6 @@ +from .base import AsyncDispatcher, Dispatcher from ..config import CertTypes, TimeoutTypes, VerifyTypes -from ..interfaces import AsyncDispatcher, ConcurrencyBackend, Dispatcher +from ..interfaces import ConcurrencyBackend from ..models import ( AsyncRequest, AsyncRequestData, diff --git a/httpx/dispatch/wsgi.py b/httpx/dispatch/wsgi.py index e1109c05a3..0cbe1095e2 100644 --- a/httpx/dispatch/wsgi.py +++ b/httpx/dispatch/wsgi.py @@ -1,8 +1,8 @@ import io import typing +from .base import Dispatcher from ..config import CertTypes, TimeoutTypes, VerifyTypes -from ..interfaces import Dispatcher from ..models import Request, Response diff --git a/httpx/interfaces.py b/httpx/interfaces.py index 6a39175f3e..628c110353 100644 --- a/httpx/interfaces.py +++ b/httpx/interfaces.py @@ -2,114 +2,7 @@ import typing from types import TracebackType -from .config import CertTypes, PoolLimits, TimeoutConfig, TimeoutTypes, VerifyTypes -from .models import ( - AsyncRequest, - AsyncRequestData, - AsyncResponse, - HeaderTypes, - QueryParamTypes, - Request, - RequestData, - Response, - URLTypes, -) - - -class AsyncDispatcher: - """ - Base class for async dispatcher classes, that handle sending the request. - - Stubs out the interface, as well as providing a `.request()` convenience - implementation, to make it easy to use or test stand-alone dispatchers, - without requiring a complete `Client` instance. - """ - - async def request( - self, - method: str, - url: URLTypes, - *, - data: AsyncRequestData = b"", - params: QueryParamTypes = None, - headers: HeaderTypes = None, - verify: VerifyTypes = None, - cert: CertTypes = None, - timeout: TimeoutTypes = None, - ) -> AsyncResponse: - request = AsyncRequest(method, url, data=data, params=params, headers=headers) - return await self.send(request, verify=verify, cert=cert, timeout=timeout) - - async def send( - self, - request: AsyncRequest, - verify: VerifyTypes = None, - cert: CertTypes = None, - timeout: TimeoutTypes = None, - ) -> AsyncResponse: - raise NotImplementedError() # pragma: nocover - - async def close(self) -> None: - pass # pragma: nocover - - async def __aenter__(self) -> "AsyncDispatcher": - return self - - async def __aexit__( - self, - exc_type: typing.Type[BaseException] = None, - exc_value: BaseException = None, - traceback: TracebackType = None, - ) -> None: - await self.close() - - -class Dispatcher: - """ - Base class for synchronous dispatcher classes, that handle sending the request. - - Stubs out the interface, as well as providing a `.request()` convenience - implementation, to make it easy to use or test stand-alone dispatchers, - without requiring a complete `Client` instance. - """ - - def request( - self, - method: str, - url: URLTypes, - *, - data: RequestData = b"", - params: QueryParamTypes = None, - headers: HeaderTypes = None, - verify: VerifyTypes = None, - cert: CertTypes = None, - timeout: TimeoutTypes = None, - ) -> Response: - request = Request(method, url, data=data, params=params, headers=headers) - return self.send(request, verify=verify, cert=cert, timeout=timeout) - - def send( - self, - request: Request, - verify: VerifyTypes = None, - cert: CertTypes = None, - timeout: TimeoutTypes = None, - ) -> Response: - raise NotImplementedError() # pragma: nocover - - def close(self) -> None: - pass # pragma: nocover - - def __enter__(self) -> "Dispatcher": - return self - - def __exit__( - self, - exc_type: typing.Type[BaseException] = None, - exc_value: BaseException = None, - traceback: TracebackType = None, - ) -> None: - self.close() +from .config import PoolLimits, TimeoutConfig class BaseReader: From 7ab2ce586b77566a7b73b8eaf3d7c17e7e68c238 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 20 Aug 2019 15:41:42 +0100 Subject: [PATCH 2/3] Move concurrency interfaces into concurrency/base.py --- httpx/__init__.py | 14 +++---- httpx/client.py | 2 +- httpx/concurrency/asyncio.py | 39 ++------------------ httpx/{interfaces.py => concurrency/base.py} | 34 ++++++++++++++++- httpx/dispatch/connection.py | 2 +- httpx/dispatch/connection_pool.py | 2 +- httpx/dispatch/http11.py | 3 +- httpx/dispatch/http2.py | 3 +- httpx/dispatch/threaded.py | 2 +- 9 files changed, 50 insertions(+), 51 deletions(-) rename httpx/{interfaces.py => concurrency/base.py} (77%) diff --git a/httpx/__init__.py b/httpx/__init__.py index ec9fa5273e..ebacaad87e 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -2,6 +2,13 @@ from .api import delete, get, head, options, patch, post, put, request from .client import AsyncClient, Client from .concurrency.asyncio import AsyncioBackend +from .concurrency.base import ( + BaseBackgroundManager, + BasePoolSemaphore, + BaseReader, + BaseWriter, + ConcurrencyBackend, +) from .config import ( USER_AGENT, CertTypes, @@ -33,13 +40,6 @@ TooManyRedirects, WriteTimeout, ) -from .interfaces import ( - BaseBackgroundManager, - BasePoolSemaphore, - BaseReader, - BaseWriter, - ConcurrencyBackend, -) from .models import ( URL, AsyncRequest, diff --git a/httpx/client.py b/httpx/client.py index f2e39a0d40..f7a3c5db62 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -6,6 +6,7 @@ from .auth import HTTPBasicAuth from .concurrency.asyncio import AsyncioBackend +from .concurrency.base import ConcurrencyBackend from .config import ( DEFAULT_MAX_REDIRECTS, DEFAULT_POOL_LIMITS, @@ -28,7 +29,6 @@ RedirectLoop, TooManyRedirects, ) -from .interfaces import ConcurrencyBackend from .models import ( URL, AsyncRequest, diff --git a/httpx/concurrency/asyncio.py b/httpx/concurrency/asyncio.py index d8421f7185..c6ca977155 100644 --- a/httpx/concurrency/asyncio.py +++ b/httpx/concurrency/asyncio.py @@ -14,15 +14,16 @@ import typing from types import TracebackType -from ..config import PoolLimits, TimeoutConfig -from ..exceptions import ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout -from ..interfaces import ( +from .base import ( BaseBackgroundManager, BasePoolSemaphore, BaseReader, BaseWriter, ConcurrencyBackend, + TimeoutFlag ) +from ..config import PoolLimits, TimeoutConfig +from ..exceptions import ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout SSL_MONKEY_PATCH_APPLIED = False @@ -48,38 +49,6 @@ def _fixed_write(self, data: bytes) -> None: # type: ignore MonkeyPatch.write = _fixed_write -class TimeoutFlag: - """ - A timeout flag holds a state of either read-timeout or write-timeout mode. - - We use this so that we can attempt both reads and writes concurrently, while - only enforcing timeouts in one direction. - - During a request/response cycle we start in write-timeout mode. - - Once we've sent a request fully, or once we start seeing a response, - then we switch to read-timeout mode instead. - """ - - def __init__(self) -> None: - self.raise_on_read_timeout = False - self.raise_on_write_timeout = True - - def set_read_timeouts(self) -> None: - """ - Set the flag to read-timeout mode. - """ - self.raise_on_read_timeout = True - self.raise_on_write_timeout = False - - def set_write_timeouts(self) -> None: - """ - Set the flag to write-timeout mode. - """ - self.raise_on_read_timeout = False - self.raise_on_write_timeout = True - - class Reader(BaseReader): def __init__( self, stream_reader: asyncio.StreamReader, timeout: TimeoutConfig diff --git a/httpx/interfaces.py b/httpx/concurrency/base.py similarity index 77% rename from httpx/interfaces.py rename to httpx/concurrency/base.py index 628c110353..994cbc22aa 100644 --- a/httpx/interfaces.py +++ b/httpx/concurrency/base.py @@ -2,7 +2,39 @@ import typing from types import TracebackType -from .config import PoolLimits, TimeoutConfig +from ..config import PoolLimits, TimeoutConfig + + +class TimeoutFlag: + """ + A timeout flag holds a state of either read-timeout or write-timeout mode. + + We use this so that we can attempt both reads and writes concurrently, while + only enforcing timeouts in one direction. + + During a request/response cycle we start in write-timeout mode. + + Once we've sent a request fully, or once we start seeing a response, + then we switch to read-timeout mode instead. + """ + + def __init__(self) -> None: + self.raise_on_read_timeout = False + self.raise_on_write_timeout = True + + def set_read_timeouts(self) -> None: + """ + Set the flag to read-timeout mode. + """ + self.raise_on_read_timeout = True + self.raise_on_write_timeout = False + + def set_write_timeouts(self) -> None: + """ + Set the flag to write-timeout mode. + """ + self.raise_on_read_timeout = False + self.raise_on_write_timeout = True class BaseReader: diff --git a/httpx/dispatch/connection.py b/httpx/dispatch/connection.py index 1aa79c60e4..36f96f8065 100644 --- a/httpx/dispatch/connection.py +++ b/httpx/dispatch/connection.py @@ -4,6 +4,7 @@ from .base import AsyncDispatcher from ..concurrency.asyncio import AsyncioBackend +from ..concurrency.base import ConcurrencyBackend from ..config import ( DEFAULT_TIMEOUT_CONFIG, CertTypes, @@ -14,7 +15,6 @@ TimeoutTypes, VerifyTypes, ) -from ..interfaces import ConcurrencyBackend from ..models import AsyncRequest, AsyncResponse, Origin from .http2 import HTTP2Connection from .http11 import HTTP11Connection diff --git a/httpx/dispatch/connection_pool.py b/httpx/dispatch/connection_pool.py index d6ea027eef..0e14bf8354 100644 --- a/httpx/dispatch/connection_pool.py +++ b/httpx/dispatch/connection_pool.py @@ -2,6 +2,7 @@ from .base import AsyncDispatcher from ..concurrency.asyncio import AsyncioBackend +from ..concurrency.base import ConcurrencyBackend from ..config import ( DEFAULT_POOL_LIMITS, DEFAULT_TIMEOUT_CONFIG, @@ -11,7 +12,6 @@ TimeoutTypes, VerifyTypes, ) -from ..interfaces import ConcurrencyBackend from ..models import AsyncRequest, AsyncResponse, Origin from .connection import HTTPConnection diff --git a/httpx/dispatch/http11.py b/httpx/dispatch/http11.py index 46d7309cc3..554591f804 100644 --- a/httpx/dispatch/http11.py +++ b/httpx/dispatch/http11.py @@ -2,9 +2,8 @@ import h11 -from ..concurrency.asyncio import TimeoutFlag +from ..concurrency.base import BaseReader, BaseWriter, ConcurrencyBackend, TimeoutFlag from ..config import TimeoutConfig, TimeoutTypes -from ..interfaces import BaseReader, BaseWriter, ConcurrencyBackend from ..models import AsyncRequest, AsyncResponse H11Event = typing.Union[ diff --git a/httpx/dispatch/http2.py b/httpx/dispatch/http2.py index 199ee8fa19..bf258e3a7a 100644 --- a/httpx/dispatch/http2.py +++ b/httpx/dispatch/http2.py @@ -4,9 +4,8 @@ import h2.connection import h2.events -from ..concurrency.asyncio import TimeoutFlag +from ..concurrency.base import BaseReader, BaseWriter, ConcurrencyBackend, TimeoutFlag from ..config import TimeoutConfig, TimeoutTypes -from ..interfaces import BaseReader, BaseWriter, ConcurrencyBackend from ..models import AsyncRequest, AsyncResponse diff --git a/httpx/dispatch/threaded.py b/httpx/dispatch/threaded.py index 0381cccde2..8176608729 100644 --- a/httpx/dispatch/threaded.py +++ b/httpx/dispatch/threaded.py @@ -1,6 +1,6 @@ from .base import AsyncDispatcher, Dispatcher +from ..concurrency.base import ConcurrencyBackend from ..config import CertTypes, TimeoutTypes, VerifyTypes -from ..interfaces import ConcurrencyBackend from ..models import ( AsyncRequest, AsyncRequestData, From b83a4383b7450edc8ff0d6f55716fa8dd3217e42 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 20 Aug 2019 22:08:50 -0500 Subject: [PATCH 3/3] , --- httpx/concurrency/asyncio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpx/concurrency/asyncio.py b/httpx/concurrency/asyncio.py index c6ca977155..b8674bd855 100644 --- a/httpx/concurrency/asyncio.py +++ b/httpx/concurrency/asyncio.py @@ -20,7 +20,7 @@ BaseReader, BaseWriter, ConcurrencyBackend, - TimeoutFlag + TimeoutFlag, ) from ..config import PoolLimits, TimeoutConfig from ..exceptions import ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout