From 53f3dc4a6628dabe2732fe7dfec334dacb235da9 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 24 Apr 2019 15:48:18 +0100 Subject: [PATCH 01/23] First pass at HTTP/2 support --- httpcore/__init__.py | 2 + httpcore/config.py | 18 ++++- httpcore/connection.py | 106 ++++++++++++++++++++++++++ httpcore/connectionpool.py | 10 +-- httpcore/datastructures.py | 13 ++-- httpcore/http11.py | 82 +++++++------------- httpcore/http2.py | 152 +++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/test_connections.py | 4 +- tests/test_requests.py | 4 +- 10 files changed, 322 insertions(+), 70 deletions(-) create mode 100644 httpcore/connection.py create mode 100644 httpcore/http2.py diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 9824e854e5..48e3426ac4 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -1,4 +1,5 @@ from .config import PoolLimits, SSLConfig, TimeoutConfig +from .connection import HTTPConnection from .connectionpool import ConnectionPool from .datastructures import URL, Origin, Request, Response from .exceptions import ( @@ -10,6 +11,7 @@ StreamConsumed, Timeout, ) +from .http2 import HTTP2Connection from .http11 import HTTP11Connection from .sync import SyncClient, SyncConnectionPool diff --git a/httpcore/config.py b/httpcore/config.py index 8cc784b3f7..5b7ab4e07b 100644 --- a/httpcore/config.py +++ b/httpcore/config.py @@ -73,7 +73,23 @@ def load_ssl_context_verify(self) -> ssl.SSLContext: "invalid path: {}".format(self.verify) ) - context = ssl.create_default_context() + context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) + + context.options |= ssl.OP_NO_SSLv2 + context.options |= ssl.OP_NO_SSLv3 + context.options |= ssl.OP_NO_COMPRESSION + + # RFC 7540 Section 9.2.2: "deployments of HTTP/2 that use TLS 1.2 MUST + # support TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256". In practice, the + # blacklist defined in this section allows only the AES GCM and ChaCha20 + # cipher suites with ephemeral key negotiation. + context.set_ciphers("ECDHE+AESGCM:ECDHE+CHACHA20:DHE+AESGCM:DHE+CHACHA20") + + if ssl.HAS_ALPN: + context.set_alpn_protocols(["h2", "http/1.1"]) + if ssl.HAS_NPN: + context.set_npn_protocols(["h2", "http/1.1"]) + if os.path.isfile(ca_bundle_path): context.load_verify_locations(cafile=ca_bundle_path) elif os.path.isdir(ca_bundle_path): diff --git a/httpcore/connection.py b/httpcore/connection.py new file mode 100644 index 0000000000..33f3459488 --- /dev/null +++ b/httpcore/connection.py @@ -0,0 +1,106 @@ +import asyncio +import typing + +import h2.connection +import h11 + +from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig +from .datastructures import Client, Origin, Request, Response +from .exceptions import ConnectTimeout +from .http2 import HTTP2Connection +from .http11 import HTTP11Connection + + +class HTTPConnection(Client): + def __init__( + self, + origin: typing.Union[str, Origin], + ssl: SSLConfig = DEFAULT_SSL_CONFIG, + timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, + on_release: typing.Callable = None, + ): + self.origin = Origin(origin) if isinstance(origin, str) else origin + self.ssl = ssl + self.timeout = timeout + self.on_release = on_release + self.h11_connection = None # type: typing.Optional[HTTP11Connection] + self.h2_connection = None # type: typing.Optional[HTTP2Connection] + + async def send( + self, + request: Request, + *, + ssl: typing.Optional[SSLConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None, + ) -> Response: + if self.h11_connection is None and self.h2_connection is None: + if ssl is None: + ssl = self.ssl + if timeout is None: + timeout = self.timeout + + reader, writer, protocol = await self.connect(ssl, timeout) + if protocol == "h2": + self.h2_connection = HTTP2Connection( + reader, + writer, + origin=self.origin, + timeout=self.timeout, + on_release=self.on_release, + ) + else: + self.h11_connection = HTTP11Connection( + reader, + writer, + origin=self.origin, + timeout=self.timeout, + on_release=self.on_release, + ) + + if self.h2_connection is not None: + response = await self.h2_connection.send(request, ssl=ssl, timeout=timeout) + else: + assert self.h11_connection is not None + response = await self.h11_connection.send(request, ssl=ssl, timeout=timeout) + + return response + + async def close(self) -> None: + if self.h2_connection is not None: + await self.h2_connection.close() + else: + assert self.h11_connection is not None + await self.h11_connection.close() + + @property + def is_closed(self) -> bool: + if self.h2_connection is not None: + return self.h2_connection.is_closed + else: + assert self.h11_connection is not None + return self.h11_connection.is_closed + + async def connect( + self, ssl: SSLConfig, timeout: TimeoutConfig + ) -> typing.Tuple[asyncio.StreamReader, asyncio.StreamWriter, str]: + hostname = self.origin.hostname + port = self.origin.port + ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None + + try: + reader, writer = await asyncio.wait_for( # type: ignore + asyncio.open_connection(hostname, port, ssl=ssl_context), + timeout.connect_timeout, + ) + except asyncio.TimeoutError: + raise ConnectTimeout() + + ssl_object = writer.get_extra_info("ssl_object") + if ssl_object is None: + protocol = "http/1.1" + else: + protocol = ssl_object.selected_alpn_protocol() + if protocol is None: + protocol = ssl_object.selected_npn_protocol() + + return (reader, writer, protocol) diff --git a/httpcore/connectionpool.py b/httpcore/connectionpool.py index a2040dbc41..54bb32f591 100644 --- a/httpcore/connectionpool.py +++ b/httpcore/connectionpool.py @@ -10,9 +10,9 @@ SSLConfig, TimeoutConfig, ) +from .connection import HTTPConnection from .datastructures import Client, Origin, Request, Response from .exceptions import PoolTimeout -from .http11 import HTTP11Connection class ConnectionPool(Client): @@ -31,7 +31,7 @@ def __init__( self.num_keepalive_connections = 0 self._keepalive_connections = ( {} - ) # type: typing.Dict[Origin, typing.List[HTTP11Connection]] + ) # type: typing.Dict[Origin, typing.List[HTTPConnection]] self._max_connections = ConnectionSemaphore( max_connections=self.limits.hard_limit ) @@ -53,7 +53,7 @@ def num_connections(self) -> int: async def acquire_connection( self, origin: Origin, timeout: typing.Optional[TimeoutConfig] = None - ) -> HTTP11Connection: + ) -> HTTPConnection: try: connection = self._keepalive_connections[origin].pop() if not self._keepalive_connections[origin]: @@ -71,7 +71,7 @@ async def acquire_connection( await asyncio.wait_for(self._max_connections.acquire(), pool_timeout) except asyncio.TimeoutError: raise PoolTimeout() - connection = HTTP11Connection( + connection = HTTPConnection( origin, ssl=self.ssl, timeout=self.timeout, @@ -81,7 +81,7 @@ async def acquire_connection( return connection - async def release_connection(self, connection: HTTP11Connection) -> None: + async def release_connection(self, connection: HTTPConnection) -> None: if connection.is_closed: self._max_connections.release() self.num_active_connections -= 1 diff --git a/httpcore/datastructures.py b/httpcore/datastructures.py index de5ee2f5a3..e4a809af5b 100644 --- a/httpcore/datastructures.py +++ b/httpcore/datastructures.py @@ -52,7 +52,7 @@ def port(self) -> int: return port @property - def target(self) -> str: + def full_path(self) -> str: path = self.path or "/" query = self.query if query: @@ -138,10 +138,11 @@ def _auto_headers(self) -> typing.List[typing.Tuple[bytes, bytes]]: return headers async def stream(self) -> typing.AsyncIterator[bytes]: - assert self.is_streaming - - async for part in self.body_aiter: - yield part + if self.is_streaming: + async for part in self.body_aiter: + yield part + elif self.body: + yield self.body class Response: @@ -150,6 +151,7 @@ def __init__( status_code: int, *, reason: typing.Optional[str] = None, + protocol: typing.Optional[str] = None, headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", on_close: typing.Callable = None, @@ -162,6 +164,7 @@ def __init__( self.reason = "" else: self.reason = reason + self.protocol = protocol self.headers = list(headers) self.on_close = on_close self.is_closed = False diff --git a/httpcore/http11.py b/httpcore/http11.py index 23cc27ceca..f660867abc 100644 --- a/httpcore/http11.py +++ b/httpcore/http11.py @@ -20,55 +20,43 @@ class HTTP11Connection(Client): def __init__( self, - origin: typing.Union[str, Origin], - ssl: SSLConfig = DEFAULT_SSL_CONFIG, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + origin: Origin, timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, on_release: typing.Callable = None, ): - self.origin = Origin(origin) if isinstance(origin, str) else origin - self.ssl = ssl + self.origin = origin + self.reader = reader + self.writer = writer self.timeout = timeout self.on_release = on_release - self._reader = None - self._writer = None - self._h11_state = h11.Connection(our_role=h11.CLIENT) + self.h11_state = h11.Connection(our_role=h11.CLIENT) @property def is_closed(self) -> bool: - return self._h11_state.our_state in (h11.CLOSED, h11.ERROR) + return self.h11_state.our_state in (h11.CLOSED, h11.ERROR) async def send( self, request: Request, *, ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None ) -> Response: - assert request.url.origin == self.origin - - if ssl is None: - ssl = self.ssl if timeout is None: timeout = self.timeout - # Make the connection - if self._reader is None: - await self._connect(ssl, timeout) - #  Start sending the request. method = request.method.encode() - target = request.url.target + target = request.url.full_path headers = request.headers event = h11.Request(method=method, target=target, headers=headers) await self._send_event(event) # Send the request body. - if request.is_streaming: - async for data in request.stream(): - event = h11.Data(data=data) - await self._send_event(event) - elif request.body: - event = h11.Data(data=request.body) + async for data in request.stream(): + event = h11.Data(data=data) await self._send_event(event) # Finalize sending the request. @@ -79,32 +67,22 @@ async def send( event = await self._receive_event(timeout) if isinstance(event, h11.InformationalResponse): event = await self._receive_event(timeout) + assert isinstance(event, h11.Response) reason = event.reason.decode("latin1") status_code = event.status_code headers = event.headers body = self._body_iter(timeout) + return Response( status_code=status_code, reason=reason, + protocol="HTTP/1.1", headers=headers, body=body, on_close=self._release, ) - async def _connect(self, ssl: SSLConfig, timeout: TimeoutConfig) -> None: - hostname = self.origin.hostname - port = self.origin.port - ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None - - try: - self._reader, self._writer = await asyncio.wait_for( # type: ignore - asyncio.open_connection(hostname, port, ssl=ssl_context), - timeout.connect_timeout, - ) - except asyncio.TimeoutError: - raise ConnectTimeout() - async def _body_iter(self, timeout: TimeoutConfig) -> typing.AsyncIterator[bytes]: event = await self._receive_event(timeout) while isinstance(event, h11.Data): @@ -113,36 +91,30 @@ async def _body_iter(self, timeout: TimeoutConfig) -> typing.AsyncIterator[bytes assert isinstance(event, h11.EndOfMessage) async def _send_event(self, event: H11Event) -> None: - assert self._writer is not None - - data = self._h11_state.send(event) - self._writer.write(data) + data = self.h11_state.send(event) + self.writer.write(data) async def _receive_event(self, timeout: TimeoutConfig) -> H11Event: - assert self._reader is not None - - event = self._h11_state.next_event() + event = self.h11_state.next_event() while event is h11.NEED_DATA: try: data = await asyncio.wait_for( - self._reader.read(2048), timeout.read_timeout + self.reader.read(2048), timeout.read_timeout ) except asyncio.TimeoutError: raise ReadTimeout() - self._h11_state.receive_data(data) - event = self._h11_state.next_event() + self.h11_state.receive_data(data) + event = self.h11_state.next_event() return event async def _release(self) -> None: - assert self._writer is not None - if ( - self._h11_state.our_state is h11.DONE - and self._h11_state.their_state is h11.DONE + self.h11_state.our_state is h11.DONE + and self.h11_state.their_state is h11.DONE ): - self._h11_state.start_next_cycle() + self.h11_state.start_next_cycle() else: await self.close() @@ -153,11 +125,11 @@ async def close(self) -> None: event = h11.ConnectionClosed() try: # If we're in h11.MUST_CLOSE then we'll end up in h11.CLOSED. - self._h11_state.send(event) + self.h11_state.send(event) except h11.ProtocolError: # If we're in some other state then it's a premature close, # and we'll end up in h11.ERROR. pass - if self._writer is not None: - self._writer.close() + if self.writer is not None: + self.writer.close() diff --git a/httpcore/http2.py b/httpcore/http2.py new file mode 100644 index 0000000000..084a87ede4 --- /dev/null +++ b/httpcore/http2.py @@ -0,0 +1,152 @@ +import asyncio +import typing + +import h2.connection +import h2.events + +from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig +from .datastructures import Client, Origin, Request, Response +from .exceptions import ConnectTimeout, ReadTimeout + + +class HTTP2Connection(Client): + def __init__( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + origin: Origin, + timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, + on_release: typing.Callable = None, + ): + self.origin = origin + self.reader = reader + self.writer = writer + self.timeout = timeout + self.on_release = on_release + self.h2_state = h2.connection.H2Connection() + self.events = [] # type: typing.List[h2.events.Event] + + @property + def is_closed(self) -> bool: + return False + + async def send( + self, + request: Request, + *, + ssl: typing.Optional[SSLConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None + ) -> Response: + if timeout is None: + timeout = self.timeout + + #  Start sending the request. + await self._initiate_connection() + await self._send_headers(request) + + # Send the request body. + if request.body: + await self._send_data(request.body) + + # Finalize sending the request. + await self._end_stream() + + # Start getting the response. + while True: + event = await self._receive_event(timeout) + if isinstance(event, h2.events.ResponseReceived): + break + + status_code = 200 + headers = [] + for k, v in event.headers: + if k == b":status": + status_code = int(v.decode()) + elif not k.startswith(b":"): + headers.append((k, v)) + + body = self._body_iter(timeout) + return Response( + status_code=status_code, + protocol="HTTP/2", + headers=headers, + body=body, + on_close=self._release, + ) + + async def _initiate_connection(self) -> None: + self.h2_state.initiate_connection() + data_to_send = self.h2_state.data_to_send() + self.writer.write(data_to_send) + + async def _send_headers(self, request: Request) -> None: + headers = [ + (b":method", request.method.encode()), + (b":authority", request.url.hostname.encode()), + (b":scheme", request.url.scheme.encode()), + (b":path", request.url.full_path.encode()), + ] + request.headers + self.h2_state.send_headers(1, headers) + data_to_send = self.h2_state.data_to_send() + self.writer.write(data_to_send) + + async def _send_data(self, data: bytes) -> None: + self.h2_state.send_data(1, data) + data_to_send = self.h2_state.data_to_send() + self.writer.write(data_to_send) + + async def _end_stream(self) -> None: + self.h2_state.end_stream(1) + data_to_send = self.h2_state.data_to_send() + self.writer.write(data_to_send) + + async def _body_iter(self, timeout: TimeoutConfig) -> typing.AsyncIterator[bytes]: + while True: + event = await self._receive_event(timeout) + if isinstance(event, h2.events.DataReceived): + yield event.data + elif isinstance(event, h2.events.StreamEnded): + break + + async def _receive_event(self, timeout: TimeoutConfig) -> h2.events.Event: + while not self.events: + try: + data = await asyncio.wait_for( + self.reader.read(2048), timeout.read_timeout + ) + except asyncio.TimeoutError: + raise ReadTimeout() + + events = self.h2_state.receive_data(data) + self.events.extend(events) + + data_to_send = self.h2_state.data_to_send() + if data_to_send: + self.writer.write(data_to_send) + + return self.events.pop(0) + + async def _release(self) -> None: + # if ( + # self.h11_state.our_state is h11.DONE + # and self.h11_state.their_state is h11.DONE + # ): + # self.h11_state.start_next_cycle() + # else: + # await self.close() + + if self.on_release is not None: + await self.on_release(self) + + async def close(self) -> None: + # event = h11.ConnectionClosed() + # try: + # # If we're in h11.MUST_CLOSE then we'll end up in h11.CLOSED. + # self.h11_state.send(event) + # except h11.ProtocolError: + # # If we're in some other state then it's a premature close, + # # and we'll end up in h11.ERROR. + # pass + + if self.writer is not None: + self.writer.close() diff --git a/requirements.txt b/requirements.txt index 5108a8d6d3..18f9c5fe4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ certifi h11 +h2 # Optional brotlipy diff --git a/tests/test_connections.py b/tests/test_connections.py index f15901404c..1103110677 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -5,7 +5,7 @@ @pytest.mark.asyncio async def test_get(server): - http = httpcore.HTTP11Connection(origin="http://127.0.0.1:8000/") + http = httpcore.HTTPConnection(origin="http://127.0.0.1:8000/") response = await http.request("GET", "http://127.0.0.1:8000/") assert response.status_code == 200 assert response.body == b"Hello, world!" @@ -13,7 +13,7 @@ async def test_get(server): @pytest.mark.asyncio async def test_post(server): - http = httpcore.HTTP11Connection(origin="http://127.0.0.1:8000/") + http = httpcore.HTTPConnection(origin="http://127.0.0.1:8000/") response = await http.request( "POST", "http://127.0.0.1:8000/", body=b"Hello, world!" ) diff --git a/tests/test_requests.py b/tests/test_requests.py index bdbf2caa08..c88b70a037 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -73,12 +73,12 @@ def test_url(): request = httpcore.Request("GET", "http://example.org") assert request.url.scheme == "http" assert request.url.port == 80 - assert request.url.target == "/" + assert request.url.full_path == "/" request = httpcore.Request("GET", "https://example.org/abc?foo=bar") assert request.url.scheme == "https" assert request.url.port == 443 - assert request.url.target == "/abc?foo=bar" + assert request.url.full_path == "/abc?foo=bar" def test_invalid_urls(): From 0bdcbaef72c569aaaac548b99f2ecfee5a0a76dd Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 24 Apr 2019 17:04:15 +0100 Subject: [PATCH 02/23] Add stream ID to HTTP2Connection --- httpcore/__init__.py | 4 +- httpcore/connection.py | 2 +- .../{connectionpool.py => connection_pool.py} | 2 +- httpcore/http11.py | 2 +- httpcore/http2.py | 81 +++++++++---------- httpcore/{datastructures.py => models.py} | 0 httpcore/sync.py | 4 +- 7 files changed, 45 insertions(+), 50 deletions(-) rename httpcore/{connectionpool.py => connection_pool.py} (98%) rename httpcore/{datastructures.py => models.py} (100%) diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 48e3426ac4..738d7ecd76 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -1,7 +1,6 @@ from .config import PoolLimits, SSLConfig, TimeoutConfig from .connection import HTTPConnection -from .connectionpool import ConnectionPool -from .datastructures import URL, Origin, Request, Response +from .connection_pool import ConnectionPool from .exceptions import ( ConnectTimeout, PoolTimeout, @@ -13,6 +12,7 @@ ) from .http2 import HTTP2Connection from .http11 import HTTP11Connection +from .models import URL, Origin, Request, Response from .sync import SyncClient, SyncConnectionPool __version__ = "0.2.1" diff --git a/httpcore/connection.py b/httpcore/connection.py index 33f3459488..db3a17e6c3 100644 --- a/httpcore/connection.py +++ b/httpcore/connection.py @@ -5,10 +5,10 @@ import h11 from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .datastructures import Client, Origin, Request, Response from .exceptions import ConnectTimeout from .http2 import HTTP2Connection from .http11 import HTTP11Connection +from .models import Client, Origin, Request, Response class HTTPConnection(Client): diff --git a/httpcore/connectionpool.py b/httpcore/connection_pool.py similarity index 98% rename from httpcore/connectionpool.py rename to httpcore/connection_pool.py index 54bb32f591..6ec3028991 100644 --- a/httpcore/connectionpool.py +++ b/httpcore/connection_pool.py @@ -11,8 +11,8 @@ TimeoutConfig, ) from .connection import HTTPConnection -from .datastructures import Client, Origin, Request, Response from .exceptions import PoolTimeout +from .models import Client, Origin, Request, Response class ConnectionPool(Client): diff --git a/httpcore/http11.py b/httpcore/http11.py index f660867abc..45994164ff 100644 --- a/httpcore/http11.py +++ b/httpcore/http11.py @@ -4,8 +4,8 @@ import h11 from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .datastructures import Client, Origin, Request, Response from .exceptions import ConnectTimeout, ReadTimeout +from .models import Client, Origin, Request, Response H11Event = typing.Union[ h11.Request, diff --git a/httpcore/http2.py b/httpcore/http2.py index 084a87ede4..0b32bc1592 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -5,8 +5,8 @@ import h2.events from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .datastructures import Client, Origin, Request, Response from .exceptions import ConnectTimeout, ReadTimeout +from .models import Client, Origin, Request, Response class HTTP2Connection(Client): @@ -24,7 +24,8 @@ def __init__( self.timeout = timeout self.on_release = on_release self.h2_state = h2.connection.H2Connection() - self.events = [] # type: typing.List[h2.events.Event] + self.events = {} # type: typing.Dict[int, typing.List[h2.events.Event]] + self.initialized = False @property def is_closed(self) -> bool: @@ -40,20 +41,24 @@ async def send( if timeout is None: timeout = self.timeout + if not self.initialized: + self.initiate_connection() + #  Start sending the request. - await self._initiate_connection() - await self._send_headers(request) + stream_id = self.h2_state.get_next_available_stream_id() + self.events[stream_id] = [] + await self.send_headers(stream_id, request) # Send the request body. - if request.body: - await self._send_data(request.body) + async for data in request.stream(): + await self.send_data(stream_id, data) # Finalize sending the request. - await self._end_stream() + await self.end_stream(stream_id) # Start getting the response. while True: - event = await self._receive_event(timeout) + event = await self.receive_event(stream_id, timeout) if isinstance(event, h2.events.ResponseReceived): break @@ -65,51 +70,57 @@ async def send( elif not k.startswith(b":"): headers.append((k, v)) - body = self._body_iter(timeout) + body = self.body_iter(stream_id, timeout) return Response( status_code=status_code, protocol="HTTP/2", headers=headers, body=body, - on_close=self._release, + on_close=self.release, ) - async def _initiate_connection(self) -> None: + def initiate_connection(self) -> None: self.h2_state.initiate_connection() data_to_send = self.h2_state.data_to_send() self.writer.write(data_to_send) + self.initialized = True - async def _send_headers(self, request: Request) -> None: + async def send_headers(self, stream_id: int, request: Request) -> None: headers = [ (b":method", request.method.encode()), (b":authority", request.url.hostname.encode()), (b":scheme", request.url.scheme.encode()), (b":path", request.url.full_path.encode()), ] + request.headers - self.h2_state.send_headers(1, headers) + self.h2_state.send_headers(stream_id, headers) data_to_send = self.h2_state.data_to_send() self.writer.write(data_to_send) - async def _send_data(self, data: bytes) -> None: - self.h2_state.send_data(1, data) + async def send_data(self, stream_id: int, data: bytes) -> None: + self.h2_state.send_data(stream_id, data) data_to_send = self.h2_state.data_to_send() self.writer.write(data_to_send) - async def _end_stream(self) -> None: - self.h2_state.end_stream(1) + async def end_stream(self, stream_id: int) -> None: + self.h2_state.end_stream(stream_id) data_to_send = self.h2_state.data_to_send() self.writer.write(data_to_send) - async def _body_iter(self, timeout: TimeoutConfig) -> typing.AsyncIterator[bytes]: + async def body_iter( + self, stream_id: int, timeout: TimeoutConfig + ) -> typing.AsyncIterator[bytes]: while True: - event = await self._receive_event(timeout) + event = await self.receive_event(stream_id, timeout) if isinstance(event, h2.events.DataReceived): yield event.data elif isinstance(event, h2.events.StreamEnded): + del self.events[stream_id] break - async def _receive_event(self, timeout: TimeoutConfig) -> h2.events.Event: - while not self.events: + async def receive_event( + self, stream_id: int, timeout: TimeoutConfig + ) -> h2.events.Event: + while not self.events[stream_id]: try: data = await asyncio.wait_for( self.reader.read(2048), timeout.read_timeout @@ -118,35 +129,19 @@ async def _receive_event(self, timeout: TimeoutConfig) -> h2.events.Event: raise ReadTimeout() events = self.h2_state.receive_data(data) - self.events.extend(events) + for event in events: + if getattr(event, "stream_id", 0): + self.events[event.stream_id].append(event) data_to_send = self.h2_state.data_to_send() if data_to_send: self.writer.write(data_to_send) - return self.events.pop(0) - - async def _release(self) -> None: - # if ( - # self.h11_state.our_state is h11.DONE - # and self.h11_state.their_state is h11.DONE - # ): - # self.h11_state.start_next_cycle() - # else: - # await self.close() + return self.events[stream_id].pop(0) + async def release(self) -> None: if self.on_release is not None: await self.on_release(self) async def close(self) -> None: - # event = h11.ConnectionClosed() - # try: - # # If we're in h11.MUST_CLOSE then we'll end up in h11.CLOSED. - # self.h11_state.send(event) - # except h11.ProtocolError: - # # If we're in some other state then it's a premature close, - # # and we'll end up in h11.ERROR. - # pass - - if self.writer is not None: - self.writer.close() + self.writer.close() diff --git a/httpcore/datastructures.py b/httpcore/models.py similarity index 100% rename from httpcore/datastructures.py rename to httpcore/models.py diff --git a/httpcore/sync.py b/httpcore/sync.py index 1d560faf91..b1f98f50e1 100644 --- a/httpcore/sync.py +++ b/httpcore/sync.py @@ -3,8 +3,8 @@ from types import TracebackType from .config import SSLConfig, TimeoutConfig -from .connectionpool import ConnectionPool -from .datastructures import URL, Client, Response +from .connection_pool import ConnectionPool +from .models import URL, Client, Response class SyncResponse: From 13b28b6ebb4077d361ce8499fd99609fb4745e7d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 24 Apr 2019 17:12:05 +0100 Subject: [PATCH 03/23] Tighten up how we hand out stream ids --- httpcore/http2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/httpcore/http2.py b/httpcore/http2.py index 0b32bc1592..08904388da 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -45,9 +45,8 @@ async def send( self.initiate_connection() #  Start sending the request. - stream_id = self.h2_state.get_next_available_stream_id() + stream_id = await self.send_headers(stream_id, request) self.events[stream_id] = [] - await self.send_headers(stream_id, request) # Send the request body. async for data in request.stream(): @@ -85,7 +84,8 @@ def initiate_connection(self) -> None: self.writer.write(data_to_send) self.initialized = True - async def send_headers(self, stream_id: int, request: Request) -> None: + async def send_headers(self, stream_id: int, request: Request) -> int: + stream_id = self.h2_state.get_next_available_stream_id() headers = [ (b":method", request.method.encode()), (b":authority", request.url.hostname.encode()), @@ -95,6 +95,7 @@ async def send_headers(self, stream_id: int, request: Request) -> None: self.h2_state.send_headers(stream_id, headers) data_to_send = self.h2_state.data_to_send() self.writer.write(data_to_send) + return stream_id async def send_data(self, stream_id: int, data: bytes) -> None: self.h2_state.send_data(stream_id, data) From abe46b8a0e2b19c3b2e46e170f21ac5aed20392e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 24 Apr 2019 19:38:53 +0100 Subject: [PATCH 04/23] Add h2 to install_requires --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d709be286..93d02ad562 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ def get_packages(package): author_email="tom@tomchristie.com", packages=get_packages("httpcore"), data_files=[("", ["LICENSE.md"])], - install_requires=["h11", "certifi"], + install_requires=["h11", "h2", "certifi"], classifiers=[ "Development Status :: 3 - Alpha", "Environment :: Web Environment", From a44190ff24558c2ee5efc9dd422f836af376150b Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 12:05:23 +0100 Subject: [PATCH 05/23] Stream refactoring and HTTP/2 test case --- httpcore/__init__.py | 1 + httpcore/config.py | 14 ++++- httpcore/connection.py | 40 ++++---------- httpcore/exceptions.py | 6 +++ httpcore/http11.py | 38 ++++++-------- httpcore/http2.py | 47 +++++++++-------- httpcore/streams.py | 115 +++++++++++++++++++++++++++++++++++++++++ tests/test_config.py | 2 +- tests/test_http2.py | 76 +++++++++++++++++++++++++++ 9 files changed, 261 insertions(+), 78 deletions(-) create mode 100644 httpcore/streams.py create mode 100644 tests/test_http2.py diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 738d7ecd76..00cb5fb80b 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -13,6 +13,7 @@ from .http2 import HTTP2Connection from .http11 import HTTP11Connection from .models import URL, Origin, Request, Response +from .streams import BaseReader, BaseWriter, Protocol, Reader, Writer, connect from .sync import SyncClient, SyncConnectionPool __version__ = "0.2.1" diff --git a/httpcore/config.py b/httpcore/config.py index 5b7ab4e07b..b0fadc4017 100644 --- a/httpcore/config.py +++ b/httpcore/config.py @@ -115,20 +115,24 @@ def __init__( *, connect_timeout: float = None, read_timeout: float = None, + write_timeout: float = None, pool_timeout: float = None, ): if timeout is not None: # Specified as a single timeout value assert connect_timeout is None assert read_timeout is None + assert write_timeout is None assert pool_timeout is None connect_timeout = timeout read_timeout = timeout + write_timeout = timeout pool_timeout = timeout self.timeout = timeout self.connect_timeout = connect_timeout self.read_timeout = read_timeout + self.write_timeout = write_timeout self.pool_timeout = pool_timeout def __eq__(self, other: typing.Any) -> bool: @@ -136,18 +140,24 @@ def __eq__(self, other: typing.Any) -> bool: isinstance(other, self.__class__) and self.connect_timeout == other.connect_timeout and self.read_timeout == other.read_timeout + and self.write_timeout == other.write_timeout and self.pool_timeout == other.pool_timeout ) def __hash__(self) -> int: - as_tuple = (self.connect_timeout, self.read_timeout, self.pool_timeout) + as_tuple = ( + self.connect_timeout, + self.read_timeout, + self.write_timeout, + self.pool_timeout, + ) return hash(as_tuple) def __repr__(self) -> str: class_name = self.__class__.__name__ if self.timeout is not None: return f"{class_name}(timeout={self.timeout})" - return f"{class_name}(connect_timeout={self.connect_timeout}, read_timeout={self.read_timeout}, pool_timeout={self.pool_timeout})" + return f"{class_name}(connect_timeout={self.connect_timeout}, read_timeout={self.read_timeout}, write_timeout={self.write_timeout}, pool_timeout={self.pool_timeout})" class PoolLimits: diff --git a/httpcore/connection.py b/httpcore/connection.py index db3a17e6c3..f164232ffe 100644 --- a/httpcore/connection.py +++ b/httpcore/connection.py @@ -1,4 +1,3 @@ -import asyncio import typing import h2.connection @@ -9,6 +8,7 @@ from .http2 import HTTP2Connection from .http11 import HTTP11Connection from .models import Client, Origin, Request, Response +from .streams import Protocol, connect class HTTPConnection(Client): @@ -39,8 +39,14 @@ async def send( if timeout is None: timeout = self.timeout - reader, writer, protocol = await self.connect(ssl, timeout) - if protocol == "h2": + hostname = self.origin.hostname + port = self.origin.port + ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None + + reader, writer, protocol = await connect( + hostname, port, ssl_context, timeout + ) + if protocol == Protocol.HTTP_2: self.h2_connection = HTTP2Connection( reader, writer, @@ -68,8 +74,7 @@ async def send( async def close(self) -> None: if self.h2_connection is not None: await self.h2_connection.close() - else: - assert self.h11_connection is not None + elif self.h11_connection is not None: await self.h11_connection.close() @property @@ -79,28 +84,3 @@ def is_closed(self) -> bool: else: assert self.h11_connection is not None return self.h11_connection.is_closed - - async def connect( - self, ssl: SSLConfig, timeout: TimeoutConfig - ) -> typing.Tuple[asyncio.StreamReader, asyncio.StreamWriter, str]: - hostname = self.origin.hostname - port = self.origin.port - ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None - - try: - reader, writer = await asyncio.wait_for( # type: ignore - asyncio.open_connection(hostname, port, ssl=ssl_context), - timeout.connect_timeout, - ) - except asyncio.TimeoutError: - raise ConnectTimeout() - - ssl_object = writer.get_extra_info("ssl_object") - if ssl_object is None: - protocol = "http/1.1" - else: - protocol = ssl_object.selected_alpn_protocol() - if protocol is None: - protocol = ssl_object.selected_npn_protocol() - - return (reader, writer, protocol) diff --git a/httpcore/exceptions.py b/httpcore/exceptions.py index 30814332c3..285b64039b 100644 --- a/httpcore/exceptions.py +++ b/httpcore/exceptions.py @@ -16,6 +16,12 @@ class ReadTimeout(Timeout): """ +class WriteTimeout(Timeout): + """ + Timeout while writing request data. + """ + + class PoolTimeout(Timeout): """ Timeout while waiting to acquire a connection from the pool. diff --git a/httpcore/http11.py b/httpcore/http11.py index 45994164ff..253865fe92 100644 --- a/httpcore/http11.py +++ b/httpcore/http11.py @@ -1,4 +1,3 @@ -import asyncio import typing import h11 @@ -6,6 +5,7 @@ from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig from .exceptions import ConnectTimeout, ReadTimeout from .models import Client, Origin, Request, Response +from .streams import BaseReader, BaseWriter H11Event = typing.Union[ h11.Request, @@ -17,11 +17,16 @@ ] +OptionalTimeout = typing.Optional[TimeoutConfig] + + class HTTP11Connection(Client): + READ_NUM_BYTES = 4096 + def __init__( self, - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, + reader: BaseReader, + writer: BaseWriter, origin: Origin, timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, on_release: typing.Callable = None, @@ -44,24 +49,21 @@ async def send( ssl: typing.Optional[SSLConfig] = None, timeout: typing.Optional[TimeoutConfig] = None ) -> Response: - if timeout is None: - timeout = self.timeout - #  Start sending the request. method = request.method.encode() target = request.url.full_path headers = request.headers event = h11.Request(method=method, target=target, headers=headers) - await self._send_event(event) + await self._send_event(event, timeout) # Send the request body. async for data in request.stream(): event = h11.Data(data=data) - await self._send_event(event) + await self._send_event(event, timeout) # Finalize sending the request. event = h11.EndOfMessage() - await self._send_event(event) + await self._send_event(event, timeout) # Start getting the response. event = await self._receive_event(timeout) @@ -83,27 +85,22 @@ async def send( on_close=self._release, ) - async def _body_iter(self, timeout: TimeoutConfig) -> typing.AsyncIterator[bytes]: + async def _body_iter(self, timeout: OptionalTimeout) -> typing.AsyncIterator[bytes]: event = await self._receive_event(timeout) while isinstance(event, h11.Data): yield event.data event = await self._receive_event(timeout) assert isinstance(event, h11.EndOfMessage) - async def _send_event(self, event: H11Event) -> None: + async def _send_event(self, event: H11Event, timeout: OptionalTimeout) -> None: data = self.h11_state.send(event) - self.writer.write(data) + await self.writer.write(data, timeout) - async def _receive_event(self, timeout: TimeoutConfig) -> H11Event: + async def _receive_event(self, timeout: OptionalTimeout) -> H11Event: event = self.h11_state.next_event() while event is h11.NEED_DATA: - try: - data = await asyncio.wait_for( - self.reader.read(2048), timeout.read_timeout - ) - except asyncio.TimeoutError: - raise ReadTimeout() + data = await self.reader.read(self.READ_NUM_BYTES, timeout) self.h11_state.receive_data(data) event = self.h11_state.next_event() @@ -131,5 +128,4 @@ async def close(self) -> None: # and we'll end up in h11.ERROR. pass - if self.writer is not None: - self.writer.close() + await self.writer.close() diff --git a/httpcore/http2.py b/httpcore/http2.py index 08904388da..41a0900cad 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -1,4 +1,3 @@ -import asyncio import typing import h2.connection @@ -7,13 +6,18 @@ from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig from .exceptions import ConnectTimeout, ReadTimeout from .models import Client, Origin, Request, Response +from .streams import BaseReader, BaseWriter + +OptionalTimeout = typing.Optional[TimeoutConfig] class HTTP2Connection(Client): + READ_NUM_BYTES = 4096 + def __init__( self, - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, + reader: BaseReader, + writer: BaseWriter, origin: Origin, timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, on_release: typing.Callable = None, @@ -45,15 +49,15 @@ async def send( self.initiate_connection() #  Start sending the request. - stream_id = await self.send_headers(stream_id, request) + stream_id = await self.send_headers(request, timeout) self.events[stream_id] = [] # Send the request body. async for data in request.stream(): - await self.send_data(stream_id, data) + await self.send_data(stream_id, data, timeout) # Finalize sending the request. - await self.end_stream(stream_id) + await self.end_stream(stream_id, timeout) # Start getting the response. while True: @@ -81,10 +85,10 @@ async def send( def initiate_connection(self) -> None: self.h2_state.initiate_connection() data_to_send = self.h2_state.data_to_send() - self.writer.write(data_to_send) + self.writer.write_no_block(data_to_send) self.initialized = True - async def send_headers(self, stream_id: int, request: Request) -> int: + async def send_headers(self, request: Request, timeout: OptionalTimeout) -> int: stream_id = self.h2_state.get_next_available_stream_id() headers = [ (b":method", request.method.encode()), @@ -94,21 +98,23 @@ async def send_headers(self, stream_id: int, request: Request) -> int: ] + request.headers self.h2_state.send_headers(stream_id, headers) data_to_send = self.h2_state.data_to_send() - self.writer.write(data_to_send) + await self.writer.write(data_to_send, timeout) return stream_id - async def send_data(self, stream_id: int, data: bytes) -> None: + async def send_data( + self, stream_id: int, data: bytes, timeout: OptionalTimeout + ) -> None: self.h2_state.send_data(stream_id, data) data_to_send = self.h2_state.data_to_send() - self.writer.write(data_to_send) + await self.writer.write(data_to_send, timeout) - async def end_stream(self, stream_id: int) -> None: + async def end_stream(self, stream_id: int, timeout: OptionalTimeout) -> None: self.h2_state.end_stream(stream_id) data_to_send = self.h2_state.data_to_send() - self.writer.write(data_to_send) + await self.writer.write(data_to_send, timeout) async def body_iter( - self, stream_id: int, timeout: TimeoutConfig + self, stream_id: int, timeout: OptionalTimeout ) -> typing.AsyncIterator[bytes]: while True: event = await self.receive_event(stream_id, timeout) @@ -119,24 +125,17 @@ async def body_iter( break async def receive_event( - self, stream_id: int, timeout: TimeoutConfig + self, stream_id: int, timeout: OptionalTimeout ) -> h2.events.Event: while not self.events[stream_id]: - try: - data = await asyncio.wait_for( - self.reader.read(2048), timeout.read_timeout - ) - except asyncio.TimeoutError: - raise ReadTimeout() - + data = await self.reader.read(self.READ_NUM_BYTES, timeout) events = self.h2_state.receive_data(data) for event in events: if getattr(event, "stream_id", 0): self.events[event.stream_id].append(event) data_to_send = self.h2_state.data_to_send() - if data_to_send: - self.writer.write(data_to_send) + await self.writer.write(data_to_send, timeout) return self.events[stream_id].pop(0) diff --git a/httpcore/streams.py b/httpcore/streams.py new file mode 100644 index 0000000000..5a9a0abb31 --- /dev/null +++ b/httpcore/streams.py @@ -0,0 +1,115 @@ +""" +The `Reader` and `Writer` classes here provide a lightweight layer over +`asyncio.StreamReader` and `asyncio.StreamWriter`. + +They help encapsulate the timeout logic, make it easier to unit-test +protocols, and help keep the rest of the package more `async`/`await` +based, and less strictly `asyncio`-specific. +""" +import asyncio +import enum +import ssl +import typing + +from .config import TimeoutConfig, DEFAULT_TIMEOUT_CONFIG +from .exceptions import ConnectTimeout, ReadTimeout, WriteTimeout + +OptionalTimeout = typing.Optional[TimeoutConfig] + + +class Protocol(enum.Enum): + HTTP_11 = 1 + HTTP_2 = 2 + + +class BaseReader: + async def read(self, n: int, timeout: OptionalTimeout = None) -> bytes: + raise NotImplementedError() # pragma: no cover + + +class BaseWriter: + def write_no_block(self, data: bytes) -> None: + raise NotImplementedError() # pragma: no cover + + async def write(self, data: bytes, timeout: OptionalTimeout = None) -> None: + raise NotImplementedError() # pragma: no cover + + async def close(self) -> None: + raise NotImplementedError() # pragma: no cover + + +class Reader(BaseReader): + def __init__( + self, stream_reader: asyncio.StreamReader, timeout: TimeoutConfig + ) -> None: + self.stream_reader = stream_reader + self.timeout = timeout + + async def read(self, n: int, timeout: OptionalTimeout = None) -> bytes: + if timeout is None: + timeout = self.timeout + + try: + data = await asyncio.wait_for( + self.stream_reader.read(n), timeout.read_timeout + ) + except asyncio.TimeoutError: + raise ReadTimeout() + + return data + + +class Writer(BaseWriter): + def __init__(self, stream_writer: asyncio.StreamWriter, timeout: TimeoutConfig): + self.stream_writer = stream_writer + self.timeout = timeout + + def write_no_block(self, data: bytes) -> None: + self.stream_writer.write(data) + + async def write(self, data: bytes, timeout: OptionalTimeout = None) -> None: + if not data: + return + + if timeout is None: + timeout = self.timeout + + self.stream_writer.write(data) + try: + data = await asyncio.wait_for( # type: ignore + self.stream_writer.drain(), timeout.write_timeout + ) + except asyncio.TimeoutError: + raise WriteTimeout() + + async def close(self) -> None: + self.stream_writer.close() + + +async def connect( + hostname: str, + port: int, + ssl_context: typing.Optional[ssl.SSLContext] = None, + timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, +) -> typing.Tuple[Reader, Writer, Protocol]: + try: + stream_reader, stream_writer = await asyncio.wait_for( # type: ignore + asyncio.open_connection(hostname, port, ssl=ssl_context), + timeout.connect_timeout, + ) + except asyncio.TimeoutError: + raise ConnectTimeout() + + ssl_object = stream_writer.get_extra_info("ssl_object") + if ssl_object is None: + ident = "http/1.1" + else: + ident = ssl_object.selected_alpn_protocol() + if ident is None: + ident = ssl_object.selected_npn_protocol() + + reader = Reader(stream_reader=stream_reader, timeout=timeout) + writer = Writer(stream_writer=stream_writer, timeout=timeout) + protocol = Protocol.HTTP_2 if ident == "h2" else Protocol.HTTP_11 + + return (reader, writer, protocol) diff --git a/tests/test_config.py b/tests/test_config.py index daf0e1ec97..8112d7c247 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -13,7 +13,7 @@ def test_timeout_repr(): timeout = httpcore.TimeoutConfig(read_timeout=5.0) assert ( repr(timeout) - == "TimeoutConfig(connect_timeout=None, read_timeout=5.0, pool_timeout=None)" + == "TimeoutConfig(connect_timeout=None, read_timeout=5.0, write_timeout=None, pool_timeout=None)" ) diff --git a/tests/test_http2.py b/tests/test_http2.py new file mode 100644 index 0000000000..7bfe301316 --- /dev/null +++ b/tests/test_http2.py @@ -0,0 +1,76 @@ +import h2.config +import h2.connection +import h2.events +import pytest + +import httpcore + + +class MockServer(httpcore.BaseReader, httpcore.BaseWriter): + """ + This class exposes Reader and Writer style interfaces + """ + + def __init__(self): + config = h2.config.H2Configuration(client_side=False) + self.conn = h2.connection.H2Connection(config=config) + self.buffer = b"" + self.requests = {} + + # BaseReader interface + + async def read(self, n, timeout) -> bytes: + send, self.buffer = self.buffer[:n], self.buffer[n:] + return send + + # BaseWriter interface + + def write_no_block(self, data: bytes) -> None: + events = self.conn.receive_data(data) + self.buffer += self.conn.data_to_send() + for event in events: + if isinstance(event, h2.events.RequestReceived): + self.request_received(event.headers, event.stream_id) + elif isinstance(event, h2.events.DataReceived): + self.receive_data(event.data, event.stream_id) + elif isinstance(event, h2.events.StreamEnded): + self.stream_complete(event.stream_id) + + async def write(self, data: bytes, timeout) -> None: + self.write_no_block(data) + + async def close(self) -> None: + pass + + # Server implementation + + def request_received(self, headers, stream_id): + if stream_id not in self.requests: + self.requests[stream_id] = [] + self.requests[stream_id].append({"headers": headers, "data": b""}) + + def receive_data(self, data, stream_id): + self.requests[stream_id][-1]["data"] += data + + def stream_complete(self, stream_id): + requests = self.requests[stream_id].pop(0) + if not self.requests[stream_id]: + del self.requests[stream_id] + + response_headers = ( + (b":status", b"200"), + ) + response_body = b"Hello, world!" + self.conn.send_headers(stream_id, response_headers) + self.conn.send_data(stream_id, response_body, end_stream=True) + self.buffer += self.conn.data_to_send() + + +@pytest.mark.asyncio +async def test_http2(): + server = MockServer() + origin = httpcore.Origin("http://example.org") + client = httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) + response = await client.request("GET", "http://example.org") + assert response.status_code == 200 + assert response.body == b"Hello, world!" From 9ab607d8a3316ee2456b74219ed882d486304714 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 12:23:20 +0100 Subject: [PATCH 06/23] More HTTP/2 tests --- httpcore/http2.py | 2 +- tests/test_http2.py | 50 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/httpcore/http2.py b/httpcore/http2.py index 41a0900cad..f8d2b648bf 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -144,4 +144,4 @@ async def release(self) -> None: await self.on_release(self) async def close(self) -> None: - self.writer.close() + await self.writer.close() diff --git a/tests/test_http2.py b/tests/test_http2.py index 7bfe301316..fbdb52d8b3 100644 --- a/tests/test_http2.py +++ b/tests/test_http2.py @@ -2,6 +2,7 @@ import h2.connection import h2.events import pytest +import json import httpcore @@ -53,24 +54,61 @@ def receive_data(self, data, stream_id): self.requests[stream_id][-1]["data"] += data def stream_complete(self, stream_id): - requests = self.requests[stream_id].pop(0) + request = self.requests[stream_id].pop(0) if not self.requests[stream_id]: del self.requests[stream_id] + request_headers = dict(request["headers"]) + request_data = request["data"] + + response_body = json.dumps({ + "method": request_headers[b":method"].decode(), + "path": request_headers[b":path"].decode(), + "body": request_data.decode() + }).encode() + response_headers = ( (b":status", b"200"), ) - response_body = b"Hello, world!" self.conn.send_headers(stream_id, response_headers) self.conn.send_data(stream_id, response_body, end_stream=True) self.buffer += self.conn.data_to_send() @pytest.mark.asyncio -async def test_http2(): +async def test_http2_get_request(): + server = MockServer() + origin = httpcore.Origin("http://example.org") + async with httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) as client: + response = await client.request("GET", "http://example.org") + assert response.status_code == 200 + assert json.loads(response.body) == {"method": "GET", "path": "/", "body": ""} + + +@pytest.mark.asyncio +async def test_http2_post_request(): server = MockServer() origin = httpcore.Origin("http://example.org") - client = httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) - response = await client.request("GET", "http://example.org") + async with httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) as client: + response = await client.request("POST", "http://example.org", body=b"") assert response.status_code == 200 - assert response.body == b"Hello, world!" + assert json.loads(response.body) == {"method": "POST", "path": "/", "body": ""} + + +@pytest.mark.asyncio +async def test_http2_multiple_requests(): + server = MockServer() + origin = httpcore.Origin("http://example.org") + async with httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) as client: + response_1 = await client.request("GET", "http://example.org/1") + response_2 = await client.request("GET", "http://example.org/2") + response_3 = await client.request("GET", "http://example.org/3") + + assert response_1.status_code == 200 + assert json.loads(response_1.body) == {"method": "GET", "path": "/1", "body": ""} + + assert response_2.status_code == 200 + assert json.loads(response_2.body) == {"method": "GET", "path": "/2", "body": ""} + + assert response_3.status_code == 200 + assert json.loads(response_3.body) == {"method": "GET", "path": "/3", "body": ""} From 286f04f1a641473d124c5e8403de24b323861054 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 12:57:18 +0100 Subject: [PATCH 07/23] Add PoolSemaphore --- httpcore/connection_pool.py | 38 +++----------------------- httpcore/streams.py | 53 ++++++++++++++++++++++++++++++++++--- tests/test_http2.py | 37 +++++++++++++++++--------- 3 files changed, 77 insertions(+), 51 deletions(-) diff --git a/httpcore/connection_pool.py b/httpcore/connection_pool.py index 6ec3028991..894212ab6d 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/connection_pool.py @@ -1,4 +1,3 @@ -import asyncio import typing from .config import ( @@ -13,6 +12,7 @@ from .connection import HTTPConnection from .exceptions import PoolTimeout from .models import Client, Origin, Request, Response +from .streams import PoolSemaphore class ConnectionPool(Client): @@ -32,9 +32,7 @@ def __init__( self._keepalive_connections = ( {} ) # type: typing.Dict[Origin, typing.List[HTTPConnection]] - self._max_connections = ConnectionSemaphore( - max_connections=self.limits.hard_limit - ) + self._max_connections = PoolSemaphore(limits, timeout) async def send( self, @@ -62,15 +60,7 @@ async def acquire_connection( self.num_active_connections += 1 except (KeyError, IndexError): - if timeout is None: - pool_timeout = self.timeout.pool_timeout - else: - pool_timeout = timeout.pool_timeout - - try: - await asyncio.wait_for(self._max_connections.acquire(), pool_timeout) - except asyncio.TimeoutError: - raise PoolTimeout() + await self._max_connections.acquire(timeout) connection = HTTPConnection( origin, ssl=self.ssl, @@ -108,25 +98,3 @@ async def close(self) -> None: self._keepalive_connections.clear() for connection in all_connections: await connection.close() - - -class ConnectionSemaphore: - def __init__(self, max_connections: int = None): - self.max_connections = max_connections - - @property - def semaphore(self) -> typing.Optional[asyncio.BoundedSemaphore]: - if not hasattr(self, "_semaphore"): - if self.max_connections is None: - self._semaphore = None - else: - self._semaphore = asyncio.BoundedSemaphore(value=self.max_connections) - return self._semaphore - - async def acquire(self) -> None: - if self.semaphore is not None: - await self.semaphore.acquire() - - def release(self) -> None: - if self.semaphore is not None: - self.semaphore.release() diff --git a/httpcore/streams.py b/httpcore/streams.py index 5a9a0abb31..cba51fd731 100644 --- a/httpcore/streams.py +++ b/httpcore/streams.py @@ -2,7 +2,9 @@ The `Reader` and `Writer` classes here provide a lightweight layer over `asyncio.StreamReader` and `asyncio.StreamWriter`. -They help encapsulate the timeout logic, make it easier to unit-test +Similarly `PoolSemaphore` is a lightweight layer over `BoundedSemaphore`. + +These classes help encapsulate the timeout logic, make it easier to unit-test protocols, and help keep the rest of the package more `async`/`await` based, and less strictly `asyncio`-specific. """ @@ -11,8 +13,8 @@ import ssl import typing -from .config import TimeoutConfig, DEFAULT_TIMEOUT_CONFIG -from .exceptions import ConnectTimeout, ReadTimeout, WriteTimeout +from .config import DEFAULT_TIMEOUT_CONFIG, PoolLimits, TimeoutConfig +from .exceptions import ConnectTimeout, ReadTimeout, PoolTimeout, WriteTimeout OptionalTimeout = typing.Optional[TimeoutConfig] @@ -38,6 +40,17 @@ async def close(self) -> None: raise NotImplementedError() # pragma: no cover +class BasePoolSemaphore: + def __init__(self, limits: PoolLimits, timeout: TimeoutConfig): + raise NotImplementedError() # pragma: no cover + + async def acquire(self, timeout: OptionalTimeout = None) -> None: + raise NotImplementedError() # pragma: no cover + + def release(self) -> None: + raise NotImplementedError() # pragma: no cover + + class Reader(BaseReader): def __init__( self, stream_reader: asyncio.StreamReader, timeout: TimeoutConfig @@ -86,6 +99,40 @@ async def close(self) -> None: self.stream_writer.close() +class PoolSemaphore(BasePoolSemaphore): + def __init__(self, limits: PoolLimits, timeout: TimeoutConfig): + self.limits = limits + self.timeout = timeout + + @property + def semaphore(self) -> typing.Optional[asyncio.BoundedSemaphore]: + if not hasattr(self, "_semaphore"): + max_connections = self.limits.hard_limit + if max_connections is None: + self._semaphore = None + else: + self._semaphore = asyncio.BoundedSemaphore(value=max_connections) + return self._semaphore + + async def acquire(self, timeout: OptionalTimeout = None) -> None: + if self.semaphore is None: + return + + if timeout is None: + timeout = self.timeout + + try: + await asyncio.wait_for(self.semaphore.acquire(), timeout.pool_timeout) + except asyncio.TimeoutError: + raise PoolTimeout() + + def release(self) -> None: + if self.semaphore is None: + return + + self.semaphore.release() + + async def connect( hostname: str, port: int, diff --git a/tests/test_http2.py b/tests/test_http2.py index fbdb52d8b3..f17d0f9822 100644 --- a/tests/test_http2.py +++ b/tests/test_http2.py @@ -1,8 +1,9 @@ +import json + import h2.config import h2.connection import h2.events import pytest -import json import httpcore @@ -61,15 +62,15 @@ def stream_complete(self, stream_id): request_headers = dict(request["headers"]) request_data = request["data"] - response_body = json.dumps({ - "method": request_headers[b":method"].decode(), - "path": request_headers[b":path"].decode(), - "body": request_data.decode() - }).encode() + response_body = json.dumps( + { + "method": request_headers[b":method"].decode(), + "path": request_headers[b":path"].decode(), + "body": request_data.decode(), + } + ).encode() - response_headers = ( - (b":status", b"200"), - ) + response_headers = ((b":status", b"200"),) self.conn.send_headers(stream_id, response_headers) self.conn.send_data(stream_id, response_body, end_stream=True) self.buffer += self.conn.data_to_send() @@ -79,7 +80,9 @@ def stream_complete(self, stream_id): async def test_http2_get_request(): server = MockServer() origin = httpcore.Origin("http://example.org") - async with httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) as client: + async with httpcore.HTTP2Connection( + reader=server, writer=server, origin=origin + ) as client: response = await client.request("GET", "http://example.org") assert response.status_code == 200 assert json.loads(response.body) == {"method": "GET", "path": "/", "body": ""} @@ -89,17 +92,25 @@ async def test_http2_get_request(): async def test_http2_post_request(): server = MockServer() origin = httpcore.Origin("http://example.org") - async with httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) as client: + async with httpcore.HTTP2Connection( + reader=server, writer=server, origin=origin + ) as client: response = await client.request("POST", "http://example.org", body=b"") assert response.status_code == 200 - assert json.loads(response.body) == {"method": "POST", "path": "/", "body": ""} + assert json.loads(response.body) == { + "method": "POST", + "path": "/", + "body": "", + } @pytest.mark.asyncio async def test_http2_multiple_requests(): server = MockServer() origin = httpcore.Origin("http://example.org") - async with httpcore.HTTP2Connection(reader=server, writer=server, origin=origin) as client: + async with httpcore.HTTP2Connection( + reader=server, writer=server, origin=origin + ) as client: response_1 = await client.request("GET", "http://example.org/1") response_2 = await client.request("GET", "http://example.org/2") response_3 = await client.request("GET", "http://example.org/3") From 28c505a70f080bca9fd9e5549a65c7dbdc9ce195 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 15:31:47 +0100 Subject: [PATCH 08/23] Tighten up connection acquiry/release --- httpcore/connection.py | 14 ++-- httpcore/connection_pool.py | 121 +++++++++++++++++++++++---------- httpcore/http11.py | 6 +- httpcore/http2.py | 2 +- httpcore/streams.py | 2 +- tests/test_connection_pools.py | 60 ++++++++-------- 6 files changed, 130 insertions(+), 75 deletions(-) diff --git a/httpcore/connection.py b/httpcore/connection.py index f164232ffe..a07a062aad 100644 --- a/httpcore/connection.py +++ b/httpcore/connection.py @@ -1,3 +1,4 @@ +import functools import typing import h2.connection @@ -17,12 +18,12 @@ def __init__( origin: typing.Union[str, Origin], ssl: SSLConfig = DEFAULT_SSL_CONFIG, timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, - on_release: typing.Callable = None, + pool_release_func: typing.Callable = None, ): self.origin = Origin(origin) if isinstance(origin, str) else origin self.ssl = ssl self.timeout = timeout - self.on_release = on_release + self.pool_release_func = pool_release_func self.h11_connection = None # type: typing.Optional[HTTP11Connection] self.h2_connection = None # type: typing.Optional[HTTP2Connection] @@ -43,6 +44,11 @@ async def send( port = self.origin.port ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None + if self.pool_release_func is None: + on_release = None + else: + on_release = functools.partial(self.pool_release_func, self) + reader, writer, protocol = await connect( hostname, port, ssl_context, timeout ) @@ -52,7 +58,7 @@ async def send( writer, origin=self.origin, timeout=self.timeout, - on_release=self.on_release, + on_release=on_release, ) else: self.h11_connection = HTTP11Connection( @@ -60,7 +66,7 @@ async def send( writer, origin=self.origin, timeout=self.timeout, - on_release=self.on_release, + on_release=on_release, ) if self.h2_connection is not None: diff --git a/httpcore/connection_pool.py b/httpcore/connection_pool.py index 894212ab6d..b541435787 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/connection_pool.py @@ -1,3 +1,4 @@ +import collections.abc import typing from .config import ( @@ -14,6 +15,66 @@ from .models import Client, Origin, Request, Response from .streams import PoolSemaphore +CONNECTIONS_DICT = typing.Dict[Origin, typing.List[HTTPConnection]] + + +class ConnectionStore(collections.abc.Sequence): + """ + We need to maintain collections of connections in a way that allows us to: + + * Lookup connections by origin. + * Iterate over connections by insertion time. + * Return the total number of connections. + """ + + def __init__(self) -> None: + self.all = {} # type: typing.Dict[HTTPConnection, float] + self.by_origin = ( + {} + ) # type: typing.Dict[Origin, typing.Dict[HTTPConnection, float]] + + def pop_by_origin(self, origin: Origin) -> typing.Optional[HTTPConnection]: + try: + connections = self.by_origin[origin] + except KeyError: + return None + + connection = next(reversed(list(connections.keys()))) + del connections[connection] + if not connections: + del self.by_origin[origin] + del self.all[connection] + + return connection + + def add(self, connection: HTTPConnection) -> None: + self.all[connection] = 0.0 + try: + self.by_origin[connection.origin][connection] = 0.0 + except KeyError: + self.by_origin[connection.origin] = {connection: 0.0} + + def remove(self, connection: HTTPConnection) -> None: + del self.all[connection] + del self.by_origin[connection.origin][connection] + if not self.by_origin[connection.origin]: + del self.by_origin[connection.origin] + + def clear(self) -> None: + self.all.clear() + self.by_origin.clear() + + def __iter__(self) -> typing.Iterator[HTTPConnection]: + return iter(self.all.keys()) + + def __getitem__(self, key: typing.Any) -> typing.Any: + if key in self.all: + return key + return None + + def __len__(self) -> int: + return len(self.all) + class ConnectionPool(Client): def __init__( @@ -27,12 +88,14 @@ def __init__( self.timeout = timeout self.limits = limits self.is_closed = False - self.num_active_connections = 0 - self.num_keepalive_connections = 0 - self._keepalive_connections = ( - {} - ) # type: typing.Dict[Origin, typing.List[HTTPConnection]] - self._max_connections = PoolSemaphore(limits, timeout) + + self.max_connections = PoolSemaphore(limits, timeout) + self.keepalive_connections = ConnectionStore() + self.active_connections = ConnectionStore() + + @property + def num_connections(self) -> int: + return len(self.keepalive_connections) + len(self.active_connections) async def send( self, @@ -45,56 +108,42 @@ async def send( response = await connection.send(request, ssl=ssl, timeout=timeout) return response - @property - def num_connections(self) -> int: - return self.num_active_connections + self.num_keepalive_connections - async def acquire_connection( self, origin: Origin, timeout: typing.Optional[TimeoutConfig] = None ) -> HTTPConnection: - try: - connection = self._keepalive_connections[origin].pop() - if not self._keepalive_connections[origin]: - del self._keepalive_connections[origin] - self.num_keepalive_connections -= 1 - self.num_active_connections += 1 - - except (KeyError, IndexError): - await self._max_connections.acquire(timeout) + connection = self.keepalive_connections.pop_by_origin(origin) + + if connection is None: + await self.max_connections.acquire(timeout) connection = HTTPConnection( origin, ssl=self.ssl, timeout=self.timeout, - on_release=self.release_connection, + pool_release_func=self.release_connection, ) - self.num_active_connections += 1 + + self.active_connections.add(connection) return connection async def release_connection(self, connection: HTTPConnection) -> None: if connection.is_closed: - self._max_connections.release() - self.num_active_connections -= 1 + self.active_connections.remove(connection) + self.max_connections.release() elif ( self.limits.soft_limit is not None and self.num_connections > self.limits.soft_limit ): - self._max_connections.release() - self.num_active_connections -= 1 + self.active_connections.remove(connection) + self.max_connections.release() await connection.close() else: - self.num_active_connections -= 1 - self.num_keepalive_connections += 1 - try: - self._keepalive_connections[connection.origin].append(connection) - except KeyError: - self._keepalive_connections[connection.origin] = [connection] + self.active_connections.remove(connection) + self.keepalive_connections.add(connection) async def close(self) -> None: self.is_closed = True - all_connections = [] - for connections in self._keepalive_connections.values(): - all_connections.extend(list(connections)) - self._keepalive_connections.clear() - for connection in all_connections: + connections = list(self.keepalive_connections) + self.keepalive_connections.clear() + for connection in connections: await connection.close() diff --git a/httpcore/http11.py b/httpcore/http11.py index 253865fe92..3280e1def8 100644 --- a/httpcore/http11.py +++ b/httpcore/http11.py @@ -82,7 +82,7 @@ async def send( protocol="HTTP/1.1", headers=headers, body=body, - on_close=self._release, + on_close=self.response_closed, ) async def _body_iter(self, timeout: OptionalTimeout) -> typing.AsyncIterator[bytes]: @@ -106,7 +106,7 @@ async def _receive_event(self, timeout: OptionalTimeout) -> H11Event: return event - async def _release(self) -> None: + async def response_closed(self) -> None: if ( self.h11_state.our_state is h11.DONE and self.h11_state.their_state is h11.DONE @@ -116,7 +116,7 @@ async def _release(self) -> None: await self.close() if self.on_release is not None: - await self.on_release(self) + await self.on_release() async def close(self) -> None: event = h11.ConnectionClosed() diff --git a/httpcore/http2.py b/httpcore/http2.py index f8d2b648bf..2b6c8e6dba 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -141,7 +141,7 @@ async def receive_event( async def release(self) -> None: if self.on_release is not None: - await self.on_release(self) + await self.on_release() async def close(self) -> None: await self.writer.close() diff --git a/httpcore/streams.py b/httpcore/streams.py index cba51fd731..e46ffee485 100644 --- a/httpcore/streams.py +++ b/httpcore/streams.py @@ -14,7 +14,7 @@ import typing from .config import DEFAULT_TIMEOUT_CONFIG, PoolLimits, TimeoutConfig -from .exceptions import ConnectTimeout, ReadTimeout, PoolTimeout, WriteTimeout +from .exceptions import ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout OptionalTimeout = typing.Optional[TimeoutConfig] diff --git a/tests/test_connection_pools.py b/tests/test_connection_pools.py index 77a2215754..7d478c5ac6 100644 --- a/tests/test_connection_pools.py +++ b/tests/test_connection_pools.py @@ -10,12 +10,12 @@ async def test_keepalive_connections(server): """ async with httpcore.ConnectionPool() as http: response = await http.request("GET", "http://127.0.0.1:8000/") - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 1 response = await http.request("GET", "http://127.0.0.1:8000/") - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 1 @pytest.mark.asyncio @@ -25,12 +25,12 @@ async def test_differing_connection_keys(server): """ async with httpcore.ConnectionPool() as http: response = await http.request("GET", "http://127.0.0.1:8000/") - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 1 response = await http.request("GET", "http://localhost:8000/") - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 2 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 2 @pytest.mark.asyncio @@ -42,12 +42,12 @@ async def test_soft_limit(server): async with httpcore.ConnectionPool(limits=limits) as http: response = await http.request("GET", "http://127.0.0.1:8000/") - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 1 response = await http.request("GET", "http://localhost:8000/") - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 1 @pytest.mark.asyncio @@ -57,13 +57,13 @@ async def test_streaming_response_holds_connection(server): """ async with httpcore.ConnectionPool() as http: response = await http.request("GET", "http://127.0.0.1:8000/", stream=True) - assert http.num_active_connections == 1 - assert http.num_keepalive_connections == 0 + assert len(http.active_connections) == 1 + assert len(http.keepalive_connections) == 0 await response.read() - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 1 @pytest.mark.asyncio @@ -73,20 +73,20 @@ async def test_multiple_concurrent_connections(server): """ async with httpcore.ConnectionPool() as http: response_a = await http.request("GET", "http://127.0.0.1:8000/", stream=True) - assert http.num_active_connections == 1 - assert http.num_keepalive_connections == 0 + assert len(http.active_connections) == 1 + assert len(http.keepalive_connections) == 0 response_b = await http.request("GET", "http://127.0.0.1:8000/", stream=True) - assert http.num_active_connections == 2 - assert http.num_keepalive_connections == 0 + assert len(http.active_connections) == 2 + assert len(http.keepalive_connections) == 0 await response_b.read() - assert http.num_active_connections == 1 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 1 + assert len(http.keepalive_connections) == 1 await response_a.read() - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 2 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 2 @pytest.mark.asyncio @@ -97,8 +97,8 @@ async def test_close_connections(server): headers = [(b"connection", b"close")] async with httpcore.ConnectionPool() as http: response = await http.request("GET", "http://127.0.0.1:8000/", headers=headers) - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 0 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 0 @pytest.mark.asyncio @@ -110,8 +110,8 @@ async def test_standard_response_close(server): response = await http.request("GET", "http://127.0.0.1:8000/", stream=True) await response.read() await response.close() - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 1 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 1 @pytest.mark.asyncio @@ -122,5 +122,5 @@ async def test_premature_response_close(server): async with httpcore.ConnectionPool() as http: response = await http.request("GET", "http://127.0.0.1:8000/", stream=True) await response.close() - assert http.num_active_connections == 0 - assert http.num_keepalive_connections == 0 + assert len(http.active_connections) == 0 + assert len(http.keepalive_connections) == 0 From f942eb94f113f29b1415e47a1c0640d9cdf0ce26 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 16:17:42 +0100 Subject: [PATCH 09/23] Add http2 connection re-use --- httpcore/connection.py | 75 +++++++++++++++++++++---------------- httpcore/connection_pool.py | 11 +++++- httpcore/http2.py | 12 ++++-- 3 files changed, 59 insertions(+), 39 deletions(-) diff --git a/httpcore/connection.py b/httpcore/connection.py index a07a062aad..d5caadeeef 100644 --- a/httpcore/connection.py +++ b/httpcore/connection.py @@ -35,39 +35,7 @@ async def send( timeout: typing.Optional[TimeoutConfig] = None, ) -> Response: if self.h11_connection is None and self.h2_connection is None: - if ssl is None: - ssl = self.ssl - if timeout is None: - timeout = self.timeout - - hostname = self.origin.hostname - port = self.origin.port - ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None - - if self.pool_release_func is None: - on_release = None - else: - on_release = functools.partial(self.pool_release_func, self) - - reader, writer, protocol = await connect( - hostname, port, ssl_context, timeout - ) - if protocol == Protocol.HTTP_2: - self.h2_connection = HTTP2Connection( - reader, - writer, - origin=self.origin, - timeout=self.timeout, - on_release=on_release, - ) - else: - self.h11_connection = HTTP11Connection( - reader, - writer, - origin=self.origin, - timeout=self.timeout, - on_release=on_release, - ) + await self.connect(ssl, timeout) if self.h2_connection is not None: response = await self.h2_connection.send(request, ssl=ssl, timeout=timeout) @@ -77,12 +45,53 @@ async def send( return response + async def connect( + self, + ssl: typing.Optional[SSLConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None, + ) -> None: + if ssl is None: + ssl = self.ssl + if timeout is None: + timeout = self.timeout + + hostname = self.origin.hostname + port = self.origin.port + ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None + + if self.pool_release_func is None: + on_release = None + else: + on_release = functools.partial(self.pool_release_func, self) + + reader, writer, protocol = await connect(hostname, port, ssl_context, timeout) + if protocol == Protocol.HTTP_2: + self.h2_connection = HTTP2Connection( + reader, + writer, + origin=self.origin, + timeout=self.timeout, + on_release=on_release, + ) + else: + self.h11_connection = HTTP11Connection( + reader, + writer, + origin=self.origin, + timeout=self.timeout, + on_release=on_release, + ) + async def close(self) -> None: if self.h2_connection is not None: await self.h2_connection.close() elif self.h11_connection is not None: await self.h11_connection.close() + @property + def is_http2(self) -> bool: + return self.h2_connection is not None + @property def is_closed(self) -> bool: if self.h2_connection is not None: diff --git a/httpcore/connection_pool.py b/httpcore/connection_pool.py index b541435787..709ad0f713 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/connection_pool.py @@ -33,13 +33,18 @@ def __init__(self) -> None: {} ) # type: typing.Dict[Origin, typing.Dict[HTTPConnection, float]] - def pop_by_origin(self, origin: Origin) -> typing.Optional[HTTPConnection]: + def pop_by_origin( + self, origin: Origin, http2_only: bool = False + ) -> typing.Optional[HTTPConnection]: try: connections = self.by_origin[origin] except KeyError: return None connection = next(reversed(list(connections.keys()))) + if http2_only and not connection.is_http2: + return None + del connections[connection] if not connections: del self.by_origin[origin] @@ -111,7 +116,9 @@ async def send( async def acquire_connection( self, origin: Origin, timeout: typing.Optional[TimeoutConfig] = None ) -> HTTPConnection: - connection = self.keepalive_connections.pop_by_origin(origin) + connection = self.active_connections.pop_by_origin(origin, http2_only=True) + if connection is None: + connection = self.keepalive_connections.pop_by_origin(origin) if connection is None: await self.max_connections.acquire(timeout) diff --git a/httpcore/http2.py b/httpcore/http2.py index 2b6c8e6dba..e48c4de8e0 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -1,3 +1,4 @@ +import functools import typing import h2.connection @@ -74,12 +75,14 @@ async def send( headers.append((k, v)) body = self.body_iter(stream_id, timeout) + on_close = functools.partial(self.response_closed, stream_id=stream_id) + return Response( status_code=status_code, protocol="HTTP/2", headers=headers, body=body, - on_close=self.release, + on_close=on_close, ) def initiate_connection(self) -> None: @@ -121,7 +124,6 @@ async def body_iter( if isinstance(event, h2.events.DataReceived): yield event.data elif isinstance(event, h2.events.StreamEnded): - del self.events[stream_id] break async def receive_event( @@ -139,8 +141,10 @@ async def receive_event( return self.events[stream_id].pop(0) - async def release(self) -> None: - if self.on_release is not None: + async def response_closed(self, stream_id: int) -> None: + del self.events[stream_id] + + if not self.events and self.on_release is not None: await self.on_release() async def close(self) -> None: From 06f6d50084343e894fb7535e0dcf1da3e8caa725 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 17:03:14 +0100 Subject: [PATCH 10/23] More specfic annotations on callbacks --- httpcore/connection.py | 11 +++++++---- httpcore/connection_pool.py | 2 +- httpcore/http11.py | 9 +++++++-- httpcore/http2.py | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/httpcore/connection.py b/httpcore/connection.py index d5caadeeef..a9c9890e80 100644 --- a/httpcore/connection.py +++ b/httpcore/connection.py @@ -11,6 +11,9 @@ from .models import Client, Origin, Request, Response from .streams import Protocol, connect +# Callback signature: async def callback(conn: HTTPConnection) -> None +ReleaseCallback = typing.Callable[["HTTPConnection"], typing.Awaitable[None]] + class HTTPConnection(Client): def __init__( @@ -18,12 +21,12 @@ def __init__( origin: typing.Union[str, Origin], ssl: SSLConfig = DEFAULT_SSL_CONFIG, timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, - pool_release_func: typing.Callable = None, + release_func: typing.Optional[ReleaseCallback] = None, ): self.origin = Origin(origin) if isinstance(origin, str) else origin self.ssl = ssl self.timeout = timeout - self.pool_release_func = pool_release_func + self.release_func = release_func self.h11_connection = None # type: typing.Optional[HTTP11Connection] self.h2_connection = None # type: typing.Optional[HTTP2Connection] @@ -59,10 +62,10 @@ async def connect( port = self.origin.port ssl_context = await ssl.load_ssl_context() if self.origin.is_ssl else None - if self.pool_release_func is None: + if self.release_func is None: on_release = None else: - on_release = functools.partial(self.pool_release_func, self) + on_release = functools.partial(self.release_func, self) reader, writer, protocol = await connect(hostname, port, ssl_context, timeout) if protocol == Protocol.HTTP_2: diff --git a/httpcore/connection_pool.py b/httpcore/connection_pool.py index 709ad0f713..4ba83ec3dc 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/connection_pool.py @@ -126,7 +126,7 @@ async def acquire_connection( origin, ssl=self.ssl, timeout=self.timeout, - pool_release_func=self.release_connection, + release_func=self.release_connection, ) self.active_connections.add(connection) diff --git a/httpcore/http11.py b/httpcore/http11.py index 3280e1def8..3fda6d1f0d 100644 --- a/httpcore/http11.py +++ b/httpcore/http11.py @@ -19,6 +19,11 @@ OptionalTimeout = typing.Optional[TimeoutConfig] +# Callback signature: async def callback() -> None +# In practice the callback will be a functools partial, which binds +# the `ConnectionPool.release_connection(conn: HTTPConnection)` method. +OnReleaseCallback = typing.Callable[[], typing.Awaitable[None]] + class HTTP11Connection(Client): READ_NUM_BYTES = 4096 @@ -29,11 +34,11 @@ def __init__( writer: BaseWriter, origin: Origin, timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, - on_release: typing.Callable = None, + on_release: typing.Optional[OnReleaseCallback] = None, ): - self.origin = origin self.reader = reader self.writer = writer + self.origin = origin self.timeout = timeout self.on_release = on_release self.h11_state = h11.Connection(our_role=h11.CLIENT) diff --git a/httpcore/http2.py b/httpcore/http2.py index e48c4de8e0..c60490ce12 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -23,9 +23,9 @@ def __init__( timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, on_release: typing.Callable = None, ): - self.origin = origin self.reader = reader self.writer = writer + self.origin = origin self.timeout = timeout self.on_release = on_release self.h2_state = h2.connection.H2Connection() From 6d35506277f66f6e88e85a75a2dadd758d98cce2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 17:05:40 +0100 Subject: [PATCH 11/23] Drop unneccessary __hash__ methods --- httpcore/config.py | 17 ----------------- tests/test_config.py | 21 --------------------- 2 files changed, 38 deletions(-) diff --git a/httpcore/config.py b/httpcore/config.py index b0fadc4017..7166c05f75 100644 --- a/httpcore/config.py +++ b/httpcore/config.py @@ -27,10 +27,6 @@ def __eq__(self, other: typing.Any) -> bool: and self.verify == other.verify ) - def __hash__(self) -> int: - as_tuple = (self.cert, self.verify) - return hash(as_tuple) - def __repr__(self) -> str: class_name = self.__class__.__name__ return f"{class_name}(cert={self.cert}, verify={self.verify})" @@ -144,15 +140,6 @@ def __eq__(self, other: typing.Any) -> bool: and self.pool_timeout == other.pool_timeout ) - def __hash__(self) -> int: - as_tuple = ( - self.connect_timeout, - self.read_timeout, - self.write_timeout, - self.pool_timeout, - ) - return hash(as_tuple) - def __repr__(self) -> str: class_name = self.__class__.__name__ if self.timeout is not None: @@ -181,10 +168,6 @@ def __eq__(self, other: typing.Any) -> bool: and self.hard_limit == other.hard_limit ) - def __hash__(self) -> int: - as_tuple = (self.soft_limit, self.hard_limit) - return hash(as_tuple) - def __repr__(self) -> str: class_name = self.__class__.__name__ return ( diff --git a/tests/test_config.py b/tests/test_config.py index 8112d7c247..74c1267ea1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -35,24 +35,3 @@ def test_timeout_eq(): def test_limits_eq(): limits = httpcore.PoolLimits(hard_limit=100) assert limits == httpcore.PoolLimits(hard_limit=100) - - -def test_ssl_hash(): - cache = {} - ssl = httpcore.SSLConfig(verify=False) - cache[ssl] = "example" - assert cache[httpcore.SSLConfig(verify=False)] == "example" - - -def test_timeout_hash(): - cache = {} - timeout = httpcore.TimeoutConfig(timeout=5.0) - cache[timeout] = "example" - assert cache[httpcore.TimeoutConfig(timeout=5.0)] == "example" - - -def test_limits_hash(): - cache = {} - limits = httpcore.PoolLimits(hard_limit=100) - cache[limits] = "example" - assert cache[httpcore.PoolLimits(hard_limit=100)] == "example" From ddf1cec65a0dc6c18d7a163ff2fc73b2eb134cce Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Apr 2019 17:16:37 +0100 Subject: [PATCH 12/23] Drop connections from pool on connection exceptions --- httpcore/connection_pool.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/httpcore/connection_pool.py b/httpcore/connection_pool.py index 4ba83ec3dc..6eaef9bef2 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/connection_pool.py @@ -110,7 +110,12 @@ async def send( timeout: typing.Optional[TimeoutConfig] = None, ) -> Response: connection = await self.acquire_connection(request.url.origin, timeout=timeout) - response = await connection.send(request, ssl=ssl, timeout=timeout) + try: + response = await connection.send(request, ssl=ssl, timeout=timeout) + except BaseException as exc: + self.active_connections.remove(connection) + self.max_connections.release() + raise exc return response async def acquire_connection( From fab6fcd397e43e97286ba284816762cf6e7e55c9 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 26 Apr 2019 15:34:04 +0100 Subject: [PATCH 13/23] Adapters --- httpcore/__init__.py | 2 + httpcore/adapters.py | 40 ++++++++++++ httpcore/auth.py | 18 ++++++ httpcore/client.py | 124 ++++++++++++++++++++++++++++++++++++ httpcore/config.py | 21 +++--- httpcore/connection.py | 50 +++++---------- httpcore/connection_pool.py | 28 ++++---- httpcore/cookies.py | 18 ++++++ httpcore/environment.py | 27 ++++++++ httpcore/exceptions.py | 6 ++ httpcore/http11.py | 62 ++++++++++-------- httpcore/http2.py | 53 ++++++++------- httpcore/models.py | 48 +------------- httpcore/redirects.py | 35 ++++++++++ httpcore/streams.py | 16 ++--- httpcore/sync.py | 21 ++---- tests/test_api.py | 16 ++--- tests/test_config.py | 6 +- tests/test_http2.py | 25 +++----- tests/test_timeouts.py | 5 +- 20 files changed, 404 insertions(+), 217 deletions(-) create mode 100644 httpcore/adapters.py create mode 100644 httpcore/auth.py create mode 100644 httpcore/client.py create mode 100644 httpcore/cookies.py create mode 100644 httpcore/environment.py create mode 100644 httpcore/redirects.py diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 00cb5fb80b..30ed38e607 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -1,3 +1,5 @@ +from .adapters import Adapter +from .client import Client from .config import PoolLimits, SSLConfig, TimeoutConfig from .connection import HTTPConnection from .connection_pool import ConnectionPool diff --git a/httpcore/adapters.py b/httpcore/adapters.py new file mode 100644 index 0000000000..0c14e89ceb --- /dev/null +++ b/httpcore/adapters.py @@ -0,0 +1,40 @@ +import typing +from types import TracebackType + +from .models import URL, Request, Response + + +class Adapter: + async def request( + self, + method: str, + url: typing.Union[str, URL], + *, + headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", + **options: typing.Any, + ) -> Response: + request = Request(method, url, headers=headers, body=body) + self.prepare_request(request) + response = await self.send(request, **options) + return response + + def prepare_request(self, request: Request) -> None: + raise NotImplementedError() # pragma: nocover + + async def send(self, request: Request, **options: typing.Any) -> Response: + raise NotImplementedError() # pragma: nocover + + async def close(self) -> None: + raise NotImplementedError() # pragma: nocover + + async def __aenter__(self) -> "Adapter": + return self + + async def __aexit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + await self.close() diff --git a/httpcore/auth.py b/httpcore/auth.py new file mode 100644 index 0000000000..949d8a9fb6 --- /dev/null +++ b/httpcore/auth.py @@ -0,0 +1,18 @@ +import typing + +from .adapters import Adapter +from .models import Request, Response + + +class AuthAdapter(Adapter): + def __init__(self, dispatch: Adapter): + self.dispatch = dispatch + + def prepare_request(self, request: Request) -> None: + self.dispatch.prepare_request(request) + + async def send(self, request: Request, **options: typing.Any) -> Response: + return await self.dispatch.send(request, **options) + + async def close(self) -> None: + self.dispatch.close() diff --git a/httpcore/client.py b/httpcore/client.py new file mode 100644 index 0000000000..022ae7ffe5 --- /dev/null +++ b/httpcore/client.py @@ -0,0 +1,124 @@ +import typing +from types import TracebackType + +from .auth import AuthAdapter +from .config import ( + DEFAULT_MAX_REDIRECTS, + DEFAULT_POOL_LIMITS, + DEFAULT_SSL_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + PoolLimits, + SSLConfig, + TimeoutConfig, +) +from .connection_pool import ConnectionPool +from .cookies import CookieAdapter +from .environment import EnvironmentAdapter +from .models import URL, Request, Response +from .redirects import RedirectAdapter + + +class Client: + def __init__( + self, + ssl: SSLConfig = DEFAULT_SSL_CONFIG, + timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, + limits: PoolLimits = DEFAULT_POOL_LIMITS, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + ): + connection_pool = ConnectionPool(ssl=ssl, timeout=timeout, limits=limits) + cookie_adapter = CookieAdapter(dispatch=connection_pool) + auth_adapter = AuthAdapter(dispatch=cookie_adapter) + redirect_adapter = RedirectAdapter( + dispatch=auth_adapter, max_redirects=max_redirects + ) + self.adapter = EnvironmentAdapter(dispatch=redirect_adapter) + + async def request( + self, + method: str, + url: typing.Union[str, URL], + *, + body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", + headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + stream: bool = False, + allow_redirects: bool = True, + ssl: typing.Optional[SSLConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None, + ) -> Response: + request = Request(method, url, headers=headers, body=body) + self.prepare_request(request) + response = await self.send( + request, + stream=stream, + allow_redirects=allow_redirects, + ssl=ssl, + timeout=timeout, + ) + return response + + async def get( + self, + url: typing.Union[str, URL], + *, + headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + stream: bool = False, + ssl: typing.Optional[SSLConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None, + ) -> Response: + return await self.request( + "GET", url, headers=headers, stream=stream, ssl=ssl, timeout=timeout + ) + + async def post( + self, + url: typing.Union[str, URL], + *, + body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", + headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + stream: bool = False, + ssl: typing.Optional[SSLConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None, + ) -> Response: + return await self.request( + "POST", + url, + body=body, + headers=headers, + stream=stream, + ssl=ssl, + timeout=timeout, + ) + + def prepare_request(self, request: Request) -> None: + self.adapter.prepare_request(request) + + async def send( + self, + request: Request, + *, + stream: bool = False, + allow_redirects: bool = True, + ssl: typing.Optional[SSLConfig] = None, + timeout: typing.Optional[TimeoutConfig] = None, + ) -> Response: + options = {"stream": stream} # type: typing.Dict[str, typing.Any] + if ssl is not None: + options["ssl"] = ssl + if timeout is not None: + options["timeout"] = timeout + return await self.adapter.send(request, **options) + + async def close(self) -> None: + await self.adapter.close() + + async def __aenter__(self) -> "Client": + return self + + async def __aexit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + await self.close() diff --git a/httpcore/config.py b/httpcore/config.py index 7166c05f75..5ce24707c3 100644 --- a/httpcore/config.py +++ b/httpcore/config.py @@ -112,24 +112,20 @@ def __init__( connect_timeout: float = None, read_timeout: float = None, write_timeout: float = None, - pool_timeout: float = None, ): if timeout is not None: # Specified as a single timeout value assert connect_timeout is None assert read_timeout is None assert write_timeout is None - assert pool_timeout is None connect_timeout = timeout read_timeout = timeout write_timeout = timeout - pool_timeout = timeout self.timeout = timeout self.connect_timeout = connect_timeout self.read_timeout = read_timeout self.write_timeout = write_timeout - self.pool_timeout = pool_timeout def __eq__(self, other: typing.Any) -> bool: return ( @@ -137,14 +133,13 @@ def __eq__(self, other: typing.Any) -> bool: and self.connect_timeout == other.connect_timeout and self.read_timeout == other.read_timeout and self.write_timeout == other.write_timeout - and self.pool_timeout == other.pool_timeout ) def __repr__(self) -> str: class_name = self.__class__.__name__ if self.timeout is not None: return f"{class_name}(timeout={self.timeout})" - return f"{class_name}(connect_timeout={self.connect_timeout}, read_timeout={self.read_timeout}, write_timeout={self.write_timeout}, pool_timeout={self.pool_timeout})" + return f"{class_name}(connect_timeout={self.connect_timeout}, read_timeout={self.read_timeout}, write_timeout={self.write_timeout})" class PoolLimits: @@ -155,27 +150,29 @@ class PoolLimits: def __init__( self, *, - soft_limit: typing.Optional[int] = None, - hard_limit: typing.Optional[int] = None, + soft_limit: int = None, + hard_limit: int = None, + pool_timeout: float = None, ): self.soft_limit = soft_limit self.hard_limit = hard_limit + self.pool_timeout = pool_timeout def __eq__(self, other: typing.Any) -> bool: return ( isinstance(other, self.__class__) and self.soft_limit == other.soft_limit and self.hard_limit == other.hard_limit + and self.pool_timeout == other.pool_timeout ) def __repr__(self) -> str: class_name = self.__class__.__name__ - return ( - f"{class_name}(soft_limit={self.soft_limit}, hard_limit={self.hard_limit})" - ) + return f"{class_name}(soft_limit={self.soft_limit}, hard_limit={self.hard_limit}, pool_timeout={self.pool_timeout})" DEFAULT_SSL_CONFIG = SSLConfig(cert=None, verify=True) DEFAULT_TIMEOUT_CONFIG = TimeoutConfig(timeout=5.0) -DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100) +DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100, pool_timeout=5.0) DEFAULT_CA_BUNDLE_PATH = certifi.where() +DEFAULT_MAX_REDIRECTS = 30 diff --git a/httpcore/connection.py b/httpcore/connection.py index a9c9890e80..1662018a59 100644 --- a/httpcore/connection.py +++ b/httpcore/connection.py @@ -4,18 +4,19 @@ import h2.connection import h11 +from .adapters import Adapter from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig from .exceptions import ConnectTimeout from .http2 import HTTP2Connection from .http11 import HTTP11Connection -from .models import Client, Origin, Request, Response +from .models import Origin, Request, Response from .streams import Protocol, connect # Callback signature: async def callback(conn: HTTPConnection) -> None ReleaseCallback = typing.Callable[["HTTPConnection"], typing.Awaitable[None]] -class HTTPConnection(Client): +class HTTPConnection(Adapter): def __init__( self, origin: typing.Union[str, Origin], @@ -30,33 +31,26 @@ def __init__( self.h11_connection = None # type: typing.Optional[HTTP11Connection] self.h2_connection = None # type: typing.Optional[HTTP2Connection] - async def send( - self, - request: Request, - *, - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None, - ) -> Response: + def prepare_request(self, request: Request) -> None: + pass + + async def send(self, request: Request, **options: typing.Any) -> Response: if self.h11_connection is None and self.h2_connection is None: - await self.connect(ssl, timeout) + await self.connect(**options) if self.h2_connection is not None: - response = await self.h2_connection.send(request, ssl=ssl, timeout=timeout) + response = await self.h2_connection.send(request, **options) else: assert self.h11_connection is not None - response = await self.h11_connection.send(request, ssl=ssl, timeout=timeout) + response = await self.h11_connection.send(request, **options) return response - async def connect( - self, - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None, - ) -> None: - if ssl is None: - ssl = self.ssl - if timeout is None: - timeout = self.timeout + async def connect(self, **options: typing.Any) -> None: + ssl = options.get("ssl", self.ssl) + timeout = options.get("timeout", self.timeout) + assert isinstance(ssl, SSLConfig) + assert isinstance(timeout, TimeoutConfig) hostname = self.origin.hostname port = self.origin.port @@ -69,20 +63,10 @@ async def connect( reader, writer, protocol = await connect(hostname, port, ssl_context, timeout) if protocol == Protocol.HTTP_2: - self.h2_connection = HTTP2Connection( - reader, - writer, - origin=self.origin, - timeout=self.timeout, - on_release=on_release, - ) + self.h2_connection = HTTP2Connection(reader, writer, on_release=on_release) else: self.h11_connection = HTTP11Connection( - reader, - writer, - origin=self.origin, - timeout=self.timeout, - on_release=on_release, + reader, writer, on_release=on_release ) async def close(self) -> None: diff --git a/httpcore/connection_pool.py b/httpcore/connection_pool.py index 6eaef9bef2..ab394a1963 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/connection_pool.py @@ -1,6 +1,7 @@ import collections.abc import typing +from .adapters import Adapter from .config import ( DEFAULT_CA_BUNDLE_PATH, DEFAULT_POOL_LIMITS, @@ -12,7 +13,7 @@ ) from .connection import HTTPConnection from .exceptions import PoolTimeout -from .models import Client, Origin, Request, Response +from .models import Origin, Request, Response from .streams import PoolSemaphore CONNECTIONS_DICT = typing.Dict[Origin, typing.List[HTTPConnection]] @@ -81,7 +82,7 @@ def __len__(self) -> int: return len(self.all) -class ConnectionPool(Client): +class ConnectionPool(Adapter): def __init__( self, *, @@ -94,7 +95,7 @@ def __init__( self.limits = limits self.is_closed = False - self.max_connections = PoolSemaphore(limits, timeout) + self.max_connections = PoolSemaphore(limits) self.keepalive_connections = ConnectionStore() self.active_connections = ConnectionStore() @@ -102,31 +103,26 @@ def __init__( def num_connections(self) -> int: return len(self.keepalive_connections) + len(self.active_connections) - async def send( - self, - request: Request, - *, - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None, - ) -> Response: - connection = await self.acquire_connection(request.url.origin, timeout=timeout) + def prepare_request(self, request: Request) -> None: + pass + + async def send(self, request: Request, **options: typing.Any) -> Response: + connection = await self.acquire_connection(request.url.origin) try: - response = await connection.send(request, ssl=ssl, timeout=timeout) + response = await connection.send(request, **options) except BaseException as exc: self.active_connections.remove(connection) self.max_connections.release() raise exc return response - async def acquire_connection( - self, origin: Origin, timeout: typing.Optional[TimeoutConfig] = None - ) -> HTTPConnection: + async def acquire_connection(self, origin: Origin) -> HTTPConnection: connection = self.active_connections.pop_by_origin(origin, http2_only=True) if connection is None: connection = self.keepalive_connections.pop_by_origin(origin) if connection is None: - await self.max_connections.acquire(timeout) + await self.max_connections.acquire() connection = HTTPConnection( origin, ssl=self.ssl, diff --git a/httpcore/cookies.py b/httpcore/cookies.py new file mode 100644 index 0000000000..f6fd2b0381 --- /dev/null +++ b/httpcore/cookies.py @@ -0,0 +1,18 @@ +import typing + +from .adapters import Adapter +from .models import Request, Response + + +class CookieAdapter(Adapter): + def __init__(self, dispatch: Adapter): + self.dispatch = dispatch + + def prepare_request(self, request: Request) -> None: + self.dispatch.prepare_request(request) + + async def send(self, request: Request, **options: typing.Any) -> Response: + return await self.dispatch.send(request, **options) + + async def close(self) -> None: + self.dispatch.close() diff --git a/httpcore/environment.py b/httpcore/environment.py new file mode 100644 index 0000000000..5065eed884 --- /dev/null +++ b/httpcore/environment.py @@ -0,0 +1,27 @@ +import typing + +from .adapters import Adapter +from .models import Request, Response + + +class EnvironmentAdapter(Adapter): + def __init__(self, dispatch: Adapter, trust_env: bool = True): + self.dispatch = dispatch + self.trust_env = trust_env + + def prepare_request(self, request: Request) -> None: + self.dispatch.prepare_request(request) + + async def send(self, request: Request, **options: typing.Any) -> Response: + if self.trust_env: + self.merge_environment_options(options) + return await self.dispatch.send(request, **options) + + async def close(self) -> None: + await self.dispatch.close() + + def merge_environment_options(self, options: dict) -> None: + """ + Add environment options. + """ + #  TODO diff --git a/httpcore/exceptions.py b/httpcore/exceptions.py index 285b64039b..94154b073e 100644 --- a/httpcore/exceptions.py +++ b/httpcore/exceptions.py @@ -28,6 +28,12 @@ class PoolTimeout(Timeout): """ +class TooManyRedirects(Exception): + """ + Too many redirects. + """ + + class ProtocolError(Exception): """ Malformed HTTP. diff --git a/httpcore/http11.py b/httpcore/http11.py index 3fda6d1f0d..0075b524ce 100644 --- a/httpcore/http11.py +++ b/httpcore/http11.py @@ -2,9 +2,10 @@ import h11 +from .adapters import Adapter from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig from .exceptions import ConnectTimeout, ReadTimeout -from .models import Client, Origin, Request, Response +from .models import Request, Response from .streams import BaseReader, BaseWriter H11Event = typing.Union[ @@ -25,35 +26,28 @@ OnReleaseCallback = typing.Callable[[], typing.Awaitable[None]] -class HTTP11Connection(Client): +class HTTP11Connection(Adapter): READ_NUM_BYTES = 4096 def __init__( self, reader: BaseReader, writer: BaseWriter, - origin: Origin, - timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, on_release: typing.Optional[OnReleaseCallback] = None, ): self.reader = reader self.writer = writer - self.origin = origin - self.timeout = timeout self.on_release = on_release self.h11_state = h11.Connection(our_role=h11.CLIENT) - @property - def is_closed(self) -> bool: - return self.h11_state.our_state in (h11.CLOSED, h11.ERROR) + def prepare_request(self, request: Request) -> None: + pass + + async def send(self, request: Request, **options: typing.Any) -> Response: + timeout = options.get("timeout") + stream = options.get("stream", False) + assert timeout is None or isinstance(timeout, TimeoutConfig) - async def send( - self, - request: Request, - *, - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None - ) -> Response: #  Start sending the request. method = request.method.encode() target = request.url.full_path @@ -81,7 +75,7 @@ async def send( headers = event.headers body = self._body_iter(timeout) - return Response( + response = Response( status_code=status_code, reason=reason, protocol="HTTP/1.1", @@ -90,6 +84,26 @@ async def send( on_close=self.response_closed, ) + if not stream: + try: + await response.read() + finally: + await response.close() + + return response + + async def close(self) -> None: + event = h11.ConnectionClosed() + try: + # If we're in h11.MUST_CLOSE then we'll end up in h11.CLOSED. + self.h11_state.send(event) + except h11.ProtocolError: + # If we're in some other state then it's a premature close, + # and we'll end up in h11.ERROR. + pass + + await self.writer.close() + async def _body_iter(self, timeout: OptionalTimeout) -> typing.AsyncIterator[bytes]: event = await self._receive_event(timeout) while isinstance(event, h11.Data): @@ -123,14 +137,6 @@ async def response_closed(self) -> None: if self.on_release is not None: await self.on_release() - async def close(self) -> None: - event = h11.ConnectionClosed() - try: - # If we're in h11.MUST_CLOSE then we'll end up in h11.CLOSED. - self.h11_state.send(event) - except h11.ProtocolError: - # If we're in some other state then it's a premature close, - # and we'll end up in h11.ERROR. - pass - - await self.writer.close() + @property + def is_closed(self) -> bool: + return self.h11_state.our_state in (h11.CLOSED, h11.ERROR) diff --git a/httpcore/http2.py b/httpcore/http2.py index c60490ce12..e89029cf94 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -4,52 +4,39 @@ import h2.connection import h2.events +from .adapters import Adapter from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig from .exceptions import ConnectTimeout, ReadTimeout -from .models import Client, Origin, Request, Response +from .models import Request, Response from .streams import BaseReader, BaseWriter OptionalTimeout = typing.Optional[TimeoutConfig] -class HTTP2Connection(Client): +class HTTP2Connection(Adapter): READ_NUM_BYTES = 4096 def __init__( - self, - reader: BaseReader, - writer: BaseWriter, - origin: Origin, - timeout: TimeoutConfig = DEFAULT_TIMEOUT_CONFIG, - on_release: typing.Callable = None, + self, reader: BaseReader, writer: BaseWriter, on_release: typing.Callable = None ): self.reader = reader self.writer = writer - self.origin = origin - self.timeout = timeout self.on_release = on_release self.h2_state = h2.connection.H2Connection() self.events = {} # type: typing.Dict[int, typing.List[h2.events.Event]] self.initialized = False - @property - def is_closed(self) -> bool: - return False + def prepare_request(self, request: Request) -> None: + pass - async def send( - self, - request: Request, - *, - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None - ) -> Response: - if timeout is None: - timeout = self.timeout + async def send(self, request: Request, **options: typing.Any) -> Response: + timeout = options.get("timeout") + stream = options.get("stream", False) + assert timeout is None or isinstance(timeout, TimeoutConfig) + #  Start sending the request. if not self.initialized: self.initiate_connection() - - #  Start sending the request. stream_id = await self.send_headers(request, timeout) self.events[stream_id] = [] @@ -77,7 +64,7 @@ async def send( body = self.body_iter(stream_id, timeout) on_close = functools.partial(self.response_closed, stream_id=stream_id) - return Response( + response = Response( status_code=status_code, protocol="HTTP/2", headers=headers, @@ -85,6 +72,17 @@ async def send( on_close=on_close, ) + if not stream: + try: + await response.read() + finally: + await response.close() + + return response + + async def close(self) -> None: + await self.writer.close() + def initiate_connection(self) -> None: self.h2_state.initiate_connection() data_to_send = self.h2_state.data_to_send() @@ -147,5 +145,6 @@ async def response_closed(self, stream_id: int) -> None: if not self.events and self.on_release is not None: await self.on_release() - async def close(self) -> None: - await self.writer.close() + @property + def is_closed(self) -> bool: + return False diff --git a/httpcore/models.py b/httpcore/models.py index e4a809af5b..7229a76c18 100644 --- a/httpcore/models.py +++ b/httpcore/models.py @@ -1,6 +1,5 @@ import http import typing -from types import TracebackType from urllib.parse import urlsplit from .config import SSLConfig, TimeoutConfig @@ -237,47 +236,6 @@ async def close(self) -> None: if self.on_close is not None: await self.on_close() - -class Client: - async def request( - self, - method: str, - url: typing.Union[str, URL], - *, - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), - body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None, - stream: bool = False, - ) -> Response: - request = Request(method, url, headers=headers, body=body) - response = await self.send(request, ssl=ssl, timeout=timeout) - if not stream: - try: - await response.read() - finally: - await response.close() - return response - - async def send( - self, - request: Request, - *, - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None, - ) -> Response: - raise NotImplementedError() # pragma: nocover - - async def close(self) -> None: - raise NotImplementedError() # pragma: nocover - - async def __aenter__(self) -> "Client": - return self - - async def __aexit__( - self, - exc_type: typing.Type[BaseException] = None, - exc_value: BaseException = None, - traceback: TracebackType = None, - ) -> None: - await self.close() + @property + def is_redirect(self) -> bool: + return self.status_code in (301, 302, 303, 307, 308) diff --git a/httpcore/redirects.py b/httpcore/redirects.py new file mode 100644 index 0000000000..0657ebca5c --- /dev/null +++ b/httpcore/redirects.py @@ -0,0 +1,35 @@ +import typing + +from .adapters import Adapter +from .exceptions import TooManyRedirects +from .models import Request, Response + + +class RedirectAdapter(Adapter): + def __init__(self, dispatch: Adapter, max_redirects: int): + self.dispatch = dispatch + self.max_redirects = max_redirects + + def prepare_request(self, request: Request) -> None: + self.dispatch.prepare_request(request) + + async def send(self, request: Request, **options: typing.Any) -> Response: + allow_redirects = options.pop("allow_redirects", True) + history = [] + + while True: + response = await self.dispatch.send(request, **options) + if not allow_redirects or not response.is_redirect: + break + history.append(response) + if len(history) > self.max_redirects: + raise TooManyRedirects() + request = self.build_redirect_request(request, response) + + return response + + async def close(self) -> None: + self.dispatch.close() + + def build_redirect_request(self, request: Request, response: Response) -> Request: + raise NotImplementedError() diff --git a/httpcore/streams.py b/httpcore/streams.py index e46ffee485..03bba17ca6 100644 --- a/httpcore/streams.py +++ b/httpcore/streams.py @@ -41,10 +41,7 @@ async def close(self) -> None: class BasePoolSemaphore: - def __init__(self, limits: PoolLimits, timeout: TimeoutConfig): - raise NotImplementedError() # pragma: no cover - - async def acquire(self, timeout: OptionalTimeout = None) -> None: + async def acquire(self) -> None: raise NotImplementedError() # pragma: no cover def release(self) -> None: @@ -100,9 +97,8 @@ async def close(self) -> None: class PoolSemaphore(BasePoolSemaphore): - def __init__(self, limits: PoolLimits, timeout: TimeoutConfig): + def __init__(self, limits: PoolLimits): self.limits = limits - self.timeout = timeout @property def semaphore(self) -> typing.Optional[asyncio.BoundedSemaphore]: @@ -114,15 +110,13 @@ def semaphore(self) -> typing.Optional[asyncio.BoundedSemaphore]: self._semaphore = asyncio.BoundedSemaphore(value=max_connections) return self._semaphore - async def acquire(self, timeout: OptionalTimeout = None) -> None: + async def acquire(self) -> None: if self.semaphore is None: return - if timeout is None: - timeout = self.timeout - + timeout = self.limits.pool_timeout try: - await asyncio.wait_for(self.semaphore.acquire(), timeout.pool_timeout) + await asyncio.wait_for(self.semaphore.acquire(), timeout) except asyncio.TimeoutError: raise PoolTimeout() diff --git a/httpcore/sync.py b/httpcore/sync.py index b1f98f50e1..737d3fcfe2 100644 --- a/httpcore/sync.py +++ b/httpcore/sync.py @@ -2,9 +2,10 @@ import typing from types import TracebackType +from .adapters import Adapter from .config import SSLConfig, TimeoutConfig from .connection_pool import ConnectionPool -from .models import URL, Client, Response +from .models import URL, Response class SyncResponse: @@ -44,8 +45,8 @@ def close(self) -> None: class SyncClient: - def __init__(self, client: Client): - self._client = client + def __init__(self, adapter: Adapter): + self._client = adapter self._loop = asyncio.new_event_loop() def request( @@ -55,20 +56,10 @@ def request( *, headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", - ssl: typing.Optional[SSLConfig] = None, - timeout: typing.Optional[TimeoutConfig] = None, - stream: bool = False, + **options: typing.Any ) -> SyncResponse: response = self._loop.run_until_complete( - self._client.request( - method, - url, - headers=headers, - body=body, - ssl=ssl, - timeout=timeout, - stream=stream, - ) + self._client.request(method, url, headers=headers, body=body, **options) ) return SyncResponse(response, self._loop) diff --git a/tests/test_api.py b/tests/test_api.py index 6b80587dbc..4622849b56 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -5,16 +5,16 @@ @pytest.mark.asyncio async def test_get(server): - async with httpcore.ConnectionPool() as http: - response = await http.request("GET", "http://127.0.0.1:8000/") + async with httpcore.Client() as client: + response = await client.request("GET", "http://127.0.0.1:8000/") assert response.status_code == 200 assert response.body == b"Hello, world!" @pytest.mark.asyncio async def test_post(server): - async with httpcore.ConnectionPool() as http: - response = await http.request( + async with httpcore.Client() as client: + response = await client.request( "POST", "http://127.0.0.1:8000/", body=b"Hello, world!" ) assert response.status_code == 200 @@ -22,8 +22,8 @@ async def test_post(server): @pytest.mark.asyncio async def test_stream_response(server): - async with httpcore.ConnectionPool() as http: - response = await http.request("GET", "http://127.0.0.1:8000/", stream=True) + async with httpcore.Client() as client: + response = await client.request("GET", "http://127.0.0.1:8000/", stream=True) assert response.status_code == 200 assert not hasattr(response, "body") body = await response.read() @@ -36,8 +36,8 @@ async def hello_world(): yield b"Hello, " yield b"world!" - async with httpcore.ConnectionPool() as http: - response = await http.request( + async with httpcore.Client() as client: + response = await client.request( "POST", "http://127.0.0.1:8000/", body=hello_world() ) assert response.status_code == 200 diff --git a/tests/test_config.py b/tests/test_config.py index 74c1267ea1..e4ce64a46c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -13,13 +13,15 @@ def test_timeout_repr(): timeout = httpcore.TimeoutConfig(read_timeout=5.0) assert ( repr(timeout) - == "TimeoutConfig(connect_timeout=None, read_timeout=5.0, write_timeout=None, pool_timeout=None)" + == "TimeoutConfig(connect_timeout=None, read_timeout=5.0, write_timeout=None)" ) def test_limits_repr(): limits = httpcore.PoolLimits(hard_limit=100) - assert repr(limits) == "PoolLimits(soft_limit=None, hard_limit=100)" + assert ( + repr(limits) == "PoolLimits(soft_limit=None, hard_limit=100, pool_timeout=None)" + ) def test_ssl_eq(): diff --git a/tests/test_http2.py b/tests/test_http2.py index f17d0f9822..dc287bce33 100644 --- a/tests/test_http2.py +++ b/tests/test_http2.py @@ -79,11 +79,8 @@ def stream_complete(self, stream_id): @pytest.mark.asyncio async def test_http2_get_request(): server = MockServer() - origin = httpcore.Origin("http://example.org") - async with httpcore.HTTP2Connection( - reader=server, writer=server, origin=origin - ) as client: - response = await client.request("GET", "http://example.org") + async with httpcore.HTTP2Connection(reader=server, writer=server) as conn: + response = await conn.request("GET", "http://example.org") assert response.status_code == 200 assert json.loads(response.body) == {"method": "GET", "path": "/", "body": ""} @@ -91,11 +88,8 @@ async def test_http2_get_request(): @pytest.mark.asyncio async def test_http2_post_request(): server = MockServer() - origin = httpcore.Origin("http://example.org") - async with httpcore.HTTP2Connection( - reader=server, writer=server, origin=origin - ) as client: - response = await client.request("POST", "http://example.org", body=b"") + async with httpcore.HTTP2Connection(reader=server, writer=server) as conn: + response = await conn.request("POST", "http://example.org", body=b"") assert response.status_code == 200 assert json.loads(response.body) == { "method": "POST", @@ -107,13 +101,10 @@ async def test_http2_post_request(): @pytest.mark.asyncio async def test_http2_multiple_requests(): server = MockServer() - origin = httpcore.Origin("http://example.org") - async with httpcore.HTTP2Connection( - reader=server, writer=server, origin=origin - ) as client: - response_1 = await client.request("GET", "http://example.org/1") - response_2 = await client.request("GET", "http://example.org/2") - response_3 = await client.request("GET", "http://example.org/3") + async with httpcore.HTTP2Connection(reader=server, writer=server) as conn: + response_1 = await conn.request("GET", "http://example.org/1") + response_2 = await conn.request("GET", "http://example.org/2") + response_3 = await conn.request("GET", "http://example.org/3") assert response_1.status_code == 200 assert json.loads(response_1.body) == {"method": "GET", "path": "/1", "body": ""} diff --git a/tests/test_timeouts.py b/tests/test_timeouts.py index b1ceef93d4..d91cb79949 100644 --- a/tests/test_timeouts.py +++ b/tests/test_timeouts.py @@ -24,10 +24,9 @@ async def test_connect_timeout(server): @pytest.mark.asyncio async def test_pool_timeout(server): - timeout = httpcore.TimeoutConfig(pool_timeout=0.0001) - limits = httpcore.PoolLimits(hard_limit=1) + limits = httpcore.PoolLimits(hard_limit=1, pool_timeout=0.0001) - async with httpcore.ConnectionPool(timeout=timeout, limits=limits) as http: + async with httpcore.ConnectionPool(limits=limits) as http: response = await http.request("GET", "http://127.0.0.1:8000/", stream=True) with pytest.raises(httpcore.PoolTimeout): From 8a29a0a1ad6b9eafce34f810c61b43bb00440448 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 26 Apr 2019 17:00:47 +0100 Subject: [PATCH 14/23] Rollin' --- httpcore/__init__.py | 2 +- httpcore/adapters.py | 2 +- httpcore/auth.py | 2 +- httpcore/client.py | 6 +- httpcore/cookies.py | 2 +- httpcore/decoders.py | 14 ++-- httpcore/exceptions.py | 5 ++ httpcore/http11.py | 2 +- httpcore/http2.py | 2 +- httpcore/models.py | 149 ++++++++++++++++++++++++++++++++++----- httpcore/redirects.py | 55 ++++++++++++++- httpcore/status_codes.py | 123 ++++++++++++++++++++++++++++++++ httpcore/structures.py | 20 ++++++ httpcore/sync.py | 6 +- httpcore/utils.py | 52 ++++++++++++++ tests/test_requests.py | 57 ++++++++------- 16 files changed, 432 insertions(+), 67 deletions(-) create mode 100644 httpcore/status_codes.py create mode 100644 httpcore/structures.py create mode 100644 httpcore/utils.py diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 30ed38e607..ef8d6a5d44 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -14,7 +14,7 @@ ) from .http2 import HTTP2Connection from .http11 import HTTP11Connection -from .models import URL, Origin, Request, Response +from .models import URL, Headers, Origin, Request, Response from .streams import BaseReader, BaseWriter, Protocol, Reader, Writer, connect from .sync import SyncClient, SyncConnectionPool diff --git a/httpcore/adapters.py b/httpcore/adapters.py index 0c14e89ceb..72d5880f0d 100644 --- a/httpcore/adapters.py +++ b/httpcore/adapters.py @@ -10,7 +10,7 @@ async def request( method: str, url: typing.Union[str, URL], *, - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + headers: typing.List[typing.Tuple[bytes, bytes]] = [], body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", **options: typing.Any, ) -> Response: diff --git a/httpcore/auth.py b/httpcore/auth.py index 949d8a9fb6..38c434a241 100644 --- a/httpcore/auth.py +++ b/httpcore/auth.py @@ -15,4 +15,4 @@ async def send(self, request: Request, **options: typing.Any) -> Response: return await self.dispatch.send(request, **options) async def close(self) -> None: - self.dispatch.close() + await self.dispatch.close() diff --git a/httpcore/client.py b/httpcore/client.py index 022ae7ffe5..2d98ba3078 100644 --- a/httpcore/client.py +++ b/httpcore/client.py @@ -40,7 +40,7 @@ async def request( url: typing.Union[str, URL], *, body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + headers: typing.List[typing.Tuple[bytes, bytes]] = [], stream: bool = False, allow_redirects: bool = True, ssl: typing.Optional[SSLConfig] = None, @@ -61,7 +61,7 @@ async def get( self, url: typing.Union[str, URL], *, - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + headers: typing.List[typing.Tuple[bytes, bytes]] = [], stream: bool = False, ssl: typing.Optional[SSLConfig] = None, timeout: typing.Optional[TimeoutConfig] = None, @@ -75,7 +75,7 @@ async def post( url: typing.Union[str, URL], *, body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + headers: typing.List[typing.Tuple[bytes, bytes]] = [], stream: bool = False, ssl: typing.Optional[SSLConfig] = None, timeout: typing.Optional[TimeoutConfig] = None, diff --git a/httpcore/cookies.py b/httpcore/cookies.py index f6fd2b0381..ed9e97c4f9 100644 --- a/httpcore/cookies.py +++ b/httpcore/cookies.py @@ -15,4 +15,4 @@ async def send(self, request: Request, **options: typing.Any) -> Response: return await self.dispatch.send(request, **options) async def close(self) -> None: - self.dispatch.close() + await self.dispatch.close() diff --git a/httpcore/decoders.py b/httpcore/decoders.py index b56745c49a..e47143d740 100644 --- a/httpcore/decoders.py +++ b/httpcore/decoders.py @@ -110,17 +110,17 @@ def flush(self) -> bytes: SUPPORTED_DECODERS = { - b"identity": IdentityDecoder, - b"deflate": DeflateDecoder, - b"gzip": GZipDecoder, - b"br": BrotliDecoder, + "identity": IdentityDecoder, + "deflate": DeflateDecoder, + "gzip": GZipDecoder, + "br": BrotliDecoder, } if brotli is None: - SUPPORTED_DECODERS.pop(b"br") # pragma: nocover + SUPPORTED_DECODERS.pop("br") # pragma: nocover -ACCEPT_ENCODING = b", ".join( - [key for key in SUPPORTED_DECODERS.keys() if key != b"identity"] +ACCEPT_ENCODING = ", ".join( + [key for key in SUPPORTED_DECODERS.keys() if key != "identity"] ) diff --git a/httpcore/exceptions.py b/httpcore/exceptions.py index 94154b073e..337b74a2e4 100644 --- a/httpcore/exceptions.py +++ b/httpcore/exceptions.py @@ -52,3 +52,8 @@ class ResponseClosed(Exception): Attempted to read or stream response content, but the request has been closed without loading the body. """ + + +class InvalidURL(Exception): + """ + """ diff --git a/httpcore/http11.py b/httpcore/http11.py index 0075b524ce..97e8b6a82a 100644 --- a/httpcore/http11.py +++ b/httpcore/http11.py @@ -51,7 +51,7 @@ async def send(self, request: Request, **options: typing.Any) -> Response: #  Start sending the request. method = request.method.encode() target = request.url.full_path - headers = request.headers + headers = request.headers.raw event = h11.Request(method=method, target=target, headers=headers) await self._send_event(event, timeout) diff --git a/httpcore/http2.py b/httpcore/http2.py index e89029cf94..4044817623 100644 --- a/httpcore/http2.py +++ b/httpcore/http2.py @@ -96,7 +96,7 @@ async def send_headers(self, request: Request, timeout: OptionalTimeout) -> int: (b":authority", request.url.hostname.encode()), (b":scheme", request.url.scheme.encode()), (b":path", request.url.full_path.encode()), - ] + request.headers + ] + request.headers.raw self.h2_state.send_headers(stream_id, headers) data_to_send = self.h2_state.data_to_send() await self.writer.write(data_to_send, timeout) diff --git a/httpcore/models.py b/httpcore/models.py index 7229a76c18..791bbc906f 100644 --- a/httpcore/models.py +++ b/httpcore/models.py @@ -39,6 +39,10 @@ def path(self) -> str: def query(self) -> str: return self.components.query + @property + def fragment(self) -> str: + return self.components.fragment + @property def hostname(self) -> str: return self.components.hostname @@ -87,32 +91,140 @@ def __hash__(self) -> int: return hash((self.is_ssl, self.hostname, self.port)) +class Headers(typing.MutableMapping[str, str]): + """ + A case-insensitive multidict. + """ + + def __init__(self, headers: typing.List[typing.Tuple[bytes, bytes]]) -> None: + self._list = [(k.lower(), v) for k, v in headers] + + @property + def raw(self) -> typing.List[typing.Tuple[bytes, bytes]]: + return self._list + + def keys(self) -> typing.List[str]: # type: ignore + return [key.decode("latin-1") for key, value in self._list] + + def values(self) -> typing.List[str]: # type: ignore + return [value.decode("latin-1") for key, value in self._list] + + def items(self) -> typing.List[typing.Tuple[str, str]]: # type: ignore + return [ + (key.decode("latin-1"), value.decode("latin-1")) + for key, value in self._list + ] + + def get(self, key: str, default: typing.Any = None) -> typing.Any: + try: + return self[key] + except KeyError: + return default + + def getlist(self, key: str) -> typing.List[str]: + get_header_key = key.lower().encode("latin-1") + return [ + item_value.decode("latin-1") + for item_key, item_value in self._list + if item_key == get_header_key + ] + + def __getitem__(self, key: str) -> str: + get_header_key = key.lower().encode("latin-1") + for header_key, header_value in self._list: + if header_key == get_header_key: + return header_value.decode("latin-1") + raise KeyError(key) + + def __setitem__(self, key: str, value: str) -> None: + """ + Set the header `key` to `value`, removing any duplicate entries. + Retains insertion order. + """ + set_key = key.lower().encode("latin-1") + set_value = value.encode("latin-1") + + found_indexes = [] + for idx, (item_key, item_value) in enumerate(self._list): + if item_key == set_key: + found_indexes.append(idx) + + for idx in reversed(found_indexes[1:]): + del self._list[idx] + + if found_indexes: + idx = found_indexes[0] + self._list[idx] = (set_key, set_value) + else: + self._list.append((set_key, set_value)) + + def __delitem__(self, key: str) -> None: + """ + Remove the header `key`. + """ + del_key = key.lower().encode("latin-1") + + pop_indexes = [] + for idx, (item_key, item_value) in enumerate(self._list): + if item_key == del_key: + pop_indexes.append(idx) + + for idx in reversed(pop_indexes): + del self._list[idx] + + def __contains__(self, key: typing.Any) -> bool: + get_header_key = key.lower().encode("latin-1") + for header_key, header_value in self._list: + if header_key == get_header_key: + return True + return False + + def __iter__(self) -> typing.Iterator[typing.Any]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self._list) + + def __eq__(self, other: typing.Any) -> bool: + if not isinstance(other, Headers): + return False + return sorted(self._list) == sorted(other._list) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + as_dict = dict(self.items()) + if len(as_dict) == len(self): + return f"{class_name}({as_dict!r})" + return f"{class_name}(raw={self.raw!r})" + + class Request: def __init__( self, method: str, url: typing.Union[str, URL], *, - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + headers: typing.List[typing.Tuple[bytes, bytes]] = [], body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", ): self.method = method.upper() self.url = URL(url) if isinstance(url, str) else url - self.headers = list(headers) if isinstance(body, bytes): self.is_streaming = False self.body = body else: self.is_streaming = True self.body_aiter = body - self.headers = self._auto_headers() + self.headers + self.headers = self.build_headers(headers) - def _auto_headers(self) -> typing.List[typing.Tuple[bytes, bytes]]: + def build_headers( + self, init_headers: typing.List[typing.Tuple[bytes, bytes]] + ) -> Headers: has_host = False has_content_length = False has_accept_encoding = False - for header, value in self.headers: + for header, value in init_headers: header = header.strip().lower() if header == b"host": has_host = True @@ -121,20 +233,20 @@ def _auto_headers(self) -> typing.List[typing.Tuple[bytes, bytes]]: elif header == b"accept-encoding": has_accept_encoding = True - headers = [] # type: typing.List[typing.Tuple[bytes, bytes]] + auto_headers = [] # type: typing.List[typing.Tuple[bytes, bytes]] if not has_host: - headers.append((b"host", self.url.netloc.encode("ascii"))) + auto_headers.append((b"host", self.url.netloc.encode("ascii"))) if not has_content_length: if self.is_streaming: - headers.append((b"transfer-encoding", b"chunked")) + auto_headers.append((b"transfer-encoding", b"chunked")) elif self.body: content_length = str(len(self.body)).encode() - headers.append((b"content-length", content_length)) + auto_headers.append((b"content-length", content_length)) if not has_accept_encoding: - headers.append((b"accept-encoding", ACCEPT_ENCODING)) + auto_headers.append((b"accept-encoding", ACCEPT_ENCODING.encode())) - return headers + return Headers(auto_headers + init_headers) async def stream(self) -> typing.AsyncIterator[bytes]: if self.is_streaming: @@ -151,7 +263,7 @@ def __init__( *, reason: typing.Optional[str] = None, protocol: typing.Optional[str] = None, - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + headers: typing.List[typing.Tuple[bytes, bytes]] = [], body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", on_close: typing.Callable = None, ): @@ -164,18 +276,17 @@ def __init__( else: self.reason = reason self.protocol = protocol - self.headers = list(headers) + self.headers = Headers(headers) self.on_close = on_close self.is_closed = False self.is_streamed = False decoders = [] # type: typing.List[Decoder] - for header, value in self.headers: - if header.strip().lower() == b"content-encoding": - for part in value.split(b","): - part = part.strip().lower() - decoder_cls = SUPPORTED_DECODERS[part] - decoders.append(decoder_cls()) + value = self.headers.get("content-encoding", "identity") + for part in value.split(","): + part = part.strip().lower() + decoder_cls = SUPPORTED_DECODERS[part] + decoders.append(decoder_cls()) if len(decoders) == 0: self.decoder = IdentityDecoder() # type: Decoder diff --git a/httpcore/redirects.py b/httpcore/redirects.py index 0657ebca5c..557b7468a6 100644 --- a/httpcore/redirects.py +++ b/httpcore/redirects.py @@ -1,8 +1,11 @@ import typing +from urllib.parse import urljoin, urlparse from .adapters import Adapter from .exceptions import TooManyRedirects -from .models import Request, Response +from .models import URL, Request, Response +from .status_codes import codes +from .utils import requote_uri class RedirectAdapter(Adapter): @@ -29,7 +32,55 @@ async def send(self, request: Request, **options: typing.Any) -> Response: return response async def close(self) -> None: - self.dispatch.close() + await self.dispatch.close() def build_redirect_request(self, request: Request, response: Response) -> Request: + method = self.redirect_method(request, response) + url = self.redirect_url(request, response) raise NotImplementedError() + + def redirect_method(self, request: Request, response: Response) -> str: + """ + When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + """ + method = request.method + + # https://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes["see_other"] and method != "HEAD": + method = "GET" + + # Do what the browsers do, despite standards... + # First, turn 302s into GETs. + if response.status_code == codes["found"] and method != "HEAD": + method = "GET" + + # Second, if a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in Issue 1704. + if response.status_code == codes["moved"] and method == "POST": + method = "GET" + + return method + + def redirect_url(self, request: Request, response: Response) -> URL: + location = response.headers["Location"] + + # Handle redirection without scheme (see: RFC 1808 Section 4) + if location.startswith("//"): + location = f"{request.url.scheme}:{location}" + + # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) + parsed = urlparse(location) + if parsed.fragment == "" and request.url.fragment: + parsed = parsed._replace(fragment=request.url.fragment) + url = parsed.geturl() + + # Facilitate relative 'location' headers, as allowed by RFC 7231. + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + # Compliant with RFC3986, we percent encode the url. + if not parsed.netloc: + url = urljoin(str(request.url), requote_uri(url)) + else: + url = requote_uri(url) + + return URL(url) diff --git a/httpcore/status_codes.py b/httpcore/status_codes.py new file mode 100644 index 0000000000..9265a71ba1 --- /dev/null +++ b/httpcore/status_codes.py @@ -0,0 +1,123 @@ +""" +The ``codes`` object defines a mapping from common names for HTTP statuses +to their numerical codes, accessible either as attributes or as dictionary +items. +>>> requests.codes['temporary_redirect'] +307 +>>> requests.codes.teapot +418 +Some codes have multiple names, and both upper- and lower-case versions of +the names are allowed. For example, ``codes.ok``, ``codes.OK``, and +``codes.okay`` all correspond to the HTTP status code 200. +""" + +import typing + +from .structures import LookupDict + +_codes = { + # Informational. + 100: ("continue",), + 101: ("switching_protocols",), + 102: ("processing",), + 103: ("checkpoint",), + 122: ("uri_too_long", "request_uri_too_long"), + 200: ("ok", "okay", "all_ok", "all_okay", "all_good", "\\o/", "✓"), + 201: ("created",), + 202: ("accepted",), + 203: ("non_authoritative_info", "non_authoritative_information"), + 204: ("no_content",), + 205: ("reset_content", "reset"), + 206: ("partial_content", "partial"), + 207: ("multi_status", "multiple_status", "multi_stati", "multiple_stati"), + 208: ("already_reported",), + 226: ("im_used",), + # Redirection. + 300: ("multiple_choices",), + 301: ("moved_permanently", "moved", "\\o-"), + 302: ("found",), + 303: ("see_other", "other"), + 304: ("not_modified",), + 305: ("use_proxy",), + 306: ("switch_proxy",), + 307: ("temporary_redirect", "temporary_moved", "temporary"), + 308: ( + "permanent_redirect", + "resume_incomplete", + "resume", + ), # These 2 to be removed in 3.0 + # Client Error. + 400: ("bad_request", "bad"), + 401: ("unauthorized",), + 402: ("payment_required", "payment"), + 403: ("forbidden",), + 404: ("not_found", "-o-"), + 405: ("method_not_allowed", "not_allowed"), + 406: ("not_acceptable",), + 407: ("proxy_authentication_required", "proxy_auth", "proxy_authentication"), + 408: ("request_timeout", "timeout"), + 409: ("conflict",), + 410: ("gone",), + 411: ("length_required",), + 412: ("precondition_failed", "precondition"), + 413: ("request_entity_too_large",), + 414: ("request_uri_too_large",), + 415: ("unsupported_media_type", "unsupported_media", "media_type"), + 416: ( + "requested_range_not_satisfiable", + "requested_range", + "range_not_satisfiable", + ), + 417: ("expectation_failed",), + 418: ("im_a_teapot", "teapot", "i_am_a_teapot"), + 421: ("misdirected_request",), + 422: ("unprocessable_entity", "unprocessable"), + 423: ("locked",), + 424: ("failed_dependency", "dependency"), + 425: ("unordered_collection", "unordered"), + 426: ("upgrade_required", "upgrade"), + 428: ("precondition_required", "precondition"), + 429: ("too_many_requests", "too_many"), + 431: ("header_fields_too_large", "fields_too_large"), + 444: ("no_response", "none"), + 449: ("retry_with", "retry"), + 450: ("blocked_by_windows_parental_controls", "parental_controls"), + 451: ("unavailable_for_legal_reasons", "legal_reasons"), + 499: ("client_closed_request",), + # Server Error. + 500: ("internal_server_error", "server_error", "/o\\", "✗"), + 501: ("not_implemented",), + 502: ("bad_gateway",), + 503: ("service_unavailable", "unavailable"), + 504: ("gateway_timeout",), + 505: ("http_version_not_supported", "http_version"), + 506: ("variant_also_negotiates",), + 507: ("insufficient_storage",), + 509: ("bandwidth_limit_exceeded", "bandwidth"), + 510: ("not_extended",), + 511: ("network_authentication_required", "network_auth", "network_authentication"), +} # type: typing.Dict[int, typing.Sequence[str]] + +codes = LookupDict(name="status_codes") + + +def _init() -> None: + for code, titles in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith(("\\", "/")): + setattr(codes, title.upper(), code) + + def doc(code: int) -> str: + names = ", ".join("``%s``" % n for n in _codes[code]) + return "* %d: %s" % (code, names) + + global __doc__ + __doc__ = ( + __doc__ + "\n" + "\n".join(doc(code) for code in sorted(_codes)) + if __doc__ is not None + else None + ) + + +_init() diff --git a/httpcore/structures.py b/httpcore/structures.py new file mode 100644 index 0000000000..127af5c649 --- /dev/null +++ b/httpcore/structures.py @@ -0,0 +1,20 @@ +import typing + + +class LookupDict(dict): + """Dictionary lookup object.""" + + def __init__(self, name: str = None) -> None: + self.name = name + super(LookupDict, self).__init__() + + def __repr__(self) -> str: + return "" % (self.name) + + def __getitem__(self, key: typing.Any) -> typing.Any: + # We allow fall-through here, so values default to None + + return self.__dict__.get(key, None) + + def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: + return self.__dict__.get(key, default) diff --git a/httpcore/sync.py b/httpcore/sync.py index 737d3fcfe2..89b586287a 100644 --- a/httpcore/sync.py +++ b/httpcore/sync.py @@ -5,7 +5,7 @@ from .adapters import Adapter from .config import SSLConfig, TimeoutConfig from .connection_pool import ConnectionPool -from .models import URL, Response +from .models import URL, Headers, Response class SyncResponse: @@ -22,7 +22,7 @@ def reason(self) -> str: return self._response.reason @property - def headers(self) -> typing.List[typing.Tuple[bytes, bytes]]: + def headers(self) -> Headers: return self._response.headers @property @@ -54,7 +54,7 @@ def request( method: str, url: typing.Union[str, URL], *, - headers: typing.Sequence[typing.Tuple[bytes, bytes]] = (), + headers: typing.List[typing.Tuple[bytes, bytes]] = [], body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", **options: typing.Any ) -> SyncResponse: diff --git a/httpcore/utils.py b/httpcore/utils.py new file mode 100644 index 0000000000..dc7f1d282c --- /dev/null +++ b/httpcore/utils.py @@ -0,0 +1,52 @@ +from urllib.parse import quote + +from .exceptions import InvalidURL + +# The unreserved URI characters (RFC 3986) +UNRESERVED_SET = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~" +) + + +def unquote_unreserved(uri: str) -> str: + """Un-escape any percent-escape sequences in a URI that are unreserved + characters. This leaves all reserved, illegal and non-ASCII bytes encoded. + :rtype: str + """ + parts = uri.split("%") + for i in range(1, len(parts)): + h = parts[i][0:2] + if len(h) == 2 and h.isalnum(): + try: + c = chr(int(h, 16)) + except ValueError: + raise InvalidURL("Invalid percent-escape sequence: '%s'" % h) + + if c in UNRESERVED_SET: + parts[i] = c + parts[i][2:] + else: + parts[i] = "%" + parts[i] + else: + parts[i] = "%" + parts[i] + return "".join(parts) + + +def requote_uri(uri: str) -> str: + """ + Re-quote the given URI. + + This function passes the given URI through an unquote/quote cycle to + ensure that it is fully and consistently quoted. + """ + safe_with_percent = "!#$%&'()*+,/:;=?@[]~" + safe_without_percent = "!#$&'()*+,/:;=?@[]~" + try: + # Unquote only the unreserved characters + # Then quote only illegal characters (do not quote reserved, + # unreserved, or '%') + return quote(unquote_unreserved(uri), safe=safe_with_percent) + except InvalidURL: + # We couldn't unquote the given URI, so let's try quoting it, but + # there may be unquoted '%'s in the URI. We need to make sure they're + # properly quoted so they do not cause issues elsewhere. + return quote(uri, safe=safe_without_percent) diff --git a/tests/test_requests.py b/tests/test_requests.py index c88b70a037..840dde1a8a 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -5,19 +5,20 @@ def test_host_header(): request = httpcore.Request("GET", "http://example.org") - assert request.headers == [ - (b"host", b"example.org"), - (b"accept-encoding", b"deflate, gzip, br"), - ] + assert request.headers == httpcore.Headers( + [(b"host", b"example.org"), (b"accept-encoding", b"deflate, gzip, br")] + ) def test_content_length_header(): request = httpcore.Request("POST", "http://example.org", body=b"test 123") - assert request.headers == [ - (b"host", b"example.org"), - (b"content-length", b"8"), - (b"accept-encoding", b"deflate, gzip, br"), - ] + assert request.headers == httpcore.Headers( + [ + (b"host", b"example.org"), + (b"content-length", b"8"), + (b"accept-encoding", b"deflate, gzip, br"), + ] + ) def test_transfer_encoding_header(): @@ -27,31 +28,31 @@ async def streaming_body(data): body = streaming_body(b"test 123") request = httpcore.Request("POST", "http://example.org", body=body) - assert request.headers == [ - (b"host", b"example.org"), - (b"transfer-encoding", b"chunked"), - (b"accept-encoding", b"deflate, gzip, br"), - ] + assert request.headers == httpcore.Headers( + [ + (b"host", b"example.org"), + (b"transfer-encoding", b"chunked"), + (b"accept-encoding", b"deflate, gzip, br"), + ] + ) def test_override_host_header(): headers = [(b"host", b"1.2.3.4:80")] request = httpcore.Request("GET", "http://example.org", headers=headers) - assert request.headers == [ - (b"accept-encoding", b"deflate, gzip, br"), - (b"host", b"1.2.3.4:80"), - ] + assert request.headers == httpcore.Headers( + [(b"accept-encoding", b"deflate, gzip, br"), (b"host", b"1.2.3.4:80")] + ) def test_override_accept_encoding_header(): headers = [(b"accept-encoding", b"identity")] request = httpcore.Request("GET", "http://example.org", headers=headers) - assert request.headers == [ - (b"host", b"example.org"), - (b"accept-encoding", b"identity"), - ] + assert request.headers == httpcore.Headers( + [(b"host", b"example.org"), (b"accept-encoding", b"identity")] + ) def test_override_content_length_header(): @@ -62,11 +63,13 @@ async def streaming_body(data): headers = [(b"content-length", b"8")] request = httpcore.Request("POST", "http://example.org", body=body, headers=headers) - assert request.headers == [ - (b"host", b"example.org"), - (b"accept-encoding", b"deflate, gzip, br"), - (b"content-length", b"8"), - ] + assert request.headers == httpcore.Headers( + [ + (b"host", b"example.org"), + (b"accept-encoding", b"deflate, gzip, br"), + (b"content-length", b"8"), + ] + ) def test_url(): From fe43c08adccfe309135aaf6fdbce82db9587d5c2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 26 Apr 2019 17:08:38 +0100 Subject: [PATCH 15/23] Add API sketch --- API.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 API.md diff --git a/API.md b/API.md new file mode 100644 index 0000000000..56f34fe562 --- /dev/null +++ b/API.md @@ -0,0 +1,90 @@ +Client(...) + + .request(method, url, ...) + + .get(url, ...) + .options(url, ...) + .head(url, ...) + .post(url, ...) + .put(url, ...) + .patch(url, ...) + .delete(url, ...) + + .prepare_request(request) + .send(request, ...) + .close() + + +Adapter() + + .prepare_request(request) + .send(request) + .close() + + ++ EnvironmentAdapter ++ RedirectAdapter ++ CookieAdapter ++ AuthAdapter ++ ConnectionPool + + HTTPConnection + + HTTP11Connection + + HTTP2Connection + + + +Response(...) + .status_code - int + .reason_phrase - str + .protocol - "HTTP/2" or "HTTP/1.1" + .url - URL + .headers - Headers + + .content - bytes + .text - str + .encoding - str + .json() - Any + + .read() - bytes + .stream() - bytes iterator + .raw() - bytes iterator + .close() - None + + .is_redirect - bool + .request - Request + .cookies - Cookies + .history - List[Response] + + .raise_for_status() + .next() + + +Request(...) + .method + .url + .headers + + ... + + +Headers + +URL + +Origin + +Cookies + + +# Sync + +SyncClient +SyncResponse +SyncRequest +SyncAdapter + + + +SSE +HTTP/2 server push support +Concurrency From e09fc3741f8b5f311a463634afcef0f3cfd97243 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 11:54:48 +0100 Subject: [PATCH 16/23] Restructuring --- httpcore/__init__.py | 12 +- httpcore/adapters/__init__.py | 4 + .../{auth.py => adapters/authentication.py} | 6 +- httpcore/{ => adapters}/cookies.py | 4 +- httpcore/{ => adapters}/environment.py | 4 +- httpcore/{ => adapters}/redirects.py | 27 +-- httpcore/client.py | 12 +- httpcore/config.py | 2 +- httpcore/dispatch/__init__.py | 4 + httpcore/{ => dispatch}/connection.py | 15 +- httpcore/{ => dispatch}/connection_pool.py | 10 +- httpcore/{ => dispatch}/http11.py | 15 +- httpcore/{ => dispatch}/http2.py | 15 +- httpcore/{adapters.py => interfaces.py} | 27 +++ httpcore/models.py | 3 + httpcore/status_codes.py | 181 ++++++------------ httpcore/streams.py | 25 +-- httpcore/structures.py | 20 -- httpcore/sync.py | 4 +- httpcore/utils.py | 4 +- tests/adapters/test_redirects.py | 31 +++ 21 files changed, 202 insertions(+), 223 deletions(-) create mode 100644 httpcore/adapters/__init__.py rename httpcore/{auth.py => adapters/authentication.py} (78%) rename httpcore/{ => adapters}/cookies.py (85%) rename httpcore/{ => adapters}/environment.py (90%) rename httpcore/{ => adapters}/redirects.py (77%) create mode 100644 httpcore/dispatch/__init__.py rename httpcore/{ => dispatch}/connection.py (91%) rename httpcore/{ => dispatch}/connection_pool.py (96%) rename httpcore/{ => dispatch}/http11.py (93%) rename httpcore/{ => dispatch}/http2.py (94%) rename httpcore/{adapters.py => interfaces.py} (60%) delete mode 100644 httpcore/structures.py create mode 100644 tests/adapters/test_redirects.py diff --git a/httpcore/__init__.py b/httpcore/__init__.py index ef8d6a5d44..b81c877924 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -1,8 +1,10 @@ -from .adapters import Adapter +from .adapters.redirects import RedirectAdapter from .client import Client from .config import PoolLimits, SSLConfig, TimeoutConfig -from .connection import HTTPConnection -from .connection_pool import ConnectionPool +from .dispatch.connection import HTTPConnection +from .dispatch.connection_pool import ConnectionPool +from .dispatch.http2 import HTTP2Connection +from .dispatch.http11 import HTTP11Connection from .exceptions import ( ConnectTimeout, PoolTimeout, @@ -12,9 +14,9 @@ StreamConsumed, Timeout, ) -from .http2 import HTTP2Connection -from .http11 import HTTP11Connection +from .interfaces import Adapter from .models import URL, Headers, Origin, Request, Response +from .status_codes import codes from .streams import BaseReader, BaseWriter, Protocol, Reader, Writer, connect from .sync import SyncClient, SyncConnectionPool diff --git a/httpcore/adapters/__init__.py b/httpcore/adapters/__init__.py new file mode 100644 index 0000000000..8d4629f7ad --- /dev/null +++ b/httpcore/adapters/__init__.py @@ -0,0 +1,4 @@ +""" +Adapter classes layer additional behavior over the raw dispatching of the +HTTP request/response. +""" diff --git a/httpcore/auth.py b/httpcore/adapters/authentication.py similarity index 78% rename from httpcore/auth.py rename to httpcore/adapters/authentication.py index 38c434a241..cb5ae99ac1 100644 --- a/httpcore/auth.py +++ b/httpcore/adapters/authentication.py @@ -1,10 +1,10 @@ import typing -from .adapters import Adapter -from .models import Request, Response +from ..interfaces import Adapter +from ..models import Request, Response -class AuthAdapter(Adapter): +class AuthenticationAdapter(Adapter): def __init__(self, dispatch: Adapter): self.dispatch = dispatch diff --git a/httpcore/cookies.py b/httpcore/adapters/cookies.py similarity index 85% rename from httpcore/cookies.py rename to httpcore/adapters/cookies.py index ed9e97c4f9..1105152149 100644 --- a/httpcore/cookies.py +++ b/httpcore/adapters/cookies.py @@ -1,7 +1,7 @@ import typing -from .adapters import Adapter -from .models import Request, Response +from ..interfaces import Adapter +from ..models import Request, Response class CookieAdapter(Adapter): diff --git a/httpcore/environment.py b/httpcore/adapters/environment.py similarity index 90% rename from httpcore/environment.py rename to httpcore/adapters/environment.py index 5065eed884..9840a92678 100644 --- a/httpcore/environment.py +++ b/httpcore/adapters/environment.py @@ -1,7 +1,7 @@ import typing -from .adapters import Adapter -from .models import Request, Response +from ..interfaces import Adapter +from ..models import Request, Response class EnvironmentAdapter(Adapter): diff --git a/httpcore/redirects.py b/httpcore/adapters/redirects.py similarity index 77% rename from httpcore/redirects.py rename to httpcore/adapters/redirects.py index 557b7468a6..0cf3f9fbb7 100644 --- a/httpcore/redirects.py +++ b/httpcore/adapters/redirects.py @@ -1,15 +1,16 @@ import typing from urllib.parse import urljoin, urlparse -from .adapters import Adapter -from .exceptions import TooManyRedirects -from .models import URL, Request, Response -from .status_codes import codes -from .utils import requote_uri +from ..config import DEFAULT_MAX_REDIRECTS +from ..exceptions import TooManyRedirects +from ..interfaces import Adapter +from ..models import URL, Request, Response +from ..status_codes import codes +from ..utils import requote_uri class RedirectAdapter(Adapter): - def __init__(self, dispatch: Adapter, max_redirects: int): + def __init__(self, dispatch: Adapter, max_redirects: int = DEFAULT_MAX_REDIRECTS): self.dispatch = dispatch self.max_redirects = max_redirects @@ -37,7 +38,7 @@ async def close(self) -> None: def build_redirect_request(self, request: Request, response: Response) -> Request: method = self.redirect_method(request, response) url = self.redirect_url(request, response) - raise NotImplementedError() + return Request(method=method, url=url) def redirect_method(self, request: Request, response: Response) -> str: """ @@ -47,17 +48,17 @@ def redirect_method(self, request: Request, response: Response) -> str: method = request.method # https://tools.ietf.org/html/rfc7231#section-6.4.4 - if response.status_code == codes["see_other"] and method != "HEAD": + if response.status_code == codes.see_other and method != "HEAD": method = "GET" # Do what the browsers do, despite standards... - # First, turn 302s into GETs. - if response.status_code == codes["found"] and method != "HEAD": + # Turn 302s into GETs. + if response.status_code == codes.found and method != "HEAD": method = "GET" - # Second, if a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in Issue 1704. - if response.status_code == codes["moved"] and method == "POST": + # If a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in 'requests' issue 1704. + if response.status_code == codes.moved_permanently and method == "POST": method = "GET" return method diff --git a/httpcore/client.py b/httpcore/client.py index 2d98ba3078..139a568679 100644 --- a/httpcore/client.py +++ b/httpcore/client.py @@ -1,7 +1,10 @@ import typing from types import TracebackType -from .auth import AuthAdapter +from .adapters.authentication import AuthenticationAdapter +from .adapters.cookies import CookieAdapter +from .adapters.environment import EnvironmentAdapter +from .adapters.redirects import RedirectAdapter from .config import ( DEFAULT_MAX_REDIRECTS, DEFAULT_POOL_LIMITS, @@ -11,11 +14,8 @@ SSLConfig, TimeoutConfig, ) -from .connection_pool import ConnectionPool -from .cookies import CookieAdapter -from .environment import EnvironmentAdapter +from .dispatch.connection_pool import ConnectionPool from .models import URL, Request, Response -from .redirects import RedirectAdapter class Client: @@ -28,7 +28,7 @@ def __init__( ): connection_pool = ConnectionPool(ssl=ssl, timeout=timeout, limits=limits) cookie_adapter = CookieAdapter(dispatch=connection_pool) - auth_adapter = AuthAdapter(dispatch=cookie_adapter) + auth_adapter = AuthenticationAdapter(dispatch=cookie_adapter) redirect_adapter = RedirectAdapter( dispatch=auth_adapter, max_redirects=max_redirects ) diff --git a/httpcore/config.py b/httpcore/config.py index 5ce24707c3..ef24a8b1a0 100644 --- a/httpcore/config.py +++ b/httpcore/config.py @@ -175,4 +175,4 @@ def __repr__(self) -> str: DEFAULT_TIMEOUT_CONFIG = TimeoutConfig(timeout=5.0) DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100, pool_timeout=5.0) DEFAULT_CA_BUNDLE_PATH = certifi.where() -DEFAULT_MAX_REDIRECTS = 30 +DEFAULT_MAX_REDIRECTS = 20 diff --git a/httpcore/dispatch/__init__.py b/httpcore/dispatch/__init__.py new file mode 100644 index 0000000000..4057d6ead5 --- /dev/null +++ b/httpcore/dispatch/__init__.py @@ -0,0 +1,4 @@ +""" +Dispatch classes handle the raw network connections and the implementation +details of making the HTTP request and receiving the response. +""" diff --git a/httpcore/connection.py b/httpcore/dispatch/connection.py similarity index 91% rename from httpcore/connection.py rename to httpcore/dispatch/connection.py index 1662018a59..45134456b9 100644 --- a/httpcore/connection.py +++ b/httpcore/dispatch/connection.py @@ -4,13 +4,18 @@ import h2.connection import h11 -from .adapters import Adapter -from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .exceptions import ConnectTimeout +from ..config import ( + DEFAULT_SSL_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + SSLConfig, + TimeoutConfig, +) +from ..exceptions import ConnectTimeout +from ..interfaces import Adapter +from ..models import Origin, Request, Response +from ..streams import Protocol, connect from .http2 import HTTP2Connection from .http11 import HTTP11Connection -from .models import Origin, Request, Response -from .streams import Protocol, connect # Callback signature: async def callback(conn: HTTPConnection) -> None ReleaseCallback = typing.Callable[["HTTPConnection"], typing.Awaitable[None]] diff --git a/httpcore/connection_pool.py b/httpcore/dispatch/connection_pool.py similarity index 96% rename from httpcore/connection_pool.py rename to httpcore/dispatch/connection_pool.py index ab394a1963..a536d5f38d 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/dispatch/connection_pool.py @@ -1,8 +1,7 @@ import collections.abc import typing -from .adapters import Adapter -from .config import ( +from ..config import ( DEFAULT_CA_BUNDLE_PATH, DEFAULT_POOL_LIMITS, DEFAULT_SSL_CONFIG, @@ -11,10 +10,11 @@ SSLConfig, TimeoutConfig, ) +from ..exceptions import PoolTimeout +from ..interfaces import Adapter +from ..models import Origin, Request, Response +from ..streams import PoolSemaphore from .connection import HTTPConnection -from .exceptions import PoolTimeout -from .models import Origin, Request, Response -from .streams import PoolSemaphore CONNECTIONS_DICT = typing.Dict[Origin, typing.List[HTTPConnection]] diff --git a/httpcore/http11.py b/httpcore/dispatch/http11.py similarity index 93% rename from httpcore/http11.py rename to httpcore/dispatch/http11.py index 97e8b6a82a..e4a5c83a56 100644 --- a/httpcore/http11.py +++ b/httpcore/dispatch/http11.py @@ -2,11 +2,16 @@ import h11 -from .adapters import Adapter -from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .exceptions import ConnectTimeout, ReadTimeout -from .models import Request, Response -from .streams import BaseReader, BaseWriter +from ..config import ( + DEFAULT_SSL_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + SSLConfig, + TimeoutConfig, +) +from ..exceptions import ConnectTimeout, ReadTimeout +from ..interfaces import Adapter +from ..models import Request, Response +from ..streams import BaseReader, BaseWriter H11Event = typing.Union[ h11.Request, diff --git a/httpcore/http2.py b/httpcore/dispatch/http2.py similarity index 94% rename from httpcore/http2.py rename to httpcore/dispatch/http2.py index 4044817623..7bd124b768 100644 --- a/httpcore/http2.py +++ b/httpcore/dispatch/http2.py @@ -4,11 +4,16 @@ import h2.connection import h2.events -from .adapters import Adapter -from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .exceptions import ConnectTimeout, ReadTimeout -from .models import Request, Response -from .streams import BaseReader, BaseWriter +from ..config import ( + DEFAULT_SSL_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + SSLConfig, + TimeoutConfig, +) +from ..exceptions import ConnectTimeout, ReadTimeout +from ..interfaces import Adapter +from ..models import Request, Response +from ..streams import BaseReader, BaseWriter OptionalTimeout = typing.Optional[TimeoutConfig] diff --git a/httpcore/adapters.py b/httpcore/interfaces.py similarity index 60% rename from httpcore/adapters.py rename to httpcore/interfaces.py index 72d5880f0d..d3cdca47dc 100644 --- a/httpcore/adapters.py +++ b/httpcore/interfaces.py @@ -1,8 +1,11 @@ import typing from types import TracebackType +from .config import TimeoutConfig from .models import URL, Request, Response +OptionalTimeout = typing.Optional[TimeoutConfig] + class Adapter: async def request( @@ -38,3 +41,27 @@ async def __aexit__( traceback: TracebackType = None, ) -> None: await self.close() + + +class BaseReader: + async def read(self, n: int, timeout: OptionalTimeout = None) -> bytes: + raise NotImplementedError() # pragma: no cover + + +class BaseWriter: + def write_no_block(self, data: bytes) -> None: + raise NotImplementedError() # pragma: no cover + + async def write(self, data: bytes, timeout: OptionalTimeout = None) -> None: + raise NotImplementedError() # pragma: no cover + + async def close(self) -> None: + raise NotImplementedError() # pragma: no cover + + +class BasePoolSemaphore: + async def acquire(self) -> None: + raise NotImplementedError() # pragma: no cover + + def release(self) -> None: + raise NotImplementedError() # pragma: no cover diff --git a/httpcore/models.py b/httpcore/models.py index 791bbc906f..b08da934a3 100644 --- a/httpcore/models.py +++ b/httpcore/models.py @@ -70,6 +70,9 @@ def is_secure(self) -> bool: def origin(self) -> "Origin": return Origin(self) + def __str__(self) -> str: + return self.components.geturl() + class Origin: def __init__(self, url: typing.Union[str, URL]) -> None: diff --git a/httpcore/status_codes.py b/httpcore/status_codes.py index 9265a71ba1..19b849f045 100644 --- a/httpcore/status_codes.py +++ b/httpcore/status_codes.py @@ -1,123 +1,58 @@ -""" -The ``codes`` object defines a mapping from common names for HTTP statuses -to their numerical codes, accessible either as attributes or as dictionary -items. ->>> requests.codes['temporary_redirect'] -307 ->>> requests.codes.teapot -418 -Some codes have multiple names, and both upper- and lower-case versions of -the names are allowed. For example, ``codes.ok``, ``codes.OK``, and -``codes.okay`` all correspond to the HTTP status code 200. -""" - -import typing - -from .structures import LookupDict - -_codes = { - # Informational. - 100: ("continue",), - 101: ("switching_protocols",), - 102: ("processing",), - 103: ("checkpoint",), - 122: ("uri_too_long", "request_uri_too_long"), - 200: ("ok", "okay", "all_ok", "all_okay", "all_good", "\\o/", "✓"), - 201: ("created",), - 202: ("accepted",), - 203: ("non_authoritative_info", "non_authoritative_information"), - 204: ("no_content",), - 205: ("reset_content", "reset"), - 206: ("partial_content", "partial"), - 207: ("multi_status", "multiple_status", "multi_stati", "multiple_stati"), - 208: ("already_reported",), - 226: ("im_used",), - # Redirection. - 300: ("multiple_choices",), - 301: ("moved_permanently", "moved", "\\o-"), - 302: ("found",), - 303: ("see_other", "other"), - 304: ("not_modified",), - 305: ("use_proxy",), - 306: ("switch_proxy",), - 307: ("temporary_redirect", "temporary_moved", "temporary"), - 308: ( - "permanent_redirect", - "resume_incomplete", - "resume", - ), # These 2 to be removed in 3.0 - # Client Error. - 400: ("bad_request", "bad"), - 401: ("unauthorized",), - 402: ("payment_required", "payment"), - 403: ("forbidden",), - 404: ("not_found", "-o-"), - 405: ("method_not_allowed", "not_allowed"), - 406: ("not_acceptable",), - 407: ("proxy_authentication_required", "proxy_auth", "proxy_authentication"), - 408: ("request_timeout", "timeout"), - 409: ("conflict",), - 410: ("gone",), - 411: ("length_required",), - 412: ("precondition_failed", "precondition"), - 413: ("request_entity_too_large",), - 414: ("request_uri_too_large",), - 415: ("unsupported_media_type", "unsupported_media", "media_type"), - 416: ( - "requested_range_not_satisfiable", - "requested_range", - "range_not_satisfiable", - ), - 417: ("expectation_failed",), - 418: ("im_a_teapot", "teapot", "i_am_a_teapot"), - 421: ("misdirected_request",), - 422: ("unprocessable_entity", "unprocessable"), - 423: ("locked",), - 424: ("failed_dependency", "dependency"), - 425: ("unordered_collection", "unordered"), - 426: ("upgrade_required", "upgrade"), - 428: ("precondition_required", "precondition"), - 429: ("too_many_requests", "too_many"), - 431: ("header_fields_too_large", "fields_too_large"), - 444: ("no_response", "none"), - 449: ("retry_with", "retry"), - 450: ("blocked_by_windows_parental_controls", "parental_controls"), - 451: ("unavailable_for_legal_reasons", "legal_reasons"), - 499: ("client_closed_request",), - # Server Error. - 500: ("internal_server_error", "server_error", "/o\\", "✗"), - 501: ("not_implemented",), - 502: ("bad_gateway",), - 503: ("service_unavailable", "unavailable"), - 504: ("gateway_timeout",), - 505: ("http_version_not_supported", "http_version"), - 506: ("variant_also_negotiates",), - 507: ("insufficient_storage",), - 509: ("bandwidth_limit_exceeded", "bandwidth"), - 510: ("not_extended",), - 511: ("network_authentication_required", "network_auth", "network_authentication"), -} # type: typing.Dict[int, typing.Sequence[str]] - -codes = LookupDict(name="status_codes") - - -def _init() -> None: - for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(("\\", "/")): - setattr(codes, title.upper(), code) - - def doc(code: int) -> str: - names = ", ".join("``%s``" % n for n in _codes[code]) - return "* %d: %s" % (code, names) - - global __doc__ - __doc__ = ( - __doc__ + "\n" + "\n".join(doc(code) for code in sorted(_codes)) - if __doc__ is not None - else None - ) - - -_init() +import enum + +codes = enum.IntEnum( + "StatusCode", + [ + ("continue", 100), + ("switching_protocols", 101), + ("ok", 200), + ("created", 201), + ("accepted", 202), + ("non_authoritative_information", 203), + ("no_content", 204), + ("reset_content", 205), + ("partial_content", 206), + ("multi_status", 207), + ("multiple_choices", 300), + ("moved_permanently", 301), + ("found", 302), + ("see_other", 303), + ("not_modified", 304), + ("use_proxy", 305), + ("reserved", 306), + ("temporary_redirect", 307), + ("bad_request", 400), + ("unauthorized", 401), + ("payment_required", 402), + ("forbidden", 403), + ("not_found", 404), + ("method_not_allowed", 405), + ("not_acceptable", 406), + ("proxy_authentication_required", 407), + ("request_timeout", 408), + ("conflict", 409), + ("gone", 410), + ("length_required", 411), + ("precondition_failed", 412), + ("request_entity_too_large", 413), + ("request_uri_too_long", 414), + ("unsupported_media_type", 415), + ("requested_range_not_satisfiable", 416), + ("expectation_failed", 417), + ("unprocessable_entity", 422), + ("locked", 423), + ("failed_dependency", 424), + ("precondition_required", 428), + ("too_many_requests", 429), + ("request_header_fields_too_large", 431), + ("unavailable_for_legal_reasons", 451), + ("internal_server_error", 500), + ("not_implemented", 501), + ("bad_gateway", 502), + ("service_unavailable", 503), + ("gateway_timeout", 504), + ("http_version_not_supported", 505), + ("insufficient_storage", 507), + ("network_authentication_required", 511), + ], +) diff --git a/httpcore/streams.py b/httpcore/streams.py index 03bba17ca6..a8590db420 100644 --- a/httpcore/streams.py +++ b/httpcore/streams.py @@ -15,6 +15,7 @@ from .config import DEFAULT_TIMEOUT_CONFIG, PoolLimits, TimeoutConfig from .exceptions import ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout +from .interfaces import BasePoolSemaphore, BaseReader, BaseWriter OptionalTimeout = typing.Optional[TimeoutConfig] @@ -24,30 +25,6 @@ class Protocol(enum.Enum): HTTP_2 = 2 -class BaseReader: - async def read(self, n: int, timeout: OptionalTimeout = None) -> bytes: - raise NotImplementedError() # pragma: no cover - - -class BaseWriter: - def write_no_block(self, data: bytes) -> None: - raise NotImplementedError() # pragma: no cover - - async def write(self, data: bytes, timeout: OptionalTimeout = None) -> None: - raise NotImplementedError() # pragma: no cover - - async def close(self) -> None: - raise NotImplementedError() # pragma: no cover - - -class BasePoolSemaphore: - async def acquire(self) -> None: - raise NotImplementedError() # pragma: no cover - - def release(self) -> None: - raise NotImplementedError() # pragma: no cover - - class Reader(BaseReader): def __init__( self, stream_reader: asyncio.StreamReader, timeout: TimeoutConfig diff --git a/httpcore/structures.py b/httpcore/structures.py deleted file mode 100644 index 127af5c649..0000000000 --- a/httpcore/structures.py +++ /dev/null @@ -1,20 +0,0 @@ -import typing - - -class LookupDict(dict): - """Dictionary lookup object.""" - - def __init__(self, name: str = None) -> None: - self.name = name - super(LookupDict, self).__init__() - - def __repr__(self) -> str: - return "" % (self.name) - - def __getitem__(self, key: typing.Any) -> typing.Any: - # We allow fall-through here, so values default to None - - return self.__dict__.get(key, None) - - def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: - return self.__dict__.get(key, default) diff --git a/httpcore/sync.py b/httpcore/sync.py index 89b586287a..2d58f9a187 100644 --- a/httpcore/sync.py +++ b/httpcore/sync.py @@ -2,9 +2,9 @@ import typing from types import TracebackType -from .adapters import Adapter from .config import SSLConfig, TimeoutConfig -from .connection_pool import ConnectionPool +from .dispatch.connection_pool import ConnectionPool +from .interfaces import Adapter from .models import URL, Headers, Response diff --git a/httpcore/utils.py b/httpcore/utils.py index dc7f1d282c..cd11858a8a 100644 --- a/httpcore/utils.py +++ b/httpcore/utils.py @@ -9,9 +9,9 @@ def unquote_unreserved(uri: str) -> str: - """Un-escape any percent-escape sequences in a URI that are unreserved + """ + Un-escape any percent-escape sequences in a URI that are unreserved characters. This leaves all reserved, illegal and non-ASCII bytes encoded. - :rtype: str """ parts = uri.split("%") for i in range(1, len(parts)): diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py new file mode 100644 index 0000000000..f4a98105a6 --- /dev/null +++ b/tests/adapters/test_redirects.py @@ -0,0 +1,31 @@ +import pytest + +from httpcore import Adapter, RedirectAdapter, Request, Response, codes + + +class MockDispatch(Adapter): + def prepare_request(self, request: Request) -> None: + pass + + async def send(self, request: Request, **options) -> Response: + if request.url.path == "/redirect_303": + return Response( + codes.see_other, headers=[(b"location", b"https://example.org/")] + ) + elif request.url.path == "/relative_redirect": + return Response(codes.see_other, headers=[(b"location", b"/")]) + return Response(codes.ok, body=b"Hello, world!") + + +@pytest.mark.asyncio +async def test_redirect_303(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("GET", "https://example.org/redirect_303") + assert response.status_code == codes.ok + + +@pytest.mark.asyncio +async def test_relative_redirect(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("GET", "https://example.org/relative_redirect") + assert response.status_code == codes.ok From 8b4ef87a892c0b8aff61ede0445ec3ca999fe26e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 12:12:08 +0100 Subject: [PATCH 17/23] Redirect tests --- httpcore/__init__.py | 1 + tests/adapters/test_redirects.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/httpcore/__init__.py b/httpcore/__init__.py index b81c877924..d9c1364c7f 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -13,6 +13,7 @@ ResponseClosed, StreamConsumed, Timeout, + TooManyRedirects ) from .interfaces import Adapter from .models import URL, Headers, Origin, Request, Response diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py index f4a98105a6..dbcb233857 100644 --- a/tests/adapters/test_redirects.py +++ b/tests/adapters/test_redirects.py @@ -1,6 +1,7 @@ import pytest +from urllib.parse import parse_qs -from httpcore import Adapter, RedirectAdapter, Request, Response, codes +from httpcore import Adapter, RedirectAdapter, Request, Response, TooManyRedirects, codes class MockDispatch(Adapter): @@ -14,6 +15,13 @@ async def send(self, request: Request, **options) -> Response: ) elif request.url.path == "/relative_redirect": return Response(codes.see_other, headers=[(b"location", b"/")]) + elif request.url.path == "/multiple_redirects": + params = parse_qs(request.url.query) + count = int(params.get("count", "0")[0]) + code = codes.see_other if count else codes.ok + location = "/multiple_redirects?count=" + str(count - 1) + headers = [(b"location", location.encode())] if count else [] + return Response(code, headers=headers) return Response(codes.ok, body=b"Hello, world!") @@ -29,3 +37,17 @@ async def test_relative_redirect(): client = RedirectAdapter(MockDispatch()) response = await client.request("GET", "https://example.org/relative_redirect") assert response.status_code == codes.ok + + +@pytest.mark.asyncio +async def test_multiple_redirects(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("GET", "https://example.org/multiple_redirects?count=20") + assert response.status_code == codes.ok + + +@pytest.mark.asyncio +async def test_too_many_redirects(): + client = RedirectAdapter(MockDispatch()) + with pytest.raises(TooManyRedirects): + await client.request("GET", "https://example.org/multiple_redirects?count=21") From 041813bc4a7bd45550dd8d558cb2a0893ca93cd2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 12:35:58 +0100 Subject: [PATCH 18/23] Redirections --- httpcore/__init__.py | 3 +- httpcore/adapters/redirects.py | 6 ++- httpcore/exceptions.py | 14 +++++- httpcore/models.py | 6 +++ tests/adapters/test_redirects.py | 73 +++++++++++++++++++++++++++++--- 5 files changed, 92 insertions(+), 10 deletions(-) diff --git a/httpcore/__init__.py b/httpcore/__init__.py index d9c1364c7f..fdd992534f 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -10,10 +10,11 @@ PoolTimeout, ProtocolError, ReadTimeout, + RedirectLoop, ResponseClosed, StreamConsumed, Timeout, - TooManyRedirects + TooManyRedirects, ) from .interfaces import Adapter from .models import URL, Headers, Origin, Request, Response diff --git a/httpcore/adapters/redirects.py b/httpcore/adapters/redirects.py index 0cf3f9fbb7..9ef287f720 100644 --- a/httpcore/adapters/redirects.py +++ b/httpcore/adapters/redirects.py @@ -2,7 +2,7 @@ from urllib.parse import urljoin, urlparse from ..config import DEFAULT_MAX_REDIRECTS -from ..exceptions import TooManyRedirects +from ..exceptions import RedirectLoop, TooManyRedirects from ..interfaces import Adapter from ..models import URL, Request, Response from ..status_codes import codes @@ -20,6 +20,7 @@ def prepare_request(self, request: Request) -> None: async def send(self, request: Request, **options: typing.Any) -> Response: allow_redirects = options.pop("allow_redirects", True) history = [] + seen_urls = set((request.url,)) while True: response = await self.dispatch.send(request, **options) @@ -29,6 +30,9 @@ async def send(self, request: Request, **options: typing.Any) -> Response: if len(history) > self.max_redirects: raise TooManyRedirects() request = self.build_redirect_request(request, response) + if request.url in seen_urls: + raise RedirectLoop() + seen_urls.add(request.url) return response diff --git a/httpcore/exceptions.py b/httpcore/exceptions.py index 337b74a2e4..12aac53b85 100644 --- a/httpcore/exceptions.py +++ b/httpcore/exceptions.py @@ -28,12 +28,24 @@ class PoolTimeout(Timeout): """ -class TooManyRedirects(Exception): +class RedirectError(Exception): + """ + Base class for HTTP redirect errors. + """ + + +class TooManyRedirects(RedirectError): """ Too many redirects. """ +class RedirectLoop(RedirectError): + """ + Infinite redirect loop. + """ + + class ProtocolError(Exception): """ Malformed HTTP. diff --git a/httpcore/models.py b/httpcore/models.py index b08da934a3..b0a4723d6a 100644 --- a/httpcore/models.py +++ b/httpcore/models.py @@ -70,6 +70,12 @@ def is_secure(self) -> bool: def origin(self) -> "Origin": return Origin(self) + def __hash__(self) -> int: + return hash(str(self)) + + def __eq__(self, other: typing.Any) -> bool: + return isinstance(other, URL) and str(self) == str(other) + def __str__(self) -> str: return self.components.geturl() diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py index dbcb233857..3609dd7342 100644 --- a/tests/adapters/test_redirects.py +++ b/tests/adapters/test_redirects.py @@ -1,7 +1,16 @@ -import pytest from urllib.parse import parse_qs -from httpcore import Adapter, RedirectAdapter, Request, Response, TooManyRedirects, codes +import pytest + +from httpcore import ( + Adapter, + RedirectAdapter, + RedirectLoop, + Request, + Response, + TooManyRedirects, + codes, +) class MockDispatch(Adapter): @@ -9,12 +18,21 @@ def prepare_request(self, request: Request) -> None: pass async def send(self, request: Request, **options) -> Response: - if request.url.path == "/redirect_303": - return Response( - codes.see_other, headers=[(b"location", b"https://example.org/")] - ) + if request.url.path == "/redirect_301": # "Moved Permanently" + return Response(301, headers=[(b"location", b"https://example.org/")]) + + elif request.url.path == "/redirect_302": # "Found" + return Response(302, headers=[(b"location", b"https://example.org/")]) + + elif request.url.path == "/redirect_303": # "See Other" + return Response(303, headers=[(b"location", b"https://example.org/")]) + elif request.url.path == "/relative_redirect": return Response(codes.see_other, headers=[(b"location", b"/")]) + + elif request.url.path == "/no_scheme_redirect": + return Response(codes.see_other, headers=[(b"location", b"//example.org/")]) + elif request.url.path == "/multiple_redirects": params = parse_qs(request.url.query) count = int(params.get("count", "0")[0]) @@ -22,9 +40,27 @@ async def send(self, request: Request, **options) -> Response: location = "/multiple_redirects?count=" + str(count - 1) headers = [(b"location", location.encode())] if count else [] return Response(code, headers=headers) + + if request.url.path == "/redirect_loop": + return Response(codes.see_other, headers=[(b"location", b"/redirect_loop")]) + return Response(codes.ok, body=b"Hello, world!") +@pytest.mark.asyncio +async def test_redirect_301(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("POST", "https://example.org/redirect_301") + assert response.status_code == codes.ok + + +@pytest.mark.asyncio +async def test_redirect_302(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("POST", "https://example.org/redirect_302") + assert response.status_code == codes.ok + + @pytest.mark.asyncio async def test_redirect_303(): client = RedirectAdapter(MockDispatch()) @@ -39,10 +75,26 @@ async def test_relative_redirect(): assert response.status_code == codes.ok +@pytest.mark.asyncio +async def test_no_scheme_redirect(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("GET", "https://example.org/no_scheme_redirect") + assert response.status_code == codes.ok + + +@pytest.mark.asyncio +async def test_fragment_redirect(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("GET", "https://example.org/relative_redirect#fragment") + assert response.status_code == codes.ok + + @pytest.mark.asyncio async def test_multiple_redirects(): client = RedirectAdapter(MockDispatch()) - response = await client.request("GET", "https://example.org/multiple_redirects?count=20") + response = await client.request( + "GET", "https://example.org/multiple_redirects?count=20" + ) assert response.status_code == codes.ok @@ -51,3 +103,10 @@ async def test_too_many_redirects(): client = RedirectAdapter(MockDispatch()) with pytest.raises(TooManyRedirects): await client.request("GET", "https://example.org/multiple_redirects?count=21") + + +@pytest.mark.asyncio +async def test_redirect_loop(): + client = RedirectAdapter(MockDispatch()) + with pytest.raises(RedirectLoop): + await client.request("GET", "https://example.org/redirect_loop") From 4f23ab4f0d3fa074d44d055fabaa194412e28661 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 14:25:24 +0100 Subject: [PATCH 19/23] Rejig request preparing --- httpcore/adapters/redirects.py | 14 ++++- httpcore/dispatch/connection.py | 2 +- httpcore/dispatch/connection_pool.py | 3 +- httpcore/dispatch/http11.py | 3 +- httpcore/dispatch/http2.py | 3 +- httpcore/models.py | 74 ++++++++++++++--------- tests/adapters/test_redirects.py | 88 ++++++++++++++++++++++++---- tests/test_requests.py | 6 ++ 8 files changed, 147 insertions(+), 46 deletions(-) diff --git a/httpcore/adapters/redirects.py b/httpcore/adapters/redirects.py index 9ef287f720..aa6c557e57 100644 --- a/httpcore/adapters/redirects.py +++ b/httpcore/adapters/redirects.py @@ -4,7 +4,7 @@ from ..config import DEFAULT_MAX_REDIRECTS from ..exceptions import RedirectLoop, TooManyRedirects from ..interfaces import Adapter -from ..models import URL, Request, Response +from ..models import URL, Headers, Request, Response from ..status_codes import codes from ..utils import requote_uri @@ -19,11 +19,12 @@ def prepare_request(self, request: Request) -> None: async def send(self, request: Request, **options: typing.Any) -> Response: allow_redirects = options.pop("allow_redirects", True) - history = [] + history = [] # type: typing.List[Response] seen_urls = set((request.url,)) while True: response = await self.dispatch.send(request, **options) + response.history = list(history) if not allow_redirects or not response.is_redirect: break history.append(response) @@ -42,7 +43,8 @@ async def close(self) -> None: def build_redirect_request(self, request: Request, response: Response) -> Request: method = self.redirect_method(request, response) url = self.redirect_url(request, response) - return Request(method=method, url=url) + headers = self.redirect_headers(request, url) + return Request(method=method, url=url, headers=headers) def redirect_method(self, request: Request, response: Response) -> str: """ @@ -89,3 +91,9 @@ def redirect_url(self, request: Request, response: Response) -> URL: url = requote_uri(url) return URL(url) + + def redirect_headers(self, request: Request, url: URL) -> Headers: + headers = Headers(request.headers) + if url.origin != request.url.origin: + del headers["Authorization"] + return headers diff --git a/httpcore/dispatch/connection.py b/httpcore/dispatch/connection.py index 45134456b9..ffe82fcb0e 100644 --- a/httpcore/dispatch/connection.py +++ b/httpcore/dispatch/connection.py @@ -37,7 +37,7 @@ def __init__( self.h2_connection = None # type: typing.Optional[HTTP2Connection] def prepare_request(self, request: Request) -> None: - pass + request.prepare() async def send(self, request: Request, **options: typing.Any) -> Response: if self.h11_connection is None and self.h2_connection is None: diff --git a/httpcore/dispatch/connection_pool.py b/httpcore/dispatch/connection_pool.py index a536d5f38d..ca7a837fa5 100644 --- a/httpcore/dispatch/connection_pool.py +++ b/httpcore/dispatch/connection_pool.py @@ -10,6 +10,7 @@ SSLConfig, TimeoutConfig, ) +from ..decoders import ACCEPT_ENCODING from ..exceptions import PoolTimeout from ..interfaces import Adapter from ..models import Origin, Request, Response @@ -104,7 +105,7 @@ def num_connections(self) -> int: return len(self.keepalive_connections) + len(self.active_connections) def prepare_request(self, request: Request) -> None: - pass + request.prepare() async def send(self, request: Request, **options: typing.Any) -> Response: connection = await self.acquire_connection(request.url.origin) diff --git a/httpcore/dispatch/http11.py b/httpcore/dispatch/http11.py index e4a5c83a56..8128dc18fd 100644 --- a/httpcore/dispatch/http11.py +++ b/httpcore/dispatch/http11.py @@ -46,7 +46,7 @@ def __init__( self.h11_state = h11.Connection(our_role=h11.CLIENT) def prepare_request(self, request: Request) -> None: - pass + request.prepare() async def send(self, request: Request, **options: typing.Any) -> Response: timeout = options.get("timeout") @@ -87,6 +87,7 @@ async def send(self, request: Request, **options: typing.Any) -> Response: headers=headers, body=body, on_close=self.response_closed, + request=request, ) if not stream: diff --git a/httpcore/dispatch/http2.py b/httpcore/dispatch/http2.py index 7bd124b768..d44c5f6047 100644 --- a/httpcore/dispatch/http2.py +++ b/httpcore/dispatch/http2.py @@ -32,7 +32,7 @@ def __init__( self.initialized = False def prepare_request(self, request: Request) -> None: - pass + request.prepare() async def send(self, request: Request, **options: typing.Any) -> Response: timeout = options.get("timeout") @@ -75,6 +75,7 @@ async def send(self, request: Request, **options: typing.Any) -> Response: headers=headers, body=body, on_close=on_close, + request=request, ) if not stream: diff --git a/httpcore/models.py b/httpcore/models.py index b0a4723d6a..9235240309 100644 --- a/httpcore/models.py +++ b/httpcore/models.py @@ -79,6 +79,11 @@ def __eq__(self, other: typing.Any) -> bool: def __str__(self) -> str: return self.components.geturl() + def __repr__(self) -> str: + class_name = self.__class__.__name__ + url_str = str(self) + return f"{class_name}({url_str!r})" + class Origin: def __init__(self, url: typing.Union[str, URL]) -> None: @@ -100,13 +105,21 @@ def __hash__(self) -> int: return hash((self.is_ssl, self.hostname, self.port)) +HeaderTypes = typing.Union["Headers", typing.List[typing.Tuple[bytes, bytes]]] + + class Headers(typing.MutableMapping[str, str]): """ A case-insensitive multidict. """ - def __init__(self, headers: typing.List[typing.Tuple[bytes, bytes]]) -> None: - self._list = [(k.lower(), v) for k, v in headers] + def __init__(self, headers: HeaderTypes = None) -> None: + if headers is None: + self._list = [] # type: typing.List[typing.Tuple[bytes, bytes]] + elif isinstance(headers, Headers): + self._list = list(headers.raw) + else: + self._list = [(k.lower(), v) for k, v in headers] @property def raw(self) -> typing.List[typing.Tuple[bytes, bytes]]: @@ -213,7 +226,7 @@ def __init__( method: str, url: typing.Union[str, URL], *, - headers: typing.List[typing.Tuple[bytes, bytes]] = [], + headers: HeaderTypes = None, body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", ): self.method = method.upper() @@ -224,26 +237,24 @@ def __init__( else: self.is_streaming = True self.body_aiter = body - self.headers = self.build_headers(headers) - - def build_headers( - self, init_headers: typing.List[typing.Tuple[bytes, bytes]] - ) -> Headers: - has_host = False - has_content_length = False - has_accept_encoding = False - - for header, value in init_headers: - header = header.strip().lower() - if header == b"host": - has_host = True - elif header in (b"content-length", b"transfer-encoding"): - has_content_length = True - elif header == b"accept-encoding": - has_accept_encoding = True + self.headers = Headers(headers) + + async def stream(self) -> typing.AsyncIterator[bytes]: + if self.is_streaming: + async for part in self.body_aiter: + yield part + elif self.body: + yield self.body + def prepare(self) -> None: auto_headers = [] # type: typing.List[typing.Tuple[bytes, bytes]] + has_host = "host" in self.headers + has_content_length = ( + "content-length" in self.headers or "transfer-encoding" in self.headers + ) + has_accept_encoding = "accept-encoding" in self.headers + if not has_host: auto_headers.append((b"host", self.url.netloc.encode("ascii"))) if not has_content_length: @@ -255,14 +266,8 @@ def build_headers( if not has_accept_encoding: auto_headers.append((b"accept-encoding", ACCEPT_ENCODING.encode())) - return Headers(auto_headers + init_headers) - - async def stream(self) -> typing.AsyncIterator[bytes]: - if self.is_streaming: - async for part in self.body_aiter: - yield part - elif self.body: - yield self.body + for item in reversed(auto_headers): + self.headers.raw.insert(0, item) class Response: @@ -275,6 +280,8 @@ def __init__( headers: typing.List[typing.Tuple[bytes, bytes]] = [], body: typing.Union[bytes, typing.AsyncIterator[bytes]] = b"", on_close: typing.Callable = None, + request: Request = None, + history: typing.List["Response"] = None, ): self.status_code = status_code if not reason: @@ -310,6 +317,13 @@ def __init__( else: self.body_aiter = body + self.request = request + self.history = [] if history is None else list(history) + + @property + def url(self) -> typing.Optional[URL]: + return None if self.request is None else self.request.url + async def read(self) -> bytes: """ Read and return the response content. @@ -358,4 +372,6 @@ async def close(self) -> None: @property def is_redirect(self) -> bool: - return self.status_code in (301, 302, 303, 307, 308) + return ( + self.status_code in (301, 302, 303, 307, 308) and "location" in self.headers + ) diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py index 3609dd7342..ab371a047d 100644 --- a/tests/adapters/test_redirects.py +++ b/tests/adapters/test_redirects.py @@ -1,8 +1,10 @@ +import json from urllib.parse import parse_qs import pytest from httpcore import ( + URL, Adapter, RedirectAdapter, RedirectLoop, @@ -19,32 +21,60 @@ def prepare_request(self, request: Request) -> None: async def send(self, request: Request, **options) -> Response: if request.url.path == "/redirect_301": # "Moved Permanently" - return Response(301, headers=[(b"location", b"https://example.org/")]) + return Response( + 301, headers=[(b"location", b"https://example.org/")], request=request + ) elif request.url.path == "/redirect_302": # "Found" - return Response(302, headers=[(b"location", b"https://example.org/")]) + return Response( + 302, headers=[(b"location", b"https://example.org/")], request=request + ) elif request.url.path == "/redirect_303": # "See Other" - return Response(303, headers=[(b"location", b"https://example.org/")]) + return Response( + 303, headers=[(b"location", b"https://example.org/")], request=request + ) elif request.url.path == "/relative_redirect": - return Response(codes.see_other, headers=[(b"location", b"/")]) + return Response( + codes.see_other, headers=[(b"location", b"/")], request=request + ) elif request.url.path == "/no_scheme_redirect": - return Response(codes.see_other, headers=[(b"location", b"//example.org/")]) + return Response( + codes.see_other, + headers=[(b"location", b"//example.org/")], + request=request, + ) elif request.url.path == "/multiple_redirects": params = parse_qs(request.url.query) count = int(params.get("count", "0")[0]) + redirect_count = count - 1 code = codes.see_other if count else codes.ok - location = "/multiple_redirects?count=" + str(count - 1) + location = "/multiple_redirects" + if redirect_count: + location += "?count=" + str(redirect_count) headers = [(b"location", location.encode())] if count else [] - return Response(code, headers=headers) + return Response(code, headers=headers, request=request) if request.url.path == "/redirect_loop": - return Response(codes.see_other, headers=[(b"location", b"/redirect_loop")]) + return Response( + codes.see_other, + headers=[(b"location", b"/redirect_loop")], + request=request, + ) - return Response(codes.ok, body=b"Hello, world!") + elif request.url.path == "/cross_domain": + location = b"https://example.org/cross_domain_target" + return Response(301, headers=[(b"location", location)], request=request) + + elif request.url.path == "/cross_domain_target": + headers = {k.decode(): v.decode() for k, v in request.headers.raw} + body = json.dumps({"headers": headers}).encode() + return Response(codes.ok, body=body, request=request) + + return Response(codes.ok, body=b"Hello, world!", request=request) @pytest.mark.asyncio @@ -52,6 +82,8 @@ async def test_redirect_301(): client = RedirectAdapter(MockDispatch()) response = await client.request("POST", "https://example.org/redirect_301") assert response.status_code == codes.ok + assert response.url == URL("https://example.org/") + assert len(response.history) == 1 @pytest.mark.asyncio @@ -59,6 +91,8 @@ async def test_redirect_302(): client = RedirectAdapter(MockDispatch()) response = await client.request("POST", "https://example.org/redirect_302") assert response.status_code == codes.ok + assert response.url == URL("https://example.org/") + assert len(response.history) == 1 @pytest.mark.asyncio @@ -66,6 +100,8 @@ async def test_redirect_303(): client = RedirectAdapter(MockDispatch()) response = await client.request("GET", "https://example.org/redirect_303") assert response.status_code == codes.ok + assert response.url == URL("https://example.org/") + assert len(response.history) == 1 @pytest.mark.asyncio @@ -73,6 +109,8 @@ async def test_relative_redirect(): client = RedirectAdapter(MockDispatch()) response = await client.request("GET", "https://example.org/relative_redirect") assert response.status_code == codes.ok + assert response.url == URL("https://example.org/") + assert len(response.history) == 1 @pytest.mark.asyncio @@ -80,13 +118,19 @@ async def test_no_scheme_redirect(): client = RedirectAdapter(MockDispatch()) response = await client.request("GET", "https://example.org/no_scheme_redirect") assert response.status_code == codes.ok + assert response.url == URL("https://example.org/") + assert len(response.history) == 1 @pytest.mark.asyncio async def test_fragment_redirect(): client = RedirectAdapter(MockDispatch()) - response = await client.request("GET", "https://example.org/relative_redirect#fragment") + response = await client.request( + "GET", "https://example.org/relative_redirect#fragment" + ) assert response.status_code == codes.ok + assert response.url == URL("https://example.org/#fragment") + assert len(response.history) == 1 @pytest.mark.asyncio @@ -96,6 +140,8 @@ async def test_multiple_redirects(): "GET", "https://example.org/multiple_redirects?count=20" ) assert response.status_code == codes.ok + assert response.url == URL("https://example.org/multiple_redirects") + assert len(response.history) == 20 @pytest.mark.asyncio @@ -110,3 +156,25 @@ async def test_redirect_loop(): client = RedirectAdapter(MockDispatch()) with pytest.raises(RedirectLoop): await client.request("GET", "https://example.org/redirect_loop") + + +@pytest.mark.asyncio +async def test_cross_domain_redirect(): + client = RedirectAdapter(MockDispatch()) + headers = [(b"Authorization", b"abc")] + url = "https://example.com/cross_domain" + response = await client.request("GET", url, headers=headers) + data = json.loads(response.body.decode()) + assert response.url == URL("https://example.org/cross_domain_target") + assert data == {"headers": {}} + + +@pytest.mark.asyncio +async def test_same_domain_redirect(): + client = RedirectAdapter(MockDispatch()) + headers = [(b"Authorization", b"abc")] + url = "https://example.org/cross_domain" + response = await client.request("GET", url, headers=headers) + data = json.loads(response.body.decode()) + assert response.url == URL("https://example.org/cross_domain_target") + assert data == {"headers": {"authorization": "abc"}} diff --git a/tests/test_requests.py b/tests/test_requests.py index 840dde1a8a..4df3529cd5 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -5,6 +5,7 @@ def test_host_header(): request = httpcore.Request("GET", "http://example.org") + request.prepare() assert request.headers == httpcore.Headers( [(b"host", b"example.org"), (b"accept-encoding", b"deflate, gzip, br")] ) @@ -12,6 +13,7 @@ def test_host_header(): def test_content_length_header(): request = httpcore.Request("POST", "http://example.org", body=b"test 123") + request.prepare() assert request.headers == httpcore.Headers( [ (b"host", b"example.org"), @@ -28,6 +30,7 @@ async def streaming_body(data): body = streaming_body(b"test 123") request = httpcore.Request("POST", "http://example.org", body=body) + request.prepare() assert request.headers == httpcore.Headers( [ (b"host", b"example.org"), @@ -41,6 +44,7 @@ def test_override_host_header(): headers = [(b"host", b"1.2.3.4:80")] request = httpcore.Request("GET", "http://example.org", headers=headers) + request.prepare() assert request.headers == httpcore.Headers( [(b"accept-encoding", b"deflate, gzip, br"), (b"host", b"1.2.3.4:80")] ) @@ -50,6 +54,7 @@ def test_override_accept_encoding_header(): headers = [(b"accept-encoding", b"identity")] request = httpcore.Request("GET", "http://example.org", headers=headers) + request.prepare() assert request.headers == httpcore.Headers( [(b"host", b"example.org"), (b"accept-encoding", b"identity")] ) @@ -63,6 +68,7 @@ async def streaming_body(data): headers = [(b"content-length", b"8")] request = httpcore.Request("POST", "http://example.org", body=body, headers=headers) + request.prepare() assert request.headers == httpcore.Headers( [ (b"host", b"example.org"), From 83d322f7368155abe5129272b74d7a989fe79a85 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 15:25:44 +0100 Subject: [PATCH 20/23] Handle body in redirects --- httpcore/__init__.py | 1 + httpcore/adapters/redirects.py | 12 ++++++++++-- httpcore/exceptions.py | 7 +++++++ httpcore/status_codes.py | 5 ++++- tests/adapters/test_redirects.py | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/httpcore/__init__.py b/httpcore/__init__.py index fdd992534f..813b2b56c0 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -10,6 +10,7 @@ PoolTimeout, ProtocolError, ReadTimeout, + RedirectBodyUnavailable, RedirectLoop, ResponseClosed, StreamConsumed, diff --git a/httpcore/adapters/redirects.py b/httpcore/adapters/redirects.py index aa6c557e57..584efec26e 100644 --- a/httpcore/adapters/redirects.py +++ b/httpcore/adapters/redirects.py @@ -2,7 +2,7 @@ from urllib.parse import urljoin, urlparse from ..config import DEFAULT_MAX_REDIRECTS -from ..exceptions import RedirectLoop, TooManyRedirects +from ..exceptions import RedirectBodyUnavailable, RedirectLoop, TooManyRedirects from ..interfaces import Adapter from ..models import URL, Headers, Request, Response from ..status_codes import codes @@ -44,7 +44,8 @@ def build_redirect_request(self, request: Request, response: Response) -> Reques method = self.redirect_method(request, response) url = self.redirect_url(request, response) headers = self.redirect_headers(request, url) - return Request(method=method, url=url, headers=headers) + body = self.redirect_body(request, method) + return Request(method=method, url=url, headers=headers, body=body) def redirect_method(self, request: Request, response: Response) -> str: """ @@ -97,3 +98,10 @@ def redirect_headers(self, request: Request, url: URL) -> Headers: if url.origin != request.url.origin: del headers["Authorization"] return headers + + def redirect_body(self, request: Request, method: str) -> bytes: + if method != request.method and method == "GET": + return b"" + if request.is_streaming: + raise RedirectBodyUnavailable() + return request.body diff --git a/httpcore/exceptions.py b/httpcore/exceptions.py index 12aac53b85..cf8d3f18ec 100644 --- a/httpcore/exceptions.py +++ b/httpcore/exceptions.py @@ -40,6 +40,13 @@ class TooManyRedirects(RedirectError): """ +class RedirectBodyUnavailable(RedirectError): + """ + Got a redirect response, but the request body was streaming, and is + no longer available. + """ + + class RedirectLoop(RedirectError): """ Infinite redirect loop. diff --git a/httpcore/status_codes.py b/httpcore/status_codes.py index 19b849f045..6a224d0434 100644 --- a/httpcore/status_codes.py +++ b/httpcore/status_codes.py @@ -5,6 +5,7 @@ [ ("continue", 100), ("switching_protocols", 101), + ("processing", 102), ("ok", 200), ("created", 201), ("accepted", 202), @@ -13,14 +14,16 @@ ("reset_content", 205), ("partial_content", 206), ("multi_status", 207), + ("already_reported", 208), + ("im_used", 226), ("multiple_choices", 300), ("moved_permanently", 301), ("found", 302), ("see_other", 303), ("not_modified", 304), ("use_proxy", 305), - ("reserved", 306), ("temporary_redirect", 307), + ("permanent_redirect", 308), ("bad_request", 400), ("unauthorized", 401), ("payment_required", 402), diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py index ab371a047d..6499c762f2 100644 --- a/tests/adapters/test_redirects.py +++ b/tests/adapters/test_redirects.py @@ -7,6 +7,7 @@ URL, Adapter, RedirectAdapter, + RedirectBodyUnavailable, RedirectLoop, Request, Response, @@ -74,6 +75,14 @@ async def send(self, request: Request, **options) -> Response: body = json.dumps({"headers": headers}).encode() return Response(codes.ok, body=body, request=request) + elif request.url.path == "/redirect_body": + headers = [(b"location", b"/redirect_body_target")] + return Response(codes.permanent_redirect, headers=headers, request=request) + + elif request.url.path == "/redirect_body_target": + body = json.dumps({"body": request.body.decode()}).encode() + return Response(codes.ok, body=body, request=request) + return Response(codes.ok, body=b"Hello, world!", request=request) @@ -178,3 +187,26 @@ async def test_same_domain_redirect(): data = json.loads(response.body.decode()) assert response.url == URL("https://example.org/cross_domain_target") assert data == {"headers": {"authorization": "abc"}} + + +@pytest.mark.asyncio +async def test_body_redirect(): + client = RedirectAdapter(MockDispatch()) + body = b"Example request body" + url = "https://example.org/redirect_body" + response = await client.request("POST", url, body=body) + data = json.loads(response.body.decode()) + assert response.url == URL("https://example.org/redirect_body_target") + assert data == {"body": "Example request body"} + + +@pytest.mark.asyncio +async def test_cannot_redirect_streaming_body(): + client = RedirectAdapter(MockDispatch()) + url = "https://example.org/redirect_body" + + async def body(): + yield b"Example request body" + + with pytest.raises(RedirectBodyUnavailable): + await client.request("POST", url, body=body()) From 05e0649e8fee91913b59c61515a387f0a389a8ac Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 16:05:05 +0100 Subject: [PATCH 21/23] Nicer headers interface --- httpcore/models.py | 27 +++++++++++- httpcore/utils.py | 19 +++++++++ tests/adapters/test_redirects.py | 73 +++++++++++++++----------------- 3 files changed, 77 insertions(+), 42 deletions(-) diff --git a/httpcore/models.py b/httpcore/models.py index 9235240309..43f799a629 100644 --- a/httpcore/models.py +++ b/httpcore/models.py @@ -11,6 +11,7 @@ MultiDecoder, ) from .exceptions import ResponseClosed, StreamConsumed +from .utils import normalize_header_key, normalize_header_value class URL: @@ -105,7 +106,11 @@ def __hash__(self) -> int: return hash((self.is_ssl, self.hostname, self.port)) -HeaderTypes = typing.Union["Headers", typing.List[typing.Tuple[bytes, bytes]]] +HeaderTypes = typing.Union[ + "Headers", + typing.Dict[typing.AnyStr, typing.AnyStr], + typing.List[typing.Tuple[typing.AnyStr, typing.AnyStr]], +] class Headers(typing.MutableMapping[str, str]): @@ -118,8 +123,15 @@ def __init__(self, headers: HeaderTypes = None) -> None: self._list = [] # type: typing.List[typing.Tuple[bytes, bytes]] elif isinstance(headers, Headers): self._list = list(headers.raw) + elif isinstance(headers, dict): + self._list = [ + (normalize_header_key(k), normalize_header_value(v)) + for k, v in headers.items() + ] else: - self._list = [(k.lower(), v) for k, v in headers] + self._list = [ + (normalize_header_key(k), normalize_header_value(v)) for k, v in headers + ] @property def raw(self) -> typing.List[typing.Tuple[bytes, bytes]]: @@ -239,6 +251,17 @@ def __init__( self.body_aiter = body self.headers = Headers(headers) + async def read(self) -> bytes: + """ + Read and return the response content. + """ + if not hasattr(self, "body"): + body = b"" + async for part in self.stream(): + body += part + self.body = body + return self.body + async def stream(self) -> typing.AsyncIterator[bytes]: if self.is_streaming: async for part in self.body_aiter: diff --git a/httpcore/utils.py b/httpcore/utils.py index cd11858a8a..419e7ec274 100644 --- a/httpcore/utils.py +++ b/httpcore/utils.py @@ -1,3 +1,4 @@ +import typing from urllib.parse import quote from .exceptions import InvalidURL @@ -50,3 +51,21 @@ def requote_uri(uri: str) -> str: # there may be unquoted '%'s in the URI. We need to make sure they're # properly quoted so they do not cause issues elsewhere. return quote(uri, safe=safe_without_percent) + + +def normalize_header_key(value: typing.AnyStr) -> bytes: + """ + Coerce str/bytes into a strictly byte-wise HTTP header key. + """ + if isinstance(value, bytes): + return value.lower() + return value.encode("latin-1").lower() + + +def normalize_header_value(value: typing.AnyStr) -> bytes: + """ + Coerce str/bytes into a strictly byte-wise HTTP header value. + """ + if isinstance(value, bytes): + return value + return value.encode("latin-1") diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py index 6499c762f2..56ee919751 100644 --- a/tests/adapters/test_redirects.py +++ b/tests/adapters/test_redirects.py @@ -21,32 +21,28 @@ def prepare_request(self, request: Request) -> None: pass async def send(self, request: Request, **options) -> Response: - if request.url.path == "/redirect_301": # "Moved Permanently" - return Response( - 301, headers=[(b"location", b"https://example.org/")], request=request - ) + if request.url.path == "/redirect_301": + status_code = codes.moved_permanently + headers = {"location": "https://example.org/"} + return Response(status_code, headers=headers, request=request) - elif request.url.path == "/redirect_302": # "Found" - return Response( - 302, headers=[(b"location", b"https://example.org/")], request=request - ) + elif request.url.path == "/redirect_302": + status_code = codes.found + headers = {"location": "https://example.org/"} + return Response(status_code, headers=headers, request=request) - elif request.url.path == "/redirect_303": # "See Other" - return Response( - 303, headers=[(b"location", b"https://example.org/")], request=request - ) + elif request.url.path == "/redirect_303": + status_code = codes.see_other + headers = {"location": "https://example.org/"} + return Response(status_code, headers=headers, request=request) elif request.url.path == "/relative_redirect": - return Response( - codes.see_other, headers=[(b"location", b"/")], request=request - ) + headers = {"location": "/"} + return Response(codes.see_other, headers=headers, request=request) elif request.url.path == "/no_scheme_redirect": - return Response( - codes.see_other, - headers=[(b"location", b"//example.org/")], - request=request, - ) + headers = {"location": "//example.org/"} + return Response(codes.see_other, headers=headers, request=request) elif request.url.path == "/multiple_redirects": params = parse_qs(request.url.query) @@ -56,31 +52,30 @@ async def send(self, request: Request, **options) -> Response: location = "/multiple_redirects" if redirect_count: location += "?count=" + str(redirect_count) - headers = [(b"location", location.encode())] if count else [] + headers = {"location": location} if count else {} return Response(code, headers=headers, request=request) if request.url.path == "/redirect_loop": - return Response( - codes.see_other, - headers=[(b"location", b"/redirect_loop")], - request=request, - ) + headers = {"location": "/redirect_loop"} + return Response(codes.see_other, headers=headers, request=request) elif request.url.path == "/cross_domain": - location = b"https://example.org/cross_domain_target" - return Response(301, headers=[(b"location", location)], request=request) + headers = {"location": "https://example.org/cross_domain_target"} + return Response(codes.see_other, headers=headers, request=request) elif request.url.path == "/cross_domain_target": - headers = {k.decode(): v.decode() for k, v in request.headers.raw} + headers = dict(request.headers.items()) body = json.dumps({"headers": headers}).encode() return Response(codes.ok, body=body, request=request) elif request.url.path == "/redirect_body": - headers = [(b"location", b"/redirect_body_target")] + body = await request.read() + headers = {"location": "/redirect_body_target"} return Response(codes.permanent_redirect, headers=headers, request=request) elif request.url.path == "/redirect_body_target": - body = json.dumps({"body": request.body.decode()}).encode() + body = await request.read() + body = json.dumps({"body": body.decode()}).encode() return Response(codes.ok, body=body, request=request) return Response(codes.ok, body=b"Hello, world!", request=request) @@ -134,9 +129,8 @@ async def test_no_scheme_redirect(): @pytest.mark.asyncio async def test_fragment_redirect(): client = RedirectAdapter(MockDispatch()) - response = await client.request( - "GET", "https://example.org/relative_redirect#fragment" - ) + url = "https://example.org/relative_redirect#fragment" + response = await client.request("GET", url) assert response.status_code == codes.ok assert response.url == URL("https://example.org/#fragment") assert len(response.history) == 1 @@ -145,9 +139,8 @@ async def test_fragment_redirect(): @pytest.mark.asyncio async def test_multiple_redirects(): client = RedirectAdapter(MockDispatch()) - response = await client.request( - "GET", "https://example.org/multiple_redirects?count=20" - ) + url = "https://example.org/multiple_redirects?count=20" + response = await client.request("GET", url) assert response.status_code == codes.ok assert response.url == URL("https://example.org/multiple_redirects") assert len(response.history) == 20 @@ -170,8 +163,8 @@ async def test_redirect_loop(): @pytest.mark.asyncio async def test_cross_domain_redirect(): client = RedirectAdapter(MockDispatch()) - headers = [(b"Authorization", b"abc")] url = "https://example.com/cross_domain" + headers = {"Authorization": "abc"} response = await client.request("GET", url, headers=headers) data = json.loads(response.body.decode()) assert response.url == URL("https://example.org/cross_domain_target") @@ -181,8 +174,8 @@ async def test_cross_domain_redirect(): @pytest.mark.asyncio async def test_same_domain_redirect(): client = RedirectAdapter(MockDispatch()) - headers = [(b"Authorization", b"abc")] url = "https://example.org/cross_domain" + headers = {"Authorization": "abc"} response = await client.request("GET", url, headers=headers) data = json.loads(response.body.decode()) assert response.url == URL("https://example.org/cross_domain_target") @@ -192,8 +185,8 @@ async def test_same_domain_redirect(): @pytest.mark.asyncio async def test_body_redirect(): client = RedirectAdapter(MockDispatch()) - body = b"Example request body" url = "https://example.org/redirect_body" + body = b"Example request body" response = await client.request("POST", url, body=body) data = json.loads(response.body.decode()) assert response.url == URL("https://example.org/redirect_body_target") From b392b7ea591734813e94c9f0237365a53e17f91e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 16:08:31 +0100 Subject: [PATCH 22/23] Test allow_redirects=False --- tests/adapters/test_redirects.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py index 56ee919751..3197cc3a1f 100644 --- a/tests/adapters/test_redirects.py +++ b/tests/adapters/test_redirects.py @@ -108,6 +108,15 @@ async def test_redirect_303(): assert len(response.history) == 1 +@pytest.mark.asyncio +async def test_disallow_redirects(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("POST", "https://example.org/redirect_303", allow_redirects=False) + assert response.status_code == codes.see_other + assert response.url == URL("https://example.org/redirect_303") + assert len(response.history) == 0 + + @pytest.mark.asyncio async def test_relative_redirect(): client = RedirectAdapter(MockDispatch()) From 450ea25b5ad8194d180d16ede4537166ab7b3b18 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 29 Apr 2019 16:18:41 +0100 Subject: [PATCH 23/23] Add await response.next() interface --- httpcore/adapters/redirects.py | 26 ++++++++++++++++++++++---- tests/adapters/test_redirects.py | 5 +++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/httpcore/adapters/redirects.py b/httpcore/adapters/redirects.py index 584efec26e..ecbb172825 100644 --- a/httpcore/adapters/redirects.py +++ b/httpcore/adapters/redirects.py @@ -1,3 +1,4 @@ +import functools import typing from urllib.parse import urljoin, urlparse @@ -19,18 +20,25 @@ def prepare_request(self, request: Request) -> None: async def send(self, request: Request, **options: typing.Any) -> Response: allow_redirects = options.pop("allow_redirects", True) - history = [] # type: typing.List[Response] - seen_urls = set((request.url,)) + history = options.pop("history", []) # type: typing.List[Response] + seen_urls = options.pop("seen_urls", set()) # type: typing.Set[URL] + seen_urls.add(request.url) while True: response = await self.dispatch.send(request, **options) response.history = list(history) - if not allow_redirects or not response.is_redirect: + if not response.is_redirect: break history.append(response) + request = self.build_redirect_request(request, response) + if not allow_redirects: + next_options = dict(options) + next_options["seen_urls"] = seen_urls + next_options["history"] = history + response.next = functools.partial(self.send, request=request, **next_options) + break if len(history) > self.max_redirects: raise TooManyRedirects() - request = self.build_redirect_request(request, response) if request.url in seen_urls: raise RedirectLoop() seen_urls.add(request.url) @@ -71,6 +79,9 @@ def redirect_method(self, request: Request, response: Response) -> str: return method def redirect_url(self, request: Request, response: Response) -> URL: + """ + Return the URL for the redirect to follow. + """ location = response.headers["Location"] # Handle redirection without scheme (see: RFC 1808 Section 4) @@ -94,12 +105,19 @@ def redirect_url(self, request: Request, response: Response) -> URL: return URL(url) def redirect_headers(self, request: Request, url: URL) -> Headers: + """ + Strip Authorization headers when responses are redirected away from + the origin. + """ headers = Headers(request.headers) if url.origin != request.url.origin: del headers["Authorization"] return headers def redirect_body(self, request: Request, method: str) -> bytes: + """ + Return the body that should be used for the redirect request. + """ if method != request.method and method == "GET": return b"" if request.is_streaming: diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py index 3197cc3a1f..ce6d4d3f8e 100644 --- a/tests/adapters/test_redirects.py +++ b/tests/adapters/test_redirects.py @@ -116,6 +116,11 @@ async def test_disallow_redirects(): assert response.url == URL("https://example.org/redirect_303") assert len(response.history) == 0 + response = await response.next() + assert response.status_code == codes.ok + assert response.url == URL("https://example.org/") + assert len(response.history) == 1 + @pytest.mark.asyncio async def test_relative_redirect():