From 0426eb0604edf79bcf7aa0aa991c2bc15493014c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 5 Jan 2022 09:53:06 +0000 Subject: [PATCH 1/7] First pass at socks support --- httpcore/_async/socks_proxy.py | 88 ++++++++++++++++++++++++++++++++ httpcore/_sync/socks_proxy.py | 88 ++++++++++++++++++++++++++++++++ tests/_async/test_socks_proxy.py | 85 ++++++++++++++++++++++++++++++ tests/_sync/test_socks_proxy.py | 85 ++++++++++++++++++++++++++++++ 4 files changed, 346 insertions(+) create mode 100644 httpcore/_async/socks_proxy.py create mode 100644 httpcore/_sync/socks_proxy.py create mode 100644 tests/_async/test_socks_proxy.py create mode 100644 tests/_sync/test_socks_proxy.py diff --git a/httpcore/_async/socks_proxy.py b/httpcore/_async/socks_proxy.py new file mode 100644 index 000000000..2451e669b --- /dev/null +++ b/httpcore/_async/socks_proxy.py @@ -0,0 +1,88 @@ +import typing + +from socksio import socks5 + +from .._exceptions import ProxyError +from ..backends.base import AsyncNetworkStream + +AUTH_METHODS = { + b"\x00": "NO AUTHENTICATION REQUIRED", + b"\x01": "GSSAPI", + b"\x02": "USERNAME/PASSWORD", + b"\xff": "NO ACCEPTABLE METHODS", +} + +REPLY_CODES = { + b"\x00": "Succeeded", + b"\x01": "General SOCKS server failure", + b"\x02": "Connection not allowed by ruleset", + b"\x03": "Network unreachable", + b"\x04": "Host unreachable", + b"\x05": "Connection refused", + b"\x06": "TTL expired", + b"\x07": "Command not supported", + b"\x08": "Address type not supported", +} + + +async def _init_socks5_connection( + stream: AsyncNetworkStream, + *, + host: bytes, + port: int, + auth: typing.Tuple[bytes, bytes] = None, +) -> None: + conn = socks5.SOCKS5Connection() + + # Auth method request + auth_method = ( + socks5.SOCKS5AuthMethod.NO_AUTH_REQUIRED + if auth is None + else socks5.SOCKS5AuthMethod.USERNAME_PASSWORD + ) + conn.send(socks5.SOCKS5AuthMethodsRequest([auth_method])) + outgoing_bytes = conn.data_to_send() + await stream.write(outgoing_bytes) + + # Auth method response + incoming_bytes = await stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socks5.SOCKS5AuthReply) + if response.method != auth_method: + requested = AUTH_METHODS.get(auth_method, "UNKNOWN") + responded = AUTH_METHODS.get(response.method, "UNKNOWN") + raise ProxyError( + f"Requested {requested} from proxy server, but got {responded}." + ) + + if response.method == socks5.SOCKS5AuthMethod.USERNAME_PASSWORD: + # Username/password request + assert auth is not None + username, password = auth + conn.send(socks5.SOCKS5UsernamePasswordRequest(username, password)) + outgoing_bytes = conn.data_to_send() + await stream.write(outgoing_bytes) + + # Username/password response + incoming_bytes = await stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socks5.SOCKS5UsernamePasswordReply) + if not response.success: + raise ProxyError("Invalid username/password") + + # Connect request + conn.send( + socks5.SOCKS5CommandRequest.from_address( + socks5.SOCKS5Command.CONNECT, (host, port) + ) + ) + outgoing_bytes = conn.data_to_send() + await stream.write(outgoing_bytes) + + # Connect response + incoming_bytes = await stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socks5.SOCKS5Reply) + if response.reply_code != socks5.SOCKS5ReplyCode.SUCCEEDED: + reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") + raise ProxyError(f"Proxy Server could not connect: {reply_code}.") diff --git a/httpcore/_sync/socks_proxy.py b/httpcore/_sync/socks_proxy.py new file mode 100644 index 000000000..8bf1189fb --- /dev/null +++ b/httpcore/_sync/socks_proxy.py @@ -0,0 +1,88 @@ +import typing + +from socksio import socks5 + +from .._exceptions import ProxyError +from ..backends.base import NetworkStream + +AUTH_METHODS = { + b"\x00": "NO AUTHENTICATION REQUIRED", + b"\x01": "GSSAPI", + b"\x02": "USERNAME/PASSWORD", + b"\xff": "NO ACCEPTABLE METHODS", +} + +REPLY_CODES = { + b"\x00": "Succeeded", + b"\x01": "General SOCKS server failure", + b"\x02": "Connection not allowed by ruleset", + b"\x03": "Network unreachable", + b"\x04": "Host unreachable", + b"\x05": "Connection refused", + b"\x06": "TTL expired", + b"\x07": "Command not supported", + b"\x08": "Address type not supported", +} + + +def _init_socks5_connection( + stream: NetworkStream, + *, + host: bytes, + port: int, + auth: typing.Tuple[bytes, bytes] = None, +) -> None: + conn = socks5.SOCKS5Connection() + + # Auth method request + auth_method = ( + socks5.SOCKS5AuthMethod.NO_AUTH_REQUIRED + if auth is None + else socks5.SOCKS5AuthMethod.USERNAME_PASSWORD + ) + conn.send(socks5.SOCKS5AuthMethodsRequest([auth_method])) + outgoing_bytes = conn.data_to_send() + stream.write(outgoing_bytes) + + # Auth method response + incoming_bytes = stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socks5.SOCKS5AuthReply) + if response.method != auth_method: + requested = AUTH_METHODS.get(auth_method, "UNKNOWN") + responded = AUTH_METHODS.get(response.method, "UNKNOWN") + raise ProxyError( + f"Requested {requested} from proxy server, but got {responded}." + ) + + if response.method == socks5.SOCKS5AuthMethod.USERNAME_PASSWORD: + # Username/password request + assert auth is not None + username, password = auth + conn.send(socks5.SOCKS5UsernamePasswordRequest(username, password)) + outgoing_bytes = conn.data_to_send() + stream.write(outgoing_bytes) + + # Username/password response + incoming_bytes = stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socks5.SOCKS5UsernamePasswordReply) + if not response.success: + raise ProxyError("Invalid username/password") + + # Connect request + conn.send( + socks5.SOCKS5CommandRequest.from_address( + socks5.SOCKS5Command.CONNECT, (host, port) + ) + ) + outgoing_bytes = conn.data_to_send() + stream.write(outgoing_bytes) + + # Connect response + incoming_bytes = stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socks5.SOCKS5Reply) + if response.reply_code != socks5.SOCKS5ReplyCode.SUCCEEDED: + reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") + raise ProxyError(f"Proxy Server could not connect: {reply_code}.") diff --git a/tests/_async/test_socks_proxy.py b/tests/_async/test_socks_proxy.py new file mode 100644 index 000000000..a05bac708 --- /dev/null +++ b/tests/_async/test_socks_proxy.py @@ -0,0 +1,85 @@ +import pytest + +from httpcore import ProxyError +from httpcore._async.socks_proxy import _init_socks5_connection +from httpcore.backends.mock import AsyncMockStream + + +@pytest.mark.anyio +async def test_init_socks5_proxy(): + stream = AsyncMockStream( + [ + # v5 NOAUTH + b"\x05\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + ] + ) + await _init_socks5_connection(stream, host=b"google.com", port=80) + + +@pytest.mark.anyio +async def test_init_socks5_proxy_failed(): + stream = AsyncMockStream( + [ + # v5 NOAUTH + b"\x05\x00", + # v5 NO RSV IP4 0 .0 .0 .0 :00 + b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00", + ] + ) + with pytest.raises(ProxyError) as exc_info: + await _init_socks5_connection(stream, host=b"google.com", port=80) + assert str(exc_info.value) == "Proxy Server could not connect: Connection refused." + + +@pytest.mark.anyio +async def test_init_socks5_proxy_invalid_auth_method(): + stream = AsyncMockStream( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + ] + ) + with pytest.raises(ProxyError) as exc_info: + await _init_socks5_connection(stream, host=b"google.com", port=80) + assert ( + str(exc_info.value) + == "Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD." + ) + + +@pytest.mark.anyio +async def test_init_socks5_proxy_invalid_username_password(): + stream = AsyncMockStream( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v5 INVALID USERNAME/PASSWORD + b"\x05\x01", + ] + ) + with pytest.raises(ProxyError) as exc_info: + await _init_socks5_connection( + stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") + ) + assert str(exc_info.value) == "Invalid username/password" + + +@pytest.mark.anyio +async def test_init_socks5_proxy_valid_username_password(): + stream = AsyncMockStream( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v5 VALID USERNAME/PASSWORD + b"\x05\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + ] + ) + with pytest.raises(ProxyError) as exc_info: + await _init_socks5_connection( + stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") + ) + assert str(exc_info.value) == "Invalid username/password" diff --git a/tests/_sync/test_socks_proxy.py b/tests/_sync/test_socks_proxy.py new file mode 100644 index 000000000..5444f44e6 --- /dev/null +++ b/tests/_sync/test_socks_proxy.py @@ -0,0 +1,85 @@ +import pytest + +from httpcore import ProxyError +from httpcore._async.socks_proxy import _init_socks5_connection +from httpcore.backends.mock import MockStream + + + +def test_init_socks5_proxy(): + stream = MockStream( + [ + # v5 NOAUTH + b"\x05\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + ] + ) + _init_socks5_connection(stream, host=b"google.com", port=80) + + + +def test_init_socks5_proxy_failed(): + stream = MockStream( + [ + # v5 NOAUTH + b"\x05\x00", + # v5 NO RSV IP4 0 .0 .0 .0 :00 + b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00", + ] + ) + with pytest.raises(ProxyError) as exc_info: + _init_socks5_connection(stream, host=b"google.com", port=80) + assert str(exc_info.value) == "Proxy Server could not connect: Connection refused." + + + +def test_init_socks5_proxy_invalid_auth_method(): + stream = MockStream( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + ] + ) + with pytest.raises(ProxyError) as exc_info: + _init_socks5_connection(stream, host=b"google.com", port=80) + assert ( + str(exc_info.value) + == "Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD." + ) + + + +def test_init_socks5_proxy_invalid_username_password(): + stream = MockStream( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v5 INVALID USERNAME/PASSWORD + b"\x05\x01", + ] + ) + with pytest.raises(ProxyError) as exc_info: + _init_socks5_connection( + stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") + ) + assert str(exc_info.value) == "Invalid username/password" + + + +def test_init_socks5_proxy_valid_username_password(): + stream = MockStream( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v5 VALID USERNAME/PASSWORD + b"\x05\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + ] + ) + with pytest.raises(ProxyError) as exc_info: + _init_socks5_connection( + stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") + ) + assert str(exc_info.value) == "Invalid username/password" From edd2092056e7ab64fd4fb000fc0a0ed2eeedd102 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 12 Jan 2022 13:07:25 +0000 Subject: [PATCH 2/7] Initial work for socks5 proxy support --- setup.py | 1 + tests/_sync/test_socks_proxy.py | 2 +- unasync.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c0d0a4486..9bfc613be 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ def get_packages(package): ], extras_require={ "http2": ["h2>=3,<5"], + "socks": ["socksio==1.*"] }, classifiers=[ "Development Status :: 3 - Alpha", diff --git a/tests/_sync/test_socks_proxy.py b/tests/_sync/test_socks_proxy.py index 5444f44e6..ce6a5bfd2 100644 --- a/tests/_sync/test_socks_proxy.py +++ b/tests/_sync/test_socks_proxy.py @@ -1,7 +1,7 @@ import pytest from httpcore import ProxyError -from httpcore._async.socks_proxy import _init_socks5_connection +from httpcore._sync.socks_proxy import _init_socks5_connection from httpcore.backends.mock import MockStream diff --git a/unasync.py b/unasync.py index a7b2e46fe..b71a54150 100755 --- a/unasync.py +++ b/unasync.py @@ -6,6 +6,7 @@ SUBS = [ ('from .._compat import asynccontextmanager', 'from contextlib import contextmanager'), ('from ..backends.auto import AutoBackend', 'from ..backends.sync import SyncBackend'), + ('from httpcore._async', 'from httpcore._sync'), ('import trio as concurrency', 'from tests import concurrency'), ('AsyncByteStream', 'SyncByteStream'), ('AsyncIterator', 'Iterator'), From cdd6520ba6c8f38a41b689490de585a48cbb956d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 12 Jan 2022 13:28:37 +0000 Subject: [PATCH 3/7] Install socks in dev environment --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6794c6740..328cfa722 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ --e .[http2] +-e .[http2,socks] # Optionals trio==0.19.0 From 96837e69b694a4e90a83440ec21653fc17186626 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 13 Jan 2022 14:47:41 +0000 Subject: [PATCH 4/7] Add AsyncSOCKSProxy --- httpcore/_async/socks_proxy.py | 230 ++++++++++++++++++++++++++++++- httpcore/_models.py | 2 +- httpcore/_sync/socks_proxy.py | 230 ++++++++++++++++++++++++++++++- tests/_async/test_socks_proxy.py | 95 ++++++++++++- tests/_sync/test_socks_proxy.py | 95 ++++++++++++- 5 files changed, 641 insertions(+), 11 deletions(-) diff --git a/httpcore/_async/socks_proxy.py b/httpcore/_async/socks_proxy.py index 2451e669b..638b426ab 100644 --- a/httpcore/_async/socks_proxy.py +++ b/httpcore/_async/socks_proxy.py @@ -1,9 +1,18 @@ +import ssl import typing from socksio import socks5 -from .._exceptions import ProxyError -from ..backends.base import AsyncNetworkStream +from .._exceptions import ConnectionNotAvailable, ProxyError +from .._models import URL, Origin, Request, Response, enforce_url +from .._ssl import default_ssl_context +from .._synchronization import AsyncLock +from .._trace import Trace +from ..backends.auto import AutoBackend +from ..backends.base import AsyncNetworkBackend, AsyncNetworkStream +from .connection_pool import AsyncConnectionPool +from .http11 import AsyncHTTP11Connection +from .interfaces import AsyncConnectionInterface AUTH_METHODS = { b"\x00": "NO AUTHENTICATION REQUIRED", @@ -86,3 +95,220 @@ async def _init_socks5_connection( if response.reply_code != socks5.SOCKS5ReplyCode.SUCCEEDED: reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") raise ProxyError(f"Proxy Server could not connect: {reply_code}.") + + +class AsyncSOCKSProxy(AsyncConnectionPool): + """ + A connection pool that sends requests via an HTTP proxy. + """ + + def __init__( + self, + proxy_url: typing.Union[URL, bytes, str], + ssl_context: ssl.SSLContext = None, + max_connections: typing.Optional[int] = 10, + max_keepalive_connections: int = None, + keepalive_expiry: float = None, + http1: bool = True, + http2: bool = False, + network_backend: AsyncNetworkBackend = None, + ) -> None: + """ + A connection pool for making HTTP requests. + + Parameters: + proxy_url: The URL to use when connecting to the proxy server. + For example `"http://127.0.0.1:8080/"`. + ssl_context: An SSL context to use for verifying connections. + If not specified, the default `httpcore.default_ssl_context()` + will be used. + max_connections: The maximum number of concurrent HTTP connections that + the pool should allow. Any attempt to send a request on a pool that + would exceed this amount will block until a connection is available. + max_keepalive_connections: The maximum number of idle HTTP connections + that will be maintained in the pool. + keepalive_expiry: The duration in seconds that an idle HTTP connection + may be maintained for before being expired from the pool. + http1: A boolean indicating if HTTP/1.1 requests should be supported + by the connection pool. Defaults to True. + http2: A boolean indicating if HTTP/2 requests should be supported by + the connection pool. Defaults to False. + retries: The maximum number of retries when trying to establish + a connection. + local_address: Local address to connect from. Can also be used to + connect using a particular address family. Using + `local_address="0.0.0.0"` will connect using an `AF_INET` address + (IPv4), while using `local_address="::"` will connect using an + `AF_INET6` address (IPv6). + uds: Path to a Unix Domain Socket to use instead of TCP sockets. + network_backend: A backend instance to use for handling network I/O. + """ + super().__init__( + ssl_context=ssl_context, + max_connections=max_connections, + max_keepalive_connections=max_keepalive_connections, + keepalive_expiry=keepalive_expiry, + http1=http1, + http2=http2, + network_backend=network_backend, + ) + self._ssl_context = ssl_context + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + + def create_connection(self, origin: Origin) -> AsyncConnectionInterface: + return AsyncSocks5Connection( + proxy_origin=self._proxy_url.origin, + remote_origin=origin, + ssl_context=self._ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + + +class AsyncSocks5Connection(AsyncConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + ssl_context: ssl.SSLContext = None, + keepalive_expiry: float = None, + http1: bool = True, + http2: bool = False, + network_backend: AsyncNetworkBackend = None, + ) -> None: + self._proxy_origin = proxy_origin + self._remote_origin = remote_origin + self._ssl_context = ssl_context + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + + self._network_backend: AsyncNetworkBackend = ( + AutoBackend() if network_backend is None else network_backend + ) + self._connect_lock = AsyncLock() + self._connection: typing.Optional[AsyncConnectionInterface] = None + self._connection_failed = False + + async def handle_async_request(self, request: Request) -> Response: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + async with self._connect_lock: + if self._connection is None: + try: + # Connect to the proxy + kwargs = { + "host": self._proxy_origin.host.decode("ascii"), + "port": self._proxy_origin.port, + "timeout": timeout, + } + with Trace("connection.connect_tcp", request, kwargs) as trace: + stream = await self._network_backend.connect_tcp(**kwargs) + trace.return_value = stream + + # Connect to the remote host using socks5 + kwargs = { + "stream": stream, + "host": self._remote_origin.host.decode("ascii"), + "port": self._remote_origin.port, + } + with Trace( + "connection.setup_socks5_connection", request, kwargs + ) as trace: + await _init_socks5_connection(**kwargs) + trace.return_value = stream + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + async with Trace("connection.start_tls", request, kwargs) as trace: + stream = await stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or ( + self._http2 and not self._http1 + ): # pragma: nocover + from .http2 import AsyncHTTP2Connection + + self._connection = AsyncHTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = AsyncHTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + except Exception as exc: + self._connect_failed = True + raise exc + elif not self._connection.is_available(): # pragma: nocover + raise ConnectionNotAvailable() + + return await self._connection.handle_async_request(request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + async def aclose(self) -> None: + if self._connection is not None: + await self._connection.aclose() + + def is_available(self) -> bool: + if self._connection is None: # pragma: nocover + # If HTTP/2 support is enabled, and the resulting connection could + # end up as HTTP/2 then we should indicate the connection as being + # available to service multiple requests. + return ( + self._http2 + and (self._remote_origin.scheme == b"https" or not self._http1) + and not self._connect_failed + ) + return self._connection.is_available() + + def has_expired(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.has_expired() + + def is_idle(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_idle() + + def is_closed(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_closed() + + def info(self) -> str: + if self._connection is None: # pragma: nocover + return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" + return self._connection.info() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/httpcore/_models.py b/httpcore/_models.py index 0241b70e7..d135e0d45 100644 --- a/httpcore/_models.py +++ b/httpcore/_models.py @@ -282,7 +282,7 @@ def __init__( @property def origin(self) -> Origin: - default_port = {b"http": 80, b"https": 443}[self.scheme] + default_port = {b"http": 80, b"https": 443, b"socks5": 1080}[self.scheme] return Origin( scheme=self.scheme, host=self.host, port=self.port or default_port ) diff --git a/httpcore/_sync/socks_proxy.py b/httpcore/_sync/socks_proxy.py index 8bf1189fb..55e26ae2e 100644 --- a/httpcore/_sync/socks_proxy.py +++ b/httpcore/_sync/socks_proxy.py @@ -1,9 +1,18 @@ +import ssl import typing from socksio import socks5 -from .._exceptions import ProxyError -from ..backends.base import NetworkStream +from .._exceptions import ConnectionNotAvailable, ProxyError +from .._models import URL, Origin, Request, Response, enforce_url +from .._ssl import default_ssl_context +from .._synchronization import Lock +from .._trace import Trace +from ..backends.sync import SyncBackend +from ..backends.base import NetworkBackend, NetworkStream +from .connection_pool import ConnectionPool +from .http11 import HTTP11Connection +from .interfaces import ConnectionInterface AUTH_METHODS = { b"\x00": "NO AUTHENTICATION REQUIRED", @@ -86,3 +95,220 @@ def _init_socks5_connection( if response.reply_code != socks5.SOCKS5ReplyCode.SUCCEEDED: reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") raise ProxyError(f"Proxy Server could not connect: {reply_code}.") + + +class SOCKSProxy(ConnectionPool): + """ + A connection pool that sends requests via an HTTP proxy. + """ + + def __init__( + self, + proxy_url: typing.Union[URL, bytes, str], + ssl_context: ssl.SSLContext = None, + max_connections: typing.Optional[int] = 10, + max_keepalive_connections: int = None, + keepalive_expiry: float = None, + http1: bool = True, + http2: bool = False, + network_backend: NetworkBackend = None, + ) -> None: + """ + A connection pool for making HTTP requests. + + Parameters: + proxy_url: The URL to use when connecting to the proxy server. + For example `"http://127.0.0.1:8080/"`. + ssl_context: An SSL context to use for verifying connections. + If not specified, the default `httpcore.default_ssl_context()` + will be used. + max_connections: The maximum number of concurrent HTTP connections that + the pool should allow. Any attempt to send a request on a pool that + would exceed this amount will block until a connection is available. + max_keepalive_connections: The maximum number of idle HTTP connections + that will be maintained in the pool. + keepalive_expiry: The duration in seconds that an idle HTTP connection + may be maintained for before being expired from the pool. + http1: A boolean indicating if HTTP/1.1 requests should be supported + by the connection pool. Defaults to True. + http2: A boolean indicating if HTTP/2 requests should be supported by + the connection pool. Defaults to False. + retries: The maximum number of retries when trying to establish + a connection. + local_address: Local address to connect from. Can also be used to + connect using a particular address family. Using + `local_address="0.0.0.0"` will connect using an `AF_INET` address + (IPv4), while using `local_address="::"` will connect using an + `AF_INET6` address (IPv6). + uds: Path to a Unix Domain Socket to use instead of TCP sockets. + network_backend: A backend instance to use for handling network I/O. + """ + super().__init__( + ssl_context=ssl_context, + max_connections=max_connections, + max_keepalive_connections=max_keepalive_connections, + keepalive_expiry=keepalive_expiry, + http1=http1, + http2=http2, + network_backend=network_backend, + ) + self._ssl_context = ssl_context + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + + def create_connection(self, origin: Origin) -> ConnectionInterface: + return Socks5Connection( + proxy_origin=self._proxy_url.origin, + remote_origin=origin, + ssl_context=self._ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + + +class Socks5Connection(ConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + ssl_context: ssl.SSLContext = None, + keepalive_expiry: float = None, + http1: bool = True, + http2: bool = False, + network_backend: NetworkBackend = None, + ) -> None: + self._proxy_origin = proxy_origin + self._remote_origin = remote_origin + self._ssl_context = ssl_context + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + + self._network_backend: NetworkBackend = ( + SyncBackend() if network_backend is None else network_backend + ) + self._connect_lock = Lock() + self._connection: typing.Optional[ConnectionInterface] = None + self._connection_failed = False + + def handle_request(self, request: Request) -> Response: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + with self._connect_lock: + if self._connection is None: + try: + # Connect to the proxy + kwargs = { + "host": self._proxy_origin.host.decode("ascii"), + "port": self._proxy_origin.port, + "timeout": timeout, + } + with Trace("connection.connect_tcp", request, kwargs) as trace: + stream = self._network_backend.connect_tcp(**kwargs) + trace.return_value = stream + + # Connect to the remote host using socks5 + kwargs = { + "stream": stream, + "host": self._remote_origin.host.decode("ascii"), + "port": self._remote_origin.port, + } + with Trace( + "connection.setup_socks5_connection", request, kwargs + ) as trace: + _init_socks5_connection(**kwargs) + trace.return_value = stream + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + with Trace("connection.start_tls", request, kwargs) as trace: + stream = stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or ( + self._http2 and not self._http1 + ): # pragma: nocover + from .http2 import HTTP2Connection + + self._connection = HTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = HTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + except Exception as exc: + self._connect_failed = True + raise exc + elif not self._connection.is_available(): # pragma: nocover + raise ConnectionNotAvailable() + + return self._connection.handle_request(request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + def close(self) -> None: + if self._connection is not None: + self._connection.close() + + def is_available(self) -> bool: + if self._connection is None: # pragma: nocover + # If HTTP/2 support is enabled, and the resulting connection could + # end up as HTTP/2 then we should indicate the connection as being + # available to service multiple requests. + return ( + self._http2 + and (self._remote_origin.scheme == b"https" or not self._http1) + and not self._connect_failed + ) + return self._connection.is_available() + + def has_expired(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.has_expired() + + def is_idle(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_idle() + + def is_closed(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_closed() + + def info(self) -> str: + if self._connection is None: # pragma: nocover + return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" + return self._connection.info() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/tests/_async/test_socks_proxy.py b/tests/_async/test_socks_proxy.py index a05bac708..d8e399961 100644 --- a/tests/_async/test_socks_proxy.py +++ b/tests/_async/test_socks_proxy.py @@ -1,8 +1,8 @@ import pytest -from httpcore import ProxyError -from httpcore._async.socks_proxy import _init_socks5_connection -from httpcore.backends.mock import AsyncMockStream +from httpcore import Origin, ProxyError +from httpcore._async.socks_proxy import AsyncSOCKSProxy, _init_socks5_connection +from httpcore.backends.mock import AsyncMockBackend, AsyncMockStream @pytest.mark.anyio @@ -83,3 +83,92 @@ async def test_init_socks5_proxy_valid_username_password(): stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") ) assert str(exc_info.value) == "Invalid username/password" + + +@pytest.mark.anyio +async def test_socks5_proxy_request(): + """ + Send an HTTPS request via a proxy. + """ + network_backend = AsyncMockBackend( + [ + # The initial socks CONNECT + # v5 NOAUTH + b"\x05\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + # The actual response from the remote server + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: plain/text\r\n", + b"Content-Length: 13\r\n", + b"\r\n", + b"Hello, world!", + ] + ) + + async with AsyncSOCKSProxy( + proxy_url="socks5://localhost:8080/", + max_connections=10, + network_backend=network_backend, + ) as proxy: + # Sending an intial request, which once complete will return to the pool, IDLE. + async with proxy.stream("GET", "https://example.com/") as response: + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + await response.aread() + + assert response.status == 200 + assert response.content == b"Hello, world!" + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + assert proxy.connections[0].is_idle() + assert proxy.connections[0].is_available() + assert not proxy.connections[0].is_closed() + + # A connection on a tunneled proxy can only handle HTTPS requests to the same origin. + assert not proxy.connections[0].can_handle_request( + Origin(b"http", b"example.com", 80) + ) + assert not proxy.connections[0].can_handle_request( + Origin(b"http", b"other.com", 80) + ) + assert proxy.connections[0].can_handle_request( + Origin(b"https", b"example.com", 443) + ) + assert not proxy.connections[0].can_handle_request( + Origin(b"https", b"other.com", 443) + ) + + +@pytest.mark.anyio +async def test_socks5_proxy_request_failed(): + """ + Send an HTTPS request via a proxy. + """ + network_backend = AsyncMockBackend( + [ + # The initial socks CONNECT + # v5 NOAUTH + b"\x05\x00", + # v5 NO RSV IP4 0 .0 .0 .0 :00 + b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00", + ] + ) + + async with AsyncSOCKSProxy( + proxy_url="socks5://localhost:8080/", + max_connections=10, + network_backend=network_backend, + ) as proxy: + # Sending a request, which the proxy rejects + with pytest.raises(ProxyError) as exc_info: + await proxy.request("GET", "https://example.com/") + assert ( + str(exc_info.value) == "Proxy Server could not connect: Connection refused." + ) + + assert not proxy.connections diff --git a/tests/_sync/test_socks_proxy.py b/tests/_sync/test_socks_proxy.py index ce6a5bfd2..93f61e48d 100644 --- a/tests/_sync/test_socks_proxy.py +++ b/tests/_sync/test_socks_proxy.py @@ -1,8 +1,8 @@ import pytest -from httpcore import ProxyError -from httpcore._sync.socks_proxy import _init_socks5_connection -from httpcore.backends.mock import MockStream +from httpcore import Origin, ProxyError +from httpcore._sync.socks_proxy import SOCKSProxy, _init_socks5_connection +from httpcore.backends.mock import MockBackend, MockStream @@ -83,3 +83,92 @@ def test_init_socks5_proxy_valid_username_password(): stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") ) assert str(exc_info.value) == "Invalid username/password" + + + +def test_socks5_proxy_request(): + """ + Send an HTTPS request via a proxy. + """ + network_backend = MockBackend( + [ + # The initial socks CONNECT + # v5 NOAUTH + b"\x05\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + # The actual response from the remote server + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: plain/text\r\n", + b"Content-Length: 13\r\n", + b"\r\n", + b"Hello, world!", + ] + ) + + with SOCKSProxy( + proxy_url="socks5://localhost:8080/", + max_connections=10, + network_backend=network_backend, + ) as proxy: + # Sending an intial request, which once complete will return to the pool, IDLE. + with proxy.stream("GET", "https://example.com/") as response: + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + response.read() + + assert response.status == 200 + assert response.content == b"Hello, world!" + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + assert proxy.connections[0].is_idle() + assert proxy.connections[0].is_available() + assert not proxy.connections[0].is_closed() + + # A connection on a tunneled proxy can only handle HTTPS requests to the same origin. + assert not proxy.connections[0].can_handle_request( + Origin(b"http", b"example.com", 80) + ) + assert not proxy.connections[0].can_handle_request( + Origin(b"http", b"other.com", 80) + ) + assert proxy.connections[0].can_handle_request( + Origin(b"https", b"example.com", 443) + ) + assert not proxy.connections[0].can_handle_request( + Origin(b"https", b"other.com", 443) + ) + + + +def test_socks5_proxy_request_failed(): + """ + Send an HTTPS request via a proxy. + """ + network_backend = MockBackend( + [ + # The initial socks CONNECT + # v5 NOAUTH + b"\x05\x00", + # v5 NO RSV IP4 0 .0 .0 .0 :00 + b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00", + ] + ) + + with SOCKSProxy( + proxy_url="socks5://localhost:8080/", + max_connections=10, + network_backend=network_backend, + ) as proxy: + # Sending a request, which the proxy rejects + with pytest.raises(ProxyError) as exc_info: + proxy.request("GET", "https://example.com/") + assert ( + str(exc_info.value) == "Proxy Server could not connect: Connection refused." + ) + + assert not proxy.connections From db9a450b457ec8afb84da2f19fdac420a3b7872c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 14 Jan 2022 11:25:58 +0000 Subject: [PATCH 5/7] Add SOCKSProxy and AsyncSOCKSProxy classes --- httpcore/_async/socks_proxy.py | 6 + httpcore/_sync/socks_proxy.py | 6 + tests/_async/test_socks_proxy.py | 197 +++++++++++++++++-------------- tests/_sync/test_socks_proxy.py | 197 +++++++++++++++++-------------- 4 files changed, 230 insertions(+), 176 deletions(-) diff --git a/httpcore/_async/socks_proxy.py b/httpcore/_async/socks_proxy.py index 638b426ab..6c44e263e 100644 --- a/httpcore/_async/socks_proxy.py +++ b/httpcore/_async/socks_proxy.py @@ -105,6 +105,7 @@ class AsyncSOCKSProxy(AsyncConnectionPool): def __init__( self, proxy_url: typing.Union[URL, bytes, str], + proxy_auth: typing.Tuple[bytes, bytes] = None, ssl_context: ssl.SSLContext = None, max_connections: typing.Optional[int] = 10, max_keepalive_connections: int = None, @@ -154,11 +155,13 @@ def __init__( ) self._ssl_context = ssl_context self._proxy_url = enforce_url(proxy_url, name="proxy_url") + self._proxy_auth = proxy_auth def create_connection(self, origin: Origin) -> AsyncConnectionInterface: return AsyncSocks5Connection( proxy_origin=self._proxy_url.origin, remote_origin=origin, + proxy_auth=self._proxy_auth, ssl_context=self._ssl_context, keepalive_expiry=self._keepalive_expiry, http1=self._http1, @@ -172,6 +175,7 @@ def __init__( self, proxy_origin: Origin, remote_origin: Origin, + proxy_auth: typing.Tuple[bytes, bytes] = None, ssl_context: ssl.SSLContext = None, keepalive_expiry: float = None, http1: bool = True, @@ -180,6 +184,7 @@ def __init__( ) -> None: self._proxy_origin = proxy_origin self._remote_origin = remote_origin + self._proxy_auth = proxy_auth self._ssl_context = ssl_context self._keepalive_expiry = keepalive_expiry self._http1 = http1 @@ -214,6 +219,7 @@ async def handle_async_request(self, request: Request) -> Response: "stream": stream, "host": self._remote_origin.host.decode("ascii"), "port": self._remote_origin.port, + "auth": self._proxy_auth, } with Trace( "connection.setup_socks5_connection", request, kwargs diff --git a/httpcore/_sync/socks_proxy.py b/httpcore/_sync/socks_proxy.py index 55e26ae2e..d1dbd9ec4 100644 --- a/httpcore/_sync/socks_proxy.py +++ b/httpcore/_sync/socks_proxy.py @@ -105,6 +105,7 @@ class SOCKSProxy(ConnectionPool): def __init__( self, proxy_url: typing.Union[URL, bytes, str], + proxy_auth: typing.Tuple[bytes, bytes] = None, ssl_context: ssl.SSLContext = None, max_connections: typing.Optional[int] = 10, max_keepalive_connections: int = None, @@ -154,11 +155,13 @@ def __init__( ) self._ssl_context = ssl_context self._proxy_url = enforce_url(proxy_url, name="proxy_url") + self._proxy_auth = proxy_auth def create_connection(self, origin: Origin) -> ConnectionInterface: return Socks5Connection( proxy_origin=self._proxy_url.origin, remote_origin=origin, + proxy_auth=self._proxy_auth, ssl_context=self._ssl_context, keepalive_expiry=self._keepalive_expiry, http1=self._http1, @@ -172,6 +175,7 @@ def __init__( self, proxy_origin: Origin, remote_origin: Origin, + proxy_auth: typing.Tuple[bytes, bytes] = None, ssl_context: ssl.SSLContext = None, keepalive_expiry: float = None, http1: bool = True, @@ -180,6 +184,7 @@ def __init__( ) -> None: self._proxy_origin = proxy_origin self._remote_origin = remote_origin + self._proxy_auth = proxy_auth self._ssl_context = ssl_context self._keepalive_expiry = keepalive_expiry self._http1 = http1 @@ -214,6 +219,7 @@ def handle_request(self, request: Request) -> Response: "stream": stream, "host": self._remote_origin.host.decode("ascii"), "port": self._remote_origin.port, + "auth": self._proxy_auth, } with Trace( "connection.setup_socks5_connection", request, kwargs diff --git a/tests/_async/test_socks_proxy.py b/tests/_async/test_socks_proxy.py index d8e399961..0124bc34e 100644 --- a/tests/_async/test_socks_proxy.py +++ b/tests/_async/test_socks_proxy.py @@ -1,94 +1,14 @@ import pytest from httpcore import Origin, ProxyError -from httpcore._async.socks_proxy import AsyncSOCKSProxy, _init_socks5_connection -from httpcore.backends.mock import AsyncMockBackend, AsyncMockStream +from httpcore._async.socks_proxy import AsyncSOCKSProxy +from httpcore.backends.mock import AsyncMockBackend @pytest.mark.anyio -async def test_init_socks5_proxy(): - stream = AsyncMockStream( - [ - # v5 NOAUTH - b"\x05\x00", - # v5 SUC RSV IP4 127 .0 .0 .1 :80 - b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", - ] - ) - await _init_socks5_connection(stream, host=b"google.com", port=80) - - -@pytest.mark.anyio -async def test_init_socks5_proxy_failed(): - stream = AsyncMockStream( - [ - # v5 NOAUTH - b"\x05\x00", - # v5 NO RSV IP4 0 .0 .0 .0 :00 - b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00", - ] - ) - with pytest.raises(ProxyError) as exc_info: - await _init_socks5_connection(stream, host=b"google.com", port=80) - assert str(exc_info.value) == "Proxy Server could not connect: Connection refused." - - -@pytest.mark.anyio -async def test_init_socks5_proxy_invalid_auth_method(): - stream = AsyncMockStream( - [ - # v5 USERNAME/PASSWORD - b"\x05\x02", - ] - ) - with pytest.raises(ProxyError) as exc_info: - await _init_socks5_connection(stream, host=b"google.com", port=80) - assert ( - str(exc_info.value) - == "Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD." - ) - - -@pytest.mark.anyio -async def test_init_socks5_proxy_invalid_username_password(): - stream = AsyncMockStream( - [ - # v5 USERNAME/PASSWORD - b"\x05\x02", - # v5 INVALID USERNAME/PASSWORD - b"\x05\x01", - ] - ) - with pytest.raises(ProxyError) as exc_info: - await _init_socks5_connection( - stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") - ) - assert str(exc_info.value) == "Invalid username/password" - - -@pytest.mark.anyio -async def test_init_socks5_proxy_valid_username_password(): - stream = AsyncMockStream( - [ - # v5 USERNAME/PASSWORD - b"\x05\x02", - # v5 VALID USERNAME/PASSWORD - b"\x05\x00", - # v5 SUC RSV IP4 127 .0 .0 .1 :80 - b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", - ] - ) - with pytest.raises(ProxyError) as exc_info: - await _init_socks5_connection( - stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") - ) - assert str(exc_info.value) == "Invalid username/password" - - -@pytest.mark.anyio -async def test_socks5_proxy_request(): +async def test_socks5_request(): """ - Send an HTTPS request via a proxy. + Send an HTTP request via a SOCKS proxy. """ network_backend = AsyncMockBackend( [ @@ -108,7 +28,6 @@ async def test_socks5_proxy_request(): async with AsyncSOCKSProxy( proxy_url="socks5://localhost:8080/", - max_connections=10, network_backend=network_backend, ) as proxy: # Sending an intial request, which once complete will return to the pool, IDLE. @@ -145,9 +64,56 @@ async def test_socks5_proxy_request(): @pytest.mark.anyio -async def test_socks5_proxy_request_failed(): +async def test_authenticated_socks5_request(): + """ + Send an HTTP request via a SOCKS proxy. + """ + network_backend = AsyncMockBackend( + [ + # The initial socks CONNECT + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v1 VALID USERNAME/PASSWORD + b"\x01\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + # The actual response from the remote server + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: plain/text\r\n", + b"Content-Length: 13\r\n", + b"\r\n", + b"Hello, world!", + ] + ) + + async with AsyncSOCKSProxy( + proxy_url="socks5://localhost:8080/", + proxy_auth=(b"username", b"password"), + network_backend=network_backend, + ) as proxy: + # Sending an intial request, which once complete will return to the pool, IDLE. + async with proxy.stream("GET", "https://example.com/") as response: + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + await response.aread() + + assert response.status == 200 + assert response.content == b"Hello, world!" + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + assert proxy.connections[0].is_idle() + assert proxy.connections[0].is_available() + assert not proxy.connections[0].is_closed() + + +@pytest.mark.anyio +async def test_socks5_request_connect_failed(): """ - Send an HTTPS request via a proxy. + Attempt to send an HTTP request via a SOCKS proxy, resulting in a connect failure. """ network_backend = AsyncMockBackend( [ @@ -161,7 +127,6 @@ async def test_socks5_proxy_request_failed(): async with AsyncSOCKSProxy( proxy_url="socks5://localhost:8080/", - max_connections=10, network_backend=network_backend, ) as proxy: # Sending a request, which the proxy rejects @@ -172,3 +137,59 @@ async def test_socks5_proxy_request_failed(): ) assert not proxy.connections + + +@pytest.mark.anyio +async def test_socks5_request_failed_to_provide_auth(): + """ + Attempt to send an HTTP request via an authenticated SOCKS proxy, + without providing authentication credentials. + """ + network_backend = AsyncMockBackend( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + ] + ) + + async with AsyncSOCKSProxy( + proxy_url="socks5://localhost:8080/", + network_backend=network_backend, + ) as proxy: + # Sending a request, which the proxy rejects + with pytest.raises(ProxyError) as exc_info: + await proxy.request("GET", "https://example.com/") + assert ( + str(exc_info.value) + == "Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD." + ) + + assert not proxy.connections + + +@pytest.mark.anyio +async def test_socks5_request_incorrect_auth(): + """ + Attempt to send an HTTP request via an authenticated SOCKS proxy, + wit incorrect authentication credentials. + """ + network_backend = AsyncMockBackend( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v1 INVALID USERNAME/PASSWORD + b"\x01\x01", + ] + ) + + async with AsyncSOCKSProxy( + proxy_url="socks5://localhost:8080/", + proxy_auth=(b"invalid", b"invalid"), + network_backend=network_backend, + ) as proxy: + # Sending a request, which the proxy rejects + with pytest.raises(ProxyError) as exc_info: + await proxy.request("GET", "https://example.com/") + assert str(exc_info.value) == "Invalid username/password" + + assert not proxy.connections diff --git a/tests/_sync/test_socks_proxy.py b/tests/_sync/test_socks_proxy.py index 93f61e48d..a19556d5e 100644 --- a/tests/_sync/test_socks_proxy.py +++ b/tests/_sync/test_socks_proxy.py @@ -1,94 +1,14 @@ import pytest from httpcore import Origin, ProxyError -from httpcore._sync.socks_proxy import SOCKSProxy, _init_socks5_connection -from httpcore.backends.mock import MockBackend, MockStream +from httpcore._sync.socks_proxy import SOCKSProxy +from httpcore.backends.mock import MockBackend -def test_init_socks5_proxy(): - stream = MockStream( - [ - # v5 NOAUTH - b"\x05\x00", - # v5 SUC RSV IP4 127 .0 .0 .1 :80 - b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", - ] - ) - _init_socks5_connection(stream, host=b"google.com", port=80) - - - -def test_init_socks5_proxy_failed(): - stream = MockStream( - [ - # v5 NOAUTH - b"\x05\x00", - # v5 NO RSV IP4 0 .0 .0 .0 :00 - b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00", - ] - ) - with pytest.raises(ProxyError) as exc_info: - _init_socks5_connection(stream, host=b"google.com", port=80) - assert str(exc_info.value) == "Proxy Server could not connect: Connection refused." - - - -def test_init_socks5_proxy_invalid_auth_method(): - stream = MockStream( - [ - # v5 USERNAME/PASSWORD - b"\x05\x02", - ] - ) - with pytest.raises(ProxyError) as exc_info: - _init_socks5_connection(stream, host=b"google.com", port=80) - assert ( - str(exc_info.value) - == "Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD." - ) - - - -def test_init_socks5_proxy_invalid_username_password(): - stream = MockStream( - [ - # v5 USERNAME/PASSWORD - b"\x05\x02", - # v5 INVALID USERNAME/PASSWORD - b"\x05\x01", - ] - ) - with pytest.raises(ProxyError) as exc_info: - _init_socks5_connection( - stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") - ) - assert str(exc_info.value) == "Invalid username/password" - - - -def test_init_socks5_proxy_valid_username_password(): - stream = MockStream( - [ - # v5 USERNAME/PASSWORD - b"\x05\x02", - # v5 VALID USERNAME/PASSWORD - b"\x05\x00", - # v5 SUC RSV IP4 127 .0 .0 .1 :80 - b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", - ] - ) - with pytest.raises(ProxyError) as exc_info: - _init_socks5_connection( - stream, host=b"google.com", port=80, auth=(b"invalid", b"invalid") - ) - assert str(exc_info.value) == "Invalid username/password" - - - -def test_socks5_proxy_request(): +def test_socks5_request(): """ - Send an HTTPS request via a proxy. + Send an HTTP request via a SOCKS proxy. """ network_backend = MockBackend( [ @@ -108,7 +28,6 @@ def test_socks5_proxy_request(): with SOCKSProxy( proxy_url="socks5://localhost:8080/", - max_connections=10, network_backend=network_backend, ) as proxy: # Sending an intial request, which once complete will return to the pool, IDLE. @@ -145,9 +64,56 @@ def test_socks5_proxy_request(): -def test_socks5_proxy_request_failed(): +def test_authenticated_socks5_request(): + """ + Send an HTTP request via a SOCKS proxy. + """ + network_backend = MockBackend( + [ + # The initial socks CONNECT + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v1 VALID USERNAME/PASSWORD + b"\x01\x00", + # v5 SUC RSV IP4 127 .0 .0 .1 :80 + b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", + # The actual response from the remote server + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: plain/text\r\n", + b"Content-Length: 13\r\n", + b"\r\n", + b"Hello, world!", + ] + ) + + with SOCKSProxy( + proxy_url="socks5://localhost:8080/", + proxy_auth=(b"username", b"password"), + network_backend=network_backend, + ) as proxy: + # Sending an intial request, which once complete will return to the pool, IDLE. + with proxy.stream("GET", "https://example.com/") as response: + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + response.read() + + assert response.status == 200 + assert response.content == b"Hello, world!" + info = [repr(c) for c in proxy.connections] + assert info == [ + "" + ] + assert proxy.connections[0].is_idle() + assert proxy.connections[0].is_available() + assert not proxy.connections[0].is_closed() + + + +def test_socks5_request_connect_failed(): """ - Send an HTTPS request via a proxy. + Attempt to send an HTTP request via a SOCKS proxy, resulting in a connect failure. """ network_backend = MockBackend( [ @@ -161,7 +127,6 @@ def test_socks5_proxy_request_failed(): with SOCKSProxy( proxy_url="socks5://localhost:8080/", - max_connections=10, network_backend=network_backend, ) as proxy: # Sending a request, which the proxy rejects @@ -172,3 +137,59 @@ def test_socks5_proxy_request_failed(): ) assert not proxy.connections + + + +def test_socks5_request_failed_to_provide_auth(): + """ + Attempt to send an HTTP request via an authenticated SOCKS proxy, + without providing authentication credentials. + """ + network_backend = MockBackend( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + ] + ) + + with SOCKSProxy( + proxy_url="socks5://localhost:8080/", + network_backend=network_backend, + ) as proxy: + # Sending a request, which the proxy rejects + with pytest.raises(ProxyError) as exc_info: + proxy.request("GET", "https://example.com/") + assert ( + str(exc_info.value) + == "Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD." + ) + + assert not proxy.connections + + + +def test_socks5_request_incorrect_auth(): + """ + Attempt to send an HTTP request via an authenticated SOCKS proxy, + wit incorrect authentication credentials. + """ + network_backend = MockBackend( + [ + # v5 USERNAME/PASSWORD + b"\x05\x02", + # v1 INVALID USERNAME/PASSWORD + b"\x01\x01", + ] + ) + + with SOCKSProxy( + proxy_url="socks5://localhost:8080/", + proxy_auth=(b"invalid", b"invalid"), + network_backend=network_backend, + ) as proxy: + # Sending a request, which the proxy rejects + with pytest.raises(ProxyError) as exc_info: + proxy.request("GET", "https://example.com/") + assert str(exc_info.value) == "Invalid username/password" + + assert not proxy.connections From 0cd68697ef5929a565bec72e668fe25c7e62d4d8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 14 Jan 2022 11:42:17 +0000 Subject: [PATCH 6/7] Expose SOCKSProxy, AsyncSOCKSProxy at top-level --- httpcore/__init__.py | 4 ++++ httpcore/_async/__init__.py | 13 +++++++++++++ httpcore/_sync/__init__.py | 13 +++++++++++++ tests/_async/test_socks_proxy.py | 3 +-- tests/_sync/test_socks_proxy.py | 3 +-- unasync.py | 1 - 6 files changed, 32 insertions(+), 5 deletions(-) diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 74f4de20d..a3b73e62b 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -6,6 +6,7 @@ AsyncHTTP11Connection, AsyncHTTPConnection, AsyncHTTPProxy, + AsyncSOCKSProxy, ) from ._exceptions import ( ConnectError, @@ -33,6 +34,7 @@ HTTP11Connection, HTTPConnection, HTTPProxy, + SOCKSProxy, ) __all__ = [ @@ -51,6 +53,7 @@ "AsyncHTTP11Connection", "AsyncHTTP2Connection", "AsyncConnectionInterface", + "AsyncSOCKSProxy", # sync "HTTPConnection", "ConnectionPool", @@ -58,6 +61,7 @@ "HTTP11Connection", "HTTP2Connection", "ConnectionInterface", + "SOCKSProxy", # util "default_ssl_context", # exceptions diff --git a/httpcore/_async/__init__.py b/httpcore/_async/__init__.py index a230ebeb8..88dc7f01e 100644 --- a/httpcore/_async/__init__.py +++ b/httpcore/_async/__init__.py @@ -16,6 +16,18 @@ def __init__(self, *args, **kwargs) -> None: # type: ignore ) +try: + from .socks_proxy import AsyncSOCKSProxy +except ImportError: # pragma: nocover + + class AsyncSOCKSProxy: # type: ignore + def __init__(self, *args, **kwargs) -> None: # type: ignore + raise RuntimeError( + "Attempted to use SOCKS support, but the `socksio` package is not " + "installed. Use 'pip install httpcore[socks]'." + ) + + __all__ = [ "AsyncHTTPConnection", "AsyncConnectionPool", @@ -23,4 +35,5 @@ def __init__(self, *args, **kwargs) -> None: # type: ignore "AsyncHTTP11Connection", "AsyncHTTP2Connection", "AsyncConnectionInterface", + "AsyncSOCKSProxy", ] diff --git a/httpcore/_sync/__init__.py b/httpcore/_sync/__init__.py index 73de5c1b3..b476d76d9 100644 --- a/httpcore/_sync/__init__.py +++ b/httpcore/_sync/__init__.py @@ -16,6 +16,18 @@ def __init__(self, *args, **kwargs) -> None: # type: ignore ) +try: + from .socks_proxy import SOCKSProxy +except ImportError: # pragma: nocover + + class SOCKSProxy: # type: ignore + def __init__(self, *args, **kwargs) -> None: # type: ignore + raise RuntimeError( + "Attempted to use SOCKS support, but the `socksio` package is not " + "installed. Use 'pip install httpcore[socks]'." + ) + + __all__ = [ "HTTPConnection", "ConnectionPool", @@ -23,4 +35,5 @@ def __init__(self, *args, **kwargs) -> None: # type: ignore "HTTP11Connection", "HTTP2Connection", "ConnectionInterface", + "SOCKSProxy", ] diff --git a/tests/_async/test_socks_proxy.py b/tests/_async/test_socks_proxy.py index 0124bc34e..168bc21e1 100644 --- a/tests/_async/test_socks_proxy.py +++ b/tests/_async/test_socks_proxy.py @@ -1,7 +1,6 @@ import pytest -from httpcore import Origin, ProxyError -from httpcore._async.socks_proxy import AsyncSOCKSProxy +from httpcore import AsyncSOCKSProxy, Origin, ProxyError from httpcore.backends.mock import AsyncMockBackend diff --git a/tests/_sync/test_socks_proxy.py b/tests/_sync/test_socks_proxy.py index a19556d5e..39108c8f2 100644 --- a/tests/_sync/test_socks_proxy.py +++ b/tests/_sync/test_socks_proxy.py @@ -1,7 +1,6 @@ import pytest -from httpcore import Origin, ProxyError -from httpcore._sync.socks_proxy import SOCKSProxy +from httpcore import SOCKSProxy, Origin, ProxyError from httpcore.backends.mock import MockBackend diff --git a/unasync.py b/unasync.py index b71a54150..a7b2e46fe 100755 --- a/unasync.py +++ b/unasync.py @@ -6,7 +6,6 @@ SUBS = [ ('from .._compat import asynccontextmanager', 'from contextlib import contextmanager'), ('from ..backends.auto import AutoBackend', 'from ..backends.sync import SyncBackend'), - ('from httpcore._async', 'from httpcore._sync'), ('import trio as concurrency', 'from tests import concurrency'), ('AsyncByteStream', 'SyncByteStream'), ('AsyncIterator', 'Iterator'), From e5e43f10e7be390bf73c4427d9b40aaf43bab923 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 14 Jan 2022 14:34:43 +0000 Subject: [PATCH 7/7] proxy_auth becomes ascii-str/bytes instead of bytes --- httpcore/_async/socks_proxy.py | 17 ++++++++++++++--- httpcore/_sync/socks_proxy.py | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/httpcore/_async/socks_proxy.py b/httpcore/_async/socks_proxy.py index 6c44e263e..cecb0a152 100644 --- a/httpcore/_async/socks_proxy.py +++ b/httpcore/_async/socks_proxy.py @@ -4,7 +4,7 @@ from socksio import socks5 from .._exceptions import ConnectionNotAvailable, ProxyError -from .._models import URL, Origin, Request, Response, enforce_url +from .._models import URL, Origin, Request, Response, enforce_bytes, enforce_url from .._ssl import default_ssl_context from .._synchronization import AsyncLock from .._trace import Trace @@ -105,7 +105,9 @@ class AsyncSOCKSProxy(AsyncConnectionPool): def __init__( self, proxy_url: typing.Union[URL, bytes, str], - proxy_auth: typing.Tuple[bytes, bytes] = None, + proxy_auth: typing.Tuple[ + typing.Union[bytes, str], typing.Union[bytes, str] + ] = None, ssl_context: ssl.SSLContext = None, max_connections: typing.Optional[int] = 10, max_keepalive_connections: int = None, @@ -155,7 +157,16 @@ def __init__( ) self._ssl_context = ssl_context self._proxy_url = enforce_url(proxy_url, name="proxy_url") - self._proxy_auth = proxy_auth + if proxy_auth is not None: + username, password = proxy_auth + username_bytes = enforce_bytes(username, name="proxy_auth") + password_bytes = enforce_bytes(password, name="proxy_auth") + self._proxy_auth: typing.Optional[typing.Tuple[bytes, bytes]] = ( + username_bytes, + password_bytes, + ) + else: + self._proxy_auth = None def create_connection(self, origin: Origin) -> AsyncConnectionInterface: return AsyncSocks5Connection( diff --git a/httpcore/_sync/socks_proxy.py b/httpcore/_sync/socks_proxy.py index d1dbd9ec4..a6a3a9969 100644 --- a/httpcore/_sync/socks_proxy.py +++ b/httpcore/_sync/socks_proxy.py @@ -4,7 +4,7 @@ from socksio import socks5 from .._exceptions import ConnectionNotAvailable, ProxyError -from .._models import URL, Origin, Request, Response, enforce_url +from .._models import URL, Origin, Request, Response, enforce_bytes, enforce_url from .._ssl import default_ssl_context from .._synchronization import Lock from .._trace import Trace @@ -105,7 +105,9 @@ class SOCKSProxy(ConnectionPool): def __init__( self, proxy_url: typing.Union[URL, bytes, str], - proxy_auth: typing.Tuple[bytes, bytes] = None, + proxy_auth: typing.Tuple[ + typing.Union[bytes, str], typing.Union[bytes, str] + ] = None, ssl_context: ssl.SSLContext = None, max_connections: typing.Optional[int] = 10, max_keepalive_connections: int = None, @@ -155,7 +157,16 @@ def __init__( ) self._ssl_context = ssl_context self._proxy_url = enforce_url(proxy_url, name="proxy_url") - self._proxy_auth = proxy_auth + if proxy_auth is not None: + username, password = proxy_auth + username_bytes = enforce_bytes(username, name="proxy_auth") + password_bytes = enforce_bytes(password, name="proxy_auth") + self._proxy_auth: typing.Optional[typing.Tuple[bytes, bytes]] = ( + username_bytes, + password_bytes, + ) + else: + self._proxy_auth = None def create_connection(self, origin: Origin) -> ConnectionInterface: return Socks5Connection(