Fix up proxy header behaviour

httpcore/_async/http_proxy.py

@@ -8,6 +8,23 @@
 from .connection_pool import AsyncConnectionPool, ResponseByteStream
 
 
+def merge_headers(
+    default_headers: Headers = None, override_headers: Headers = None
+) -> Headers:
+    """
+    Append default_headers and override_headers, de-duplicating if a key existing in both cases.
+    """
+    default_headers = [] if default_headers is None else default_headers
+    override_headers = [] if override_headers is None else override_headers
+    has_override = set([key.lower() for key, value in override_headers])
+    default_headers = [
+        (key, value)
+        for key, value in default_headers
+        if key.lower() not in has_override
+    ]
+    return default_headers + override_headers
+
+
 class AsyncHTTPProxy(AsyncConnectionPool):
     """
     A connection pool for making HTTP requests via an HTTP proxy.
@@ -105,7 +122,7 @@ async def _forward_request(
         # [headers]
         target = b"%b://%b:%d%b" % url
         url = self.proxy_origin + (target,)
-        headers = self.proxy_headers + ([] if headers is None else headers)
+        headers = merge_headers(self.proxy_headers, headers)
 
         response = await connection.request(
             method, url, headers=headers, stream=stream, timeout=timeout
@@ -144,8 +161,10 @@ async def _tunnel_request(
             # [proxy-headers]
             target = b"%b:%d" % (url[1], url[2])
             connect_url = self.proxy_origin + (target,)
+            connect_headers = [(b"Host", target), (b"Accept", b"*/*")]
+            connect_headers = merge_headers(connect_headers, self.proxy_headers)
             proxy_response = await proxy_connection.request(
-                b"CONNECT", connect_url, headers=self.proxy_headers, timeout=timeout
+                b"CONNECT", connect_url, headers=connect_headers, timeout=timeout
             )
             proxy_status_code = proxy_response[1]
             proxy_reason_phrase = proxy_response[2]

httpcore/_sync/http_proxy.py

@@ -8,6 +8,23 @@
 from .connection_pool import SyncConnectionPool, ResponseByteStream
 
 
+def merge_headers(
+    default_headers: Headers = None, override_headers: Headers = None
+) -> Headers:
+    """
+    Append default_headers and override_headers, de-duplicating if a key existing in both cases.
+    """
+    default_headers = [] if default_headers is None else default_headers
+    override_headers = [] if override_headers is None else override_headers
+    has_override = set([key.lower() for key, value in override_headers])
+    default_headers = [
+        (key, value)
+        for key, value in default_headers
+        if key.lower() not in has_override
+    ]
+    return default_headers + override_headers
+
+
 class SyncHTTPProxy(SyncConnectionPool):
     """
     A connection pool for making HTTP requests via an HTTP proxy.
@@ -105,7 +122,7 @@ def _forward_request(
         # [headers]
         target = b"%b://%b:%d%b" % url
         url = self.proxy_origin + (target,)
-        headers = self.proxy_headers + ([] if headers is None else headers)
+        headers = merge_headers(self.proxy_headers, headers)
 
         response = connection.request(
             method, url, headers=headers, stream=stream, timeout=timeout
@@ -144,8 +161,10 @@ def _tunnel_request(
             # [proxy-headers]
             target = b"%b:%d" % (url[1], url[2])
             connect_url = self.proxy_origin + (target,)
+            connect_headers = [(b"Host", target), (b"Accept", b"*/*")]
+            connect_headers = merge_headers(connect_headers, self.proxy_headers)
             proxy_response = proxy_connection.request(
-                b"CONNECT", connect_url, headers=self.proxy_headers, timeout=timeout
+                b"CONNECT", connect_url, headers=connect_headers, timeout=timeout
             )
             proxy_status_code = proxy_response[1]
             proxy_reason_phrase = proxy_response[2]

tests/async_tests/test_interfaces.py

@@ -188,12 +188,8 @@ async def test_http_proxy(
     headers = [(b"host", b"example.org")]
     max_connections = 1
     max_keepalive = 2
-    # Tunnel requires the host header to be present,
-    # Forwarding will use the request headers
-    proxy_headers = headers if proxy_mode == "TUNNEL_ONLY" else None
     async with httpcore.AsyncHTTPProxy(
         proxy_server,
-        proxy_headers=proxy_headers,
         proxy_mode=proxy_mode,
         max_connections=max_connections,
         max_keepalive=max_keepalive,
@@ -220,12 +216,11 @@ async def test_proxy_https_requests(
 ) -> None:
     method = b"GET"
     url = (b"https", b"example.org", 443, b"/")
-    headers = proxy_headers = [(b"host", b"example.org")]
+    headers = [(b"host", b"example.org")]
     max_connections = 1
     max_keepalive = 2
     async with httpcore.AsyncHTTPProxy(
         proxy_server,
-        proxy_headers=proxy_headers,
         proxy_mode=proxy_mode,
         ssl_context=ca_ssl_context,
         max_connections=max_connections,

tests/sync_tests/test_interfaces.py

@@ -188,12 +188,8 @@ def test_http_proxy(
     headers = [(b"host", b"example.org")]
     max_connections = 1
     max_keepalive = 2
-    # Tunnel requires the host header to be present,
-    # Forwarding will use the request headers
-    proxy_headers = headers if proxy_mode == "TUNNEL_ONLY" else None
     with httpcore.SyncHTTPProxy(
         proxy_server,
-        proxy_headers=proxy_headers,
         proxy_mode=proxy_mode,
         max_connections=max_connections,
         max_keepalive=max_keepalive,
@@ -220,12 +216,11 @@ def test_proxy_https_requests(
 ) -> None:
     method = b"GET"
     url = (b"https", b"example.org", 443, b"/")
-    headers = proxy_headers = [(b"host", b"example.org")]
+    headers = [(b"host", b"example.org")]
     max_connections = 1
     max_keepalive = 2
     with httpcore.SyncHTTPProxy(
         proxy_server,
-        proxy_headers=proxy_headers,
         proxy_mode=proxy_mode,
         ssl_context=ca_ssl_context,
         max_connections=max_connections,