Allow providing credentials to http_proxy

changelog.d/10360.feature

@@ -0,0 +1 @@
+Allow providing credentials to `http_proxy`.

synapse/http/proxyagent.py

@@ -117,7 +117,8 @@ def __init__(
             https_proxy = proxies["https"].encode() if "https" in proxies else None
             no_proxy = proxies["no"] if "no" in proxies else None
 
-        # Parse credentials from https proxy connection string if present
+        # Parse credentials from http and https proxy connection string if present
+        self.http_proxy_creds, http_proxy = parse_username_password(http_proxy)
         self.https_proxy_creds, https_proxy = parse_username_password(https_proxy)
 
         self.http_proxy_endpoint = _http_proxy_endpoint(
@@ -189,11 +190,28 @@ def request(self, method, uri, headers=None, bodyProducer=None):
             and self.http_proxy_endpoint
             and not should_skip_proxy
         ):
-            # Cache *all* connections under the same key, since we are only
-            # connecting to a single destination, the proxy:
-            pool_key = ("http-proxy", self.http_proxy_endpoint)
-            endpoint = self.http_proxy_endpoint
-            request_path = uri
+            # Determine whether we need to set Proxy-Authorization headers
+            if self.http_proxy_creds:
+                # Set a Proxy-Authorization header
+                connect_headers = Headers()
+                connect_headers.addRawHeader(
+                    b"Proxy-Authorization",
+                    self.http_proxy_creds.as_proxy_authorization_value(),
+                )
+                # if authentication is required, use tunnel instead of transparent mode
+                endpoint = HTTPConnectProxyEndpoint(
+                    self.proxy_reactor,
+                    self.http_proxy_endpoint,
+                    parsed_uri.host,
+                    parsed_uri.port,
+                    headers=connect_headers,
+                )
+            else:
+                # Cache *all* connections under the same key, since we are only
+                # connecting to a single destination, the proxy:
+                pool_key = ("http-proxy", self.http_proxy_endpoint)
+                endpoint = self.http_proxy_endpoint
+                request_path = uri
         elif (
             parsed_uri.scheme == b"https"
             and self.https_proxy_endpoint

tests/http/test_proxyagent.py

@@ -205,6 +205,10 @@ def test_https_request_via_no_proxy_star(self):
 
     @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "unused.com"})
     def test_http_request_via_proxy(self):
+        """
+        Tests that requests can be made through a proxy
+        This use transparent proxy.
+        """
         agent = ProxyAgent(self.reactor, use_proxy=True)
 
         self.reactor.lookups["proxy.com"] = "1.2.3.5"
@@ -229,6 +233,7 @@ def test_http_request_via_proxy(self):
         self.assertEqual(len(http_server.requests), 1)
 
         request = http_server.requests[0]
+
         self.assertEqual(request.method, b"GET")
         self.assertEqual(request.path, b"http://test.com")
         self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
@@ -241,6 +246,96 @@ def test_http_request_via_proxy(self):
         body = self.successResultOf(treq.content(resp))
         self.assertEqual(body, b"result")
 
+    @patch.dict(
+        os.environ,
+        {"http_proxy": "bob:[email protected]:8888", "no_proxy": "unused.com"},
+    )
+    def test_http_request_via_proxy_with_auth(self):
+        """
+        Tests that authenticated requests can be made through a proxy
+        This use a proxy tunnel via CONNECT method similar to https.
+        """
+        agent = ProxyAgent(self.reactor, use_proxy=True)
+
+        self.reactor.lookups["proxy.com"] = "1.2.3.5"
+        d = agent.request(b"GET", b"http://test.com/abc")
+
+        # there should be a pending TCP connection
+        clients = self.reactor.tcpClients
+        self.assertEqual(len(clients), 1)
+        (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+        self.assertEqual(host, "1.2.3.5")
+        self.assertEqual(port, 8888)
+
+        # make a test HTTP server, and wire up the client
+        proxy_server = self._make_connection(
+            client_factory, _get_test_protocol_factory()
+        )
+
+        # fish the transports back out so that we can do the old switcheroo
+        s2c_transport = proxy_server.transport
+        client_protocol = s2c_transport.other
+        c2s_transport = client_protocol.transport
+
+        # the FakeTransport is async, so we need to pump the reactor
+        self.reactor.advance(0)
+
+        # now there should be a pending CONNECT request
+        self.assertEqual(len(proxy_server.requests), 1)
+
+        request = proxy_server.requests[0]
+        self.assertEqual(request.method, b"CONNECT")
+        self.assertEqual(request.path, b"test.com:80")
+
+        # Check whether auth credentials have been supplied to the proxy
+        proxy_auth_header_values = request.requestHeaders.getRawHeaders(
+            b"Proxy-Authorization"
+        )
+        # Compute the correct header value for Proxy-Authorization
+        encoded_credentials = base64.b64encode(b"bob:pinkponies")
+        expected_header_value = b"Basic " + encoded_credentials
+
+        # Validate the header's value
+        self.assertIn(expected_header_value, proxy_auth_header_values)
+
+        # tell the proxy server not to close the connection
+        proxy_server.persistent = True
+
+        # this just stops the http Request trying to do a chunked response
+        # request.setHeader(b"Content-Length", b"0")
+        request.finish()
+
+        http_server = _get_test_protocol_factory().buildProtocol(None)
+        http_server.makeConnection(
+            FakeTransport(client_protocol, self.reactor, http_server)
+        )
+        c2s_transport.other = http_server
+
+        self.reactor.advance(0)
+
+        # now there should be a pending request
+        self.assertEqual(len(http_server.requests), 1)
+
+        request = http_server.requests[0]
+        self.assertEqual(request.method, b"GET")
+        self.assertEqual(request.path, b"/abc")
+        self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
+
+        # Check that the destination server DID NOT receive proxy credentials
+        proxy_auth_header_values = request.requestHeaders.getRawHeaders(
+            b"Proxy-Authorization"
+        )
+        self.assertIsNone(proxy_auth_header_values)
+
+        request.write(b"result")
+        request.finish()
+
+        self.reactor.advance(0)
+
+        resp = self.successResultOf(d)
+        body = self.successResultOf(treq.content(resp))
+        self.assertEqual(body, b"result")
+
     @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"})
     def test_https_request_via_proxy(self):
         """Tests that TLS-encrypted requests can be made through a proxy"""