diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 507ca4c9bbf..b3266533c20 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -384,6 +384,8 @@ def _create_connection(self, req): _SSL_OP_NO_COMPRESSION = getattr(ssl, "OP_NO_COMPRESSION", 0) _SSH_HAS_CREATE_DEFAULT_CONTEXT = hasattr(ssl, 'create_default_context') +_marker = object() + class TCPConnector(BaseConnector): """TCP connector. @@ -400,7 +402,8 @@ class TCPConnector(BaseConnector): """ def __init__(self, *, verify_ssl=True, fingerprint=None, - resolve=False, family=socket.AF_INET, ssl_context=None, + resolve=_marker, use_dns_cache=_marker, + family=socket.AF_INET, ssl_context=None, **kwargs): super().__init__(**kwargs) @@ -419,10 +422,26 @@ def __init__(self, *, verify_ssl=True, fingerprint=None, self._hashfunc = hashfunc self._fingerprint = fingerprint + if resolve is not _marker: + warnings.warn(("resolve parameter is deprecated, " + "use use_dns_cache instead"), + DeprecationWarning, stacklevel=2) + + if use_dns_cache is not _marker and resolve is not _marker: + if use_dns_cache != resolve: + raise ValueError("use_dns_cache must agree with resolve") + _use_dns_cache = use_dns_cache + elif use_dns_cache is not _marker: + _use_dns_cache = use_dns_cache + elif resolve is not _marker: + _use_dns_cache = resolve + else: + _use_dns_cache = False + + self._use_dns_cache = _use_dns_cache + self._cached_hosts = {} self._ssl_context = ssl_context self._family = family - self._resolve = resolve - self._resolved_hosts = {} @property def verify_ssl(self): @@ -466,31 +485,58 @@ def family(self): """Socket family like AF_INET.""" return self._family + @property + def use_dns_cache(self): + """True if local DNS caching is enabled.""" + return self._use_dns_cache + + @property + def cached_hosts(self): + """Read-only dict of cached DNS record.""" + return MappingProxyType(self._cached_hosts) + + def clear_dns_cache(self, host=None, port=None): + """Remove specified host/port or clear all dns local cache.""" + if host is not None and port is not None: + self._cached_hosts.pop((host, port), None) + elif host is not None or port is not None: + raise ValueError("either both host and port " + "or none of them are allowed") + else: + self._cached_hosts.clear() + @property def resolve(self): """Do DNS lookup for host name?""" - return self._resolve + warnings.warn((".resolve property is deprecated, " + "use .dns_cache instead"), + DeprecationWarning, stacklevel=2) + return self.use_dns_cache @property def resolved_hosts(self): """The dict of (host, port) -> (ipaddr, port) pairs.""" - return MappingProxyType(self._resolved_hosts) + warnings.warn((".resolved_hosts property is deprecated, " + "use .cached_hosts instead"), + DeprecationWarning, stacklevel=2) + return self.cached_hosts def clear_resolved_hosts(self, host=None, port=None): """Remove specified host/port or clear all resolve cache.""" + warnings.warn((".clear_resolved_hosts() is deprecated, " + "use .clear_dns_cache() instead"), + DeprecationWarning, stacklevel=2) if host is not None and port is not None: - key = (host, port) - if key in self._resolved_hosts: - del self._resolved_hosts[key] + self.clear_dns_cache(host, port) else: - self._resolved_hosts.clear() + self.clear_dns_cache() @asyncio.coroutine def _resolve_host(self, host, port): - if self._resolve: + if self._use_dns_cache: key = (host, port) - if key not in self._resolved_hosts: + if key not in self._cached_hosts: infos = yield from self._loop.getaddrinfo( host, port, type=socket.SOCK_STREAM, family=self._family) @@ -501,9 +547,9 @@ def _resolve_host(self, host, port): 'host': address[0], 'port': address[1], 'family': family, 'proto': proto, 'flags': socket.AI_NUMERICHOST}) - self._resolved_hosts[key] = hosts + self._cached_hosts[key] = hosts - return list(self._resolved_hosts[key]) + return list(self._cached_hosts[key]) else: return [{'hostname': host, 'host': host, 'port': port, 'family': self._family, 'proto': 0, 'flags': 0}] diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 757302dc1f2..624938e215d 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -464,7 +464,7 @@ BaseConnector TCPConnector ^^^^^^^^^^^^ -.. class:: TCPConnector(*, verify_ssl=True, fingerprint=None, resolve=False, \ +.. class:: TCPConnector(*, verify_ssl=True, fingerprint=None, use_dns_cache=False, \ family=socket.AF_INET, \ ssl_context=None, conn_timeout=None, \ keepalive_timeout=30, limit=None, share_cookies=False, \ @@ -492,13 +492,19 @@ TCPConnector .. versionadded:: 0.16 - :param bool resolve: use internal cache for DNS lookups, ``False`` + :param bool use_dns_cache: use internal cache for DNS lookups, ``False`` by default. Enabling an option *may* speedup connection establishing a bit but may introduce some *side effects* also. + .. versionadded:: 0.17 + + :param bool resolve: alias for *use_dns_cache* parameter. + + .. deprecated:: 0.17 + :param int family: TCP socket family, ``AF_INET`` by default (*IPv4*). For *IPv6* use ``AF_INET6``. @@ -525,24 +531,34 @@ TCPConnector Read-only property. - .. attribute:: resolve + .. attribute:: dns_cache Use quick lookup in internal *DNS* cache for host names if ``True``. Read-only :class:`bool` property. + .. versionadded:: 0.17 + .. attribute:: resolve - Use quick lookup in internal *DNS* cache for host names if ``True``. + Alias for :attr:`dns_cache`. - Read-only :class:`bool` property. + .. deprecated:: 0.17 - .. attribute:: resolved_hosts + .. attribute:: cached_hosts - The cache of resolved hosts if :attr:`resolve` is enabled. + The cache of resolved hosts if :attr:`dns_cache` is enabled. Read-only :class:`types.MappingProxyType` property. + .. versionadded:: 0.17 + + .. attribute:: resolved_hosts + + Alias for :attr:`cached_hosts` + + .. deprecated:: 0.17 + .. attribute:: fingerprint md5, sha1, or sha256 hash of the expected certificate in DER @@ -553,13 +569,21 @@ TCPConnector .. versionadded:: 0.16 - .. method:: clear_resolved_hosts(self, host=None, port=None) + .. method:: clear_dns_cache(self, host=None, port=None) Clear internal *DNS* cache. Remove specific entry if both *host* and *port* are specified, clear all cache otherwise. + .. versionadded:: 0.17 + + .. method:: clear_resolved_hosts(self, host=None, port=None) + + Alias for :meth:`clear_dns_cache`. + + .. deprecated:: 0.17 + diff --git a/tests/test_connector.py b/tests/test_connector.py index 53f7a9a6b83..2030402c5cb 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -458,8 +458,15 @@ def test_tcp_connector_ctor(self): conn = aiohttp.TCPConnector(loop=self.loop) self.assertTrue(conn.verify_ssl) self.assertIs(conn.fingerprint, None) - self.assertFalse(conn.resolve) + + with self.assertWarns(DeprecationWarning): + self.assertFalse(conn.resolve) + self.assertFalse(conn.use_dns_cache) + self.assertEqual(conn.family, socket.AF_INET) + + with self.assertWarns(DeprecationWarning): + self.assertEqual(conn.resolved_hosts, {}) self.assertEqual(conn.resolved_hosts, {}) def test_tcp_connector_ctor_fingerprint_valid(self): @@ -514,17 +521,37 @@ def test_tcp_connector_fingerprint(self): def test_tcp_connector_clear_resolved_hosts(self): conn = aiohttp.TCPConnector(loop=self.loop) info = object() - conn._resolved_hosts[('localhost', 123)] = info - conn._resolved_hosts[('localhost', 124)] = info + conn._cached_hosts[('localhost', 123)] = info + conn._cached_hosts[('localhost', 124)] = info conn.clear_resolved_hosts('localhost', 123) self.assertEqual( conn.resolved_hosts, {('localhost', 124): info}) conn.clear_resolved_hosts('localhost', 123) self.assertEqual( conn.resolved_hosts, {('localhost', 124): info}) - conn.clear_resolved_hosts() + with self.assertWarns(DeprecationWarning): + conn.clear_resolved_hosts() self.assertEqual(conn.resolved_hosts, {}) + def test_tcp_connector_clear_dns_cache(self): + conn = aiohttp.TCPConnector(loop=self.loop) + info = object() + conn._cached_hosts[('localhost', 123)] = info + conn._cached_hosts[('localhost', 124)] = info + conn.clear_dns_cache('localhost', 123) + self.assertEqual( + conn.cached_hosts, {('localhost', 124): info}) + conn.clear_dns_cache('localhost', 123) + self.assertEqual( + conn.cached_hosts, {('localhost', 124): info}) + conn.clear_dns_cache() + self.assertEqual(conn.cached_hosts, {}) + + def test_tcp_connector_clear_dns_cache_bad_args(self): + conn = aiohttp.TCPConnector(loop=self.loop) + with self.assertRaises(ValueError): + conn.clear_dns_cache('localhost') + def test_ambigous_verify_ssl_and_ssl_context(self): with self.assertRaises(ValueError): aiohttp.TCPConnector( @@ -756,6 +783,25 @@ def test_connector_cookie_deprecation(self): conn = aiohttp.TCPConnector(share_cookies=True, loop=self.loop) conn.close() + def test_ambiguous_ctor_params(self): + with self.assertRaises(ValueError): + aiohttp.TCPConnector(resolve=True, use_dns_cache=False, + loop=self.loop) + + def test_both_resolve_and_use_dns_cache(self): + conn = aiohttp.TCPConnector(resolve=True, use_dns_cache=True, + loop=self.loop) + self.assertTrue(conn.use_dns_cache) + with self.assertWarns(DeprecationWarning): + self.assertTrue(conn.resolve) + + def test_both_use_dns_cache_only(self): + conn = aiohttp.TCPConnector(use_dns_cache=True, + loop=self.loop) + self.assertTrue(conn.use_dns_cache) + with self.assertWarns(DeprecationWarning): + self.assertTrue(conn.resolve) + class TestProxyConnector(unittest.TestCase):