From b2bdac6d17709ddcb68674e5ba69902e6f657617 Mon Sep 17 00:00:00 2001 From: "TAHRI Ahmed R." Date: Tue, 24 Sep 2024 09:32:07 +0200 Subject: [PATCH] :bookmark: Release 3.8.0 (#144) **Added** - Support for HTTP Trailers. - Help script now yield warnings if update are available for each sub dependencies. **Fixed** - Setting a list of Resolver. **Changed** - urllib3-future lower bound version is raised to 2.9.900 (for http trailer support). - relax strict kwargs passing in Session adapters (required for some plugins). --- .pre-commit-config.yaml | 2 +- HISTORY.md | 14 +++++++++ README.md | 61 ++++++++++++++++++++----------------- docs/index.rst | 2 ++ docs/user/advanced.rst | 22 +++++++++++++ pyproject.toml | 2 +- src/niquests/__version__.py | 4 +-- src/niquests/help.py | 54 ++++++++++++++++++++++++++------ src/niquests/models.py | 24 +++++++++++++++ src/niquests/sessions.py | 27 ++++++++++------ src/niquests/utils.py | 14 +++++++++ tests/test_async.py | 23 ++++++++++++++ tests/test_live.py | 23 ++++++++++++++ 13 files changed, 221 insertions(+), 51 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b23cd6542..6a234e1add 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,4 +28,4 @@ repos: - id: mypy args: [--check-untyped-defs] exclude: 'tests/|noxfile.py' - additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.7.904', 'wassima>=1.0.1', 'idna', 'kiss_headers'] + additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.9.900', 'wassima>=1.0.1', 'idna', 'kiss_headers'] diff --git a/HISTORY.md b/HISTORY.md index 4a4a850bce..95bb336000 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,20 @@ Release History =============== +3.8.0 (2024-09-24) +------------------ + +**Added** +- Support for HTTP Trailers. +- Help script now yield warnings if update are available for each sub dependencies. + +**Fixed** +- Setting a list of Resolver. + +**Changed** +- urllib3-future lower bound version is raised to 2.9.900 (for http trailer support). +- relax strict kwargs passing in Session adapters (required for some plugins). + 3.7.2 (2024-07-09) ------------------ diff --git a/README.md b/README.md index a3f9e8fe58..68ff24632f 100644 --- a/README.md +++ b/README.md @@ -12,34 +12,36 @@ Niquests, is the β€œ**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc
πŸ‘† Look at the feature table comparison against requests, httpx and aiohttp! -| Feature | niquests | requests | httpx | aiohttp | -|--------------------------------------------------------------------------|:-------------------------:|:----------------------------------:|:-----------------------------:|----------------------| -| `HTTP/1.1` | βœ… | βœ… | βœ… | βœ… | -| `HTTP/2` | βœ… | ❌ | βœ…[^7] | ❌ | -| `HTTP/3 over QUIC` | βœ… | ❌ | ❌ | ❌ | -| `Synchronous` | βœ… | βœ… | βœ… | ❌ | -| `Asynchronous` | βœ… | ❌ | βœ… | βœ… | -| `Thread Safe` | βœ… | βœ… | ❌[^5] | _N/A_[^1] | -| `Task Safe` | βœ… | _N/A_[^2] | βœ… | βœ… | -| `OS Trust Store` | βœ… | ❌ | ❌ | ❌ | -| `Multiplexing` | βœ… | ❌ | _Limited_[^3] | ❌ | -| `DNSSEC` | βœ…[^11] | ❌ | ❌ | ❌ | -| `Customizable DNS Resolution` | βœ… | ❌ | ❌ | βœ… | -| `DNS over HTTPS` | βœ… | ❌ | ❌ | ❌ | -| `DNS over QUIC` | βœ… | ❌ | ❌ | ❌ | -| `DNS over TLS` | βœ… | ❌ | ❌ | ❌ | -| `Multiple DNS Resolver` | βœ… | ❌ | ❌ | ❌ | -| `Network Fine Tuning & Inspect` | βœ… | ❌ | _Limited_[^6] | _Limited_[^6] | -| `Certificate Revocation Protection` | βœ… | ❌ | ❌ | ❌ | -| `Session Persistence` | βœ… | βœ… | βœ… | βœ… | -| `In-memory Certificate CA & mTLS` | βœ… | ❌ | _Limited_[^4] | _Limited_[^4] | -| `SOCKS 4/5 Proxies` | βœ… | βœ… | βœ… | ❌ | -| `HTTP/HTTPS Proxies` | βœ… | βœ… | βœ… | βœ… | -| `TLS-in-TLS Support` | βœ… | βœ… | βœ… | βœ… | -| `Direct HTTP/3 Negotiation` | βœ…[^9] | N/A[^8] | N/A[^8] | N/A[^8] | -| `Happy Eyeballs` | βœ… | ❌ | ❌ | βœ… | -| `Package / SLSA Signed` | βœ… | ❌ | ❌ | βœ… | -| `HTTP/2 with prior knowledge (h2c)` | βœ… | ❌ | βœ… | ❌ | +| Feature | niquests | requests | httpx | aiohttp | +|-------------------------------------|:--------------:|:---------:|:-------------:|---------------| +| `HTTP/1.1` | βœ… | βœ… | βœ… | βœ… | +| `HTTP/2` | βœ… | ❌ | βœ…[^7] | ❌ | +| `HTTP/3 over QUIC` | βœ… | ❌ | ❌ | ❌ | +| `Synchronous` | βœ… | βœ… | βœ… | ❌ | +| `Asynchronous` | βœ… | ❌ | βœ… | βœ… | +| `Thread Safe` | βœ… | βœ… | ❌[^5] | _N/A_[^1] | +| `Task Safe` | βœ… | _N/A_[^2] | βœ… | βœ… | +| `OS Trust Store` | βœ… | ❌ | ❌ | ❌ | +| `Multiplexing` | βœ… | ❌ | _Limited_[^3] | ❌ | +| `DNSSEC` | βœ…[^11] | ❌ | ❌ | ❌ | +| `Customizable DNS Resolution` | βœ… | ❌ | ❌ | βœ… | +| `DNS over HTTPS` | βœ… | ❌ | ❌ | ❌ | +| `DNS over QUIC` | βœ… | ❌ | ❌ | ❌ | +| `DNS over TLS` | βœ… | ❌ | ❌ | ❌ | +| `Multiple DNS Resolver` | βœ… | ❌ | ❌ | ❌ | +| `Network Fine Tuning & Inspect` | βœ… | ❌ | _Limited_[^6] | _Limited_[^6] | +| `Certificate Revocation Protection` | βœ… | ❌ | ❌ | ❌ | +| `Session Persistence` | βœ… | βœ… | βœ… | βœ… | +| `In-memory Certificate CA & mTLS` | βœ… | ❌ | _Limited_[^4] | _Limited_[^4] | +| `SOCKS 4/5 Proxies` | βœ… | βœ… | βœ… | ❌ | +| `HTTP/HTTPS Proxies` | βœ… | βœ… | βœ… | βœ… | +| `TLS-in-TLS Support` | βœ… | βœ… | βœ… | βœ… | +| `Direct HTTP/3 Negotiation` | βœ…[^9] | N/A[^8] | N/A[^8] | N/A[^8] | +| `Happy Eyeballs` | βœ… | ❌ | ❌ | βœ… | +| `Package / SLSA Signed` | βœ… | ❌ | ❌ | βœ… | +| `HTTP/2 with prior knowledge (h2c)` | βœ… | ❌ | βœ… | ❌ | +| `Post-Quantum Security` | _Limited_[^12] | ❌ | ❌ | ❌ | +| `HTTP Trailers` | βœ… | ❌ | ❌ | ❌ |
@@ -148,6 +150,7 @@ Niquests is ready for the demands of building scalable, robust and reliable HTTP - HTTP/2 with prior knowledge - Object-oriented headers - Multi-part File Uploads +- Post-Quantum Security - Chunked HTTP Requests - Fully type-annotated! - SOCKS Proxy Support @@ -158,6 +161,7 @@ Niquests is ready for the demands of building scalable, robust and reliable HTTP - Happy Eyeballs - Multiplexed! - Thread-safe! +- Trailers! - DNSSEC! - Async! @@ -198,3 +202,4 @@ Niquests is a highly improved HTTP client that is based (forked) on Requests. Th [^9]: you must use a custom DNS resolver so that it can preemptively connect using HTTP/3 over QUIC when remote is compatible. [^10]: performance measured when leveraging a multiplexed connection with or without uses of any form of concurrency as of July 2024. The research compared `httpx`, `requests`, `aiohttp` against `niquests`. See https://github.com/Ousret/niquests-stats [^11]: enabled when using a custom DNS resolver. +[^12]: available only when using HTTP/3 over QUIC and that the remote server support also the same post-quantum key-exchange algorithm. Also, the `qh3` installed version must be >= 1.1. diff --git a/docs/index.rst b/docs/index.rst index 1cd545c51b..6495bbd6ff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,6 +78,7 @@ Niquests is ready for today's web. - Familiar `dict`–like Cookies - Object-oriented headers - Multi-part File Uploads +- Post-Quantum Security - Chunked HTTP Requests - Fully type-annotated! - SOCKS Proxy Support @@ -88,6 +89,7 @@ Niquests is ready for today's web. - Happy Eyeballs - Multiplexed! - Thread-safe! +- Trailers! - DNSSEC! - Async! diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index af25ef4826..4a9bd9c0c5 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -1485,3 +1485,25 @@ Here is a basic example of how you would proceed:: for chunk in r.iter_content(): # do anything you want with chunk print(r.download_progress.total) # this actually contain the amt of bytes (raw) downloaded from the socket. + + +HTTP Trailers +------------- + +.. note:: Available since Niquests 3.8+ + +HTTP response may contain one or several trailer headers. Those special headers are received +after the reception of the body. Before this, those headers were unreachable and dropped silently. + +Quoted from Mozilla MDN: "The Trailer response header allows the sender to include additional fields +at the end of chunked messages in order to supply metadata that might be dynamically generated while the +message body is sent, such as a message integrity check, digital signature, or post-processing status." + +For example, we retrieve our trailers this way:: + + >>> url = 'https://httpbingo.org/trailers?foo=baz' + >>> r = niquests.get(url) + >>> r.trailers # output: {'foo': 'baz'} + + +.. warning:: The ``trailers`` property is only filled when the response has been consumed entirely. The server only send them after finishing sending the body. By default, ``trailers`` is an empty CaseInsensibleDict. diff --git a/pyproject.toml b/pyproject.toml index 493c78a2ef..ffeb9bc294 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dynamic = ["version"] dependencies = [ "charset_normalizer>=2,<4", "idna>=2.5,<4", - "urllib3.future>=2.8.902,<3", + "urllib3.future>=2.9.900,<3", "wassima>=1.0.1,<2", "kiss_headers>=2,<4", ] diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 684b315a1c..41217d0dc3 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.7.2" +__version__ = "3.8.0" -__build__: int = 0x030702 +__build__: int = 0x030800 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/help.py b/src/niquests/help.py index d412cb6b49..cd5f0abc39 100644 --- a/src/niquests/help.py +++ b/src/niquests/help.py @@ -15,9 +15,9 @@ import idna import wassima -from . import RequestException +from . import RequestException, HTTPError from . import __version__ as niquests_version -from . import get +from . import Session from ._compat import HAS_LEGACY_URLLIB3 if HAS_LEGACY_URLLIB3 is True: @@ -142,26 +142,60 @@ def info(): } -def main() -> None: - """Pretty-print the bug information as JSON.""" +pypi_session = Session() + + +def check_update(package_name: str, actual_version: str) -> None: + """ + Small and concise utility to check for updates. + """ try: - response = get("https://pypi.org/pypi/niquests/json") - package_info = response.json() + response = pypi_session.get(f"https://pypi.org/pypi/{package_name}/json") + package_info = response.raise_for_status().json() if ( isinstance(package_info, dict) and "info" in package_info and "version" in package_info["info"] ): - if package_info["info"]["version"] != niquests_version: + if package_info["info"]["version"] != actual_version: warnings.warn( - f"You are using Niquests {niquests_version} and PyPI yield version ({package_info['info']['version']}) as the stable one. " - "We invite you to install this version as soon as possible. Run `python -m pip install niquests -U`.", + f"You are using {package_name} {actual_version} and PyPI yield version ({package_info['info']['version']}) as the stable one. " + f"We invite you to install this version as soon as possible. Run `python -m pip install {package_name} -U`.", UserWarning, ) - except (RequestException, JSONDecodeError): + except (RequestException, JSONDecodeError, HTTPError): pass + +PACKAGE_TO_CHECK_FOR_UPGRADE = { + "niquests": niquests_version, + "urllib3-future": urllib3.__version__, + "qh3": qh3.__version__ if qh3 is not None else None, + "jh2": jh2.__version__, + "h11": h11.__version__, + "charset-normalizer": charset_normalizer.__version__, + "wassima": wassima.__version__, + "idna": idna.__version__, +} + + +def main() -> None: + """Pretty-print the bug information as JSON.""" + for package, actual_version in PACKAGE_TO_CHECK_FOR_UPGRADE.items(): + if actual_version is None: + continue + check_update(package, actual_version) + + if __legacy_urllib3_version__ is not None: + warnings.warn( + "urllib3-future is installed alongside (legacy) urllib3. This may cause compatibility issues." + "Some (Requests) 3rd parties may be bound to urllib3, therefor the plugins may wrongfully invoke" + "urllib3 (legacy) instead of urllib3-future. To remediate this, run " + "`python -m pip uninstall -y urllib3 urllib3-future`, then run `python -m pip install urllib3-future`.", + UserWarning, + ) + print(json.dumps(info(), sort_keys=True, indent=2)) diff --git a/src/niquests/models.py b/src/niquests/models.py index bee9e6cbaf..7e97e7c465 100644 --- a/src/niquests/models.py +++ b/src/niquests/models.py @@ -976,6 +976,10 @@ def __init__(self) -> None: #: value of a ``'Content-Encoding'`` response header. self.headers: CaseInsensitiveDict = CaseInsensitiveDict() + #: Case-insensitive Dictionary of Response Trailer Headers. + #: This can only be filled after response consumption. + self.trailers: CaseInsensitiveDict = CaseInsensitiveDict() + #: File-like object representation of response (for advanced usage). #: Use of ``raw`` requires that ``stream=True`` be set on the request. #: This requirement does not apply for use internally to Requests. @@ -1245,6 +1249,9 @@ def generate() -> typing.Generator[bytes, None, None]: break yield chunk + if self.raw is not None and hasattr(self.raw, "trailers"): + self.trailers = CaseInsensitiveDict(self.raw.trailers) + self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): @@ -1347,6 +1354,15 @@ def oheaders(self) -> Headers: return headers return parse_it(self.headers) + @property + def otrailers(self) -> Headers: + """ + Retrieve trailers as they were objects. There is no need to parse headers yourself. + """ + if self.raw: + return parse_it(self.raw.trailers) + return parse_it(self.trailers) + @property def content(self) -> bytes | None: """Content of the response, in bytes.""" @@ -1374,6 +1390,8 @@ def content(self) -> bytes | None: raise RequestsSSLError(e) self._content_consumed = True + if self.raw is not None and hasattr(self.raw, "trailers"): + self.trailers = CaseInsensitiveDict(self.raw.trailers) # don't need to release the connection; that's been handled by urllib3 # since we exhausted the data. return self._content @@ -1662,6 +1680,9 @@ async def generate() -> ( yield chunk + if self.raw is not None and hasattr(self.raw, "trailers"): + self.trailers = CaseInsensitiveDict(self.raw.trailers) + self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): @@ -1761,6 +1782,9 @@ async def content(self) -> bytes | None: # type: ignore[override] except SSLError as e: raise RequestsSSLError(e) + if self.raw is not None and hasattr(self.raw, "trailers"): + self.trailers = CaseInsensitiveDict(self.raw.trailers) + self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 # since we exhausted the data. diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index ceec2e107a..c2caff048b 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -1200,15 +1200,24 @@ def handle_upload_progress( # Send the request r = adapter.send(request, **kwargs) except TypeError: - # this is required because some people may do an incomplete migration. - # this will hint them appropriately. - raise TypeError( - "You probably tried to add a Requests adapter into a Niquests session. " - "Make sure you replaced the 'import requests.adapters' into 'import niquests.adapters' " - "and made required adjustment. If you did this to increase pool_maxsize, know that the " - "Session constructor support kwargs for it. " - "See https://niquests.readthedocs.io/en/latest/user/quickstart.html#scale-your-session-pool to learn more." - ) + if "requests." in str(type(adapter)): + # this is required because some people may do an incomplete migration. + # this will hint them appropriately. + raise TypeError( + "You probably tried to add a Requests adapter into a Niquests session. " + "Make sure you replaced the 'import requests.adapters' into 'import niquests.adapters' " + "and made required adjustment. If you did this to increase pool_maxsize, know that the " + "Session constructor support kwargs for it. " + "See https://niquests.readthedocs.io/en/latest/user/quickstart.html#scale-your-session-pool to learn more." + ) + else: + # probably using a plugin that don't support extra kwargs! + # try nonetheless! + del kwargs["multiplexed"] + del kwargs["on_upload_body"] + del kwargs["on_post_connection"] + + r = adapter.send(request, **kwargs) # Make sure the timings data are kept as is, conn_info is a reference to # urllib3-future conn_info. diff --git a/src/niquests/utils.py b/src/niquests/utils.py index 4051577b43..4e8fc63359 100644 --- a/src/niquests/utils.py +++ b/src/niquests/utils.py @@ -1013,6 +1013,13 @@ def create_resolver(definition: ResolverType | None) -> BaseResolver: resolver = [ResolverDescription.from_url(definition)] elif isinstance(definition, ResolverDescription): resolver = [definition] + elif isinstance(definition, list): + if not definition: + return ResolverDescription(ProtocolResolver.SYSTEM).new() + if isinstance(definition[0], str): + resolver = [ResolverDescription.from_url(e) for e in definition] # type: ignore[arg-type] + else: + resolver = definition # type: ignore[assignment] else: raise ValueError("invalid resolver definition given") @@ -1076,6 +1083,13 @@ def create_async_resolver(definition: AsyncResolverType | None) -> AsyncBaseReso resolver = [AsyncResolverDescription.from_url(definition)] elif isinstance(definition, AsyncResolverDescription): resolver = [definition] + elif isinstance(definition, list): # can either be list of str or list of Resolver + if not definition: + return AsyncResolverDescription(ProtocolResolver.SYSTEM).new() + if isinstance(definition[0], str): + resolver = [AsyncResolverDescription.from_url(e) for e in definition] # type: ignore[arg-type] + else: + resolver = definition # type: ignore[assignment] else: raise ValueError("invalid resolver definition given") diff --git a/tests/test_async.py b/tests/test_async.py index 20ae8321a3..79a8ddf682 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -89,6 +89,29 @@ async def fake_aauth(p): assert r.status_code == 200 assert "X-Async-Auth" in r.json()["headers"] + # async def test_http_trailer_preload(self) -> None: + # async with AsyncSession() as s: + # r = await s.get("https://httpbingo.org/trailers?foo=baz") + # + # assert r.ok + # assert r.trailers + # assert "foo" in r.trailers + # assert r.trailers["foo"] == "baz" + # + # async def test_http_trailer_no_preload(self) -> None: + # async with AsyncSession() as s: + # r = await s.get("https://httpbingo.org/trailers?foo=baz", stream=True) + # + # assert r.ok + # assert not r.trailers + # assert "foo" not in r.trailers + # + # await r.content + # + # assert r.trailers + # assert "foo" in r.trailers + # assert r.trailers["foo"] == "baz" + @pytest.mark.usefixtures("requires_wan") @pytest.mark.asyncio diff --git a/tests/test_live.py b/tests/test_live.py index 5aa7b72d23..bdf45dd410 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -110,3 +110,26 @@ def test_happy_eyeballs(self) -> None: r = s.get("https://pie.dev/get") assert r.ok + + # def test_http_trailer_preload(self) -> None: + # with Session() as s: + # r = s.get("https://httpbingo.org/trailers?foo=baz") + # + # assert r.ok + # assert r.trailers + # assert "foo" in r.trailers + # assert r.trailers["foo"] == "baz" + # + # def test_http_trailer_no_preload(self) -> None: + # with Session() as s: + # r = s.get("https://httpbingo.org/trailers?foo=baz", stream=True) + # + # assert r.ok + # assert not r.trailers + # assert "foo" not in r.trailers + # + # r.content + # + # assert r.trailers + # assert "foo" in r.trailers + # assert r.trailers["foo"] == "baz"