feat(server): retire HTTP/2 downstreams with GOAWAY when the drain starts

Cargo.toml

@@ -41,7 +41,11 @@ async-trait = "0.1"
 
 # HTTP server
 axum = { version = "0.7", features = ["macros", "ws", "tracing", "multipart"] }
-axum-server = { version = "0.7", features = ["tls-rustls"] }
+# The gateway serves its own accept loop on hyper directly; see
+# `serve_http` in aisix-server for why neither `axum::serve` nor
+# `axum_server` can express what the listener has to do.
+hyper = { version = "1", features = ["server", "http1", "http2"] }
+tokio-rustls = "0.26"
 tower = "0.5"
 tower-http = { version = "0.6", features = ["trace", "cors", "limit", "compression-gzip", "set-header"] }
 http = "1.2"

crates/aisix-proxy/src/lib.rs

@@ -609,13 +609,11 @@ fn hold_until_body_done(response: Response, drain: DrainGuard) -> Response {
 /// this header's.
 ///
 /// HTTP/2 forbids this header (RFC 9113 §8.2.2 — it is connection-specific)
-/// and has no header in its place: the h2 retirement signal is GOAWAY, a
-/// connection-level frame. hyper emits one when the listener shuts down,
-/// which is after the drain, so an h2 downstream goes the whole window
-/// unsignalled. Emitting it at SIGTERM instead needs the hyper connection
-/// future, which `axum_server` owns internally and only retires together
-/// with the listener (AISIX-Cloud#1395) — nothing added to the response
-/// here can reach an h2 peer.
+/// and has no header in its place, so nothing added to the response here
+/// can reach an h2 peer. Its retirement signal is GOAWAY, a
+/// connection-level frame, and the listener sends it when the drain
+/// starts — see `serve_connection` in the aisix-server binary
+/// (AISIX-Cloud#1395).
 fn retire_connection(version: &axum::http::Version, response: &mut Response) {
     if *version == axum::http::Version::HTTP_2 || *version == axum::http::Version::HTTP_3 {
         return;

crates/aisix-proxy/src/request_id.rs

@@ -105,6 +105,34 @@ fn downstream_request_id(headers: &axum::http::HeaderMap) -> Option<&str> {
 #[derive(Debug, Clone)]
 pub(crate) struct RequestId(pub String);
 
+/// The downstream protocol, as a bounded label for the request span.
+///
+/// It rides the span alongside `peer` because it is the same kind of
+/// fact — a property of the connection rather than of the request — and
+/// so reaches the access log, the per-attempt `provider call completed`
+/// line, and every diagnostic in between without 29 call sites having to
+/// thread it down.
+///
+/// It is worth a field because the two protocols retire differently
+/// during a drain, and an operator sizing
+/// `terminationGracePeriodSeconds` has to know which they have: an
+/// HTTP/1.1 connection retires itself through `Connection: close` as it
+/// is used, while an HTTP/2 one retires on the GOAWAY the listener sends
+/// when the drain starts (AISIX-Cloud#1395). Until this field existed
+/// nothing the gateway logged said which a deployment was seeing.
+///
+/// The set is closed, so it is safe as a metric label too.
+fn http_version_label(version: axum::http::Version) -> &'static str {
+    match version {
+        axum::http::Version::HTTP_09 => "0.9",
+        axum::http::Version::HTTP_10 => "1.0",
+        axum::http::Version::HTTP_11 => "1.1",
+        axum::http::Version::HTTP_2 => "2",
+        axum::http::Version::HTTP_3 => "3",
+        _ => "other",
+    }
+}
+
 /// Ingress+egress middleware that resolves the request id — the caller's
 /// own when they supplied one, a fresh UUID otherwise — and gives every
 /// proxied response an `x-aisix-request-id` header derived from the same id
@@ -159,6 +187,9 @@ pub(crate) struct RequestId(pub String);
 /// Both are recorded rather than declared, so a request that carried
 /// neither logs neither field instead of an empty one.
 ///
+/// `http_version` is the third, and it is always known, so it is
+/// declared. See [`http_version_label`] for why it is worth a field.
+///
 /// Response-body streams (SSE) are polled after this middleware returns,
 /// so they fall outside the span. Generators that moderate streamed
 /// output re-attach it explicitly — see `chat::build_sse_stream`.
@@ -191,6 +222,7 @@ pub(crate) async fn ensure_request_id(
     let span = tracing::info_span!(
         "request",
         request_id = %id,
+        http_version = http_version_label(request.version()),
         peer = tracing::field::Empty,
         downstream_request_id = tracing::field::Empty,
     );
@@ -322,6 +354,18 @@ mod tests {
     use std::sync::Arc;
     use tower::ServiceExt;
 
+    /// The label is a metric-safe closed set, and the two versions that
+    /// actually reach the proxy listener have to be told apart — that is
+    /// the whole reason the field exists.
+    #[test]
+    fn http_version_labels_are_a_closed_set() {
+        assert_eq!(http_version_label(axum::http::Version::HTTP_11), "1.1");
+        assert_eq!(http_version_label(axum::http::Version::HTTP_2), "2");
+        assert_eq!(http_version_label(axum::http::Version::HTTP_10), "1.0");
+        assert_eq!(http_version_label(axum::http::Version::HTTP_09), "0.9");
+        assert_eq!(http_version_label(axum::http::Version::HTTP_3), "3");
+    }
+
     fn state_accepting(accept_headers: &[&str]) -> ProxyState {
         ProxyState::new(
             SnapshotHandle::new(AisixSnapshot::new()),

crates/aisix-server/Cargo.toml

@@ -29,7 +29,10 @@ aisix-ratelimit = { path = "../aisix-ratelimit" }
 aisix-cache = { path = "../aisix-cache", features = ["redis"] }
 tokio.workspace = true
 axum.workspace = true
-axum-server.workspace = true
+hyper.workspace = true
+tokio-rustls.workspace = true
+tower.workspace = true
+rustls-pemfile.workspace = true
 rustls.workspace = true
 clap.workspace = true
 config.workspace = true
@@ -51,10 +54,10 @@ x509-parser = "0.16"
 # without the feature today only because other dependencies happen to
 # enable it on the same socket2, which a version bump would end.
 socket2 = { version = "0.5", features = ["all"] }
-# Only for the hyper connection builder `axum_server::Server::http_builder()`
-# hands back — where `downstream.idle_timeout_secs` is applied. Same 0.1.x
-# axum-server itself depends on, so the builder types unify.
-hyper-util = { version = "0.1", features = ["server-auto", "tokio"] }
+# `TokioIo`/`TokioExecutor`/`TokioTimer` and the tower->hyper service
+# bridge the accept loop needs. Version dispatch is the gateway's own
+# (`Downstream`), so `server-auto` is deliberately not enabled.
+hyper-util = { version = "0.1", features = ["tokio", "service"] }
 
 # jemalloc as the global allocator, only on the targets we ship and
 # bench (Linux glibc — the Docker image and both supported production
@@ -70,5 +73,9 @@ tikv-jemalloc-ctl = "0.6"
 
 [dev-dependencies]
 tempfile = "3"
+# An HTTP/2 client, so the accept loop's retirement behaviour can be
+# asserted against a real peer rather than inspected.
+hyper = { workspace = true, features = ["client"] }
+http-body-util.workspace = true
 wiremock = "0.6"
 tokio = { workspace = true, features = ["test-util"] }