access_log: Added new command operators to retrieve upstream information

docs/root/configuration/observability/access_log/usage.rst

@@ -195,6 +195,16 @@ The following command operators are supported:
   In typed JSON logs, PROTOCOL will render the string ``"-"`` if the protocol is not
   available (e.g. in TCP logs).
 
+%UPSTREAM_PROTOCOL%
+  HTTP
+    Upstream protocol. Currently either *HTTP/1.1* *HTTP/2* or *HTTP/3*.
+
+  TCP/UDP
+    Not implemented ("-").
+
+  In typed JSON logs, UPSTREAM_PROTOCOL will render the string ``"-"`` if the protocol is not
+  available (e.g. in TCP logs).
+
 %RESPONSE_CODE%
   HTTP
     HTTP response code. Note that a response code of '0' means that the server never sent the
@@ -844,6 +854,80 @@ The following command operators are supported:
   DOWNSTREAM_PEER_CERT_V_END can be customized using a `format string <https://en.cppreference.com/w/cpp/io/manip/put_time>`_.
   See :ref:`START_TIME <config_access_log_format_start_time>` for additional format specifiers and examples.
 
+%UPSTREAM_PEER_SUBJECT%
+  HTTP
+    The subject present in the peer certificate used to establish the upstream TLS connection.
+  TCP
+    The subject present in the peer certificate used to establish the upstream TLS connection.
+  UDP
+    Not implemented ("-").
+
+%UPSTREAM_PEER_ISSUER%
+  HTTP
+    The issuer present in the peer certificate used to establish the upstream TLS connection.
+  TCP
+    The issuer present in the peer certificate used to establish the upstream TLS connection.
+  UDP
+    Not implemented ("-").
+
+%UPSTREAM_TLS_SESSION_ID%
+  HTTP
+    The session ID for the established upstream TLS connection.
+  TCP
+    The session ID for the established upstream TLS connection.
+  UDP
+    Not implemented (0).
+
+%UPSTREAM_TLS_CIPHER%
+  HTTP
+    The OpenSSL name for the set of ciphers used to establish the upstream TLS connection.
+  TCP
+    The OpenSSL name for the set of ciphers used to establish the upstream TLS connection.
+  UDP
+    Not implemented ("-").
+
+%UPSTREAM_TLS_VERSION%
+  HTTP
+    The TLS version (e.g., ``TLSv1.2``, ``TLSv1.3``) used to establish the upstream TLS connection.
+  TCP
+    The TLS version (e.g., ``TLSv1.2``, ``TLSv1.3``) used to establish the upstream TLS connection.
+  UDP
+    Not implemented ("-").
+
+%UPSTREAM_PEER_CERT%
+  HTTP
+    The server certificate in the URL-encoded PEM format used to establish the upstream TLS connection.
+  TCP
+    The server certificate in the URL-encoded PEM format used to establish the upstream TLS connection.
+  UDP
+    Not implemented ("-").
+
+.. _config_access_log_format_upstream_peer_cert_v_start:
+
+%UPSTREAM_PEER_CERT_V_START%
+  HTTP
+    The validity start date of the upstream server certificate used to establish the upstream TLS connection.
+  TCP
+    The validity start date of the upstream server certificate used to establish the upstream TLS connection.
+  UDP
+    Not implemented ("-").
+
+  UPSTREAM_PEER_CERT_V_START can be customized using a `format string <https://en.cppreference.com/w/cpp/io/manip/put_time>`_.
+  See :ref:`START_TIME <config_access_log_format_start_time>` for additional format specifiers and examples.
+
+.. _config_access_log_format_upstream_peer_cert_v_end:
+
+%UPSTREAM_PEER_CERT_V_END%
+  HTTP
+    The validity end date of the upstream server certificate used to establish the upstream TLS connection.
+  TCP
+    The validity end date of the upstream server certificate used to establish the upstream TLS connection.
+  UDP
+    Not implemented ("-").
+
+  UPSTREAM_PEER_CERT_V_END can be customized using a `format string <https://en.cppreference.com/w/cpp/io/manip/put_time>`_.
+  See :ref:`START_TIME <config_access_log_format_start_time>` for additional format specifiers and examples.
+
 %HOSTNAME%
   The system hostname.
 

docs/root/version_history/current.rst

@@ -23,6 +23,7 @@ Removed Config or Runtime
 
 New Features
 ------------
+* access_log: added new access_log command operators to retrieve upstream connection information: ``%UPSTREAM_PROTOCOL%``, ``%UPSTREAM_PEER_SUBJECT%``, ``%UPSTREAM_PEER_ISSUER%``, ``%UPSTREAM_TLS_SESSION_ID%``, ``%UPSTREAM_TLS_CIPHER%``, ``%UPSTREAM_TLS_VERSION%``, ``%UPSTREAM_PEER_CERT_V_START%``, ``%UPSTREAM_PEER_CERT_V_END%`` and ``%UPSTREAM_PEER_CERT%``.
 * dns_resolver: added :ref:`include_unroutable_families<envoy_v3_api_field_extensions.network.dns_resolver.apple.v3.AppleDnsResolverConfig.include_unroutable_families>` to the Apple DNS resolver.
 * ext_proc: added support for per-route :ref:`grpc_service <envoy_v3_api_field_extensions.filters.http.ext_proc.v3.ExtProcOverrides.grpc_service>`.
 

envoy/stream_info/stream_info.h

@@ -430,6 +430,9 @@ class UpstreamInfo {
    */
   virtual void setUpstreamNumStreams(uint64_t num_streams) PURE;
   virtual uint64_t upstreamNumStreams() const PURE;
+
+  virtual void setUpstreamProtocol(Http::Protocol protocol) PURE;
+  virtual absl::optional<Http::Protocol> upstreamProtocol() const PURE;
 };
 
 /**

source/common/formatter/substitution_formatter.cc

@@ -459,6 +459,16 @@ SubstitutionFormatParser::getKnownFormatters() {
          [](const std::string& format, const absl::optional<size_t>&) {
            return std::make_unique<DownstreamPeerCertVEndFormatter>(format);
          }}},
+       {"UPSTREAM_PEER_CERT_V_START",
+        {CommandSyntaxChecker::PARAMS_OPTIONAL,
+         [](const std::string& format, const absl::optional<size_t>&) {
+           return std::make_unique<UpstreamPeerCertVStartFormatter>(format);
+         }}},
+       {"UPSTREAM_PEER_CERT_V_END",
+        {CommandSyntaxChecker::PARAMS_OPTIONAL,
+         [](const std::string& format, const absl::optional<size_t>&) {
+           return std::make_unique<UpstreamPeerCertVEndFormatter>(format);
+         }}},
        {"ENVIRONMENT",
         {CommandSyntaxChecker::PARAMS_REQUIRED | CommandSyntaxChecker::LENGTH_ALLOWED,
          [](const std::string& key, absl::optional<size_t>& max_length) {
@@ -794,6 +804,46 @@ class StreamInfoSslConnectionInfoFieldExtractor : public StreamInfoFormatter::Fi
   FieldExtractor field_extractor_;
 };
 
+class StreamInfoUpstreamSslConnectionInfoFieldExtractor
+    : public StreamInfoFormatter::FieldExtractor {
+public:
+  using FieldExtractor =
+      std::function<absl::optional<std::string>(const Ssl::ConnectionInfo& connection_info)>;
+
+  StreamInfoUpstreamSslConnectionInfoFieldExtractor(FieldExtractor f) : field_extractor_(f) {}
+
+  absl::optional<std::string> extract(const StreamInfo::StreamInfo& stream_info) const override {
+    if (!stream_info.upstreamInfo() ||
+        stream_info.upstreamInfo()->upstreamSslConnection() == nullptr) {
+      return absl::nullopt;
+    }
+
+    const auto value = field_extractor_(*(stream_info.upstreamInfo()->upstreamSslConnection()));
+    if (value && value->empty()) {
+      return absl::nullopt;
+    }
+
+    return value;
+  }
+
+  ProtobufWkt::Value extractValue(const StreamInfo::StreamInfo& stream_info) const override {
+    if (!stream_info.upstreamInfo() ||
+        stream_info.upstreamInfo()->upstreamSslConnection() == nullptr) {
+      return unspecifiedValue();
+    }
+
+    const auto value = field_extractor_(*(stream_info.upstreamInfo()->upstreamSslConnection()));
+    if (value && value->empty()) {
+      return unspecifiedValue();
+    }
+
+    return ValueUtil::optionalStringValue(value);
+  }
+
+private:
+  FieldExtractor field_extractor_;
+};
+
 const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnownFieldExtractors() {
   CONSTRUCT_ON_FIRST_USE(FieldExtractorLookupTbl,
                          {{"REQUEST_DURATION",
@@ -892,6 +942,17 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow
                                         stream_info.protocol());
                                   });
                             }}},
+                          {"UPSTREAM_PROTOCOL",
+                           {CommandSyntaxChecker::COMMAND_ONLY,
+                            [](const std::string&, const absl::optional<size_t>&) {
+                              return std::make_unique<StreamInfoStringFieldExtractor>(
+                                  [](const StreamInfo::StreamInfo& stream_info) {
+                                    return stream_info.upstreamInfo()
+                                               ? SubstitutionFormatUtils::protocolToString(
+                                                     stream_info.upstreamInfo()->upstreamProtocol())
+                                               : absl::nullopt;
+                                  });
+                            }}},
                           {"RESPONSE_CODE",
                            {CommandSyntaxChecker::COMMAND_ONLY,
                             [](const std::string&, const absl::optional<size_t>&) {
@@ -1097,6 +1158,60 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow
                                     return stream_info.attemptCount().value_or(0);
                                   });
                             }}},
+                          {"UPSTREAM_TLS_CIPHER",
+                           {CommandSyntaxChecker::COMMAND_ONLY,
+                            [](const std::string&, const absl::optional<size_t>&) {
+                              return std::make_unique<
+                                  StreamInfoUpstreamSslConnectionInfoFieldExtractor>(
+                                  [](const Ssl::ConnectionInfo& connection_info) {
+                                    return connection_info.ciphersuiteString();
+                                  });
+                            }}},
+                          {"UPSTREAM_TLS_VERSION",
+                           {CommandSyntaxChecker::COMMAND_ONLY,
+                            [](const std::string&, const absl::optional<size_t>&) {
+                              return std::make_unique<
+                                  StreamInfoUpstreamSslConnectionInfoFieldExtractor>(
+                                  [](const Ssl::ConnectionInfo& connection_info) {
+                                    return connection_info.tlsVersion();
+                                  });
+                            }}},
+                          {"UPSTREAM_TLS_SESSION_ID",
+                           {CommandSyntaxChecker::COMMAND_ONLY,
+                            [](const std::string&, const absl::optional<size_t>&) {
+                              return std::make_unique<
+                                  StreamInfoUpstreamSslConnectionInfoFieldExtractor>(
+                                  [](const Ssl::ConnectionInfo& connection_info) {
+                                    return connection_info.sessionId();
+                                  });
+                            }}},
+                          {"UPSTREAM_PEER_ISSUER",
+                           {CommandSyntaxChecker::COMMAND_ONLY,
+                            [](const std::string&, const absl::optional<size_t>&) {
+                              return std::make_unique<
+                                  StreamInfoUpstreamSslConnectionInfoFieldExtractor>(
+                                  [](const Ssl::ConnectionInfo& connection_info) {
+                                    return connection_info.issuerPeerCertificate();
+                                  });
+                            }}},
+                          {"UPSTREAM_PEER_CERT",
+                           {CommandSyntaxChecker::COMMAND_ONLY,
+                            [](const std::string&, const absl::optional<size_t>&) {
+                              return std::make_unique<
+                                  StreamInfoUpstreamSslConnectionInfoFieldExtractor>(
+                                  [](const Ssl::ConnectionInfo& connection_info) {
+                                    return connection_info.urlEncodedPemEncodedPeerCertificate();
+                                  });
+                            }}},
+                          {"UPSTREAM_PEER_SUBJECT",
+                           {CommandSyntaxChecker::COMMAND_ONLY,
+                            [](const std::string&, const absl::optional<size_t>&) {
+                              return std::make_unique<
+                                  StreamInfoUpstreamSslConnectionInfoFieldExtractor>(
+                                  [](const Ssl::ConnectionInfo& connection_info) {
+                                    return connection_info.subjectPeerCertificate();
+                                  });
+                            }}},
                           {"DOWNSTREAM_LOCAL_ADDRESS",
                            {CommandSyntaxChecker::COMMAND_ONLY,
                             [](const std::string&, const absl::optional<size_t>&) {
@@ -1807,6 +1922,30 @@ DownstreamPeerCertVEndFormatter::DownstreamPeerCertVEndFormatter(const std::stri
                                    ? connection_info->expirationPeerCertificate()
                                    : absl::optional<SystemTime>();
                       })) {}
+UpstreamPeerCertVStartFormatter::UpstreamPeerCertVStartFormatter(const std::string& format)
+    : SystemTimeFormatter(
+          format, std::make_unique<SystemTimeFormatter::TimeFieldExtractor>(
+                      [](const StreamInfo::StreamInfo& stream_info) -> absl::optional<SystemTime> {
+                        return stream_info.upstreamInfo() &&
+                                       stream_info.upstreamInfo()->upstreamSslConnection() !=
+                                           nullptr
+                                   ? stream_info.upstreamInfo()
+                                         ->upstreamSslConnection()
+                                         ->validFromPeerCertificate()
+                                   : absl::optional<SystemTime>();
+                      })) {}
+UpstreamPeerCertVEndFormatter::UpstreamPeerCertVEndFormatter(const std::string& format)
+    : SystemTimeFormatter(
+          format, std::make_unique<SystemTimeFormatter::TimeFieldExtractor>(
+                      [](const StreamInfo::StreamInfo& stream_info) -> absl::optional<SystemTime> {
+                        return stream_info.upstreamInfo() &&
+                                       stream_info.upstreamInfo()->upstreamSslConnection() !=
+                                           nullptr
+                                   ? stream_info.upstreamInfo()
+                                         ->upstreamSslConnection()
+                                         ->expirationPeerCertificate()
+                                   : absl::optional<SystemTime>();
+                      })) {}
 
 SystemTimeFormatter::SystemTimeFormatter(const std::string& format, TimeFieldExtractorPtr f)
     : date_formatter_(format), time_field_extractor_(std::move(f)) {

source/common/formatter/substitution_formatter.h

@@ -585,6 +585,24 @@ class DownstreamPeerCertVEndFormatter : public SystemTimeFormatter {
   DownstreamPeerCertVEndFormatter(const std::string& format);
 };
 
+/**
+ * SystemTimeFormatter (FormatterProvider) for upstream cert start time from the StreamInfo's
+ * upstreamInfo.
+ */
+class UpstreamPeerCertVStartFormatter : public SystemTimeFormatter {
+public:
+  UpstreamPeerCertVStartFormatter(const std::string& format);
+};
+
+/**
+ * SystemTimeFormatter (FormatterProvider) for upstream cert end time from the StreamInfo's
+ * upstreamInfo.
+ */
+class UpstreamPeerCertVEndFormatter : public SystemTimeFormatter {
+public:
+  UpstreamPeerCertVEndFormatter(const std::string& format);
+};
+
 /**
  * FormatterProvider for environment. If no valid environment value then
  */

source/common/router/upstream_request.cc

@@ -465,6 +465,9 @@ void UpstreamRequest::onPoolReady(
   stream_info_.setUpstreamBytesMeter(upstream_->bytesMeter());
   StreamInfo::StreamInfo::syncUpstreamAndDownstreamBytesMeter(parent_.callbacks()->streamInfo(),
                                                               stream_info_);
+  if (protocol) {
+    upstream_info.setUpstreamProtocol(protocol.value());
+  }
 
   if (parent_.downstreamEndStream()) {
     setupPerTryTimeout();

source/common/stream_info/stream_info_impl.h

@@ -79,6 +79,9 @@ struct UpstreamInfoImpl : public UpstreamInfo {
   void setUpstreamNumStreams(uint64_t num_streams) override { num_streams_ = num_streams; }
   uint64_t upstreamNumStreams() const override { return num_streams_; }
 
+  void setUpstreamProtocol(Http::Protocol protocol) override { upstream_protocol_ = protocol; }
+  absl::optional<Http::Protocol> upstreamProtocol() const override { return upstream_protocol_; }
+
   Upstream::HostDescriptionConstSharedPtr upstream_host_{};
   Network::Address::InstanceConstSharedPtr upstream_local_address_;
   UpstreamTiming upstream_timing_;
@@ -88,6 +91,7 @@ struct UpstreamInfoImpl : public UpstreamInfo {
   std::string upstream_transport_failure_reason_;
   FilterStateSharedPtr upstream_filter_state_;
   size_t num_streams_{};
+  absl::optional<Http::Protocol> upstream_protocol_;
 };
 
 struct StreamInfoImpl : public StreamInfo {