From df6d49f8dc553dd742a1f38a4f91bf2d0a0b3d51 Mon Sep 17 00:00:00 2001 From: grial Date: Sun, 25 Oct 2020 22:39:27 +0100 Subject: [PATCH 001/106] Macro 'SAFE_MEMCPY added according to comments on issue#9328' Signed-off-by: grial --- source/common/common/macros.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index f2b06b84f340b..c989714fba2a9 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -2,6 +2,15 @@ namespace Envoy { +/** + * Assert memory bounds to avoid copy errors. +*/ +#define SAFE_MEMCPY(dst, src) \ + do { \ + static_assert(sizeof(*(src)) == sizeof(*(dst))); \ + memcpy(dst, src, sizeof(*(src))); \ + } while (0) + /** * @return the size of a C array. */ From 6a9d849c983a9cab6596e7561998f251b65223e1 Mon Sep 17 00:00:00 2001 From: grial Date: Tue, 27 Oct 2020 01:08:38 +0100 Subject: [PATCH 002/106] WIP: SAFE_MEMCPY modified with memmove, and some minor changes made. Signed-off-by: grial --- source/common/common/macros.h | 4 ++-- .../filters/listener/proxy_protocol/proxy_protocol.cc | 4 ++-- source/extensions/filters/udp/dns_filter/dns_parser.cc | 4 ++-- source/server/hot_restarting_base.cc | 6 +++++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index c989714fba2a9..f36e8a51f2687 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -7,8 +7,8 @@ namespace Envoy { */ #define SAFE_MEMCPY(dst, src) \ do { \ - static_assert(sizeof(*(src)) == sizeof(*(dst))); \ - memcpy(dst, src, sizeof(*(src))); \ + static_assert(src != nullptr && dst != nullptr); \ + memmove(dst, src, std::min(sizeof(*(src)), sizeof(*(dst)))); \ } while (0) /** diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index d50117b3df716..7820499a29e35 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -209,11 +209,11 @@ void Filter::parseV2Header(char* buf) { memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; ra6.sin6_port = v6->src_port; - memcpy(ra6.sin6_addr.s6_addr, v6->src_addr, sizeof(ra6.sin6_addr.s6_addr)); + SAFE_MEMCPY(ra6.sin6_addr.s6_addr, v6->src_addr); la6.sin6_family = AF_INET6; la6.sin6_port = v6->dst_port; - memcpy(la6.sin6_addr.s6_addr, v6->dst_addr, sizeof(la6.sin6_addr.s6_addr)); + SAFE_MEMCPY(la6.sin6_addr.s6_addr, v6->dst_addr); proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 763424a63324b..c3b442632bd29 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - ::memcpy(static_cast(&header_.flags), &data, field_size); + SAFE_MEMCPY(&header_.flags, &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -741,7 +741,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(response_header_.id); uint16_t flags; - ::memcpy(&flags, static_cast(&response_header_.flags), sizeof(uint16_t)); + SAFE_MEMCPY(&flags, &response_header_.flags); buffer.writeBEInt(flags); buffer.writeBEInt(response_header_.questions); diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 5cf2f8378a6d1..5c3cd5f7d03b3 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -1,6 +1,7 @@ #include "server/hot_restarting_base.h" #include "common/api/os_sys_calls_impl.h" +#include "common/common/mem_block_builder.h" #include "common/common/utility.h" #include "common/network/address_impl.h" #include "common/stats/utility.h" @@ -36,8 +37,11 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - memcpy(&address, addr.sockAddr(), addr.sockAddrLen()); + MemBlockBuilder mem_builder(addr.sockAddrLen()); + mem_builder.appendOne(addr.sockAddr()); + SAFE_MEMCPY(&address, mem_builder.release()); fchmod(my_domain_socket_, socket_mode); + return address; } From 5a29b9a858c604fe993ed727120fe5b12ed2d5dc Mon Sep 17 00:00:00 2001 From: grial Date: Tue, 27 Oct 2020 09:44:22 +0100 Subject: [PATCH 003/106] macros.h format fixed Signed-off-by: grial --- source/common/common/macros.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index f36e8a51f2687..4aab59d83374a 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -5,15 +5,15 @@ namespace Envoy { /** * Assert memory bounds to avoid copy errors. */ -#define SAFE_MEMCPY(dst, src) \ - do { \ - static_assert(src != nullptr && dst != nullptr); \ - memmove(dst, src, std::min(sizeof(*(src)), sizeof(*(dst)))); \ +#define SAFE_MEMCPY(dst, src) \ + do { \ + static_assert(src != nullptr && dst != nullptr); \ + memmove(dst, src, std::min(sizeof(*(src)), sizeof(*(dst)))); \ } while (0) /** * @return the size of a C array. - */ +*/ #define ARRAY_SIZE(X) (sizeof(X) / sizeof(X[0])) /** From 814894b9072a075dd00504b63c467e955dabe564 Mon Sep 17 00:00:00 2001 From: grial Date: Tue, 27 Oct 2020 22:19:01 +0100 Subject: [PATCH 004/106] testing CI pipeline Signed-off-by: grial --- source/common/common/macros.h | 4 ++-- source/server/hot_restarting_base.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index 4aab59d83374a..7e9e2b1559604 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -4,7 +4,7 @@ namespace Envoy { /** * Assert memory bounds to avoid copy errors. -*/ + */ #define SAFE_MEMCPY(dst, src) \ do { \ static_assert(src != nullptr && dst != nullptr); \ @@ -13,7 +13,7 @@ namespace Envoy { /** * @return the size of a C array. -*/ + */ #define ARRAY_SIZE(X) (sizeof(X) / sizeof(X[0])) /** diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 5c3cd5f7d03b3..c33970c2a423e 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -38,7 +38,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); MemBlockBuilder mem_builder(addr.sockAddrLen()); - mem_builder.appendOne(addr.sockAddr()); + mem_builder.appendOne(addr.sockAddr()); SAFE_MEMCPY(&address, mem_builder.release()); fchmod(my_domain_socket_, socket_mode); From 45565c22e56940ebff128d2e40c33d2038a49748 Mon Sep 17 00:00:00 2001 From: grial Date: Wed, 28 Oct 2020 00:02:01 +0100 Subject: [PATCH 005/106] static_assert deleted from macro SAFE_MEMCPY to avoid troubles with reinterpret_cast Signed-off-by: grial --- source/common/common/macros.h | 1 - .../filters/network/mongo_proxy/bson_impl.cc | 13 ++++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index 7e9e2b1559604..9318e60923a09 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -7,7 +7,6 @@ namespace Envoy { */ #define SAFE_MEMCPY(dst, src) \ do { \ - static_assert(src != nullptr && dst != nullptr); \ memmove(dst, src, std::min(sizeof(*(src)), sizeof(*(dst)))); \ } while (0) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 4d76387626469..630dd6ebe2a87 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -8,6 +8,7 @@ #include "common/common/byte_order.h" #include "common/common/fmt.h" #include "common/common/hex.h" +#include "common/common/mem_block_builder.h" #include "common/common/utility.h" namespace Envoy { @@ -22,8 +23,7 @@ int32_t BufferHelper::peekInt32(Buffer::Instance& data) { } int32_t val; - void* mem = data.linearize(sizeof(int32_t)); - std::memcpy(reinterpret_cast(&val), mem, sizeof(int32_t)); + SAFE_MEMCPY(&val, data.linearize(sizeof(int32_t))); return le32toh(val); } @@ -43,8 +43,9 @@ void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_ throw EnvoyException("invalid buffer size"); } - void* mem = data.linearize(out_len); - std::memcpy(out, mem, out_len); + MemBlockBuilder mem_builder(out_len); + mem_builder.appendData(data.linearize(out_len)); + out = mem_builder.release().get(); data.drain(out_len); } @@ -87,9 +88,7 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { throw EnvoyException("invalid buffer size"); } - int64_t val; - void* mem = data.linearize(sizeof(int64_t)); - std::memcpy(reinterpret_cast(&val), mem, sizeof(int64_t)); + SAFE_MEMCPY(&val, data.linearize(sizeof(int64_t))); data.drain(sizeof(int64_t)); return le64toh(val); } From 2ad12d4a68c89aa0048473a8d6d25244ee47f4e4 Mon Sep 17 00:00:00 2001 From: grial Date: Wed, 28 Oct 2020 09:45:08 +0100 Subject: [PATCH 006/106] list of dependencies changed Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index 2e281e1f67896..7fcc745155268 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -34,6 +34,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:byte_order_lib", "//source/common/common:hex_lib", + "//source/common/common/mem_block_builder_lib", "//source/common/common:minimal_logger_lib", "//source/common/common:utility_lib", ], From 22d2bfac4617c9cb178b356ba3d2c0e363da08cc Mon Sep 17 00:00:00 2001 From: grial Date: Wed, 28 Oct 2020 13:42:49 +0100 Subject: [PATCH 007/106] label corrected in .../mongo_proxy/BUILD Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index 7fcc745155268..0a300779f2679 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -34,7 +34,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:byte_order_lib", "//source/common/common:hex_lib", - "//source/common/common/mem_block_builder_lib", + "//source/common/common:mem_block_builder_lib", "//source/common/common:minimal_logger_lib", "//source/common/common:utility_lib", ], From 9d72d1125c8ef5f64d57afcc3b1b86969ccb30c1 Mon Sep 17 00:00:00 2001 From: grial Date: Wed, 28 Oct 2020 15:59:49 +0100 Subject: [PATCH 008/106] casts added and deleted var val included Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/bson_impl.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 630dd6ebe2a87..c78873264d8c2 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -23,7 +23,7 @@ int32_t BufferHelper::peekInt32(Buffer::Instance& data) { } int32_t val; - SAFE_MEMCPY(&val, data.linearize(sizeof(int32_t))); + SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int32_t)))); return le32toh(val); } @@ -87,8 +87,9 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { if (data.length() < sizeof(int64_t)) { throw EnvoyException("invalid buffer size"); } - - SAFE_MEMCPY(&val, data.linearize(sizeof(int64_t))); + + int64_t val; + SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int64_t)))); data.drain(sizeof(int64_t)); return le64toh(val); } From 1b43b5360a603c5025f0f4f14e6787164bcea5b0 Mon Sep 17 00:00:00 2001 From: grial Date: Wed, 28 Oct 2020 16:40:44 +0100 Subject: [PATCH 009/106] format corrected Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/bson_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index c78873264d8c2..05c7393a262e0 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -87,7 +87,7 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { if (data.length() < sizeof(int64_t)) { throw EnvoyException("invalid buffer size"); } - + int64_t val; SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int64_t)))); data.drain(sizeof(int64_t)); From 0aab7bc28951f3a5a8fc2d23566c83d5c2024695 Mon Sep 17 00:00:00 2001 From: grial Date: Wed, 28 Oct 2020 17:50:36 +0100 Subject: [PATCH 010/106] cast added to avoid using void* in MemBlockBuilder Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/bson_impl.cc | 2 +- source/server/BUILD | 1 + source/server/hot_restarting_base.cc | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 05c7393a262e0..6e698ab80703d 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -44,7 +44,7 @@ void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_ } MemBlockBuilder mem_builder(out_len); - mem_builder.appendData(data.linearize(out_len)); + mem_builder.appendData(static_cast(data.linearize(out_len))); out = mem_builder.release().get(); data.drain(out_len); } diff --git a/source/server/BUILD b/source/server/BUILD index 852cc2bda6045..42b94595aabd6 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -153,6 +153,7 @@ envoy_cc_library( "//include/envoy/stats:stats_interface", "//source/common/api:os_sys_calls_lib", "//source/common/common:assert_lib", + "//source/common/common:mem_block_builder_lib", "//source/common/common:utility_lib", "//source/common/network:utility_lib", "//source/common/stats:utility_lib", diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index c33970c2a423e..93ce63fe4c5c5 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -37,9 +37,9 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - MemBlockBuilder mem_builder(addr.sockAddrLen()); - mem_builder.appendOne(addr.sockAddr()); - SAFE_MEMCPY(&address, mem_builder.release()); + MemBlockBuilder mem_builder(addr.sockAddrLen()); + mem_builder.appendData(addr.sockAddr()); + SAFE_MEMCPY(&address, mem_builder.release().get()); fchmod(my_domain_socket_, socket_mode); return address; From d34e261dcb4ea5721cd4295013587dd1742365ad Mon Sep 17 00:00:00 2001 From: grial Date: Wed, 28 Oct 2020 23:10:52 +0100 Subject: [PATCH 011/106] Initializing Span before calling appendData Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/bson_impl.cc | 2 +- source/server/hot_restarting_base.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 6e698ab80703d..564dd793b9b93 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -44,7 +44,7 @@ void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_ } MemBlockBuilder mem_builder(out_len); - mem_builder.appendData(static_cast(data.linearize(out_len))); + mem_builder.appendData(absl::Span(static_cast(data.linearize(out_len)))); out = mem_builder.release().get(); data.drain(out_len); } diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 93ce63fe4c5c5..dad900f0d16f4 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -38,7 +38,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); MemBlockBuilder mem_builder(addr.sockAddrLen()); - mem_builder.appendData(addr.sockAddr()); + mem_builder.appendData(absl::Span(addr.sockAddr())); SAFE_MEMCPY(&address, mem_builder.release().get()); fchmod(my_domain_socket_, socket_mode); From 6e3a2571c929a31d19969eb4042c7396e26616d5 Mon Sep 17 00:00:00 2001 From: grial Date: Thu, 29 Oct 2020 09:55:48 +0100 Subject: [PATCH 012/106] Span constructor corrected Signed-off-by: grial --- .../filters/network/mongo_proxy/bson_impl.cc | 583 ------------------ source/server/hot_restarting_base.cc | 252 -------- 2 files changed, 835 deletions(-) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 564dd793b9b93..e69de29bb2d1d 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -1,583 +0,0 @@ -#include "extensions/filters/network/mongo_proxy/bson_impl.h" - -#include -#include -#include - -#include "common/common/assert.h" -#include "common/common/byte_order.h" -#include "common/common/fmt.h" -#include "common/common/hex.h" -#include "common/common/mem_block_builder.h" -#include "common/common/utility.h" - -namespace Envoy { -namespace Extensions { -namespace NetworkFilters { -namespace MongoProxy { -namespace Bson { - -int32_t BufferHelper::peekInt32(Buffer::Instance& data) { - if (data.length() < sizeof(int32_t)) { - throw EnvoyException("invalid buffer size"); - } - - int32_t val; - SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int32_t)))); - return le32toh(val); -} - -uint8_t BufferHelper::removeByte(Buffer::Instance& data) { - if (data.length() == 0) { - throw EnvoyException("invalid buffer size"); - } - - void* mem = data.linearize(sizeof(uint8_t)); - uint8_t ret = *reinterpret_cast(mem); - data.drain(sizeof(uint8_t)); - return ret; -} - -void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_len) { - if (data.length() < out_len) { - throw EnvoyException("invalid buffer size"); - } - - MemBlockBuilder mem_builder(out_len); - mem_builder.appendData(absl::Span(static_cast(data.linearize(out_len)))); - out = mem_builder.release().get(); - data.drain(out_len); -} - -std::string BufferHelper::removeCString(Buffer::Instance& data) { - char end = '\0'; - ssize_t index = data.search(&end, sizeof(end), 0); - if (index == -1) { - throw EnvoyException("invalid CString"); - } - - char* start = reinterpret_cast(data.linearize(index + 1)); - std::string ret(start); - data.drain(index + 1); - return ret; -} - -double BufferHelper::removeDouble(Buffer::Instance& data) { - ASSERT(sizeof(double) == 8); - - // There is not really official endian support for floating point so we unpack an 8 byte integer - // into a union with a double. - union { - int64_t i; - double d; - } memory; - - static_assert(sizeof(memory.i) == sizeof(memory.d), "invalid type size"); - memory.i = removeInt64(data); - return memory.d; -} - -int32_t BufferHelper::removeInt32(Buffer::Instance& data) { - int32_t ret = peekInt32(data); - data.drain(sizeof(int32_t)); - return ret; -} - -int64_t BufferHelper::removeInt64(Buffer::Instance& data) { - if (data.length() < sizeof(int64_t)) { - throw EnvoyException("invalid buffer size"); - } - - int64_t val; - SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int64_t)))); - data.drain(sizeof(int64_t)); - return le64toh(val); -} - -std::string BufferHelper::removeString(Buffer::Instance& data) { - int32_t length = removeInt32(data); - if (static_cast(length) > data.length()) { - throw EnvoyException("invalid buffer size"); - } - - char* start = reinterpret_cast(data.linearize(length)); - std::string ret(start); - data.drain(length); - return ret; -} - -std::string BufferHelper::removeBinary(Buffer::Instance& data) { - // Read out the subtype but do not store it for now. - int32_t length = removeInt32(data); - removeByte(data); - if (static_cast(length) > data.length()) { - throw EnvoyException("invalid buffer size"); - } - - char* start = reinterpret_cast(data.linearize(length)); - std::string ret(start, length); - data.drain(length); - return ret; -} - -void BufferHelper::writeCString(Buffer::Instance& data, const std::string& value) { - data.add(value.c_str(), value.size() + 1); -} - -void BufferHelper::writeDouble(Buffer::Instance& data, double value) { - // We need to hack converting a double into little endian. - int64_t* to_write = reinterpret_cast(&value); - writeInt64(data, *to_write); -} - -void BufferHelper::writeInt32(Buffer::Instance& data, int32_t value) { - value = htole32(value); - data.add(&value, sizeof(value)); -} - -void BufferHelper::writeInt64(Buffer::Instance& data, int64_t value) { - value = htole64(value); - data.add(&value, sizeof(value)); -} - -void BufferHelper::writeString(Buffer::Instance& data, const std::string& value) { - writeInt32(data, value.size() + 1); - data.add(value.c_str(), value.size() + 1); -} - -void BufferHelper::writeBinary(Buffer::Instance& data, const std::string& value) { - // Right now we do not actually store the binary subtype and always use zero. - writeInt32(data, value.size()); - uint8_t subtype = 0; - data.add(&subtype, sizeof(subtype)); - data.add(value.c_str(), value.size()); -} - -int32_t FieldImpl::byteSize() const { - // 1 byte type, cstring key, field. - int32_t total = 1 + key_.size() + 1; - - switch (type_) { - case Type::Double: - case Type::Datetime: - case Type::Timestamp: - case Type::Int64: { - return total + 8; - } - - case Type::String: - case Type::Symbol: { - return total + 4 + value_.string_value_.size() + 1; - } - - case Type::Document: - case Type::Array: { - return total + value_.document_value_->byteSize(); - } - - case Type::Binary: { - return total + 5 + value_.string_value_.size(); - } - - case Type::ObjectId: { - return total + sizeof(ObjectId); - } - - case Type::Boolean: { - return total + 1; - } - - case Type::NullValue: { - return total; - } - - case Type::Regex: { - return total + value_.regex_value_.pattern_.size() + value_.regex_value_.options_.size() + 2; - } - - case Type::Int32: { - return total + 4; - } - } - - NOT_REACHED_GCOVR_EXCL_LINE; -} - -void FieldImpl::encode(Buffer::Instance& output) const { - output.add(&type_, sizeof(type_)); - BufferHelper::writeCString(output, key_); - - switch (type_) { - case Type::Double: { - return BufferHelper::writeDouble(output, value_.double_value_); - } - - case Type::String: - case Type::Symbol: { - return BufferHelper::writeString(output, value_.string_value_); - } - - case Type::Document: - case Type::Array: { - return value_.document_value_->encode(output); - } - - case Type::Binary: { - return BufferHelper::writeBinary(output, value_.string_value_); - } - - case Type::ObjectId: { - return output.add(&value_.object_id_value_[0], value_.object_id_value_.size()); - } - - case Type::Boolean: { - uint8_t to_write = value_.bool_value_ ? 1 : 0; - return output.add(&to_write, sizeof(to_write)); - } - - case Type::Datetime: - case Type::Timestamp: - case Type::Int64: { - return BufferHelper::writeInt64(output, value_.int64_value_); - } - - case Type::NullValue: { - return; - } - - case Type::Regex: { - BufferHelper::writeCString(output, value_.regex_value_.pattern_); - return BufferHelper::writeCString(output, value_.regex_value_.options_); - } - - case Type::Int32: - return BufferHelper::writeInt32(output, value_.int32_value_); - } - - NOT_REACHED_GCOVR_EXCL_LINE; -} - -bool FieldImpl::operator==(const Field& rhs) const { - if (type() != rhs.type()) { - return false; - } - - switch (type_) { - case Type::Double: { - return asDouble() == rhs.asDouble(); - } - - case Type::String: { - return asString() == rhs.asString(); - } - - case Type::Symbol: { - return asSymbol() == rhs.asSymbol(); - } - - case Type::Document: { - return asDocument() == rhs.asDocument(); - } - - case Type::Array: { - return asArray() == rhs.asArray(); - } - - case Type::Binary: { - return asBinary() == rhs.asBinary(); - } - - case Type::ObjectId: { - return asObjectId() == rhs.asObjectId(); - } - - case Type::Boolean: { - return asBoolean() == rhs.asBoolean(); - } - - case Type::NullValue: { - return true; - } - - case Type::Regex: { - return asRegex() == rhs.asRegex(); - } - - case Type::Int32: { - return asInt32() == rhs.asInt32(); - } - - case Type::Datetime: { - return asDatetime() == rhs.asDatetime(); - } - - case Type::Timestamp: { - return asTimestamp() == rhs.asTimestamp(); - } - - case Type::Int64: { - return asInt64() == rhs.asInt64(); - } - } - - NOT_REACHED_GCOVR_EXCL_LINE; -} - -std::string FieldImpl::toString() const { - switch (type_) { - case Type::Double: { - return std::to_string(value_.double_value_); - } - - case Type::String: - case Type::Symbol: - case Type::Binary: { - return fmt::format("\"{}\"", StringUtil::escape(value_.string_value_)); - } - - case Type::Document: - case Type::Array: { - return value_.document_value_->toString(); - } - - case Type::ObjectId: { - return fmt::format("\"{}\"", - Hex::encode(&value_.object_id_value_[0], value_.object_id_value_.size())); - } - - case Type::Boolean: { - return value_.bool_value_ ? "true" : "false"; - } - - case Type::NullValue: { - return "null"; - } - - case Type::Regex: { - return fmt::format("[\"{}\", \"{}\"]", value_.regex_value_.pattern_, - value_.regex_value_.options_); - } - - case Type::Int32: { - return std::to_string(value_.int32_value_); - } - - case Type::Datetime: - case Type::Timestamp: - case Type::Int64: { - return std::to_string(value_.int64_value_); - } - } - - NOT_REACHED_GCOVR_EXCL_LINE; -} - -void DocumentImpl::fromBuffer(Buffer::Instance& data) { - uint64_t original_buffer_length = data.length(); - int32_t message_length = BufferHelper::removeInt32(data); - if (static_cast(message_length) > original_buffer_length) { - throw EnvoyException("invalid BSON message length"); - } - - ENVOY_LOG(trace, "BSON document length: {} data length: {}", message_length, - original_buffer_length); - - while (true) { - uint64_t document_bytes_remaining = data.length() - (original_buffer_length - message_length); - ENVOY_LOG(trace, "BSON document bytes remaining: {}", document_bytes_remaining); - if (document_bytes_remaining == 1) { - uint8_t last_byte = BufferHelper::removeByte(data); - if (last_byte != 0) { - throw EnvoyException("invalid document"); - } - - return; - } - - uint8_t element_type = BufferHelper::removeByte(data); - std::string key = BufferHelper::removeCString(data); - ENVOY_LOG(trace, "BSON element type: {:#x} key: {}", element_type, key); - switch (static_cast(element_type)) { - case Field::Type::Double: { - double value = BufferHelper::removeDouble(data); - ENVOY_LOG(trace, "BSON double: {}", value); - addDouble(key, value); - break; - } - - case Field::Type::String: { - std::string value = BufferHelper::removeString(data); - ENVOY_LOG(trace, "BSON string: {}", value); - addString(key, std::move(value)); - break; - } - - case Field::Type::Symbol: { - std::string value = BufferHelper::removeString(data); - ENVOY_LOG(trace, "BSON symbol: {}", value); - addSymbol(key, std::move(value)); - break; - } - - case Field::Type::Document: { - ENVOY_LOG(trace, "BSON document"); - addDocument(key, DocumentImpl::create(data)); - break; - } - - case Field::Type::Array: { - ENVOY_LOG(trace, "BSON array"); - addArray(key, DocumentImpl::create(data)); - break; - } - - case Field::Type::Binary: { - std::string value = BufferHelper::removeBinary(data); - ENVOY_LOG(trace, "BSON binary: {}", value); - addBinary(key, std::move(value)); - break; - } - - case Field::Type::ObjectId: { - Field::ObjectId value; - BufferHelper::removeBytes(data, &value[0], value.size()); - addObjectId(key, std::move(value)); - break; - } - - case Field::Type::Boolean: { - const bool value = BufferHelper::removeByte(data) != 0; - ENVOY_LOG(trace, "BSON boolean: {}", value); - addBoolean(key, value); - break; - } - - case Field::Type::Datetime: { - const int64_t value = BufferHelper::removeInt64(data); - ENVOY_LOG(trace, "BSON datetime: {}", value); - addDatetime(key, value); - break; - } - - case Field::Type::NullValue: { - ENVOY_LOG(trace, "BSON null value"); - addNull(key); - break; - } - - case Field::Type::Regex: { - Field::Regex value; - value.pattern_ = BufferHelper::removeCString(data); - value.options_ = BufferHelper::removeCString(data); - ENVOY_LOG(trace, "BSON regex pattern: {} options: {}", value.pattern_, value.options_); - addRegex(key, std::move(value)); - break; - } - - case Field::Type::Int32: { - int32_t value = BufferHelper::removeInt32(data); - ENVOY_LOG(trace, "BSON int32: {}", value); - addInt32(key, value); - break; - } - - case Field::Type::Timestamp: { - int64_t value = BufferHelper::removeInt64(data); - ENVOY_LOG(trace, "BSON timestamp: {}", value); - addTimestamp(key, value); - break; - } - - case Field::Type::Int64: { - int64_t value = BufferHelper::removeInt64(data); - ENVOY_LOG(trace, "BSON int64: {}", value); - addInt64(key, value); - break; - } - - default: - throw EnvoyException( - fmt::format("invalid BSON element type: {:#x} key: {}", element_type, key)); - } - } -} - -int32_t DocumentImpl::byteSize() const { - // Minimum size is 5. - int32_t total_size = sizeof(int32_t) + 1; - for (const FieldPtr& field : fields_) { - total_size += field->byteSize(); - } - - return total_size; -} - -void DocumentImpl::encode(Buffer::Instance& output) const { - BufferHelper::writeInt32(output, byteSize()); - for (const FieldPtr& field : fields_) { - field->encode(output); - } - - uint8_t done = 0; - output.add(&done, sizeof(done)); -} - -bool DocumentImpl::operator==(const Document& rhs) const { - if (values().size() != rhs.values().size()) { - return false; - } - - for (auto i1 = values().begin(), i2 = rhs.values().begin(); i1 != values().end(); i1++, i2++) { - if (**i1 == **i2) { - continue; - } - - return false; - } - - return true; -} - -std::string DocumentImpl::toString() const { - std::stringstream out; - out << "{"; - - bool first = true; - for (const FieldPtr& field : fields_) { - if (!first) { - out << ", "; - } - - out << fmt::format("\"{}\": {}", field->key(), field->toString()); - first = false; - } - - out << "}"; - return out.str(); -} - -const Field* DocumentImpl::find(const std::string& name) const { - for (const FieldPtr& field : fields_) { - if (field->key() == name) { - return field.get(); - } - } - - return nullptr; -} - -const Field* DocumentImpl::find(const std::string& name, Field::Type type) const { - for (const FieldPtr& field : fields_) { - if (field->key() == name && field->type() == type) { - return field.get(); - } - } - - return nullptr; -} - -} // namespace Bson -} // namespace MongoProxy -} // namespace NetworkFilters -} // namespace Extensions -} // namespace Envoy diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index dad900f0d16f4..e69de29bb2d1d 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -1,252 +0,0 @@ -#include "server/hot_restarting_base.h" - -#include "common/api/os_sys_calls_impl.h" -#include "common/common/mem_block_builder.h" -#include "common/common/utility.h" -#include "common/network/address_impl.h" -#include "common/stats/utility.h" - -namespace Envoy { -namespace Server { - -using HotRestartMessage = envoy::HotRestartMessage; - -static constexpr uint64_t MaxSendmsgSize = 4096; - -HotRestartingBase::~HotRestartingBase() { - if (my_domain_socket_ != -1) { - Api::OsSysCalls& os_sys_calls = Api::OsSysCallsSingleton::get(); - Api::SysCallIntResult result = os_sys_calls.close(my_domain_socket_); - ASSERT(result.rc_ == 0); - } -} - -void HotRestartingBase::initDomainSocketAddress(sockaddr_un* address) { - memset(address, 0, sizeof(*address)); - address->sun_family = AF_UNIX; -} - -sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std::string& role, - const std::string& socket_path, - mode_t socket_mode) { - // Right now we only allow a maximum of 3 concurrent envoy processes to be running. When the third - // starts up it will kill the oldest parent. - static constexpr uint64_t MaxConcurrentProcesses = 3; - id = id % MaxConcurrentProcesses; - sockaddr_un address; - initDomainSocketAddress(&address); - Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), - socket_mode, nullptr); - MemBlockBuilder mem_builder(addr.sockAddrLen()); - mem_builder.appendData(absl::Span(addr.sockAddr())); - SAFE_MEMCPY(&address, mem_builder.release().get()); - fchmod(my_domain_socket_, socket_mode); - - return address; -} - -void HotRestartingBase::bindDomainSocket(uint64_t id, const std::string& role, - const std::string& socket_path, mode_t socket_mode) { - Api::OsSysCalls& os_sys_calls = Api::OsSysCallsSingleton::get(); - // This actually creates the socket and binds it. We use the socket in datagram mode so we can - // easily read single messages. - my_domain_socket_ = socket(AF_UNIX, SOCK_DGRAM | SOCK_NONBLOCK, 0); - sockaddr_un address = createDomainSocketAddress(id, role, socket_path, socket_mode); - unlink(address.sun_path); - Api::SysCallIntResult result = - os_sys_calls.bind(my_domain_socket_, reinterpret_cast(&address), sizeof(address)); - if (result.rc_ != 0) { - const auto msg = fmt::format( - "unable to bind domain socket with base_id={}, id={}, errno={} (see --base-id option)", - base_id_, id, result.errno_); - if (result.errno_ == SOCKET_ERROR_ADDR_IN_USE) { - throw HotRestartDomainSocketInUseException(msg); - } - throw EnvoyException(msg); - } -} - -void HotRestartingBase::sendHotRestartMessage(sockaddr_un& address, - const HotRestartMessage& proto) { - const uint64_t serialized_size = proto.ByteSizeLong(); - const uint64_t total_size = sizeof(uint64_t) + serialized_size; - // Fill with uint64_t 'length' followed by the serialized HotRestartMessage. - std::vector send_buf; - send_buf.resize(total_size); - *reinterpret_cast(send_buf.data()) = htobe64(serialized_size); - RELEASE_ASSERT(proto.SerializeWithCachedSizesToArray(send_buf.data() + sizeof(uint64_t)), - "failed to serialize a HotRestartMessage"); - - RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, 0) != -1, - fmt::format("Set domain socket blocking failed, errno = {}", errno)); - - uint8_t* next_byte_to_send = send_buf.data(); - uint64_t sent = 0; - while (sent < total_size) { - const uint64_t cur_chunk_size = std::min(MaxSendmsgSize, total_size - sent); - iovec iov[1]; - iov[0].iov_base = next_byte_to_send; - iov[0].iov_len = cur_chunk_size; - next_byte_to_send += cur_chunk_size; - sent += cur_chunk_size; - msghdr message; - memset(&message, 0, sizeof(message)); - message.msg_name = &address; - message.msg_namelen = sizeof(address); - message.msg_iov = iov; - message.msg_iovlen = 1; - - // Control data stuff, only relevant for the fd passing done with PassListenSocketReply. - uint8_t control_buffer[CMSG_SPACE(sizeof(int))]; - if (replyIsExpectedType(&proto, HotRestartMessage::Reply::kPassListenSocket) && - proto.reply().pass_listen_socket().fd() != -1) { - memset(control_buffer, 0, CMSG_SPACE(sizeof(int))); - message.msg_control = control_buffer; - message.msg_controllen = CMSG_SPACE(sizeof(int)); - cmsghdr* control_message = CMSG_FIRSTHDR(&message); - control_message->cmsg_level = SOL_SOCKET; - control_message->cmsg_type = SCM_RIGHTS; - control_message->cmsg_len = CMSG_LEN(sizeof(int)); - *reinterpret_cast(CMSG_DATA(control_message)) = proto.reply().pass_listen_socket().fd(); - ASSERT(sent == total_size, "an fd passing message was too long for one sendmsg()."); - } - - const int rc = sendmsg(my_domain_socket_, &message, 0); - RELEASE_ASSERT(rc == static_cast(cur_chunk_size), - fmt::format("hot restart sendmsg() failed: returned {}, errno {}", rc, errno)); - } - RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, O_NONBLOCK) != -1, - fmt::format("Set domain socket nonblocking failed, errno = {}", errno)); -} - -bool HotRestartingBase::replyIsExpectedType(const HotRestartMessage* proto, - HotRestartMessage::Reply::ReplyCase oneof_type) const { - return proto != nullptr && proto->requestreply_case() == HotRestartMessage::kReply && - proto->reply().reply_case() == oneof_type; -} - -// Pull the cloned fd, if present, out of the control data and write it into the -// PassListenSocketReply proto; the higher level code will see a listening fd that Just Works. We -// should only get control data in a PassListenSocketReply, it should only be the fd passing type, -// and there should only be one at a time. Crash on any other control data. -void HotRestartingBase::getPassedFdIfPresent(HotRestartMessage* out, msghdr* message) { - cmsghdr* cmsg = CMSG_FIRSTHDR(message); - if (cmsg != nullptr) { - RELEASE_ASSERT(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS && - replyIsExpectedType(out, HotRestartMessage::Reply::kPassListenSocket), - "recvmsg() came with control data when the message's purpose was not to pass a " - "file descriptor."); - - out->mutable_reply()->mutable_pass_listen_socket()->set_fd( - *reinterpret_cast(CMSG_DATA(cmsg))); - - RELEASE_ASSERT(CMSG_NXTHDR(message, cmsg) == nullptr, - "More than one control data on a single hot restart recvmsg()."); - } -} - -// While in use, recv_buf_ is always >= MaxSendmsgSize. In between messages, it is kept empty, -// to be grown back to MaxSendmsgSize at the start of the next message. -void HotRestartingBase::initRecvBufIfNewMessage() { - if (recv_buf_.empty()) { - ASSERT(cur_msg_recvd_bytes_ == 0); - ASSERT(!expected_proto_length_.has_value()); - recv_buf_.resize(MaxSendmsgSize); - } -} - -// Must only be called when recv_buf_ contains a full proto. Returns that proto, and resets all of -// our receive-buffering state back to empty, to await a new message. -std::unique_ptr HotRestartingBase::parseProtoAndResetState() { - auto ret = std::make_unique(); - RELEASE_ASSERT( - ret->ParseFromArray(recv_buf_.data() + sizeof(uint64_t), expected_proto_length_.value()), - "failed to parse a HotRestartMessage."); - recv_buf_.resize(0); - cur_msg_recvd_bytes_ = 0; - expected_proto_length_.reset(); - return ret; -} - -std::unique_ptr HotRestartingBase::receiveHotRestartMessage(Blocking block) { - // By default the domain socket is non blocking. If we need to block, make it blocking first. - if (block == Blocking::Yes) { - RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, 0) != -1, - fmt::format("Set domain socket blocking failed, errno = {}", errno)); - } - - initRecvBufIfNewMessage(); - - iovec iov[1]; - msghdr message; - uint8_t control_buffer[CMSG_SPACE(sizeof(int))]; - std::unique_ptr ret = nullptr; - while (!ret) { - iov[0].iov_base = recv_buf_.data() + cur_msg_recvd_bytes_; - iov[0].iov_len = MaxSendmsgSize; - - // We always setup to receive an FD even though most messages do not pass one. - memset(control_buffer, 0, CMSG_SPACE(sizeof(int))); - memset(&message, 0, sizeof(message)); - message.msg_iov = iov; - message.msg_iovlen = 1; - message.msg_control = control_buffer; - message.msg_controllen = CMSG_SPACE(sizeof(int)); - - const int recvmsg_rc = recvmsg(my_domain_socket_, &message, 0); - if (block == Blocking::No && recvmsg_rc == -1 && errno == SOCKET_ERROR_AGAIN) { - return nullptr; - } - RELEASE_ASSERT(recvmsg_rc != -1, fmt::format("recvmsg() returned -1, errno = {}", errno)); - RELEASE_ASSERT(message.msg_flags == 0, - fmt::format("recvmsg() left msg_flags = {}", message.msg_flags)); - cur_msg_recvd_bytes_ += recvmsg_rc; - - // If we don't already know 'length', we're at the start of a new length+protobuf message! - if (!expected_proto_length_.has_value()) { - // We are not ok with messages so fragmented that the length doesn't even come in one piece. - RELEASE_ASSERT(recvmsg_rc >= 8, "received a brokenly tiny message fragment."); - - expected_proto_length_ = be64toh(*reinterpret_cast(recv_buf_.data())); - // Expand the buffer from its default 4096 if this message is going to be longer. - if (expected_proto_length_.value() > MaxSendmsgSize - sizeof(uint64_t)) { - recv_buf_.resize(expected_proto_length_.value() + sizeof(uint64_t)); - cur_msg_recvd_bytes_ = recvmsg_rc; - } - } - // If we have received beyond the end of the current in-flight proto, then next is misaligned. - RELEASE_ASSERT(cur_msg_recvd_bytes_ <= sizeof(uint64_t) + expected_proto_length_.value(), - "received a length+protobuf message not aligned to start of sendmsg()."); - - if (cur_msg_recvd_bytes_ == sizeof(uint64_t) + expected_proto_length_.value()) { - ret = parseProtoAndResetState(); - } - } - - // Turn non-blocking back on if we made it blocking. - if (block == Blocking::Yes) { - RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, O_NONBLOCK) != -1, - fmt::format("Set domain socket nonblocking failed, errno = {}", errno)); - } - getPassedFdIfPresent(ret.get(), &message); - return ret; -} - -Stats::Gauge& HotRestartingBase::hotRestartGeneration(Stats::Scope& scope) { - // Track the hot-restart generation. Using gauge's accumulate semantics, - // the increments will be combined across hot-restart. This may be useful - // at some point, though the main motivation for this stat is to enable - // an integration test showing that dynamic stat-names can be coalesced - // across hot-restarts. There's no other reason this particular stat-name - // needs to be created dynamically. - // - // Note also, this stat cannot currently be represented as a counter due to - // the way stats get latched on sink update. See the comment in - // InstanceUtil::flushMetricsToSinks. - return Stats::Utility::gaugeFromElements(scope, - {Stats::DynamicName("server.hot_restart_generation")}, - Stats::Gauge::ImportMode::Accumulate); -} - -} // namespace Server -} // namespace Envoy From d4b3c50ff0fe798371f227699910f6e3e31f586c Mon Sep 17 00:00:00 2001 From: grial Date: Thu, 29 Oct 2020 10:00:44 +0100 Subject: [PATCH 013/106] Revert "Span constructor corrected" This reverts commit 6e3a2571c929a31d19969eb4042c7396e26616d5. Signed-off-by: grial --- .../filters/network/mongo_proxy/bson_impl.cc | 583 ++++++++++++++++++ source/server/hot_restarting_base.cc | 252 ++++++++ 2 files changed, 835 insertions(+) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index e69de29bb2d1d..564dd793b9b93 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -0,0 +1,583 @@ +#include "extensions/filters/network/mongo_proxy/bson_impl.h" + +#include +#include +#include + +#include "common/common/assert.h" +#include "common/common/byte_order.h" +#include "common/common/fmt.h" +#include "common/common/hex.h" +#include "common/common/mem_block_builder.h" +#include "common/common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace MongoProxy { +namespace Bson { + +int32_t BufferHelper::peekInt32(Buffer::Instance& data) { + if (data.length() < sizeof(int32_t)) { + throw EnvoyException("invalid buffer size"); + } + + int32_t val; + SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int32_t)))); + return le32toh(val); +} + +uint8_t BufferHelper::removeByte(Buffer::Instance& data) { + if (data.length() == 0) { + throw EnvoyException("invalid buffer size"); + } + + void* mem = data.linearize(sizeof(uint8_t)); + uint8_t ret = *reinterpret_cast(mem); + data.drain(sizeof(uint8_t)); + return ret; +} + +void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_len) { + if (data.length() < out_len) { + throw EnvoyException("invalid buffer size"); + } + + MemBlockBuilder mem_builder(out_len); + mem_builder.appendData(absl::Span(static_cast(data.linearize(out_len)))); + out = mem_builder.release().get(); + data.drain(out_len); +} + +std::string BufferHelper::removeCString(Buffer::Instance& data) { + char end = '\0'; + ssize_t index = data.search(&end, sizeof(end), 0); + if (index == -1) { + throw EnvoyException("invalid CString"); + } + + char* start = reinterpret_cast(data.linearize(index + 1)); + std::string ret(start); + data.drain(index + 1); + return ret; +} + +double BufferHelper::removeDouble(Buffer::Instance& data) { + ASSERT(sizeof(double) == 8); + + // There is not really official endian support for floating point so we unpack an 8 byte integer + // into a union with a double. + union { + int64_t i; + double d; + } memory; + + static_assert(sizeof(memory.i) == sizeof(memory.d), "invalid type size"); + memory.i = removeInt64(data); + return memory.d; +} + +int32_t BufferHelper::removeInt32(Buffer::Instance& data) { + int32_t ret = peekInt32(data); + data.drain(sizeof(int32_t)); + return ret; +} + +int64_t BufferHelper::removeInt64(Buffer::Instance& data) { + if (data.length() < sizeof(int64_t)) { + throw EnvoyException("invalid buffer size"); + } + + int64_t val; + SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int64_t)))); + data.drain(sizeof(int64_t)); + return le64toh(val); +} + +std::string BufferHelper::removeString(Buffer::Instance& data) { + int32_t length = removeInt32(data); + if (static_cast(length) > data.length()) { + throw EnvoyException("invalid buffer size"); + } + + char* start = reinterpret_cast(data.linearize(length)); + std::string ret(start); + data.drain(length); + return ret; +} + +std::string BufferHelper::removeBinary(Buffer::Instance& data) { + // Read out the subtype but do not store it for now. + int32_t length = removeInt32(data); + removeByte(data); + if (static_cast(length) > data.length()) { + throw EnvoyException("invalid buffer size"); + } + + char* start = reinterpret_cast(data.linearize(length)); + std::string ret(start, length); + data.drain(length); + return ret; +} + +void BufferHelper::writeCString(Buffer::Instance& data, const std::string& value) { + data.add(value.c_str(), value.size() + 1); +} + +void BufferHelper::writeDouble(Buffer::Instance& data, double value) { + // We need to hack converting a double into little endian. + int64_t* to_write = reinterpret_cast(&value); + writeInt64(data, *to_write); +} + +void BufferHelper::writeInt32(Buffer::Instance& data, int32_t value) { + value = htole32(value); + data.add(&value, sizeof(value)); +} + +void BufferHelper::writeInt64(Buffer::Instance& data, int64_t value) { + value = htole64(value); + data.add(&value, sizeof(value)); +} + +void BufferHelper::writeString(Buffer::Instance& data, const std::string& value) { + writeInt32(data, value.size() + 1); + data.add(value.c_str(), value.size() + 1); +} + +void BufferHelper::writeBinary(Buffer::Instance& data, const std::string& value) { + // Right now we do not actually store the binary subtype and always use zero. + writeInt32(data, value.size()); + uint8_t subtype = 0; + data.add(&subtype, sizeof(subtype)); + data.add(value.c_str(), value.size()); +} + +int32_t FieldImpl::byteSize() const { + // 1 byte type, cstring key, field. + int32_t total = 1 + key_.size() + 1; + + switch (type_) { + case Type::Double: + case Type::Datetime: + case Type::Timestamp: + case Type::Int64: { + return total + 8; + } + + case Type::String: + case Type::Symbol: { + return total + 4 + value_.string_value_.size() + 1; + } + + case Type::Document: + case Type::Array: { + return total + value_.document_value_->byteSize(); + } + + case Type::Binary: { + return total + 5 + value_.string_value_.size(); + } + + case Type::ObjectId: { + return total + sizeof(ObjectId); + } + + case Type::Boolean: { + return total + 1; + } + + case Type::NullValue: { + return total; + } + + case Type::Regex: { + return total + value_.regex_value_.pattern_.size() + value_.regex_value_.options_.size() + 2; + } + + case Type::Int32: { + return total + 4; + } + } + + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void FieldImpl::encode(Buffer::Instance& output) const { + output.add(&type_, sizeof(type_)); + BufferHelper::writeCString(output, key_); + + switch (type_) { + case Type::Double: { + return BufferHelper::writeDouble(output, value_.double_value_); + } + + case Type::String: + case Type::Symbol: { + return BufferHelper::writeString(output, value_.string_value_); + } + + case Type::Document: + case Type::Array: { + return value_.document_value_->encode(output); + } + + case Type::Binary: { + return BufferHelper::writeBinary(output, value_.string_value_); + } + + case Type::ObjectId: { + return output.add(&value_.object_id_value_[0], value_.object_id_value_.size()); + } + + case Type::Boolean: { + uint8_t to_write = value_.bool_value_ ? 1 : 0; + return output.add(&to_write, sizeof(to_write)); + } + + case Type::Datetime: + case Type::Timestamp: + case Type::Int64: { + return BufferHelper::writeInt64(output, value_.int64_value_); + } + + case Type::NullValue: { + return; + } + + case Type::Regex: { + BufferHelper::writeCString(output, value_.regex_value_.pattern_); + return BufferHelper::writeCString(output, value_.regex_value_.options_); + } + + case Type::Int32: + return BufferHelper::writeInt32(output, value_.int32_value_); + } + + NOT_REACHED_GCOVR_EXCL_LINE; +} + +bool FieldImpl::operator==(const Field& rhs) const { + if (type() != rhs.type()) { + return false; + } + + switch (type_) { + case Type::Double: { + return asDouble() == rhs.asDouble(); + } + + case Type::String: { + return asString() == rhs.asString(); + } + + case Type::Symbol: { + return asSymbol() == rhs.asSymbol(); + } + + case Type::Document: { + return asDocument() == rhs.asDocument(); + } + + case Type::Array: { + return asArray() == rhs.asArray(); + } + + case Type::Binary: { + return asBinary() == rhs.asBinary(); + } + + case Type::ObjectId: { + return asObjectId() == rhs.asObjectId(); + } + + case Type::Boolean: { + return asBoolean() == rhs.asBoolean(); + } + + case Type::NullValue: { + return true; + } + + case Type::Regex: { + return asRegex() == rhs.asRegex(); + } + + case Type::Int32: { + return asInt32() == rhs.asInt32(); + } + + case Type::Datetime: { + return asDatetime() == rhs.asDatetime(); + } + + case Type::Timestamp: { + return asTimestamp() == rhs.asTimestamp(); + } + + case Type::Int64: { + return asInt64() == rhs.asInt64(); + } + } + + NOT_REACHED_GCOVR_EXCL_LINE; +} + +std::string FieldImpl::toString() const { + switch (type_) { + case Type::Double: { + return std::to_string(value_.double_value_); + } + + case Type::String: + case Type::Symbol: + case Type::Binary: { + return fmt::format("\"{}\"", StringUtil::escape(value_.string_value_)); + } + + case Type::Document: + case Type::Array: { + return value_.document_value_->toString(); + } + + case Type::ObjectId: { + return fmt::format("\"{}\"", + Hex::encode(&value_.object_id_value_[0], value_.object_id_value_.size())); + } + + case Type::Boolean: { + return value_.bool_value_ ? "true" : "false"; + } + + case Type::NullValue: { + return "null"; + } + + case Type::Regex: { + return fmt::format("[\"{}\", \"{}\"]", value_.regex_value_.pattern_, + value_.regex_value_.options_); + } + + case Type::Int32: { + return std::to_string(value_.int32_value_); + } + + case Type::Datetime: + case Type::Timestamp: + case Type::Int64: { + return std::to_string(value_.int64_value_); + } + } + + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void DocumentImpl::fromBuffer(Buffer::Instance& data) { + uint64_t original_buffer_length = data.length(); + int32_t message_length = BufferHelper::removeInt32(data); + if (static_cast(message_length) > original_buffer_length) { + throw EnvoyException("invalid BSON message length"); + } + + ENVOY_LOG(trace, "BSON document length: {} data length: {}", message_length, + original_buffer_length); + + while (true) { + uint64_t document_bytes_remaining = data.length() - (original_buffer_length - message_length); + ENVOY_LOG(trace, "BSON document bytes remaining: {}", document_bytes_remaining); + if (document_bytes_remaining == 1) { + uint8_t last_byte = BufferHelper::removeByte(data); + if (last_byte != 0) { + throw EnvoyException("invalid document"); + } + + return; + } + + uint8_t element_type = BufferHelper::removeByte(data); + std::string key = BufferHelper::removeCString(data); + ENVOY_LOG(trace, "BSON element type: {:#x} key: {}", element_type, key); + switch (static_cast(element_type)) { + case Field::Type::Double: { + double value = BufferHelper::removeDouble(data); + ENVOY_LOG(trace, "BSON double: {}", value); + addDouble(key, value); + break; + } + + case Field::Type::String: { + std::string value = BufferHelper::removeString(data); + ENVOY_LOG(trace, "BSON string: {}", value); + addString(key, std::move(value)); + break; + } + + case Field::Type::Symbol: { + std::string value = BufferHelper::removeString(data); + ENVOY_LOG(trace, "BSON symbol: {}", value); + addSymbol(key, std::move(value)); + break; + } + + case Field::Type::Document: { + ENVOY_LOG(trace, "BSON document"); + addDocument(key, DocumentImpl::create(data)); + break; + } + + case Field::Type::Array: { + ENVOY_LOG(trace, "BSON array"); + addArray(key, DocumentImpl::create(data)); + break; + } + + case Field::Type::Binary: { + std::string value = BufferHelper::removeBinary(data); + ENVOY_LOG(trace, "BSON binary: {}", value); + addBinary(key, std::move(value)); + break; + } + + case Field::Type::ObjectId: { + Field::ObjectId value; + BufferHelper::removeBytes(data, &value[0], value.size()); + addObjectId(key, std::move(value)); + break; + } + + case Field::Type::Boolean: { + const bool value = BufferHelper::removeByte(data) != 0; + ENVOY_LOG(trace, "BSON boolean: {}", value); + addBoolean(key, value); + break; + } + + case Field::Type::Datetime: { + const int64_t value = BufferHelper::removeInt64(data); + ENVOY_LOG(trace, "BSON datetime: {}", value); + addDatetime(key, value); + break; + } + + case Field::Type::NullValue: { + ENVOY_LOG(trace, "BSON null value"); + addNull(key); + break; + } + + case Field::Type::Regex: { + Field::Regex value; + value.pattern_ = BufferHelper::removeCString(data); + value.options_ = BufferHelper::removeCString(data); + ENVOY_LOG(trace, "BSON regex pattern: {} options: {}", value.pattern_, value.options_); + addRegex(key, std::move(value)); + break; + } + + case Field::Type::Int32: { + int32_t value = BufferHelper::removeInt32(data); + ENVOY_LOG(trace, "BSON int32: {}", value); + addInt32(key, value); + break; + } + + case Field::Type::Timestamp: { + int64_t value = BufferHelper::removeInt64(data); + ENVOY_LOG(trace, "BSON timestamp: {}", value); + addTimestamp(key, value); + break; + } + + case Field::Type::Int64: { + int64_t value = BufferHelper::removeInt64(data); + ENVOY_LOG(trace, "BSON int64: {}", value); + addInt64(key, value); + break; + } + + default: + throw EnvoyException( + fmt::format("invalid BSON element type: {:#x} key: {}", element_type, key)); + } + } +} + +int32_t DocumentImpl::byteSize() const { + // Minimum size is 5. + int32_t total_size = sizeof(int32_t) + 1; + for (const FieldPtr& field : fields_) { + total_size += field->byteSize(); + } + + return total_size; +} + +void DocumentImpl::encode(Buffer::Instance& output) const { + BufferHelper::writeInt32(output, byteSize()); + for (const FieldPtr& field : fields_) { + field->encode(output); + } + + uint8_t done = 0; + output.add(&done, sizeof(done)); +} + +bool DocumentImpl::operator==(const Document& rhs) const { + if (values().size() != rhs.values().size()) { + return false; + } + + for (auto i1 = values().begin(), i2 = rhs.values().begin(); i1 != values().end(); i1++, i2++) { + if (**i1 == **i2) { + continue; + } + + return false; + } + + return true; +} + +std::string DocumentImpl::toString() const { + std::stringstream out; + out << "{"; + + bool first = true; + for (const FieldPtr& field : fields_) { + if (!first) { + out << ", "; + } + + out << fmt::format("\"{}\": {}", field->key(), field->toString()); + first = false; + } + + out << "}"; + return out.str(); +} + +const Field* DocumentImpl::find(const std::string& name) const { + for (const FieldPtr& field : fields_) { + if (field->key() == name) { + return field.get(); + } + } + + return nullptr; +} + +const Field* DocumentImpl::find(const std::string& name, Field::Type type) const { + for (const FieldPtr& field : fields_) { + if (field->key() == name && field->type() == type) { + return field.get(); + } + } + + return nullptr; +} + +} // namespace Bson +} // namespace MongoProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index e69de29bb2d1d..dad900f0d16f4 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -0,0 +1,252 @@ +#include "server/hot_restarting_base.h" + +#include "common/api/os_sys_calls_impl.h" +#include "common/common/mem_block_builder.h" +#include "common/common/utility.h" +#include "common/network/address_impl.h" +#include "common/stats/utility.h" + +namespace Envoy { +namespace Server { + +using HotRestartMessage = envoy::HotRestartMessage; + +static constexpr uint64_t MaxSendmsgSize = 4096; + +HotRestartingBase::~HotRestartingBase() { + if (my_domain_socket_ != -1) { + Api::OsSysCalls& os_sys_calls = Api::OsSysCallsSingleton::get(); + Api::SysCallIntResult result = os_sys_calls.close(my_domain_socket_); + ASSERT(result.rc_ == 0); + } +} + +void HotRestartingBase::initDomainSocketAddress(sockaddr_un* address) { + memset(address, 0, sizeof(*address)); + address->sun_family = AF_UNIX; +} + +sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std::string& role, + const std::string& socket_path, + mode_t socket_mode) { + // Right now we only allow a maximum of 3 concurrent envoy processes to be running. When the third + // starts up it will kill the oldest parent. + static constexpr uint64_t MaxConcurrentProcesses = 3; + id = id % MaxConcurrentProcesses; + sockaddr_un address; + initDomainSocketAddress(&address); + Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), + socket_mode, nullptr); + MemBlockBuilder mem_builder(addr.sockAddrLen()); + mem_builder.appendData(absl::Span(addr.sockAddr())); + SAFE_MEMCPY(&address, mem_builder.release().get()); + fchmod(my_domain_socket_, socket_mode); + + return address; +} + +void HotRestartingBase::bindDomainSocket(uint64_t id, const std::string& role, + const std::string& socket_path, mode_t socket_mode) { + Api::OsSysCalls& os_sys_calls = Api::OsSysCallsSingleton::get(); + // This actually creates the socket and binds it. We use the socket in datagram mode so we can + // easily read single messages. + my_domain_socket_ = socket(AF_UNIX, SOCK_DGRAM | SOCK_NONBLOCK, 0); + sockaddr_un address = createDomainSocketAddress(id, role, socket_path, socket_mode); + unlink(address.sun_path); + Api::SysCallIntResult result = + os_sys_calls.bind(my_domain_socket_, reinterpret_cast(&address), sizeof(address)); + if (result.rc_ != 0) { + const auto msg = fmt::format( + "unable to bind domain socket with base_id={}, id={}, errno={} (see --base-id option)", + base_id_, id, result.errno_); + if (result.errno_ == SOCKET_ERROR_ADDR_IN_USE) { + throw HotRestartDomainSocketInUseException(msg); + } + throw EnvoyException(msg); + } +} + +void HotRestartingBase::sendHotRestartMessage(sockaddr_un& address, + const HotRestartMessage& proto) { + const uint64_t serialized_size = proto.ByteSizeLong(); + const uint64_t total_size = sizeof(uint64_t) + serialized_size; + // Fill with uint64_t 'length' followed by the serialized HotRestartMessage. + std::vector send_buf; + send_buf.resize(total_size); + *reinterpret_cast(send_buf.data()) = htobe64(serialized_size); + RELEASE_ASSERT(proto.SerializeWithCachedSizesToArray(send_buf.data() + sizeof(uint64_t)), + "failed to serialize a HotRestartMessage"); + + RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, 0) != -1, + fmt::format("Set domain socket blocking failed, errno = {}", errno)); + + uint8_t* next_byte_to_send = send_buf.data(); + uint64_t sent = 0; + while (sent < total_size) { + const uint64_t cur_chunk_size = std::min(MaxSendmsgSize, total_size - sent); + iovec iov[1]; + iov[0].iov_base = next_byte_to_send; + iov[0].iov_len = cur_chunk_size; + next_byte_to_send += cur_chunk_size; + sent += cur_chunk_size; + msghdr message; + memset(&message, 0, sizeof(message)); + message.msg_name = &address; + message.msg_namelen = sizeof(address); + message.msg_iov = iov; + message.msg_iovlen = 1; + + // Control data stuff, only relevant for the fd passing done with PassListenSocketReply. + uint8_t control_buffer[CMSG_SPACE(sizeof(int))]; + if (replyIsExpectedType(&proto, HotRestartMessage::Reply::kPassListenSocket) && + proto.reply().pass_listen_socket().fd() != -1) { + memset(control_buffer, 0, CMSG_SPACE(sizeof(int))); + message.msg_control = control_buffer; + message.msg_controllen = CMSG_SPACE(sizeof(int)); + cmsghdr* control_message = CMSG_FIRSTHDR(&message); + control_message->cmsg_level = SOL_SOCKET; + control_message->cmsg_type = SCM_RIGHTS; + control_message->cmsg_len = CMSG_LEN(sizeof(int)); + *reinterpret_cast(CMSG_DATA(control_message)) = proto.reply().pass_listen_socket().fd(); + ASSERT(sent == total_size, "an fd passing message was too long for one sendmsg()."); + } + + const int rc = sendmsg(my_domain_socket_, &message, 0); + RELEASE_ASSERT(rc == static_cast(cur_chunk_size), + fmt::format("hot restart sendmsg() failed: returned {}, errno {}", rc, errno)); + } + RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, O_NONBLOCK) != -1, + fmt::format("Set domain socket nonblocking failed, errno = {}", errno)); +} + +bool HotRestartingBase::replyIsExpectedType(const HotRestartMessage* proto, + HotRestartMessage::Reply::ReplyCase oneof_type) const { + return proto != nullptr && proto->requestreply_case() == HotRestartMessage::kReply && + proto->reply().reply_case() == oneof_type; +} + +// Pull the cloned fd, if present, out of the control data and write it into the +// PassListenSocketReply proto; the higher level code will see a listening fd that Just Works. We +// should only get control data in a PassListenSocketReply, it should only be the fd passing type, +// and there should only be one at a time. Crash on any other control data. +void HotRestartingBase::getPassedFdIfPresent(HotRestartMessage* out, msghdr* message) { + cmsghdr* cmsg = CMSG_FIRSTHDR(message); + if (cmsg != nullptr) { + RELEASE_ASSERT(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS && + replyIsExpectedType(out, HotRestartMessage::Reply::kPassListenSocket), + "recvmsg() came with control data when the message's purpose was not to pass a " + "file descriptor."); + + out->mutable_reply()->mutable_pass_listen_socket()->set_fd( + *reinterpret_cast(CMSG_DATA(cmsg))); + + RELEASE_ASSERT(CMSG_NXTHDR(message, cmsg) == nullptr, + "More than one control data on a single hot restart recvmsg()."); + } +} + +// While in use, recv_buf_ is always >= MaxSendmsgSize. In between messages, it is kept empty, +// to be grown back to MaxSendmsgSize at the start of the next message. +void HotRestartingBase::initRecvBufIfNewMessage() { + if (recv_buf_.empty()) { + ASSERT(cur_msg_recvd_bytes_ == 0); + ASSERT(!expected_proto_length_.has_value()); + recv_buf_.resize(MaxSendmsgSize); + } +} + +// Must only be called when recv_buf_ contains a full proto. Returns that proto, and resets all of +// our receive-buffering state back to empty, to await a new message. +std::unique_ptr HotRestartingBase::parseProtoAndResetState() { + auto ret = std::make_unique(); + RELEASE_ASSERT( + ret->ParseFromArray(recv_buf_.data() + sizeof(uint64_t), expected_proto_length_.value()), + "failed to parse a HotRestartMessage."); + recv_buf_.resize(0); + cur_msg_recvd_bytes_ = 0; + expected_proto_length_.reset(); + return ret; +} + +std::unique_ptr HotRestartingBase::receiveHotRestartMessage(Blocking block) { + // By default the domain socket is non blocking. If we need to block, make it blocking first. + if (block == Blocking::Yes) { + RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, 0) != -1, + fmt::format("Set domain socket blocking failed, errno = {}", errno)); + } + + initRecvBufIfNewMessage(); + + iovec iov[1]; + msghdr message; + uint8_t control_buffer[CMSG_SPACE(sizeof(int))]; + std::unique_ptr ret = nullptr; + while (!ret) { + iov[0].iov_base = recv_buf_.data() + cur_msg_recvd_bytes_; + iov[0].iov_len = MaxSendmsgSize; + + // We always setup to receive an FD even though most messages do not pass one. + memset(control_buffer, 0, CMSG_SPACE(sizeof(int))); + memset(&message, 0, sizeof(message)); + message.msg_iov = iov; + message.msg_iovlen = 1; + message.msg_control = control_buffer; + message.msg_controllen = CMSG_SPACE(sizeof(int)); + + const int recvmsg_rc = recvmsg(my_domain_socket_, &message, 0); + if (block == Blocking::No && recvmsg_rc == -1 && errno == SOCKET_ERROR_AGAIN) { + return nullptr; + } + RELEASE_ASSERT(recvmsg_rc != -1, fmt::format("recvmsg() returned -1, errno = {}", errno)); + RELEASE_ASSERT(message.msg_flags == 0, + fmt::format("recvmsg() left msg_flags = {}", message.msg_flags)); + cur_msg_recvd_bytes_ += recvmsg_rc; + + // If we don't already know 'length', we're at the start of a new length+protobuf message! + if (!expected_proto_length_.has_value()) { + // We are not ok with messages so fragmented that the length doesn't even come in one piece. + RELEASE_ASSERT(recvmsg_rc >= 8, "received a brokenly tiny message fragment."); + + expected_proto_length_ = be64toh(*reinterpret_cast(recv_buf_.data())); + // Expand the buffer from its default 4096 if this message is going to be longer. + if (expected_proto_length_.value() > MaxSendmsgSize - sizeof(uint64_t)) { + recv_buf_.resize(expected_proto_length_.value() + sizeof(uint64_t)); + cur_msg_recvd_bytes_ = recvmsg_rc; + } + } + // If we have received beyond the end of the current in-flight proto, then next is misaligned. + RELEASE_ASSERT(cur_msg_recvd_bytes_ <= sizeof(uint64_t) + expected_proto_length_.value(), + "received a length+protobuf message not aligned to start of sendmsg()."); + + if (cur_msg_recvd_bytes_ == sizeof(uint64_t) + expected_proto_length_.value()) { + ret = parseProtoAndResetState(); + } + } + + // Turn non-blocking back on if we made it blocking. + if (block == Blocking::Yes) { + RELEASE_ASSERT(fcntl(my_domain_socket_, F_SETFL, O_NONBLOCK) != -1, + fmt::format("Set domain socket nonblocking failed, errno = {}", errno)); + } + getPassedFdIfPresent(ret.get(), &message); + return ret; +} + +Stats::Gauge& HotRestartingBase::hotRestartGeneration(Stats::Scope& scope) { + // Track the hot-restart generation. Using gauge's accumulate semantics, + // the increments will be combined across hot-restart. This may be useful + // at some point, though the main motivation for this stat is to enable + // an integration test showing that dynamic stat-names can be coalesced + // across hot-restarts. There's no other reason this particular stat-name + // needs to be created dynamically. + // + // Note also, this stat cannot currently be represented as a counter due to + // the way stats get latched on sink update. See the comment in + // InstanceUtil::flushMetricsToSinks. + return Stats::Utility::gaugeFromElements(scope, + {Stats::DynamicName("server.hot_restart_generation")}, + Stats::Gauge::ImportMode::Accumulate); +} + +} // namespace Server +} // namespace Envoy From 34d9ecbb1d116308385edc43f4cd406c95b655fc Mon Sep 17 00:00:00 2001 From: grial Date: Thu, 29 Oct 2020 10:04:22 +0100 Subject: [PATCH 014/106] Span constructor corrected Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/bson_impl.cc | 2 +- source/server/hot_restarting_base.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 564dd793b9b93..0af6ea957085e 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -44,7 +44,7 @@ void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_ } MemBlockBuilder mem_builder(out_len); - mem_builder.appendData(absl::Span(static_cast(data.linearize(out_len)))); + mem_builder.appendData(absl::Span(static_cast(data.linearize(out_len)), out_len)); out = mem_builder.release().get(); data.drain(out_len); } diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index dad900f0d16f4..862914c0e016b 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -38,7 +38,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); MemBlockBuilder mem_builder(addr.sockAddrLen()); - mem_builder.appendData(absl::Span(addr.sockAddr())); + mem_builder.appendData(absl::Span(addr.sockAddr(), addr.sockAddrLen())); SAFE_MEMCPY(&address, mem_builder.release().get()); fchmod(my_domain_socket_, socket_mode); From 50eee3cadb062c73f8c371fa017f2489c7d3a6bf Mon Sep 17 00:00:00 2001 From: grial Date: Thu, 29 Oct 2020 12:27:56 +0100 Subject: [PATCH 015/106] style corrected. Signed-off-by: grial --- source/extensions/filters/network/mongo_proxy/bson_impl.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 0af6ea957085e..4470028ca7278 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -44,7 +44,8 @@ void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_ } MemBlockBuilder mem_builder(out_len); - mem_builder.appendData(absl::Span(static_cast(data.linearize(out_len)), out_len)); + mem_builder.appendData( + absl::Span(static_cast(data.linearize(out_len)), out_len)); out = mem_builder.release().get(); data.drain(out_len); } From 1bc7916f79ffb72b98084ec92fe131d3243a9cfd Mon Sep 17 00:00:00 2001 From: grial Date: Thu, 29 Oct 2020 16:21:55 +0100 Subject: [PATCH 016/106] SAFE_MACRO modified to assert input arrays' sizes Signed-off-by: grial --- source/common/common/macros.h | 19 +++++++++++-------- source/server/hot_restarting_base.cc | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index 9318e60923a09..6e6b5a3c73854 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -1,14 +1,8 @@ #pragma once -namespace Envoy { +#include -/** - * Assert memory bounds to avoid copy errors. - */ -#define SAFE_MEMCPY(dst, src) \ - do { \ - memmove(dst, src, std::min(sizeof(*(src)), sizeof(*(dst)))); \ - } while (0) +namespace Envoy { /** * @return the size of a C array. @@ -49,6 +43,15 @@ namespace Envoy { return *objectptr; \ } while (0) +/** + * Assert memory bounds to avoid copy errors. + */ +#define SAFE_MEMCPY(dst, src) \ + do { \ + assert(ARRAY_SIZE(src) == ARRAY_SIZE(dst)); \ + memmove(dst, src, ARRAY_SIZE(src)); \ + } while (0) + /** * Have a generic fall-through for different versions of C++ */ diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 862914c0e016b..cef8617102246 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -38,7 +38,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); MemBlockBuilder mem_builder(addr.sockAddrLen()); - mem_builder.appendData(absl::Span(addr.sockAddr(), addr.sockAddrLen())); + mem_builder.appendData(absl::Span(addr.sockAddr(), addr.sockAddrLen())); SAFE_MEMCPY(&address, mem_builder.release().get()); fchmod(my_domain_socket_, socket_mode); From d68aa9e08377c3e89c77c257dce5525329ac94fe Mon Sep 17 00:00:00 2001 From: grial Date: Thu, 29 Oct 2020 22:30:18 +0100 Subject: [PATCH 017/106] Get size of the array Signed-off-by: grial --- source/common/common/macros.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index 6e6b5a3c73854..ff44f5c22f87c 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -48,8 +48,8 @@ namespace Envoy { */ #define SAFE_MEMCPY(dst, src) \ do { \ - assert(ARRAY_SIZE(src) == ARRAY_SIZE(dst)); \ - memmove(dst, src, ARRAY_SIZE(src)); \ + assert(sizeof(*(src)) == sizeof(*(dst))); \ + memmove(dst, src, sizeof(*(src))); \ } while (0) /** From 4d331cdf595947e9aaa1c08a3f9c267dbdd5443a Mon Sep 17 00:00:00 2001 From: grial Date: Fri, 30 Oct 2020 16:35:34 +0100 Subject: [PATCH 018/106] use of envoy ASSERT macro in SAFE_MEMCPY Signed-off-by: grial --- source/common/common/macros.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index ff44f5c22f87c..245b625917984 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "common/common/assert.h" namespace Envoy { @@ -48,8 +48,8 @@ namespace Envoy { */ #define SAFE_MEMCPY(dst, src) \ do { \ - assert(sizeof(*(src)) == sizeof(*(dst))); \ - memmove(dst, src, sizeof(*(src))); \ + ASSERT(sizeof(*(src)) == sizeof(*(dst))); \ + memmove(dst, src, sizeof(*(src))); \ } while (0) /** From 5372f8612bb1ffa03de8c04da08293caefe8c6be Mon Sep 17 00:00:00 2001 From: grial Date: Fri, 30 Oct 2020 19:49:09 +0100 Subject: [PATCH 019/106] Use static_assert again in SAFE_MEMCPY. Changes in proxy_protocol.cc Signed-off-by: grial --- source/common/common/macros.h | 4 +--- .../filters/listener/proxy_protocol/proxy_protocol.cc | 8 +++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index 245b625917984..251a53868dd70 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -1,7 +1,5 @@ #pragma once -#include "common/common/assert.h" - namespace Envoy { /** @@ -48,7 +46,7 @@ namespace Envoy { */ #define SAFE_MEMCPY(dst, src) \ do { \ - ASSERT(sizeof(*(src)) == sizeof(*(dst))); \ + static_assert(sizeof(*(src)) == sizeof(*(dst))); \ memmove(dst, src, sizeof(*(src))); \ } while (0) diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 7820499a29e35..7658b47dabc92 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -209,12 +209,14 @@ void Filter::parseV2Header(char* buf) { memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; ra6.sin6_port = v6->src_port; - SAFE_MEMCPY(ra6.sin6_addr.s6_addr, v6->src_addr); + for(int i = 0; i < ARRAY_SIZE(ra6.sin6_addr.s6_addr); ++i ) + ra6.sin6_addr.s6_addr[] = v6->src_addr; la6.sin6_family = AF_INET6; la6.sin6_port = v6->dst_port; - SAFE_MEMCPY(la6.sin6_addr.s6_addr, v6->dst_addr); - + for(int i = 0; i < ARRAY_SIZE(la6.sin6_addr.s6_addr); ++i ) + la6.sin6_addr.s6_addr[] = v6->dst_addr; + proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, std::make_shared(ra6), From 5517c535862017c3f137f66d2560d8cd8b944bed Mon Sep 17 00:00:00 2001 From: grial Date: Fri, 30 Oct 2020 20:33:25 +0100 Subject: [PATCH 020/106] proxy protocol header parse changed Signed-off-by: grial --- .../listener/proxy_protocol/proxy_protocol.cc | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 7658b47dabc92..338add4e22a44 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -196,27 +196,29 @@ void Filter::parseV2Header(char* buf) { std::make_shared(&la4)}); return; } else if (((proto_family & 0xf0) >> 4) == PROXY_PROTO_V2_AF_INET6) { - PACKED_STRUCT(struct pp_ipv6_addr { - uint8_t src_addr[16]; - uint8_t dst_addr[16]; - uint16_t src_port; - uint16_t dst_port; - }); - pp_ipv6_addr* v6; - v6 = reinterpret_cast(&buf[PROXY_PROTO_V2_HEADER_LEN]); + int offset = 0; // Offset to iterate over buf sockaddr_in6 ra6, la6; memset(&ra6, 0, sizeof(ra6)); memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; - ra6.sin6_port = v6->src_port; - for(int i = 0; i < ARRAY_SIZE(ra6.sin6_addr.s6_addr); ++i ) - ra6.sin6_addr.s6_addr[] = v6->src_addr; - la6.sin6_family = AF_INET6; - la6.sin6_port = v6->dst_port; - for(int i = 0; i < ARRAY_SIZE(la6.sin6_addr.s6_addr); ++i ) - la6.sin6_addr.s6_addr[] = v6->dst_addr; - + + for (; offset < ARRAY_SIZE(ra6.sin6_addr.s6_addr); ++offset) + ra6.sin6_addr.s6_addr[offset] = buf[PROXY_PROTO_V2_HEADER_LEN + offset]; + + for (; offset < ARRAY_SIZE(la6.sin6_addr.s6_addr); ++offset) + la6.sin6_addr.s6_addr[offset] = buf[PROXY_PROTO_V2_HEADER_LEN + offset]; + + PACKED_STRUCT(struct ports { + uint16_t src_port; + uint16_t dst_port; + }); + ports* port_pair; + port_pair = reinterpret_cast(&buf[PROXY_PROTO_V2_HEADER_LEN + offset]); + + ra6.sin6_port = port_pair->src_port; + la6.sin6_port = port_pair->dst_port; + proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, std::make_shared(ra6), From 455ceb79543dd146623e0fbfca1a9190583bc409 Mon Sep 17 00:00:00 2001 From: grial Date: Fri, 30 Oct 2020 21:06:16 +0100 Subject: [PATCH 021/106] getSockAddr public method added to PipeInstance Signed-off-by: grial --- source/common/network/address_impl.h | 1 + source/server/hot_restarting_base.cc | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 11aed23019522..35ff4f9e0b6a5 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -233,6 +233,7 @@ class PipeInstance : public InstanceBase { } return sizeof(pipe_.address_); } + void getSockAddr(sockaddr_un& address) { address = pipe_.address_; } private: struct PipeHelper : public Pipe { diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index cef8617102246..ce7b22da6fd16 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -37,9 +37,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - MemBlockBuilder mem_builder(addr.sockAddrLen()); - mem_builder.appendData(absl::Span(addr.sockAddr(), addr.sockAddrLen())); - SAFE_MEMCPY(&address, mem_builder.release().get()); + addr.getSockAddr(address); fchmod(my_domain_socket_, socket_mode); return address; From d188a94de0b86f102ee3793a0d918fecc617f0d6 Mon Sep 17 00:00:00 2001 From: grial Date: Sat, 31 Oct 2020 00:24:02 +0100 Subject: [PATCH 022/106] comparison corrected in for loop Signed-off-by: grial --- .../filters/listener/proxy_protocol/proxy_protocol.cc | 2 +- source/extensions/filters/udp/dns_filter/dns_parser.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 338add4e22a44..01c4089f8a1d1 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -196,7 +196,7 @@ void Filter::parseV2Header(char* buf) { std::make_shared(&la4)}); return; } else if (((proto_family & 0xf0) >> 4) == PROXY_PROTO_V2_AF_INET6) { - int offset = 0; // Offset to iterate over buf + std::size_t offset = 0; // Offset to iterate over buf sockaddr_in6 ra6, la6; memset(&ra6, 0, sizeof(ra6)); memset(&la6, 0, sizeof(la6)); diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index c3b442632bd29..c89d9f779ec7b 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - SAFE_MEMCPY(&header_.flags, &data); + header_.flags = reinterpret_cast(data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -741,7 +741,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(response_header_.id); uint16_t flags; - SAFE_MEMCPY(&flags, &response_header_.flags); + flags = reinterpret_cast(response_header_.flags); buffer.writeBEInt(flags); buffer.writeBEInt(response_header_.questions); From b20de9a2a3b49d65fe2d820ff6ed0c5ccc5f2a16 Mon Sep 17 00:00:00 2001 From: grial Date: Sat, 31 Oct 2020 18:26:45 +0100 Subject: [PATCH 023/106] reinterpret_cast deleted from dns_parser. Some changes added to buffer_impl Signed-off-by: grial --- source/common/buffer/buffer_impl.cc | 5 +++-- source/common/buffer/buffer_impl.h | 13 ++++++++++--- source/common/common/macros.h | 2 +- .../extensions/filters/udp/dns_filter/dns_parser.cc | 4 ++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/source/common/buffer/buffer_impl.cc b/source/common/buffer/buffer_impl.cc index 556faf73d6385..a1aabf563530c 100644 --- a/source/common/buffer/buffer_impl.cc +++ b/source/common/buffer/buffer_impl.cc @@ -125,6 +125,7 @@ void OwnedImpl::commit(RawSlice* iovecs, uint64_t num_iovecs) { void OwnedImpl::copyOut(size_t start, uint64_t size, void* data) const { uint64_t bytes_to_skip = start; uint8_t* dest = static_cast(data); + MemBlockBuilder mem_builder(size); for (const auto& slice : slices_) { if (size == 0) { break; @@ -137,14 +138,14 @@ void OwnedImpl::copyOut(size_t start, uint64_t size, void* data) const { continue; } uint64_t copy_size = std::min(size, data_size - bytes_to_skip); - memcpy(dest, slice->data() + bytes_to_skip, copy_size); + mem_builder.appendData(slice->data() + bytes_to_skip); size -= copy_size; - dest += copy_size; // Now that we've started copying, there are no bytes left to skip over. If there // is any more data to be copied, the next iteration can start copying from the very // beginning of the next slice. bytes_to_skip = 0; } + dest = mem_builder.release().get(); ASSERT(size == 0); } diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index cfbaadd823233..a3b248ffa112c 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -8,6 +8,7 @@ #include "envoy/buffer/buffer.h" #include "common/common/assert.h" +#include "common/common/mem_block_builder.h" #include "common/common/non_copyable.h" #include "common/common/utility.h" #include "common/event/libevent.h" @@ -153,7 +154,9 @@ class Slice : public SliceData { uint8_t* dest = base_ + reservable_; reservable_ += copy_size; // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) - memcpy(dest, data, copy_size); + MemBlockBuilder mem_builder(copy_size); + mem_builder.appendData(data); + dest = mem_builder.release().get(); return copy_size; } @@ -276,7 +279,9 @@ class OwnedSlice final : public Slice, public InlineStorage { static SlicePtr create(const void* data, uint64_t size) { uint64_t slice_capacity = sliceSize(size); std::unique_ptr slice(new (slice_capacity) OwnedSlice(slice_capacity)); - memcpy(slice->base_, data, size); + MemBlockBuilder mem_builder(size); + mem_builder.appendData(data); + slice->base_ = mem_builder.release().get(); slice->reservable_ = size; return slice; } @@ -653,7 +658,9 @@ class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorag OwnedBufferFragmentImpl(absl::string_view data, const Releasor& releasor) : releasor_(releasor), size_(data.size()) { ASSERT(releasor != nullptr); - memcpy(data_, data.data(), data.size()); + MemBlockBuilder mem_builder(data.size()); + mem_builder.appendData(data.data()); + data_ = mem_builder.release().get(); } const Releasor releasor_; diff --git a/source/common/common/macros.h b/source/common/common/macros.h index 251a53868dd70..4be21347e015e 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -42,7 +42,7 @@ namespace Envoy { } while (0) /** - * Assert memory bounds to avoid copy errors. + * @brief Assert memory bounds to avoid copy errors. */ #define SAFE_MEMCPY(dst, src) \ do { \ diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index c89d9f779ec7b..22956125f5235 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - header_.flags = reinterpret_cast(data); + SAFE_MEMCPY(&(header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -741,7 +741,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(response_header_.id); uint16_t flags; - flags = reinterpret_cast(response_header_.flags); + SAFE_MEMCPY(&flags, &(response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(response_header_.questions); From cfb81455f5e941f74712b746d719b146ea678b49 Mon Sep 17 00:00:00 2001 From: grial Date: Sat, 31 Oct 2020 18:34:14 +0100 Subject: [PATCH 024/106] buffer BUILD file changed Signed-off-by: grial --- source/common/buffer/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/buffer/BUILD b/source/common/buffer/BUILD index 171aa8a089872..95b942fe58c36 100644 --- a/source/common/buffer/BUILD +++ b/source/common/buffer/BUILD @@ -25,6 +25,7 @@ envoy_cc_library( hdrs = ["buffer_impl.h"], deps = [ "//include/envoy/buffer:buffer_interface", + "//source/common/common:mem_block_builder_lib", "//source/common/common:non_copyable", "//source/common/common:utility_lib", "//source/common/event:libevent_lib", From 87c59b3e3f0eef0928cf668b1fc9028834b45e5b Mon Sep 17 00:00:00 2001 From: grial Date: Sat, 31 Oct 2020 18:58:13 +0100 Subject: [PATCH 025/106] Cast added to copy data Signed-off-by: grial --- source/common/buffer/buffer_impl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index a3b248ffa112c..7c0de3c2bebdc 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -155,7 +155,7 @@ class Slice : public SliceData { reservable_ += copy_size; // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) MemBlockBuilder mem_builder(copy_size); - mem_builder.appendData(data); + mem_builder.appendData(absl::Span(reinterpret_cast(data))); dest = mem_builder.release().get(); return copy_size; } @@ -280,7 +280,7 @@ class OwnedSlice final : public Slice, public InlineStorage { uint64_t slice_capacity = sliceSize(size); std::unique_ptr slice(new (slice_capacity) OwnedSlice(slice_capacity)); MemBlockBuilder mem_builder(size); - mem_builder.appendData(data); + mem_builder.appendData(absl::Span(reinterpret_cast(data))); slice->base_ = mem_builder.release().get(); slice->reservable_ = size; return slice; @@ -659,7 +659,7 @@ class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorag : releasor_(releasor), size_(data.size()) { ASSERT(releasor != nullptr); MemBlockBuilder mem_builder(data.size()); - mem_builder.appendData(data.data()); + mem_builder.appendData(absl::Span(reinterpret_cast(data.data()))); data_ = mem_builder.release().get(); } From 22b2ee4658df8892d7798e33ba8d4da5ab6ba4b5 Mon Sep 17 00:00:00 2001 From: grial Date: Sat, 31 Oct 2020 20:34:20 +0100 Subject: [PATCH 026/106] proxy protocol implementation back in place. Method parseV2Header modified Signed-off-by: grial --- source/common/buffer/buffer_impl.h | 6 ++-- source/common/network/address_impl.h | 2 +- .../listener/proxy_protocol/proxy_protocol.cc | 30 ++++++++----------- source/server/hot_restarting_base.cc | 2 +- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 7c0de3c2bebdc..544796c9aa652 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -155,7 +155,7 @@ class Slice : public SliceData { reservable_ += copy_size; // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) MemBlockBuilder mem_builder(copy_size); - mem_builder.appendData(absl::Span(reinterpret_cast(data))); + mem_builder.appendData(absl::Span(reinterpret_cast(data))); dest = mem_builder.release().get(); return copy_size; } @@ -280,7 +280,7 @@ class OwnedSlice final : public Slice, public InlineStorage { uint64_t slice_capacity = sliceSize(size); std::unique_ptr slice(new (slice_capacity) OwnedSlice(slice_capacity)); MemBlockBuilder mem_builder(size); - mem_builder.appendData(absl::Span(reinterpret_cast(data))); + mem_builder.appendData(absl::Span(reinterpret_cast(data))); slice->base_ = mem_builder.release().get(); slice->reservable_ = size; return slice; @@ -659,7 +659,7 @@ class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorag : releasor_(releasor), size_(data.size()) { ASSERT(releasor != nullptr); MemBlockBuilder mem_builder(data.size()); - mem_builder.appendData(absl::Span(reinterpret_cast(data.data()))); + mem_builder.appendData(absl::Span(reinterpret_cast(data.data()))); data_ = mem_builder.release().get(); } diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 35ff4f9e0b6a5..7e4719c4efca9 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -227,13 +227,13 @@ class PipeInstance : public InstanceBase { const sockaddr* sockAddr() const override { return reinterpret_cast(&pipe_.address_); } + const sockaddr_un& getSockAddr() const override { return pipe_.address_; } socklen_t sockAddrLen() const override { if (pipe_.abstract_namespace_) { return offsetof(struct sockaddr_un, sun_path) + pipe_.address_length_; } return sizeof(pipe_.address_); } - void getSockAddr(sockaddr_un& address) { address = pipe_.address_; } private: struct PipeHelper : public Pipe { diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 01c4089f8a1d1..7820499a29e35 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -196,28 +196,24 @@ void Filter::parseV2Header(char* buf) { std::make_shared(&la4)}); return; } else if (((proto_family & 0xf0) >> 4) == PROXY_PROTO_V2_AF_INET6) { - std::size_t offset = 0; // Offset to iterate over buf + PACKED_STRUCT(struct pp_ipv6_addr { + uint8_t src_addr[16]; + uint8_t dst_addr[16]; + uint16_t src_port; + uint16_t dst_port; + }); + pp_ipv6_addr* v6; + v6 = reinterpret_cast(&buf[PROXY_PROTO_V2_HEADER_LEN]); sockaddr_in6 ra6, la6; memset(&ra6, 0, sizeof(ra6)); memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; - la6.sin6_family = AF_INET6; - - for (; offset < ARRAY_SIZE(ra6.sin6_addr.s6_addr); ++offset) - ra6.sin6_addr.s6_addr[offset] = buf[PROXY_PROTO_V2_HEADER_LEN + offset]; - - for (; offset < ARRAY_SIZE(la6.sin6_addr.s6_addr); ++offset) - la6.sin6_addr.s6_addr[offset] = buf[PROXY_PROTO_V2_HEADER_LEN + offset]; + ra6.sin6_port = v6->src_port; + SAFE_MEMCPY(ra6.sin6_addr.s6_addr, v6->src_addr); - PACKED_STRUCT(struct ports { - uint16_t src_port; - uint16_t dst_port; - }); - ports* port_pair; - port_pair = reinterpret_cast(&buf[PROXY_PROTO_V2_HEADER_LEN + offset]); - - ra6.sin6_port = port_pair->src_port; - la6.sin6_port = port_pair->dst_port; + la6.sin6_family = AF_INET6; + la6.sin6_port = v6->dst_port; + SAFE_MEMCPY(la6.sin6_addr.s6_addr, v6->dst_addr); proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index ce7b22da6fd16..e06b7e5d052d2 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -37,7 +37,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - addr.getSockAddr(address); + SAFE_MEMCPY(address, addr.getSockAddr()); fchmod(my_domain_socket_, socket_mode); return address; From e1dd071f95a18ab41bd1298dc5f763dacf5b6e63 Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 3 Nov 2020 15:50:28 +0000 Subject: [PATCH 027/106] changes to mem_block_builder applied, data_ type is a raw pointer pointer and owned_data_. Some memcpys sutituted Signed-off-by: grial1 --- source/common/buffer/buffer_impl.cc | 5 ++- source/common/buffer/buffer_impl.h | 22 ++++++------ source/common/chromium_url/BUILD | 5 ++- source/common/chromium_url/url_canon.h | 5 ++- source/common/common/BUILD | 1 + source/common/common/mem_block_builder.h | 36 ++++++++++++++----- source/common/common/utility.cc | 8 +++-- source/common/network/address_impl.h | 2 +- .../filters/network/mongo_proxy/bson_impl.cc | 3 +- source/server/hot_restarting_base.cc | 2 +- 10 files changed, 59 insertions(+), 30 deletions(-) diff --git a/source/common/buffer/buffer_impl.cc b/source/common/buffer/buffer_impl.cc index a1aabf563530c..56f8423198fdb 100644 --- a/source/common/buffer/buffer_impl.cc +++ b/source/common/buffer/buffer_impl.cc @@ -125,7 +125,7 @@ void OwnedImpl::commit(RawSlice* iovecs, uint64_t num_iovecs) { void OwnedImpl::copyOut(size_t start, uint64_t size, void* data) const { uint64_t bytes_to_skip = start; uint8_t* dest = static_cast(data); - MemBlockBuilder mem_builder(size); + MemBlockBuilder mem_builder(dest, size); for (const auto& slice : slices_) { if (size == 0) { break; @@ -138,14 +138,13 @@ void OwnedImpl::copyOut(size_t start, uint64_t size, void* data) const { continue; } uint64_t copy_size = std::min(size, data_size - bytes_to_skip); - mem_builder.appendData(slice->data() + bytes_to_skip); + mem_builder.appendData(absl::Span(slice->data() + bytes_to_skip, copy_size)); size -= copy_size; // Now that we've started copying, there are no bytes left to skip over. If there // is any more data to be copied, the next iteration can start copying from the very // beginning of the next slice. bytes_to_skip = 0; } - dest = mem_builder.release().get(); ASSERT(size == 0); } diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 544796c9aa652..07e135128064f 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -154,9 +154,9 @@ class Slice : public SliceData { uint8_t* dest = base_ + reservable_; reservable_ += copy_size; // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) - MemBlockBuilder mem_builder(copy_size); - mem_builder.appendData(absl::Span(reinterpret_cast(data))); - dest = mem_builder.release().get(); + MemBlockBuilder mem_builder(dest, copy_size); + uint8_t* data_non_raw = reinterpret_cast(const_cast(data)); + mem_builder.appendData(absl::Span(data_non_raw, size)); return copy_size; } @@ -186,7 +186,9 @@ class Slice : public SliceData { copy_size = std::min(size, data_); data_ -= copy_size; } - memcpy(base_ + data_, src + size - copy_size, copy_size); + MemBlockBuilder mem_builder(base_ + data_, copy_size); + mem_builder.appendData( + absl::Span(const_cast(src + size - copy_size), copy_size)); return copy_size; } @@ -279,9 +281,9 @@ class OwnedSlice final : public Slice, public InlineStorage { static SlicePtr create(const void* data, uint64_t size) { uint64_t slice_capacity = sliceSize(size); std::unique_ptr slice(new (slice_capacity) OwnedSlice(slice_capacity)); - MemBlockBuilder mem_builder(size); - mem_builder.appendData(absl::Span(reinterpret_cast(data))); - slice->base_ = mem_builder.release().get(); + MemBlockBuilder mem_builder(slice->base_, size); + uint8_t* data_non_raw = reinterpret_cast(const_cast(data)); + mem_builder.appendData(absl::Span(data_non_raw, size)); slice->reservable_ = size; return slice; } @@ -658,9 +660,9 @@ class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorag OwnedBufferFragmentImpl(absl::string_view data, const Releasor& releasor) : releasor_(releasor), size_(data.size()) { ASSERT(releasor != nullptr); - MemBlockBuilder mem_builder(data.size()); - mem_builder.appendData(absl::Span(reinterpret_cast(data.data()))); - data_ = mem_builder.release().get(); + MemBlockBuilder mem_builder(data_, data.size()); + uint8_t* data_non_raw = reinterpret_cast(const_cast(data.data())); + mem_builder.appendData(absl::Span(data_non_raw, data.size())); } const Releasor releasor_; diff --git a/source/common/chromium_url/BUILD b/source/common/chromium_url/BUILD index 2d4acb3487658..0501670ad4f86 100644 --- a/source/common/chromium_url/BUILD +++ b/source/common/chromium_url/BUILD @@ -24,5 +24,8 @@ envoy_cc_library( "url_parse.h", "url_parse_internal.h", ], - deps = ["//source/common/common:assert_lib"], + deps = [ + "//source/common/common:assert_lib", + "//source/common/common:mem_block_builder_lib" + ], ) diff --git a/source/common/chromium_url/url_canon.h b/source/common/chromium_url/url_canon.h index 0280de643ac86..200f111bb79e0 100644 --- a/source/common/chromium_url/url_canon.h +++ b/source/common/chromium_url/url_canon.h @@ -13,6 +13,7 @@ #include "common/chromium_url/envoy_shim.h" #include "common/chromium_url/url_parse.h" +#include "common/common/mem_block_builder.h" namespace chromium_url { @@ -146,7 +147,9 @@ template class RawCanonOutputT : public void Resize(int sz) override { T* new_buf = new T[sz]; - memcpy(new_buf, this->buffer_, sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz)); + size_t copy_size = sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz); + Envoy::MemBlockBuilder mem_builder(new_buf, copy_size); + mem_builder.appendData(absl::Span(this->buffer_, copy_size)); if (this->buffer_ != fixed_buffer_) delete[] this->buffer_; this->buffer_ = new_buf; diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 8c645fd32a91a..b93bbd020f577 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -356,6 +356,7 @@ envoy_cc_library( external_deps = ["abseil_node_hash_map"], deps = [ ":assert_lib", + ":mem_block_builder_lib", ":hash_lib", ":non_copyable", "//include/envoy/common:interval_set_interface", diff --git a/source/common/common/mem_block_builder.h b/source/common/common/mem_block_builder.h index a43f7e885e224..e3b93e8907398 100644 --- a/source/common/common/mem_block_builder.h +++ b/source/common/common/mem_block_builder.h @@ -26,7 +26,13 @@ template class MemBlockBuilder { public: // Constructs a MemBlockBuilder allowing for 'capacity' instances of T. explicit MemBlockBuilder(uint64_t capacity) - : data_(std::make_unique(capacity)), write_span_(data_.get(), capacity) {} + : owned_data_(std::make_unique(capacity)), write_span_(owned_data_.get(), capacity) { + data_ = owned_data_.get(); + }; + explicit MemBlockBuilder(T* data, uint64_t capacity) + : owned_data_(nullptr), write_span_(data, capacity) { + data_ = data; + }; MemBlockBuilder() = default; /** @@ -43,7 +49,7 @@ template class MemBlockBuilder { /** * @return the capacity. */ - uint64_t capacity() const { return write_span_.size() + write_span_.data() - data_.get(); } + uint64_t capacity() const { return write_span_.size() + write_span_.data() - owned_data_.get(); } /** * Appends a single object of type T, moving an internal write-pointer @@ -99,26 +105,40 @@ template class MemBlockBuilder { */ std::unique_ptr release() { write_span_ = absl::MakeSpan(static_cast(nullptr), 0); - return std::move(data_); + return std::move(owned_data_); + } + + /** + * Returns the underlying storage as a raw pointer, clearing this. + * + * @return the transferred storage. + */ + T* releasePointer() { + write_span_ = absl::MakeSpan(static_cast(nullptr), 0); + if (owned_data_) + return owned_data_.release(); + else + return data_; } /** * @return the populated data as an absl::Span. */ - absl::Span span() const { return absl::MakeSpan(data_.get(), write_span_.data()); } + absl::Span span() const { return absl::MakeSpan(owned_data_.get(), write_span_.data()); } /** * @return The number of elements the have been added to the builder. */ - uint64_t size() const { return write_span_.data() - data_.get(); } + uint64_t size() const { return write_span_.data() - owned_data_.get(); } private: void setCapacityHelper(uint64_t capacity, std::unique_ptr data) { - data_ = std::move(data); - write_span_ = absl::MakeSpan(data_.get(), capacity); + owned_data_ = std::move(data); + write_span_ = absl::MakeSpan(owned_data_.get(), capacity); } - std::unique_ptr data_; + T* data_; + std::unique_ptr owned_data_; absl::Span write_span_; }; diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 0c72f30c816e3..3eda8bb60b8fe 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -11,6 +11,7 @@ #include "envoy/common/exception.h" #include "common/common/assert.h" +#include "common/common/mem_block_builder.h" #include "common/common/fmt.h" #include "common/common/hash.h" #include "common/singleton/const_singleton.h" @@ -43,7 +44,7 @@ const std::string errorDetails(int error_code) { #ifndef WIN32 // clang-format off return strerror(error_code); - // clang-format on +// clang-format on #else // Windows error codes do not correspond to POSIX errno values // Use FormatMessage, strip trailing newline, and return "Unknown error" on failure (as on POSIX). @@ -507,7 +508,7 @@ std::string StringUtil::removeCharacters(const absl::string_view& str, const auto intervals = remove_characters.toVector(); std::vector pieces; pieces.reserve(intervals.size()); - for (const auto& [left_bound, right_bound] : intervals) { + for (const auto & [ left_bound, right_bound ] : intervals) { if (left_bound != pos) { ASSERT(right_bound <= str.size()); pieces.push_back(str.substr(pos, left_bound - pos)); @@ -569,7 +570,8 @@ double WelfordStandardDeviation::computeStandardDeviation() const { InlineString::InlineString(const char* str, size_t size) : size_(size) { RELEASE_ASSERT(size <= 0xffffffff, "size must fit in 32 bits"); - memcpy(data_, str, size); + MemBlockBuilder mem_builder(data_, size); + mem_builder.appendData(absl::Span(const_cast(str), size)); } void ExceptionUtil::throwEnvoyException(const std::string& message) { diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 7e4719c4efca9..584b5384d1405 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -227,7 +227,7 @@ class PipeInstance : public InstanceBase { const sockaddr* sockAddr() const override { return reinterpret_cast(&pipe_.address_); } - const sockaddr_un& getSockAddr() const override { return pipe_.address_; } + const sockaddr_un& getSockAddr() const { return pipe_.address_; } socklen_t sockAddrLen() const override { if (pipe_.abstract_namespace_) { return offsetof(struct sockaddr_un, sun_path) + pipe_.address_length_; diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 4470028ca7278..46d0f297cee58 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -43,10 +43,9 @@ void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_ throw EnvoyException("invalid buffer size"); } - MemBlockBuilder mem_builder(out_len); + MemBlockBuilder mem_builder(out, out_len); mem_builder.appendData( absl::Span(static_cast(data.linearize(out_len)), out_len)); - out = mem_builder.release().get(); data.drain(out_len); } diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index e06b7e5d052d2..16c89c53235af 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -37,7 +37,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - SAFE_MEMCPY(address, addr.getSockAddr()); + SAFE_MEMCPY(&address, &(addr.getSockAddr())); fchmod(my_domain_socket_, socket_mode); return address; From 57916ede6b5582b53078dee3548adeb648854292 Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 3 Nov 2020 17:05:57 +0000 Subject: [PATCH 028/106] format fixed Signed-off-by: grial1 --- source/common/chromium_url/BUILD | 2 +- source/common/common/BUILD | 2 +- source/common/common/utility.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/chromium_url/BUILD b/source/common/chromium_url/BUILD index 0501670ad4f86..0529a808f1397 100644 --- a/source/common/chromium_url/BUILD +++ b/source/common/chromium_url/BUILD @@ -26,6 +26,6 @@ envoy_cc_library( ], deps = [ "//source/common/common:assert_lib", - "//source/common/common:mem_block_builder_lib" + "//source/common/common:mem_block_builder_lib", ], ) diff --git a/source/common/common/BUILD b/source/common/common/BUILD index b93bbd020f577..ac16e8cabe393 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -356,8 +356,8 @@ envoy_cc_library( external_deps = ["abseil_node_hash_map"], deps = [ ":assert_lib", - ":mem_block_builder_lib", ":hash_lib", + ":mem_block_builder_lib", ":non_copyable", "//include/envoy/common:interval_set_interface", "//include/envoy/common:time_interface", diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 3eda8bb60b8fe..488a745de73e6 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -11,9 +11,9 @@ #include "envoy/common/exception.h" #include "common/common/assert.h" -#include "common/common/mem_block_builder.h" #include "common/common/fmt.h" #include "common/common/hash.h" +#include "common/common/mem_block_builder.h" #include "common/singleton/const_singleton.h" #include "absl/container/node_hash_map.h" @@ -508,7 +508,7 @@ std::string StringUtil::removeCharacters(const absl::string_view& str, const auto intervals = remove_characters.toVector(); std::vector pieces; pieces.reserve(intervals.size()); - for (const auto & [ left_bound, right_bound ] : intervals) { + for (const auto& [left_bound, right_bound] : intervals) { if (left_bound != pos) { ASSERT(right_bound <= str.size()); pieces.push_back(str.substr(pos, left_bound - pos)); From c71445f5dcc7f3af32fd5fa4cfc0125085f440fe Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 15:22:53 +0000 Subject: [PATCH 029/106] check_format changed as suggested in issue #9328 Signed-off-by: grial1 --- source/common/buffer/buffer_impl.h | 17 ++++------------- tools/code_format/check_format.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 07e135128064f..cfbaadd823233 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -8,7 +8,6 @@ #include "envoy/buffer/buffer.h" #include "common/common/assert.h" -#include "common/common/mem_block_builder.h" #include "common/common/non_copyable.h" #include "common/common/utility.h" #include "common/event/libevent.h" @@ -154,9 +153,7 @@ class Slice : public SliceData { uint8_t* dest = base_ + reservable_; reservable_ += copy_size; // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) - MemBlockBuilder mem_builder(dest, copy_size); - uint8_t* data_non_raw = reinterpret_cast(const_cast(data)); - mem_builder.appendData(absl::Span(data_non_raw, size)); + memcpy(dest, data, copy_size); return copy_size; } @@ -186,9 +183,7 @@ class Slice : public SliceData { copy_size = std::min(size, data_); data_ -= copy_size; } - MemBlockBuilder mem_builder(base_ + data_, copy_size); - mem_builder.appendData( - absl::Span(const_cast(src + size - copy_size), copy_size)); + memcpy(base_ + data_, src + size - copy_size, copy_size); return copy_size; } @@ -281,9 +276,7 @@ class OwnedSlice final : public Slice, public InlineStorage { static SlicePtr create(const void* data, uint64_t size) { uint64_t slice_capacity = sliceSize(size); std::unique_ptr slice(new (slice_capacity) OwnedSlice(slice_capacity)); - MemBlockBuilder mem_builder(slice->base_, size); - uint8_t* data_non_raw = reinterpret_cast(const_cast(data)); - mem_builder.appendData(absl::Span(data_non_raw, size)); + memcpy(slice->base_, data, size); slice->reservable_ = size; return slice; } @@ -660,9 +653,7 @@ class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorag OwnedBufferFragmentImpl(absl::string_view data, const Releasor& releasor) : releasor_(releasor), size_(data.size()) { ASSERT(releasor != nullptr); - MemBlockBuilder mem_builder(data_, data.size()); - uint8_t* data_non_raw = reinterpret_cast(const_cast(data.data())); - mem_builder.appendData(absl::Span(data_non_raw, data.size())); + memcpy(data_, data.data(), data.size()); } const Releasor releasor_; diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 7b3e1cac97e03..5cb6722da64ae 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -106,6 +106,10 @@ # Only one C++ file should instantiate grpc_init GRPC_INIT_ALLOWLIST = ("./source/common/grpc/google_grpc_context.cc") +# Files that should not raise an error for using memcpy +MEMCPY_WHITELIST = ("./source/common/common/mem_block_builder.h", + "./source/common/common/safe_memcpy.h") + # These files should not throw exceptions. Add HTTP/1 when exceptions removed. EXCEPTION_DENYLIST = ("./source/common/http/http2/codec_impl.h", "./source/common/http/http2/codec_impl.cc") @@ -406,6 +410,9 @@ def allowlistedForStdRegex(self, file_path): def allowlistedForGrpcInit(self, file_path): return file_path in GRPC_INIT_ALLOWLIST + def whitelistedForMemcpy(self, file_path): + return file_path in MEMCPY_WHITELIST + def allowlistedForUnpackTo(self, file_path): return file_path.startswith("./test") or file_path in [ "./source/common/protobuf/utility.cc", "./source/common/protobuf/utility.h" @@ -792,6 +799,12 @@ def checkSourceLine(self, line, file_path, reportError): reportError("Don't call grpc_init() or grpc_shutdown() directly, instantiate " + "Grpc::GoogleGrpcContext. See #8282") + if not self.whitelistedForMemcpy(file_path) and \ + not ("test/" in file_path) and \ + ("memcpy(" in line) and \ + not ("NOLINT(safe-memcpy)" in line): + reportError("Don't call memcpy() directly; use SAFE_MEMCPY or MemBlockBuilder instead.") + if self.denylistedForExceptions(file_path): # Skpping cases where 'throw' is a substring of a symbol like in "foothrowBar". if "throw" in line.split(): From f51819e11ea8cf87ee3982f0a26ddb4577a38c47 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 15:45:04 +0000 Subject: [PATCH 030/106] NOTLINT added to the line in which memcpy is used Signed-off-by: grial1 --- source/common/buffer/buffer_impl.cc | 4 ++-- source/common/buffer/buffer_impl.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/buffer/buffer_impl.cc b/source/common/buffer/buffer_impl.cc index 56f8423198fdb..01442b72659c1 100644 --- a/source/common/buffer/buffer_impl.cc +++ b/source/common/buffer/buffer_impl.cc @@ -125,7 +125,6 @@ void OwnedImpl::commit(RawSlice* iovecs, uint64_t num_iovecs) { void OwnedImpl::copyOut(size_t start, uint64_t size, void* data) const { uint64_t bytes_to_skip = start; uint8_t* dest = static_cast(data); - MemBlockBuilder mem_builder(dest, size); for (const auto& slice : slices_) { if (size == 0) { break; @@ -138,8 +137,9 @@ void OwnedImpl::copyOut(size_t start, uint64_t size, void* data) const { continue; } uint64_t copy_size = std::min(size, data_size - bytes_to_skip); - mem_builder.appendData(absl::Span(slice->data() + bytes_to_skip, copy_size)); + memcpy(dest, slice->data() + bytes_to_skip, copy_size); // NOLINT(safe-memcpy) size -= copy_size; + dest += copy_size; // Now that we've started copying, there are no bytes left to skip over. If there // is any more data to be copied, the next iteration can start copying from the very // beginning of the next slice. diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index cfbaadd823233..62c7bd2c94ed6 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -153,7 +153,7 @@ class Slice : public SliceData { uint8_t* dest = base_ + reservable_; reservable_ += copy_size; // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) - memcpy(dest, data, copy_size); + memcpy(dest, data, copy_size); // NOLINT(safe-memcpy) return copy_size; } @@ -183,7 +183,7 @@ class Slice : public SliceData { copy_size = std::min(size, data_); data_ -= copy_size; } - memcpy(base_ + data_, src + size - copy_size, copy_size); + memcpy(base_ + data_, src + size - copy_size, copy_size); // NOLINT(safe-memcpy) return copy_size; } @@ -276,7 +276,7 @@ class OwnedSlice final : public Slice, public InlineStorage { static SlicePtr create(const void* data, uint64_t size) { uint64_t slice_capacity = sliceSize(size); std::unique_ptr slice(new (slice_capacity) OwnedSlice(slice_capacity)); - memcpy(slice->base_, data, size); + memcpy(slice->base_, data, size); // NOLINT(safe-memcpy) slice->reservable_ = size; return slice; } @@ -653,7 +653,7 @@ class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorag OwnedBufferFragmentImpl(absl::string_view data, const Releasor& releasor) : releasor_(releasor), size_(data.size()) { ASSERT(releasor != nullptr); - memcpy(data_, data.data(), data.size()); + memcpy(data_, data.data(), data.size()); // NOLINT(safe-memcpy) } const Releasor releasor_; From b0960d3e8bf9e44448ac74a927972e4f7f82d696 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 15:58:00 +0000 Subject: [PATCH 031/106] memcpy changed for strcpy Signed-off-by: grial1 --- source/extensions/stat_sinks/common/statsd/statsd.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 15559a4dc59a6..3903279af2d70 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -210,10 +210,13 @@ void TcpStatsdSink::TlsSink::commonFlush(const std::string& name, uint64_t value // This written this way for maximum perf since with a large number of stats and at a high flush // rate this can become expensive. const char* snapped_current = current_slice_mem_; - memcpy(current_slice_mem_, parent_.getPrefix().c_str(), parent_.getPrefix().size()); + // memcpy(current_slice_mem_, parent_.getPrefix().c_str(), + // parent_.getPrefix().size()); + strcpy(current_slice_mem_, parent_.getPrefix().c_str()); current_slice_mem_ += parent_.getPrefix().size(); *current_slice_mem_++ = '.'; - memcpy(current_slice_mem_, name.c_str(), name.size()); + // memcpy(current_slice_mem_, name.c_str(), name.size()); + strcpy(current_slice_mem_, name.c_str()); current_slice_mem_ += name.size(); *current_slice_mem_++ = ':'; current_slice_mem_ += StringUtil::itoa(current_slice_mem_, 30, value); From 74e19f1f934bc9d14770b4c59e7cfa4c096ab376 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 17:13:30 +0000 Subject: [PATCH 032/106] memcpy reviewed in the dir source/ Signed-off-by: grial1 --- source/common/common/hash.h | 2 +- source/common/grpc/common.cc | 6 ++- source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl_legacy.cc | 3 +- source/common/network/address_impl.cc | 5 ++- source/common/network/cidr_range.cc | 2 +- .../proxy_protocol/proxy_protocol_header.cc | 12 +++--- source/extensions/common/wasm/context.cc | 37 +++++++++++-------- source/extensions/common/wasm/foreign.cc | 6 +-- .../network/dubbo_proxy/buffer_helper.cc | 4 +- .../filters/network/kafka/serialization.cc | 4 +- .../filters/network/kafka/serialization.h | 14 +++---- .../filters/network/kafka/tagged_fields.h | 2 +- .../network/thrift_proxy/buffer_helper.cc | 4 +- .../quiche/envoy_quic_client_stream.cc | 2 +- .../quiche/envoy_quic_server_stream.cc | 2 +- .../quiche/platform/string_utils.cc | 2 +- .../stat_sinks/common/statsd/statsd.cc | 9 ++--- .../transport_sockets/tls/utility.cc | 5 ++- 19 files changed, 65 insertions(+), 58 deletions(-) diff --git a/source/common/common/hash.h b/source/common/common/hash.h index c29b9effa89d2..3cb43d20c6d8b 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -58,7 +58,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - memcpy(&result, p, sizeof(result)); + memcpy(&result, p, sizeof(result)); // NOLINT(safe-memcpy) return result; } diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index 8b7551d8bea29..fc7bf872e923d 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -138,7 +138,8 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - std::memcpy(current, reinterpret_cast(&nsize), sizeof(uint32_t)); + std::memcpy(current, reinterpret_cast(&nsize), // NOLINT(safe-memcpy) + sizeof(uint32_t)); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); @@ -292,7 +293,8 @@ void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { std::array header; header[0] = 0; // flags const uint32_t nsize = htonl(buffer.length()); - std::memcpy(&header[1], reinterpret_cast(&nsize), sizeof(uint32_t)); + std::memcpy(&header[1], reinterpret_cast(&nsize), // NOLINT(safe-memcpy) + sizeof(uint32_t)); buffer.prepend(absl::string_view(&header[0], 5)); } diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 4af93895de4a0..5022b0df8b0a8 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -751,7 +751,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // to match the ping and ack. uint64_t data; static_assert(sizeof(data) == sizeof(frame->ping.opaque_data), "Sizes are equal"); - memcpy(&data, frame->ping.opaque_data, sizeof(data)); + SAFE_MEMCPY(&data, frame->ping.opaque_data); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index 88e596458fe92..d59e97d13877c 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -729,8 +729,7 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - static_assert(sizeof(data) == sizeof(frame->ping.opaque_data), "Sizes are equal"); - memcpy(&data, frame->ping.opaque_data, sizeof(data)); + SAFE_MEMCPY(&data, frame->ping.opaque_data); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 38308a491f5b4..4cfed726d215f 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -175,7 +175,8 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - memcpy(static_cast(&result), static_cast(&address_.sin6_addr.s6_addr), + memcpy(static_cast(&result),\ // NOLINT(safe-memcpy) + static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); return result; } @@ -281,7 +282,7 @@ PipeInstance::PipeInstance(const std::string& pipe_path, mode_t mode, } pipe_.abstract_namespace_ = true; pipe_.address_length_ = pipe_path.size(); - memcpy(&pipe_.address_.sun_path[0], pipe_path.data(), pipe_path.size()); + memcpy(&pipe_.address_.sun_path[0], pipe_path.data(), pipe_path.size()); // NOLINT(safe-memcpy) pipe_.address_.sun_path[0] = '\0'; pipe_.address_.sun_path[pipe_path.size()] = '\0'; friendly_name_ = friendlyNameFromAbstractPath( diff --git a/source/common/network/cidr_range.cc b/source/common/network/cidr_range.cc index 277125e17c4a3..501a8de6d6e3c 100644 --- a/source/common/network/cidr_range.cc +++ b/source/common/network/cidr_range.cc @@ -183,7 +183,7 @@ InstanceConstSharedPtr CidrRange::truncateIpAddressAndLength(InstanceConstShared absl::uint128 ip6_htonl = Utility::Ip6htonl(ip6); static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - memcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl, sizeof(absl::uint128)); + memcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl, sizeof(absl::uint128)); // NOLINT(safe-memcpy) return std::make_shared(sa6); } } diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index 0342f3d1aff3c..d80444d252709 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -72,8 +72,8 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv4Instance(src_addr, src_port).ip()->ipv4()->address(); const auto net_dst_addr = Network::Address::Ipv4Instance(dst_addr, dst_port).ip()->ipv4()->address(); - memcpy(addrs, &net_src_addr, 4); - memcpy(&addrs[4], &net_dst_addr, 4); + memcpy(addrs, &net_src_addr, 4); // NOLINT(safe-memcpy) + memcpy(&addrs[4], &net_dst_addr, 4); // NOLINT(safe-memcpy) out.add(addrs, 8); break; } @@ -86,8 +86,8 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const auto net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - memcpy(addrs, &net_src_addr, 16); - memcpy(&addrs[16], &net_dst_addr, 16); + memcpy(addrs, &net_src_addr, 16); // NOLINT(safe-memcpy) + memcpy(&addrs[16], &net_dst_addr, 16); // NOLINT(safe-memcpy) out.add(addrs, 32); break; } @@ -96,8 +96,8 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, uint8_t ports[4]; const auto net_src_port = htons(static_cast(src_port)); const auto net_dst_port = htons(static_cast(dst_port)); - memcpy(ports, &net_src_port, 2); - memcpy(&ports[2], &net_dst_port, 2); + memcpy(ports, &net_src_port, 2); // NOLINT(safe-memcpy) + memcpy(&ports[2], &net_dst_port, 2); // NOLINT(safe-memcpy) out.add(ports, 4); } diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index e6e4f8ae0f057..76575fde08abf 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -193,15 +193,17 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti auto buffer = std::unique_ptr(new char[s]); char* b = buffer.get(); uint32_t n = response.size(); - memcpy(b, &n, sizeof(uint32_t)); + memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); for (auto& e : response) { uint32_t ttl = e.ttl_.count(); - memcpy(b, &ttl, sizeof(uint32_t)); + memcpy(b, &ttl, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); }; for (auto& e : response) { - memcpy(b, e.address_->asStringView().data(), e.address_->asStringView().size()); + memcpy(b, e.address_->asStringView().data(),\ // NOLINT(safe-memcpy) + e.address_->asStringView() + .size()); b += e.address_->asStringView().size(); *b++ = 0; }; @@ -262,45 +264,48 @@ void Context::onStatsUpdate(Envoy::Stats::MetricSnapshot& snapshot) { auto buffer = std::unique_ptr(new char[counter_block_size + gauge_block_size]); char* b = buffer.get(); - memcpy(b, &counter_block_size, sizeof(uint32_t)); + memcpy(b, &counter_block_size, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); - memcpy(b, &counter_type, sizeof(uint32_t)); + memcpy(b, &counter_type, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); - memcpy(b, &num_counters, sizeof(uint32_t)); + memcpy(b, &num_counters, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); for (const auto& counter : snapshot.counters()) { if (counter.counter_.get().used()) { n = counter.counter_.get().name().size(); - memcpy(b, &n, sizeof(uint32_t)); + memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); - memcpy(b, counter.counter_.get().name().data(), counter.counter_.get().name().size()); + memcpy(b, counter.counter_.get().name().data(),\ // NOLINT(safe-memcpy) + counter.counter_.get() + .name() + .size()); b = align(b + counter.counter_.get().name().size()); v = counter.counter_.get().value(); - memcpy(b, &v, sizeof(uint64_t)); + memcpy(b, &v, sizeof(uint64_t)); // NOLINT(safe-memcpy) b += sizeof(uint64_t); v = counter.delta_; - memcpy(b, &v, sizeof(uint64_t)); + memcpy(b, &v, sizeof(uint64_t)); // NOLINT(safe-memcpy) b += sizeof(uint64_t); } } - memcpy(b, &gauge_block_size, sizeof(uint32_t)); + memcpy(b, &gauge_block_size, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); - memcpy(b, &gauge_type, sizeof(uint32_t)); + memcpy(b, &gauge_type, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); - memcpy(b, &num_gauges, sizeof(uint32_t)); + memcpy(b, &num_gauges, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); for (const auto& gauge : snapshot.gauges()) { if (gauge.get().used()) { n = gauge.get().name().size(); - memcpy(b, &n, sizeof(uint32_t)); + memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); - memcpy(b, gauge.get().name().data(), gauge.get().name().size()); + memcpy(b, gauge.get().name().data(), gauge.get().name().size()); // NOLINT(safe-memcpy) b = align(b + gauge.get().name().size()); v = gauge.get().value(); - memcpy(b, &v, sizeof(uint64_t)); + memcpy(b, &v, sizeof(uint64_t)); // NOLINT(safe-memcpy) b += sizeof(uint64_t); } } diff --git a/source/extensions/common/wasm/foreign.cc b/source/extensions/common/wasm/foreign.cc index 565ca22adb9b8..649e1fca09425 100644 --- a/source/extensions/common/wasm/foreign.cc +++ b/source/extensions/common/wasm/foreign.cc @@ -35,7 +35,7 @@ RegisterForeignFunction registerCompressForeignFunction( return WasmResult::SerializationFailure; } auto result = alloc_result(dest_len); - memcpy(result, b.get(), dest_len); + memcpy(result, b.get(), dest_len); // NOLINT(safe-memcpy) return WasmResult::Ok; }); @@ -51,7 +51,7 @@ RegisterForeignFunction registerUncompressForeignFunction( arguments.size()); if (r == Z_OK) { auto result = alloc_result(dest_len); - memcpy(result, b.get(), dest_len); + memcpy(result, b.get(), dest_len); // NOLINT(safe-memcpy) return WasmResult::Ok; } if (r != Z_BUF_ERROR) { @@ -184,7 +184,7 @@ class EvaluateExpressionFactory : public ExpressionFactory { return serialize_status; } auto output = alloc_result(result.size()); - memcpy(output, result.data(), result.size()); + memcpy(output, result.data(), result.size()); // NOLINT(safe-memcpy) return WasmResult::Ok; }; return f; diff --git a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc index ee7c762b565a0..acb7127f1292b 100644 --- a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc @@ -11,7 +11,7 @@ double BufferHelper::peekDouble(Buffer::Instance& buffer, uint64_t offset) { } double i; uint64_t j = buffer.peekBEInt(offset); - std::memcpy(&i, &j, 8); + SAFE_MEMCPY(&i, &j); return i; } @@ -21,7 +21,7 @@ float BufferHelper::peekFloat(Buffer::Instance& buffer, uint64_t offset) { } float i; uint32_t j = buffer.peekBEInt(offset); - std::memcpy(&i, &j, 4); + SAFE_MEMCPY(&i, &j); return i; } } // namespace DubboProxy diff --git a/source/extensions/filters/network/kafka/serialization.cc b/source/extensions/filters/network/kafka/serialization.cc index cc818bc7f292c..1f1d2643f3a8b 100644 --- a/source/extensions/filters/network/kafka/serialization.cc +++ b/source/extensions/filters/network/kafka/serialization.cc @@ -73,7 +73,7 @@ uint32_t feedBytesIntoBuffers(absl::string_view& data, DeserializerType& length_ const uint32_t data_consumed = std::min(required, data.size()); const uint32_t written = data_buffer.size() - required; if (data_consumed > 0) { - memcpy(data_buffer.data() + written, data.data(), data_consumed); + memcpy(data_buffer.data() + written, data.data(), data_consumed); // NOLINT(safe-memcpy) required -= data_consumed; data = {data.data() + data_consumed, data.size() - data_consumed}; } @@ -167,7 +167,7 @@ feedCompactBytesIntoBuffers(absl::string_view& data, VarUInt32Deserializer& leng const uint32_t data_consumed = std::min(required, data.size()); const uint32_t written = data_buffer.size() - required; if (data_consumed > 0) { - memcpy(data_buffer.data() + written, data.data(), data_consumed); + memcpy(data_buffer.data() + written, data.data(), data_consumed); // NOLINT(safe-memcpy) required -= data_consumed; data = {data.data() + data_consumed, data.size() - data_consumed}; } diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 65a2b18e78e79..f0318383139d1 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -69,7 +69,7 @@ template class IntDeserializer : public Deserializer { public: uint32_t feed(absl::string_view& data) override { const uint32_t available = std::min(sizeof(buf_) - written_, data.size()); - memcpy(buf_ + written_, data.data(), available); + memcpy(buf_ + written_, data.data(), available); // NOLINT(safe-memcpy) written_ += available; if (written_ == sizeof(buf_)) { @@ -96,7 +96,7 @@ class Int8Deserializer : public IntDeserializer { public: int8_t get() const override { int8_t result; - memcpy(&result, buf_, sizeof(result)); + SAFE_MEMCPY(&result, buf_); return result; } }; @@ -108,7 +108,7 @@ class Int16Deserializer : public IntDeserializer { public: int16_t get() const override { int16_t result; - memcpy(&result, buf_, sizeof(result)); + SAFE_MEMCPY(&result, buf_); return be16toh(result); } }; @@ -120,7 +120,7 @@ class Int32Deserializer : public IntDeserializer { public: int32_t get() const override { int32_t result; - memcpy(&result, buf_, sizeof(result)); + SAFE_MEMCPY(&result, buf_); return be32toh(result); } }; @@ -132,7 +132,7 @@ class UInt32Deserializer : public IntDeserializer { public: uint32_t get() const override { uint32_t result; - memcpy(&result, buf_, sizeof(result)); + SAFE_MEMCPY(&result, buf_); return be32toh(result); } }; @@ -144,7 +144,7 @@ class Int64Deserializer : public IntDeserializer { public: int64_t get() const override { int64_t result; - memcpy(&result, buf_, sizeof(result)); + SAFE_MEMCPY(&result, buf_); return be64toh(result); } }; @@ -189,7 +189,7 @@ class VarUInt32Deserializer : public Deserializer { // Read next byte from input. uint8_t el; - memcpy(&el, data.data(), sizeof(uint8_t)); + SAFE_MEMCPY(&el, data.data()); data = {data.data() + 1, data.size() - 1}; processed++; diff --git a/source/extensions/filters/network/kafka/tagged_fields.h b/source/extensions/filters/network/kafka/tagged_fields.h index 7e60952c03ce8..0678d9053f326 100644 --- a/source/extensions/filters/network/kafka/tagged_fields.h +++ b/source/extensions/filters/network/kafka/tagged_fields.h @@ -70,7 +70,7 @@ class TaggedFieldDeserializer : public Deserializer { const uint32_t data_consumed = std::min(required_, data.size()); const uint32_t written = data_buffer_.size() - required_; if (data_consumed > 0) { - memcpy(data_buffer_.data() + written, data.data(), data_consumed); + memcpy(data_buffer_.data() + written, data.data(), data_consumed); // NOLINT(safe-memcpy) required_ -= data_consumed; data = {data.data() + data_consumed, data.size() - data_consumed}; } diff --git a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc index 8267b0313441c..049726a190481 100644 --- a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc @@ -21,7 +21,7 @@ double BufferHelper::drainBEDouble(Buffer::Instance& buffer) { // 4. Implementation of last resort is to manually copy from i to d via unsigned char*. uint64_t i = buffer.drainBEInt(); double d; - std::memcpy(&d, &i, 8); + SAFE_MEMCPY(&d, &i); return d; } @@ -121,7 +121,7 @@ void BufferHelper::writeBEDouble(Buffer::Instance& buffer, double value) { // See drainDouble for implementation details. uint64_t i; - std::memcpy(&i, &value, 8); + SAFE_MEMCPY(&i, &value); buffer.writeBEInt(i); } diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_client_stream.cc b/source/extensions/quic_listeners/quiche/envoy_quic_client_stream.cc index 866e35416b0b3..ae174f282795a 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_client_stream.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_client_stream.cc @@ -154,7 +154,7 @@ void EnvoyQuicClientStream::OnBodyAvailable() { buffer->reserve(bytes_read, &slice, 1); ASSERT(slice.len_ >= bytes_read); slice.len_ = bytes_read; - memcpy(slice.mem_, iov.iov_base, iov.iov_len); + memcpy(slice.mem_, iov.iov_base, iov.iov_len); // NOLINT(safe-memcpy) buffer->commit(&slice, 1); MarkConsumed(bytes_read); } diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc index d5e5726bf3693..b9e0de6ae64c0 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc @@ -174,7 +174,7 @@ void EnvoyQuicServerStream::OnBodyAvailable() { buffer->reserve(bytes_read, &slice, 1); ASSERT(slice.len_ >= bytes_read); slice.len_ = bytes_read; - memcpy(slice.mem_, iov.iov_base, iov.iov_len); + memcpy(slice.mem_, iov.iov_base, iov.iov_len); // NOLINT(safe-memcpy) buffer->commit(&slice, 1); MarkConsumed(bytes_read); } diff --git a/source/extensions/quic_listeners/quiche/platform/string_utils.cc b/source/extensions/quic_listeners/quiche/platform/string_utils.cc index 24ef55bfe94a2..e72130705720a 100644 --- a/source/extensions/quic_listeners/quiche/platform/string_utils.cc +++ b/source/extensions/quic_listeners/quiche/platform/string_utils.cc @@ -87,7 +87,7 @@ bool HexDecodeToUInt32(absl::string_view data, uint32_t* out) { RELEASE_ASSERT(byte_string.size() == 4u, "padded data is not 4 byte long."); uint32_t bytes; - memcpy(&bytes, byte_string.data(), byte_string.length()); + memcpy(&bytes, byte_string.data(), byte_string.length()); // NOLINT(safe-memcpy) *out = ntohl(bytes); return true; } diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 3903279af2d70..26c7317c8db76 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -210,13 +210,12 @@ void TcpStatsdSink::TlsSink::commonFlush(const std::string& name, uint64_t value // This written this way for maximum perf since with a large number of stats and at a high flush // rate this can become expensive. const char* snapped_current = current_slice_mem_; - // memcpy(current_slice_mem_, parent_.getPrefix().c_str(), - // parent_.getPrefix().size()); - strcpy(current_slice_mem_, parent_.getPrefix().c_str()); + memcpy(current_slice_mem_, parent_.getPrefix().c_str(),\ // NOLINT(safe-memcpy) + parent_.getPrefix() + .size()); current_slice_mem_ += parent_.getPrefix().size(); *current_slice_mem_++ = '.'; - // memcpy(current_slice_mem_, name.c_str(), name.size()); - strcpy(current_slice_mem_, name.c_str()); + memcpy(current_slice_mem_, name.c_str(), name.size()); // NOLINT(safe-memcpy) current_slice_mem_ += name.size(); *current_slice_mem_++ = ':'; current_slice_mem_ += StringUtil::itoa(current_slice_mem_, 30, value); diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index e5b8bc17085f8..e7d96f614ea51 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -116,14 +116,15 @@ std::string Utility::generalNameAsString(const GENERAL_NAME* general_name) { sockaddr_in sin; sin.sin_port = 0; sin.sin_family = AF_INET; - memcpy(&sin.sin_addr, general_name->d.ip->data, sizeof(sin.sin_addr)); + memcpy(&sin.sin_addr, general_name->d.ip->data, sizeof(sin.sin_addr)); // NOLINT(safe-memcpy) Network::Address::Ipv4Instance addr(&sin); san = addr.ip()->addressAsString(); } else if (general_name->d.ip->length == 16) { sockaddr_in6 sin6; sin6.sin6_port = 0; sin6.sin6_family = AF_INET6; - memcpy(&sin6.sin6_addr, general_name->d.ip->data, sizeof(sin6.sin6_addr)); + memcpy(&sin6.sin6_addr, general_name->d.ip->data, // NOLINT(safe-memcpy) + sizeof(sin6.sin6_addr)); Network::Address::Ipv6Instance addr(sin6); san = addr.ip()->addressAsString(); } From dc0d096daee3ae60dc00c64fbd739103d74cbf37 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 18:39:37 +0000 Subject: [PATCH 033/106] Other applications of memcpy reviewed Signed-off-by: grial1 --- source/common/chromium_url/url_canon.h | 6 ++---- source/common/common/utility.cc | 6 ++---- source/extensions/filters/network/kafka/serialization.h | 8 ++++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/source/common/chromium_url/url_canon.h b/source/common/chromium_url/url_canon.h index 200f111bb79e0..47a3f881403b3 100644 --- a/source/common/chromium_url/url_canon.h +++ b/source/common/chromium_url/url_canon.h @@ -13,7 +13,6 @@ #include "common/chromium_url/envoy_shim.h" #include "common/chromium_url/url_parse.h" -#include "common/common/mem_block_builder.h" namespace chromium_url { @@ -147,9 +146,8 @@ template class RawCanonOutputT : public void Resize(int sz) override { T* new_buf = new T[sz]; - size_t copy_size = sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz); - Envoy::MemBlockBuilder mem_builder(new_buf, copy_size); - mem_builder.appendData(absl::Span(this->buffer_, copy_size)); + memcpy(new_buf, this->buffer_, // NOLINT(safe-memcpy) + sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz)); if (this->buffer_ != fixed_buffer_) delete[] this->buffer_; this->buffer_ = new_buf; diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 488a745de73e6..bfe1eef609f24 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -13,7 +13,6 @@ #include "common/common/assert.h" #include "common/common/fmt.h" #include "common/common/hash.h" -#include "common/common/mem_block_builder.h" #include "common/singleton/const_singleton.h" #include "absl/container/node_hash_map.h" @@ -44,7 +43,7 @@ const std::string errorDetails(int error_code) { #ifndef WIN32 // clang-format off return strerror(error_code); -// clang-format on + // clang-format on #else // Windows error codes do not correspond to POSIX errno values // Use FormatMessage, strip trailing newline, and return "Unknown error" on failure (as on POSIX). @@ -570,8 +569,7 @@ double WelfordStandardDeviation::computeStandardDeviation() const { InlineString::InlineString(const char* str, size_t size) : size_(size) { RELEASE_ASSERT(size <= 0xffffffff, "size must fit in 32 bits"); - MemBlockBuilder mem_builder(data_, size); - mem_builder.appendData(absl::Span(const_cast(str), size)); + memcpy(data_, str, size); // NOLINT(safe-memcpy) } void ExceptionUtil::throwEnvoyException(const std::string& message) { diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index f0318383139d1..c316f49124590 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -96,7 +96,7 @@ class Int8Deserializer : public IntDeserializer { public: int8_t get() const override { int8_t result; - SAFE_MEMCPY(&result, buf_); + memcpy(&result, buf_, sizeof(int8_t)); // NOLINT(safe-memcpy) return result; } }; @@ -120,7 +120,7 @@ class Int32Deserializer : public IntDeserializer { public: int32_t get() const override { int32_t result; - SAFE_MEMCPY(&result, buf_); + memcpy(&result, buf_, sizeof(int32_t)); // NOLINT(safe-memcpy) return be32toh(result); } }; @@ -132,7 +132,7 @@ class UInt32Deserializer : public IntDeserializer { public: uint32_t get() const override { uint32_t result; - SAFE_MEMCPY(&result, buf_); + memcpy(&result, buf_, sizeof(uint32_t)); // NOLINT(safe-memcpy) return be32toh(result); } }; @@ -144,7 +144,7 @@ class Int64Deserializer : public IntDeserializer { public: int64_t get() const override { int64_t result; - SAFE_MEMCPY(&result, buf_); + memcpy(&result, buf_, sizeof(int64_t)); // NOLINT(safe-memcpy) return be64toh(result); } }; From a980bbdc8b7d2d971a87abf4594251d0d44c7737 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 19:16:21 +0000 Subject: [PATCH 034/106] assert error corrected for extensions/filters/network/kafka/serialization.h Signed-off-by: grial1 --- source/extensions/filters/network/kafka/serialization.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index c316f49124590..ecdc7d36807b4 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -108,7 +108,7 @@ class Int16Deserializer : public IntDeserializer { public: int16_t get() const override { int16_t result; - SAFE_MEMCPY(&result, buf_); + memcpy(&result, buf_, sizeof(int16_t)); // NOLINT(safe-memcpy) return be16toh(result); } }; From d1a389821f10b3e32ce74e9bcd282e049d1e1dc2 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 21:20:48 +0000 Subject: [PATCH 035/106] SAFE_MEMCPY reviewed due to difference in array sizes Signed-off-by: grial1 --- source/common/chromium_url/url_canon.h | 2 +- source/common/http/http2/codec_impl.cc | 2 +- source/common/network/address_impl.cc | 5 ++--- source/extensions/common/wasm/context.cc | 2 +- source/extensions/filters/network/kafka/serialization.h | 2 +- source/extensions/stat_sinks/common/statsd/statsd.cc | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/common/chromium_url/url_canon.h b/source/common/chromium_url/url_canon.h index 47a3f881403b3..77f12739bcdd4 100644 --- a/source/common/chromium_url/url_canon.h +++ b/source/common/chromium_url/url_canon.h @@ -147,7 +147,7 @@ template class RawCanonOutputT : public void Resize(int sz) override { T* new_buf = new T[sz]; memcpy(new_buf, this->buffer_, // NOLINT(safe-memcpy) - sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz)); + sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz)); if (this->buffer_ != fixed_buffer_) delete[] this->buffer_; this->buffer_ = new_buf; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 5022b0df8b0a8..318e1c8686fd8 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -751,7 +751,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // to match the ping and ack. uint64_t data; static_assert(sizeof(data) == sizeof(frame->ping.opaque_data), "Sizes are equal"); - SAFE_MEMCPY(&data, frame->ping.opaque_data); + memcpy(&data, frame->ping.opaque_data, sizeof(data)); // NOLINT(safe-memcpy) ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 4cfed726d215f..f4f5162349483 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -175,9 +175,8 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - memcpy(static_cast(&result),\ // NOLINT(safe-memcpy) - static_cast(&address_.sin6_addr.s6_addr), - sizeof(absl::uint128)); + memcpy(static_cast(&result), // NOLINT(safe-memcpy) + static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); return result; } diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index 76575fde08abf..6865ee06c302b 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -201,7 +201,7 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti b += sizeof(uint32_t); }; for (auto& e : response) { - memcpy(b, e.address_->asStringView().data(),\ // NOLINT(safe-memcpy) + memcpy(b, e.address_->asStringView().data(), // NOLINT(safe-memcpy) e.address_->asStringView() .size()); b += e.address_->asStringView().size(); diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index ecdc7d36807b4..2c30a36cc0cad 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -132,7 +132,7 @@ class UInt32Deserializer : public IntDeserializer { public: uint32_t get() const override { uint32_t result; - memcpy(&result, buf_, sizeof(uint32_t)); // NOLINT(safe-memcpy) + memcpy(&result, buf_, sizeof(uint32_t)); // NOLINT(safe-memcpy) return be32toh(result); } }; diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 26c7317c8db76..d9d7149d767c5 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -210,7 +210,7 @@ void TcpStatsdSink::TlsSink::commonFlush(const std::string& name, uint64_t value // This written this way for maximum perf since with a large number of stats and at a high flush // rate this can become expensive. const char* snapped_current = current_slice_mem_; - memcpy(current_slice_mem_, parent_.getPrefix().c_str(),\ // NOLINT(safe-memcpy) + memcpy(current_slice_mem_, parent_.getPrefix().c_str(), // NOLINT(safe-memcpy) parent_.getPrefix() .size()); current_slice_mem_ += parent_.getPrefix().size(); From f5944441a66ec10bb7083a15624a70f99fa671b8 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 7 Nov 2020 21:58:54 +0000 Subject: [PATCH 036/106] minor corrections Signed-off-by: grial1 --- source/common/http/http2/codec_impl_legacy.cc | 3 ++- source/extensions/common/wasm/context.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index d59e97d13877c..f6e7bbe67a21b 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -729,7 +729,8 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - SAFE_MEMCPY(&data, frame->ping.opaque_data); + static_assert(sizeof(data) == sizeof(frame->ping.opaque_data), "Sizes are equal"); + memcpy(&data, frame->ping.opaque_data, sizeof(data)); // NOLINT(safe-memcpy) ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index 6865ee06c302b..5c4b0de7dd97d 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -276,7 +276,7 @@ void Context::onStatsUpdate(Envoy::Stats::MetricSnapshot& snapshot) { n = counter.counter_.get().name().size(); memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); - memcpy(b, counter.counter_.get().name().data(),\ // NOLINT(safe-memcpy) + memcpy(b, counter.counter_.get().name().data(), // NOLINT(safe-memcpy) counter.counter_.get() .name() .size()); From ead0075bd356b1ac8b9aad9225b41a5e8d91823d Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 8 Nov 2020 10:01:55 +0000 Subject: [PATCH 037/106] Format fixed Signed-off-by: grial1 --- source/extensions/common/wasm/context.cc | 7 ++----- source/extensions/stat_sinks/common/statsd/statsd.cc | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index 5c4b0de7dd97d..09a7451d6e840 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -202,8 +202,7 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti }; for (auto& e : response) { memcpy(b, e.address_->asStringView().data(), // NOLINT(safe-memcpy) - e.address_->asStringView() - .size()); + e.address_->asStringView().size()); b += e.address_->asStringView().size(); *b++ = 0; }; @@ -277,9 +276,7 @@ void Context::onStatsUpdate(Envoy::Stats::MetricSnapshot& snapshot) { memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) b += sizeof(uint32_t); memcpy(b, counter.counter_.get().name().data(), // NOLINT(safe-memcpy) - counter.counter_.get() - .name() - .size()); + counter.counter_.get().name().size()); b = align(b + counter.counter_.get().name().size()); v = counter.counter_.get().value(); memcpy(b, &v, sizeof(uint64_t)); // NOLINT(safe-memcpy) diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index d9d7149d767c5..2aab8dc983c41 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -211,8 +211,7 @@ void TcpStatsdSink::TlsSink::commonFlush(const std::string& name, uint64_t value // rate this can become expensive. const char* snapped_current = current_slice_mem_; memcpy(current_slice_mem_, parent_.getPrefix().c_str(), // NOLINT(safe-memcpy) - parent_.getPrefix() - .size()); + parent_.getPrefix().size()); current_slice_mem_ += parent_.getPrefix().size(); *current_slice_mem_++ = '.'; memcpy(current_slice_mem_, name.c_str(), name.size()); // NOLINT(safe-memcpy) From f3b29f6231168fdd0bb9fbce3223ae4cbcf79760 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 8 Nov 2020 12:47:41 +0000 Subject: [PATCH 038/106] wrong file submitted. dns_parser.cc fix added Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 05ac0438eac53..89ba86ee76c82 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - SAFE_MEMCPY(&(header_.flags), &data); + SAFE_MEMCPY(&(context->header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -743,7 +743,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - SAFE_MEMCPY(&flags, &(response_header_.flags)); + SAFE_MEMCPY(&flags, &(query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); From 4e502c89fd093b4615dc4c33950f6e1146286278 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 11 Nov 2020 00:15:05 +0000 Subject: [PATCH 039/106] changes suggested by jmarantz were added and some tests approved. Signed-off-by: grial1 --- source/common/chromium_url/url_canon.h | 8 ++++---- source/common/common/BUILD | 1 + source/common/common/hash.h | 3 ++- source/common/http/http2/codec_impl.cc | 3 +-- source/common/http/http2/codec_impl_legacy.cc | 3 +-- source/common/network/address_impl.cc | 5 +++-- .../filters/listener/proxy_protocol/proxy_protocol.cc | 4 ++-- .../filters/network/mongo_proxy/bson_impl.cc | 11 ++++++----- 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/source/common/chromium_url/url_canon.h b/source/common/chromium_url/url_canon.h index 77f12739bcdd4..2beef41914b67 100644 --- a/source/common/chromium_url/url_canon.h +++ b/source/common/chromium_url/url_canon.h @@ -11,6 +11,7 @@ #include #include +#include "common/common/mem_block_builder.h" #include "common/chromium_url/envoy_shim.h" #include "common/chromium_url/url_parse.h" @@ -145,12 +146,11 @@ template class RawCanonOutputT : public } void Resize(int sz) override { - T* new_buf = new T[sz]; - memcpy(new_buf, this->buffer_, // NOLINT(safe-memcpy) - sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz)); + Envoy::MemBlockBuilder new_buf(sz); + new_buf.appendData(absl::Span(this->buffer, std::min(this->cur_len_, sz))); if (this->buffer_ != fixed_buffer_) delete[] this->buffer_; - this->buffer_ = new_buf; + this->buffer_ = new_buf.releasePointer(); this->buffer_len_ = sz; } diff --git a/source/common/common/BUILD b/source/common/common/BUILD index ac16e8cabe393..8e4072b9821d1 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -106,6 +106,7 @@ envoy_cc_library( name = "hash_lib", srcs = ["hash.cc"], hdrs = ["hash.h"], + deps = [":macros"], external_deps = ["xxhash"], ) diff --git a/source/common/common/hash.h b/source/common/common/hash.h index 3cb43d20c6d8b..eaf697f3f9925 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -6,6 +6,7 @@ #include "absl/container/flat_hash_set.h" #include "absl/strings/ascii.h" #include "absl/strings/string_view.h" +#include "common/common/macros.h" #include "xxhash.h" namespace Envoy { @@ -58,7 +59,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - memcpy(&result, p, sizeof(result)); // NOLINT(safe-memcpy) + SAFE_MEMCPY(&result, reinterpret_cast(p)); return result; } diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index f474e9bc5e6e8..fee94a2910748 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -754,8 +754,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - static_assert(sizeof(data) == sizeof(frame->ping.opaque_data), "Sizes are equal"); - memcpy(&data, frame->ping.opaque_data, sizeof(data)); // NOLINT(safe-memcpy) + SAFE_MEMCPY(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index a8a836c55a5e6..6fcd5e44c7f5e 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -730,8 +730,7 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - static_assert(sizeof(data) == sizeof(frame->ping.opaque_data), "Sizes are equal"); - memcpy(&data, frame->ping.opaque_data, sizeof(data)); // NOLINT(safe-memcpy) + SAFE_MEMCPY(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index f4f5162349483..e5178a6c72afd 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -175,8 +175,9 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - memcpy(static_cast(&result), // NOLINT(safe-memcpy) - static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); + SAFE_MEMCPY(&result, &address_.sin6_addr.s6_addr); + //memcpy(static_cast(&result), // NOLINT(safe-memcpy) + // static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); return result; } diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index e2a4673ab5dc3..05072e2840de2 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -208,11 +208,11 @@ void Filter::parseV2Header(char* buf) { memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; ra6.sin6_port = v6->src_port; - SAFE_MEMCPY(ra6.sin6_addr.s6_addr, v6->src_addr); + SAFE_MEMCPY(&(ra6.sin6_addr.s6_addr), &(v6->src_addr)); la6.sin6_family = AF_INET6; la6.sin6_port = v6->dst_port; - SAFE_MEMCPY(la6.sin6_addr.s6_addr, v6->dst_addr); + SAFE_MEMCPY(&(la6.sin6_addr.s6_addr), &(v6->dst_addr)); proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 46d0f297cee58..b169653608f94 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -23,7 +23,8 @@ int32_t BufferHelper::peekInt32(Buffer::Instance& data) { } int32_t val; - SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int32_t)))); + void* mem = data.linearize(sizeof(int32_t)); + std::memcpy(reinterpret_cast(&val), mem, sizeof(int32_t)); // NOLINT(safe-memcpy) return le32toh(val); } @@ -43,9 +44,8 @@ void BufferHelper::removeBytes(Buffer::Instance& data, uint8_t* out, size_t out_ throw EnvoyException("invalid buffer size"); } - MemBlockBuilder mem_builder(out, out_len); - mem_builder.appendData( - absl::Span(static_cast(data.linearize(out_len)), out_len)); + void* mem = data.linearize(out_len); + std::memcpy(out, mem, out_len); // NOLINT(safe-memcpy) data.drain(out_len); } @@ -89,7 +89,8 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { } int64_t val; - SAFE_MEMCPY(&val, static_cast(data.linearize(sizeof(int64_t)))); + void* mem = data.linearize(sizeof(int64_t)); + std::memcpy(reinterpret_cast(&val), mem, sizeof(int64_t)); // NOLINT(safe-memcpy) data.drain(sizeof(int64_t)); return le64toh(val); } From 42105e0aa56c87038af1f19a712ebdfafe2c11c6 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 11 Nov 2020 00:56:27 +0000 Subject: [PATCH 040/106] format fixed Signed-off-by: grial1 --- source/common/chromium_url/url_canon.h | 2 +- source/common/common/BUILD | 2 +- source/common/common/hash.h | 3 ++- source/common/network/address_impl.cc | 2 -- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/common/chromium_url/url_canon.h b/source/common/chromium_url/url_canon.h index 2beef41914b67..a5d0939cb7616 100644 --- a/source/common/chromium_url/url_canon.h +++ b/source/common/chromium_url/url_canon.h @@ -11,9 +11,9 @@ #include #include -#include "common/common/mem_block_builder.h" #include "common/chromium_url/envoy_shim.h" #include "common/chromium_url/url_parse.h" +#include "common/common/mem_block_builder.h" namespace chromium_url { diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 8e4072b9821d1..b29e005b66691 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -106,8 +106,8 @@ envoy_cc_library( name = "hash_lib", srcs = ["hash.cc"], hdrs = ["hash.h"], - deps = [":macros"], external_deps = ["xxhash"], + deps = [":macros"], ) envoy_cc_library( diff --git a/source/common/common/hash.h b/source/common/common/hash.h index eaf697f3f9925..6cb8096f09225 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -2,11 +2,12 @@ #include +#include "common/common/macros.h" + #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/strings/ascii.h" #include "absl/strings/string_view.h" -#include "common/common/macros.h" #include "xxhash.h" namespace Envoy { diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index e5178a6c72afd..09af78bb1e153 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -176,8 +176,6 @@ absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); SAFE_MEMCPY(&result, &address_.sin6_addr.s6_addr); - //memcpy(static_cast(&result), // NOLINT(safe-memcpy) - // static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); return result; } From d714e2ebe69eb65ec3b191025128874683e5f8dc Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 11 Nov 2020 11:32:49 +0000 Subject: [PATCH 041/106] address_impl.cc compilation error fixed Signed-off-by: grial1 --- source/common/network/address_impl.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 09af78bb1e153..ea13c59c054ef 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -175,7 +175,15 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - SAFE_MEMCPY(&result, &address_.sin6_addr.s6_addr); +#ifdef __SIZEOF_INT128__ + unsigned __int128 ipv6_addr; + SAFE_MEMCPY(&ipv6_addr, &(address_.sin6_addr.s6_addr)); + result = ipv6_addr; // Use assignment operator which defaults to copy-constructor +#else + memcpy(static_cast(&result), + static_cast(&address_.sin6_addr.s6_addr), // NOLINT(safe-memcpy) + sizeof(absl::uint128)); +#endif return result; } From b7f136c061a22103875dceb42509fe9e09bf3b9d Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 11 Nov 2020 17:09:03 +0000 Subject: [PATCH 042/106] comment fixed in address_impl Signed-off-by: grial1 --- source/common/network/address_impl.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index ea13c59c054ef..7b4d916149d42 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -180,9 +180,8 @@ absl::uint128 Ipv6Instance::Ipv6Helper::address() const { SAFE_MEMCPY(&ipv6_addr, &(address_.sin6_addr.s6_addr)); result = ipv6_addr; // Use assignment operator which defaults to copy-constructor #else - memcpy(static_cast(&result), - static_cast(&address_.sin6_addr.s6_addr), // NOLINT(safe-memcpy) - sizeof(absl::uint128)); + memcpy(static_cast(&result), // NOLINT(safe-memcpy) + static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); #endif return result; } From 596c784fe37958a7e5602a029bcded014ba3496f Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 11 Nov 2020 17:55:15 +0000 Subject: [PATCH 043/106] initial set of changes. Added MemBlockBuilder in proxy protocol implementation Signed-off-by: grial1 --- source/common/grpc/common.cc | 13 +++++----- source/common/network/address_impl.cc | 1 + source/extensions/common/proxy_protocol/BUILD | 1 + .../proxy_protocol/proxy_protocol_header.cc | 24 +++++++++---------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index fc7bf872e923d..c3b56e8675758 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -138,8 +138,7 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - std::memcpy(current, reinterpret_cast(&nsize), // NOLINT(safe-memcpy) - sizeof(uint32_t)); + SAFE_MEMCPY(reinterpret_cast(current), nsize); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); @@ -290,12 +289,12 @@ std::string Common::typeUrl(const std::string& qualified_name) { } void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { - std::array header; - header[0] = 0; // flags + + Envoy::MemBlockBuilder header(5); + header.appendOne(0); // flags const uint32_t nsize = htonl(buffer.length()); - std::memcpy(&header[1], reinterpret_cast(&nsize), // NOLINT(safe-memcpy) - sizeof(uint32_t)); - buffer.prepend(absl::string_view(&header[0], 5)); + header.appendData(absl::Span(reinterpret_cast(nsize), sizeof(uint32_t))); + buffer.prepend(absl::string_view(header.releasePointer(), 5)); } bool Common::parseBufferInstance(Buffer::InstancePtr&& buffer, Protobuf::Message& proto) { diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 7b4d916149d42..830a251c32d4f 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -287,6 +287,7 @@ PipeInstance::PipeInstance(const std::string& pipe_path, mode_t mode, } pipe_.abstract_namespace_ = true; pipe_.address_length_ = pipe_path.size(); + // The following statement is safe since pipe_path size was checked at the beginning of this function memcpy(&pipe_.address_.sun_path[0], pipe_path.data(), pipe_path.size()); // NOLINT(safe-memcpy) pipe_.address_.sun_path[0] = '\0'; pipe_.address_.sun_path[pipe_path.size()] = '\0'; diff --git a/source/extensions/common/proxy_protocol/BUILD b/source/extensions/common/proxy_protocol/BUILD index 7a2b9bf66d034..c42efab15efae 100644 --- a/source/extensions/common/proxy_protocol/BUILD +++ b/source/extensions/common/proxy_protocol/BUILD @@ -19,6 +19,7 @@ envoy_cc_library( "//include/envoy/network:address_interface", "//include/envoy/network:connection_interface", "//source/common/network:address_lib", + "//source/common/common:mem_block_builder_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index d80444d252709..9244d99bd6751 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -4,7 +4,7 @@ #include "envoy/buffer/buffer.h" #include "envoy/network/address.h" - +#include "common/common/mem_builder_impl.h" #include "common/network/address_impl.h" namespace Envoy { @@ -67,37 +67,37 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, addr_length[1] = PROXY_PROTO_V2_ADDR_LEN_INET; out.add(addr_length, 2); - uint8_t addrs[8]; + MemBlockBuilder addrs(8); const auto net_src_addr = Network::Address::Ipv4Instance(src_addr, src_port).ip()->ipv4()->address(); const auto net_dst_addr = Network::Address::Ipv4Instance(dst_addr, dst_port).ip()->ipv4()->address(); - memcpy(addrs, &net_src_addr, 4); // NOLINT(safe-memcpy) - memcpy(&addrs[4], &net_dst_addr, 4); // NOLINT(safe-memcpy) - out.add(addrs, 8); + addrs.appendData(absl::Span(&net_src_addr, 4)); + addrs.appendData(absl::Span(&net_dst_addr, 4)); + out.add(addrs.releasePointer(), 8); break; } case Network::Address::IpVersion::v6: { addr_length[1] = PROXY_PROTO_V2_ADDR_LEN_INET6; out.add(addr_length, 2); - uint8_t addrs[32]; + MemBlockBuilder addrs(32); const auto net_src_addr = Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const auto net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - memcpy(addrs, &net_src_addr, 16); // NOLINT(safe-memcpy) - memcpy(&addrs[16], &net_dst_addr, 16); // NOLINT(safe-memcpy) - out.add(addrs, 32); + addrs.appendData(absl::Span(&net_src_addr, 16)); + addrs.appendData(absl::Span(&net_dst_addr, 16)); + out.add(addrs.realsePointer(), 32); break; } } - uint8_t ports[4]; + MemBlockBuilder ports(4); const auto net_src_port = htons(static_cast(src_port)); const auto net_dst_port = htons(static_cast(dst_port)); - memcpy(ports, &net_src_port, 2); // NOLINT(safe-memcpy) - memcpy(&ports[2], &net_dst_port, 2); // NOLINT(safe-memcpy) + ports.appendData(absl::Span(&net_src_port, 2)); + ports.appendData(absl::Span(&net_dst_port, 2)); out.add(ports, 4); } From 6f0305f02be3b977e061f7ecd5c5a15230af1006 Mon Sep 17 00:00:00 2001 From: grial1 Date: Fri, 13 Nov 2020 21:44:45 +0000 Subject: [PATCH 044/106] proposed changes applied Signed-off-by: grial1 --- source/common/common/mem_block_builder.h | 28 ++++++------------- source/common/grpc/BUILD | 1 + source/common/grpc/common.cc | 7 +++-- source/common/network/address_impl.cc | 8 +----- .../proxy_protocol/proxy_protocol_header.cc | 18 ++++++------ 5 files changed, 24 insertions(+), 38 deletions(-) diff --git a/source/common/common/mem_block_builder.h b/source/common/common/mem_block_builder.h index e3b93e8907398..6d02d2d789587 100644 --- a/source/common/common/mem_block_builder.h +++ b/source/common/common/mem_block_builder.h @@ -26,13 +26,7 @@ template class MemBlockBuilder { public: // Constructs a MemBlockBuilder allowing for 'capacity' instances of T. explicit MemBlockBuilder(uint64_t capacity) - : owned_data_(std::make_unique(capacity)), write_span_(owned_data_.get(), capacity) { - data_ = owned_data_.get(); - }; - explicit MemBlockBuilder(T* data, uint64_t capacity) - : owned_data_(nullptr), write_span_(data, capacity) { - data_ = data; - }; + : data_(std::make_unique(capacity)), write_span_(data_.get(), capacity) {}; MemBlockBuilder() = default; /** @@ -49,7 +43,7 @@ template class MemBlockBuilder { /** * @return the capacity. */ - uint64_t capacity() const { return write_span_.size() + write_span_.data() - owned_data_.get(); } + uint64_t capacity() const { return write_span_.size() + write_span_.data() - data_.get(); } /** * Appends a single object of type T, moving an internal write-pointer @@ -105,7 +99,7 @@ template class MemBlockBuilder { */ std::unique_ptr release() { write_span_ = absl::MakeSpan(static_cast(nullptr), 0); - return std::move(owned_data_); + return std::move(data_); } /** @@ -115,30 +109,26 @@ template class MemBlockBuilder { */ T* releasePointer() { write_span_ = absl::MakeSpan(static_cast(nullptr), 0); - if (owned_data_) - return owned_data_.release(); - else - return data_; + return data_.release(); } /** * @return the populated data as an absl::Span. */ - absl::Span span() const { return absl::MakeSpan(owned_data_.get(), write_span_.data()); } + absl::Span span() const { return absl::MakeSpan(data_.get(), write_span_.data()); } /** * @return The number of elements the have been added to the builder. */ - uint64_t size() const { return write_span_.data() - owned_data_.get(); } + uint64_t size() const { return write_span_.data() - data_.get(); } private: void setCapacityHelper(uint64_t capacity, std::unique_ptr data) { - owned_data_ = std::move(data); - write_span_ = absl::MakeSpan(owned_data_.get(), capacity); + data_ = std::move(data); + write_span_ = absl::MakeSpan(data_.get(), capacity); } - T* data_; - std::unique_ptr owned_data_; + std::unique_ptr data_; absl::Span write_span_; }; diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 3daea1ce4395f..b10ce307fed65 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -93,6 +93,7 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:hash_lib", "//source/common/common:macros", + "//source/common/common:mem_block_builder_lib", "//source/common/common:utility_lib", "//source/common/grpc:status_lib", "//source/common/http:header_utility_lib", diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index c3b56e8675758..f6981e217d5ab 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -13,6 +13,7 @@ #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/macros.h" +#include "common/common/mem_block_builder.h" #include "common/common/utility.h" #include "common/http/header_utility.h" #include "common/http/headers.h" @@ -138,7 +139,7 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - SAFE_MEMCPY(reinterpret_cast(current), nsize); + SAFE_MEMCPY(reinterpret_cast(current), &nsize); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); @@ -290,10 +291,10 @@ std::string Common::typeUrl(const std::string& qualified_name) { void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { - Envoy::MemBlockBuilder header(5); + MemBlockBuilder header(5); header.appendOne(0); // flags const uint32_t nsize = htonl(buffer.length()); - header.appendData(absl::Span(reinterpret_cast(nsize), sizeof(uint32_t))); + header.appendData(absl::Span(reinterpret_cast(nsize), sizeof(uint32_t))); buffer.prepend(absl::string_view(header.releasePointer(), 5)); } diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 830a251c32d4f..04788c92d8d12 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -173,16 +173,10 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { } absl::uint128 Ipv6Instance::Ipv6Helper::address() const { - absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); -#ifdef __SIZEOF_INT128__ unsigned __int128 ipv6_addr; SAFE_MEMCPY(&ipv6_addr, &(address_.sin6_addr.s6_addr)); - result = ipv6_addr; // Use assignment operator which defaults to copy-constructor -#else - memcpy(static_cast(&result), // NOLINT(safe-memcpy) - static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); -#endif + absl::uint128 result{ipv6_addr}; return result; } diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index 9244d99bd6751..2dcfb95984d83 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -4,7 +4,7 @@ #include "envoy/buffer/buffer.h" #include "envoy/network/address.h" -#include "common/common/mem_builder_impl.h" +#include "common/common/mem_block_builder.h" #include "common/network/address_impl.h" namespace Envoy { @@ -72,8 +72,8 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv4Instance(src_addr, src_port).ip()->ipv4()->address(); const auto net_dst_addr = Network::Address::Ipv4Instance(dst_addr, dst_port).ip()->ipv4()->address(); - addrs.appendData(absl::Span(&net_src_addr, 4)); - addrs.appendData(absl::Span(&net_dst_addr, 4)); + addrs.appendData(absl::Span(reinterpret_cast(&net_src_addr), 4)); + addrs.appendData(absl::Span(reinterpret_cast(&net_dst_addr), 4)); out.add(addrs.releasePointer(), 8); break; } @@ -86,9 +86,9 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const auto net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - addrs.appendData(absl::Span(&net_src_addr, 16)); - addrs.appendData(absl::Span(&net_dst_addr, 16)); - out.add(addrs.realsePointer(), 32); + addrs.appendData(absl::Span(reinterpret_cast(&net_src_addr), 16)); + addrs.appendData(absl::Span(reinterpret_cast(&net_dst_addr), 16)); + out.add(addrs.releasePointer(), 32); break; } } @@ -96,9 +96,9 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, MemBlockBuilder ports(4); const auto net_src_port = htons(static_cast(src_port)); const auto net_dst_port = htons(static_cast(dst_port)); - ports.appendData(absl::Span(&net_src_port, 2)); - ports.appendData(absl::Span(&net_dst_port, 2)); - out.add(ports, 4); + ports.appendData(absl::Span(reinterpret_cast(&net_src_port), 2)); + ports.appendData(absl::Span(reinterpret_cast(&net_dst_port), 2)); + out.add(ports.releasePointer(), 4); } void generateV2Header(const Network::Address::Ip& source_address, From 671a5f1fb6292b61cd162d90eee4b33bb20227eb Mon Sep 17 00:00:00 2001 From: grial1 Date: Fri, 13 Nov 2020 22:33:38 +0000 Subject: [PATCH 045/106] format fixed Signed-off-by: grial1 --- source/common/common/mem_block_builder.h | 2 +- source/common/network/address_impl.cc | 3 ++- source/extensions/common/proxy_protocol/BUILD | 2 +- .../common/proxy_protocol/proxy_protocol_header.cc | 7 +++++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/common/common/mem_block_builder.h b/source/common/common/mem_block_builder.h index 6d02d2d789587..5d90cc66a2aca 100644 --- a/source/common/common/mem_block_builder.h +++ b/source/common/common/mem_block_builder.h @@ -26,7 +26,7 @@ template class MemBlockBuilder { public: // Constructs a MemBlockBuilder allowing for 'capacity' instances of T. explicit MemBlockBuilder(uint64_t capacity) - : data_(std::make_unique(capacity)), write_span_(data_.get(), capacity) {}; + : data_(std::make_unique(capacity)), write_span_(data_.get(), capacity){}; MemBlockBuilder() = default; /** diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 04788c92d8d12..acac111781d89 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -281,7 +281,8 @@ PipeInstance::PipeInstance(const std::string& pipe_path, mode_t mode, } pipe_.abstract_namespace_ = true; pipe_.address_length_ = pipe_path.size(); - // The following statement is safe since pipe_path size was checked at the beginning of this function + // The following statement is safe since pipe_path size was checked at the beginning of this + // function memcpy(&pipe_.address_.sun_path[0], pipe_path.data(), pipe_path.size()); // NOLINT(safe-memcpy) pipe_.address_.sun_path[0] = '\0'; pipe_.address_.sun_path[pipe_path.size()] = '\0'; diff --git a/source/extensions/common/proxy_protocol/BUILD b/source/extensions/common/proxy_protocol/BUILD index c42efab15efae..fa800fb62838e 100644 --- a/source/extensions/common/proxy_protocol/BUILD +++ b/source/extensions/common/proxy_protocol/BUILD @@ -18,8 +18,8 @@ envoy_cc_library( "//include/envoy/buffer:buffer_interface", "//include/envoy/network:address_interface", "//include/envoy/network:connection_interface", - "//source/common/network:address_lib", "//source/common/common:mem_block_builder_lib", + "//source/common/network:address_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index 2dcfb95984d83..f3b2cc4ededa5 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -4,6 +4,7 @@ #include "envoy/buffer/buffer.h" #include "envoy/network/address.h" + #include "common/common/mem_block_builder.h" #include "common/network/address_impl.h" @@ -86,8 +87,10 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const auto net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - addrs.appendData(absl::Span(reinterpret_cast(&net_src_addr), 16)); - addrs.appendData(absl::Span(reinterpret_cast(&net_dst_addr), 16)); + addrs.appendData( + absl::Span(reinterpret_cast(&net_src_addr), 16)); + addrs.appendData( + absl::Span(reinterpret_cast(&net_dst_addr), 16)); out.add(addrs.releasePointer(), 32); break; } From c51bbf281ee6d474b6d0ce25869965547fd1411e Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 14 Nov 2020 11:07:31 +0000 Subject: [PATCH 046/106] bug fixed Signed-off-by: grial1 --- source/common/grpc/common.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index f6981e217d5ab..0b0719674688f 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -294,7 +294,7 @@ void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { MemBlockBuilder header(5); header.appendOne(0); // flags const uint32_t nsize = htonl(buffer.length()); - header.appendData(absl::Span(reinterpret_cast(nsize), sizeof(uint32_t))); + header.appendData(absl::Span(reinterpret_cast(&nsize), sizeof(uint32_t))); buffer.prepend(absl::string_view(header.releasePointer(), 5)); } From 882752a8a186780a075967841af401139c962764 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 14 Nov 2020 20:28:21 +0000 Subject: [PATCH 047/106] format fixed Signed-off-by: grial1 --- source/common/grpc/common.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index 0b0719674688f..3f106619c71d9 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -294,7 +294,8 @@ void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { MemBlockBuilder header(5); header.appendOne(0); // flags const uint32_t nsize = htonl(buffer.length()); - header.appendData(absl::Span(reinterpret_cast(&nsize), sizeof(uint32_t))); + header.appendData( + absl::Span(reinterpret_cast(&nsize), sizeof(uint32_t))); buffer.prepend(absl::string_view(header.releasePointer(), 5)); } From fecccfe6bb7c496b8f56a5bfe2ae8cb3d04dcd34 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 15 Nov 2020 16:51:55 +0000 Subject: [PATCH 048/106] SAFE_MEMCPY deleted from macros.h Signed-off-by: grial1 --- source/common/common/macros.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/source/common/common/macros.h b/source/common/common/macros.h index 4be21347e015e..f2b06b84f340b 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -41,15 +41,6 @@ namespace Envoy { return *objectptr; \ } while (0) -/** - * @brief Assert memory bounds to avoid copy errors. - */ -#define SAFE_MEMCPY(dst, src) \ - do { \ - static_assert(sizeof(*(src)) == sizeof(*(dst))); \ - memmove(dst, src, sizeof(*(src))); \ - } while (0) - /** * Have a generic fall-through for different versions of C++ */ From 36921666536061216c52e8a10368d1e68b6dabeb Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 15 Nov 2020 20:23:39 +0000 Subject: [PATCH 049/106] SAFE_MEMCPY macro changed for safe_memcpy template function Signed-off-by: grial1 --- source/common/common/BUILD | 10 +++++++++- source/common/common/hash.h | 3 ++- source/common/common/safe_memcpy.h | 16 ++++++++++++++++ source/common/grpc/BUILD | 1 + source/common/grpc/common.cc | 13 ++++++------- source/common/http/http2/BUILD | 1 + source/common/http/http2/codec_impl.cc | 3 ++- source/common/http/http2/codec_impl_legacy.cc | 3 ++- source/common/network/BUILD | 1 + source/common/network/address_impl.cc | 11 +++++++++-- .../filters/listener/proxy_protocol/BUILD | 1 + .../listener/proxy_protocol/proxy_protocol.cc | 5 +++-- .../extensions/filters/network/dubbo_proxy/BUILD | 1 + .../filters/network/dubbo_proxy/buffer_helper.cc | 4 ++-- .../filters/network/dubbo_proxy/buffer_helper.h | 1 + source/extensions/filters/network/kafka/BUILD | 1 + .../filters/network/kafka/serialization.h | 3 ++- .../filters/network/thrift_proxy/BUILD | 1 + .../network/thrift_proxy/buffer_helper.cc | 5 +++-- source/extensions/filters/udp/dns_filter/BUILD | 1 + .../filters/udp/dns_filter/dns_parser.cc | 4 ++-- .../filters/udp/dns_filter/dns_parser.h | 1 + source/server/hot_restarting_base.cc | 3 ++- tools/code_format/check_format.py | 2 +- 24 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 source/common/common/safe_memcpy.h diff --git a/source/common/common/BUILD b/source/common/common/BUILD index b29e005b66691..3cccfc660b8d6 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -90,6 +90,11 @@ envoy_cc_library( hdrs = ["enum_to_int.h"], ) +envoy_cc_library( + name = "safe_memcpy_lib", + hdrs = ["safe_memcpy.h"], +) + # fmt_lib is automatically a dependency of all envoy_cc_library definitions. envoy_basic_cc_library( name = "fmt_lib", @@ -107,7 +112,10 @@ envoy_cc_library( srcs = ["hash.cc"], hdrs = ["hash.h"], external_deps = ["xxhash"], - deps = [":macros"], + deps = [ + ":macros", + ":safe_memcpy_lib", + ], ) envoy_cc_library( diff --git a/source/common/common/hash.h b/source/common/common/hash.h index 6cb8096f09225..c31c237692e06 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -3,6 +3,7 @@ #include #include "common/common/macros.h" +#include "common/common/safe_memcpy.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" @@ -60,7 +61,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - SAFE_MEMCPY(&result, reinterpret_cast(p)); + safe_memcpy(&result, reinterpret_cast(p)); return result; } diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h new file mode 100644 index 0000000000000..e2367a016ad49 --- /dev/null +++ b/source/common/common/safe_memcpy.h @@ -0,0 +1,16 @@ +#pragma once + +namespace Envoy { + +/** + * @brief Assert memory bounds to avoid copy errors. + */ +template +inline void safe_memcpy(T1* dst, T2* src) { + + static_assert(sizeof(T1) == sizeof(T2)); + memcpy(dst, src, sizeof(T1)); + +} + +} // namespace Envoy diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index b10ce307fed65..74d799f947041 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -93,6 +93,7 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:hash_lib", "//source/common/common:macros", + "//source/common/common:safe_memcpy_lib", "//source/common/common:mem_block_builder_lib", "//source/common/common:utility_lib", "//source/common/grpc:status_lib", diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index 3f106619c71d9..ecd222f1ecf53 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -14,6 +14,7 @@ #include "common/common/fmt.h" #include "common/common/macros.h" #include "common/common/mem_block_builder.h" +#include "common/common/safe_memcpy.h" #include "common/common/utility.h" #include "common/http/header_utility.h" #include "common/http/headers.h" @@ -139,7 +140,7 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - SAFE_MEMCPY(reinterpret_cast(current), &nsize); + safe_memcpy(reinterpret_cast(current), &nsize); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); @@ -290,13 +291,11 @@ std::string Common::typeUrl(const std::string& qualified_name) { } void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { - - MemBlockBuilder header(5); - header.appendOne(0); // flags + std::array header; + header[0] = 0; // flags const uint32_t nsize = htonl(buffer.length()); - header.appendData( - absl::Span(reinterpret_cast(&nsize), sizeof(uint32_t))); - buffer.prepend(absl::string_view(header.releasePointer(), 5)); + std::memcpy(&header[1], reinterpret_cast(&nsize), sizeof(uint32_t)); // NOLINT(safe-memcpy) + buffer.prepend(absl::string_view(&header[0], 5)); } bool Common::parseBufferInstance(Buffer::InstancePtr&& buffer, Protobuf::Message& proto) { diff --git a/source/common/http/http2/BUILD b/source/common/http/http2/BUILD index 832decc73f681..9ddf7b9b86ef6 100644 --- a/source/common/http/http2/BUILD +++ b/source/common/http/http2/BUILD @@ -38,6 +38,7 @@ CODEC_LIB_DEPS = [ "//source/common/common:minimal_logger_lib", "//source/common/common:statusor_lib", "//source/common/common:utility_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/http:codec_helper_lib", "//source/common/http:codes_lib", "//source/common/http:exception_lib", diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index fee94a2910748..78c38c31600b9 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -14,6 +14,7 @@ #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/utility.h" +#include "common/common/safe_memcpy.h" #include "common/http/codes.h" #include "common/http/exception.h" #include "common/http/header_utility.h" @@ -754,7 +755,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - SAFE_MEMCPY(&data, &(frame->ping.opaque_data)); + safe_memcpy(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index 6fcd5e44c7f5e..ab86f4e9feb01 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -14,6 +14,7 @@ #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/utility.h" +#include "common/common/safe_memcpy.h" #include "common/http/codes.h" #include "common/http/exception.h" #include "common/http/header_utility.h" @@ -730,7 +731,7 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - SAFE_MEMCPY(&data, &(frame->ping.opaque_data)); + safe_memcpy(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/network/BUILD b/source/common/network/BUILD index 969e49f206732..6e44232283fa7 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( "//include/envoy/network:address_interface", "//source/common/api:os_sys_calls_lib", "//source/common/common:assert_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", ], ) diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index acac111781d89..4ef600c2b5d57 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -9,6 +9,7 @@ #include "common/common/assert.h" #include "common/common/fmt.h" +#include "common/common/safe_memcpy.h" #include "common/common/utility.h" #include "common/network/socket_interface.h" @@ -173,10 +174,16 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { } absl::uint128 Ipv6Instance::Ipv6Helper::address() const { + absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); +#ifdef __SIZEOF_INT128__ unsigned __int128 ipv6_addr; - SAFE_MEMCPY(&ipv6_addr, &(address_.sin6_addr.s6_addr)); - absl::uint128 result{ipv6_addr}; + safe_memcpy(&ipv6_addr, &(address_.sin6_addr.s6_addr)); + result = ipv6_addr; // Use assignment operator which defaults to copy-constructor +#else + memcpy(static_cast(&result), // NOLINT(safe-memcpy) + static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); +#endif return result; } diff --git a/source/extensions/filters/listener/proxy_protocol/BUILD b/source/extensions/filters/listener/proxy_protocol/BUILD index 302940fff6b79..affdb0bcc9aa8 100644 --- a/source/extensions/filters/listener/proxy_protocol/BUILD +++ b/source/extensions/filters/listener/proxy_protocol/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:empty_string", "//source/common/common:minimal_logger_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", "//source/common/network:address_lib", "//source/common/network:utility_lib", diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 05072e2840de2..626d19144a4c0 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -17,6 +17,7 @@ #include "common/common/assert.h" #include "common/common/empty_string.h" #include "common/common/fmt.h" +#include "common/common/safe_memcpy.h" #include "common/common/utility.h" #include "common/network/address_impl.h" #include "common/network/utility.h" @@ -208,11 +209,11 @@ void Filter::parseV2Header(char* buf) { memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; ra6.sin6_port = v6->src_port; - SAFE_MEMCPY(&(ra6.sin6_addr.s6_addr), &(v6->src_addr)); + safe_memcpy(&(ra6.sin6_addr.s6_addr), &(v6->src_addr)); la6.sin6_family = AF_INET6; la6.sin6_port = v6->dst_port; - SAFE_MEMCPY(&(la6.sin6_addr.s6_addr), &(v6->dst_addr)); + safe_memcpy(&(la6.sin6_addr.s6_addr), &(v6->dst_addr)); proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, diff --git a/source/extensions/filters/network/dubbo_proxy/BUILD b/source/extensions/filters/network/dubbo_proxy/BUILD index bf83e91ad0fd1..ee1d1edd9bd1b 100644 --- a/source/extensions/filters/network/dubbo_proxy/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/BUILD @@ -16,6 +16,7 @@ envoy_cc_library( deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:byte_order_lib", ], ) diff --git a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc index acb7127f1292b..0a2feee3e853f 100644 --- a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc @@ -11,7 +11,7 @@ double BufferHelper::peekDouble(Buffer::Instance& buffer, uint64_t offset) { } double i; uint64_t j = buffer.peekBEInt(offset); - SAFE_MEMCPY(&i, &j); + safe_memcpy(&i, &j); return i; } @@ -21,7 +21,7 @@ float BufferHelper::peekFloat(Buffer::Instance& buffer, uint64_t offset) { } float i; uint32_t j = buffer.peekBEInt(offset); - SAFE_MEMCPY(&i, &j); + safe_memcpy(&i, &j); return i; } } // namespace DubboProxy diff --git a/source/extensions/filters/network/dubbo_proxy/buffer_helper.h b/source/extensions/filters/network/dubbo_proxy/buffer_helper.h index d3020c39c0f96..cdd25c939f4b3 100644 --- a/source/extensions/filters/network/dubbo_proxy/buffer_helper.h +++ b/source/extensions/filters/network/dubbo_proxy/buffer_helper.h @@ -4,6 +4,7 @@ #include "envoy/common/exception.h" #include "common/common/assert.h" +#include "common/common/safe_memcpy.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/filters/network/kafka/BUILD b/source/extensions/filters/network/kafka/BUILD index 30b2251bbb55a..c8b672410bd79 100644 --- a/source/extensions/filters/network/kafka/BUILD +++ b/source/extensions/filters/network/kafka/BUILD @@ -230,6 +230,7 @@ envoy_cc_library( ":kafka_types_lib", "//include/envoy/buffer:buffer_interface", "//source/common/common:byte_order_lib", + "//source/common/common:safe_memcpy_lib", ], ) diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 2c30a36cc0cad..2236a1c7d0526 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -11,6 +11,7 @@ #include "common/common/byte_order.h" #include "common/common/fmt.h" +#include "common/common/safe_memcpy.h" #include "common/common/utility.h" #include "extensions/filters/network/kafka/kafka_types.h" @@ -189,7 +190,7 @@ class VarUInt32Deserializer : public Deserializer { // Read next byte from input. uint8_t el; - SAFE_MEMCPY(&el, data.data()); + safe_memcpy(&el, data.data()); data = {data.data() + 1, data.size() - 1}; processed++; diff --git a/source/extensions/filters/network/thrift_proxy/BUILD b/source/extensions/filters/network/thrift_proxy/BUILD index 78f484da3f9e8..7ed0f513db2b4 100644 --- a/source/extensions/filters/network/thrift_proxy/BUILD +++ b/source/extensions/filters/network/thrift_proxy/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:byte_order_lib", ], ) diff --git a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc index 049726a190481..05a4d2a811d42 100644 --- a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc @@ -1,6 +1,7 @@ #include "extensions/filters/network/thrift_proxy/buffer_helper.h" #include "common/common/byte_order.h" +#include "common/common/safe_memcpy.h" namespace Envoy { namespace Extensions { @@ -21,7 +22,7 @@ double BufferHelper::drainBEDouble(Buffer::Instance& buffer) { // 4. Implementation of last resort is to manually copy from i to d via unsigned char*. uint64_t i = buffer.drainBEInt(); double d; - SAFE_MEMCPY(&d, &i); + safe_memcpy(&d, &i); return d; } @@ -121,7 +122,7 @@ void BufferHelper::writeBEDouble(Buffer::Instance& buffer, double value) { // See drainDouble for implementation details. uint64_t i; - SAFE_MEMCPY(&i, &value); + safe_memcpy(&i, &value); buffer.writeBEInt(i); } diff --git a/source/extensions/filters/udp/dns_filter/BUILD b/source/extensions/filters/udp/dns_filter/BUILD index 5684b6569ed92..0fd4b5a1bbbb2 100644 --- a/source/extensions/filters/udp/dns_filter/BUILD +++ b/source/extensions/filters/udp/dns_filter/BUILD @@ -35,6 +35,7 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:empty_string", "//source/common/common:matchers_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/config:config_provider_lib", "//source/common/config:datasource_lib", "//source/common/network:address_lib", diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 89ba86ee76c82..609b39931ea47 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - SAFE_MEMCPY(&(context->header_.flags), &data); + safe_memcpy(&(context->header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -743,7 +743,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - SAFE_MEMCPY(&flags, &(query_context->response_header_.flags)); + safe_memcpy(&flags, &(query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.h b/source/extensions/filters/udp/dns_filter/dns_parser.h index d604f0784ba74..8da0fbe136bca 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.h +++ b/source/extensions/filters/udp/dns_filter/dns_parser.h @@ -9,6 +9,7 @@ #include "common/buffer/buffer_impl.h" #include "common/common/empty_string.h" +#include "common/common/safe_memcpy.h" #include "common/runtime/runtime_impl.h" #include "common/stats/timespan_impl.h" diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 16c89c53235af..02eb38ab89f45 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -2,6 +2,7 @@ #include "common/api/os_sys_calls_impl.h" #include "common/common/mem_block_builder.h" +#include "common/common/safe_memcpy.h" #include "common/common/utility.h" #include "common/network/address_impl.h" #include "common/stats/utility.h" @@ -37,7 +38,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - SAFE_MEMCPY(&address, &(addr.getSockAddr())); + safe_memcpy(&address, &(addr.getSockAddr())); fchmod(my_domain_socket_, socket_mode); return address; diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index e8c690554767a..b712e88af1afe 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -803,7 +803,7 @@ def checkSourceLine(self, line, file_path, reportError): not ("test/" in file_path) and \ ("memcpy(" in line) and \ not ("NOLINT(safe-memcpy)" in line): - reportError("Don't call memcpy() directly; use SAFE_MEMCPY or MemBlockBuilder instead.") + reportError("Don't call memcpy() directly; use safe_memcpy or MemBlockBuilder instead.") if self.denylistedForExceptions(file_path): # Skpping cases where 'throw' is a substring of a symbol like in "foothrowBar". From e3f469bc88f84fdd9754faf979fe4d08def73bd3 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 15 Nov 2020 20:38:59 +0000 Subject: [PATCH 050/106] safe_memcpy added to common/network/cidr_range.cc Signed-off-by: grial1 --- source/common/grpc/common.cc | 1 - source/common/network/BUILD | 1 + source/common/network/cidr_range.cc | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index ecd222f1ecf53..c89ca41ca2873 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -13,7 +13,6 @@ #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/macros.h" -#include "common/common/mem_block_builder.h" #include "common/common/safe_memcpy.h" #include "common/common/utility.h" #include "common/http/header_utility.h" diff --git a/source/common/network/BUILD b/source/common/network/BUILD index 6e44232283fa7..f4ec5b72d023c 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -41,6 +41,7 @@ envoy_cc_library( ":utility_lib", "//include/envoy/network:address_interface", "//source/common/common:assert_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/common/network/cidr_range.cc b/source/common/network/cidr_range.cc index 501a8de6d6e3c..57ef7eb9d8edc 100644 --- a/source/common/network/cidr_range.cc +++ b/source/common/network/cidr_range.cc @@ -11,6 +11,7 @@ #include "common/common/assert.h" #include "common/common/fmt.h" +#include "common/common/safe_memcpy.h" #include "common/common/utility.h" #include "common/network/address_impl.h" #include "common/network/utility.h" @@ -183,7 +184,7 @@ InstanceConstSharedPtr CidrRange::truncateIpAddressAndLength(InstanceConstShared absl::uint128 ip6_htonl = Utility::Ip6htonl(ip6); static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - memcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl, sizeof(absl::uint128)); // NOLINT(safe-memcpy) + safe_memcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl); return std::make_shared(sa6); } } From 40dc126ac9c32d3d160827d7a194b1b3cc50d425 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 15 Nov 2020 20:48:53 +0000 Subject: [PATCH 051/106] proxy_protocol_header.cc: function generateV2Header uses memcpy in a controlled manner Signed-off-by: grial1 --- source/common/grpc/BUILD | 1 - source/extensions/common/proxy_protocol/BUILD | 1 - .../proxy_protocol/proxy_protocol_header.cc | 29 ++++++++----------- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 74d799f947041..03028d11a3555 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -94,7 +94,6 @@ envoy_cc_library( "//source/common/common:hash_lib", "//source/common/common:macros", "//source/common/common:safe_memcpy_lib", - "//source/common/common:mem_block_builder_lib", "//source/common/common:utility_lib", "//source/common/grpc:status_lib", "//source/common/http:header_utility_lib", diff --git a/source/extensions/common/proxy_protocol/BUILD b/source/extensions/common/proxy_protocol/BUILD index fa800fb62838e..7a2b9bf66d034 100644 --- a/source/extensions/common/proxy_protocol/BUILD +++ b/source/extensions/common/proxy_protocol/BUILD @@ -18,7 +18,6 @@ envoy_cc_library( "//include/envoy/buffer:buffer_interface", "//include/envoy/network:address_interface", "//include/envoy/network:connection_interface", - "//source/common/common:mem_block_builder_lib", "//source/common/network:address_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index f3b2cc4ededa5..e069f2cef7520 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -5,7 +5,6 @@ #include "envoy/buffer/buffer.h" #include "envoy/network/address.h" -#include "common/common/mem_block_builder.h" #include "common/network/address_impl.h" namespace Envoy { @@ -67,41 +66,37 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, case Network::Address::IpVersion::v4: { addr_length[1] = PROXY_PROTO_V2_ADDR_LEN_INET; out.add(addr_length, 2); - - MemBlockBuilder addrs(8); + uint8_t addrs[8]; const auto net_src_addr = Network::Address::Ipv4Instance(src_addr, src_port).ip()->ipv4()->address(); const auto net_dst_addr = Network::Address::Ipv4Instance(dst_addr, dst_port).ip()->ipv4()->address(); - addrs.appendData(absl::Span(reinterpret_cast(&net_src_addr), 4)); - addrs.appendData(absl::Span(reinterpret_cast(&net_dst_addr), 4)); - out.add(addrs.releasePointer(), 8); + memcpy(addrs, &net_src_addr, 4); // NOLINT(safe-memcpy) + memcpy(&addrs[4], &net_dst_addr, 4); // NOLINT(safe-memcpy) + out.add(addrs, 8); break; } case Network::Address::IpVersion::v6: { addr_length[1] = PROXY_PROTO_V2_ADDR_LEN_INET6; out.add(addr_length, 2); - - MemBlockBuilder addrs(32); + uint8_t addrs[32]; const auto net_src_addr = Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const auto net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - addrs.appendData( - absl::Span(reinterpret_cast(&net_src_addr), 16)); - addrs.appendData( - absl::Span(reinterpret_cast(&net_dst_addr), 16)); - out.add(addrs.releasePointer(), 32); + memcpy(addrs, &net_src_addr, 16); // NOLINT(safe-memcpy) + memcpy(&addrs[16], &net_dst_addr, 16); // NOLINT(safe-memcpy) + out.add(addrs, 32); break; } } - MemBlockBuilder ports(4); + uint8_t ports[4]; const auto net_src_port = htons(static_cast(src_port)); const auto net_dst_port = htons(static_cast(dst_port)); - ports.appendData(absl::Span(reinterpret_cast(&net_src_port), 2)); - ports.appendData(absl::Span(reinterpret_cast(&net_dst_port), 2)); - out.add(ports.releasePointer(), 4); + memcpy(ports, &net_src_port, 2); // NOLINT(saf-memcpy) + memcpy(&ports[2], &net_dst_port, 2); // NOLINT(safe-memcpy) + out.add(ports, 4); } void generateV2Header(const Network::Address::Ip& source_address, From 616f36d1a3f7ad84e974eca6cadae213b53c682e Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 15 Nov 2020 20:59:55 +0000 Subject: [PATCH 052/106] format fixed Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 4 +--- source/common/grpc/common.cc | 3 ++- source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl_legacy.cc | 2 +- .../common/proxy_protocol/proxy_protocol_header.cc | 6 +++--- source/extensions/filters/network/dubbo_proxy/BUILD | 2 +- source/extensions/filters/network/thrift_proxy/BUILD | 2 +- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index e2367a016ad49..bb78f9de6abf4 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -5,12 +5,10 @@ namespace Envoy { /** * @brief Assert memory bounds to avoid copy errors. */ -template -inline void safe_memcpy(T1* dst, T2* src) { +template inline void safe_memcpy(T1* dst, T2* src) { static_assert(sizeof(T1) == sizeof(T2)); memcpy(dst, src, sizeof(T1)); - } } // namespace Envoy diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index c89ca41ca2873..9a96ae5048f5d 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -293,7 +293,8 @@ void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { std::array header; header[0] = 0; // flags const uint32_t nsize = htonl(buffer.length()); - std::memcpy(&header[1], reinterpret_cast(&nsize), sizeof(uint32_t)); // NOLINT(safe-memcpy) + std::memcpy(&header[1], reinterpret_cast(&nsize), // NOLINT(safe-memcpy) + sizeof(uint32_t)); buffer.prepend(absl::string_view(&header[0], 5)); } diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 78c38c31600b9..9d7d1d45aa938 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -13,8 +13,8 @@ #include "common/common/cleanup.h" #include "common/common/enum_to_int.h" #include "common/common/fmt.h" -#include "common/common/utility.h" #include "common/common/safe_memcpy.h" +#include "common/common/utility.h" #include "common/http/codes.h" #include "common/http/exception.h" #include "common/http/header_utility.h" diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index ab86f4e9feb01..f116c5638ad61 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -13,8 +13,8 @@ #include "common/common/cleanup.h" #include "common/common/enum_to_int.h" #include "common/common/fmt.h" -#include "common/common/utility.h" #include "common/common/safe_memcpy.h" +#include "common/common/utility.h" #include "common/http/codes.h" #include "common/http/exception.h" #include "common/http/header_utility.h" diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index e069f2cef7520..6140283957b55 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -71,7 +71,7 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv4Instance(src_addr, src_port).ip()->ipv4()->address(); const auto net_dst_addr = Network::Address::Ipv4Instance(dst_addr, dst_port).ip()->ipv4()->address(); - memcpy(addrs, &net_src_addr, 4); // NOLINT(safe-memcpy) + memcpy(addrs, &net_src_addr, 4); // NOLINT(safe-memcpy) memcpy(&addrs[4], &net_dst_addr, 4); // NOLINT(safe-memcpy) out.add(addrs, 8); break; @@ -84,7 +84,7 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const auto net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - memcpy(addrs, &net_src_addr, 16); // NOLINT(safe-memcpy) + memcpy(addrs, &net_src_addr, 16); // NOLINT(safe-memcpy) memcpy(&addrs[16], &net_dst_addr, 16); // NOLINT(safe-memcpy) out.add(addrs, 32); break; @@ -94,7 +94,7 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, uint8_t ports[4]; const auto net_src_port = htons(static_cast(src_port)); const auto net_dst_port = htons(static_cast(dst_port)); - memcpy(ports, &net_src_port, 2); // NOLINT(saf-memcpy) + memcpy(ports, &net_src_port, 2); // NOLINT(safe-memcpy) memcpy(&ports[2], &net_dst_port, 2); // NOLINT(safe-memcpy) out.add(ports, 4); } diff --git a/source/extensions/filters/network/dubbo_proxy/BUILD b/source/extensions/filters/network/dubbo_proxy/BUILD index ee1d1edd9bd1b..9cd5038225889 100644 --- a/source/extensions/filters/network/dubbo_proxy/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/BUILD @@ -16,8 +16,8 @@ envoy_cc_library( deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", - "//source/common/common:safe_memcpy_lib", "//source/common/common:byte_order_lib", + "//source/common/common:safe_memcpy_lib", ], ) diff --git a/source/extensions/filters/network/thrift_proxy/BUILD b/source/extensions/filters/network/thrift_proxy/BUILD index 7ed0f513db2b4..1df62b5864a3f 100644 --- a/source/extensions/filters/network/thrift_proxy/BUILD +++ b/source/extensions/filters/network/thrift_proxy/BUILD @@ -27,8 +27,8 @@ envoy_cc_library( deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", - "//source/common/common:safe_memcpy_lib", "//source/common/common:byte_order_lib", + "//source/common/common:safe_memcpy_lib", ], ) From 2a8c45794f031c09d8100e37705dbeb9fe6d0552 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 18 Nov 2020 20:39:05 +0000 Subject: [PATCH 053/106] explicit types deleted from safe_memcpy, check_format.py updated, and releasePointer method corrected to call the release method in MemBlockBuilder Signed-off-by: grial1 --- source/common/common/hash.h | 2 +- source/common/common/mem_block_builder.h | 3 +-- source/common/common/safe_memcpy.h | 1 - source/common/grpc/common.cc | 2 +- source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl_legacy.cc | 2 +- source/common/network/address_impl.cc | 11 ++--------- source/common/network/cidr_range.cc | 2 +- .../filters/listener/proxy_protocol/proxy_protocol.cc | 4 ++-- .../filters/network/dubbo_proxy/buffer_helper.cc | 4 ++-- .../extensions/filters/network/kafka/serialization.h | 2 +- source/extensions/filters/network/mongo_proxy/BUILD | 2 +- .../filters/network/mongo_proxy/bson_impl.cc | 4 ++-- .../filters/network/thrift_proxy/buffer_helper.cc | 4 ++-- .../extensions/filters/udp/dns_filter/dns_parser.cc | 4 ++-- source/extensions/stat_sinks/common/statsd/statsd.cc | 6 +++--- source/extensions/transport_sockets/tls/BUILD | 1 + source/extensions/transport_sockets/tls/utility.cc | 6 +++--- source/server/hot_restarting_base.cc | 2 +- tools/code_format/check_format.py | 1 + 20 files changed, 29 insertions(+), 36 deletions(-) diff --git a/source/common/common/hash.h b/source/common/common/hash.h index c31c237692e06..26972b9cc74e7 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -61,7 +61,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - safe_memcpy(&result, reinterpret_cast(p)); + safe_memcpy(&result, reinterpret_cast(p)); return result; } diff --git a/source/common/common/mem_block_builder.h b/source/common/common/mem_block_builder.h index 5d90cc66a2aca..8dc31a3dfb9af 100644 --- a/source/common/common/mem_block_builder.h +++ b/source/common/common/mem_block_builder.h @@ -108,8 +108,7 @@ template class MemBlockBuilder { * @return the transferred storage. */ T* releasePointer() { - write_span_ = absl::MakeSpan(static_cast(nullptr), 0); - return data_.release(); + return this->release().release(); } /** diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index bb78f9de6abf4..21652279ca58e 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -6,7 +6,6 @@ namespace Envoy { * @brief Assert memory bounds to avoid copy errors. */ template inline void safe_memcpy(T1* dst, T2* src) { - static_assert(sizeof(T1) == sizeof(T2)); memcpy(dst, src, sizeof(T1)); } diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index 9a96ae5048f5d..f748745a5ca4b 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -139,7 +139,7 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - safe_memcpy(reinterpret_cast(current), &nsize); + safe_memcpy(reinterpret_cast(current), &nsize); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 9d7d1d45aa938..9587ed591c9f6 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -755,7 +755,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safe_memcpy(&data, &(frame->ping.opaque_data)); + safe_memcpy(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index f116c5638ad61..9bf99c06a775d 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -731,7 +731,7 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safe_memcpy(&data, &(frame->ping.opaque_data)); + safe_memcpy(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 4ef600c2b5d57..af1bc8c180e86 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -174,16 +174,9 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { } absl::uint128 Ipv6Instance::Ipv6Helper::address() const { - absl::uint128 result{0}; + absl::uint128 result; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); -#ifdef __SIZEOF_INT128__ - unsigned __int128 ipv6_addr; - safe_memcpy(&ipv6_addr, &(address_.sin6_addr.s6_addr)); - result = ipv6_addr; // Use assignment operator which defaults to copy-constructor -#else - memcpy(static_cast(&result), // NOLINT(safe-memcpy) - static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); -#endif + safe_memcpy(&result, reinterpret_cast(&address_.sin6_addr.s6_addr)); return result; } diff --git a/source/common/network/cidr_range.cc b/source/common/network/cidr_range.cc index 57ef7eb9d8edc..4735da44ad85f 100644 --- a/source/common/network/cidr_range.cc +++ b/source/common/network/cidr_range.cc @@ -184,7 +184,7 @@ InstanceConstSharedPtr CidrRange::truncateIpAddressAndLength(InstanceConstShared absl::uint128 ip6_htonl = Utility::Ip6htonl(ip6); static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - safe_memcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl); + safe_memcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl); return std::make_shared(sa6); } } diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 626d19144a4c0..4a3118f665ddf 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -209,11 +209,11 @@ void Filter::parseV2Header(char* buf) { memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; ra6.sin6_port = v6->src_port; - safe_memcpy(&(ra6.sin6_addr.s6_addr), &(v6->src_addr)); + safe_memcpy(&(ra6.sin6_addr.s6_addr), &(v6->src_addr)); la6.sin6_family = AF_INET6; la6.sin6_port = v6->dst_port; - safe_memcpy(&(la6.sin6_addr.s6_addr), &(v6->dst_addr)); + safe_memcpy(&(la6.sin6_addr.s6_addr), &(v6->dst_addr)); proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, diff --git a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc index 0a2feee3e853f..63b7d5952560f 100644 --- a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc @@ -11,7 +11,7 @@ double BufferHelper::peekDouble(Buffer::Instance& buffer, uint64_t offset) { } double i; uint64_t j = buffer.peekBEInt(offset); - safe_memcpy(&i, &j); + safe_memcpy(&i, &j); return i; } @@ -21,7 +21,7 @@ float BufferHelper::peekFloat(Buffer::Instance& buffer, uint64_t offset) { } float i; uint32_t j = buffer.peekBEInt(offset); - safe_memcpy(&i, &j); + safe_memcpy(&i, &j); return i; } } // namespace DubboProxy diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 2236a1c7d0526..845f9f5b02add 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -190,7 +190,7 @@ class VarUInt32Deserializer : public Deserializer { // Read next byte from input. uint8_t el; - safe_memcpy(&el, data.data()); + safe_memcpy(&el, data.data()); data = {data.data() + 1, data.size() - 1}; processed++; diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index 0a300779f2679..b794141edb1c0 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -34,7 +34,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:byte_order_lib", "//source/common/common:hex_lib", - "//source/common/common:mem_block_builder_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:minimal_logger_lib", "//source/common/common:utility_lib", ], diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index b169653608f94..6fce9b7516542 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -8,7 +8,7 @@ #include "common/common/byte_order.h" #include "common/common/fmt.h" #include "common/common/hex.h" -#include "common/common/mem_block_builder.h" +#include "common/common/safe_memcpy.h" #include "common/common/utility.h" namespace Envoy { @@ -90,7 +90,7 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { int64_t val; void* mem = data.linearize(sizeof(int64_t)); - std::memcpy(reinterpret_cast(&val), mem, sizeof(int64_t)); // NOLINT(safe-memcpy) + safe_memcpy(&val, reinterpret_cast(mem)); data.drain(sizeof(int64_t)); return le64toh(val); } diff --git a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc index 05a4d2a811d42..31bce8e5dab8d 100644 --- a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc @@ -22,7 +22,7 @@ double BufferHelper::drainBEDouble(Buffer::Instance& buffer) { // 4. Implementation of last resort is to manually copy from i to d via unsigned char*. uint64_t i = buffer.drainBEInt(); double d; - safe_memcpy(&d, &i); + safe_memcpy(&d, &i); return d; } @@ -122,7 +122,7 @@ void BufferHelper::writeBEDouble(Buffer::Instance& buffer, double value) { // See drainDouble for implementation details. uint64_t i; - safe_memcpy(&i, &value); + safe_memcpy(&i, &value); buffer.writeBEInt(i); } diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 609b39931ea47..3d4dd64a17ca5 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - safe_memcpy(&(context->header_.flags), &data); + safe_memcpy(&(context->header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -743,7 +743,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safe_memcpy(&flags, &(query_context->response_header_.flags)); + safe_memcpy(&flags, &(query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 2aab8dc983c41..17124aaf8edb9 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -210,9 +210,9 @@ void TcpStatsdSink::TlsSink::commonFlush(const std::string& name, uint64_t value // This written this way for maximum perf since with a large number of stats and at a high flush // rate this can become expensive. const char* snapped_current = current_slice_mem_; - memcpy(current_slice_mem_, parent_.getPrefix().c_str(), // NOLINT(safe-memcpy) - parent_.getPrefix().size()); - current_slice_mem_ += parent_.getPrefix().size(); + const std::string prefix = parent_.getPrefix(); + memcpy(current_slice_mem_, prefix.data(), prefix.size()); // NOLINT(safe-memcpy) + current_slice_mem_ += prefix.size(); *current_slice_mem_++ = '.'; memcpy(current_slice_mem_, name.c_str(), name.size()); // NOLINT(safe-memcpy) current_slice_mem_ += name.size(); diff --git a/source/extensions/transport_sockets/tls/BUILD b/source/extensions/transport_sockets/tls/BUILD index 860b56203c301..acf7d2c61996d 100644 --- a/source/extensions/transport_sockets/tls/BUILD +++ b/source/extensions/transport_sockets/tls/BUILD @@ -177,6 +177,7 @@ envoy_cc_library( ], deps = [ "//source/common/common:assert_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", "//source/common/network:address_lib", ], diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index e7d96f614ea51..c43430b929445 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -1,6 +1,7 @@ #include "extensions/transport_sockets/tls/utility.h" #include "common/common/assert.h" +#include "common/common/safe_memcpy.h" #include "common/network/address_impl.h" #include "absl/strings/str_join.h" @@ -116,15 +117,14 @@ std::string Utility::generalNameAsString(const GENERAL_NAME* general_name) { sockaddr_in sin; sin.sin_port = 0; sin.sin_family = AF_INET; - memcpy(&sin.sin_addr, general_name->d.ip->data, sizeof(sin.sin_addr)); // NOLINT(safe-memcpy) + safe_memcpy(&sin.sin_addr, reinterpret_cast(&(general_name->d.ip->data))); Network::Address::Ipv4Instance addr(&sin); san = addr.ip()->addressAsString(); } else if (general_name->d.ip->length == 16) { sockaddr_in6 sin6; sin6.sin6_port = 0; sin6.sin6_family = AF_INET6; - memcpy(&sin6.sin6_addr, general_name->d.ip->data, // NOLINT(safe-memcpy) - sizeof(sin6.sin6_addr)); + safe_memcpy(&sin6.sin6_addr, reinterpret_cast(&(general_name->d.ip->data))); Network::Address::Ipv6Instance addr(sin6); san = addr.ip()->addressAsString(); } diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 02eb38ab89f45..3f391a2b25145 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -38,7 +38,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - safe_memcpy(&address, &(addr.getSockAddr())); + safe_memcpy(&address, &(addr.getSockAddr())); fchmod(my_domain_socket_, socket_mode); return address; diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index b712e88af1afe..7844775171ac8 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -801,6 +801,7 @@ def checkSourceLine(self, line, file_path, reportError): if not self.whitelistedForMemcpy(file_path) and \ not ("test/" in file_path) and \ + not ("safe_memcpy(" in line) and \ ("memcpy(" in line) and \ not ("NOLINT(safe-memcpy)" in line): reportError("Don't call memcpy() directly; use safe_memcpy or MemBlockBuilder instead.") From c4211cab0705199a0b7168ea0dc658594691fbc9 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 18 Nov 2020 21:27:39 +0000 Subject: [PATCH 054/106] format fixed Signed-off-by: grial1 --- source/common/common/mem_block_builder.h | 4 +--- source/extensions/filters/network/mongo_proxy/BUILD | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/common/common/mem_block_builder.h b/source/common/common/mem_block_builder.h index 8dc31a3dfb9af..ce2323c44fa80 100644 --- a/source/common/common/mem_block_builder.h +++ b/source/common/common/mem_block_builder.h @@ -107,9 +107,7 @@ template class MemBlockBuilder { * * @return the transferred storage. */ - T* releasePointer() { - return this->release().release(); - } + T* releasePointer() { return this->release().release(); } /** * @return the populated data as an absl::Span. diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index b794141edb1c0..17fa01d86349a 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -34,8 +34,8 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:byte_order_lib", "//source/common/common:hex_lib", - "//source/common/common:safe_memcpy_lib", "//source/common/common:minimal_logger_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", ], ) From 9488f4ebecfdcc79af919090d888e9ce606f2907 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 18 Nov 2020 23:31:30 +0000 Subject: [PATCH 055/106] C library added for memcpy Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 21652279ca58e..366ed34bc5d1a 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace Envoy { /** From 3b44b88f828e4eddfb9f6b7c0c1b610cc5fdceaa Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 21 Nov 2020 22:54:12 +0000 Subject: [PATCH 056/106] dereference applied and safe_memcpy_lib registered for hot_restarting_base Signed-off-by: grial1 --- source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl_legacy.cc | 2 +- source/extensions/transport_sockets/tls/utility.cc | 4 ++-- source/server/BUILD | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 9587ed591c9f6..a19ddcbd8fb00 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -755,7 +755,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safe_memcpy(&data, &(frame->ping.opaque_data)); + safe_memcpy(&data, reinterpret_cast(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index 9bf99c06a775d..b107c8d9527e1 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -731,7 +731,7 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safe_memcpy(&data, &(frame->ping.opaque_data)); + safe_memcpy(&data, reinterpret_cast(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index c43430b929445..febff7ec5a0a6 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -117,14 +117,14 @@ std::string Utility::generalNameAsString(const GENERAL_NAME* general_name) { sockaddr_in sin; sin.sin_port = 0; sin.sin_family = AF_INET; - safe_memcpy(&sin.sin_addr, reinterpret_cast(&(general_name->d.ip->data))); + safe_memcpy(&sin.sin_addr, reinterpret_cast(general_name->d.ip->data)); Network::Address::Ipv4Instance addr(&sin); san = addr.ip()->addressAsString(); } else if (general_name->d.ip->length == 16) { sockaddr_in6 sin6; sin6.sin6_port = 0; sin6.sin6_family = AF_INET6; - safe_memcpy(&sin6.sin6_addr, reinterpret_cast(&(general_name->d.ip->data))); + safe_memcpy(&sin6.sin6_addr, reinterpret_cast(general_name->d.ip->data)); Network::Address::Ipv6Instance addr(sin6); san = addr.ip()->addressAsString(); } diff --git a/source/server/BUILD b/source/server/BUILD index dd1bf5b2a12ca..d81fb59142c0b 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -155,6 +155,7 @@ envoy_cc_library( "//source/common/api:os_sys_calls_lib", "//source/common/common:assert_lib", "//source/common/common:mem_block_builder_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", "//source/common/network:utility_lib", "//source/common/stats:utility_lib", From cdcecd3063e55443b2585ac59ea9f00f33406cc4 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 22 Nov 2020 10:53:05 +0000 Subject: [PATCH 057/106] coding style error fixed: safe_memcpy signature changed to safeMemcpy Signed-off-by: grial1 --- source/common/common/hash.h | 2 +- source/common/common/safe_memcpy.h | 2 +- source/common/grpc/common.cc | 2 +- source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl_legacy.cc | 2 +- source/common/network/address_impl.cc | 2 +- source/common/network/cidr_range.cc | 2 +- .../filters/listener/proxy_protocol/proxy_protocol.cc | 4 ++-- .../extensions/filters/network/dubbo_proxy/buffer_helper.cc | 4 ++-- source/extensions/filters/network/kafka/serialization.h | 2 +- source/extensions/filters/network/mongo_proxy/bson_impl.cc | 2 +- .../extensions/filters/network/thrift_proxy/buffer_helper.cc | 4 ++-- source/extensions/filters/udp/dns_filter/dns_parser.cc | 4 ++-- source/extensions/transport_sockets/tls/utility.cc | 4 ++-- source/server/hot_restarting_base.cc | 2 +- tools/code_format/check_format.py | 1 - 16 files changed, 20 insertions(+), 21 deletions(-) diff --git a/source/common/common/hash.h b/source/common/common/hash.h index 26972b9cc74e7..11a61916829ce 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -61,7 +61,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - safe_memcpy(&result, reinterpret_cast(p)); + safeMemcpy(&result, reinterpret_cast(p)); return result; } diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 366ed34bc5d1a..a131c0031e0c4 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -7,7 +7,7 @@ namespace Envoy { /** * @brief Assert memory bounds to avoid copy errors. */ -template inline void safe_memcpy(T1* dst, T2* src) { +template inline void safeMemcpy(T1* dst, T2* src) { static_assert(sizeof(T1) == sizeof(T2)); memcpy(dst, src, sizeof(T1)); } diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index f748745a5ca4b..a35a972a86d51 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -139,7 +139,7 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - safe_memcpy(reinterpret_cast(current), &nsize); + safeMemcpy(reinterpret_cast(current), &nsize); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index a19ddcbd8fb00..63891294c944a 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -755,7 +755,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safe_memcpy(&data, reinterpret_cast(frame->ping.opaque_data)); + safeMemcpy(&data, reinterpret_cast(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index b107c8d9527e1..ffca1da595e0b 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -731,7 +731,7 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safe_memcpy(&data, reinterpret_cast(frame->ping.opaque_data)); + safeMemcpy(&data, reinterpret_cast(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index af1bc8c180e86..ac402ff04a2b1 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -176,7 +176,7 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - safe_memcpy(&result, reinterpret_cast(&address_.sin6_addr.s6_addr)); + safeMemcpy(&result, reinterpret_cast(&address_.sin6_addr.s6_addr)); return result; } diff --git a/source/common/network/cidr_range.cc b/source/common/network/cidr_range.cc index 4735da44ad85f..2ee249f265c1b 100644 --- a/source/common/network/cidr_range.cc +++ b/source/common/network/cidr_range.cc @@ -184,7 +184,7 @@ InstanceConstSharedPtr CidrRange::truncateIpAddressAndLength(InstanceConstShared absl::uint128 ip6_htonl = Utility::Ip6htonl(ip6); static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - safe_memcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl); + safeMemcpy(&sa6.sin6_addr.s6_addr, &ip6_htonl); return std::make_shared(sa6); } } diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 4a3118f665ddf..951b6cdf860cb 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -209,11 +209,11 @@ void Filter::parseV2Header(char* buf) { memset(&la6, 0, sizeof(la6)); ra6.sin6_family = AF_INET6; ra6.sin6_port = v6->src_port; - safe_memcpy(&(ra6.sin6_addr.s6_addr), &(v6->src_addr)); + safeMemcpy(&(ra6.sin6_addr.s6_addr), &(v6->src_addr)); la6.sin6_family = AF_INET6; la6.sin6_port = v6->dst_port; - safe_memcpy(&(la6.sin6_addr.s6_addr), &(v6->dst_addr)); + safeMemcpy(&(la6.sin6_addr.s6_addr), &(v6->dst_addr)); proxy_protocol_header_.emplace(WireHeader{ hdr_addr_len - PROXY_PROTO_V2_ADDR_LEN_INET6, Network::Address::IpVersion::v6, diff --git a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc index 63b7d5952560f..934a8ffb780ae 100644 --- a/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/dubbo_proxy/buffer_helper.cc @@ -11,7 +11,7 @@ double BufferHelper::peekDouble(Buffer::Instance& buffer, uint64_t offset) { } double i; uint64_t j = buffer.peekBEInt(offset); - safe_memcpy(&i, &j); + safeMemcpy(&i, &j); return i; } @@ -21,7 +21,7 @@ float BufferHelper::peekFloat(Buffer::Instance& buffer, uint64_t offset) { } float i; uint32_t j = buffer.peekBEInt(offset); - safe_memcpy(&i, &j); + safeMemcpy(&i, &j); return i; } } // namespace DubboProxy diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 845f9f5b02add..758e4392095d5 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -190,7 +190,7 @@ class VarUInt32Deserializer : public Deserializer { // Read next byte from input. uint8_t el; - safe_memcpy(&el, data.data()); + safeMemcpy(&el, data.data()); data = {data.data() + 1, data.size() - 1}; processed++; diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 6fce9b7516542..02726c1c73c41 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -90,7 +90,7 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { int64_t val; void* mem = data.linearize(sizeof(int64_t)); - safe_memcpy(&val, reinterpret_cast(mem)); + safeMemcpy(&val, reinterpret_cast(mem)); data.drain(sizeof(int64_t)); return le64toh(val); } diff --git a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc index 31bce8e5dab8d..48167e024bede 100644 --- a/source/extensions/filters/network/thrift_proxy/buffer_helper.cc +++ b/source/extensions/filters/network/thrift_proxy/buffer_helper.cc @@ -22,7 +22,7 @@ double BufferHelper::drainBEDouble(Buffer::Instance& buffer) { // 4. Implementation of last resort is to manually copy from i to d via unsigned char*. uint64_t i = buffer.drainBEInt(); double d; - safe_memcpy(&d, &i); + safeMemcpy(&d, &i); return d; } @@ -122,7 +122,7 @@ void BufferHelper::writeBEDouble(Buffer::Instance& buffer, double value) { // See drainDouble for implementation details. uint64_t i; - safe_memcpy(&i, &value); + safeMemcpy(&i, &value); buffer.writeBEInt(i); } diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 3d4dd64a17ca5..6255a5feb4dac 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - safe_memcpy(&(context->header_.flags), &data); + safeMemcpy(&(context->header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -743,7 +743,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safe_memcpy(&flags, &(query_context->response_header_.flags)); + safeMemcpy(&flags, &(query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index febff7ec5a0a6..861dfeb4fa937 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -117,14 +117,14 @@ std::string Utility::generalNameAsString(const GENERAL_NAME* general_name) { sockaddr_in sin; sin.sin_port = 0; sin.sin_family = AF_INET; - safe_memcpy(&sin.sin_addr, reinterpret_cast(general_name->d.ip->data)); + safeMemcpy(&sin.sin_addr, reinterpret_cast(general_name->d.ip->data)); Network::Address::Ipv4Instance addr(&sin); san = addr.ip()->addressAsString(); } else if (general_name->d.ip->length == 16) { sockaddr_in6 sin6; sin6.sin6_port = 0; sin6.sin6_family = AF_INET6; - safe_memcpy(&sin6.sin6_addr, reinterpret_cast(general_name->d.ip->data)); + safeMemcpy(&sin6.sin6_addr, reinterpret_cast(general_name->d.ip->data)); Network::Address::Ipv6Instance addr(sin6); san = addr.ip()->addressAsString(); } diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 3f391a2b25145..3f806367c6d85 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -38,7 +38,7 @@ sockaddr_un HotRestartingBase::createDomainSocketAddress(uint64_t id, const std: initDomainSocketAddress(&address); Network::Address::PipeInstance addr(fmt::format(socket_path + "_{}_{}", role, base_id_ + id), socket_mode, nullptr); - safe_memcpy(&address, &(addr.getSockAddr())); + safeMemcpy(&address, &(addr.getSockAddr())); fchmod(my_domain_socket_, socket_mode); return address; diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 7844775171ac8..b712e88af1afe 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -801,7 +801,6 @@ def checkSourceLine(self, line, file_path, reportError): if not self.whitelistedForMemcpy(file_path) and \ not ("test/" in file_path) and \ - not ("safe_memcpy(" in line) and \ ("memcpy(" in line) and \ not ("NOLINT(safe-memcpy)" in line): reportError("Don't call memcpy() directly; use safe_memcpy or MemBlockBuilder instead.") From 6fc72644859a66c5cdd8c327a3c5bbaa6b49b8ab Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 22 Nov 2020 15:05:58 +0000 Subject: [PATCH 058/106] memcpy error message changed in check_format.py Signed-off-by: grial1 --- tools/code_format/check_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index b712e88af1afe..9fc982b2e107f 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -803,7 +803,7 @@ def checkSourceLine(self, line, file_path, reportError): not ("test/" in file_path) and \ ("memcpy(" in line) and \ not ("NOLINT(safe-memcpy)" in line): - reportError("Don't call memcpy() directly; use safe_memcpy or MemBlockBuilder instead.") + reportError("Don't call memcpy() directly; use safeMemcpy or MemBlockBuilder instead.") if self.denylistedForExceptions(file_path): # Skpping cases where 'throw' is a substring of a symbol like in "foothrowBar". From b7dcf0401b634ac030c4644de7d7771a2aa7c669 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 22 Nov 2020 17:54:26 +0000 Subject: [PATCH 059/106] two simple tests added for safeMemcpy Signed-off-by: grial1 --- test/common/common/BUILD | 6 ++++ test/common/common/safe_memcpy_test.cc | 40 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 test/common/common/safe_memcpy_test.cc diff --git a/test/common/common/BUILD b/test/common/common/BUILD index 8f9ec5324dc86..58aa0436be7f3 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -75,6 +75,12 @@ envoy_cc_test( deps = ["//source/common/common:mem_block_builder_lib"], ) +envoy_cc_test( + name = "safe_memcpy_test", + srcs = ["safe_memcpy_test.cc"], + deps = ["//source/common/common:safe_memcpy_lib"], +) + envoy_cc_test( name = "phantom_test", srcs = ["phantom_test.cc"], diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc new file mode 100644 index 0000000000000..85ccbf528a238 --- /dev/null +++ b/test/common/common/safe_memcpy_test.cc @@ -0,0 +1,40 @@ +#include + +#include "common/grpc/common.h" +#include "common/http/headers.h" +#include "common/http/message_impl.h" +#include "common/http/utility.h" +#include "test/test_common/utility.h" +#include "common/common/safe_memcpy.h" + +#include "gtest/gtest.h" + +using namespace testing; + +namespace Envoy { + +TEST(SafeMemcpyTest, CopyUint8) { + + const uint8_t src[] = {0, 1, 1, 2, 3, 5, 8, 13}; + std::array dst; + safeMemcpy(reinterpret_cast(dst.data()), &src); + Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); + +} + +// Additional (integration) test - not ordinary copy +TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { + + auto buffer = std::make_unique(); + buffer->add("test", 4); + std::array expected_header; + expected_header[0] = 0; // flags + const uint32_t nsize = htonl(4); + safeMemcpy(reinterpret_cast(&expected_header[1]), &nsize); + std::string header_string(&expected_header[0], 5); + Grpc::Common::prependGrpcFrameHeader(*buffer); + EXPECT_EQ(buffer->toString(), header_string + "test"); + +} + +} // namespace Envoy From 93622eb8e117a946bf552d0d0b4df9c1f9066f50 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 22 Nov 2020 18:00:42 +0000 Subject: [PATCH 060/106] format corrected and application of safeMemcpy in common/grpc/common.cc Signed-off-by: grial1 --- source/common/buffer/BUILD | 1 - source/common/grpc/common.cc | 3 +-- test/common/common/safe_memcpy_test.cc | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/source/common/buffer/BUILD b/source/common/buffer/BUILD index 95b942fe58c36..171aa8a089872 100644 --- a/source/common/buffer/BUILD +++ b/source/common/buffer/BUILD @@ -25,7 +25,6 @@ envoy_cc_library( hdrs = ["buffer_impl.h"], deps = [ "//include/envoy/buffer:buffer_interface", - "//source/common/common:mem_block_builder_lib", "//source/common/common:non_copyable", "//source/common/common:utility_lib", "//source/common/event:libevent_lib", diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index a35a972a86d51..eca146d095a87 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -293,8 +293,7 @@ void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { std::array header; header[0] = 0; // flags const uint32_t nsize = htonl(buffer.length()); - std::memcpy(&header[1], reinterpret_cast(&nsize), // NOLINT(safe-memcpy) - sizeof(uint32_t)); + safeMemcpy(reinterpret_cast(&header[1]), &nsize); buffer.prepend(absl::string_view(&header[0], 5)); } diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 85ccbf528a238..e19b11ccb0448 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -1,11 +1,12 @@ #include +#include "common/common/safe_memcpy.h" #include "common/grpc/common.h" #include "common/http/headers.h" #include "common/http/message_impl.h" #include "common/http/utility.h" + #include "test/test_common/utility.h" -#include "common/common/safe_memcpy.h" #include "gtest/gtest.h" @@ -19,7 +20,6 @@ TEST(SafeMemcpyTest, CopyUint8) { std::array dst; safeMemcpy(reinterpret_cast(dst.data()), &src); Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); - } // Additional (integration) test - not ordinary copy @@ -34,7 +34,6 @@ TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { std::string header_string(&expected_header[0], 5); Grpc::Common::prependGrpcFrameHeader(*buffer); EXPECT_EQ(buffer->toString(), header_string + "test"); - } } // namespace Envoy From ade5461539abc7ffadb7b425e5535682dddb0ea9 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 22 Nov 2020 18:40:22 +0000 Subject: [PATCH 061/106] address taken from frame->ping.opaque_data in codec_impl*.cc Signed-off-by: grial1 --- source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl_legacy.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 63891294c944a..1b06c61686300 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -755,7 +755,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safeMemcpy(&data, reinterpret_cast(frame->ping.opaque_data)); + safeMemcpy(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc index ffca1da595e0b..3fcc0316a8ca0 100644 --- a/source/common/http/http2/codec_impl_legacy.cc +++ b/source/common/http/http2/codec_impl_legacy.cc @@ -731,7 +731,7 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // was the current time when the ping was sent. This can be useful while debugging // to match the ping and ack. uint64_t data; - safeMemcpy(&data, reinterpret_cast(frame->ping.opaque_data)); + safeMemcpy(&data, &(frame->ping.opaque_data)); ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); onKeepaliveResponse(); From da34c9d42321633989be87a8854f22d69bf89242 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 22 Nov 2020 22:55:56 +0000 Subject: [PATCH 062/106] safeMemcpy switch back to memcpy in function parseDnsObject due to static_assert error when compiling with MSVC Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 6255a5feb4dac..8749a6c4d2dd5 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,8 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - safeMemcpy(&(context->header_.flags), &data); + ::memcpy(static_cast(&context->header_.flags), &data, // NOLINT(safe-memcpy) + field_size); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: From ab19cbd71a85347916b5a24dcf68bec9f324edac Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 23 Nov 2020 00:25:31 +0000 Subject: [PATCH 063/106] dns_filter_lib reverted due to inability to compile with MSVC Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/BUILD | 1 - source/extensions/filters/udp/dns_filter/dns_parser.cc | 3 ++- source/extensions/filters/udp/dns_filter/dns_parser.h | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/udp/dns_filter/BUILD b/source/extensions/filters/udp/dns_filter/BUILD index 0fd4b5a1bbbb2..5684b6569ed92 100644 --- a/source/extensions/filters/udp/dns_filter/BUILD +++ b/source/extensions/filters/udp/dns_filter/BUILD @@ -35,7 +35,6 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:empty_string", "//source/common/common:matchers_lib", - "//source/common/common:safe_memcpy_lib", "//source/common/config:config_provider_lib", "//source/common/config:datasource_lib", "//source/common/network:address_lib", diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 8749a6c4d2dd5..2cf85262e5c51 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -744,7 +744,8 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safeMemcpy(&flags, &(query_context->response_header_.flags)); + ::memcpy(&flags, // NOLINT(safe-memcpy) + static_cast(&query_context->response_header_.flags), sizeof(uint16_t)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.h b/source/extensions/filters/udp/dns_filter/dns_parser.h index 8da0fbe136bca..d604f0784ba74 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.h +++ b/source/extensions/filters/udp/dns_filter/dns_parser.h @@ -9,7 +9,6 @@ #include "common/buffer/buffer_impl.h" #include "common/common/empty_string.h" -#include "common/common/safe_memcpy.h" #include "common/runtime/runtime_impl.h" #include "common/stats/timespan_impl.h" From 5f76f5ebc57213686b1ec9b727cd963675a6908e Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 23 Nov 2020 09:56:45 +0000 Subject: [PATCH 064/106] suggested changes applied Signed-off-by: grial1 --- source/common/common/BUILD | 1 - source/common/common/safe_memcpy.h | 2 +- source/extensions/stat_sinks/common/statsd/statsd.cc | 2 +- test/common/common/safe_memcpy_test.cc | 4 +--- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 3cccfc660b8d6..83a326dfe8425 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -366,7 +366,6 @@ envoy_cc_library( deps = [ ":assert_lib", ":hash_lib", - ":mem_block_builder_lib", ":non_copyable", "//include/envoy/common:interval_set_interface", "//include/envoy/common:time_interface", diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index a131c0031e0c4..0d69f27dab2ca 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -5,7 +5,7 @@ namespace Envoy { /** - * @brief Assert memory bounds to avoid copy errors. + * @brief Copies src to dst based on their sizes, which must be the same. */ template inline void safeMemcpy(T1* dst, T2* src) { static_assert(sizeof(T1) == sizeof(T2)); diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 17124aaf8edb9..64a4a9dfc1add 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -214,7 +214,7 @@ void TcpStatsdSink::TlsSink::commonFlush(const std::string& name, uint64_t value memcpy(current_slice_mem_, prefix.data(), prefix.size()); // NOLINT(safe-memcpy) current_slice_mem_ += prefix.size(); *current_slice_mem_++ = '.'; - memcpy(current_slice_mem_, name.c_str(), name.size()); // NOLINT(safe-memcpy) + memcpy(current_slice_mem_, name.data(), name.size()); // NOLINT(safe-memcpy) current_slice_mem_ += name.size(); *current_slice_mem_++ = ':'; current_slice_mem_ += StringUtil::itoa(current_slice_mem_, 30, value); diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index e19b11ccb0448..0c740fefd1945 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -10,12 +10,11 @@ #include "gtest/gtest.h" -using namespace testing; +using testing::Eq; namespace Envoy { TEST(SafeMemcpyTest, CopyUint8) { - const uint8_t src[] = {0, 1, 1, 2, 3, 5, 8, 13}; std::array dst; safeMemcpy(reinterpret_cast(dst.data()), &src); @@ -24,7 +23,6 @@ TEST(SafeMemcpyTest, CopyUint8) { // Additional (integration) test - not ordinary copy TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { - auto buffer = std::make_unique(); buffer->add("test", 4); std::array expected_header; From 60f3da64847002d7986d47d48d51e80cd8c1de10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Rial?= <19420029+grial1@users.noreply.github.com> Date: Sun, 8 Nov 2020 00:52:42 +0100 Subject: [PATCH 065/106] Merge branch 'master' into issue-9328 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: grial1 Signed-off-by: Gastón Rial <19420029+grial1@users.noreply.github.com> --- source/common/common/hash.h | 2 +- source/common/common/safe_memcpy.h | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/source/common/common/hash.h b/source/common/common/hash.h index 11a61916829ce..6be204cb0184f 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -61,7 +61,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - safeMemcpy(&result, reinterpret_cast(p)); + safeMemcpySrc(&result, p, 8); return result; } diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 0d69f27dab2ca..744b33df45b5d 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -12,4 +12,22 @@ template inline void safeMemcpy(T1* dst, T2* src) { memcpy(dst, src, sizeof(T1)); } +/** + * @brief Copies src to dst based on the size of dst, which must be specified. + */ +#define safeMemcpySrc(dst, src, size) \ + do { \ + static_assert(sizeof(*(dst)) == size); \ + memcpy(dst, src, size); \ + } while (0) + +/** + * @brief Copies src to dst based on the size of src, which must be specified. + */ +#define safeMemcpyDst(dst, src, size) \ + do { \ + static_assert(sizeof(*(src)) == size); \ + memcpy(dst, src, size); \ + } while (0) + } // namespace Envoy From 5f06fc06d1455823a4a7a1bb2f8a4d7dc6a38afb Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 29 Nov 2020 01:40:25 +0000 Subject: [PATCH 066/106] Merge remote-tracking branch 'upstream/master' Signed-off-by: grial1 --- bazel/PPROF.md | 11 +-- docs/root/faq/extensions/contract.rst | 12 +-- docs/root/operations/admin.rst | 3 - docs/root/version_history/current.rst | 1 + include/envoy/server/options.h | 5 -- include/envoy/stats/symbol_table.h | 19 ---- .../common/grpc/async_client_manager_impl.cc | 22 +++++ source/common/stats/symbol_table_impl.cc | 7 +- source/common/stats/symbol_table_impl.h | 13 +-- source/common/stats/thread_local_store.cc | 10 +-- source/docs/stats.md | 7 +- source/server/options_impl.cc | 13 +-- source/server/options_impl.h | 6 -- source/server/server.cc | 8 ++ source/server/server.h | 2 +- .../grpc/async_client_manager_impl_test.cc | 38 ++++++++ test/common/stats/symbol_table_impl_test.cc | 3 +- .../network/kafka/serialization_test.cc | 13 +++ .../network/kafka/serialization_utilities.h | 13 ++- .../filters/network/thrift_proxy/BUILD | 2 - .../filters/network/thrift_proxy/driver/BUILD | 3 + .../network/thrift_proxy/driver/client.py | 5 ++ .../thrift_proxy/driver/generate_fixture.sh | 89 +++++++++++++------ .../network/thrift_proxy/driver/server.py | 5 ++ .../network/thrift_proxy/integration.cc | 4 + .../network/thrift_proxy/integration_test.cc | 4 + test/integration/hotrestart_test.sh | 60 +++---------- test/mocks/server/options.h | 1 - test/server/options_impl_test.cc | 20 +---- .../server_corpus/grpc_illegal_characters | 64 +++++++++++++ test/server/server_test.cc | 10 +++ 31 files changed, 290 insertions(+), 183 deletions(-) create mode 100644 test/server/server_corpus/grpc_illegal_characters diff --git a/bazel/PPROF.md b/bazel/PPROF.md index 97e1c0541181d..fa0a4f0125557 100644 --- a/bazel/PPROF.md +++ b/bazel/PPROF.md @@ -1,7 +1,8 @@ # CPU or memory consumption testing with `pprof` To use `pprof` to analyze performance and memory consumption in Envoy, you can -use the built-in statically linked profiler, or dynamically link it in to a +use the built-in statically linked profiler provided by +[gperftools](https://github.com/gperftools/gperftools), or dynamically link it in to a specific place yourself. ## Collecting CPU or heap profile for a full execution of envoy @@ -14,7 +15,7 @@ inside Build the static binary using bazel: - $ bazel build //source/exe:envoy-static + $ bazel build --define tcmalloc=gperftools //source/exe:envoy-static ### Collecting the profile @@ -41,15 +42,15 @@ The profiler library is automatically linked into envoy_cc_test targets. Run a test with heap profiling enabled, like so: - $ bazel test --test_env=HEAPPROFILE=/tmp/heapprof + $ bazel test --test_env=HEAPPROFILE=/tmp/heapprof --define tcmalloc=gperftools Run a test with CPU profiling enabled, like so: - $ bazel test --test_env=CPUPROFILE=/tmp/cpuprof + $ bazel test --test_env=CPUPROFILE=/tmp/cpuprof --define tcmalloc=gperftools Note that heap checks and heap profile collection in tests have noticiable performance implications. Use the following command to collect a CPU profile from a test target with heap check and heap profile collection disabled: - $ bazel test --test_env=CPUPROFILE=/tmp/cpuprof --test_env=HEAPPROFILE= --test_env=HEAPCHECK= + $ bazel test --test_env=CPUPROFILE=/tmp/cpuprof --test_env=HEAPPROFILE= --test_env=HEAPCHECK= --define tcmalloc=gperftools ## Starting and stopping profile programmatically diff --git a/docs/root/faq/extensions/contract.rst b/docs/root/faq/extensions/contract.rst index 314701aca805a..755e15eb080b7 100644 --- a/docs/root/faq/extensions/contract.rst +++ b/docs/root/faq/extensions/contract.rst @@ -5,12 +5,13 @@ Is there a contract my HTTP filter must adhere to? * Headers encoding/decoding - * During encoding/decoding of headers if a filter returns ``FilterHeadersStatus::StopIteration``, - the processing can be resumed if ``encodeData()``/``decodeData()`` return + * During encoding/decoding of headers if a local reply wasn't sent and a filter + returns ``FilterHeadersStatus::StopIteration``, the processing can be resumed + if ``encodeData()``/``decodeData()`` return ``FilterDataStatus::Continue`` or by explicitly calling ``continueEncoding()``/``continueDecoding()``. - * During encoding/decoding of headers if a filter returns + * During encoding/decoding of headers if a local reply wasn't sent and a filter returns ``FilterHeadersStatus::StopAllIterationAndBuffer`` or ``FilterHeadersStatus::StopAllIterationAndWatermark``, the processing can be resumed by calling ``continueEncoding()``/``continueDecoding()``. @@ -24,7 +25,7 @@ Is there a contract my HTTP filter must adhere to? * Data encoding/decoding - * During encoding/decoding of data if a filter returns + * During encoding/decoding of data if a local reply wasn't sent and a filter returns ``FilterDataStatus::StopIterationAndBuffer``, ``FilterDataStatus::StopIterationAndWatermark``, or ``FilterDataStatus::StopIterationNoBuffer``, the processing can be resumed if ``encodeData()``/``decodeData()`` return ``FilterDataStatus::Continue`` or by explicitly @@ -32,7 +33,8 @@ Is there a contract my HTTP filter must adhere to? * Trailers encoding/decoding - * During encoding/decoding of trailers if a filter returns ``FilterTrailersStatus::StopIteration``, + * During encoding/decoding of trailers if a local reply wasn't sent and a filter + returns ``FilterTrailersStatus::StopIteration``, the processing can be resumed by explicitly calling ``continueEncoding()``/``continueDecoding()``. Are there well-known headers that will appear in the given headers map of ``decodeHeaders()``? diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index ae2dd425a28e6..514c25b51e093 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -511,9 +511,6 @@ modify different aspects of the server: but in response to user requests on high core-count machines, this can cause performance issues due to mutex contention. - This admin endpoint requires Envoy to be started with option - `--use-fake-symbol-table 0`. - See :repo:`source/docs/stats.md` for more details. Note also that actual mutex contention can be tracked via :http:get:`/contention`. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 5bad3f54a38ad..a19010abf84dc 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -85,3 +85,4 @@ Deprecated * gzip: :ref:`HTTP Gzip filter ` is rejected now unless explicitly allowed with :ref:`runtime override ` `envoy.deprecated_features.allow_deprecated_gzip_http_filter` set to `true`. * logging: the `--log-format-prefix-with-location` option is removed. * ratelimit: the :ref:`dynamic metadata ` action is deprecated in favor of the more generic :ref:`metadata ` action. +* stats: the `--use-fake-symbol-table` option is removed. diff --git a/include/envoy/server/options.h b/include/envoy/server/options.h index 362857899bc26..9b3709af846b7 100644 --- a/include/envoy/server/options.h +++ b/include/envoy/server/options.h @@ -234,11 +234,6 @@ class Options { */ virtual bool mutexTracingEnabled() const PURE; - /** - * @return whether to use the fake symbol table implementation. - */ - virtual bool fakeSymbolTableEnabled() const PURE; - /** * @return bool indicating whether cpuset size should determine the number of worker threads. */ diff --git a/include/envoy/stats/symbol_table.h b/include/envoy/stats/symbol_table.h index b84d340f79d10..e7f171b6f8c91 100644 --- a/include/envoy/stats/symbol_table.h +++ b/include/envoy/stats/symbol_table.h @@ -136,25 +136,6 @@ class SymbolTable { virtual void debugPrint() const PURE; #endif - /** - * Calls the provided function with a string-view representation of the - * elaborated name. This is useful during the interim period when we - * are using FakeSymbolTableImpl, to avoid an extra allocation. Once - * we migrate to using SymbolTableImpl, this interface will no longer - * be helpful and can be removed. The reason it's useful now is that - * it makes up, in part, for some extra runtime overhead that is spent - * on the SymbolTable abstraction and API, without getting full benefit - * from the improved representation. - * - * TODO(#6307): Remove this when the transition from FakeSymbolTableImpl to - * SymbolTableImpl is complete. - * - * @param stat_name The stat name. - * @param fn The function to call with the elaborated stat name as a string_view. - */ - virtual void callWithStringView(StatName stat_name, - const std::function& fn) const PURE; - using RecentLookupsFn = std::function; /** diff --git a/source/common/grpc/async_client_manager_impl.cc b/source/common/grpc/async_client_manager_impl.cc index 5f809755f89d1..14866aa25d0ef 100644 --- a/source/common/grpc/async_client_manager_impl.cc +++ b/source/common/grpc/async_client_manager_impl.cc @@ -11,6 +11,20 @@ namespace Envoy { namespace Grpc { +namespace { + +// Validates a string for gRPC header key compliance. This is a subset of legal HTTP characters. +// See https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md +bool validateGrpcHeaderChars(absl::string_view key) { + for (auto ch : key) { + if (!(absl::ascii_isalnum(ch) || ch == '_' || ch == '.' || ch == '-')) { + return false; + } + } + return true; +} + +} // namespace AsyncClientFactoryImpl::AsyncClientFactoryImpl(Upstream::ClusterManager& cm, const envoy::config::core::v3::GrpcService& config, @@ -66,6 +80,14 @@ GoogleAsyncClientFactoryImpl::GoogleAsyncClientFactoryImpl( #else ASSERT(google_tls_slot_ != nullptr); #endif + + // Check metadata for gRPC API compliance. Uppercase characters are lowered in the HeaderParser. + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md + for (const auto& header : config.initial_metadata()) { + if (!validateGrpcHeaderChars(header.key()) || !validateGrpcHeaderChars(header.value())) { + throw EnvoyException("Illegal characters in gRPC initial metadata."); + } + } } RawAsyncClientPtr GoogleAsyncClientFactoryImpl::create() { diff --git a/source/common/stats/symbol_table_impl.cc b/source/common/stats/symbol_table_impl.cc index 5a9a6df7461de..8b6479c6d840a 100644 --- a/source/common/stats/symbol_table_impl.cc +++ b/source/common/stats/symbol_table_impl.cc @@ -236,11 +236,6 @@ std::string SymbolTableImpl::toString(const StatName& stat_name) const { return absl::StrJoin(decodeStrings(stat_name.data(), stat_name.dataSize()), "."); } -void SymbolTableImpl::callWithStringView(StatName stat_name, - const std::function& fn) const { - fn(toString(stat_name)); -} - void SymbolTableImpl::incRefCount(const StatName& stat_name) { // Before taking the lock, decode the array of symbols from the SymbolTable::Storage. const SymbolVec symbols = Encoding::decodeSymbols(stat_name.data(), stat_name.dataSize()); @@ -615,7 +610,7 @@ void StatNameList::clear(SymbolTable& symbol_table) { } StatNameSet::StatNameSet(SymbolTable& symbol_table, absl::string_view name) - : name_(std::string(name)), symbol_table_(symbol_table), pool_(symbol_table) { + : name_(std::string(name)), pool_(symbol_table) { builtin_stat_names_[""] = StatName(); } diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index 816799461803f..97c457c0e0a9b 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -186,8 +186,6 @@ class SymbolTableImpl : public SymbolTable { void populateList(const StatName* names, uint32_t num_names, StatNameList& list) override; StoragePtr encode(absl::string_view name) override; StoragePtr makeDynamicStorage(absl::string_view name) override; - void callWithStringView(StatName stat_name, - const std::function& fn) const override; #ifndef ENVOY_CONFIG_COVERAGE void debugPrint() const override; @@ -585,7 +583,7 @@ class StatNamePool { * SymbolTable lock, but tokens are not shared across StatNames. * * The SymbolTable is required as a constructor argument to assist in encoding - * the stat-names, which differs between FakeSymbolTableImpl and SymbolTableImpl. + * the stat-names. * * Example usage: * StatNameDynamicPool pool(symbol_table); @@ -652,7 +650,6 @@ class StatNameList { void clear(SymbolTable& symbol_table); private: - friend class FakeSymbolTableImpl; friend class SymbolTableImpl; /** @@ -666,10 +663,8 @@ class StatNameList { * ... * * - * For FakeSymbolTableImpl, each symbol is a single char, casted into a - * uint8_t. For SymbolTableImpl, each symbol is 1 or more bytes, in a - * variable-length encoding. See SymbolTableImpl::Encoding::addSymbol for - * details. + * For SymbolTableImpl, each symbol is 1 or more bytes, in a variable-length + * encoding. See SymbolTableImpl::Encoding::addSymbol for details. */ void moveStorageIntoList(SymbolTable::StoragePtr&& storage) { storage_ = std::move(storage); } @@ -841,13 +836,11 @@ class StatNameSet { } private: - friend class FakeSymbolTableImpl; friend class SymbolTableImpl; StatNameSet(SymbolTable& symbol_table, absl::string_view name); const std::string name_; - Stats::SymbolTable& symbol_table_; Stats::StatNamePool pool_ ABSL_GUARDED_BY(mutex_); mutable absl::Mutex mutex_; using StringStatNameMap = absl::flat_hash_map; diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 11b9774408371..e936a8654371b 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -349,9 +349,8 @@ class StatNameTagHelper { : pool_(tls.symbolTable()), stat_name_tags_(stat_name_tags.value_or(StatNameTagVector())) { if (!stat_name_tags) { TagVector tags; - tls.symbolTable().callWithStringView(name, [&tags, &tls, this](absl::string_view name_str) { - tag_extracted_name_ = pool_.add(tls.tagProducer().produceTags(name_str, tags)); - }); + tag_extracted_name_ = + pool_.add(tls.tagProducer().produceTags(tls.symbolTable().toString(name), tags)); StatName empty; for (const auto& tag : tags) { StatName tag_name = tls.wellKnownTags().getBuiltin(tag.name_, empty); @@ -603,10 +602,7 @@ Histogram& ThreadLocalStoreImpl::ScopeImpl::histogramFromStatNameWithTags( StatNameTagHelper tag_helper(parent_, joiner.tagExtractedName(), stat_name_tags); ConstSupportedBuckets* buckets = nullptr; - symbolTable().callWithStringView(final_stat_name, - [&buckets, this](absl::string_view stat_name) { - buckets = &parent_.histogram_settings_->buckets(stat_name); - }); + buckets = &parent_.histogram_settings_->buckets(symbolTable().toString(final_stat_name)); RefcountPtr stat; { diff --git a/source/docs/stats.md b/source/docs/stats.md index 418a04f628d74..8b67355e24c89 100644 --- a/source/docs/stats.md +++ b/source/docs/stats.md @@ -193,11 +193,6 @@ with a format-check, but we can determine whether symbol-table lookups are occurring during via an admin endpoint that shows 20 recent lookups by name, at `ENVOY_HOST:ADMIN_PORT/stats?recentlookups`. -As of October 6, 2020, the "fake" symbol table implementation has been removed -from the system, and the "--use-fake-symbol-table" option is now a no-op, -triggering a warning if set to "1". The option will be removed in a later -release. - ### Symbol Table Class Overview Class | Superclass | Description @@ -205,7 +200,7 @@ Class | Superclass | Description SymbolTable | | Abstract class providing an interface for symbol tables SymbolTableImpl | SymbolTable | Implementation of SymbolTable API where StatName share symbols held in a table SymbolTableImpl::Encoding | | Helper class for incrementally encoding strings into symbols -StatName | | Provides an API and a view into a StatName (dynamic orsymbolized). Like absl::string_view, the backing store must be separately maintained. +StatName | | Provides an API and a view into a StatName (dynamic or symbolized). Like absl::string_view, the backing store must be separately maintained. StatNameStorageBase | | Holds storage (an array of bytes) for a dynamic or symbolized StatName StatNameStorage | StatNameStorageBase | Holds storage for a symbolized StatName. Must be explicitly freed (not just destructed). StatNameManagedStorage | StatNameStorage | Like StatNameStorage, but is 8 bytes larger, and can be destructed without free(). diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index 9a488b1fe2e7e..d4623d5a41412 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -147,10 +147,6 @@ OptionsImpl::OptionsImpl(std::vector args, TCLAP::SwitchArg cpuset_threads( "", "cpuset-threads", "Get the default # of worker threads from cpuset size", cmd, false); - TCLAP::ValueArg use_fake_symbol_table("", "use-fake-symbol-table", - "Use fake symbol table implementation", false, false, - "bool", cmd); - TCLAP::ValueArg disable_extensions("", "disable-extensions", "Comma-separated list of extensions to disable", false, "", "string", cmd); @@ -181,11 +177,6 @@ OptionsImpl::OptionsImpl(std::vector args, hot_restart_disabled_ = disable_hot_restart.getValue(); mutex_tracing_enabled_ = enable_mutex_tracing.getValue(); - fake_symbol_table_enabled_ = use_fake_symbol_table.getValue(); - if (fake_symbol_table_enabled_) { - ENVOY_LOG(warn, "Fake symbol tables have been removed. Please remove references to " - "--use-fake-symbol-table"); - } cpuset_threads_ = cpuset_threads.getValue(); @@ -423,8 +414,8 @@ OptionsImpl::OptionsImpl(const std::string& service_cluster, const std::string& service_zone_(service_zone), file_flush_interval_msec_(10000), drain_time_(600), parent_shutdown_time_(900), drain_strategy_(Server::DrainStrategy::Gradual), mode_(Server::Mode::Serve), hot_restart_disabled_(false), signal_handling_enabled_(true), - mutex_tracing_enabled_(false), cpuset_threads_(false), fake_symbol_table_enabled_(false), - socket_path_("@envoy_domain_socket"), socket_mode_(0) {} + mutex_tracing_enabled_(false), cpuset_threads_(false), socket_path_("@envoy_domain_socket"), + socket_mode_(0) {} void OptionsImpl::disableExtensions(const std::vector& names) { for (const auto& name : names) { diff --git a/source/server/options_impl.h b/source/server/options_impl.h index cacf88685aa90..85897ac7d8227 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -100,10 +100,6 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable disabled_extensions_; uint32_t count_; diff --git a/source/server/server.cc b/source/server/server.cc index 214f9d0459388..0e238f9825d88 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -246,6 +246,14 @@ void InstanceImpl::flushStatsInternal() { bool InstanceImpl::healthCheckFailed() { return !live_.load(); } +ProcessContextOptRef InstanceImpl::processContext() { + if (process_context_ == nullptr) { + return absl::nullopt; + } + + return *process_context_; +} + namespace { // Loads a bootstrap object, potentially at a specific version (upgrading if necessary). void loadBootstrap(absl::optional bootstrap_version, diff --git a/source/server/server.h b/source/server/server.h index 1c06562b5a0e4..0409b9db529bf 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -251,7 +251,7 @@ class InstanceImpl final : Logger::Loggable, Stats::Store& stats() override { return stats_store_; } Grpc::Context& grpcContext() override { return grpc_context_; } Http::Context& httpContext() override { return http_context_; } - ProcessContextOptRef processContext() override { return *process_context_; } + ProcessContextOptRef processContext() override; ThreadLocal::Instance& threadLocal() override { return thread_local_; } const LocalInfo::LocalInfo& localInfo() const override { return *local_info_; } TimeSource& timeSource() override { return time_source_; } diff --git a/test/common/grpc/async_client_manager_impl_test.cc b/test/common/grpc/async_client_manager_impl_test.cc index 448f52aa6140b..76c7c869a3b8e 100644 --- a/test/common/grpc/async_client_manager_impl_test.cc +++ b/test/common/grpc/async_client_manager_impl_test.cc @@ -88,6 +88,44 @@ TEST_F(AsyncClientManagerImplTest, GoogleGrpc) { #endif } +TEST_F(AsyncClientManagerImplTest, GoogleGrpcIllegalChars) { + EXPECT_CALL(scope_, createScope_("grpc.foo.")); + envoy::config::core::v3::GrpcService grpc_service; + grpc_service.mutable_google_grpc()->set_stat_prefix("foo"); + + auto& metadata = *grpc_service.mutable_initial_metadata()->Add(); + metadata.set_key("illegalcharacter;"); + metadata.set_value("value"); + +#ifdef ENVOY_GOOGLE_GRPC + EXPECT_THROW_WITH_MESSAGE( + async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + "Illegal characters in gRPC initial metadata."); +#else + EXPECT_THROW_WITH_MESSAGE( + async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + "Google C++ gRPC client is not linked"); +#endif +} + +TEST_F(AsyncClientManagerImplTest, LegalGoogleGrpcChar) { + EXPECT_CALL(scope_, createScope_("grpc.foo.")); + envoy::config::core::v3::GrpcService grpc_service; + grpc_service.mutable_google_grpc()->set_stat_prefix("foo"); + + auto& metadata = *grpc_service.mutable_initial_metadata()->Add(); + metadata.set_key("_legal-character."); + metadata.set_value("value"); + +#ifdef ENVOY_GOOGLE_GRPC + EXPECT_NE(nullptr, async_client_manager_.factoryForGrpcService(grpc_service, scope_, false)); +#else + EXPECT_THROW_WITH_MESSAGE( + async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + "Google C++ gRPC client is not linked"); +#endif +} + TEST_F(AsyncClientManagerImplTest, EnvoyGrpcUnknownOk) { envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index 5ac09db06a4ba..7fd540803a151 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -704,8 +704,7 @@ TEST_F(StatNameTest, SupportsAbslHash) { // Tests the memory savings realized from using symbol tables with 1k // clusters. This test shows the memory drops from almost 8M to less than -// 2M. Note that only SymbolTableImpl is tested for memory consumption, -// and not FakeSymbolTableImpl. +// 2M. TEST(SymbolTableTest, Memory) { // Tests a stat-name allocation strategy. auto test_memory_usage = [](std::function fn) -> size_t { diff --git a/test/extensions/filters/network/kafka/serialization_test.cc b/test/extensions/filters/network/kafka/serialization_test.cc index 2cc656926d645..c7e28d14ca879 100644 --- a/test/extensions/filters/network/kafka/serialization_test.cc +++ b/test/extensions/filters/network/kafka/serialization_test.cc @@ -101,10 +101,12 @@ TEST(VarUInt32Deserializer, ShouldDeserializeEdgeValues) { Buffer::OwnedImpl buffer; // when + const uint32_t expected_size = encoder.computeCompactSize(values[i]); const uint32_t written = encoder.encodeCompact(values[i], buffer); // then ASSERT_EQ(written, i + 1); + ASSERT_EQ(written, expected_size); absl::string_view data = {getRawData(buffer), 1024}; // All bits in lower bytes need to be set. for (auto j = 0; j + 1 < i; ++j) { @@ -434,6 +436,17 @@ TEST(NullableCompactArrayDeserializer, ShouldConsumeNullArray) { NullableCompactArrayDeserializer>(value); } +TEST(NullableCompactArrayDeserializer, ShouldConsumeCorrectAmountOfDataForLargeInput) { + std::vector raw; + raw.reserve(4096); + for (int32_t i = 0; i < 4096; ++i) { + raw.push_back(i); + } + const NullableArray value{raw}; + serializeCompactThenDeserializeAndCheckEquality< + NullableCompactArrayDeserializer>(value); +} + // Tagged fields. TEST(TaggedFieldDeserializer, ShouldConsumeCorrectAmountOfData) { diff --git a/test/extensions/filters/network/kafka/serialization_utilities.h b/test/extensions/filters/network/kafka/serialization_utilities.h index e9fb2a4491b82..5c252ab4a8709 100644 --- a/test/extensions/filters/network/kafka/serialization_utilities.h +++ b/test/extensions/filters/network/kafka/serialization_utilities.h @@ -114,12 +114,16 @@ void serializeCompactThenDeserializeAndCheckEqualityInOneGo(AT expected) { Buffer::OwnedImpl buffer; EncodingContext encoder{-1}; + const uint32_t expected_written_size = encoder.computeCompactSize(expected); const uint32_t written = encoder.encodeCompact(expected, buffer); + ASSERT_EQ(written, expected_written_size); // Insert garbage after serialized payload. const uint32_t garbage_size = encoder.encode(Bytes(10000), buffer); + const char* raw_buffer_ptr = + reinterpret_cast(buffer.linearize(written + garbage_size)); // Tell parser that there is more data, it should never consume more than written. - const absl::string_view orig_data = {getRawData(buffer), written + garbage_size}; + const absl::string_view orig_data = {raw_buffer_ptr, written + garbage_size}; absl::string_view data = orig_data; // when @@ -147,11 +151,16 @@ void serializeCompactThenDeserializeAndCheckEqualityWithChunks(AT expected) { Buffer::OwnedImpl buffer; EncodingContext encoder{-1}; + const uint32_t expected_written_size = encoder.computeCompactSize(expected); const uint32_t written = encoder.encodeCompact(expected, buffer); + ASSERT_EQ(written, expected_written_size); // Insert garbage after serialized payload. const uint32_t garbage_size = encoder.encode(Bytes(10000), buffer); - const absl::string_view orig_data = {getRawData(buffer), written + garbage_size}; + const char* raw_buffer_ptr = + reinterpret_cast(buffer.linearize(written + garbage_size)); + // Tell parser that there is more data, it should never consume more than written. + const absl::string_view orig_data = {raw_buffer_ptr, written + garbage_size}; // when absl::string_view data = orig_data; diff --git a/test/extensions/filters/network/thrift_proxy/BUILD b/test/extensions/filters/network/thrift_proxy/BUILD index 0d842f34a5aca..24eb2f1937035 100644 --- a/test/extensions/filters/network/thrift_proxy/BUILD +++ b/test/extensions/filters/network/thrift_proxy/BUILD @@ -333,7 +333,6 @@ envoy_extension_cc_test( "//test/extensions/filters/network/thrift_proxy/driver:generate_fixture", ], extension_name = "envoy.filters.network.thrift_proxy", - tags = ["fails_on_windows"], deps = [ ":integration_lib", ":utility_lib", @@ -350,7 +349,6 @@ envoy_extension_cc_test( "//test/extensions/filters/network/thrift_proxy/driver:generate_fixture", ], extension_name = "envoy.filters.network.thrift_proxy", - tags = ["fails_on_windows"], deps = [ ":integration_lib", ":utility_lib", diff --git a/test/extensions/filters/network/thrift_proxy/driver/BUILD b/test/extensions/filters/network/thrift_proxy/driver/BUILD index 4e5d0f47d1d4e..e27580ed82cd7 100644 --- a/test/extensions/filters/network/thrift_proxy/driver/BUILD +++ b/test/extensions/filters/network/thrift_proxy/driver/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") +load("@thrift_pip3//:requirements.bzl", "requirement") licenses(["notice"]) # Apache 2 @@ -23,6 +24,7 @@ py_binary( "//test/extensions/filters/network/thrift_proxy/driver/finagle:finagle_lib", "//test/extensions/filters/network/thrift_proxy/driver/generated/example:example_lib", "@com_github_twitter_common_rpc//:twitter_common_rpc", + requirement("six"), ], ) @@ -35,5 +37,6 @@ py_binary( "//test/extensions/filters/network/thrift_proxy/driver/finagle:finagle_lib", "//test/extensions/filters/network/thrift_proxy/driver/generated/example:example_lib", "@com_github_twitter_common_rpc//:twitter_common_rpc", + requirement("six"), ], ) diff --git a/test/extensions/filters/network/thrift_proxy/driver/client.py b/test/extensions/filters/network/thrift_proxy/driver/client.py index f323cd7f3a49c..544e5acff16a3 100755 --- a/test/extensions/filters/network/thrift_proxy/driver/client.py +++ b/test/extensions/filters/network/thrift_proxy/driver/client.py @@ -14,6 +14,11 @@ from fbthrift import THeaderTransport from twitter.common.rpc.finagle.protocol import TFinagleProtocol +# On Windows we run this test on Python3 +if sys.version_info[0] != 2: + sys.stdin.reconfigure(encoding='utf-8') + sys.stdout.reconfigure(encoding='utf-8') + class TRecordingTransport(TTransport.TTransportBase): diff --git a/test/extensions/filters/network/thrift_proxy/driver/generate_fixture.sh b/test/extensions/filters/network/thrift_proxy/driver/generate_fixture.sh index a4c58cb798d84..6304d28c83141 100755 --- a/test/extensions/filters/network/thrift_proxy/driver/generate_fixture.sh +++ b/test/extensions/filters/network/thrift_proxy/driver/generate_fixture.sh @@ -7,20 +7,11 @@ set -e function usage() { - echo "Usage: $0 -s [multiplex-service] -H [headers] method [param...]" + echo "Usage: $0 -s [multiplex-service] -H [headers] -T [TempPath] method [param...]" echo "where mode is success, exception, or idl-exception" exit 1 } -FIXTURE_DIR="${TEST_TMPDIR}" -mkdir -p "${FIXTURE_DIR}" - -DRIVER_DIR="${TEST_SRCDIR}/envoy/test/extensions/filters/network/thrift_proxy/driver" - -if [[ -z "${TEST_UDSDIR}" ]]; then - TEST_UDSDIR=$(mktemp -d /tmp/envoy_test_thrift.XXXXXX) -fi - MODE="$1" TRANSPORT="$2" PROTOCOL="$3" @@ -35,7 +26,8 @@ fi MULTIPLEX= HEADERS= -while getopts ":s:H:" opt; do +TEST_TMPDIR= +while getopts ":s:H:T:" opt; do case ${opt} in s) MULTIPLEX=$OPTARG @@ -43,7 +35,9 @@ while getopts ":s:H:" opt; do H) HEADERS=$OPTARG ;; - + T) + TEST_TMPDIR=$OPTARG + ;; \?) echo "Invalid Option: -$OPTARG" >&2 exit 1 @@ -62,11 +56,33 @@ if [[ "${METHOD}" == "" ]]; then fi shift -SOCKET="${TEST_UDSDIR}/fixture.sock" -rm -f "${SOCKET}" +FIXTURE_DIR="${TEST_TMPDIR}" +mkdir -p "${FIXTURE_DIR}" + +DRIVER_DIR="${TEST_SRCDIR}/envoy/test/extensions/filters/network/thrift_proxy/driver" + +# On UNIX python supports AF_UNIX socket which are more reliable and efficient for communication +# between the client and the server, so we use it. On Windows, we find a random unused port +# on and let the communication happen via TCP. +SOCKET="" +if [[ "$OSTYPE" == "msys" ]]; then + while + port=$(shuf -n 1 -i 49152-65535) + netstat -atn | grep -q "$port" >> /dev/null + do + continue + done + SOCKET="127.0.0.1:${port}" +else + if [[ -z "${TEST_UDSDIR}" ]]; then + TEST_UDSDIR=$(mktemp -d /tmp/envoy_test_thrift.XXXXXX) + fi + SOCKET="${TEST_UDSDIR}/fixture.sock" + rm -f "${SOCKET}" +fi +echo "Using address ${SOCKET}" SERVICE_FLAGS=("--addr" "${SOCKET}" - "--unix" "--response" "${MODE}" "--transport" "${TRANSPORT}" "--protocol" "${PROTOCOL}") @@ -83,26 +99,41 @@ else fi # start server -"${DRIVER_DIR}/server" "${SERVICE_FLAGS[@]}" & -SERVER_PID="$!" +if [[ "$OSTYPE" == "msys" ]]; then + echo "${SERVICE_FLAGS[@]}" + "${DRIVER_DIR}/server.exe" "${SERVICE_FLAGS[@]}" & + SERVER_PID="$!" +else + SERVICE_FLAGS+=("--unix") + "${DRIVER_DIR}/server" "${SERVICE_FLAGS[@]}" & + SERVER_PID="$!" + while [[ ! -a "${SOCKET}" ]]; do + sleep 0.1 + + if ! kill -0 "${SERVER_PID}"; then + echo "server failed to start" + exit 1 + fi + done +fi trap 'kill ${SERVER_PID}' EXIT; -while [[ ! -a "${SOCKET}" ]]; do - sleep 0.1 - - if ! kill -0 "${SERVER_PID}"; then - echo "server failed to start" - exit 1 - fi -done +CLIENT_COMMAND="" +if [[ "$OSTYPE" == "msys" ]]; then + CLIENT_COMMAND="${DRIVER_DIR}/client.exe" +else + CLIENT_COMMAND="${DRIVER_DIR}/client" +fi if [[ -n "$HEADERS" ]]; then SERVICE_FLAGS+=("--headers") SERVICE_FLAGS+=("$HEADERS") fi -"${DRIVER_DIR}/client" "${SERVICE_FLAGS[@]}" \ - --request "${REQUEST_FILE}" \ - --response "${RESPONSE_FILE}" \ - "${METHOD}" "$@" +echo "${METHOD}" "$@" + +$CLIENT_COMMAND "${SERVICE_FLAGS[@]}" \ + --request "${REQUEST_FILE}" \ + --response "${RESPONSE_FILE}" \ + "${METHOD}" "$@" diff --git a/test/extensions/filters/network/thrift_proxy/driver/server.py b/test/extensions/filters/network/thrift_proxy/driver/server.py index e4d5a5c7cc5b9..650280919eaa0 100755 --- a/test/extensions/filters/network/thrift_proxy/driver/server.py +++ b/test/extensions/filters/network/thrift_proxy/driver/server.py @@ -15,6 +15,11 @@ from fbthrift import THeaderTransport from finagle import TFinagleServerProcessor, TFinagleServerProtocol +# On Windows we run this test on Python3 +if sys.version_info[0] != 2: + sys.stdin.reconfigure(encoding='utf-8') + sys.stdout.reconfigure(encoding='utf-8') + class SuccessHandler: diff --git a/test/extensions/filters/network/thrift_proxy/integration.cc b/test/extensions/filters/network/thrift_proxy/integration.cc index 62f33950f4744..cd907b4e084a2 100644 --- a/test/extensions/filters/network/thrift_proxy/integration.cc +++ b/test/extensions/filters/network/thrift_proxy/integration.cc @@ -80,6 +80,10 @@ void BaseThriftIntegrationTest::preparePayloads(const PayloadOptions& options, args.push_back(absl::StrJoin(headers, ",")); } + auto temp_path = TestEnvironment::temporaryDirectory(); + args.push_back("-T"); + args.push_back(temp_path); + args.push_back(options.method_name_); std::copy(options.method_args_.begin(), options.method_args_.end(), std::back_inserter(args)); diff --git a/test/extensions/filters/network/thrift_proxy/integration_test.cc b/test/extensions/filters/network/thrift_proxy/integration_test.cc index 2766bf61d5403..c3ea956fb23fd 100644 --- a/test/extensions/filters/network/thrift_proxy/integration_test.cc +++ b/test/extensions/filters/network/thrift_proxy/integration_test.cc @@ -401,6 +401,9 @@ INSTANTIATE_TEST_SUITE_P(FramedTwitter, ThriftTwitterConnManagerIntegrationTest, // Because of the protocol upgrade requests and the difficulty of separating them, we test this // protocol independently. TEST_P(ThriftTwitterConnManagerIntegrationTest, Success) { +// This test relies on an old Apache Thrift Python package +// that is only available in Python2. Disabling the test on Windows. +#ifndef WIN32 initializeCall(DriverMode::Success); uint32_t upgrade_request_size = request_bytes_.peekBEInt() + 4; @@ -457,6 +460,7 @@ TEST_P(ThriftTwitterConnManagerIntegrationTest, Success) { EXPECT_EQ(2U, counter->value()); counter = test_server_->counter("thrift.thrift_stats.response_success"); EXPECT_EQ(2U, counter->value()); +#endif } } // namespace ThriftProxy diff --git a/test/integration/hotrestart_test.sh b/test/integration/hotrestart_test.sh index 729cc09bdc3c6..f53dde86700d7 100755 --- a/test/integration/hotrestart_test.sh +++ b/test/integration/hotrestart_test.sh @@ -78,18 +78,18 @@ echo "Hot restart test using dynamic base id" TEST_INDEX=0 function run_testsuite() { - local BASE_ID BASE_ID_PATH HOT_RESTART_JSON="$1" FAKE_SYMBOL_TABLE="$2" + local BASE_ID BASE_ID_PATH HOT_RESTART_JSON="$1" local SOCKET_PATH=@envoy_domain_socket local SOCKET_MODE=0 - if [ -n "$3" ] && [ -n "$4" ] + if [ -n "$2" ] && [ -n "$3" ] then - SOCKET_PATH="$3" - SOCKET_MODE="$4" + SOCKET_PATH="$2" + SOCKET_MODE="$3" fi start_test validation check "${ENVOY_BIN}" -c "${HOT_RESTART_JSON}" --mode validate --service-cluster cluster \ - --use-fake-symbol-table "$FAKE_SYMBOL_TABLE" --service-node node --disable-hot-restart + --service-node node --disable-hot-restart BASE_ID_PATH=$(mktemp 'envoy_test_base_id.XXXXXX') echo "Selected dynamic base id path ${BASE_ID_PATH}" @@ -101,8 +101,7 @@ function run_testsuite() { ADMIN_ADDRESS_PATH_0="${TEST_TMPDIR}"/admin.0."${TEST_INDEX}".address run_in_background_saving_pid "${ENVOY_BIN}" -c "${HOT_RESTART_JSON}" \ --restart-epoch 0 --use-dynamic-base-id --base-id-path "${BASE_ID_PATH}" \ - --service-cluster cluster --service-node node --use-fake-symbol-table "$FAKE_SYMBOL_TABLE" \ - --admin-address-path "${ADMIN_ADDRESS_PATH_0}" \ + --service-cluster cluster --service-node node --admin-address-path "${ADMIN_ADDRESS_PATH_0}" \ --socket-path "${SOCKET_PATH}" --socket-mode "${SOCKET_MODE}" BASE_ID=$(cat "${BASE_ID_PATH}") @@ -140,22 +139,13 @@ function run_testsuite() { echo "The Envoy's hot restart version is ${CLI_HOT_RESTART_VERSION}" echo "Now checking that the above version is what we expected." check [ "${CLI_HOT_RESTART_VERSION}" = "${EXPECTED_CLI_HOT_RESTART_VERSION}" ] - - start_test "Checking for consistency of /hot_restart_version with --use-fake-symbol-table ${FAKE_SYMBOL_TABLE}" - CLI_HOT_RESTART_VERSION=$("${ENVOY_BIN}" --hot-restart-version --base-id "${BASE_ID}" \ - --use-fake-symbol-table "$FAKE_SYMBOL_TABLE" 2>&1) - CLI_HOT_RESTART_VERSION=$(strip_fake_symbol_table_warning "$CLI_HOT_RESTART_VERSION" "$FAKE_SYMBOL_TABLE") - EXPECTED_CLI_HOT_RESTART_VERSION="11.${SHARED_MEMORY_SIZE}" - check [ "${CLI_HOT_RESTART_VERSION}" = "${EXPECTED_CLI_HOT_RESTART_VERSION}" ] - + start_test "Checking for match of --hot-restart-version and admin /hot_restart_version" ADMIN_ADDRESS_0=$(cat "${ADMIN_ADDRESS_PATH_0}") echo "fetching hot restart version from http://${ADMIN_ADDRESS_0}/hot_restart_version ..." ADMIN_HOT_RESTART_VERSION=$(curl -sg "http://${ADMIN_ADDRESS_0}/hot_restart_version") echo "Fetched ADMIN_HOT_RESTART_VERSION is ${ADMIN_HOT_RESTART_VERSION}" - CLI_HOT_RESTART_VERSION=$("${ENVOY_BIN}" --hot-restart-version --base-id "${BASE_ID}" \ - --use-fake-symbol-table "$FAKE_SYMBOL_TABLE" 2>&1) - CLI_HOT_RESTART_VERSION=$(strip_fake_symbol_table_warning "$CLI_HOT_RESTART_VERSION" "$FAKE_SYMBOL_TABLE") + CLI_HOT_RESTART_VERSION=$("${ENVOY_BIN}" --hot-restart-version --base-id "${BASE_ID}" 2>&1) check [ "${ADMIN_HOT_RESTART_VERSION}" = "${CLI_HOT_RESTART_VERSION}" ] start_test "Checking server.hot_restart_generation 1" @@ -175,7 +165,7 @@ function run_testsuite() { ADMIN_ADDRESS_PATH_1="${TEST_TMPDIR}"/admin.1."${TEST_INDEX}".address run_in_background_saving_pid "${ENVOY_BIN}" -c "${UPDATED_HOT_RESTART_JSON}" \ --restart-epoch 1 --base-id "${BASE_ID}" --service-cluster cluster --service-node node \ - --use-fake-symbol-table "$FAKE_SYMBOL_TABLE" --admin-address-path "${ADMIN_ADDRESS_PATH_1}" \ + --admin-address-path "${ADMIN_ADDRESS_PATH_1}" \ --socket-path "${SOCKET_PATH}" --socket-mode "${SOCKET_MODE}" SERVER_1_PID=$BACKGROUND_PID @@ -216,7 +206,7 @@ function run_testsuite() { start_test "Starting epoch 2" run_in_background_saving_pid "${ENVOY_BIN}" -c "${UPDATED_HOT_RESTART_JSON}" \ --restart-epoch 2 --base-id "${BASE_ID}" --service-cluster cluster --service-node node \ - --use-fake-symbol-table "$FAKE_SYMBOL_TABLE" --admin-address-path "${ADMIN_ADDRESS_PATH_2}" \ + --admin-address-path "${ADMIN_ADDRESS_PATH_2}" \ --parent-shutdown-time-s 3 \ --socket-path "${SOCKET_PATH}" --socket-mode "${SOCKET_MODE}" @@ -267,37 +257,14 @@ function run_testsuite() { wait "${SERVER_2_PID}" } -# TODO(#13399): remove this helper function and the references to it, as long as -# the references to $FAKE_SYMBOL_TABLE. -function strip_fake_symbol_table_warning() { - local INPUT="$1" - local FAKE_SYMBOL_TABLE="$2" - if [ "$FAKE_SYMBOL_TABLE" = "1" ]; then - echo "$INPUT" | grep -v "Fake symbol tables have been removed" - else - echo "$INPUT" - fi -} - # Hotrestart in abstract namespace for HOT_RESTART_JSON in "${JSON_TEST_ARRAY[@]}" do - # Run one of the tests with real symbol tables. No need to do all of them. - if [ "$TEST_INDEX" = "0" ]; then - run_testsuite "$HOT_RESTART_JSON" "0" || exit 1 - fi - - run_testsuite "$HOT_RESTART_JSON" "1" || exit 1 + run_testsuite "$HOT_RESTART_JSON" || exit 1 done # Hotrestart in specified UDS -# Real symbol tables are the default, so I had run just one with fake symbol tables -# (Switch the "0" and "1" in the second arg in the two run_testsuite calls below). -if [ "$TEST_INDEX" = "0" ]; then - run_testsuite "${HOT_RESTART_JSON_V4}" "0" "${SOCKET_DIR}/envoy_domain_socket" "600" || exit 1 -fi - -run_testsuite "${HOT_RESTART_JSON_V4}" "1" "${SOCKET_DIR}/envoy_domain_socket" "600" || exit 1 +run_testsuite "${HOT_RESTART_JSON_V4}" "${SOCKET_DIR}/envoy_domain_socket" "600" || exit 1 start_test "disabling hot_restart by command line." CLI_HOT_RESTART_VERSION=$("${ENVOY_BIN}" --hot-restart-version --disable-hot-restart 2>&1) @@ -308,8 +275,7 @@ start_test socket-mode for socket path run_in_background_saving_pid "${ENVOY_BIN}" -c "${HOT_RESTART_JSON}" \ --restart-epoch 0 --base-id 0 --base-id-path "${BASE_ID_PATH}" \ --socket-path "${SOCKET_DIR}"/envoy_domain_socket --socket-mode 644 \ - --service-cluster cluster --service-node node --use-fake-symbol-table "$FAKE_SYMBOL_TABLE" \ - --admin-address-path "${ADMIN_ADDRESS_PATH_0}" + --service-cluster cluster --service-node node --admin-address-path "${ADMIN_ADDRESS_PATH_0}" sleep 3 EXPECTED_SOCKET_MODE=$(stat -c '%a' "${SOCKET_DIR}"/envoy_domain_socket_parent_0) check [ "644" = "${EXPECTED_SOCKET_MODE}" ] diff --git a/test/mocks/server/options.h b/test/mocks/server/options.h index 51eeadf3d3b28..d28fd47b143ae 100644 --- a/test/mocks/server/options.h +++ b/test/mocks/server/options.h @@ -47,7 +47,6 @@ class MockOptions : public Options { MOCK_METHOD(bool, hotRestartDisabled, (), (const)); MOCK_METHOD(bool, signalHandlingEnabled, (), (const)); MOCK_METHOD(bool, mutexTracingEnabled, (), (const)); - MOCK_METHOD(bool, fakeSymbolTableEnabled, (), (const)); MOCK_METHOD(bool, cpusetThreadsEnabled, (), (const)); MOCK_METHOD(const std::vector&, disabledExtensions, (), (const)); MOCK_METHOD(Server::CommandLineOptionsPtr, toCommandLineOptions, (), (const)); diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index bf22008af5a3a..e618bb6343498 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -94,7 +94,7 @@ TEST_F(OptionsImplTest, All) { "--log-path " "/foo/bar " "--disable-hot-restart --cpuset-threads --allow-unknown-static-fields " - "--reject-unknown-dynamic-fields --use-fake-symbol-table 0 --base-id 5 " + "--reject-unknown-dynamic-fields --base-id 5 " "--use-dynamic-base-id --base-id-path /foo/baz " "--socket-path /foo/envoy_domain_socket --socket-mode 644"); EXPECT_EQ(Server::Mode::Validate, options->mode()); @@ -118,7 +118,6 @@ TEST_F(OptionsImplTest, All) { EXPECT_TRUE(options->cpusetThreadsEnabled()); EXPECT_TRUE(options->allowUnknownStaticFields()); EXPECT_TRUE(options->rejectUnknownDynamicFields()); - EXPECT_FALSE(options->fakeSymbolTableEnabled()); EXPECT_EQ(5U, options->baseId()); EXPECT_TRUE(options->useDynamicBaseId()); EXPECT_EQ("/foo/baz", options->baseIdPath()); @@ -129,13 +128,6 @@ TEST_F(OptionsImplTest, All) { EXPECT_EQ(Server::Mode::InitOnly, options->mode()); } -// TODO(#13399): remove this test once we remove the option. -TEST_F(OptionsImplTest, FakeSymtabWarning) { - EXPECT_LOG_CONTAINS("warning", "Fake symbol tables have been removed", - createOptionsImpl("envoy --use-fake-symbol-table 1")); - EXPECT_NO_LOGS(createOptionsImpl("envoy --use-fake-symbol-table 0")); -} - // Either variants of allow-unknown-[static-]-fields works. TEST_F(OptionsImplTest, AllowUnknownFields) { { @@ -161,7 +153,6 @@ TEST_F(OptionsImplTest, SetAll) { bool hot_restart_disabled = options->hotRestartDisabled(); bool signal_handling_enabled = options->signalHandlingEnabled(); bool cpuset_threads_enabled = options->cpusetThreadsEnabled(); - bool fake_symbol_table_enabled = options->fakeSymbolTableEnabled(); options->setBaseId(109876); options->setConcurrency(42); @@ -189,7 +180,6 @@ TEST_F(OptionsImplTest, SetAll) { options->setCpusetThreads(!options->cpusetThreadsEnabled()); options->setAllowUnkownFields(true); options->setRejectUnknownFieldsDynamic(true); - options->setFakeSymbolTableEnabled(!options->fakeSymbolTableEnabled()); options->setSocketPath("/foo/envoy_domain_socket"); options->setSocketMode(0644); @@ -219,7 +209,6 @@ TEST_F(OptionsImplTest, SetAll) { EXPECT_EQ(!cpuset_threads_enabled, options->cpusetThreadsEnabled()); EXPECT_TRUE(options->allowUnknownStaticFields()); EXPECT_TRUE(options->rejectUnknownDynamicFields()); - EXPECT_EQ(!fake_symbol_table_enabled, options->fakeSymbolTableEnabled()); EXPECT_EQ("/foo/envoy_domain_socket", options->socketPath()); EXPECT_EQ(0644, options->socketMode()); @@ -293,14 +282,13 @@ TEST_F(OptionsImplTest, OptionsAreInSyncWithProto) { // Failure of this condition indicates that the server_info proto is not in sync with the options. // If an option is added/removed, please update server_info proto as well to keep it in sync. - // Currently the following 7 options are not defined in proto, hence the count differs by 7. + // Currently the following 6 options are not defined in proto, hence the count differs by 6. // 1. version - default TCLAP argument. // 2. help - default TCLAP argument. // 3. ignore_rest - default TCLAP argument. // 4. allow-unknown-fields - deprecated alias of allow-unknown-static-fields. - // 5. use-fake-symbol-table - short-term override for rollout of real symbol-table implementation. - // 6. hot restart version - print the hot restart version and exit. - const uint32_t options_not_in_proto = 6; + // 5. hot restart version - print the hot restart version and exit. + const uint32_t options_not_in_proto = 5; // There are two deprecated options: "max_stats" and "max_obj_name_len". const uint32_t deprecated_options = 2; diff --git a/test/server/server_corpus/grpc_illegal_characters b/test/server/server_corpus/grpc_illegal_characters new file mode 100644 index 0000000000000..8309bb56e7d7e --- /dev/null +++ b/test/server/server_corpus/grpc_illegal_characters @@ -0,0 +1,64 @@ +cluster_manager { + load_stats_config { + api_type: GRPC + grpc_services { + google_grpc { + target_uri: "48?" + stat_prefix: "$" + } + initial_metadata { + key: "2" + value: "$$" + } + initial_metadata { + key: "2" + value: "2" + } + initial_metadata { + key: "2" + value: "$$" + } + initial_metadata { + key: "2" + value: "$$" + } + initial_metadata { + key: "2;" + value: "$$" + } + initial_metadata { + key: "1" + value: "$$" + } + initial_metadata { + key: "2" + value: "$$" + } + initial_metadata { + key: "2" + value: "$$" + } + initial_metadata { + key: "2" + value: "2" + } + initial_metadata { + key: "2" + value: "$$" + } + initial_metadata { + key: "4" + value: "$$" + } + initial_metadata { + key: "2" + value: "1" + } + initial_metadata { + key: "10" + value: "$$" + } + } + } +} +enable_dispatcher_stats: true \ No newline at end of file diff --git a/test/server/server_test.cc b/test/server/server_test.cc index aa097fc787cf3..1db932a7c8c4c 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -1411,6 +1411,16 @@ TEST_P(ServerInstanceImplTest, DisabledExtension) { ASSERT_TRUE(disabled_filter_found); } +TEST_P(ServerInstanceImplTest, NullProcessContextTest) { + // These are already the defaults. Repeated here for clarity. + process_object_ = nullptr; + process_context_ = nullptr; + + initialize("test/server/test_data/server/empty_bootstrap.yaml"); + ProcessContextOptRef context = server_->processContext(); + EXPECT_FALSE(context.has_value()); +} + } // namespace } // namespace Server } // namespace Envoy From 10f45968084da9c702209fbacb1feb8710659975 Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 30 Nov 2020 22:28:44 +0000 Subject: [PATCH 067/106] Merge remote-tracking branch 'upstream/master' Signed-off-by: grial1 --- api/envoy/api/v2/route/route_components.proto | 19 +- .../config/route/v3/route_components.proto | 19 +- .../route/v4alpha/route_components.proto | 19 +- .../thrift_proxy/v3/thrift_proxy.proto | 8 +- .../thrift_proxy/v4alpha/thrift_proxy.proto | 8 +- bazel/foreign_cc/BUILD | 4 +- bazel/repository_locations.bzl | 12 +- ci/repokitteh/modules/ownerscheck.star | 53 ++- docs/root/version_history/current.rst | 1 + .../envoy/api/v2/route/route_components.proto | 19 +- .../config/route/v3/route_components.proto | 19 +- .../route/v4alpha/route_components.proto | 19 +- .../thrift_proxy/v3/thrift_proxy.proto | 8 +- .../thrift_proxy/v4alpha/thrift_proxy.proto | 8 +- repokitteh.star | 1 + .../filters/network/thrift_proxy/config.cc | 3 +- .../filters/network/thrift_proxy/config.h | 2 + .../network/thrift_proxy/conn_manager.cc | 34 ++ .../network/thrift_proxy/conn_manager.h | 5 + .../filters/network/thrift_proxy/decoder.cc | 28 +- .../filters/network/thrift_proxy/decoder.h | 17 +- .../network/thrift_proxy/decoder_events.h | 8 + .../network/thrift_proxy/filters/filter.h | 6 + .../filters/pass_through_filter.h | 6 + .../network/thrift_proxy/protocol_converter.h | 5 + .../thrift_proxy/router/router_impl.cc | 6 + .../network/thrift_proxy/router/router_impl.h | 6 +- .../network/thrift_proxy/thrift_object_impl.h | 2 + test/config/utility.cc | 20 +- test/config/utility.h | 34 ++ .../filters/network/thrift_proxy/BUILD | 2 + .../network/thrift_proxy/config_test.cc | 18 + .../network/thrift_proxy/conn_manager_test.cc | 264 ++++++++++++-- .../network/thrift_proxy/decoder_test.cc | 332 ++++++++++++++++-- .../network/thrift_proxy/integration_test.cc | 60 +++- .../filters/network/thrift_proxy/mocks.h | 4 + .../network/thrift_proxy/router_test.cc | 104 +++++- tools/dependency/release_dates.py | 1 + 38 files changed, 1031 insertions(+), 153 deletions(-) diff --git a/api/envoy/api/v2/route/route_components.proto b/api/envoy/api/v2/route/route_components.proto index c1e84a5618a76..d73fbb8674c90 100644 --- a/api/envoy/api/v2/route/route_components.proto +++ b/api/envoy/api/v2/route/route_components.proto @@ -1133,13 +1133,18 @@ message HedgePolicy { // [#not-implemented-hide:] type.FractionalPercent additional_request_chance = 2; - // Indicates that a hedged request should be sent when the per-try timeout - // is hit. This will only occur if the retry policy also indicates that a - // timed out request should be retried. - // Once a timed out request is retried due to per try timeout, the router - // filter will ensure that it is not retried again even if the returned - // response headers would otherwise be retried according the specified - // :ref:`RetryPolicy `. + // Indicates that a hedged request should be sent when the per-try timeout is hit. + // This means that a retry will be issued without resetting the original request, leaving multiple upstream requests in flight. + // The first request to complete successfully will be the one returned to the caller. + // + // * At any time, a successful response (i.e. not triggering any of the retry-on conditions) would be returned to the client. + // * Before per-try timeout, an error response (per retry-on conditions) would be retried immediately or returned ot the client + // if there are no more retries left. + // * After per-try timeout, an error response would be discarded, as a retry in the form of a hedged request is already in progress. + // + // Note: For this to have effect, you must have a :ref:`RetryPolicy ` that retries at least + // one error code and specifies a maximum number of retries. + // // Defaults to false. bool hedge_on_per_try_timeout = 3; } diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 62633012cf477..6915c62922fa3 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1310,13 +1310,18 @@ message HedgePolicy { // [#not-implemented-hide:] type.v3.FractionalPercent additional_request_chance = 2; - // Indicates that a hedged request should be sent when the per-try timeout - // is hit. This will only occur if the retry policy also indicates that a - // timed out request should be retried. - // Once a timed out request is retried due to per try timeout, the router - // filter will ensure that it is not retried again even if the returned - // response headers would otherwise be retried according the specified - // :ref:`RetryPolicy `. + // Indicates that a hedged request should be sent when the per-try timeout is hit. + // This means that a retry will be issued without resetting the original request, leaving multiple upstream requests in flight. + // The first request to complete successfully will be the one returned to the caller. + // + // * At any time, a successful response (i.e. not triggering any of the retry-on conditions) would be returned to the client. + // * Before per-try timeout, an error response (per retry-on conditions) would be retried immediately or returned ot the client + // if there are no more retries left. + // * After per-try timeout, an error response would be discarded, as a retry in the form of a hedged request is already in progress. + // + // Note: For this to have effect, you must have a :ref:`RetryPolicy ` that retries at least + // one error code and specifies a maximum number of retries. + // // Defaults to false. bool hedge_on_per_try_timeout = 3; } diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto index 9f3da5376ae0b..3d6dbfd41130d 100644 --- a/api/envoy/config/route/v4alpha/route_components.proto +++ b/api/envoy/config/route/v4alpha/route_components.proto @@ -1257,13 +1257,18 @@ message HedgePolicy { // [#not-implemented-hide:] type.v3.FractionalPercent additional_request_chance = 2; - // Indicates that a hedged request should be sent when the per-try timeout - // is hit. This will only occur if the retry policy also indicates that a - // timed out request should be retried. - // Once a timed out request is retried due to per try timeout, the router - // filter will ensure that it is not retried again even if the returned - // response headers would otherwise be retried according the specified - // :ref:`RetryPolicy `. + // Indicates that a hedged request should be sent when the per-try timeout is hit. + // This means that a retry will be issued without resetting the original request, leaving multiple upstream requests in flight. + // The first request to complete successfully will be the one returned to the caller. + // + // * At any time, a successful response (i.e. not triggering any of the retry-on conditions) would be returned to the client. + // * Before per-try timeout, an error response (per retry-on conditions) would be retried immediately or returned ot the client + // if there are no more retries left. + // * After per-try timeout, an error response would be discarded, as a retry in the form of a hedged request is already in progress. + // + // Note: For this to have effect, you must have a :ref:`RetryPolicy ` that retries at least + // one error code and specifies a maximum number of retries. + // // Defaults to false. bool hedge_on_per_try_timeout = 3; } diff --git a/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto b/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto index 2b9863e91ffac..53bec0436116a 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto +++ b/api/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto @@ -58,7 +58,7 @@ enum ProtocolType { TWITTER = 4; } -// [#next-free-field: 6] +// [#next-free-field: 7] message ThriftProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.thrift_proxy.v2alpha1.ThriftProxy"; @@ -82,6 +82,12 @@ message ThriftProxy { // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. repeated ThriftFilter thrift_filters = 5; + + // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. + // This mode will only work if the upstream and downstream protocols are the same and the transport + // is the same, the transport type is framed and the protocol is not Twitter. Otherwise Envoy will + // fallback to decode the data. + bool payload_passthrough = 6; } // ThriftFilter configures a Thrift filter. diff --git a/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto b/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto index b75d0e39eaf29..8e7bf3c91a032 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto +++ b/api/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto @@ -58,7 +58,7 @@ enum ProtocolType { TWITTER = 4; } -// [#next-free-field: 6] +// [#next-free-field: 7] message ThriftProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy"; @@ -82,6 +82,12 @@ message ThriftProxy { // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. repeated ThriftFilter thrift_filters = 5; + + // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. + // This mode will only work if the upstream and downstream protocols are the same and the transport + // is the same, the transport type is framed and the protocol is not Twitter. Otherwise Envoy will + // fallback to decode the data. + bool payload_passthrough = 6; } // ThriftFilter configures a Thrift filter. diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 43dbfe3459aeb..a374a7d905de4 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -91,8 +91,8 @@ envoy_cmake_external( "//conditions:default": [], }), postfix_script = select({ - "//bazel:windows_x86_64": "cp -L $EXT_BUILD_ROOT/external/com_github_c_ares_c_ares/src/lib/nameser.h $INSTALLDIR/include/nameser.h && cp -L $EXT_BUILD_ROOT/external/com_github_c_ares_c_ares/src/lib/ares_dns.h $INSTALLDIR/include/ares_dns.h", - "//conditions:default": "cp -L $EXT_BUILD_ROOT/external/com_github_c_ares_c_ares/src/lib/ares_dns.h $INSTALLDIR/include/ares_dns.h", + "//bazel:windows_x86_64": "cp -L $EXT_BUILD_ROOT/external/com_github_c_ares_c_ares/src/lib/nameser.h $INSTALLDIR/include/nameser.h && cp -L $EXT_BUILD_ROOT/external/com_github_c_ares_c_ares/include/ares_dns.h $INSTALLDIR/include/ares_dns.h", + "//conditions:default": "cp -L $EXT_BUILD_ROOT/external/com_github_c_ares_c_ares/include/ares_dns.h $INSTALLDIR/include/ares_dns.h", }), static_libraries = select({ "//bazel:windows_x86_64": ["cares.lib"], diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index abdaa8a432068..a5c61cc113ccb 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -101,12 +101,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "c-ares", project_desc = "C library for asynchronous DNS requests", project_url = "https://c-ares.haxx.se/", - version = "c15f403875ababb1149215d49683d720b3d035c7", - sha256 = "0ad4f9748752909b00a0ca8d2b6a075e0a7a06ee922d9dcf9625d2562d7c894a", + version = "1.17.1", + sha256 = "d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40", strip_prefix = "c-ares-{version}", - urls = ["https://github.com/c-ares/c-ares/archive/{version}.tar.gz"], + urls = ["https://github.com/c-ares/c-ares/releases/download/cares-{underscore_version}/c-ares-{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2020-10-21", + release_date = "2020-11-19", cpe = "cpe:2.3:a:c-ares_project:c-ares:*", ), com_github_circonus_labs_libcircllhist = dict( @@ -414,7 +414,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/google/jwt_verify_lib/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.jwt_authn"], - release_date = "2020-11-04", + release_date = "2020-11-05", cpe = "N/A", ), com_github_nodejs_http_parser = dict( @@ -513,7 +513,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.grpc_json_transcoder"], - release_date = "2020-11-12", + release_date = "2020-11-13", cpe = "N/A", ), io_bazel_rules_go = dict( diff --git a/ci/repokitteh/modules/ownerscheck.star b/ci/repokitteh/modules/ownerscheck.star index e93010f89a7f6..08b728a9daccf 100644 --- a/ci/repokitteh/modules/ownerscheck.star +++ b/ci/repokitteh/modules/ownerscheck.star @@ -8,7 +8,8 @@ # "path": "api/", # "label": "api", # "allow_global_approval": True, -# "github_status_label" = "any API change", +# "github_status_label": "any API change", +# "auto_assign": True, # }, # ], # ) @@ -27,8 +28,13 @@ # # 'label' refers to a GitHub label applied to any matching PR. The GitHub check status # can be customized with `github_status_label`. +# +# If 'auto_assign' is set True, a randomly selected reviwer from the owner team will +# be selected and set as a reviewer on the PR if there is not already a member of the +# owner team set as reviewer or assignee for the PR. load("text", "match") +load("time", "now") load("github.com/repokitteh/modules/lib/utils.star", "react") def _store_partial_approval(who, files): @@ -64,7 +70,8 @@ def _get_relevant_specs(specs, changed_files): label=spec.get("label", None), path_match=path_match, allow_global_approval=allow_global_approval, - status_label=status_label)) + status_label=status_label, + auto_assign=spec.get("auto_assign", False))) print("specs: %s" % relevant) @@ -152,20 +159,19 @@ def _reconcile(config, specs=None): return results -def _comment(config, results, force=False): +def _comment(config, results, assignees, sender, force=False): lines = [] for spec, approved in results: if approved: continue - mention = spec.owner + owner = spec.owner - if mention[0] != '@': - mention = '@' + mention + if owner[-1] == '!': + owner = owner[:-1] - if mention[-1] == '!': - mention = mention[:-1] + mention = '@' + owner match_description = spec.path_match if match_description: @@ -185,21 +191,40 @@ def _comment(config, results, force=False): elif mode == 'fyi': lines.append('CC %s: FYI only%s.' % (mention, match_description)) + if mode != 'skip' and spec.auto_assign: + api_assignee = None + # Find owners via github.team_get_by_name, github.team_list_members + team_name = owner.split('/')[1] + team = github.team_get_by_name(team_name) + # Exclude author from assignment. + members = [m['login'] for m in github.team_list_members(team['id']) if m['login'] != sender] + # Is a team member already assigned? The first assigned team member is picked. Bad O(n^2) as + # Starlark doesn't have sets, n is small. + for assignee in assignees: + if assignee in members: + api_assignee = assignee + break + # Otherwise, pick at "random" (we just use timestamp). + if not api_assignee: + api_assignee = members[now().second % len(members)] + lines.append('API shepherd assignee is @%s' % api_assignee) + github.issue_assign(api_assignee) + if lines: github.issue_create_comment('\n'.join(lines)) -def _reconcile_and_comment(config): - _comment(config, _reconcile(config)) +def _reconcile_and_comment(config, assignees, sender): + _comment(config, _reconcile(config), assignees, sender) -def _force_reconcile_and_comment(config): - _comment(config, _reconcile(config), force=True) +def _force_reconcile_and_comment(config, assignees, sender): + _comment(config, _reconcile(config), assignees, sender, force=True) -def _pr(action, config): +def _pr(action, config, assignees, sender): if action in ['synchronize', 'opened']: - _reconcile_and_comment(config) + _reconcile_and_comment(config, assignees, sender) def _pr_review(action, review_state, config): diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index a19010abf84dc..90389259cc2e1 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -75,6 +75,7 @@ New Features :ref:`CertificateValidationContext `. * signal: added an extension point for custom actions to run on the thread that has encountered a fatal error. Actions are configurable via :ref:`fatal_actions `. * tcp: added a new :ref:`envoy.overload_actions.reject_incoming_connections ` action to reject incoming TCP connections. +* thrift_proxy: added a new :ref: `payload_passthrough ` option to skip decoding body in the Thrift message. * tls: added support for RSA certificates with 4096-bit keys in FIPS mode. * tracing: added SkyWalking tracer. * xds: added support for resource TTLs. A TTL is specified on the :ref:`Resource `. For SotW, a :ref:`Resource ` can be embedded diff --git a/generated_api_shadow/envoy/api/v2/route/route_components.proto b/generated_api_shadow/envoy/api/v2/route/route_components.proto index c1e84a5618a76..d73fbb8674c90 100644 --- a/generated_api_shadow/envoy/api/v2/route/route_components.proto +++ b/generated_api_shadow/envoy/api/v2/route/route_components.proto @@ -1133,13 +1133,18 @@ message HedgePolicy { // [#not-implemented-hide:] type.FractionalPercent additional_request_chance = 2; - // Indicates that a hedged request should be sent when the per-try timeout - // is hit. This will only occur if the retry policy also indicates that a - // timed out request should be retried. - // Once a timed out request is retried due to per try timeout, the router - // filter will ensure that it is not retried again even if the returned - // response headers would otherwise be retried according the specified - // :ref:`RetryPolicy `. + // Indicates that a hedged request should be sent when the per-try timeout is hit. + // This means that a retry will be issued without resetting the original request, leaving multiple upstream requests in flight. + // The first request to complete successfully will be the one returned to the caller. + // + // * At any time, a successful response (i.e. not triggering any of the retry-on conditions) would be returned to the client. + // * Before per-try timeout, an error response (per retry-on conditions) would be retried immediately or returned ot the client + // if there are no more retries left. + // * After per-try timeout, an error response would be discarded, as a retry in the form of a hedged request is already in progress. + // + // Note: For this to have effect, you must have a :ref:`RetryPolicy ` that retries at least + // one error code and specifies a maximum number of retries. + // // Defaults to false. bool hedge_on_per_try_timeout = 3; } diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index 6b97d3ff4a126..7813038930a31 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -1317,13 +1317,18 @@ message HedgePolicy { // [#not-implemented-hide:] type.v3.FractionalPercent additional_request_chance = 2; - // Indicates that a hedged request should be sent when the per-try timeout - // is hit. This will only occur if the retry policy also indicates that a - // timed out request should be retried. - // Once a timed out request is retried due to per try timeout, the router - // filter will ensure that it is not retried again even if the returned - // response headers would otherwise be retried according the specified - // :ref:`RetryPolicy `. + // Indicates that a hedged request should be sent when the per-try timeout is hit. + // This means that a retry will be issued without resetting the original request, leaving multiple upstream requests in flight. + // The first request to complete successfully will be the one returned to the caller. + // + // * At any time, a successful response (i.e. not triggering any of the retry-on conditions) would be returned to the client. + // * Before per-try timeout, an error response (per retry-on conditions) would be retried immediately or returned ot the client + // if there are no more retries left. + // * After per-try timeout, an error response would be discarded, as a retry in the form of a hedged request is already in progress. + // + // Note: For this to have effect, you must have a :ref:`RetryPolicy ` that retries at least + // one error code and specifies a maximum number of retries. + // // Defaults to false. bool hedge_on_per_try_timeout = 3; } diff --git a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto index 5c9c2c46a202f..0860c9d56700f 100644 --- a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto @@ -1324,13 +1324,18 @@ message HedgePolicy { // [#not-implemented-hide:] type.v3.FractionalPercent additional_request_chance = 2; - // Indicates that a hedged request should be sent when the per-try timeout - // is hit. This will only occur if the retry policy also indicates that a - // timed out request should be retried. - // Once a timed out request is retried due to per try timeout, the router - // filter will ensure that it is not retried again even if the returned - // response headers would otherwise be retried according the specified - // :ref:`RetryPolicy `. + // Indicates that a hedged request should be sent when the per-try timeout is hit. + // This means that a retry will be issued without resetting the original request, leaving multiple upstream requests in flight. + // The first request to complete successfully will be the one returned to the caller. + // + // * At any time, a successful response (i.e. not triggering any of the retry-on conditions) would be returned to the client. + // * Before per-try timeout, an error response (per retry-on conditions) would be retried immediately or returned ot the client + // if there are no more retries left. + // * After per-try timeout, an error response would be discarded, as a retry in the form of a hedged request is already in progress. + // + // Note: For this to have effect, you must have a :ref:`RetryPolicy ` that retries at least + // one error code and specifies a maximum number of retries. + // // Defaults to false. bool hedge_on_per_try_timeout = 3; } diff --git a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto index 88f7b013fec78..4fc04f2d88033 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v3/thrift_proxy.proto @@ -58,7 +58,7 @@ enum ProtocolType { TWITTER = 4; } -// [#next-free-field: 6] +// [#next-free-field: 7] message ThriftProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.thrift_proxy.v2alpha1.ThriftProxy"; @@ -82,6 +82,12 @@ message ThriftProxy { // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. repeated ThriftFilter thrift_filters = 5; + + // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. + // This mode will only work if the upstream and downstream protocols are the same and the transport + // is the same, the transport type is framed and the protocol is not Twitter. Otherwise Envoy will + // fallback to decode the data. + bool payload_passthrough = 6; } // ThriftFilter configures a Thrift filter. diff --git a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto index b75d0e39eaf29..8e7bf3c91a032 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/thrift_proxy/v4alpha/thrift_proxy.proto @@ -58,7 +58,7 @@ enum ProtocolType { TWITTER = 4; } -// [#next-free-field: 6] +// [#next-free-field: 7] message ThriftProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy"; @@ -82,6 +82,12 @@ message ThriftProxy { // compatibility, if no thrift_filters are specified, a default Thrift router filter // (`envoy.filters.thrift.router`) is used. repeated ThriftFilter thrift_filters = 5; + + // If set to true, Envoy will try to skip decode data after metadata in the Thrift message. + // This mode will only work if the upstream and downstream protocols are the same and the transport + // is the same, the transport type is framed and the protocol is not Twitter. Otherwise Envoy will + // fallback to decode the data. + bool payload_passthrough = 6; } // ThriftFilter configures a Thrift filter. diff --git a/repokitteh.star b/repokitteh.star index bf4c78b592915..5181346d560a0 100644 --- a/repokitteh.star +++ b/repokitteh.star @@ -21,6 +21,7 @@ use( "path": "api/envoy/", "label": "api", "github_status_label": "any API change", + "auto_assign": True, }, { "owner": "envoyproxy/api-watchers", diff --git a/source/extensions/filters/network/thrift_proxy/config.cc b/source/extensions/filters/network/thrift_proxy/config.cc index fc2edbb54cb15..52b5030bc57f4 100644 --- a/source/extensions/filters/network/thrift_proxy/config.cc +++ b/source/extensions/filters/network/thrift_proxy/config.cc @@ -121,7 +121,8 @@ ConfigImpl::ConfigImpl( : context_(context), stats_prefix_(fmt::format("thrift.{}.", config.stat_prefix())), stats_(ThriftFilterStats::generateStats(stats_prefix_, context_.scope())), transport_(lookupTransport(config.transport())), proto_(lookupProtocol(config.protocol())), - route_matcher_(new Router::RouteMatcher(config.route_config())) { + route_matcher_(new Router::RouteMatcher(config.route_config())), + payload_passthrough_(config.payload_passthrough()) { if (config.thrift_filters().empty()) { ENVOY_LOG(debug, "using default router filter"); diff --git a/source/extensions/filters/network/thrift_proxy/config.h b/source/extensions/filters/network/thrift_proxy/config.h index 532298c380e3b..02c1fcf4d13a4 100644 --- a/source/extensions/filters/network/thrift_proxy/config.h +++ b/source/extensions/filters/network/thrift_proxy/config.h @@ -81,6 +81,7 @@ class ConfigImpl : public Config, TransportPtr createTransport() override; ProtocolPtr createProtocol() override; Router::Config& routerConfig() override { return *this; } + bool payloadPassthrough() const override { return payload_passthrough_; } private: void processFilter( @@ -94,6 +95,7 @@ class ConfigImpl : public Config, std::unique_ptr route_matcher_; std::list filter_factories_; + const bool payload_passthrough_; }; } // namespace ThriftProxy diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.cc b/source/extensions/filters/network/thrift_proxy/conn_manager.cc index 737e707369785..7f7129715edf6 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.cc +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.cc @@ -176,6 +176,17 @@ DecoderEventHandler& ConnectionManager::newDecoderEventHandler() { return **rpcs_.begin(); } +bool ConnectionManager::passthroughEnabled() const { + if (!config_.payloadPassthrough()) { + return false; + } + + // This is called right after the metadata has been parsed, and the ActiveRpc being processed must + // be in the rpcs_ list. + ASSERT(!rpcs_.empty()); + return (*rpcs_.begin())->passthroughSupported(); +} + bool ConnectionManager::ResponseDecoder::onData(Buffer::Instance& data) { upstream_buffer_.move(data); @@ -274,6 +285,10 @@ FilterStatus ConnectionManager::ResponseDecoder::transportEnd() { return FilterStatus::Continue; } +bool ConnectionManager::ResponseDecoder::passthroughEnabled() const { + return parent_.parent_.passthroughEnabled(); +} + void ConnectionManager::ActiveRpcDecoderFilter::continueDecoding() { const FilterStatus status = parent_.applyDecoderFilters(this); if (status == FilterStatus::Continue) { @@ -398,6 +413,25 @@ void ConnectionManager::ActiveRpc::finalizeRequest() { } } +bool ConnectionManager::ActiveRpc::passthroughSupported() const { + for (auto& entry : decoder_filters_) { + if (!entry->handle_->passthroughSupported()) { + return false; + } + } + return true; +} + +FilterStatus ConnectionManager::ActiveRpc::passthroughData(Buffer::Instance& data) { + filter_context_ = &data; + filter_action_ = [this](DecoderEventHandler* filter) -> FilterStatus { + Buffer::Instance* data = absl::any_cast(filter_context_); + return filter->passthroughData(*data); + }; + + return applyDecoderFilters(nullptr); +} + FilterStatus ConnectionManager::ActiveRpc::messageBegin(MessageMetadataSharedPtr metadata) { ASSERT(metadata->hasSequenceId()); ASSERT(metadata->hasMessageType()); diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.h b/source/extensions/filters/network/thrift_proxy/conn_manager.h index b7408e1a3def6..3fc0cf7b2f046 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.h +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.h @@ -39,6 +39,7 @@ class Config { virtual TransportPtr createTransport() PURE; virtual ProtocolPtr createProtocol() PURE; virtual Router::Config& routerConfig() PURE; + virtual bool payloadPassthrough() const PURE; }; /** @@ -76,6 +77,7 @@ class ConnectionManager : public Network::ReadFilter, // DecoderCallbacks DecoderEventHandler& newDecoderEventHandler() override; + bool passthroughEnabled() const override; private: struct ActiveRpc; @@ -102,6 +104,7 @@ class ConnectionManager : public Network::ReadFilter, // DecoderCallbacks DecoderEventHandler& newDecoderEventHandler() override { return *this; } + bool passthroughEnabled() const override; ActiveRpc& parent_; DecoderPtr decoder_; @@ -180,6 +183,7 @@ class ConnectionManager : public Network::ReadFilter, // DecoderEventHandler FilterStatus transportBegin(MessageMetadataSharedPtr metadata) override; FilterStatus transportEnd() override; + FilterStatus passthroughData(Buffer::Instance& data) override; FilterStatus messageBegin(MessageMetadataSharedPtr metadata) override; FilterStatus messageEnd() override; FilterStatus structBegin(absl::string_view name) override; @@ -225,6 +229,7 @@ class ConnectionManager : public Network::ReadFilter, LinkedList::moveIntoListBack(std::move(wrapper), decoder_filters_); } + bool passthroughSupported() const; FilterStatus applyDecoderFilters(ActiveRpcDecoderFilter* filter); void finalizeRequest(); diff --git a/source/extensions/filters/network/thrift_proxy/decoder.cc b/source/extensions/filters/network/thrift_proxy/decoder.cc index 73a12ff23377b..55f23af6ef88e 100644 --- a/source/extensions/filters/network/thrift_proxy/decoder.cc +++ b/source/extensions/filters/network/thrift_proxy/decoder.cc @@ -2,6 +2,7 @@ #include "envoy/common/exception.h" +#include "common/buffer/buffer_impl.h" #include "common/common/assert.h" #include "common/common/macros.h" @@ -12,8 +13,22 @@ namespace Extensions { namespace NetworkFilters { namespace ThriftProxy { +// PassthroughData -> PassthroughData +// PassthroughData -> MessageEnd (all body bytes received) +DecoderStateMachine::DecoderStatus DecoderStateMachine::passthroughData(Buffer::Instance& buffer) { + if (body_bytes_ > buffer.length()) { + return {ProtocolState::WaitForData}; + } + + Buffer::OwnedImpl body; + body.move(buffer, body_bytes_); + + return {ProtocolState::MessageEnd, handler_.passthroughData(body)}; +} + // MessageBegin -> StructBegin DecoderStateMachine::DecoderStatus DecoderStateMachine::messageBegin(Buffer::Instance& buffer) { + const auto total = buffer.length(); if (!proto_.readMessageBegin(buffer, *metadata_)) { return {ProtocolState::WaitForData}; } @@ -21,7 +36,14 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::messageBegin(Buffer::Ins stack_.clear(); stack_.emplace_back(Frame(ProtocolState::MessageEnd)); - return {ProtocolState::StructBegin, handler_.messageBegin(metadata_)}; + const auto status = handler_.messageBegin(metadata_); + + if (callbacks_.passthroughEnabled()) { + body_bytes_ = metadata_->frameSize() - (total - buffer.length()); + return {ProtocolState::PassthroughData, status}; + } + + return {ProtocolState::StructBegin, status}; } // MessageEnd -> Done @@ -293,6 +315,8 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::handleValue(Buffer::Inst DecoderStateMachine::DecoderStatus DecoderStateMachine::handleState(Buffer::Instance& buffer) { switch (state_) { + case ProtocolState::PassthroughData: + return passthroughData(buffer); case ProtocolState::MessageBegin: return messageBegin(buffer); case ProtocolState::StructBegin: @@ -416,7 +440,7 @@ FilterStatus Decoder::onData(Buffer::Instance& data, bool& buffer_underflow) { request_ = std::make_unique(callbacks_.newDecoderEventHandler()); frame_started_ = true; state_machine_ = - std::make_unique(protocol_, metadata_, request_->handler_); + std::make_unique(protocol_, metadata_, request_->handler_, callbacks_); if (request_->handler_.transportBegin(metadata_) == FilterStatus::StopIteration) { return FilterStatus::StopIteration; diff --git a/source/extensions/filters/network/thrift_proxy/decoder.h b/source/extensions/filters/network/thrift_proxy/decoder.h index 1f7675b9f76ff..6c8b8bef57551 100644 --- a/source/extensions/filters/network/thrift_proxy/decoder.h +++ b/source/extensions/filters/network/thrift_proxy/decoder.h @@ -17,6 +17,7 @@ namespace ThriftProxy { #define ALL_PROTOCOL_STATES(FUNCTION) \ FUNCTION(StopIteration) \ FUNCTION(WaitForData) \ + FUNCTION(PassthroughData) \ FUNCTION(MessageBegin) \ FUNCTION(MessageEnd) \ FUNCTION(StructBegin) \ @@ -56,6 +57,8 @@ class ProtocolStateNameValues { } }; +class DecoderCallbacks; + /** * DecoderStateMachine is the Thrift message state machine as described in * source/extensions/filters/network/thrift_proxy/docs. @@ -63,9 +66,9 @@ class ProtocolStateNameValues { class DecoderStateMachine : public Logger::Loggable { public: DecoderStateMachine(Protocol& proto, MessageMetadataSharedPtr& metadata, - DecoderEventHandler& handler) - : proto_(proto), metadata_(metadata), handler_(handler), state_(ProtocolState::MessageBegin) { - } + DecoderEventHandler& handler, DecoderCallbacks& callbacks) + : proto_(proto), metadata_(metadata), handler_(handler), callbacks_(callbacks), + state_(ProtocolState::MessageBegin) {} /** * Consumes as much data from the configured Buffer as possible and executes the decoding state @@ -129,6 +132,7 @@ class DecoderStateMachine : public Logger::Loggable { // These functions map directly to the matching ProtocolState values. Each returns the next state // or ProtocolState::WaitForData if more data is required. + DecoderStatus passthroughData(Buffer::Instance& buffer); DecoderStatus messageBegin(Buffer::Instance& buffer); DecoderStatus messageEnd(Buffer::Instance& buffer); DecoderStatus structBegin(Buffer::Instance& buffer); @@ -165,8 +169,10 @@ class DecoderStateMachine : public Logger::Loggable { Protocol& proto_; MessageMetadataSharedPtr metadata_; DecoderEventHandler& handler_; + DecoderCallbacks& callbacks_; ProtocolState state_; std::vector stack_; + uint32_t body_bytes_{}; }; using DecoderStateMachinePtr = std::unique_ptr; @@ -179,6 +185,11 @@ class DecoderCallbacks { * @return DecoderEventHandler& a new DecoderEventHandler for a message. */ virtual DecoderEventHandler& newDecoderEventHandler() PURE; + + /** + * @return True if payload passthrough is enabled and is supported by filter chain. + */ + virtual bool passthroughEnabled() const PURE; }; /** diff --git a/source/extensions/filters/network/thrift_proxy/decoder_events.h b/source/extensions/filters/network/thrift_proxy/decoder_events.h index c69db94c0d268..b5981d6bc1dfb 100644 --- a/source/extensions/filters/network/thrift_proxy/decoder_events.h +++ b/source/extensions/filters/network/thrift_proxy/decoder_events.h @@ -35,6 +35,14 @@ class DecoderEventHandler { */ virtual FilterStatus transportEnd() PURE; + /** + * Indicates raw bytes after metadata in a Thrift transport frame was detected. + * Filters should not modify data except for the router. + * @param data data to send as passthrough + * @return FilterStatus to indicate if filter chain iteration should continue + */ + virtual FilterStatus passthroughData(Buffer::Instance& data) PURE; + /** * Indicates that the start of a Thrift protocol message was detected. * @param metadata MessageMetadataSharedPtr describing the message diff --git a/source/extensions/filters/network/thrift_proxy/filters/filter.h b/source/extensions/filters/network/thrift_proxy/filters/filter.h index c2ef1a895061a..122ff019e711d 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/filter.h +++ b/source/extensions/filters/network/thrift_proxy/filters/filter.h @@ -123,6 +123,12 @@ class DecoderFilter : public virtual DecoderEventHandler { * filter should use. Callbacks will not be invoked by the filter after onDestroy() is called. */ virtual void setDecoderFilterCallbacks(DecoderFilterCallbacks& callbacks) PURE; + + /** + * @return True if payload passthrough is supported. Called by the connection manager once after + * messageBegin. + */ + virtual bool passthroughSupported() const PURE; }; using DecoderFilterSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/network/thrift_proxy/filters/pass_through_filter.h b/source/extensions/filters/network/thrift_proxy/filters/pass_through_filter.h index 6992253a11987..dfcdebb8888c1 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/pass_through_filter.h +++ b/source/extensions/filters/network/thrift_proxy/filters/pass_through_filter.h @@ -30,6 +30,12 @@ class PassThroughDecoderFilter : public DecoderFilter { ThriftProxy::FilterStatus transportEnd() override { return ThriftProxy::FilterStatus::Continue; } + bool passthroughSupported() const override { return true; } + + ThriftProxy::FilterStatus passthroughData(Buffer::Instance&) override { + return ThriftProxy::FilterStatus::Continue; + } + ThriftProxy::FilterStatus messageBegin(ThriftProxy::MessageMetadataSharedPtr) override { return ThriftProxy::FilterStatus::Continue; } diff --git a/source/extensions/filters/network/thrift_proxy/protocol_converter.h b/source/extensions/filters/network/thrift_proxy/protocol_converter.h index 2d73f4c9498bd..16a2d41115416 100644 --- a/source/extensions/filters/network/thrift_proxy/protocol_converter.h +++ b/source/extensions/filters/network/thrift_proxy/protocol_converter.h @@ -25,6 +25,11 @@ class ProtocolConverter : public virtual DecoderEventHandler { } // DecoderEventHandler + FilterStatus passthroughData(Buffer::Instance& data) override { + buffer_->move(data); + return FilterStatus::Continue; + } + FilterStatus messageBegin(MessageMetadataSharedPtr metadata) override { proto_->writeMessageBegin(*buffer_, *metadata); return FilterStatus::Continue; diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc index aa10f24d5b71c..c0529c69109ec 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc @@ -262,6 +262,12 @@ FilterStatus Router::messageBegin(MessageMetadataSharedPtr metadata) { : callbacks_->downstreamProtocolType(); ASSERT(protocol != ProtocolType::Auto); + if (callbacks_->downstreamTransportType() == TransportType::Framed && + transport == TransportType::Framed && callbacks_->downstreamProtocolType() == protocol && + protocol != ProtocolType::Twitter) { + passthrough_supported_ = true; + } + Tcp::ConnectionPool::Instance* conn_pool = cluster_manager_.tcpConnPoolForCluster( cluster_name, Upstream::ResourcePriority::Default, this); if (!conn_pool) { diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.h b/source/extensions/filters/network/thrift_proxy/router/router_impl.h index 26a94c90c7534..a2328d4777f55 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.h @@ -178,13 +178,15 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, public: Router(Upstream::ClusterManager& cluster_manager, const std::string& stat_prefix, Stats::Scope& scope) - : cluster_manager_(cluster_manager), stats_(generateStats(stat_prefix, scope)) {} + : cluster_manager_(cluster_manager), stats_(generateStats(stat_prefix, scope)), + passthrough_supported_(false) {} ~Router() override = default; // ThriftFilters::DecoderFilter void onDestroy() override; void setDecoderFilterCallbacks(ThriftFilters::DecoderFilterCallbacks& callbacks) override; + bool passthroughSupported() const override { return passthrough_supported_; } // ProtocolConverter FilterStatus transportBegin(MessageMetadataSharedPtr metadata) override; @@ -265,6 +267,8 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, std::unique_ptr upstream_request_; Buffer::OwnedImpl upstream_request_buffer_; + + bool passthrough_supported_ : 1; }; } // namespace Router diff --git a/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h b/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h index d6961ce873577..700e6b2238393 100644 --- a/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h +++ b/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h @@ -21,6 +21,7 @@ class ThriftBase : public DecoderEventHandler { ~ThriftBase() override = default; // DecoderEventHandler + FilterStatus passthroughData(Buffer::Instance&) override { return FilterStatus::Continue; } FilterStatus transportBegin(MessageMetadataSharedPtr) override { return FilterStatus::Continue; } FilterStatus transportEnd() override { return FilterStatus::Continue; } FilterStatus messageBegin(MessageMetadataSharedPtr) override { return FilterStatus::Continue; } @@ -246,6 +247,7 @@ class ThriftObjectImpl : public ThriftObject, complete_ = true; return FilterStatus::Continue; } + bool passthroughEnabled() const override { return false; } // ThriftObject bool onData(Buffer::Instance& buffer) override; diff --git a/test/config/utility.cc b/test/config/utility.cc index 54a8247dfb185..dc86c425e960b 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -1047,25 +1047,17 @@ void ConfigHelper::addListenerFilter(const std::string& filter_yaml) { bool ConfigHelper::loadHttpConnectionManager( envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { - RELEASE_ASSERT(!finalized_, ""); - auto* hcm_filter = getFilterFromListener("http"); - if (hcm_filter) { - auto* config = hcm_filter->mutable_typed_config(); - hcm = MessageUtil::anyConvert< - envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager>( - *config); - return true; - } - return false; + return loadFilter< + envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager>( + "http", hcm); } void ConfigHelper::storeHttpConnectionManager( const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { - RELEASE_ASSERT(!finalized_, ""); - auto* hcm_config_any = getFilterFromListener("http")->mutable_typed_config(); - - hcm_config_any->PackFrom(hcm); + return storeFilter< + envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager>( + "http", hcm); } void ConfigHelper::addConfigModifier(ConfigModifierFunction function) { diff --git a/test/config/utility.h b/test/config/utility.h index 0f7fc73b433e2..a0f19e90a445d 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -19,6 +19,7 @@ #include "common/config/api_version.h" #include "common/network/address_impl.h" #include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" #include "test/integration/server_stats.h" @@ -219,6 +220,19 @@ class ConfigHelper { // Modifiers will be applied just before ports are modified in finalize void addConfigModifier(HttpModifierFunction function); + // Allows callers to easily modify the filter named 'name' from the first filter chain from the + // first listener. Modifiers will be applied just before ports are modified in finalize + template + void addFilterConfigModifier(const std::string& name, + std::function function) { + addConfigModifier([name, function, this](envoy::config::bootstrap::v3::Bootstrap&) -> void { + FilterType filter_config; + loadFilter(name, filter_config); + function(filter_config); + storeFilter(name, filter_config); + }); + } + // Apply any outstanding config modifiers, stick all the listeners in a discovery response message // and write it to the lds file. void setLds(absl::string_view version_info); @@ -275,6 +289,26 @@ class ConfigHelper { // struct of the first listener. void storeHttpConnectionManager(const HttpConnectionManager& hcm); + // Load the first FilterType struct from the first listener into a parsed proto. + template bool loadFilter(const std::string& name, FilterType& filter) { + RELEASE_ASSERT(!finalized_, ""); + auto* filter_config = getFilterFromListener(name); + if (filter_config) { + auto* config = filter_config->mutable_typed_config(); + filter = MessageUtil::anyConvert(*config); + return true; + } + return false; + } + // Take the contents of the provided FilterType proto and stuff them into the first FilterType + // struct of the first listener. + template void storeFilter(const std::string& name, const FilterType& filter) { + RELEASE_ASSERT(!finalized_, ""); + auto* filter_config_any = getFilterFromListener(name)->mutable_typed_config(); + + filter_config_any->PackFrom(filter); + } + // Finds the filter named 'name' from the first filter chain from the first listener. envoy::config::listener::v3::Filter* getFilterFromListener(const std::string& name); diff --git a/test/extensions/filters/network/thrift_proxy/BUILD b/test/extensions/filters/network/thrift_proxy/BUILD index 24eb2f1937035..b4bb10f1d2d2d 100644 --- a/test/extensions/filters/network/thrift_proxy/BUILD +++ b/test/extensions/filters/network/thrift_proxy/BUILD @@ -267,6 +267,7 @@ envoy_extension_cc_test( ":mocks", ":utility_lib", "//source/extensions/filters/network/thrift_proxy:app_exception_lib", + "//source/extensions/filters/network/thrift_proxy:config", "//source/extensions/filters/network/thrift_proxy/router:config", "//source/extensions/filters/network/thrift_proxy/router:router_lib", "//test/mocks/network:network_mocks", @@ -333,6 +334,7 @@ envoy_extension_cc_test( "//test/extensions/filters/network/thrift_proxy/driver:generate_fixture", ], extension_name = "envoy.filters.network.thrift_proxy", + shard_count = 4, deps = [ ":integration_lib", ":utility_lib", diff --git a/test/extensions/filters/network/thrift_proxy/config_test.cc b/test/extensions/filters/network/thrift_proxy/config_test.cc index 6bf4afbd3f7af..9b3db4c4ebc8d 100644 --- a/test/extensions/filters/network/thrift_proxy/config_test.cc +++ b/test/extensions/filters/network/thrift_proxy/config_test.cc @@ -206,6 +206,24 @@ stat_prefix: ingress EXPECT_EQ("thrift.ingress.", factory.config_stat_prefix_); } +// Test config with payload passthrough enabled. +TEST_F(ThriftFilterConfigTest, ThriftProxyPayloadPassthrough) { + const std::string yaml = R"EOF( +stat_prefix: ingress +payload_passthrough: true +route_config: + name: local_route +thrift_filters: + - name: envoy.filters.thrift.router +)EOF"; + + envoy::extensions::filters::network::thrift_proxy::v3::ThriftProxy config = + parseThriftProxyFromV2Yaml(yaml); + testConfig(config); + + EXPECT_EQ(true, config.payload_passthrough()); +} + } // namespace ThriftProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index 9798db6e2e9c5..f58948b262fc1 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -348,7 +348,7 @@ TEST_F(ThriftConnectionManagerTest, OnDataHandlesThriftOneWay) { initializeFilter(); writeFramedBinaryMessage(buffer_, MessageType::Oneway, 0x0F); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -386,7 +386,7 @@ TEST_F(ThriftConnectionManagerTest, OnDataHandlesStopIterationAndResume) { EXPECT_EQ(&filter_callbacks_.connection_, callbacks->connection()); // Resume processing. - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); callbacks->continueDecoding(); EXPECT_EQ(1U, store_.counter("test.request").value()); @@ -461,7 +461,7 @@ TEST_F(ThriftConnectionManagerTest, OnDataHandlesProtocolError) { EXPECT_EQ(write_buffer_.toString(), buffer.toString()); })); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request_decoding_error").value()); @@ -561,7 +561,7 @@ TEST_F(ThriftConnectionManagerTest, OnEvent) { }); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); filter_->onEvent(Network::ConnectionEvent::RemoteClose); EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); @@ -580,7 +580,7 @@ TEST_F(ThriftConnectionManagerTest, OnEvent) { }); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); filter_->onEvent(Network::ConnectionEvent::LocalClose); EXPECT_EQ(1U, store_.counter("test.cx_destroy_local_with_active_rq").value()); @@ -596,7 +596,7 @@ TEST_F(ThriftConnectionManagerTest, OnEvent) { writeFramedBinaryMessage(buffer_, MessageType::Call, 0x0F); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); filter_->onEvent(Network::ConnectionEvent::RemoteClose); EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); @@ -612,7 +612,7 @@ TEST_F(ThriftConnectionManagerTest, OnEvent) { writeFramedBinaryMessage(buffer_, MessageType::Call, 0x0F); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); filter_->onEvent(Network::ConnectionEvent::LocalClose); EXPECT_EQ(1U, store_.counter("test.cx_destroy_local_with_active_rq").value()); @@ -655,7 +655,7 @@ stat_prefix: test EXPECT_NE(nullptr, route->routeEntry()); EXPECT_EQ("cluster", route->routeEntry()->clusterName()); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); callbacks->continueDecoding(); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -679,7 +679,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndResponse) { BinaryProtocolImpl proto; callbacks->startUpstreamResponse(transport, proto); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -713,7 +713,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndVoidResponse) { BinaryProtocolImpl proto; callbacks->startUpstreamResponse(transport, proto); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -755,7 +755,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndResponseSequenceIdHandling) { .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> void { EXPECT_EQ(response_buffer.toString(), buffer.toString()); })); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -789,7 +789,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndExceptionResponse) { BinaryProtocolImpl proto; callbacks->startUpstreamResponse(transport, proto); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -824,7 +824,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndErrorResponse) { BinaryProtocolImpl proto; callbacks->startUpstreamResponse(transport, proto); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -859,7 +859,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndInvalidResponse) { BinaryProtocolImpl proto; callbacks->startUpstreamResponse(transport, proto); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -901,7 +901,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndResponseProtocolError) { callbacks->startUpstreamResponse(transport, proto); EXPECT_CALL(filter_callbacks_.connection_, write(_, true)); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Reset, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -943,7 +943,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndTransportApplicationException) { BinaryProtocolImpl proto; callbacks->startUpstreamResponse(transport, proto); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Reset, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -979,7 +979,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndGarbageResponse) { BinaryProtocolImpl proto; callbacks->startUpstreamResponse(transport, proto); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(ThriftFilters::ResponseStatus::Reset, callbacks->upstreamData(write_buffer_)); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -1249,7 +1249,7 @@ TEST_F(ThriftConnectionManagerTest, OnDataWithFilterSendsLocalReply) { EXPECT_EQ(8, buffer.drainBEInt()); EXPECT_EQ("response", buffer.toString()); })); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); @@ -1293,7 +1293,7 @@ TEST_F(ThriftConnectionManagerTest, OnDataWithFilterSendsLocalErrorReply) { EXPECT_EQ(8, buffer.drainBEInt()); EXPECT_EQ("response", buffer.toString()); })); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); @@ -1328,7 +1328,7 @@ TEST_F(ThriftConnectionManagerTest, OnDataWithFilterSendLocalReplyRemoteClosedCo return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, write(_, false)).Times(0); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); // Remote closes the connection. filter_callbacks_.connection_.state_ = Network::Connection::State::Closed; @@ -1417,6 +1417,230 @@ TEST_F(ThriftConnectionManagerTest, TransportEndWhenRemoteClose) { filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); } +// TODO(caitong93): use TEST_P to avoid duplicating test cases +TEST_F(ThriftConnectionManagerTest, PayloadPassthroughOnDataHandlesThriftCall) { + const std::string yaml = R"EOF( +transport: FRAMED +protocol: BINARY +stat_prefix: test +payload_passthrough: true +)EOF"; + + initializeFilter(yaml); + writeFramedBinaryMessage(buffer_, MessageType::Call, 0x0F); + + EXPECT_CALL(*decoder_filter_, passthroughSupported()).WillRepeatedly(Return(true)); + EXPECT_CALL(*decoder_filter_, passthroughData(_)); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(0, buffer_.length()); + + EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + EXPECT_EQ(0U, store_.counter("test.request_oneway").value()); + EXPECT_EQ(0U, store_.counter("test.request_invalid_type").value()); + EXPECT_EQ(0U, store_.counter("test.request_decoding_error").value()); + EXPECT_EQ(1U, stats_.request_active_.value()); + EXPECT_EQ(0U, store_.counter("test.response").value()); +} + +TEST_F(ThriftConnectionManagerTest, PayloadPassthroughOnDataHandlesThriftOneWay) { + const std::string yaml = R"EOF( +stat_prefix: test +payload_passthrough: true +)EOF"; + + initializeFilter(yaml); + writeFramedBinaryMessage(buffer_, MessageType::Oneway, 0x0F); + + EXPECT_CALL(*decoder_filter_, passthroughSupported()).WillRepeatedly(Return(true)); + EXPECT_CALL(*decoder_filter_, passthroughData(_)); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); + + EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(0U, store_.counter("test.request_call").value()); + EXPECT_EQ(1U, store_.counter("test.request_oneway").value()); + EXPECT_EQ(0U, store_.counter("test.request_invalid_type").value()); + EXPECT_EQ(0U, store_.counter("test.request_decoding_error").value()); + EXPECT_EQ(0U, stats_.request_active_.value()); + EXPECT_EQ(0U, store_.counter("test.response").value()); +} + +TEST_F(ThriftConnectionManagerTest, PayloadPassthroughRequestAndExceptionResponse) { + const std::string yaml = R"EOF( +stat_prefix: test +payload_passthrough: true +)EOF"; + + initializeFilter(yaml); + writeFramedBinaryMessage(buffer_, MessageType::Call, 0x0F); + + EXPECT_CALL(*decoder_filter_, passthroughSupported()).WillRepeatedly(Return(true)); + EXPECT_CALL(*decoder_filter_, passthroughData(_)); + + ThriftFilters::DecoderFilterCallbacks* callbacks{}; + EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + .WillOnce( + Invoke([&](ThriftFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + + writeFramedBinaryTApplicationException(write_buffer_, 0x0F); + + FramedTransportImpl transport; + BinaryProtocolImpl proto; + callbacks->startUpstreamResponse(transport, proto); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); + EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); + + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); + + EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + EXPECT_EQ(0U, stats_.request_active_.value()); + EXPECT_EQ(1U, store_.counter("test.response").value()); + EXPECT_EQ(0U, store_.counter("test.response_reply").value()); + EXPECT_EQ(0U, store_.counter("test.response_error").value()); + EXPECT_EQ(1U, store_.counter("test.response_exception").value()); + EXPECT_EQ(0U, store_.counter("test.response_invalid_type").value()); + EXPECT_EQ(0U, store_.counter("test.response_success").value()); + EXPECT_EQ(0U, store_.counter("test.response_error").value()); +} + +TEST_F(ThriftConnectionManagerTest, PayloadPassthroughRequestAndErrorResponse) { + const std::string yaml = R"EOF( +stat_prefix: test +payload_passthrough: true +)EOF"; + + initializeFilter(yaml); + writeFramedBinaryMessage(buffer_, MessageType::Call, 0x0F); + + EXPECT_CALL(*decoder_filter_, passthroughSupported()).WillRepeatedly(Return(true)); + EXPECT_CALL(*decoder_filter_, passthroughData(_)); + + ThriftFilters::DecoderFilterCallbacks* callbacks{}; + EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + .WillOnce( + Invoke([&](ThriftFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + + writeFramedBinaryIDLException(write_buffer_, 0x0F); + + FramedTransportImpl transport; + BinaryProtocolImpl proto; + callbacks->startUpstreamResponse(transport, proto); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); + EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); + + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); + + EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + EXPECT_EQ(0U, stats_.request_active_.value()); + EXPECT_EQ(1U, store_.counter("test.response").value()); + EXPECT_EQ(1U, store_.counter("test.response_reply").value()); + EXPECT_EQ(0U, store_.counter("test.response_exception").value()); + EXPECT_EQ(0U, store_.counter("test.response_invalid_type").value()); + // In payload_passthrough mode, Envoy cannot detect response error. + EXPECT_EQ(1U, store_.counter("test.response_success").value()); + EXPECT_EQ(0U, store_.counter("test.response_error").value()); +} + +TEST_F(ThriftConnectionManagerTest, PayloadPassthroughRequestAndInvalidResponse) { + const std::string yaml = R"EOF( +stat_prefix: test +payload_passthrough: true +)EOF"; + + initializeFilter(yaml); + writeFramedBinaryMessage(buffer_, MessageType::Call, 0x0F); + + EXPECT_CALL(*decoder_filter_, passthroughSupported()).WillRepeatedly(Return(true)); + EXPECT_CALL(*decoder_filter_, passthroughData(_)); + + ThriftFilters::DecoderFilterCallbacks* callbacks{}; + EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + .WillOnce( + Invoke([&](ThriftFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + + // Call is not valid in a response + writeFramedBinaryMessage(write_buffer_, MessageType::Call, 0x0F); + + FramedTransportImpl transport; + BinaryProtocolImpl proto; + callbacks->startUpstreamResponse(transport, proto); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); + EXPECT_EQ(ThriftFilters::ResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); + + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); + + EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + EXPECT_EQ(0U, stats_.request_active_.value()); + EXPECT_EQ(1U, store_.counter("test.response").value()); + EXPECT_EQ(0U, store_.counter("test.response_reply").value()); + EXPECT_EQ(0U, store_.counter("test.response_exception").value()); + EXPECT_EQ(1U, store_.counter("test.response_invalid_type").value()); + EXPECT_EQ(0U, store_.counter("test.response_success").value()); + EXPECT_EQ(0U, store_.counter("test.response_error").value()); +} + +TEST_F(ThriftConnectionManagerTest, PayloadPassthroughRouting) { + const std::string yaml = R"EOF( +transport: FRAMED +protocol: BINARY +payload_passthrough: true +stat_prefix: test +route_config: + name: "routes" + routes: + - match: + method_name: name + route: + cluster: cluster +)EOF"; + + initializeFilter(yaml); + writeFramedBinaryMessage(buffer_, MessageType::Oneway, 0x0F); + + EXPECT_CALL(*decoder_filter_, passthroughSupported()).WillRepeatedly(Return(true)); + EXPECT_CALL(*decoder_filter_, passthroughData(_)); + + ThriftFilters::DecoderFilterCallbacks* callbacks{}; + EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + .WillOnce( + Invoke([&](ThriftFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); + EXPECT_CALL(*decoder_filter_, messageBegin(_)).WillOnce(Return(FilterStatus::StopIteration)); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(0U, store_.counter("test.request").value()); + EXPECT_EQ(1U, stats_.request_active_.value()); + + Router::RouteConstSharedPtr route = callbacks->route(); + EXPECT_NE(nullptr, route); + EXPECT_NE(nullptr, route->routeEntry()); + EXPECT_EQ("cluster", route->routeEntry()->clusterName()); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); + callbacks->continueDecoding(); + + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); +} + } // namespace ThriftProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/test/extensions/filters/network/thrift_proxy/decoder_test.cc b/test/extensions/filters/network/thrift_proxy/decoder_test.cc index 4699f3d94a911..73c8ac0b85dc7 100644 --- a/test/extensions/filters/network/thrift_proxy/decoder_test.cc +++ b/test/extensions/filters/network/thrift_proxy/decoder_test.cc @@ -185,6 +185,7 @@ class DecoderStateMachineTestBase { NiceMock proto_; MessageMetadataSharedPtr metadata_; NiceMock handler_; + NiceMock callbacks_; }; class DecoderStateMachineNonValueTest : public DecoderStateMachineTestBase, @@ -236,7 +237,7 @@ TEST_P(DecoderStateMachineNonValueTest, NoData) { ProtocolState state = GetParam(); Buffer::OwnedImpl buffer; - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(state); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); EXPECT_EQ(dsm.currentState(), state); @@ -256,7 +257,7 @@ TEST_P(DecoderStateMachineValueTest, NoFieldValueData) { EXPECT_CALL(proto_, readFieldEnd(Ref(buffer))).WillOnce(Return(true)); EXPECT_CALL(proto_, readFieldBegin(Ref(buffer), _, _, _)).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::FieldBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -280,7 +281,7 @@ TEST_P(DecoderStateMachineValueTest, FieldValue) { EXPECT_CALL(proto_, readFieldEnd(Ref(buffer))).WillOnce(Return(true)); EXPECT_CALL(proto_, readFieldBegin(Ref(buffer), _, _, _)).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::FieldBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -295,7 +296,7 @@ TEST_F(DecoderStateMachineTest, NoListValueData) { .WillOnce(DoAll(SetArgReferee<1>(FieldType::I32), SetArgReferee<2>(1), Return(true))); EXPECT_CALL(proto_, readInt32(Ref(buffer), _)).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::ListBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -310,7 +311,7 @@ TEST_F(DecoderStateMachineTest, EmptyList) { .WillOnce(DoAll(SetArgReferee<1>(FieldType::I32), SetArgReferee<2>(0), Return(true))); EXPECT_CALL(proto_, readListEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::ListBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -329,7 +330,7 @@ TEST_P(DecoderStateMachineValueTest, ListValue) { EXPECT_CALL(proto_, readListEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::ListBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -346,7 +347,7 @@ TEST_P(DecoderStateMachineValueTest, IncompleteListValue) { expectValue(proto_, handler_, field_type, false); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::ListBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -372,7 +373,7 @@ TEST_P(DecoderStateMachineValueTest, MultipleListValues) { EXPECT_CALL(proto_, readListEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::ListBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -388,7 +389,7 @@ TEST_F(DecoderStateMachineTest, NoMapKeyData) { SetArgReferee<3>(1), Return(true))); EXPECT_CALL(proto_, readInt32(Ref(buffer), _)).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -405,7 +406,7 @@ TEST_F(DecoderStateMachineTest, NoMapValueData) { EXPECT_CALL(proto_, readInt32(Ref(buffer), _)).WillOnce(Return(true)); EXPECT_CALL(proto_, readString(Ref(buffer), _)).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -421,7 +422,7 @@ TEST_F(DecoderStateMachineTest, EmptyMap) { SetArgReferee<3>(0), Return(true))); EXPECT_CALL(proto_, readMapEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -442,7 +443,7 @@ TEST_P(DecoderStateMachineValueTest, MapKeyValue) { EXPECT_CALL(proto_, readMapEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -463,7 +464,7 @@ TEST_P(DecoderStateMachineValueTest, MapValueValue) { EXPECT_CALL(proto_, readMapEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -481,7 +482,7 @@ TEST_P(DecoderStateMachineValueTest, IncompleteMapKey) { expectValue(proto_, handler_, field_type, false); // key - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -506,7 +507,7 @@ TEST_P(DecoderStateMachineValueTest, IncompleteMapValue) { expectValue(proto_, handler_, FieldType::I32); // key expectValue(proto_, handler_, field_type, false); // value - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -534,7 +535,7 @@ TEST_P(DecoderStateMachineValueTest, MultipleMapKeyValues) { EXPECT_CALL(proto_, readMapEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::MapBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -549,7 +550,7 @@ TEST_F(DecoderStateMachineTest, NoSetValueData) { .WillOnce(DoAll(SetArgReferee<1>(FieldType::I32), SetArgReferee<2>(1), Return(true))); EXPECT_CALL(proto_, readInt32(Ref(buffer), _)).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::SetBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -564,7 +565,7 @@ TEST_F(DecoderStateMachineTest, EmptySet) { .WillOnce(DoAll(SetArgReferee<1>(FieldType::I32), SetArgReferee<2>(0), Return(true))); EXPECT_CALL(proto_, readSetEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::SetBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -583,7 +584,7 @@ TEST_P(DecoderStateMachineValueTest, SetValue) { EXPECT_CALL(proto_, readSetEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::SetBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -600,7 +601,7 @@ TEST_P(DecoderStateMachineValueTest, IncompleteSetValue) { expectValue(proto_, handler_, field_type, false); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::SetBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -626,7 +627,7 @@ TEST_P(DecoderStateMachineValueTest, MultipleSetValues) { EXPECT_CALL(proto_, readSetEnd(Ref(buffer))).WillOnce(Return(false)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); dsm.setCurrentState(ProtocolState::SetBegin); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); @@ -650,7 +651,7 @@ TEST_F(DecoderStateMachineTest, EmptyStruct) { EXPECT_CALL(proto_, readStructEnd(Ref(buffer))).WillOnce(Return(true)); EXPECT_CALL(proto_, readMessageEnd(Ref(buffer))).WillOnce(Return(true)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); EXPECT_EQ(dsm.run(buffer), ProtocolState::Done); EXPECT_EQ(dsm.currentState(), ProtocolState::Done); @@ -706,7 +707,7 @@ TEST_P(DecoderStateMachineValueTest, SingleFieldStruct) { EXPECT_CALL(proto_, readMessageEnd(Ref(buffer))).WillOnce(Return(true)); EXPECT_CALL(handler_, messageEnd()).WillOnce(Return(FilterStatus::Continue)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); EXPECT_EQ(dsm.run(buffer), ProtocolState::Done); EXPECT_EQ(dsm.currentState(), ProtocolState::Done); @@ -767,7 +768,7 @@ TEST_F(DecoderStateMachineTest, MultiFieldStruct) { EXPECT_CALL(proto_, readMessageEnd(Ref(buffer))).WillOnce(Return(true)); EXPECT_CALL(handler_, messageEnd()).WillOnce(Return(FilterStatus::Continue)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); EXPECT_EQ(dsm.run(buffer), ProtocolState::Done); EXPECT_EQ(dsm.currentState(), ProtocolState::Done); @@ -823,7 +824,7 @@ TEST_P(DecoderStateMachineNestingTest, NestedTypes) { EXPECT_CALL(proto_, readMessageEnd(Ref(buffer))).WillOnce(Return(true)); EXPECT_CALL(handler_, messageEnd()).WillOnce(Return(FilterStatus::Continue)); - DecoderStateMachine dsm(proto_, metadata_, handler_); + DecoderStateMachine dsm(proto_, metadata_, handler_, callbacks_); EXPECT_EQ(dsm.run(buffer), ProtocolState::Done); EXPECT_EQ(dsm.currentState(), ProtocolState::Done); @@ -869,6 +870,7 @@ TEST(DecoderTest, OnData) { EXPECT_EQ(100U, metadata->sequenceId()); return FilterStatus::Continue; })); + EXPECT_CALL(callbacks, passthroughEnabled()).Times(1).WillRepeatedly(Return(false)); EXPECT_CALL(proto, readStructBegin(Ref(buffer), _)).WillOnce(Return(true)); EXPECT_CALL(handler, structBegin(absl::string_view())).WillOnce(Return(FilterStatus::Continue)); @@ -936,6 +938,7 @@ TEST(DecoderTest, OnDataWithProtocolHint) { EXPECT_EQ(100U, metadata->sequenceId()); return FilterStatus::Continue; })); + EXPECT_CALL(callbacks, passthroughEnabled()).Times(1); EXPECT_CALL(proto, readStructBegin(Ref(buffer), _)).WillOnce(Return(true)); EXPECT_CALL(handler, structBegin(absl::string_view())).WillOnce(Return(FilterStatus::Continue)); @@ -1178,6 +1181,7 @@ TEST(DecoderTest, OnDataHandlesStopIterationAndResumes) { EXPECT_EQ(100U, metadata->sequenceId()); return FilterStatus::StopIteration; })); + EXPECT_CALL(callbacks, passthroughEnabled()).Times(1).WillRepeatedly(Return(false)); EXPECT_EQ(FilterStatus::StopIteration, decoder.onData(buffer, underflow)); EXPECT_FALSE(underflow); @@ -1231,6 +1235,284 @@ TEST(DecoderTest, OnDataHandlesStopIterationAndResumes) { EXPECT_TRUE(underflow); } +TEST(DecoderTest, OnDataPassthrough) { + NiceMock transport; + NiceMock proto; + NiceMock callbacks; + StrictMock handler; + ON_CALL(callbacks, newDecoderEventHandler()).WillByDefault(ReturnRef(handler)); + + InSequence dummy; + Decoder decoder(transport, proto, callbacks); + Buffer::OwnedImpl buffer(std::string(100, 'a')); + + EXPECT_CALL(transport, decodeFrameStart(Ref(buffer), _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setFrameSize(100); + return true; + })); + EXPECT_CALL(handler, transportBegin(_)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata) -> FilterStatus { + EXPECT_TRUE(metadata->hasFrameSize()); + EXPECT_EQ(100U, metadata->frameSize()); + return FilterStatus::Continue; + })); + + EXPECT_CALL(proto, readMessageBegin(Ref(buffer), _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setMethodName("name"); + metadata.setMessageType(MessageType::Call); + metadata.setSequenceId(100); + buffer.drain(20); + return true; + })); + EXPECT_CALL(handler, messageBegin(_)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata) -> FilterStatus { + EXPECT_TRUE(metadata->hasMethodName()); + EXPECT_TRUE(metadata->hasMessageType()); + EXPECT_TRUE(metadata->hasSequenceId()); + EXPECT_EQ("name", metadata->methodName()); + EXPECT_EQ(MessageType::Call, metadata->messageType()); + EXPECT_EQ(100U, metadata->sequenceId()); + return FilterStatus::Continue; + })); + + EXPECT_CALL(callbacks, passthroughEnabled()).WillOnce(Return(true)); + EXPECT_CALL(handler, passthroughData(_)) + .WillOnce(Invoke([&](Buffer::Instance& data) -> FilterStatus { + EXPECT_EQ(80, data.length()); + return FilterStatus::Continue; + })); + + EXPECT_CALL(proto, readMessageEnd(Ref(buffer))).WillOnce(Return(true)); + EXPECT_CALL(handler, messageEnd()).WillOnce(Return(FilterStatus::Continue)); + + EXPECT_CALL(transport, decodeFrameEnd(Ref(buffer))).WillOnce(Return(true)); + EXPECT_CALL(handler, transportEnd()).WillOnce(Return(FilterStatus::Continue)); + + bool underflow = false; + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_TRUE(underflow); +} + +TEST(DecoderTest, OnDataPassthroughResumes) { + NiceMock transport; + NiceMock proto; + NiceMock callbacks; + NiceMock handler; + ON_CALL(callbacks, newDecoderEventHandler()).WillByDefault(ReturnRef(handler)); + + InSequence dummy; + + Decoder decoder(transport, proto, callbacks); + Buffer::OwnedImpl buffer; + buffer.add("x"); + + EXPECT_CALL(transport, decodeFrameStart(Ref(buffer), _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setFrameSize(100); + return true; + })); + EXPECT_CALL(proto, readMessageBegin(_, _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setMethodName("name"); + metadata.setMessageType(MessageType::Call); + metadata.setSequenceId(100); + return true; + })); + EXPECT_CALL(callbacks, passthroughEnabled()).WillOnce(Return(true)); + EXPECT_CALL(handler, passthroughData(_)).Times(0); + + bool underflow = false; + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_TRUE(underflow); + + buffer.add(std::string(100, 'a')); + EXPECT_CALL(handler, passthroughData(_)) + .WillOnce(Invoke([&](Buffer::Instance& data) -> FilterStatus { + EXPECT_EQ(100, data.length()); + return FilterStatus::Continue; + })); + EXPECT_CALL(proto, readMessageEnd(_)).WillOnce(Return(true)); + EXPECT_CALL(transport, decodeFrameEnd(_)).WillOnce(Return(true)); + + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_FALSE(underflow); // buffer.length() == 1 +} + +TEST(DecoderTest, OnDataPassthroughResumesTransportFrameStart) { + StrictMock transport; + StrictMock proto; + NiceMock callbacks; + NiceMock handler; + ON_CALL(callbacks, newDecoderEventHandler()).WillByDefault(ReturnRef(handler)); + + EXPECT_CALL(transport, name()).Times(AnyNumber()); + EXPECT_CALL(proto, name()).Times(AnyNumber()); + + InSequence dummy; + + Decoder decoder(transport, proto, callbacks); + Buffer::OwnedImpl buffer; + buffer.add(std::string(100, 'a')); + bool underflow = false; + + EXPECT_CALL(transport, decodeFrameStart(Ref(buffer), _)).WillOnce(Return(false)); + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_TRUE(underflow); + + EXPECT_CALL(transport, decodeFrameStart(Ref(buffer), _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setFrameSize(100); + return true; + })); + EXPECT_CALL(proto, readMessageBegin(_, _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setMethodName("name"); + metadata.setMessageType(MessageType::Call); + metadata.setSequenceId(100); + return true; + })); + EXPECT_CALL(callbacks, passthroughEnabled()).WillOnce(Return(true)); + EXPECT_CALL(handler, passthroughData(_)) + .WillOnce(Invoke([&](Buffer::Instance& data) -> FilterStatus { + EXPECT_EQ(100, data.length()); + return FilterStatus::Continue; + })); + + EXPECT_CALL(proto, readMessageEnd(_)).WillOnce(Return(true)); + EXPECT_CALL(transport, decodeFrameEnd(_)).WillOnce(Return(true)); + + underflow = false; + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_TRUE(underflow); // buffer.length() == 0 +} + +TEST(DecoderTest, OnDataPassthroughResumesTransportFrameEnd) { + StrictMock transport; + StrictMock proto; + NiceMock callbacks; + NiceMock handler; + ON_CALL(callbacks, newDecoderEventHandler()).WillByDefault(ReturnRef(handler)); + + EXPECT_CALL(transport, name()).Times(AnyNumber()); + EXPECT_CALL(proto, name()).Times(AnyNumber()); + + InSequence dummy; + + Decoder decoder(transport, proto, callbacks); + Buffer::OwnedImpl buffer; + buffer.add(std::string(100, 'a')); + + EXPECT_CALL(transport, decodeFrameStart(Ref(buffer), _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setFrameSize(100); + return true; + })); + EXPECT_CALL(proto, readMessageBegin(_, _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setMethodName("name"); + metadata.setMessageType(MessageType::Call); + metadata.setSequenceId(100); + return true; + })); + EXPECT_CALL(callbacks, passthroughEnabled()).WillOnce(Return(true)); + EXPECT_CALL(handler, passthroughData(_)) + .WillOnce(Invoke([&](Buffer::Instance& data) -> FilterStatus { + EXPECT_EQ(100, data.length()); + return FilterStatus::Continue; + })); + + EXPECT_CALL(proto, readMessageEnd(_)).WillOnce(Return(true)); + EXPECT_CALL(transport, decodeFrameEnd(_)).WillOnce(Return(false)); + + bool underflow = false; + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_TRUE(underflow); + + EXPECT_CALL(transport, decodeFrameEnd(_)).WillOnce(Return(true)); + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_TRUE(underflow); // buffer.length() == 0 +} + +TEST(DecoderTest, OnDataPassthroughHandlesStopIterationAndResumes) { + StrictMock transport; + EXPECT_CALL(transport, name()).WillRepeatedly(ReturnRef(transport.name_)); + + StrictMock proto; + EXPECT_CALL(proto, name()).WillRepeatedly(ReturnRef(proto.name_)); + + NiceMock callbacks; + StrictMock handler; + ON_CALL(callbacks, newDecoderEventHandler()).WillByDefault(ReturnRef(handler)); + + InSequence dummy; + Decoder decoder(transport, proto, callbacks); + Buffer::OwnedImpl buffer; + bool underflow = true; + + EXPECT_CALL(transport, decodeFrameStart(Ref(buffer), _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setFrameSize(100); + return true; + })); + EXPECT_CALL(handler, transportBegin(_)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata) -> FilterStatus { + EXPECT_TRUE(metadata->hasFrameSize()); + EXPECT_EQ(100U, metadata->frameSize()); + + return FilterStatus::StopIteration; + })); + EXPECT_EQ(FilterStatus::StopIteration, decoder.onData(buffer, underflow)); + EXPECT_FALSE(underflow); + + EXPECT_CALL(proto, readMessageBegin(Ref(buffer), _)) + .WillOnce(Invoke([&](Buffer::Instance&, MessageMetadata& metadata) -> bool { + metadata.setMethodName("name"); + metadata.setMessageType(MessageType::Call); + metadata.setSequenceId(100); + return true; + })); + EXPECT_CALL(handler, messageBegin(_)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata) -> FilterStatus { + EXPECT_TRUE(metadata->hasMethodName()); + EXPECT_TRUE(metadata->hasMessageType()); + EXPECT_TRUE(metadata->hasSequenceId()); + EXPECT_EQ("name", metadata->methodName()); + EXPECT_EQ(MessageType::Call, metadata->messageType()); + EXPECT_EQ(100U, metadata->sequenceId()); + return FilterStatus::StopIteration; + })); + EXPECT_CALL(callbacks, passthroughEnabled()).WillOnce(Return(true)); + EXPECT_CALL(handler, passthroughData(_)).Times(0); + + EXPECT_EQ(FilterStatus::StopIteration, decoder.onData(buffer, underflow)); + EXPECT_FALSE(underflow); + + buffer.add(std::string(100, 'a')); + EXPECT_CALL(handler, passthroughData(_)) + .WillOnce(Invoke([&](Buffer::Instance& data) -> FilterStatus { + EXPECT_EQ(100, data.length()); + return FilterStatus::StopIteration; + })); + + EXPECT_EQ(FilterStatus::StopIteration, decoder.onData(buffer, underflow)); + EXPECT_FALSE(underflow); // buffer.length() == 0 + + EXPECT_CALL(proto, readMessageEnd(Ref(buffer))).WillOnce(Return(true)); + EXPECT_CALL(handler, messageEnd()).WillOnce(Return(FilterStatus::StopIteration)); + EXPECT_EQ(FilterStatus::StopIteration, decoder.onData(buffer, underflow)); + EXPECT_FALSE(underflow); + + EXPECT_CALL(transport, decodeFrameEnd(Ref(buffer))).WillOnce(Return(true)); + EXPECT_CALL(handler, transportEnd()).WillOnce(Return(FilterStatus::StopIteration)); + EXPECT_EQ(FilterStatus::StopIteration, decoder.onData(buffer, underflow)); + EXPECT_FALSE(underflow); + + EXPECT_EQ(FilterStatus::Continue, decoder.onData(buffer, underflow)); + EXPECT_TRUE(underflow); +} + #define TEST_NAME(X) EXPECT_EQ(ProtocolStateNameValues::name(ProtocolState::X), #X); TEST(ProtocolStateNameValuesTest, ValidNames) { ALL_PROTOCOL_STATES(TEST_NAME) } diff --git a/test/extensions/filters/network/thrift_proxy/integration_test.cc b/test/extensions/filters/network/thrift_proxy/integration_test.cc index c3ea956fb23fd..16dddb2b31635 100644 --- a/test/extensions/filters/network/thrift_proxy/integration_test.cc +++ b/test/extensions/filters/network/thrift_proxy/integration_test.cc @@ -19,7 +19,7 @@ namespace NetworkFilters { namespace ThriftProxy { class ThriftConnManagerIntegrationTest - : public testing::TestWithParam>, + : public testing::TestWithParam>, public BaseThriftIntegrationTest { public: static void SetUpTestSuite() { // NOLINT(readability-identifier-naming) @@ -68,7 +68,7 @@ class ThriftConnManagerIntegrationTest } void initializeCall(DriverMode mode) { - std::tie(transport_, protocol_, multiplexed_) = GetParam(); + std::tie(transport_, protocol_, multiplexed_, std::ignore) = GetParam(); absl::optional service_name; if (multiplexed_) { @@ -92,7 +92,7 @@ class ThriftConnManagerIntegrationTest } void initializeOneway() { - std::tie(transport_, protocol_, multiplexed_) = GetParam(); + std::tie(transport_, protocol_, multiplexed_, std::ignore) = GetParam(); absl::optional service_name; if (multiplexed_) { @@ -106,6 +106,21 @@ class ThriftConnManagerIntegrationTest initializeCommon(); } + void tryInitializePassthrough() { + std::tie(std::ignore, std::ignore, std::ignore, payload_passthrough_) = GetParam(); + + if (payload_passthrough_) { + config_helper_.addFilterConfigModifier< + envoy::extensions::filters::network::thrift_proxy::v3::ThriftProxy>( + "thrift", [](Protobuf::Message& filter) { + auto& conn_manager = + dynamic_cast( + filter); + conn_manager.set_payload_passthrough(true); + }); + } + } + // We allocate as many upstreams as there are clusters, with each upstream being allocated // to clusters in the order they're defined in the bootstrap config. void initializeCommon() { @@ -119,6 +134,8 @@ class ThriftConnManagerIntegrationTest } }); + tryInitializePassthrough(); + BaseThriftIntegrationTest::initialize(); } @@ -142,6 +159,7 @@ class ThriftConnManagerIntegrationTest TransportType transport_; ProtocolType protocol_; bool multiplexed_; + bool payload_passthrough_; std::string result_; @@ -150,26 +168,35 @@ class ThriftConnManagerIntegrationTest }; static std::string -paramToString(const TestParamInfo>& params) { +paramToString(const TestParamInfo>& params) { TransportType transport; ProtocolType protocol; bool multiplexed; - std::tie(transport, protocol, multiplexed) = params.param; + bool passthrough; + std::tie(transport, protocol, multiplexed, passthrough) = params.param; std::string transport_name = transportNameForTest(transport); std::string protocol_name = protocolNameForTest(protocol); + std::string result; + if (multiplexed) { - return fmt::format("{}{}Multiplexed", transport_name, protocol_name); + result = fmt::format("{}{}Multiplexed", transport_name, protocol_name); + } else { + result = fmt::format("{}{}", transport_name, protocol_name); } - return fmt::format("{}{}", transport_name, protocol_name); + if (passthrough) { + result = fmt::format("{}Passthrough", result); + } + return result; } -INSTANTIATE_TEST_SUITE_P( - TransportAndProtocol, ThriftConnManagerIntegrationTest, - Combine(Values(TransportType::Framed, TransportType::Unframed, TransportType::Header), - Values(ProtocolType::Binary, ProtocolType::Compact), Values(false, true)), - paramToString); +INSTANTIATE_TEST_SUITE_P(TransportAndProtocol, ThriftConnManagerIntegrationTest, + Combine(Values(TransportType::Framed, TransportType::Unframed, + TransportType::Header), + Values(ProtocolType::Binary, ProtocolType::Compact), + Values(false, true), Values(false, true)), + paramToString); TEST_P(ThriftConnManagerIntegrationTest, Success) { initializeCall(DriverMode::Success); @@ -222,7 +249,12 @@ TEST_P(ThriftConnManagerIntegrationTest, IDLException) { Stats::CounterSharedPtr counter = test_server_->counter("thrift.thrift_stats.request_call"); EXPECT_EQ(1U, counter->value()); counter = test_server_->counter("thrift.thrift_stats.response_error"); - EXPECT_EQ(1U, counter->value()); + if (payload_passthrough_ && transport_ == TransportType::Framed && + protocol_ != ProtocolType::Twitter) { + EXPECT_EQ(0U, counter->value()); + } else { + EXPECT_EQ(1U, counter->value()); + } } TEST_P(ThriftConnManagerIntegrationTest, Exception) { @@ -395,7 +427,7 @@ class ThriftTwitterConnManagerIntegrationTest : public ThriftConnManagerIntegrat INSTANTIATE_TEST_SUITE_P(FramedTwitter, ThriftTwitterConnManagerIntegrationTest, Combine(Values(TransportType::Framed), Values(ProtocolType::Twitter), - Values(false, true)), + Values(false, true), Values(false, true)), paramToString); // Because of the protocol upgrade requests and the difficulty of separating them, we test this diff --git a/test/extensions/filters/network/thrift_proxy/mocks.h b/test/extensions/filters/network/thrift_proxy/mocks.h index 05e4e88dbc983..b5846a878a211 100644 --- a/test/extensions/filters/network/thrift_proxy/mocks.h +++ b/test/extensions/filters/network/thrift_proxy/mocks.h @@ -132,6 +132,7 @@ class MockDecoderCallbacks : public DecoderCallbacks { // ThriftProxy::DecoderCallbacks MOCK_METHOD(DecoderEventHandler&, newDecoderEventHandler, ()); + MOCK_METHOD(bool, passthroughEnabled, (), (const)); }; class MockDecoderEventHandler : public DecoderEventHandler { @@ -140,6 +141,7 @@ class MockDecoderEventHandler : public DecoderEventHandler { ~MockDecoderEventHandler() override; // ThriftProxy::DecoderEventHandler + MOCK_METHOD(FilterStatus, passthroughData, (Buffer::Instance & data)); MOCK_METHOD(FilterStatus, transportBegin, (MessageMetadataSharedPtr metadata)); MOCK_METHOD(FilterStatus, transportEnd, ()); MOCK_METHOD(FilterStatus, messageBegin, (MessageMetadataSharedPtr metadata)); @@ -207,8 +209,10 @@ class MockDecoderFilter : public DecoderFilter { MOCK_METHOD(void, onDestroy, ()); MOCK_METHOD(void, setDecoderFilterCallbacks, (DecoderFilterCallbacks & callbacks)); MOCK_METHOD(void, resetUpstreamConnection, ()); + MOCK_METHOD(bool, passthroughSupported, (), (const)); // ThriftProxy::DecoderEventHandler + MOCK_METHOD(FilterStatus, passthroughData, (Buffer::Instance & data)); MOCK_METHOD(FilterStatus, transportBegin, (MessageMetadataSharedPtr metadata)); MOCK_METHOD(FilterStatus, transportEnd, ()); MOCK_METHOD(FilterStatus, messageBegin, (MessageMetadataSharedPtr metadata)); diff --git a/test/extensions/filters/network/thrift_proxy/router_test.cc b/test/extensions/filters/network/thrift_proxy/router_test.cc index 110c10b5d624f..e5ba97bcb8b7c 100644 --- a/test/extensions/filters/network/thrift_proxy/router_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_test.cc @@ -7,6 +7,7 @@ #include "common/buffer/buffer_impl.h" #include "extensions/filters/network/thrift_proxy/app_exception_impl.h" +#include "extensions/filters/network/thrift_proxy/config.h" #include "extensions/filters/network/thrift_proxy/router/config.h" #include "extensions/filters/network/thrift_proxy/router/router_impl.h" @@ -22,6 +23,8 @@ #include "gtest/gtest.h" using testing::_; +using testing::AtLeast; +using testing::Combine; using testing::ContainsRegex; using testing::Eq; using testing::Invoke; @@ -29,6 +32,7 @@ using testing::NiceMock; using testing::Ref; using testing::Return; using testing::ReturnRef; +using ::testing::TestParamInfo; using testing::Values; namespace Envoy { @@ -102,7 +106,9 @@ class ThriftRouterTestBase { } void startRequest(MessageType msg_type, std::string method = "method", - const bool strip_service_name = false) { + const bool strip_service_name = false, + const TransportType transport_type = TransportType::Framed, + const ProtocolType protocol_type = ProtocolType::Binary) { EXPECT_EQ(FilterStatus::Continue, router_->transportBegin(metadata_)); EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); @@ -115,8 +121,12 @@ class ThriftRouterTestBase { initializeMetadata(msg_type, method); - EXPECT_CALL(callbacks_, downstreamTransportType()).WillOnce(Return(TransportType::Framed)); - EXPECT_CALL(callbacks_, downstreamProtocolType()).WillOnce(Return(ProtocolType::Binary)); + EXPECT_CALL(callbacks_, downstreamTransportType()) + .Times(AtLeast(1)) + .WillRepeatedly(Return(transport_type)); + EXPECT_CALL(callbacks_, downstreamProtocolType()) + .Times(AtLeast(1)) + .WillRepeatedly(Return(protocol_type)); EXPECT_EQ(FilterStatus::StopIteration, router_->messageBegin(metadata_)); EXPECT_CALL(callbacks_, connection()).WillRepeatedly(Return(&connection_)); @@ -184,8 +194,12 @@ class ThriftRouterTestBase { EXPECT_EQ(nullptr, router_->metadataMatchCriteria()); EXPECT_EQ(nullptr, router_->downstreamHeaders()); - EXPECT_CALL(callbacks_, downstreamTransportType()).WillOnce(Return(TransportType::Framed)); - EXPECT_CALL(callbacks_, downstreamProtocolType()).WillOnce(Return(ProtocolType::Binary)); + EXPECT_CALL(callbacks_, downstreamTransportType()) + .Times(2) + .WillRepeatedly(Return(TransportType::Framed)); + EXPECT_CALL(callbacks_, downstreamProtocolType()) + .Times(2) + .WillRepeatedly(Return(ProtocolType::Binary)); mock_protocol_cb_ = [&](MockProtocol* protocol) -> void { ON_CALL(*protocol, type()).WillByDefault(Return(ProtocolType::Binary)); @@ -355,6 +369,37 @@ INSTANTIATE_TEST_SUITE_P(ContainerFieldTypes, ThriftRouterContainerTest, Values(FieldType::Map, FieldType::List, FieldType::Set), fieldTypeParamToString); +class ThriftRouterPassthroughTest + : public testing::TestWithParam< + std::tuple>, + public ThriftRouterTestBase { +public: +}; + +static std::string downstreamUpstreamTypesToString( + const TestParamInfo>& + params) { + TransportType downstream_transport_type; + ProtocolType downstream_protocol_type; + TransportType upstream_transport_type; + ProtocolType upstream_protocol_type; + + std::tie(downstream_transport_type, downstream_protocol_type, upstream_transport_type, + upstream_protocol_type) = params.param; + + return fmt::format("{}{}{}{}", TransportNames::get().fromType(downstream_transport_type), + ProtocolNames::get().fromType(downstream_protocol_type), + TransportNames::get().fromType(upstream_transport_type), + ProtocolNames::get().fromType(upstream_protocol_type)); +} + +INSTANTIATE_TEST_SUITE_P(DownstreamUpstreamTypes, ThriftRouterPassthroughTest, + Combine(Values(TransportType::Framed, TransportType::Unframed), + Values(ProtocolType::Binary, ProtocolType::Twitter), + Values(TransportType::Framed, TransportType::Unframed), + Values(ProtocolType::Binary, ProtocolType::Twitter)), + downstreamUpstreamTypesToString); + TEST_F(ThriftRouterTest, PoolRemoteConnectionFailure) { initializeRouter(); @@ -949,6 +994,55 @@ TEST_P(ThriftRouterContainerTest, DecoderFilterCallbacks) { destroyRouter(); } +TEST_P(ThriftRouterPassthroughTest, PassthroughEnable) { + TransportType downstream_transport_type; + ProtocolType downstream_protocol_type; + TransportType upstream_transport_type; + ProtocolType upstream_protocol_type; + + std::tie(downstream_transport_type, downstream_protocol_type, upstream_transport_type, + upstream_protocol_type) = GetParam(); + + const std::string yaml_string = R"EOF( + transport: {} + protocol: {} + )EOF"; + + envoy::extensions::filters::network::thrift_proxy::v3::ThriftProtocolOptions configuration; + TestUtility::loadFromYaml(fmt::format(yaml_string, + TransportNames::get().fromType(upstream_transport_type), + ProtocolNames::get().fromType(upstream_protocol_type)), + configuration); + + const auto protocol_option = std::make_shared(configuration); + EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + extensionProtocolOptions(_)) + .WillRepeatedly(Return(protocol_option)); + + initializeRouter(); + startRequest(MessageType::Call, "method", false, downstream_transport_type, + downstream_protocol_type); + + bool passthroughSupported = false; + if (downstream_transport_type == upstream_transport_type && + downstream_transport_type == TransportType::Framed && + downstream_protocol_type == upstream_protocol_type && + downstream_protocol_type != ProtocolType::Twitter) { + passthroughSupported = true; + } + ASSERT_EQ(passthroughSupported, router_->passthroughSupported()); + + EXPECT_CALL(callbacks_, sendLocalReply(_, _)) + .WillOnce(Invoke([&](const DirectResponse& response, bool end_stream) -> void { + auto& app_ex = dynamic_cast(response); + EXPECT_EQ(AppExceptionType::InternalError, app_ex.type_); + EXPECT_THAT(app_ex.what(), ContainsRegex(".*connection failure.*")); + EXPECT_TRUE(end_stream); + })); + context_.cluster_manager_.tcp_conn_pool_.poolFailure( + ConnectionPool::PoolFailureReason::RemoteConnectionFailure); +} + } // namespace Router } // namespace ThriftProxy } // namespace NetworkFilters diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py index 7780ff45f9282..4126d3507d847 100644 --- a/tools/dependency/release_dates.py +++ b/tools/dependency/release_dates.py @@ -105,3 +105,4 @@ def VerifyAndPrintReleaseDates(repository_locations, github_instance): except ReleaseDateError as e: print(f'An error occurred while processing {path}, please verify the correctness of the ' f'metadata: {e}') + sys.exit(1) From 57ad370ad3aa4106e475f41dfd862872ff86ff8d Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 30 Nov 2020 23:47:49 +0000 Subject: [PATCH 068/106] Chnages suggested by htuch: safeMemcpy applied to proxy_protocol_header.cc, context.cc, dns_parser.cc and bson_impl.cc . Changes to extensions/filters/network/kafka/serialization.h Signed-off-by: grial1 --- source/extensions/common/proxy_protocol/BUILD | 1 + .../common/proxy_protocol/proxy_protocol_header.cc | 12 ++++++++---- .../common/proxy_protocol/proxy_protocol_header.h | 2 ++ source/extensions/common/wasm/BUILD | 1 + source/extensions/common/wasm/context.cc | 3 ++- .../extensions/filters/network/kafka/serialization.h | 3 +-- .../filters/network/mongo_proxy/bson_impl.cc | 2 +- source/extensions/filters/udp/dns_filter/BUILD | 1 + .../extensions/filters/udp/dns_filter/dns_parser.cc | 3 +-- .../extensions/filters/udp/dns_filter/dns_parser.h | 1 + 10 files changed, 19 insertions(+), 10 deletions(-) diff --git a/source/extensions/common/proxy_protocol/BUILD b/source/extensions/common/proxy_protocol/BUILD index 7a2b9bf66d034..0b8e6b8e92383 100644 --- a/source/extensions/common/proxy_protocol/BUILD +++ b/source/extensions/common/proxy_protocol/BUILD @@ -18,6 +18,7 @@ envoy_cc_library( "//include/envoy/buffer:buffer_interface", "//include/envoy/network:address_interface", "//include/envoy/network:connection_interface", + "//source/common/common:safe_memcpy_lib", "//source/common/network:address_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index 6140283957b55..67ecbd645a278 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -71,8 +71,10 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv4Instance(src_addr, src_port).ip()->ipv4()->address(); const auto net_dst_addr = Network::Address::Ipv4Instance(dst_addr, dst_port).ip()->ipv4()->address(); - memcpy(addrs, &net_src_addr, 4); // NOLINT(safe-memcpy) - memcpy(&addrs[4], &net_dst_addr, 4); // NOLINT(safe-memcpy) + safeMemcpy(reinterpret_cast(addrs), + reinterpret_cast(&net_src_addr)); + safeMemcpy(reinterpret_cast(&addrs[4]), + reinterpret_cast(&net_dst_addr)); out.add(addrs, 8); break; } @@ -94,8 +96,10 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, uint8_t ports[4]; const auto net_src_port = htons(static_cast(src_port)); const auto net_dst_port = htons(static_cast(dst_port)); - memcpy(ports, &net_src_port, 2); // NOLINT(safe-memcpy) - memcpy(&ports[2], &net_dst_port, 2); // NOLINT(safe-memcpy) + safeMemcpy(reinterpret_cast(ports), + reinterpret_cast(&net_src_port)); + safeMemcpy(reinterpret_cast(&ports[2]), + reinterpret_cast(&net_dst_port)); out.add(ports, 4); } diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.h b/source/extensions/common/proxy_protocol/proxy_protocol_header.h index 013c842ced20a..2aeb27bef432b 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.h +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.h @@ -5,6 +5,8 @@ #include "envoy/network/address.h" #include "envoy/network/connection.h" +#include "common/common/safe_memcpy.h" + namespace Envoy { namespace Extensions { namespace Common { diff --git a/source/extensions/common/wasm/BUILD b/source/extensions/common/wasm/BUILD index 0f351b56b5bd7..b2c7d5374ff42 100644 --- a/source/extensions/common/wasm/BUILD +++ b/source/extensions/common/wasm/BUILD @@ -102,6 +102,7 @@ envoy_cc_library( "//include/envoy/server:lifecycle_notifier_interface", "//source/common/buffer:buffer_lib", "//source/common/common:enum_to_int", + "//source/common/common:safe_memcpy_lib", "//source/common/config:remote_data_fetcher_lib", "//source/common/http:message_lib", "//source/common/http:utility_lib", diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index c2182679631fb..ad066192f27ad 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -20,6 +20,7 @@ #include "common/common/empty_string.h" #include "common/common/enum_to_int.h" #include "common/common/logger.h" +#include "common/common/safe_memcpy.h" #include "common/http/header_map_impl.h" #include "common/http/message_impl.h" #include "common/http/utility.h" @@ -194,7 +195,7 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti auto buffer = std::unique_ptr(new char[s]); char* b = buffer.get(); uint32_t n = response.size(); - memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpy(reinterpret_cast(b), &n); b += sizeof(uint32_t); for (auto& e : response) { uint32_t ttl = e.ttl_.count(); diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 758e4392095d5..75d29b298dde6 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -96,8 +96,7 @@ template class IntDeserializer : public Deserializer { class Int8Deserializer : public IntDeserializer { public: int8_t get() const override { - int8_t result; - memcpy(&result, buf_, sizeof(int8_t)); // NOLINT(safe-memcpy) + int8_t result = buf_[0]; return result; } }; diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 02726c1c73c41..5549620b0a085 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -24,7 +24,7 @@ int32_t BufferHelper::peekInt32(Buffer::Instance& data) { int32_t val; void* mem = data.linearize(sizeof(int32_t)); - std::memcpy(reinterpret_cast(&val), mem, sizeof(int32_t)); // NOLINT(safe-memcpy) + safeMemcpy(&val, reinterpret_cast(mem)); return le32toh(val); } diff --git a/source/extensions/filters/udp/dns_filter/BUILD b/source/extensions/filters/udp/dns_filter/BUILD index 5684b6569ed92..0fd4b5a1bbbb2 100644 --- a/source/extensions/filters/udp/dns_filter/BUILD +++ b/source/extensions/filters/udp/dns_filter/BUILD @@ -35,6 +35,7 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:empty_string", "//source/common/common:matchers_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/config:config_provider_lib", "//source/common/config:datasource_lib", "//source/common/network:address_lib", diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 2cf85262e5c51..8749a6c4d2dd5 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -744,8 +744,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - ::memcpy(&flags, // NOLINT(safe-memcpy) - static_cast(&query_context->response_header_.flags), sizeof(uint16_t)); + safeMemcpy(&flags, &(query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.h b/source/extensions/filters/udp/dns_filter/dns_parser.h index d604f0784ba74..8da0fbe136bca 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.h +++ b/source/extensions/filters/udp/dns_filter/dns_parser.h @@ -9,6 +9,7 @@ #include "common/buffer/buffer_impl.h" #include "common/common/empty_string.h" +#include "common/common/safe_memcpy.h" #include "common/runtime/runtime_impl.h" #include "common/stats/timespan_impl.h" From 6e56f971efaabcb9eddf84a9b8f4d50352de0fb3 Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 1 Dec 2020 16:55:43 +0000 Subject: [PATCH 069/106] reinterpreting DnsHeaderFlags as uint16_t ptr Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 8749a6c4d2dd5..463364ac63af7 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,8 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - ::memcpy(static_cast(&context->header_.flags), &data, // NOLINT(safe-memcpy) - field_size); + safeMemcpy(reinterpret_cast(&context->response_header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -744,7 +743,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safeMemcpy(&flags, &(query_context->response_header_.flags)); + safeMemcpy(&flags, reinterpret_cast(&query_context->response_header_.flags); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); From 31f428a180d204efb3b654a6535b429782ad9f84 Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 1 Dec 2020 19:26:17 +0000 Subject: [PATCH 070/106] typo fixed Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 463364ac63af7..7c79ba14234bf 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -743,7 +743,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safeMemcpy(&flags, reinterpret_cast(&query_context->response_header_.flags); + safeMemcpy(&flags, reinterpret_cast(&query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); From f9d8c1f80bbe5634a6c66727b8c274c445131bed Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 1 Dec 2020 20:03:21 +0000 Subject: [PATCH 071/106] Changes suggested by lizan@ and jmarantz@ added Signed-off-by: grial1 --- source/extensions/common/proxy_protocol/BUILD | 1 - .../proxy_protocol/proxy_protocol_header.cc | 36 ++++++++----------- .../proxy_protocol/proxy_protocol_header.h | 2 -- .../filters/network/mongo_proxy/BUILD | 1 - .../filters/network/mongo_proxy/bson_impl.cc | 8 ++--- 5 files changed, 16 insertions(+), 32 deletions(-) diff --git a/source/extensions/common/proxy_protocol/BUILD b/source/extensions/common/proxy_protocol/BUILD index 0b8e6b8e92383..7a2b9bf66d034 100644 --- a/source/extensions/common/proxy_protocol/BUILD +++ b/source/extensions/common/proxy_protocol/BUILD @@ -18,7 +18,6 @@ envoy_cc_library( "//include/envoy/buffer:buffer_interface", "//include/envoy/network:address_interface", "//include/envoy/network:connection_interface", - "//source/common/common:safe_memcpy_lib", "//source/common/network:address_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index 67ecbd645a278..cacb378b01eb6 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -66,41 +66,33 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, case Network::Address::IpVersion::v4: { addr_length[1] = PROXY_PROTO_V2_ADDR_LEN_INET; out.add(addr_length, 2); - uint8_t addrs[8]; - const auto net_src_addr = + const uint32_t net_src_addr = Network::Address::Ipv4Instance(src_addr, src_port).ip()->ipv4()->address(); - const auto net_dst_addr = + const uint32_t net_dst_addr = Network::Address::Ipv4Instance(dst_addr, dst_port).ip()->ipv4()->address(); - safeMemcpy(reinterpret_cast(addrs), - reinterpret_cast(&net_src_addr)); - safeMemcpy(reinterpret_cast(&addrs[4]), - reinterpret_cast(&net_dst_addr)); - out.add(addrs, 8); + out.add(&net_src_addr, 4); + out.add(&net_dst_addr, 4); break; } case Network::Address::IpVersion::v6: { addr_length[1] = PROXY_PROTO_V2_ADDR_LEN_INET6; out.add(addr_length, 2); - uint8_t addrs[32]; - const auto net_src_addr = + const absl::uint128 net_src_addr = Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); - const auto net_dst_addr = + const absl::uint128 net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - memcpy(addrs, &net_src_addr, 16); // NOLINT(safe-memcpy) - memcpy(&addrs[16], &net_dst_addr, 16); // NOLINT(safe-memcpy) - out.add(addrs, 32); + out.add(&(net_src_addr.Uint128Low64()), 8); + out.add(&(net_src_addr.Uint128High64()), 8); + out.add(&(net_dst_addr.Uint128Low64()), 8); + out.add(&(net_dst_addr.Uint128High64()), 8); break; } } - uint8_t ports[4]; - const auto net_src_port = htons(static_cast(src_port)); - const auto net_dst_port = htons(static_cast(dst_port)); - safeMemcpy(reinterpret_cast(ports), - reinterpret_cast(&net_src_port)); - safeMemcpy(reinterpret_cast(&ports[2]), - reinterpret_cast(&net_dst_port)); - out.add(ports, 4); + const uint16_t net_src_port = htons(static_cast(src_port)); + const uint16_t net_dst_port = htons(static_cast(dst_port)); + out.add(&net_src_port, 2); + out.add(&net_dst_port, 2); } void generateV2Header(const Network::Address::Ip& source_address, diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.h b/source/extensions/common/proxy_protocol/proxy_protocol_header.h index 2aeb27bef432b..013c842ced20a 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.h +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.h @@ -5,8 +5,6 @@ #include "envoy/network/address.h" #include "envoy/network/connection.h" -#include "common/common/safe_memcpy.h" - namespace Envoy { namespace Extensions { namespace Common { diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index 17fa01d86349a..2e281e1f67896 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -35,7 +35,6 @@ envoy_cc_library( "//source/common/common:byte_order_lib", "//source/common/common:hex_lib", "//source/common/common:minimal_logger_lib", - "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", ], ) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 5549620b0a085..de0b9170b22b9 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -8,7 +8,6 @@ #include "common/common/byte_order.h" #include "common/common/fmt.h" #include "common/common/hex.h" -#include "common/common/safe_memcpy.h" #include "common/common/utility.h" namespace Envoy { @@ -23,8 +22,7 @@ int32_t BufferHelper::peekInt32(Buffer::Instance& data) { } int32_t val; - void* mem = data.linearize(sizeof(int32_t)); - safeMemcpy(&val, reinterpret_cast(mem)); + val = data.peekLEInt(); return le32toh(val); } @@ -89,9 +87,7 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { } int64_t val; - void* mem = data.linearize(sizeof(int64_t)); - safeMemcpy(&val, reinterpret_cast(mem)); - data.drain(sizeof(int64_t)); + val = data.drainLEInt(); return le64toh(val); } From b1cc18b2577b87b39ed634998b1b21777d3e3905 Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 1 Dec 2020 22:19:26 +0000 Subject: [PATCH 072/106] adding IPv6 address to the output corrected in generateV2Header Signed-off-by: grial1 --- .../common/proxy_protocol/proxy_protocol_header.cc | 12 ++++++++---- .../extensions/filters/udp/dns_filter/dns_parser.cc | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index cacb378b01eb6..14ac11e460073 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -81,10 +81,14 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const absl::uint128 net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - out.add(&(net_src_addr.Uint128Low64()), 8); - out.add(&(net_src_addr.Uint128High64()), 8); - out.add(&(net_dst_addr.Uint128Low64()), 8); - out.add(&(net_dst_addr.Uint128High64()), 8); + const uint64_t net_src_addr_lo = absl::Uint128Low64(net_src_addr); + out.add(&net_src_addr_lo, 8); + const uint64_t net_src_addr_hi = absl::Uint128High64(net_src_addr); + out.add(&net_src_addr_hi, 8); + const uint64_t net_dst_addr_lo = absl::Uint128Low64(net_dst_addr); + out.add(&net_dst_addr_lo, 8); + const uint64_t net_dst_addr_hi = absl::Uint128High64(net_dst_addr); + out.add(&net_dst_addr_hi, 8); break; } } diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 7c79ba14234bf..8556e684b9da3 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - safeMemcpy(reinterpret_cast(&context->response_header_.flags), &data); + ::memcpy(static_cast(&context->header_.flags), &data, field_size); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: From 4a8b709492231e2ff8e73bd5b1da83330bb7effc Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 1 Dec 2020 22:39:18 +0000 Subject: [PATCH 073/106] dns_parser reverted Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 8556e684b9da3..4cac20c678f32 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,7 +171,8 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - ::memcpy(static_cast(&context->header_.flags), &data, field_size); + ::memcpy(static_cast(&context->header_.flags), // NOLINT(safe-memcpy) + &data, field_size); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: From d662d87b6478b118a215ca0514ad9aa7236f988c Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 2 Dec 2020 06:27:19 +0000 Subject: [PATCH 074/106] format fixed in dns_parser.cc Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 4cac20c678f32..264fe5deef1f1 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -172,7 +172,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, break; case DnsQueryParseState::Flags: ::memcpy(static_cast(&context->header_.flags), // NOLINT(safe-memcpy) - &data, field_size); + &data, field_size); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: From 4406fa44b8043c436eca4d480e698f7dfdafbaf0 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 2 Dec 2020 13:57:45 +0000 Subject: [PATCH 075/106] missing parentheses when safeMemcpy was used in dns_parser.cc Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 264fe5deef1f1..6d980eede445c 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -744,7 +744,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safeMemcpy(&flags, reinterpret_cast(&query_context->response_header_.flags)); + safeMemcpy(&flags, reinterpret_cast(&(query_context->response_header_.flags))); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); From 3dac1e33b8181e59945155bde94caf95632ee6ec Mon Sep 17 00:00:00 2001 From: grial1 Date: Thu, 3 Dec 2020 07:42:31 +0000 Subject: [PATCH 076/106] dns_parser.cc reverted to memcpy Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/BUILD | 1 - source/extensions/filters/udp/dns_filter/dns_parser.cc | 7 ++++--- source/extensions/filters/udp/dns_filter/dns_parser.h | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/extensions/filters/udp/dns_filter/BUILD b/source/extensions/filters/udp/dns_filter/BUILD index 0fd4b5a1bbbb2..5684b6569ed92 100644 --- a/source/extensions/filters/udp/dns_filter/BUILD +++ b/source/extensions/filters/udp/dns_filter/BUILD @@ -35,7 +35,6 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:empty_string", "//source/common/common:matchers_lib", - "//source/common/common:safe_memcpy_lib", "//source/common/config:config_provider_lib", "//source/common/config:datasource_lib", "//source/common/network:address_lib", diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 6d980eede445c..2cf85262e5c51 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -171,8 +171,8 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - ::memcpy(static_cast(&context->header_.flags), // NOLINT(safe-memcpy) - &data, field_size); + ::memcpy(static_cast(&context->header_.flags), &data, // NOLINT(safe-memcpy) + field_size); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -744,7 +744,8 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safeMemcpy(&flags, reinterpret_cast(&(query_context->response_header_.flags))); + ::memcpy(&flags, // NOLINT(safe-memcpy) + static_cast(&query_context->response_header_.flags), sizeof(uint16_t)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.h b/source/extensions/filters/udp/dns_filter/dns_parser.h index 8da0fbe136bca..d604f0784ba74 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.h +++ b/source/extensions/filters/udp/dns_filter/dns_parser.h @@ -9,7 +9,6 @@ #include "common/buffer/buffer_impl.h" #include "common/common/empty_string.h" -#include "common/common/safe_memcpy.h" #include "common/runtime/runtime_impl.h" #include "common/stats/timespan_impl.h" From be23cf1120db3213b6bb4a324ecbec6726629a5d Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 5 Dec 2020 09:09:23 +0000 Subject: [PATCH 077/106] safeMemcpySrc & safeMemcpyDst refactored to match comments (drop size) Signed-off-by: grial1 --- source/common/common/hash.h | 2 +- source/common/common/safe_memcpy.h | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/source/common/common/hash.h b/source/common/common/hash.h index 6be204cb0184f..07629f74bf062 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -61,7 +61,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - safeMemcpySrc(&result, p, 8); + safeMemcpySrc(&result, p); return result; } diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 744b33df45b5d..2d3cdce759490 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -13,21 +13,15 @@ template inline void safeMemcpy(T1* dst, T2* src) { } /** - * @brief Copies src to dst based on the size of dst, which must be specified. + * @brief Copies src to dst based on the size of dst */ -#define safeMemcpySrc(dst, src, size) \ - do { \ - static_assert(sizeof(*(dst)) == size); \ - memcpy(dst, src, size); \ - } while (0) +#define safeMemcpySrc(dst, src) \ + memcpy(dst, src, sizeof(*(dst))); /** - * @brief Copies src to dst based on the size of src, which must be specified. + * @brief Copies src to dst based on the size of src */ -#define safeMemcpyDst(dst, src, size) \ - do { \ - static_assert(sizeof(*(src)) == size); \ - memcpy(dst, src, size); \ - } while (0) +#define safeMemcpyDst(dst, src) \ + memcpy(dst, src, sizeof(*(src))); } // namespace Envoy From 51c59d9f046f5d50a67d37ba9f9d5b5cf66180ef Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 5 Dec 2020 10:15:23 +0000 Subject: [PATCH 078/106] first set of changes: Do not use reinterpret_cast with safeMemcpy. Pending: refactor other memcpy Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 6 ++---- source/common/grpc/common.cc | 4 ++-- source/common/network/address_impl.cc | 2 +- source/extensions/common/wasm/context.cc | 2 +- source/extensions/transport_sockets/tls/utility.cc | 4 ++-- test/common/common/safe_memcpy_test.cc | 4 ++-- tools/code_format/check_format.py | 2 +- 7 files changed, 11 insertions(+), 13 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 2d3cdce759490..8a8977be6b84c 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -15,13 +15,11 @@ template inline void safeMemcpy(T1* dst, T2* src) { /** * @brief Copies src to dst based on the size of dst */ -#define safeMemcpySrc(dst, src) \ - memcpy(dst, src, sizeof(*(dst))); +#define safeMemcpySrc(dst, src) memcpy(dst, src, sizeof(*(dst))); /** * @brief Copies src to dst based on the size of src */ -#define safeMemcpyDst(dst, src) \ - memcpy(dst, src, sizeof(*(src))); +#define safeMemcpyDst(dst, src) memcpy(dst, src, sizeof(*(src))); } // namespace Envoy diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index eca146d095a87..c8e5099ff341e 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -139,7 +139,7 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - safeMemcpy(reinterpret_cast(current), &nsize); + safeMemcpyDst(current, &nsize); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); @@ -293,7 +293,7 @@ void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { std::array header; header[0] = 0; // flags const uint32_t nsize = htonl(buffer.length()); - safeMemcpy(reinterpret_cast(&header[1]), &nsize); + safeMemcpyDst(&header[1], &nsize); buffer.prepend(absl::string_view(&header[0], 5)); } diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index ac402ff04a2b1..c5e21e49e8bd1 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -176,7 +176,7 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - safeMemcpy(&result, reinterpret_cast(&address_.sin6_addr.s6_addr)); + safeMemcpySrc(&result, &address_.sin6_addr.s6_addr); return result; } diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index 10694454b408b..a9ef91e31b7cd 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -195,7 +195,7 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti auto buffer = std::unique_ptr(new char[s]); char* b = buffer.get(); uint32_t n = response.size(); - safeMemcpy(reinterpret_cast(b), &n); + safeMemcpyDst(b, &n); b += sizeof(uint32_t); for (auto& e : response) { uint32_t ttl = e.ttl_.count(); diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index 861dfeb4fa937..fa8f7d810a575 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -117,14 +117,14 @@ std::string Utility::generalNameAsString(const GENERAL_NAME* general_name) { sockaddr_in sin; sin.sin_port = 0; sin.sin_family = AF_INET; - safeMemcpy(&sin.sin_addr, reinterpret_cast(general_name->d.ip->data)); + safeMemcpySrc(&sin.sin_addr, general_name->d.ip->data); Network::Address::Ipv4Instance addr(&sin); san = addr.ip()->addressAsString(); } else if (general_name->d.ip->length == 16) { sockaddr_in6 sin6; sin6.sin6_port = 0; sin6.sin6_family = AF_INET6; - safeMemcpy(&sin6.sin6_addr, reinterpret_cast(general_name->d.ip->data)); + safeMemcpySrc(&sin6.sin6_addr, general_name->d.ip->data); Network::Address::Ipv6Instance addr(sin6); san = addr.ip()->addressAsString(); } diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 0c740fefd1945..5a13d92dd4276 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -17,7 +17,7 @@ namespace Envoy { TEST(SafeMemcpyTest, CopyUint8) { const uint8_t src[] = {0, 1, 1, 2, 3, 5, 8, 13}; std::array dst; - safeMemcpy(reinterpret_cast(dst.data()), &src); + safeMemcpyDst(dst.data(), &src); Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); } @@ -28,7 +28,7 @@ TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { std::array expected_header; expected_header[0] = 0; // flags const uint32_t nsize = htonl(4); - safeMemcpy(reinterpret_cast(&expected_header[1]), &nsize); + safeMemcpyDst(&expected_header[1], &nsize); std::string header_string(&expected_header[0], 5); Grpc::Common::prependGrpcFrameHeader(*buffer); EXPECT_EQ(buffer->toString(), header_string + "test"); diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 9fc982b2e107f..89fba3250ccc9 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -803,7 +803,7 @@ def checkSourceLine(self, line, file_path, reportError): not ("test/" in file_path) and \ ("memcpy(" in line) and \ not ("NOLINT(safe-memcpy)" in line): - reportError("Don't call memcpy() directly; use safeMemcpy or MemBlockBuilder instead.") + reportError("Don't call memcpy() directly; use safeMemcpy, safeMemcpySrc, safeMemcpyDst or MemBlockBuilder instead.") if self.denylistedForExceptions(file_path): # Skpping cases where 'throw' is a substring of a symbol like in "foothrowBar". From 422b0e9fe797a76d913c8b2aa6be275497b1ff4b Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 5 Dec 2020 13:24:05 +0000 Subject: [PATCH 079/106] format fixed Signed-off-by: grial1 --- tools/code_format/check_format.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 89fba3250ccc9..c37038371fa74 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -803,7 +803,9 @@ def checkSourceLine(self, line, file_path, reportError): not ("test/" in file_path) and \ ("memcpy(" in line) and \ not ("NOLINT(safe-memcpy)" in line): - reportError("Don't call memcpy() directly; use safeMemcpy, safeMemcpySrc, safeMemcpyDst or MemBlockBuilder instead.") + reportError( + "Don't call memcpy() directly; use safeMemcpy, safeMemcpySrc, safeMemcpyDst or MemBlockBuilder instead." + ) if self.denylistedForExceptions(file_path): # Skpping cases where 'throw' is a substring of a symbol like in "foothrowBar". From 23a5cf7c40e1aca39efb1199e51ef319b2cd9e06 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 5 Dec 2020 13:57:50 +0000 Subject: [PATCH 080/106] second set of changes: replace, where possible, memcpy for a safeMemcpy variant Signed-off-by: grial1 --- source/extensions/common/wasm/context.cc | 24 +++++++++---------- .../filters/network/kafka/serialization.h | 8 +++---- .../extensions/filters/udp/dns_filter/BUILD | 1 + .../filters/udp/dns_filter/dns_parser.cc | 9 ++++--- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index a9ef91e31b7cd..f5e02750324ab 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -199,7 +199,7 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti b += sizeof(uint32_t); for (auto& e : response) { uint32_t ttl = e.ttl_.count(); - memcpy(b, &ttl, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &ttl); b += sizeof(uint32_t); }; for (auto& e : response) { @@ -265,46 +265,46 @@ void Context::onStatsUpdate(Envoy::Stats::MetricSnapshot& snapshot) { auto buffer = std::unique_ptr(new char[counter_block_size + gauge_block_size]); char* b = buffer.get(); - memcpy(b, &counter_block_size, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &counter_block_size); b += sizeof(uint32_t); - memcpy(b, &counter_type, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &counter_type); b += sizeof(uint32_t); - memcpy(b, &num_counters, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &num_counters); b += sizeof(uint32_t); for (const auto& counter : snapshot.counters()) { if (counter.counter_.get().used()) { n = counter.counter_.get().name().size(); - memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &n); b += sizeof(uint32_t); memcpy(b, counter.counter_.get().name().data(), // NOLINT(safe-memcpy) counter.counter_.get().name().size()); b = align(b + counter.counter_.get().name().size()); v = counter.counter_.get().value(); - memcpy(b, &v, sizeof(uint64_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &v); b += sizeof(uint64_t); v = counter.delta_; - memcpy(b, &v, sizeof(uint64_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &v); b += sizeof(uint64_t); } } - memcpy(b, &gauge_block_size, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &gauge_block_size); b += sizeof(uint32_t); - memcpy(b, &gauge_type, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &gauge_type); b += sizeof(uint32_t); - memcpy(b, &num_gauges, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &num_gauges); b += sizeof(uint32_t); for (const auto& gauge : snapshot.gauges()) { if (gauge.get().used()) { n = gauge.get().name().size(); - memcpy(b, &n, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &n); b += sizeof(uint32_t); memcpy(b, gauge.get().name().data(), gauge.get().name().size()); // NOLINT(safe-memcpy) b = align(b + gauge.get().name().size()); v = gauge.get().value(); - memcpy(b, &v, sizeof(uint64_t)); // NOLINT(safe-memcpy) + safeMemcpyDst(b, &v); b += sizeof(uint64_t); } } diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 75d29b298dde6..1a6a80c4b0481 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -108,7 +108,7 @@ class Int16Deserializer : public IntDeserializer { public: int16_t get() const override { int16_t result; - memcpy(&result, buf_, sizeof(int16_t)); // NOLINT(safe-memcpy) + safeMemcpySrc(&result, buf_); return be16toh(result); } }; @@ -120,7 +120,7 @@ class Int32Deserializer : public IntDeserializer { public: int32_t get() const override { int32_t result; - memcpy(&result, buf_, sizeof(int32_t)); // NOLINT(safe-memcpy) + safeMemcpySrc(&result, buf_); return be32toh(result); } }; @@ -132,7 +132,7 @@ class UInt32Deserializer : public IntDeserializer { public: uint32_t get() const override { uint32_t result; - memcpy(&result, buf_, sizeof(uint32_t)); // NOLINT(safe-memcpy) + safeMemcpySrc(&result, buf_); return be32toh(result); } }; @@ -144,7 +144,7 @@ class Int64Deserializer : public IntDeserializer { public: int64_t get() const override { int64_t result; - memcpy(&result, buf_, sizeof(int64_t)); // NOLINT(safe-memcpy) + safeMemcpySrc(&result, buf_); return be64toh(result); } }; diff --git a/source/extensions/filters/udp/dns_filter/BUILD b/source/extensions/filters/udp/dns_filter/BUILD index 5684b6569ed92..0fd4b5a1bbbb2 100644 --- a/source/extensions/filters/udp/dns_filter/BUILD +++ b/source/extensions/filters/udp/dns_filter/BUILD @@ -35,6 +35,7 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:empty_string", "//source/common/common:matchers_lib", + "//source/common/common:safe_memcpy_lib", "//source/common/config:config_provider_lib", "//source/common/config:datasource_lib", "//source/common/network:address_lib", diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index 2cf85262e5c51..d9f733276a324 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -1,7 +1,7 @@ #include "extensions/filters/udp/dns_filter/dns_parser.h" +#include "common/common/safe_memcpy.h" #include "envoy/network/address.h" - #include "common/network/address_impl.h" #include "common/network/utility.h" @@ -171,8 +171,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - ::memcpy(static_cast(&context->header_.flags), &data, // NOLINT(safe-memcpy) - field_size); + safeMemcpyDst(static_cast(&context->header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -744,8 +743,8 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - ::memcpy(&flags, // NOLINT(safe-memcpy) - static_cast(&query_context->response_header_.flags), sizeof(uint16_t)); + safeMemcpySrc(&flags, + static_cast(&query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); From c789969ec462bda0b932751a080551e01c6df6f3 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 5 Dec 2020 14:13:52 +0000 Subject: [PATCH 081/106] format fixed Signed-off-by: grial1 --- source/extensions/filters/udp/dns_filter/dns_parser.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index d9f733276a324..d027deefc6de3 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -1,7 +1,8 @@ #include "extensions/filters/udp/dns_filter/dns_parser.h" -#include "common/common/safe_memcpy.h" #include "envoy/network/address.h" + +#include "common/common/safe_memcpy.h" #include "common/network/address_impl.h" #include "common/network/utility.h" @@ -743,8 +744,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safeMemcpySrc(&flags, - static_cast(&query_context->response_header_.flags)); + safeMemcpySrc(&flags, static_cast(&query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); From d79baa3b541ffdbb1099b34462938e844f4f4683 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 5 Dec 2020 20:47:32 +0000 Subject: [PATCH 082/106] address() function changed in address_impl.cc Signed-off-by: grial1 --- source/common/network/address_impl.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index c5e21e49e8bd1..21207c4ce297b 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -174,10 +174,11 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { } absl::uint128 Ipv6Instance::Ipv6Helper::address() const { - absl::uint128 result; + uint64_t result_high, result_low; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - safeMemcpySrc(&result, &address_.sin6_addr.s6_addr); - return result; + safeMemcpySrc(&result_low, &address_.sin6_addr.s6_addr); + safeMemcpySrc(&result_high, &address_.sin6_addr.s6_addr + 8); + return MakeUint128(result_high, result_low); } uint32_t Ipv6Instance::Ipv6Helper::port() const { return ntohs(address_.sin6_port); } From 3661d0af0713d8fdd01dbe92aa53980e6a85a5bb Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 5 Dec 2020 22:05:12 +0000 Subject: [PATCH 083/106] error in address_impl.cc Signed-off-by: grial1 --- source/common/network/address_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 21207c4ce297b..de83d63a68970 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -178,7 +178,7 @@ absl::uint128 Ipv6Instance::Ipv6Helper::address() const { static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); safeMemcpySrc(&result_low, &address_.sin6_addr.s6_addr); safeMemcpySrc(&result_high, &address_.sin6_addr.s6_addr + 8); - return MakeUint128(result_high, result_low); + return absl::MakeUint128(result_high, result_low); } uint32_t Ipv6Instance::Ipv6Helper::port() const { return ntohs(address_.sin6_port); } From 27e3ccfc1b25ee2d7fe03808020aefbefbc697ba Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 10:26:45 +0000 Subject: [PATCH 084/106] format fixed Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 4 ++++ test/common/common/safe_memcpy_test.cc | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 8a8977be6b84c..050506e089022 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -14,11 +14,15 @@ template inline void safeMemcpy(T1* dst, T2* src) { /** * @brief Copies src to dst based on the size of dst + * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to + * call safeMemcpyUnsafeSrc */ #define safeMemcpySrc(dst, src) memcpy(dst, src, sizeof(*(dst))); /** * @brief Copies src to dst based on the size of src + * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to + * call safeMemcpyUnsafeDst */ #define safeMemcpyDst(dst, src) memcpy(dst, src, sizeof(*(src))); diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 5a13d92dd4276..e0309322341c0 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -21,8 +21,17 @@ TEST(SafeMemcpyTest, CopyUint8) { Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); } -// Additional (integration) test - not ordinary copy -TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { +TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { + const uint8_t* src = new uint8_t[8]; + for (int i = 0; i < 8; ++i) + src[i] = i; + uint8_t dst[8]; + safeMemcpyUnsafeSrc(&dst, src); + Eq(dst == {0, 1, 2, 3, 4, 5, 6, 7}); + delete src; +} + +TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { auto buffer = std::make_unique(); buffer->add("test", 4); std::array expected_header; From 83bd5dc19652b25f5d012b5263c7a3e8b385c9f2 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 10:30:41 +0000 Subject: [PATCH 085/106] Revert "format fixed" This reverts commit 27e3ccfc1b25ee2d7fe03808020aefbefbc697ba. Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 4 ---- test/common/common/safe_memcpy_test.cc | 13 ++----------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 050506e089022..8a8977be6b84c 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -14,15 +14,11 @@ template inline void safeMemcpy(T1* dst, T2* src) { /** * @brief Copies src to dst based on the size of dst - * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to - * call safeMemcpyUnsafeSrc */ #define safeMemcpySrc(dst, src) memcpy(dst, src, sizeof(*(dst))); /** * @brief Copies src to dst based on the size of src - * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to - * call safeMemcpyUnsafeDst */ #define safeMemcpyDst(dst, src) memcpy(dst, src, sizeof(*(src))); diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index e0309322341c0..5a13d92dd4276 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -21,17 +21,8 @@ TEST(SafeMemcpyTest, CopyUint8) { Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); } -TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { - const uint8_t* src = new uint8_t[8]; - for (int i = 0; i < 8; ++i) - src[i] = i; - uint8_t dst[8]; - safeMemcpyUnsafeSrc(&dst, src); - Eq(dst == {0, 1, 2, 3, 4, 5, 6, 7}); - delete src; -} - -TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { +// Additional (integration) test - not ordinary copy +TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { auto buffer = std::make_unique(); buffer->add("test", 4); std::array expected_header; From 4263158a28127abccebd7c191bebc236dd903beb Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 10:26:45 +0000 Subject: [PATCH 086/106] format fixed Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 4 ++++ test/common/common/safe_memcpy_test.cc | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 8a8977be6b84c..050506e089022 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -14,11 +14,15 @@ template inline void safeMemcpy(T1* dst, T2* src) { /** * @brief Copies src to dst based on the size of dst + * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to + * call safeMemcpyUnsafeSrc */ #define safeMemcpySrc(dst, src) memcpy(dst, src, sizeof(*(dst))); /** * @brief Copies src to dst based on the size of src + * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to + * call safeMemcpyUnsafeDst */ #define safeMemcpyDst(dst, src) memcpy(dst, src, sizeof(*(src))); diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 5a13d92dd4276..46fa376cc7878 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -21,8 +21,22 @@ TEST(SafeMemcpyTest, CopyUint8) { Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); } +<<<<<<< HEAD // Additional (integration) test - not ordinary copy TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { +======= +TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { + const uint8_t* src = new uint8_t[8]; + for (int i = 0; i < 8; ++i) + src[i] = i; + uint8_t dst[8]; + safeMemcpyUnsafeSrc(&dst, src); + Eq(dst == {0, 1, 2, 3, 4, 5, 6, 7}); + delete src; +} + +TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { +>>>>>>> 1c7e427... format fixed auto buffer = std::make_unique(); buffer->add("test", 4); std::array expected_header; From 89bc7ed4dbed1af16a35ca979e9297d5d50f5c1f Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 10:07:09 +0000 Subject: [PATCH 087/106] names of the variants of safeMemcpy changed Signed-off-by: grial1 --- source/common/common/hash.h | 2 +- source/common/common/safe_memcpy.h | 10 +++---- source/common/grpc/common.cc | 4 +-- source/common/network/address_impl.cc | 8 +++--- source/extensions/common/wasm/context.cc | 26 +++++++++---------- .../filters/network/kafka/serialization.h | 8 +++--- .../filters/udp/dns_filter/dns_parser.cc | 4 +-- .../transport_sockets/tls/utility.cc | 4 +-- test/common/common/safe_memcpy_test.cc | 5 ---- 9 files changed, 32 insertions(+), 39 deletions(-) diff --git a/source/common/common/hash.h b/source/common/common/hash.h index 07629f74bf062..13152ef7bb628 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -61,7 +61,7 @@ class MurmurHash { private: static inline uint64_t unalignedLoad(const char* p) { uint64_t result; - safeMemcpySrc(&result, p); + safeMemcpyUnsafeSrc(&result, p); return result; } diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 050506e089022..832415ccbd62a 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -14,16 +14,14 @@ template inline void safeMemcpy(T1* dst, T2* src) { /** * @brief Copies src to dst based on the size of dst - * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to - * call safeMemcpyUnsafeSrc + * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to call safeMemcpyUnsafeSrc */ -#define safeMemcpySrc(dst, src) memcpy(dst, src, sizeof(*(dst))); +#define safeMemcpyUnsafeSrc(dst, src) memcpy(dst, src, sizeof(*(dst))); /** * @brief Copies src to dst based on the size of src - * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to - * call safeMemcpyUnsafeDst + * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to call safeMemcpyUnsafeDst */ -#define safeMemcpyDst(dst, src) memcpy(dst, src, sizeof(*(src))); +#define safeMemcpyUnsafeDst(dst, src) memcpy(dst, src, sizeof(*(src))); } // namespace Envoy diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index c8e5099ff341e..caf9f70093214 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -139,7 +139,7 @@ Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& messag uint8_t* current = reinterpret_cast(iovec.mem_); *current++ = 0; // flags const uint32_t nsize = htonl(size); - safeMemcpyDst(current, &nsize); + safeMemcpyUnsafeDst(current, &nsize); current += sizeof(uint32_t); Protobuf::io::ArrayOutputStream stream(current, size, -1); Protobuf::io::CodedOutputStream codec_stream(&stream); @@ -293,7 +293,7 @@ void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { std::array header; header[0] = 0; // flags const uint32_t nsize = htonl(buffer.length()); - safeMemcpyDst(&header[1], &nsize); + safeMemcpyUnsafeDst(&header[1], &nsize); buffer.prepend(absl::string_view(&header[0], 5)); } diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index de83d63a68970..fd371718ce497 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -174,11 +174,11 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { } absl::uint128 Ipv6Instance::Ipv6Helper::address() const { - uint64_t result_high, result_low; + absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - safeMemcpySrc(&result_low, &address_.sin6_addr.s6_addr); - safeMemcpySrc(&result_high, &address_.sin6_addr.s6_addr + 8); - return absl::MakeUint128(result_high, result_low); + memcpy(static_cast(&result), // NOLINT(safe-memcpy) + static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); + return result; } uint32_t Ipv6Instance::Ipv6Helper::port() const { return ntohs(address_.sin6_port); } diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index f5e02750324ab..d0cb396107b61 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -195,11 +195,11 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti auto buffer = std::unique_ptr(new char[s]); char* b = buffer.get(); uint32_t n = response.size(); - safeMemcpyDst(b, &n); + safeMemcpyUnsafeDst(b, &n); b += sizeof(uint32_t); for (auto& e : response) { uint32_t ttl = e.ttl_.count(); - safeMemcpyDst(b, &ttl); + safeMemcpyUnsafeDst(b, &ttl); b += sizeof(uint32_t); }; for (auto& e : response) { @@ -265,46 +265,46 @@ void Context::onStatsUpdate(Envoy::Stats::MetricSnapshot& snapshot) { auto buffer = std::unique_ptr(new char[counter_block_size + gauge_block_size]); char* b = buffer.get(); - safeMemcpyDst(b, &counter_block_size); + safeMemcpyUnsafeDst(b, &counter_block_size); b += sizeof(uint32_t); - safeMemcpyDst(b, &counter_type); + safeMemcpyUnsafeDst(b, &counter_type); b += sizeof(uint32_t); - safeMemcpyDst(b, &num_counters); + safeMemcpyUnsafeDst(b, &num_counters); b += sizeof(uint32_t); for (const auto& counter : snapshot.counters()) { if (counter.counter_.get().used()) { n = counter.counter_.get().name().size(); - safeMemcpyDst(b, &n); + safeMemcpyUnsafeDst(b, &n); b += sizeof(uint32_t); memcpy(b, counter.counter_.get().name().data(), // NOLINT(safe-memcpy) counter.counter_.get().name().size()); b = align(b + counter.counter_.get().name().size()); v = counter.counter_.get().value(); - safeMemcpyDst(b, &v); + safeMemcpyUnsafeDst(b, &v); b += sizeof(uint64_t); v = counter.delta_; - safeMemcpyDst(b, &v); + safeMemcpyUnsafeDst(b, &v); b += sizeof(uint64_t); } } - safeMemcpyDst(b, &gauge_block_size); + safeMemcpyUnsafeDst(b, &gauge_block_size); b += sizeof(uint32_t); - safeMemcpyDst(b, &gauge_type); + safeMemcpyUnsafeDst(b, &gauge_type); b += sizeof(uint32_t); - safeMemcpyDst(b, &num_gauges); + safeMemcpyUnsafeDst(b, &num_gauges); b += sizeof(uint32_t); for (const auto& gauge : snapshot.gauges()) { if (gauge.get().used()) { n = gauge.get().name().size(); - safeMemcpyDst(b, &n); + safeMemcpyUnsafeDst(b, &n); b += sizeof(uint32_t); memcpy(b, gauge.get().name().data(), gauge.get().name().size()); // NOLINT(safe-memcpy) b = align(b + gauge.get().name().size()); v = gauge.get().value(); - safeMemcpyDst(b, &v); + safeMemcpyUnsafeDst(b, &v); b += sizeof(uint64_t); } } diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 1a6a80c4b0481..6e2fe85f145b1 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -108,7 +108,7 @@ class Int16Deserializer : public IntDeserializer { public: int16_t get() const override { int16_t result; - safeMemcpySrc(&result, buf_); + safeMemcpyUnsafeSrc(&result, buf_); return be16toh(result); } }; @@ -120,7 +120,7 @@ class Int32Deserializer : public IntDeserializer { public: int32_t get() const override { int32_t result; - safeMemcpySrc(&result, buf_); + safeMemcpyUnsafeSrc(&result, buf_); return be32toh(result); } }; @@ -132,7 +132,7 @@ class UInt32Deserializer : public IntDeserializer { public: uint32_t get() const override { uint32_t result; - safeMemcpySrc(&result, buf_); + safeMemcpyUnsafeSrc(&result, buf_); return be32toh(result); } }; @@ -144,7 +144,7 @@ class Int64Deserializer : public IntDeserializer { public: int64_t get() const override { int64_t result; - safeMemcpySrc(&result, buf_); + safeMemcpyUnsafeSrc(&result, buf_); return be64toh(result); } }; diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index d027deefc6de3..26899ea8ad134 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -172,7 +172,7 @@ bool DnsMessageParser::parseDnsObject(DnsQueryContextPtr& context, state = DnsQueryParseState::Flags; break; case DnsQueryParseState::Flags: - safeMemcpyDst(static_cast(&context->header_.flags), &data); + safeMemcpyUnsafeDst(static_cast(&context->header_.flags), &data); state = DnsQueryParseState::Questions; break; case DnsQueryParseState::Questions: @@ -744,7 +744,7 @@ void DnsMessageParser::buildResponseBuffer(DnsQueryContextPtr& query_context, buffer.writeBEInt(query_context->response_header_.id); uint16_t flags; - safeMemcpySrc(&flags, static_cast(&query_context->response_header_.flags)); + safeMemcpyUnsafeSrc(&flags, static_cast(&query_context->response_header_.flags)); buffer.writeBEInt(flags); buffer.writeBEInt(query_context->response_header_.questions); diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index fa8f7d810a575..55f8b06c546d1 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -117,14 +117,14 @@ std::string Utility::generalNameAsString(const GENERAL_NAME* general_name) { sockaddr_in sin; sin.sin_port = 0; sin.sin_family = AF_INET; - safeMemcpySrc(&sin.sin_addr, general_name->d.ip->data); + safeMemcpyUnsafeSrc(&sin.sin_addr, general_name->d.ip->data); Network::Address::Ipv4Instance addr(&sin); san = addr.ip()->addressAsString(); } else if (general_name->d.ip->length == 16) { sockaddr_in6 sin6; sin6.sin6_port = 0; sin6.sin6_family = AF_INET6; - safeMemcpySrc(&sin6.sin6_addr, general_name->d.ip->data); + safeMemcpyUnsafeSrc(&sin6.sin6_addr, general_name->d.ip->data); Network::Address::Ipv6Instance addr(sin6); san = addr.ip()->addressAsString(); } diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 46fa376cc7878..e0309322341c0 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -21,10 +21,6 @@ TEST(SafeMemcpyTest, CopyUint8) { Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); } -<<<<<<< HEAD -// Additional (integration) test - not ordinary copy -TEST(SafeMemcpyTest, PrependGrpcFrameHeader) { -======= TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { const uint8_t* src = new uint8_t[8]; for (int i = 0; i < 8; ++i) @@ -36,7 +32,6 @@ TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { } TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { ->>>>>>> 1c7e427... format fixed auto buffer = std::make_unique(); buffer->add("test", 4); std::array expected_header; From 89e53c2f714107c0955a250204fcd20be87e0854 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 10:26:45 +0000 Subject: [PATCH 088/106] format fixed Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 832415ccbd62a..d7962fc11517f 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -14,13 +14,15 @@ template inline void safeMemcpy(T1* dst, T2* src) { /** * @brief Copies src to dst based on the size of dst - * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to call safeMemcpyUnsafeSrc + * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to + * call safeMemcpyUnsafeSrc */ #define safeMemcpyUnsafeSrc(dst, src) memcpy(dst, src, sizeof(*(dst))); /** * @brief Copies src to dst based on the size of src - * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to call safeMemcpyUnsafeDst + * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to + * call safeMemcpyUnsafeDst */ #define safeMemcpyUnsafeDst(dst, src) memcpy(dst, src, sizeof(*(src))); From 08007f403b3b938d9a6d6966a8e5ec8a7e302b77 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 10:55:40 +0000 Subject: [PATCH 089/106] tests corrected Signed-off-by: grial1 --- test/common/common/safe_memcpy_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index e0309322341c0..3d7f48047480a 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -16,9 +16,9 @@ namespace Envoy { TEST(SafeMemcpyTest, CopyUint8) { const uint8_t src[] = {0, 1, 1, 2, 3, 5, 8, 13}; - std::array dst; - safeMemcpyDst(dst.data(), &src); - Eq(dst == std::array{0, 1, 1, 2, 3, 5, 8, 13}); + uint64_t dst; + safeMemcpy(&dst, &src); + Eq(dst == 282583128934413); } TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { @@ -37,7 +37,7 @@ TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { std::array expected_header; expected_header[0] = 0; // flags const uint32_t nsize = htonl(4); - safeMemcpyDst(&expected_header[1], &nsize); + safeMemcpyUnsafeDst(&expected_header[1], &nsize); std::string header_string(&expected_header[0], 5); Grpc::Common::prependGrpcFrameHeader(*buffer); EXPECT_EQ(buffer->toString(), header_string + "test"); From 021efceab455745ffe68e7a2a47ba2a9fdf22dfc Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 15:19:02 +0000 Subject: [PATCH 090/106] test corrected Signed-off-by: grial1 --- test/common/common/safe_memcpy_test.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 3d7f48047480a..db37b1bb1b207 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -11,6 +11,7 @@ #include "gtest/gtest.h" using testing::Eq; +using testing::ElementsAre; namespace Envoy { @@ -22,12 +23,12 @@ TEST(SafeMemcpyTest, CopyUint8) { } TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { - const uint8_t* src = new uint8_t[8]; + uint8_t* src = new uint8_t[8]; for (int i = 0; i < 8; ++i) src[i] = i; uint8_t dst[8]; safeMemcpyUnsafeSrc(&dst, src); - Eq(dst == {0, 1, 2, 3, 4, 5, 6, 7}); + ASSERT_THAT(dst, ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); delete src; } From 9159f298c93f777ba8001a86001f0c808fb4bd70 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 15:38:27 +0000 Subject: [PATCH 091/106] test approved Signed-off-by: grial1 --- test/common/common/safe_memcpy_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index db37b1bb1b207..c005740844b11 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -10,8 +10,8 @@ #include "gtest/gtest.h" -using testing::Eq; using testing::ElementsAre; +using testing::Eq; namespace Envoy { From b561e2f2ba4ce465386496d0b8849c9248de656c Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 15:54:51 +0000 Subject: [PATCH 092/106] format message updated Signed-off-by: grial1 --- tools/code_format/check_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index c37038371fa74..492d368bfe7bb 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -804,7 +804,7 @@ def checkSourceLine(self, line, file_path, reportError): ("memcpy(" in line) and \ not ("NOLINT(safe-memcpy)" in line): reportError( - "Don't call memcpy() directly; use safeMemcpy, safeMemcpySrc, safeMemcpyDst or MemBlockBuilder instead." + "Don't call memcpy() directly; use safeMemcpy, safeMemcpyUnsafeSrc, safeMemcpyUnsafeDst or MemBlockBuilder instead." ) if self.denylistedForExceptions(file_path): From ed2a892e0087abbf28cb62de7f5f614ed07e2e99 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 6 Dec 2020 19:23:01 +0000 Subject: [PATCH 093/106] tests corrected Signed-off-by: grial1 --- test/common/common/safe_memcpy_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index c005740844b11..8a61193e6d701 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -29,7 +29,7 @@ TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { uint8_t dst[8]; safeMemcpyUnsafeSrc(&dst, src); ASSERT_THAT(dst, ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); - delete src; + delete[] src; } TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { From a792eecba3635b9c87a32054bffb791d7d090a66 Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 7 Dec 2020 16:33:18 +0000 Subject: [PATCH 094/106] test/common/common/safe_memcpy_test.cc Signed-off-by: grial1 --- test/common/common/safe_memcpy_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 8a61193e6d701..f90cfed179e14 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -24,8 +24,9 @@ TEST(SafeMemcpyTest, CopyUint8) { TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { uint8_t* src = new uint8_t[8]; - for (int i = 0; i < 8; ++i) + for (int i = 0; i < 8; ++i) { src[i] = i; + } uint8_t dst[8]; safeMemcpyUnsafeSrc(&dst, src); ASSERT_THAT(dst, ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); From f6ebfa65072f525007487bb34e4978a684963e29 Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 7 Dec 2020 22:28:37 +0000 Subject: [PATCH 095/106] tmpfile to set in motion pipeline Signed-off-by: grial1 --- file7hbpZe | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 file7hbpZe diff --git a/file7hbpZe b/file7hbpZe new file mode 100644 index 0000000000000..e69de29bb2d1d From cb3ec942e21ce3a51d81ac8ff53f2c50ae7fde0e Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 9 Dec 2020 16:42:25 +0000 Subject: [PATCH 096/106] temporary file deleted Signed-off-by: grial1 --- file7hbpZe | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 file7hbpZe diff --git a/file7hbpZe b/file7hbpZe deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 33ef5a866c56af235e8e371e83d2f7328a3f3a55 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 9 Dec 2020 18:25:46 +0000 Subject: [PATCH 097/106] changed safeMemcpyUnsafeSrc and safeMemcpyUnsafeDst to function templates Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index d7962fc11517f..1d99bf69070c2 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -17,13 +17,16 @@ template inline void safeMemcpy(T1* dst, T2* src) { * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to * call safeMemcpyUnsafeSrc */ -#define safeMemcpyUnsafeSrc(dst, src) memcpy(dst, src, sizeof(*(dst))); - +template inline void safeMemcpyUnsafeSrc(T1* dst, void* src) { + memcpy(dst, src, sizeof(T1); +} /** * @brief Copies src to dst based on the size of src * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to * call safeMemcpyUnsafeDst */ -#define safeMemcpyUnsafeDst(dst, src) memcpy(dst, src, sizeof(*(src))); +template inline void safeMemcpyUnsafeDst(void* dst, T2* src) { + memcpy(dst, src, sizeof(T2)); +} } // namespace Envoy From 0630eee8eb289894fa9a3980e52295ecbb2afdf3 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 9 Dec 2020 18:29:30 +0000 Subject: [PATCH 098/106] typo fixed Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 1d99bf69070c2..06fddade36254 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -18,7 +18,7 @@ template inline void safeMemcpy(T1* dst, T2* src) { * call safeMemcpyUnsafeSrc */ template inline void safeMemcpyUnsafeSrc(T1* dst, void* src) { - memcpy(dst, src, sizeof(T1); + memcpy(dst, src, sizeof(T1)); } /** * @brief Copies src to dst based on the size of src From c8f1808bde0a491b0908c3c87a14f4ee00847367 Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 9 Dec 2020 19:03:19 +0000 Subject: [PATCH 099/106] void* removed Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 06fddade36254..575513393c619 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -17,7 +17,7 @@ template inline void safeMemcpy(T1* dst, T2* src) { * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to * call safeMemcpyUnsafeSrc */ -template inline void safeMemcpyUnsafeSrc(T1* dst, void* src) { +template inline void safeMemcpyUnsafeSrc(T1* dst, T2* src) { memcpy(dst, src, sizeof(T1)); } /** @@ -25,7 +25,7 @@ template inline void safeMemcpyUnsafeSrc(T1* dst, void* src) { * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to * call safeMemcpyUnsafeDst */ -template inline void safeMemcpyUnsafeDst(void* dst, T2* src) { +template inline void safeMemcpyUnsafeDst(T1* dst, T2* src) { memcpy(dst, src, sizeof(T2)); } From a2dfabb5e05426120e25a800a2a5b1527782e89e Mon Sep 17 00:00:00 2001 From: grial1 Date: Wed, 9 Dec 2020 21:46:43 +0000 Subject: [PATCH 100/106] address() function modified to use safeMemcpyUnsafeSrc Signed-off-by: grial1 --- source/common/network/address_impl.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index fd371718ce497..478299a503ee3 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -176,8 +176,7 @@ std::string Ipv4Instance::sockaddrToString(const sockaddr_in& addr) { absl::uint128 Ipv6Instance::Ipv6Helper::address() const { absl::uint128 result{0}; static_assert(sizeof(absl::uint128) == 16, "The size of asbl::uint128 is not 16."); - memcpy(static_cast(&result), // NOLINT(safe-memcpy) - static_cast(&address_.sin6_addr.s6_addr), sizeof(absl::uint128)); + safeMemcpyUnsafeSrc(&result, &address_.sin6_addr.s6_addr[0]); return result; } From 4da526bef6a9e628306a8d69d6698420bf121a16 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sun, 13 Dec 2020 22:26:21 +0000 Subject: [PATCH 101/106] merged with master: fixed buffer_impl.cc Signed-off-by: grial1 --- source/common/buffer/buffer_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/buffer/buffer_impl.cc b/source/common/buffer/buffer_impl.cc index 838833c15e1f5..2fcae93b4150e 100644 --- a/source/common/buffer/buffer_impl.cc +++ b/source/common/buffer/buffer_impl.cc @@ -137,7 +137,7 @@ void OwnedImpl::copyOut(size_t start, uint64_t size, void* data) const { continue; } uint64_t copy_size = std::min(size, data_size - bytes_to_skip); - memcpy(dest, slice->data() + bytes_to_skip, copy_size); // NOLINT(safe-memcpy) + memcpy(dest, slice.data() + bytes_to_skip, copy_size); // NOLINT(safe-memcpy) size -= copy_size; dest += copy_size; // Now that we've started copying, there are no bytes left to skip over. If there From 2b603f2b73e4c6c4713e97d6cd6078e5e44da896 Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 21 Dec 2020 11:56:22 +0000 Subject: [PATCH 102/106] codec_impl_legacy.cc deleted Signed-off-by: grial1 --- source/common/http/http2/codec_impl_legacy.cc | 1534 ----------------- 1 file changed, 1534 deletions(-) delete mode 100644 source/common/http/http2/codec_impl_legacy.cc diff --git a/source/common/http/http2/codec_impl_legacy.cc b/source/common/http/http2/codec_impl_legacy.cc deleted file mode 100644 index ffe587cad056b..0000000000000 --- a/source/common/http/http2/codec_impl_legacy.cc +++ /dev/null @@ -1,1534 +0,0 @@ -#include "common/http/http2/codec_impl_legacy.h" - -#include -#include -#include - -#include "envoy/event/dispatcher.h" -#include "envoy/http/codes.h" -#include "envoy/http/header_map.h" -#include "envoy/network/connection.h" - -#include "common/common/assert.h" -#include "common/common/cleanup.h" -#include "common/common/enum_to_int.h" -#include "common/common/fmt.h" -#include "common/common/safe_memcpy.h" -#include "common/common/utility.h" -#include "common/http/codes.h" -#include "common/http/exception.h" -#include "common/http/header_utility.h" -#include "common/http/headers.h" -#include "common/http/http2/codec_stats.h" -#include "common/http/utility.h" -#include "common/runtime/runtime_features.h" - -#include "absl/container/fixed_array.h" - -namespace Envoy { -namespace Http { -namespace Legacy { -namespace Http2 { - -class Http2ResponseCodeDetailValues { -public: - // Invalid HTTP header field was received and stream is going to be - // closed. - const absl::string_view ng_http2_err_http_header_ = "http2.invalid.header.field"; - // Violation in HTTP messaging rule. - const absl::string_view ng_http2_err_http_messaging_ = "http2.violation.of.messaging.rule"; - // none of the above - const absl::string_view ng_http2_err_unknown_ = "http2.unknown.nghttp2.error"; - // The number of headers (or trailers) exceeded the configured limits - const absl::string_view too_many_headers = "http2.too_many_headers"; - // Envoy detected an HTTP/2 frame flood from the server. - const absl::string_view outbound_frame_flood = "http2.outbound_frames_flood"; - // Envoy detected an inbound HTTP/2 frame flood. - const absl::string_view inbound_empty_frame_flood = "http2.inbound_empty_frames_flood"; - // Envoy was configured to drop requests with header keys beginning with underscores. - const absl::string_view invalid_underscore = "http2.unexpected_underscore"; - // The upstream refused the stream. - const absl::string_view remote_refused = "http2.remote_refuse"; - // The upstream reset the stream. - const absl::string_view remote_reset = "http2.remote_reset"; - - const absl::string_view errorDetails(int error_code) const { - switch (error_code) { - case NGHTTP2_ERR_HTTP_HEADER: - return ng_http2_err_http_header_; - case NGHTTP2_ERR_HTTP_MESSAGING: - return ng_http2_err_http_messaging_; - default: - return ng_http2_err_unknown_; - } - } -}; - -int reasonToReset(StreamResetReason reason) { - switch (reason) { - case StreamResetReason::LocalRefusedStreamReset: - return NGHTTP2_REFUSED_STREAM; - case StreamResetReason::ConnectError: - return NGHTTP2_CONNECT_ERROR; - default: - return NGHTTP2_NO_ERROR; - } -} - -using Http2ResponseCodeDetails = ConstSingleton; -using Http::Http2::CodecStats; -using Http::Http2::MetadataDecoder; -using Http::Http2::MetadataEncoder; - -bool Utility::reconstituteCrumbledCookies(const HeaderString& key, const HeaderString& value, - HeaderString& cookies) { - if (key != Headers::get().Cookie.get().c_str()) { - return false; - } - - if (!cookies.empty()) { - cookies.append("; ", 2); - } - - const absl::string_view value_view = value.getStringView(); - cookies.append(value_view.data(), value_view.size()); - return true; -} - -ConnectionImpl::Http2Callbacks ConnectionImpl::http2_callbacks_; - -nghttp2_session* ProdNghttp2SessionFactory::create(const nghttp2_session_callbacks* callbacks, - ConnectionImpl* connection, - const nghttp2_option* options) { - nghttp2_session* session; - nghttp2_session_client_new2(&session, callbacks, connection, options); - return session; -} - -void ProdNghttp2SessionFactory::init(nghttp2_session*, ConnectionImpl* connection, - const envoy::config::core::v3::Http2ProtocolOptions& options) { - connection->sendSettings(options, true); -} - -/** - * Helper to remove const during a cast. nghttp2 takes non-const pointers for headers even though - * it copies them. - */ -template static T* removeConst(const void* object) { - return const_cast(reinterpret_cast(object)); -} - -ConnectionImpl::StreamImpl::StreamImpl(ConnectionImpl& parent, uint32_t buffer_limit) - : parent_(parent), local_end_stream_sent_(false), remote_end_stream_(false), - data_deferred_(false), received_noninformational_headers_(false), - pending_receive_buffer_high_watermark_called_(false), - pending_send_buffer_high_watermark_called_(false), reset_due_to_messaging_error_(false) { - parent_.stats_.streams_active_.inc(); - if (buffer_limit > 0) { - setWriteBufferWatermarks(buffer_limit / 2, buffer_limit); - } -} - -ConnectionImpl::StreamImpl::~StreamImpl() { ASSERT(stream_idle_timer_ == nullptr); } - -void ConnectionImpl::StreamImpl::destroy() { - disarmStreamIdleTimer(); - parent_.stats_.streams_active_.dec(); - parent_.stats_.pending_send_bytes_.sub(pending_send_data_.length()); -} - -static void insertHeader(std::vector& headers, const HeaderEntry& header) { - uint8_t flags = 0; - if (header.key().isReference()) { - flags |= NGHTTP2_NV_FLAG_NO_COPY_NAME; - } - if (header.value().isReference()) { - flags |= NGHTTP2_NV_FLAG_NO_COPY_VALUE; - } - const absl::string_view header_key = header.key().getStringView(); - const absl::string_view header_value = header.value().getStringView(); - headers.push_back({removeConst(header_key.data()), - removeConst(header_value.data()), header_key.size(), - header_value.size(), flags}); -} - -void ConnectionImpl::StreamImpl::buildHeaders(std::vector& final_headers, - const HeaderMap& headers) { - final_headers.reserve(headers.size()); - headers.iterate([&final_headers](const HeaderEntry& header) -> HeaderMap::Iterate { - insertHeader(final_headers, header); - return HeaderMap::Iterate::Continue; - }); -} - -void ConnectionImpl::ServerStreamImpl::encode100ContinueHeaders(const ResponseHeaderMap& headers) { - ASSERT(headers.Status()->value() == "100"); - encodeHeaders(headers, false); -} - -void ConnectionImpl::StreamImpl::encodeHeadersBase(const std::vector& final_headers, - bool end_stream) { - nghttp2_data_provider provider; - if (!end_stream) { - provider.source.ptr = this; - provider.read_callback = [](nghttp2_session*, int32_t, uint8_t*, size_t length, - uint32_t* data_flags, nghttp2_data_source* source, - void*) -> ssize_t { - return static_cast(source->ptr)->onDataSourceRead(length, data_flags); - }; - } - - local_end_stream_ = end_stream; - submitHeaders(final_headers, end_stream ? nullptr : &provider); - parent_.sendPendingFrames(); - parent_.checkProtocolConstraintViolation(); -} - -Status ConnectionImpl::ClientStreamImpl::encodeHeaders(const RequestHeaderMap& headers, - bool end_stream) { - // This must exist outside of the scope of isUpgrade as the underlying memory is - // needed until encodeHeadersBase has been called. - std::vector final_headers; - Http::RequestHeaderMapPtr modified_headers; - if (Http::Utility::isUpgrade(headers)) { - modified_headers = createHeaderMap(headers); - upgrade_type_ = std::string(headers.getUpgradeValue()); - Http::Utility::transformUpgradeRequestFromH1toH2(*modified_headers); - buildHeaders(final_headers, *modified_headers); - } else if (headers.Method() && headers.Method()->value() == "CONNECT") { - // If this is not an upgrade style connect (above branch) it is a bytestream - // connect and should have :path and :protocol set accordingly - // As HTTP/1.1 does not require a path for CONNECT, we may have to add one - // if shifting codecs. For now, default to "/" - this can be made - // configurable if necessary. - // https://tools.ietf.org/html/draft-kinnear-httpbis-http2-transport-02 - modified_headers = createHeaderMap(headers); - modified_headers->setProtocol(Headers::get().ProtocolValues.Bytestream); - if (!headers.Path()) { - modified_headers->setPath("/"); - } - buildHeaders(final_headers, *modified_headers); - } else { - buildHeaders(final_headers, headers); - } - encodeHeadersBase(final_headers, end_stream); - return okStatus(); -} - -void ConnectionImpl::ServerStreamImpl::encodeHeaders(const ResponseHeaderMap& headers, - bool end_stream) { - // The contract is that client codecs must ensure that :status is present. - ASSERT(headers.Status() != nullptr); - - // This must exist outside of the scope of isUpgrade as the underlying memory is - // needed until encodeHeadersBase has been called. - std::vector final_headers; - Http::ResponseHeaderMapPtr modified_headers; - if (Http::Utility::isUpgrade(headers)) { - modified_headers = createHeaderMap(headers); - Http::Utility::transformUpgradeResponseFromH1toH2(*modified_headers); - buildHeaders(final_headers, *modified_headers); - } else { - buildHeaders(final_headers, headers); - } - encodeHeadersBase(final_headers, end_stream); -} - -void ConnectionImpl::StreamImpl::encodeTrailersBase(const HeaderMap& trailers) { - ASSERT(!local_end_stream_); - local_end_stream_ = true; - if (pending_send_data_.length() > 0) { - // In this case we want trailers to come after we release all pending body data that is - // waiting on window updates. We need to save the trailers so that we can emit them later. - // However, for empty trailers, we don't need to to save the trailers. - ASSERT(!pending_trailers_to_encode_); - const bool skip_encoding_empty_trailers = - trailers.empty() && parent_.skip_encoding_empty_trailers_; - if (!skip_encoding_empty_trailers) { - pending_trailers_to_encode_ = cloneTrailers(trailers); - createPendingFlushTimer(); - } - } else { - submitTrailers(trailers); - parent_.sendPendingFrames(); - parent_.checkProtocolConstraintViolation(); - } -} - -void ConnectionImpl::StreamImpl::encodeMetadata(const MetadataMapVector& metadata_map_vector) { - ASSERT(parent_.allow_metadata_); - MetadataEncoder& metadata_encoder = getMetadataEncoder(); - if (!metadata_encoder.createPayload(metadata_map_vector)) { - return; - } - for (uint8_t flags : metadata_encoder.payloadFrameFlagBytes()) { - submitMetadata(flags); - } - parent_.sendPendingFrames(); - parent_.checkProtocolConstraintViolation(); -} - -void ConnectionImpl::StreamImpl::readDisable(bool disable) { - ENVOY_CONN_LOG(debug, "Stream {} {}, unconsumed_bytes {} read_disable_count {}", - parent_.connection_, stream_id_, (disable ? "disabled" : "enabled"), - unconsumed_bytes_, read_disable_count_); - if (disable) { - ++read_disable_count_; - } else { - ASSERT(read_disable_count_ > 0); - --read_disable_count_; - if (!buffersOverrun()) { - nghttp2_session_consume(parent_.session_, stream_id_, unconsumed_bytes_); - unconsumed_bytes_ = 0; - parent_.sendPendingFrames(); - parent_.checkProtocolConstraintViolation(); - } - } -} - -void ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() { - ENVOY_CONN_LOG(debug, "recv buffer over limit ", parent_.connection_); - ASSERT(!pending_receive_buffer_high_watermark_called_); - pending_receive_buffer_high_watermark_called_ = true; - readDisable(true); -} - -void ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark() { - ENVOY_CONN_LOG(debug, "recv buffer under limit ", parent_.connection_); - ASSERT(pending_receive_buffer_high_watermark_called_); - pending_receive_buffer_high_watermark_called_ = false; - readDisable(false); -} - -void ConnectionImpl::ClientStreamImpl::decodeHeaders() { - auto& headers = absl::get(headers_or_trailers_); - const uint64_t status = Http::Utility::getResponseStatus(*headers); - - if (!upgrade_type_.empty() && headers->Status()) { - Http::Utility::transformUpgradeResponseFromH2toH1(*headers, upgrade_type_); - } - - // Non-informational headers are non-1xx OR 101-SwitchingProtocols, since 101 implies that further - // proxying is on an upgrade path. - received_noninformational_headers_ = - !CodeUtility::is1xx(status) || status == enumToInt(Http::Code::SwitchingProtocols); - - if (status == enumToInt(Http::Code::Continue)) { - ASSERT(!remote_end_stream_); - response_decoder_.decode100ContinueHeaders(std::move(headers)); - } else { - response_decoder_.decodeHeaders(std::move(headers), remote_end_stream_); - } -} - -void ConnectionImpl::ClientStreamImpl::decodeTrailers() { - response_decoder_.decodeTrailers( - std::move(absl::get(headers_or_trailers_))); -} - -void ConnectionImpl::ServerStreamImpl::decodeHeaders() { - auto& headers = absl::get(headers_or_trailers_); - if (Http::Utility::isH2UpgradeRequest(*headers)) { - Http::Utility::transformUpgradeRequestFromH2toH1(*headers); - } - request_decoder_->decodeHeaders(std::move(headers), remote_end_stream_); -} - -void ConnectionImpl::ServerStreamImpl::decodeTrailers() { - request_decoder_->decodeTrailers( - std::move(absl::get(headers_or_trailers_))); -} - -void ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() { - ENVOY_CONN_LOG(debug, "send buffer over limit ", parent_.connection_); - ASSERT(!pending_send_buffer_high_watermark_called_); - pending_send_buffer_high_watermark_called_ = true; - runHighWatermarkCallbacks(); -} - -void ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark() { - ENVOY_CONN_LOG(debug, "send buffer under limit ", parent_.connection_); - ASSERT(pending_send_buffer_high_watermark_called_); - pending_send_buffer_high_watermark_called_ = false; - runLowWatermarkCallbacks(); -} - -void ConnectionImpl::StreamImpl::saveHeader(HeaderString&& name, HeaderString&& value) { - if (!Utility::reconstituteCrumbledCookies(name, value, cookies_)) { - headers().addViaMove(std::move(name), std::move(value)); - } -} - -void ConnectionImpl::StreamImpl::submitTrailers(const HeaderMap& trailers) { - ASSERT(local_end_stream_); - const bool skip_encoding_empty_trailers = - trailers.empty() && parent_.skip_encoding_empty_trailers_; - if (skip_encoding_empty_trailers) { - ENVOY_CONN_LOG(debug, "skipping submitting trailers", parent_.connection_); - - // Instead of submitting empty trailers, we send empty data instead. - Buffer::OwnedImpl empty_buffer; - encodeDataHelper(empty_buffer, /*end_stream=*/true, skip_encoding_empty_trailers); - return; - } - - std::vector final_headers; - buildHeaders(final_headers, trailers); - int rc = nghttp2_submit_trailer(parent_.session_, stream_id_, final_headers.data(), - final_headers.size()); - ASSERT(rc == 0); -} - -void ConnectionImpl::StreamImpl::submitMetadata(uint8_t flags) { - ASSERT(stream_id_ > 0); - const int result = - nghttp2_submit_extension(parent_.session_, METADATA_FRAME_TYPE, flags, stream_id_, nullptr); - ASSERT(result == 0); -} - -ssize_t ConnectionImpl::StreamImpl::onDataSourceRead(uint64_t length, uint32_t* data_flags) { - if (pending_send_data_.length() == 0 && !local_end_stream_) { - ASSERT(!data_deferred_); - data_deferred_ = true; - return NGHTTP2_ERR_DEFERRED; - } else { - *data_flags |= NGHTTP2_DATA_FLAG_NO_COPY; - if (local_end_stream_ && pending_send_data_.length() <= length) { - *data_flags |= NGHTTP2_DATA_FLAG_EOF; - if (pending_trailers_to_encode_) { - // We need to tell the library to not set end stream so that we can emit the trailers. - *data_flags |= NGHTTP2_DATA_FLAG_NO_END_STREAM; - submitTrailers(*pending_trailers_to_encode_); - pending_trailers_to_encode_.reset(); - } - } - - return std::min(length, pending_send_data_.length()); - } -} - -int ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t length) { - // In this callback we are writing out a raw DATA frame without copying. nghttp2 assumes that we - // "just know" that the frame header is 9 bytes. - // https://nghttp2.org/documentation/types.html#c.nghttp2_send_data_callback - static const uint64_t FRAME_HEADER_SIZE = 9; - - parent_.protocol_constraints_.incrementOutboundDataFrameCount(); - - Buffer::OwnedImpl output; - if (!parent_.addOutboundFrameFragment(output, framehd, FRAME_HEADER_SIZE)) { - ENVOY_CONN_LOG(debug, "error sending data frame: Too many frames in the outbound queue", - parent_.connection_); - setDetails(Http2ResponseCodeDetails::get().outbound_frame_flood); - return NGHTTP2_ERR_FLOODED; - } - - parent_.stats_.pending_send_bytes_.sub(length); - output.move(pending_send_data_, length); - parent_.connection_.write(output, false); - return 0; -} - -void ConnectionImpl::ClientStreamImpl::submitHeaders(const std::vector& final_headers, - nghttp2_data_provider* provider) { - ASSERT(stream_id_ == -1); - stream_id_ = nghttp2_submit_request(parent_.session_, nullptr, final_headers.data(), - final_headers.size(), provider, base()); - ASSERT(stream_id_ > 0); -} - -void ConnectionImpl::ServerStreamImpl::submitHeaders(const std::vector& final_headers, - nghttp2_data_provider* provider) { - ASSERT(stream_id_ != -1); - int rc = nghttp2_submit_response(parent_.session_, stream_id_, final_headers.data(), - final_headers.size(), provider); - ASSERT(rc == 0); -} - -void ConnectionImpl::ServerStreamImpl::createPendingFlushTimer() { - ASSERT(stream_idle_timer_ == nullptr); - if (stream_idle_timeout_.count() > 0) { - stream_idle_timer_ = - parent_.connection_.dispatcher().createTimer([this] { onPendingFlushTimer(); }); - stream_idle_timer_->enableTimer(stream_idle_timeout_); - } -} - -void ConnectionImpl::StreamImpl::onPendingFlushTimer() { - ENVOY_CONN_LOG(debug, "pending stream flush timeout", parent_.connection_); - stream_idle_timer_.reset(); - parent_.stats_.tx_flush_timeout_.inc(); - ASSERT(local_end_stream_ && !local_end_stream_sent_); - // This will emit a reset frame for this stream and close the stream locally. No reset callbacks - // will be run because higher layers think the stream is already finished. - resetStreamWorker(StreamResetReason::LocalReset); - parent_.sendPendingFrames(); - parent_.checkProtocolConstraintViolation(); -} - -void ConnectionImpl::StreamImpl::encodeData(Buffer::Instance& data, bool end_stream) { - ASSERT(!local_end_stream_); - encodeDataHelper(data, end_stream, /*skip_encoding_empty_trailers=*/false); -} - -void ConnectionImpl::StreamImpl::encodeDataHelper(Buffer::Instance& data, bool end_stream, - bool skip_encoding_empty_trailers) { - if (skip_encoding_empty_trailers) { - ASSERT(data.length() == 0 && end_stream); - } - - local_end_stream_ = end_stream; - parent_.stats_.pending_send_bytes_.add(data.length()); - pending_send_data_.move(data); - if (data_deferred_) { - int rc = nghttp2_session_resume_data(parent_.session_, stream_id_); - ASSERT(rc == 0); - - data_deferred_ = false; - } - - parent_.sendPendingFrames(); - parent_.checkProtocolConstraintViolation(); - - if (local_end_stream_ && pending_send_data_.length() > 0) { - createPendingFlushTimer(); - } -} - -void ConnectionImpl::StreamImpl::resetStream(StreamResetReason reason) { - // Higher layers expect calling resetStream() to immediately raise reset callbacks. - runResetCallbacks(reason); - - // If we submit a reset, nghttp2 will cancel outbound frames that have not yet been sent. - // We want these frames to go out so we defer the reset until we send all of the frames that - // end the local stream. - if (local_end_stream_ && !local_end_stream_sent_) { - parent_.pending_deferred_reset_ = true; - deferred_reset_ = reason; - ENVOY_CONN_LOG(trace, "deferred reset stream", parent_.connection_); - } else { - resetStreamWorker(reason); - } - - // We must still call sendPendingFrames() in both the deferred and not deferred path. This forces - // the cleanup logic to run which will reset the stream in all cases if all data frames could not - // be sent. - parent_.sendPendingFrames(); - parent_.checkProtocolConstraintViolation(); -} - -void ConnectionImpl::StreamImpl::resetStreamWorker(StreamResetReason reason) { - int rc = nghttp2_submit_rst_stream(parent_.session_, NGHTTP2_FLAG_NONE, stream_id_, - reasonToReset(reason)); - ASSERT(rc == 0); -} - -MetadataEncoder& ConnectionImpl::StreamImpl::getMetadataEncoder() { - if (metadata_encoder_ == nullptr) { - metadata_encoder_ = std::make_unique(); - } - return *metadata_encoder_; -} - -MetadataDecoder& ConnectionImpl::StreamImpl::getMetadataDecoder() { - if (metadata_decoder_ == nullptr) { - auto cb = [this](MetadataMapPtr&& metadata_map_ptr) { - this->onMetadataDecoded(std::move(metadata_map_ptr)); - }; - metadata_decoder_ = std::make_unique(cb); - } - return *metadata_decoder_; -} - -void ConnectionImpl::StreamImpl::onMetadataDecoded(MetadataMapPtr&& metadata_map_ptr) { - decoder().decodeMetadata(std::move(metadata_map_ptr)); -} - -ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stats, - Random::RandomGenerator& random, - const envoy::config::core::v3::Http2ProtocolOptions& http2_options, - const uint32_t max_headers_kb, const uint32_t max_headers_count) - : stats_(stats), connection_(connection), max_headers_kb_(max_headers_kb), - max_headers_count_(max_headers_count), - per_stream_buffer_limit_(http2_options.initial_stream_window_size().value()), - stream_error_on_invalid_http_messaging_( - http2_options.override_stream_error_on_invalid_http_message().value()), - flood_detected_(false), protocol_constraints_(stats, http2_options), - skip_encoding_empty_trailers_(Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.http2_skip_encoding_empty_trailers")), - dispatching_(false), raised_goaway_(false), pending_deferred_reset_(false), random_(random) { - if (http2_options.has_connection_keepalive()) { - keepalive_interval_ = std::chrono::milliseconds( - PROTOBUF_GET_MS_REQUIRED(http2_options.connection_keepalive(), interval)); - keepalive_timeout_ = std::chrono::milliseconds( - PROTOBUF_GET_MS_REQUIRED(http2_options.connection_keepalive(), timeout)); - keepalive_interval_jitter_percent_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT( - http2_options.connection_keepalive(), interval_jitter, 15.0); - - keepalive_send_timer_ = connection.dispatcher().createTimer([this]() { sendKeepalive(); }); - keepalive_timeout_timer_ = - connection.dispatcher().createTimer([this]() { onKeepaliveResponseTimeout(); }); - - // This call schedules the initial interval, with jitter. - onKeepaliveResponse(); - } -} - -ConnectionImpl::~ConnectionImpl() { - for (const auto& stream : active_streams_) { - stream->destroy(); - } - nghttp2_session_del(session_); -} - -void ConnectionImpl::sendKeepalive() { - // Include the current time as the payload to help with debugging. - SystemTime now = connection_.dispatcher().timeSource().systemTime(); - uint64_t ms_since_epoch = - std::chrono::duration_cast(now.time_since_epoch()).count(); - ENVOY_CONN_LOG(trace, "Sending keepalive PING {}", connection_, ms_since_epoch); - - // The last parameter is an opaque 8-byte buffer, so this cast is safe. - int rc = nghttp2_submit_ping(session_, 0 /*flags*/, reinterpret_cast(&ms_since_epoch)); - ASSERT(rc == 0); - sendPendingFrames(); - checkProtocolConstraintViolation(); - - keepalive_timeout_timer_->enableTimer(keepalive_timeout_); -} - -void ConnectionImpl::onKeepaliveResponse() { - // Check the timers for nullptr in case the peer sent an unsolicited PING ACK. - if (keepalive_timeout_timer_ != nullptr) { - keepalive_timeout_timer_->disableTimer(); - } - if (keepalive_send_timer_ != nullptr) { - uint64_t interval_ms = keepalive_interval_.count(); - const uint64_t jitter_percent_mod = keepalive_interval_jitter_percent_ * interval_ms / 100; - if (jitter_percent_mod > 0) { - interval_ms += random_.random() % jitter_percent_mod; - } - keepalive_send_timer_->enableTimer(std::chrono::milliseconds(interval_ms)); - } -} - -void ConnectionImpl::onKeepaliveResponseTimeout() { - ENVOY_CONN_LOG(debug, "Closing connection due to keepalive timeout", connection_); - stats_.keepalive_timeout_.inc(); - connection_.close(Network::ConnectionCloseType::NoFlush); -} - -Http::Status ConnectionImpl::dispatch(Buffer::Instance& data) { - // TODO(#10878): Remove this wrapper when exception removal is complete. innerDispatch may either - // throw an exception or return an error status. The utility wrapper catches exceptions and - // converts them to error statuses. - return Http::Utility::exceptionToStatus( - [&](Buffer::Instance& data) -> Http::Status { return innerDispatch(data); }, data); -} - -Http::Status ConnectionImpl::innerDispatch(Buffer::Instance& data) { - ENVOY_CONN_LOG(trace, "dispatching {} bytes", connection_, data.length()); - // Make sure that dispatching_ is set to false after dispatching, even when - // ConnectionImpl::dispatch returns early or throws an exception (consider removing if there is a - // single return after exception removal (#10878)). - Cleanup cleanup([this]() { dispatching_ = false; }); - for (const Buffer::RawSlice& slice : data.getRawSlices()) { - dispatching_ = true; - ssize_t rc = - nghttp2_session_mem_recv(session_, static_cast(slice.mem_), slice.len_); - if (rc == NGHTTP2_ERR_FLOODED || flood_detected_) { - throw FrameFloodException( - "Flooding was detected in this HTTP/2 session, and it must be closed"); - } - if (rc != static_cast(slice.len_)) { - throw CodecProtocolException(fmt::format("{}", nghttp2_strerror(rc))); - } - - dispatching_ = false; - } - - ENVOY_CONN_LOG(trace, "dispatched {} bytes", connection_, data.length()); - data.drain(data.length()); - - // Decoding incoming frames can generate outbound frames so flush pending. - sendPendingFrames(); - return Http::okStatus(); -} - -ConnectionImpl::StreamImpl* ConnectionImpl::getStream(int32_t stream_id) { - return static_cast(nghttp2_session_get_stream_user_data(session_, stream_id)); -} - -int ConnectionImpl::onData(int32_t stream_id, const uint8_t* data, size_t len) { - StreamImpl* stream = getStream(stream_id); - // If this results in buffering too much data, the watermark buffer will call - // pendingRecvBufferHighWatermark, resulting in ++read_disable_count_ - stream->pending_recv_data_.add(data, len); - // Update the window to the peer unless some consumer of this stream's data has hit a flow control - // limit and disabled reads on this stream - if (!stream->buffersOverrun()) { - nghttp2_session_consume(session_, stream_id, len); - } else { - stream->unconsumed_bytes_ += len; - } - return 0; -} - -void ConnectionImpl::goAway() { - int rc = nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, - nghttp2_session_get_last_proc_stream_id(session_), - NGHTTP2_NO_ERROR, nullptr, 0); - ASSERT(rc == 0); - - sendPendingFrames(); - checkProtocolConstraintViolation(); -} - -void ConnectionImpl::shutdownNotice() { - int rc = nghttp2_submit_shutdown_notice(session_); - ASSERT(rc == 0); - - sendPendingFrames(); - checkProtocolConstraintViolation(); -} - -int ConnectionImpl::onBeforeFrameReceived(const nghttp2_frame_hd* hd) { - ENVOY_CONN_LOG(trace, "about to recv frame type={}, flags={}", connection_, - static_cast(hd->type), static_cast(hd->flags)); - - // Track all the frames without padding here, since this is the only callback we receive - // for some of them (e.g. CONTINUATION frame, frames sent on closed streams, etc.). - // HEADERS frame is tracked in onBeginHeaders(), DATA frame is tracked in onFrameReceived(). - if (hd->type != NGHTTP2_HEADERS && hd->type != NGHTTP2_DATA) { - if (!trackInboundFrames(hd, 0)) { - return NGHTTP2_ERR_FLOODED; - } - } - - return 0; -} - -ABSL_MUST_USE_RESULT -enum GoAwayErrorCode ngHttp2ErrorCodeToErrorCode(uint32_t code) noexcept { - switch (code) { - case NGHTTP2_NO_ERROR: - return GoAwayErrorCode::NoError; - default: - return GoAwayErrorCode::Other; - } -} - -int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { - ENVOY_CONN_LOG(trace, "recv frame type={}", connection_, static_cast(frame->hd.type)); - - // onFrameReceived() is called with a complete HEADERS frame assembled from all the HEADERS - // and CONTINUATION frames, but we track them separately: HEADERS frames in onBeginHeaders() - // and CONTINUATION frames in onBeforeFrameReceived(). - ASSERT(frame->hd.type != NGHTTP2_CONTINUATION); - - if ((frame->hd.type == NGHTTP2_PING) && (frame->ping.hd.flags & NGHTTP2_FLAG_ACK)) { - // The ``opaque_data`` should be exactly what was sent in the ping, which is - // was the current time when the ping was sent. This can be useful while debugging - // to match the ping and ack. - uint64_t data; - safeMemcpy(&data, &(frame->ping.opaque_data)); - ENVOY_CONN_LOG(trace, "recv PING ACK {}", connection_, data); - - onKeepaliveResponse(); - return 0; - } - - if (frame->hd.type == NGHTTP2_DATA) { - if (!trackInboundFrames(&frame->hd, frame->data.padlen)) { - return NGHTTP2_ERR_FLOODED; - } - } - - // Only raise GOAWAY once, since we don't currently expose stream information. Shutdown - // notifications are the same as a normal GOAWAY. - // TODO: handle multiple GOAWAY frames. - if (frame->hd.type == NGHTTP2_GOAWAY && !raised_goaway_) { - ASSERT(frame->hd.stream_id == 0); - raised_goaway_ = true; - callbacks().onGoAway(ngHttp2ErrorCodeToErrorCode(frame->goaway.error_code)); - return 0; - } - - if (frame->hd.type == NGHTTP2_SETTINGS && frame->hd.flags == NGHTTP2_FLAG_NONE) { - onSettingsForTest(frame->settings); - } - - StreamImpl* stream = getStream(frame->hd.stream_id); - if (!stream) { - return 0; - } - - switch (frame->hd.type) { - case NGHTTP2_HEADERS: { - stream->remote_end_stream_ = frame->hd.flags & NGHTTP2_FLAG_END_STREAM; - if (!stream->cookies_.empty()) { - HeaderString key(Headers::get().Cookie); - stream->headers().addViaMove(std::move(key), std::move(stream->cookies_)); - } - - switch (frame->headers.cat) { - case NGHTTP2_HCAT_RESPONSE: - case NGHTTP2_HCAT_REQUEST: { - stream->decodeHeaders(); - break; - } - - case NGHTTP2_HCAT_HEADERS: { - // It's possible that we are waiting to send a deferred reset, so only raise headers/trailers - // if local is not complete. - if (!stream->deferred_reset_) { - if (nghttp2_session_check_server_session(session_) || - stream->received_noninformational_headers_) { - ASSERT(stream->remote_end_stream_); - stream->decodeTrailers(); - } else { - // We're a client session and still waiting for non-informational headers. - stream->decodeHeaders(); - } - } - break; - } - - default: - // We do not currently support push. - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } - - break; - } - case NGHTTP2_DATA: { - stream->remote_end_stream_ = frame->hd.flags & NGHTTP2_FLAG_END_STREAM; - - // It's possible that we are waiting to send a deferred reset, so only raise data if local - // is not complete. - if (!stream->deferred_reset_) { - stream->decoder().decodeData(stream->pending_recv_data_, stream->remote_end_stream_); - } - - stream->pending_recv_data_.drain(stream->pending_recv_data_.length()); - break; - } - case NGHTTP2_RST_STREAM: { - ENVOY_CONN_LOG(trace, "remote reset: {}", connection_, frame->rst_stream.error_code); - stats_.rx_reset_.inc(); - break; - } - } - - return 0; -} - -int ConnectionImpl::onFrameSend(const nghttp2_frame* frame) { - // The nghttp2 library does not cleanly give us a way to determine whether we received invalid - // data from our peer. Sometimes it raises the invalid frame callback, and sometimes it does not. - // In all cases however it will attempt to send a GOAWAY frame with an error status. If we see - // an outgoing frame of this type, we will return an error code so that we can abort execution. - ENVOY_CONN_LOG(trace, "sent frame type={}", connection_, static_cast(frame->hd.type)); - switch (frame->hd.type) { - case NGHTTP2_GOAWAY: { - ENVOY_CONN_LOG(debug, "sent goaway code={}", connection_, frame->goaway.error_code); - if (frame->goaway.error_code != NGHTTP2_NO_ERROR) { - // TODO(mattklein123): Returning this error code abandons standard nghttp2 frame accounting. - // As such, it is not reliable to call sendPendingFrames() again after this and we assume - // that the connection is going to get torn down immediately. One byproduct of this is that - // we need to cancel all pending flush stream timeouts since they can race with connection - // teardown. As part of the work to remove exceptions we should aim to clean up all of this - // error handling logic and only handle this type of case at the end of dispatch. - for (auto& stream : active_streams_) { - stream->disarmStreamIdleTimer(); - } - return NGHTTP2_ERR_CALLBACK_FAILURE; - } - break; - } - - case NGHTTP2_RST_STREAM: { - ENVOY_CONN_LOG(debug, "sent reset code={}", connection_, frame->rst_stream.error_code); - stats_.tx_reset_.inc(); - break; - } - - case NGHTTP2_HEADERS: - case NGHTTP2_DATA: { - StreamImpl* stream = getStream(frame->hd.stream_id); - stream->local_end_stream_sent_ = frame->hd.flags & NGHTTP2_FLAG_END_STREAM; - break; - } - } - - return 0; -} - -int ConnectionImpl::onError(absl::string_view error) { - ENVOY_CONN_LOG(debug, "invalid http2: {}", connection_, error); - return 0; -} - -int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) { - ENVOY_CONN_LOG(debug, "invalid frame: {} on stream {}", connection_, nghttp2_strerror(error_code), - stream_id); - - // Set details of error_code in the stream whenever we have one. - StreamImpl* stream = getStream(stream_id); - if (stream != nullptr) { - stream->setDetails(Http2ResponseCodeDetails::get().errorDetails(error_code)); - } - - if (error_code == NGHTTP2_ERR_HTTP_HEADER || error_code == NGHTTP2_ERR_HTTP_MESSAGING) { - stats_.rx_messaging_error_.inc(); - - if (stream_error_on_invalid_http_messaging_) { - // The stream is about to be closed due to an invalid header or messaging. Don't kill the - // entire connection if one stream has bad headers or messaging. - if (stream != nullptr) { - // See comment below in onStreamClose() for why we do this. - stream->reset_due_to_messaging_error_ = true; - } - return 0; - } - } - - // Cause dispatch to return with an error code. - return NGHTTP2_ERR_CALLBACK_FAILURE; -} - -int ConnectionImpl::onBeforeFrameSend(const nghttp2_frame* frame) { - ENVOY_CONN_LOG(trace, "about to send frame type={}, flags={}", connection_, - static_cast(frame->hd.type), static_cast(frame->hd.flags)); - ASSERT(!is_outbound_flood_monitored_control_frame_); - // Flag flood monitored outbound control frames. - is_outbound_flood_monitored_control_frame_ = - ((frame->hd.type == NGHTTP2_PING || frame->hd.type == NGHTTP2_SETTINGS) && - frame->hd.flags & NGHTTP2_FLAG_ACK) || - frame->hd.type == NGHTTP2_RST_STREAM; - return 0; -} - -bool ConnectionImpl::addOutboundFrameFragment(Buffer::OwnedImpl& output, const uint8_t* data, - size_t length) { - // Reset the outbound frame type (set in the onBeforeFrameSend callback) since the - // onBeforeFrameSend callback is not called for DATA frames. - bool is_outbound_flood_monitored_control_frame = false; - std::swap(is_outbound_flood_monitored_control_frame, is_outbound_flood_monitored_control_frame_); - try { - auto releasor = trackOutboundFrames(is_outbound_flood_monitored_control_frame); - output.add(data, length); - output.addDrainTracker(releasor); - } catch (const FrameFloodException&) { - return false; - } - return true; -} - -ssize_t ConnectionImpl::onSend(const uint8_t* data, size_t length) { - ENVOY_CONN_LOG(trace, "send data: bytes={}", connection_, length); - Buffer::OwnedImpl buffer; - if (!addOutboundFrameFragment(buffer, data, length)) { - ENVOY_CONN_LOG(debug, "error sending frame: Too many frames in the outbound queue.", - connection_); - return NGHTTP2_ERR_FLOODED; - } - - // While the buffer is transient the fragment it contains will be moved into the - // write_buffer_ of the underlying connection_ by the write method below. - // This creates lifetime dependency between the write_buffer_ of the underlying connection - // and the codec object. Specifically the write_buffer_ MUST be either fully drained or - // deleted before the codec object is deleted. This is presently guaranteed by the - // destruction order of the Network::ConnectionImpl object where write_buffer_ is - // destroyed before the filter_manager_ which owns the codec through Http::ConnectionManagerImpl. - connection_.write(buffer, false); - return length; -} - -int ConnectionImpl::onStreamClose(int32_t stream_id, uint32_t error_code) { - StreamImpl* stream = getStream(stream_id); - if (stream) { - ENVOY_CONN_LOG(debug, "stream closed: {}", connection_, error_code); - if (!stream->remote_end_stream_ || !stream->local_end_stream_) { - StreamResetReason reason; - if (stream->reset_due_to_messaging_error_) { - // Unfortunately, the nghttp2 API makes it incredibly difficult to clearly understand - // the flow of resets. I.e., did the reset originate locally? Was it remote? Here, - // we attempt to track cases in which we sent a reset locally due to an invalid frame - // received from the remote. We only do that in two cases currently (HTTP messaging layer - // errors from https://tools.ietf.org/html/rfc7540#section-8 which nghttp2 is very strict - // about). In other cases we treat invalid frames as a protocol error and just kill - // the connection. - reason = StreamResetReason::LocalReset; - } else { - if (error_code == NGHTTP2_REFUSED_STREAM) { - reason = StreamResetReason::RemoteRefusedStreamReset; - stream->setDetails(Http2ResponseCodeDetails::get().remote_refused); - } else if (error_code == NGHTTP2_CONNECT_ERROR) { - reason = StreamResetReason::ConnectError; - stream->setDetails(Http2ResponseCodeDetails::get().remote_reset); - } else { - reason = StreamResetReason::RemoteReset; - stream->setDetails(Http2ResponseCodeDetails::get().remote_reset); - } - } - - stream->runResetCallbacks(reason); - } - - stream->destroy(); - connection_.dispatcher().deferredDelete(stream->removeFromList(active_streams_)); - // Any unconsumed data must be consumed before the stream is deleted. - // nghttp2 does not appear to track this internally, and any stream deleted - // with outstanding window will contribute to a slow connection-window leak. - nghttp2_session_consume(session_, stream_id, stream->unconsumed_bytes_); - stream->unconsumed_bytes_ = 0; - nghttp2_session_set_stream_user_data(session_, stream->stream_id_, nullptr); - } - - return 0; -} - -int ConnectionImpl::onMetadataReceived(int32_t stream_id, const uint8_t* data, size_t len) { - ENVOY_CONN_LOG(trace, "recv {} bytes METADATA", connection_, len); - - StreamImpl* stream = getStream(stream_id); - if (!stream || stream->remote_end_stream_) { - return 0; - } - - bool success = stream->getMetadataDecoder().receiveMetadata(data, len); - return success ? 0 : NGHTTP2_ERR_CALLBACK_FAILURE; -} - -int ConnectionImpl::onMetadataFrameComplete(int32_t stream_id, bool end_metadata) { - ENVOY_CONN_LOG(trace, "recv METADATA frame on stream {}, end_metadata: {}", connection_, - stream_id, end_metadata); - - StreamImpl* stream = getStream(stream_id); - if (!stream || stream->remote_end_stream_) { - return 0; - } - - bool result = stream->getMetadataDecoder().onMetadataFrameComplete(end_metadata); - return result ? 0 : NGHTTP2_ERR_CALLBACK_FAILURE; -} - -ssize_t ConnectionImpl::packMetadata(int32_t stream_id, uint8_t* buf, size_t len) { - ENVOY_CONN_LOG(trace, "pack METADATA frame on stream {}", connection_, stream_id); - - StreamImpl* stream = getStream(stream_id); - if (stream == nullptr) { - return 0; - } - - MetadataEncoder& encoder = stream->getMetadataEncoder(); - return encoder.packNextFramePayload(buf, len); -} - -int ConnectionImpl::saveHeader(const nghttp2_frame* frame, HeaderString&& name, - HeaderString&& value) { - StreamImpl* stream = getStream(frame->hd.stream_id); - if (!stream) { - // We have seen 1 or 2 crashes where we get a headers callback but there is no associated - // stream data. I honestly am not sure how this can happen. However, from reading the nghttp2 - // code it looks possible that inflate_header_block() can safely inflate headers for an already - // closed stream, but will still call the headers callback. Since that seems possible, we should - // ignore this case here. - // TODO(mattklein123): Figure out a test case that can hit this. - stats_.headers_cb_no_stream_.inc(); - return 0; - } - - auto should_return = checkHeaderNameForUnderscores(name.getStringView()); - if (should_return) { - stream->setDetails(Http2ResponseCodeDetails::get().invalid_underscore); - name.clear(); - value.clear(); - return should_return.value(); - } - - stream->saveHeader(std::move(name), std::move(value)); - - if (stream->headers().byteSize() > max_headers_kb_ * 1024 || - stream->headers().size() > max_headers_count_) { - stream->setDetails(Http2ResponseCodeDetails::get().too_many_headers); - stats_.header_overflow_.inc(); - // This will cause the library to reset/close the stream. - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; - } else { - return 0; - } -} - -void ConnectionImpl::sendPendingFrames() { - if (dispatching_ || connection_.state() == Network::Connection::State::Closed) { - return; - } - - const int rc = nghttp2_session_send(session_); - if (rc != 0) { - ASSERT(rc == NGHTTP2_ERR_CALLBACK_FAILURE); - // For errors caused by the pending outbound frame flood the FrameFloodException has - // to be thrown. However the nghttp2 library returns only the generic error code for - // all failure types. Check queue limits and throw FrameFloodException if they were - // exceeded. - if (!protocol_constraints_.status().ok()) { - throw FrameFloodException("Too many frames in the outbound queue."); - } - - throw CodecProtocolException(std::string(nghttp2_strerror(rc))); - } - - // See ConnectionImpl::StreamImpl::resetStream() for why we do this. This is an uncommon event, - // so iterating through every stream to find the ones that have a deferred reset is not a big - // deal. Furthermore, queueing a reset frame does not actually invoke the close stream callback. - // This is only done when the reset frame is sent. Thus, it's safe to work directly with the - // stream map. - // NOTE: The way we handle deferred reset is essentially best effort. If we intend to do a - // deferred reset, we try to finish the stream, including writing any pending data frames. - // If we cannot do this (potentially due to not enough window), we just reset the stream. - // In general this behavior occurs only when we are trying to send immediate error messages - // to short circuit requests. In the best effort case, we complete the stream before - // resetting. In other cases, we just do the reset now which will blow away pending data - // frames and release any memory associated with the stream. - if (pending_deferred_reset_) { - pending_deferred_reset_ = false; - for (auto& stream : active_streams_) { - if (stream->deferred_reset_) { - stream->resetStreamWorker(stream->deferred_reset_.value()); - } - } - sendPendingFrames(); - } -} - -void ConnectionImpl::sendSettings( - const envoy::config::core::v3::Http2ProtocolOptions& http2_options, bool disable_push) { - absl::InlinedVector settings; - auto insertParameter = [&settings](const nghttp2_settings_entry& entry) mutable -> bool { - const auto it = std::find_if(settings.cbegin(), settings.cend(), - [&entry](const nghttp2_settings_entry& existing) { - return entry.settings_id == existing.settings_id; - }); - if (it != settings.end()) { - return false; - } - settings.push_back(entry); - return true; - }; - - // Universally disable receiving push promise frames as we don't currently support - // them. nghttp2 will fail the connection if the other side still sends them. - // TODO(mattklein123): Remove this when we correctly proxy push promise. - // NOTE: This is a special case with respect to custom parameter overrides in that server push is - // not supported and therefore not end user configurable. - if (disable_push) { - settings.push_back( - {static_cast(NGHTTP2_SETTINGS_ENABLE_PUSH), disable_push ? 0U : 1U}); - } - - for (const auto& it : http2_options.custom_settings_parameters()) { - ASSERT(it.identifier().value() <= std::numeric_limits::max()); - const bool result = - insertParameter({static_cast(it.identifier().value()), it.value().value()}); - ASSERT(result); - ENVOY_CONN_LOG(debug, "adding custom settings parameter with id {:#x} to {}", connection_, - it.identifier().value(), it.value().value()); - } - - // Insert named parameters. - settings.insert( - settings.end(), - {{NGHTTP2_SETTINGS_HEADER_TABLE_SIZE, http2_options.hpack_table_size().value()}, - {NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL, http2_options.allow_connect()}, - {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, http2_options.max_concurrent_streams().value()}, - {NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, http2_options.initial_stream_window_size().value()}}); - if (!settings.empty()) { - int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, settings.data(), settings.size()); - ASSERT(rc == 0); - } else { - // nghttp2_submit_settings need to be called at least once - int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, nullptr, 0); - ASSERT(rc == 0); - } - - const uint32_t initial_connection_window_size = - http2_options.initial_connection_window_size().value(); - // Increase connection window size up to our default size. - if (initial_connection_window_size != NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE) { - ENVOY_CONN_LOG(debug, "updating connection-level initial window size to {}", connection_, - initial_connection_window_size); - int rc = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0, - initial_connection_window_size - - NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE); - ASSERT(rc == 0); - } -} - -void ConnectionImpl::scheduleProtocolConstraintViolationCallback() { - if (!protocol_constraint_violation_callback_) { - protocol_constraint_violation_callback_ = connection_.dispatcher().createSchedulableCallback( - [this]() { onProtocolConstraintViolation(); }); - protocol_constraint_violation_callback_->scheduleCallbackCurrentIteration(); - } -} - -void ConnectionImpl::onProtocolConstraintViolation() { - // Flooded outbound queue implies that peer is not reading and it does not - // make sense to try to flush pending bytes. - connection_.close(Envoy::Network::ConnectionCloseType::NoFlush); -} - -ConnectionImpl::Http2Callbacks::Http2Callbacks() { - nghttp2_session_callbacks_new(&callbacks_); - nghttp2_session_callbacks_set_send_callback( - callbacks_, - [](nghttp2_session*, const uint8_t* data, size_t length, int, void* user_data) -> ssize_t { - return static_cast(user_data)->onSend(data, length); - }); - - nghttp2_session_callbacks_set_send_data_callback( - callbacks_, - [](nghttp2_session*, nghttp2_frame* frame, const uint8_t* framehd, size_t length, - nghttp2_data_source* source, void*) -> int { - ASSERT(frame->data.padlen == 0); - return static_cast(source->ptr)->onDataSourceSend(framehd, length); - }); - - nghttp2_session_callbacks_set_on_begin_headers_callback( - callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { - return static_cast(user_data)->onBeginHeaders(frame); - }); - - nghttp2_session_callbacks_set_on_header_callback( - callbacks_, - [](nghttp2_session*, const nghttp2_frame* frame, const uint8_t* raw_name, size_t name_length, - const uint8_t* raw_value, size_t value_length, uint8_t, void* user_data) -> int { - // TODO PERF: Can reference count here to avoid copies. - HeaderString name; - name.setCopy(reinterpret_cast(raw_name), name_length); - HeaderString value; - value.setCopy(reinterpret_cast(raw_value), value_length); - return static_cast(user_data)->onHeader(frame, std::move(name), - std::move(value)); - }); - - nghttp2_session_callbacks_set_on_data_chunk_recv_callback( - callbacks_, - [](nghttp2_session*, uint8_t, int32_t stream_id, const uint8_t* data, size_t len, - void* user_data) -> int { - return static_cast(user_data)->onData(stream_id, data, len); - }); - - nghttp2_session_callbacks_set_on_begin_frame_callback( - callbacks_, [](nghttp2_session*, const nghttp2_frame_hd* hd, void* user_data) -> int { - return static_cast(user_data)->onBeforeFrameReceived(hd); - }); - - nghttp2_session_callbacks_set_on_frame_recv_callback( - callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { - return static_cast(user_data)->onFrameReceived(frame); - }); - - nghttp2_session_callbacks_set_on_stream_close_callback( - callbacks_, - [](nghttp2_session*, int32_t stream_id, uint32_t error_code, void* user_data) -> int { - return static_cast(user_data)->onStreamClose(stream_id, error_code); - }); - - nghttp2_session_callbacks_set_on_frame_send_callback( - callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { - return static_cast(user_data)->onFrameSend(frame); - }); - - nghttp2_session_callbacks_set_before_frame_send_callback( - callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { - return static_cast(user_data)->onBeforeFrameSend(frame); - }); - - nghttp2_session_callbacks_set_on_frame_not_send_callback( - callbacks_, [](nghttp2_session*, const nghttp2_frame*, int, void*) -> int { - // We used to always return failure here but it looks now this can get called if the other - // side sends GOAWAY and we are trying to send a SETTINGS ACK. Just ignore this for now. - return 0; - }); - - nghttp2_session_callbacks_set_on_invalid_frame_recv_callback( - callbacks_, - [](nghttp2_session*, const nghttp2_frame* frame, int error_code, void* user_data) -> int { - return static_cast(user_data)->onInvalidFrame(frame->hd.stream_id, - error_code); - }); - - nghttp2_session_callbacks_set_on_extension_chunk_recv_callback( - callbacks_, - [](nghttp2_session*, const nghttp2_frame_hd* hd, const uint8_t* data, size_t len, - void* user_data) -> int { - ASSERT(hd->length >= len); - return static_cast(user_data)->onMetadataReceived(hd->stream_id, data, - len); - }); - - nghttp2_session_callbacks_set_unpack_extension_callback( - callbacks_, [](nghttp2_session*, void**, const nghttp2_frame_hd* hd, void* user_data) -> int { - return static_cast(user_data)->onMetadataFrameComplete( - hd->stream_id, hd->flags == END_METADATA_FLAG); - }); - - nghttp2_session_callbacks_set_pack_extension_callback( - callbacks_, - [](nghttp2_session*, uint8_t* buf, size_t len, const nghttp2_frame* frame, - void* user_data) -> ssize_t { - ASSERT(frame->hd.length <= len); - return static_cast(user_data)->packMetadata(frame->hd.stream_id, buf, len); - }); - - nghttp2_session_callbacks_set_error_callback2( - callbacks_, [](nghttp2_session*, int, const char* msg, size_t len, void* user_data) -> int { - return static_cast(user_data)->onError(absl::string_view(msg, len)); - }); -} - -ConnectionImpl::Http2Callbacks::~Http2Callbacks() { nghttp2_session_callbacks_del(callbacks_); } - -ConnectionImpl::Http2Options::Http2Options( - const envoy::config::core::v3::Http2ProtocolOptions& http2_options) { - nghttp2_option_new(&options_); - // Currently we do not do anything with stream priority. Setting the following option prevents - // nghttp2 from keeping around closed streams for use during stream priority dependency graph - // calculations. This saves a tremendous amount of memory in cases where there are a large - // number of kept alive HTTP/2 connections. - nghttp2_option_set_no_closed_streams(options_, 1); - nghttp2_option_set_no_auto_window_update(options_, 1); - - // The max send header block length is configured to an arbitrarily high number so as to never - // trigger the check within nghttp2, as we check request headers length in - // codec_impl::saveHeader. - nghttp2_option_set_max_send_header_block_length(options_, 0x2000000); - - if (http2_options.hpack_table_size().value() != NGHTTP2_DEFAULT_HEADER_TABLE_SIZE) { - nghttp2_option_set_max_deflate_dynamic_table_size(options_, - http2_options.hpack_table_size().value()); - } - - if (http2_options.allow_metadata()) { - nghttp2_option_set_user_recv_extension_type(options_, METADATA_FRAME_TYPE); - } else { - ENVOY_LOG(trace, "Codec does not have Metadata frame support."); - } - - // nghttp2 v1.39.2 lowered the internal flood protection limit from 10K to 1K of ACK frames. - // This new limit may cause the internal nghttp2 mitigation to trigger more often (as it - // requires just 9K of incoming bytes for smallest 9 byte SETTINGS frame), bypassing the same - // mitigation and its associated behavior in the envoy HTTP/2 codec. Since envoy does not rely - // on this mitigation, set back to the old 10K number to avoid any changes in the HTTP/2 codec - // behavior. - nghttp2_option_set_max_outbound_ack(options_, 10000); -} - -ConnectionImpl::Http2Options::~Http2Options() { nghttp2_option_del(options_); } - -ConnectionImpl::ClientHttp2Options::ClientHttp2Options( - const envoy::config::core::v3::Http2ProtocolOptions& http2_options) - : Http2Options(http2_options) { - // Temporarily disable initial max streams limit/protection, since we might want to create - // more than 100 streams before receiving the HTTP/2 SETTINGS frame from the server. - // - // TODO(PiotrSikora): remove this once multiple upstream connections or queuing are implemented. - nghttp2_option_set_peer_max_concurrent_streams( - options_, ::Envoy::Http2::Utility::OptionsLimits::DEFAULT_MAX_CONCURRENT_STREAMS); -} - -ClientConnectionImpl::ClientConnectionImpl( - Network::Connection& connection, Http::ConnectionCallbacks& callbacks, CodecStats& stats, - Random::RandomGenerator& random, - const envoy::config::core::v3::Http2ProtocolOptions& http2_options, - const uint32_t max_response_headers_kb, const uint32_t max_response_headers_count, - Nghttp2SessionFactory& http2_session_factory) - : ConnectionImpl(connection, stats, random, http2_options, max_response_headers_kb, - max_response_headers_count), - callbacks_(callbacks) { - ClientHttp2Options client_http2_options(http2_options); - session_ = http2_session_factory.create(http2_callbacks_.callbacks(), base(), - client_http2_options.options()); - http2_session_factory.init(session_, base(), http2_options); - allow_metadata_ = http2_options.allow_metadata(); -} - -RequestEncoder& ClientConnectionImpl::newStream(ResponseDecoder& decoder) { - ClientStreamImplPtr stream(new ClientStreamImpl(*this, per_stream_buffer_limit_, decoder)); - // If the connection is currently above the high watermark, make sure to inform the new stream. - // The connection can not pass this on automatically as it has no awareness that a new stream is - // created. - if (connection_.aboveHighWatermark()) { - stream->runHighWatermarkCallbacks(); - } - ClientStreamImpl& stream_ref = *stream; - LinkedList::moveIntoList(std::move(stream), active_streams_); - return stream_ref; -} - -int ClientConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { - // The client code explicitly does not currently support push promise. - RELEASE_ASSERT(frame->hd.type == NGHTTP2_HEADERS, ""); - RELEASE_ASSERT(frame->headers.cat == NGHTTP2_HCAT_RESPONSE || - frame->headers.cat == NGHTTP2_HCAT_HEADERS, - ""); - if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { - StreamImpl* stream = getStream(frame->hd.stream_id); - stream->allocTrailers(); - } - - return 0; -} - -int ClientConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& name, - HeaderString&& value) { - // The client code explicitly does not currently support push promise. - ASSERT(frame->hd.type == NGHTTP2_HEADERS); - ASSERT(frame->headers.cat == NGHTTP2_HCAT_RESPONSE || frame->headers.cat == NGHTTP2_HCAT_HEADERS); - return saveHeader(frame, std::move(name), std::move(value)); -} - -ServerConnectionImpl::ServerConnectionImpl( - Network::Connection& connection, Http::ServerConnectionCallbacks& callbacks, CodecStats& stats, - Random::RandomGenerator& random, - const envoy::config::core::v3::Http2ProtocolOptions& http2_options, - const uint32_t max_request_headers_kb, const uint32_t max_request_headers_count, - envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action) - : ConnectionImpl(connection, stats, random, http2_options, max_request_headers_kb, - max_request_headers_count), - callbacks_(callbacks), headers_with_underscores_action_(headers_with_underscores_action) { - Http2Options h2_options(http2_options); - - nghttp2_session_server_new2(&session_, http2_callbacks_.callbacks(), base(), - h2_options.options()); - sendSettings(http2_options, false); - allow_metadata_ = http2_options.allow_metadata(); -} - -int ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { - // For a server connection, we should never get push promise frames. - ASSERT(frame->hd.type == NGHTTP2_HEADERS); - - if (!trackInboundFrames(&frame->hd, frame->headers.padlen)) { - return NGHTTP2_ERR_FLOODED; - } - - if (frame->headers.cat != NGHTTP2_HCAT_REQUEST) { - stats_.trailers_.inc(); - ASSERT(frame->headers.cat == NGHTTP2_HCAT_HEADERS); - - StreamImpl* stream = getStream(frame->hd.stream_id); - stream->allocTrailers(); - return 0; - } - - ServerStreamImplPtr stream(new ServerStreamImpl(*this, per_stream_buffer_limit_)); - if (connection_.aboveHighWatermark()) { - stream->runHighWatermarkCallbacks(); - } - stream->request_decoder_ = &callbacks_.newStream(*stream); - stream->stream_id_ = frame->hd.stream_id; - LinkedList::moveIntoList(std::move(stream), active_streams_); - nghttp2_session_set_stream_user_data(session_, frame->hd.stream_id, - active_streams_.front().get()); - return 0; -} - -int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& name, - HeaderString&& value) { - // For a server connection, we should never get push promise frames. - ASSERT(frame->hd.type == NGHTTP2_HEADERS); - ASSERT(frame->headers.cat == NGHTTP2_HCAT_REQUEST || frame->headers.cat == NGHTTP2_HCAT_HEADERS); - return saveHeader(frame, std::move(name), std::move(value)); -} - -bool ServerConnectionImpl::trackInboundFrames(const nghttp2_frame_hd* hd, uint32_t padding_length) { - ENVOY_CONN_LOG(trace, "track inbound frame type={} flags={} length={} padding_length={}", - connection_, static_cast(hd->type), static_cast(hd->flags), - static_cast(hd->length), padding_length); - auto result = protocol_constraints_.trackInboundFrames(hd, padding_length); - if (!result.ok()) { - ENVOY_CONN_LOG(trace, "error reading frame: {} received in this HTTP/2 session.", connection_, - result.message()); - if (isInboundFramesWithEmptyPayloadError(result)) { - ConnectionImpl::StreamImpl* stream = getStream(hd->stream_id); - if (stream) { - stream->setDetails(Http2ResponseCodeDetails::get().inbound_empty_frame_flood); - } - } - // NGHTTP2_ERR_FLOODED is overridden within nghttp2 library and it doesn't propagate - // all the way to nghttp2_session_mem_recv() where we need it. - flood_detected_ = true; - return false; - } - - return true; -} - -Envoy::Http::Http2::ProtocolConstraints::ReleasorProc -ServerConnectionImpl::trackOutboundFrames(bool is_outbound_flood_monitored_control_frame) { - auto releasor = - protocol_constraints_.incrementOutboundFrameCount(is_outbound_flood_monitored_control_frame); - if (dispatching_downstream_data_ && !protocol_constraints_.checkOutboundFrameLimits().ok()) { - throw FrameFloodException(std::string(protocol_constraints_.status().message())); - } - return releasor; -} - -void ServerConnectionImpl::checkProtocolConstraintViolation() { - if (!protocol_constraints_.checkOutboundFrameLimits().ok()) { - scheduleProtocolConstraintViolationCallback(); - } -} - -Http::Status ServerConnectionImpl::dispatch(Buffer::Instance& data) { - // TODO(#10878): Remove this wrapper when exception removal is complete. innerDispatch may either - // throw an exception or return an error status. The utility wrapper catches exceptions and - // converts them to error statuses. - return Http::Utility::exceptionToStatus( - [&](Buffer::Instance& data) -> Http::Status { return innerDispatch(data); }, data); -} - -Http::Status ServerConnectionImpl::innerDispatch(Buffer::Instance& data) { - ASSERT(!dispatching_downstream_data_); - dispatching_downstream_data_ = true; - - // Make sure the dispatching_downstream_data_ is set to false even - // when ConnectionImpl::dispatch throws an exception. - Cleanup cleanup([this]() { dispatching_downstream_data_ = false; }); - - // Make sure downstream outbound queue was not flooded by the upstream frames. - if (!protocol_constraints_.checkOutboundFrameLimits().ok()) { - throw FrameFloodException(std::string(protocol_constraints_.status().message())); - } - - return ConnectionImpl::innerDispatch(data); -} - -absl::optional -ServerConnectionImpl::checkHeaderNameForUnderscores(absl::string_view header_name) { - if (headers_with_underscores_action_ != envoy::config::core::v3::HttpProtocolOptions::ALLOW && - Http::HeaderUtility::headerNameContainsUnderscore(header_name)) { - if (headers_with_underscores_action_ == - envoy::config::core::v3::HttpProtocolOptions::DROP_HEADER) { - ENVOY_CONN_LOG(debug, "Dropping header with invalid characters in its name: {}", connection_, - header_name); - stats_.dropped_headers_with_underscores_.inc(); - return 0; - } - ENVOY_CONN_LOG(debug, "Rejecting request due to header name with underscores: {}", connection_, - header_name); - stats_.requests_rejected_with_underscores_in_headers_.inc(); - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; - } - return absl::nullopt; -} - -} // namespace Http2 -} // namespace Legacy -} // namespace Http -} // namespace Envoy From 2632163e3cf6f7d4638e45bb923206e56d5ba249 Mon Sep 17 00:00:00 2001 From: grial1 Date: Mon, 21 Dec 2020 21:39:15 +0000 Subject: [PATCH 103/106] requested changes added Signed-off-by: grial1 --- source/common/common/safe_memcpy.h | 16 +++++++++++----- .../proxy_protocol/proxy_protocol_header.cc | 10 ++-------- test/common/common/safe_memcpy_test.cc | 13 ++++++------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/source/common/common/safe_memcpy.h b/source/common/common/safe_memcpy.h index 575513393c619..624998e763ddf 100644 --- a/source/common/common/safe_memcpy.h +++ b/source/common/common/safe_memcpy.h @@ -5,7 +5,9 @@ namespace Envoy { /** - * @brief Copies src to dst based on their sizes, which must be the same. + * Copies src to dst based on their sizes, which must be the same. + * @param dst the destination + * @param src the source */ template inline void safeMemcpy(T1* dst, T2* src) { static_assert(sizeof(T1) == sizeof(T2)); @@ -13,17 +15,21 @@ template inline void safeMemcpy(T1* dst, T2* src) { } /** - * @brief Copies src to dst based on the size of dst - * @note Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to + * Copies src to dst based on the size of dst. + * Sizes are not compared, so ensure the src is of size sizeof(*(dst)) before proceeding to * call safeMemcpyUnsafeSrc + * @param dst the destination + * @param src the source */ template inline void safeMemcpyUnsafeSrc(T1* dst, T2* src) { memcpy(dst, src, sizeof(T1)); } /** - * @brief Copies src to dst based on the size of src - * @note Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to + * Copies src to dst based on the size of src. + * Sizes are not compared, so ensure the dst is of size sizeof(*(src)) before proceeding to * call safeMemcpyUnsafeDst + * @param dst the destination + * @param src the source */ template inline void safeMemcpyUnsafeDst(T1* dst, T2* src) { memcpy(dst, src, sizeof(T2)); diff --git a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc index 14ac11e460073..c61aff8e3807b 100644 --- a/source/extensions/common/proxy_protocol/proxy_protocol_header.cc +++ b/source/extensions/common/proxy_protocol/proxy_protocol_header.cc @@ -81,14 +81,8 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr, Network::Address::Ipv6Instance(src_addr, src_port).ip()->ipv6()->address(); const absl::uint128 net_dst_addr = Network::Address::Ipv6Instance(dst_addr, dst_port).ip()->ipv6()->address(); - const uint64_t net_src_addr_lo = absl::Uint128Low64(net_src_addr); - out.add(&net_src_addr_lo, 8); - const uint64_t net_src_addr_hi = absl::Uint128High64(net_src_addr); - out.add(&net_src_addr_hi, 8); - const uint64_t net_dst_addr_lo = absl::Uint128Low64(net_dst_addr); - out.add(&net_dst_addr_lo, 8); - const uint64_t net_dst_addr_hi = absl::Uint128High64(net_dst_addr); - out.add(&net_dst_addr_hi, 8); + out.add(&net_src_addr, 16); + out.add(&net_dst_addr, 16); break; } } diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index f90cfed179e14..464709d355332 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -11,7 +11,6 @@ #include "gtest/gtest.h" using testing::ElementsAre; -using testing::Eq; namespace Envoy { @@ -19,7 +18,7 @@ TEST(SafeMemcpyTest, CopyUint8) { const uint8_t src[] = {0, 1, 1, 2, 3, 5, 8, 13}; uint64_t dst; safeMemcpy(&dst, &src); - Eq(dst == 282583128934413); + EXPECT_EQ(dst, 282583128934413); } TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { @@ -29,20 +28,20 @@ TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { } uint8_t dst[8]; safeMemcpyUnsafeSrc(&dst, src); - ASSERT_THAT(dst, ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); + EXPECT_THAT(dst, ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); delete[] src; } TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { - auto buffer = std::make_unique(); - buffer->add("test", 4); + Buffer::OwnedImpl buffer; + buffer.add("test", 4); std::array expected_header; expected_header[0] = 0; // flags const uint32_t nsize = htonl(4); safeMemcpyUnsafeDst(&expected_header[1], &nsize); std::string header_string(&expected_header[0], 5); - Grpc::Common::prependGrpcFrameHeader(*buffer); - EXPECT_EQ(buffer->toString(), header_string + "test"); + Grpc::Common::prependGrpcFrameHeader(buffer); + EXPECT_EQ(buffer.toString(), header_string + "test"); } } // namespace Envoy From 6454e2c67ba80d0a863980f33abbaa8177052171 Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 22 Dec 2020 17:25:15 +0000 Subject: [PATCH 104/106] SafeMemcpyUnsafeDstTest modified to use basic C++ Signed-off-by: grial1 --- test/common/common/safe_memcpy_test.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index 464709d355332..c3ac4987149ce 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -33,15 +33,16 @@ TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { } TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { - Buffer::OwnedImpl buffer; - buffer.add("test", 4); - std::array expected_header; - expected_header[0] = 0; // flags - const uint32_t nsize = htonl(4); - safeMemcpyUnsafeDst(&expected_header[1], &nsize); - std::string header_string(&expected_header[0], 5); - Grpc::Common::prependGrpcFrameHeader(buffer); - EXPECT_EQ(buffer.toString(), header_string + "test"); + uint8_t* dst = new uint8_t[8]; + uint8_t src[8]; + for (int i = 0; i < 8; ++i) { + src[i] = i; + } + safeMemcpyUnsafeSrc(dst, &src); + for (int i = 0; i < 8; ++i) { + EXPECT_THAT(dst[i], i); + } + delete[] dst; } } // namespace Envoy From 6d6aba8332cc10bf75978ae5c9221d5d48969d96 Mon Sep 17 00:00:00 2001 From: grial1 Date: Tue, 22 Dec 2020 23:25:08 +0000 Subject: [PATCH 105/106] safe_memcpy_test fixed Signed-off-by: grial1 --- test/common/common/safe_memcpy_test.cc | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/test/common/common/safe_memcpy_test.cc b/test/common/common/safe_memcpy_test.cc index c3ac4987149ce..bbe8010205ec8 100644 --- a/test/common/common/safe_memcpy_test.cc +++ b/test/common/common/safe_memcpy_test.cc @@ -1,4 +1,4 @@ -#include +#include #include "common/common/safe_memcpy.h" #include "common/grpc/common.h" @@ -16,9 +16,9 @@ namespace Envoy { TEST(SafeMemcpyTest, CopyUint8) { const uint8_t src[] = {0, 1, 1, 2, 3, 5, 8, 13}; - uint64_t dst; + uint8_t dst[8]; safeMemcpy(&dst, &src); - EXPECT_EQ(dst, 282583128934413); + EXPECT_THAT(dst, ElementsAre(0, 1, 1, 2, 3, 5, 8, 13)); } TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { @@ -33,15 +33,11 @@ TEST(SafeMemcpyUnsafeSrcTest, CopyUint8Pointer) { } TEST(SafeMemcpyUnsafeDstTest, PrependGrpcFrameHeader) { - uint8_t* dst = new uint8_t[8]; - uint8_t src[8]; - for (int i = 0; i < 8; ++i) { - src[i] = i; - } - safeMemcpyUnsafeSrc(dst, &src); - for (int i = 0; i < 8; ++i) { - EXPECT_THAT(dst[i], i); - } + const uint8_t src[] = {1, 2, 3, 4}; + uint8_t* dst = new uint8_t[4]; + memset(dst, 0, 4); + safeMemcpyUnsafeDst(dst, &src); + EXPECT_THAT(std::vector(dst, dst + sizeof(src)), ElementsAre(1, 2, 3, 4)); delete[] dst; } From 1151c7ae33b0acad272dad1af625511aa6bb2776 Mon Sep 17 00:00:00 2001 From: grial1 Date: Sat, 9 Jan 2021 21:17:44 +0000 Subject: [PATCH 106/106] Tests coverage acceptable threshold reduced by 0.1 for source/extensions/filters/network/mongo_proxy Signed-off-by: grial1 --- test/per_file_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 174c0cd149910..b07f0fbc23d40 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -45,7 +45,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/filters/network/common/redis:96.2" "source/extensions/filters/network/dubbo_proxy:96.1" "source/extensions/filters/network/dubbo_proxy/router:95.1" -"source/extensions/filters/network/mongo_proxy:94.1" +"source/extensions/filters/network/mongo_proxy:94.0" "source/extensions/filters/network/sni_cluster:90.3" "source/extensions/filters/network/sni_dynamic_forward_proxy:90.9" "source/extensions/health_checkers:95.9"