From df3db4bf83d18f76b8b14253ae34997c21288dda Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Fri, 1 Feb 2019 17:27:35 -0800 Subject: [PATCH 001/114] ZooKeeper proxy filter This filter decodes the ZooKeeper wire protocol and emits stats & metadata about requests, responses and events. This wire protocol parsing is based on: https://github.com/twitter/zktraffic https://github.com/rgs1/zktraffic-cpp The actual filter structure is based on the Mysql proxy filter. Signed-off-by: Raul Gutierrez Segales --- CODEOWNERS | 2 + .../network/zookeeper_proxy/v1alpha1/BUILD | 8 + .../v1alpha1/zookeeper_proxy.proto | 19 ++ .../network_filters/network_filters.rst | 1 + .../zookeeper_proxy_filter.rst | 83 ++++++ .../well_known_dynamic_metadata.rst | 1 + docs/root/intro/version_history.rst | 2 + source/extensions/extensions_build_config.bzl | 2 + .../filters/network/well_known_names.h | 2 + .../filters/network/zookeeper_proxy/BUILD | 47 +++ .../zookeeper_proxy/zookeeper_config.cc | 45 +++ .../zookeeper_proxy/zookeeper_config.h | 33 +++ .../zookeeper_proxy/zookeeper_decoder.cc | 199 +++++++++++++ .../zookeeper_proxy/zookeeper_decoder.h | 99 +++++++ .../zookeeper_proxy/zookeeper_filter.cc | 110 +++++++ .../zookeeper_proxy/zookeeper_filter.h | 116 ++++++++ .../zookeeper_proxy/zookeeper_utils.cc | 53 ++++ .../network/zookeeper_proxy/zookeeper_utils.h | 28 ++ .../filters/network/zookeeper_proxy/BUILD | 28 ++ .../zookeeper_proxy/zookeeper_filter_test.cc | 271 ++++++++++++++++++ 20 files changed, 1149 insertions(+) create mode 100644 api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD create mode 100644 api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto create mode 100644 docs/root/configuration/network_filters/zookeeper_proxy_filter.rst create mode 100644 source/extensions/filters/network/zookeeper_proxy/BUILD create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc create mode 100644 source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h create mode 100644 test/extensions/filters/network/zookeeper_proxy/BUILD create mode 100644 test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index abdf25c16f860..bf61be634b6e0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -22,3 +22,5 @@ /*/extensions/filters/network/mysql_proxy @rshriram @venilnoronha @mattklein123 # quic extension /*/extensions/quic_listeners/ @alyssawilk @danzh2010 @mattklein123 @mpwarres @wu-bin +# zookeeper_proxy extension +/*/extensions/filters/network/zookeeper_proxy @rgs1 diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD new file mode 100644 index 0000000000000..a29ebf3a88484 --- /dev/null +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD @@ -0,0 +1,8 @@ +load("//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "zookeeper_proxy", + srcs = ["zookeeper_proxy.proto"], +) diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto new file mode 100644 index 0000000000000..58f9389ea3e99 --- /dev/null +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package envoy.config.filter.network.zookeeper_proxy.v1alpha1; +option java_package = "io.envoyproxy.envoy.config.filter.network.zookeeper_proxy.v1alpha1"; +option go_package = "v1alpha1"; + +import "validate/validate.proto"; + +// [#protodoc-title: ZooKeeper proxy] +// ZooKeeper Proxy :ref:`configuration overview `. +message ZooKeeperProxy { + // The human readable prefix to use when emitting :ref:`statistics + // `. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // [#not-implemented-hide:] The optional path to use for writing ZooKeeper access logs. + // If the access log field is empty, access logs will not be written. + string access_log = 2; +} diff --git a/docs/root/configuration/network_filters/network_filters.rst b/docs/root/configuration/network_filters/network_filters.rst index dd559ddd66890..91693bc40ab05 100644 --- a/docs/root/configuration/network_filters/network_filters.rst +++ b/docs/root/configuration/network_filters/network_filters.rst @@ -21,3 +21,4 @@ filters. tcp_proxy_filter thrift_proxy_filter sni_cluster_filter + zookeeper_proxy_filter diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst new file mode 100644 index 0000000000000..9756c5dc26e48 --- /dev/null +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -0,0 +1,83 @@ +.. _config_network_filters_zookeeper_proxy: + +ZooKeeper proxy +=============== + +The ZooKeeper proxy filter decodes the wire protocol between the ZooKeeper client +and server. It decodes the requests, responses and events in the payload. +The decoded info is emitted as dynamic metadata that can be combined with +access log filters to get detailed information on paths that are accessed +as well as the requests that are performed on each path. + +.. attention:: + + The zookeeper_proxy filter is experimental and is currently under active + development. Capabilities will be expanded over time and the + configuration structures are likely to change. + +.. _config_network_filters_zookeeper_proxy_config: + +Configuration +------------- + +The ZooKeeper proxy filter should be chained with the TCP proxy filter as shown +in the configuration snippet below: + +.. code-block:: yaml + + filter_chains: + - filters: + - name: envoy.filters.network.zookeeper_proxy + config: + stat_prefix: zookeeper + - name: envoy.tcp_proxy + config: + stat_prefix: tcp + cluster: ... + + +.. _config_network_filters_zookeeper_proxy_stats: + +Statistics +---------- + +Every configured ZooKeeper proxy filter has statistics rooted at *zookeeper..* with the +following statistics: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + decoder_error, Counter, Number of times a message wasn't decoded + connect, Counter, Number of connect requests + connect_readonly, Counter, Number of connect requests with the readonly flag set + ping, Counter, Number of ping requests + auth._rq, Counter, Number of auth requests for a given type + getdata_rq, Counter, Number of getdata requests + create_rq, Counter, Number of create requests + create2_rq, Counter, Number of create2 requests + setdata_rq, Counter, Number of setdata requests + getchildren_rq, Counter, Number of getchildren requests + getchildren2, Counter, Number of getchildren2 requests + remove, Counter, Number of delete requests + exists, Counter, Number of stat requests + getacl, Counter, Number of getacl requests + setacl, Counter, Number of setacl requests + sync, Counter, Number of sync requests + multi, Counter, Number of multi transaction requests + reconfig, Counter, Number of reconfig requests + close, Counter, Number of close requests + setwatches, Counter, Number of setwatches requests + +.. _config_network_filters_zookeeper_proxy_dynamic_metadata: + +Dynamic Metadata +---------------- + +The ZooKeeper filter emits the following dynamic metadata for each message parsed: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + , string, The path associated with the request, response or event. diff --git a/docs/root/configuration/well_known_dynamic_metadata.rst b/docs/root/configuration/well_known_dynamic_metadata.rst index dd11866a42a02..73215617e46db 100644 --- a/docs/root/configuration/well_known_dynamic_metadata.rst +++ b/docs/root/configuration/well_known_dynamic_metadata.rst @@ -17,3 +17,4 @@ The following Envoy filters emit dynamic metadata that other filters can leverag * :ref:`MySQL Proxy Filter ` * :ref:`Role Based Access Control (RBAC) Filter ` * :ref:`Role Based Access Control (RBAC) Network Filter ` +* :ref:`ZooKeeper Proxy Filter ` diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index f6e6d436beea3..98b150b8f447b 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -46,6 +46,8 @@ Version history * upstream: add hash_function to specify the hash function for :ref:`ring hash` as either xxHash or `murmurHash2 `_. MurmurHash2 is compatible with std::hash in GNU libstdc++ 3.4.20 or above. This is typically the case when compiled on Linux and not macOS. * upstream: added :ref:`degraded health value` which allows routing to certain hosts only when there are insufficient healthy hosts available. +* zookeeper: added a ZooKeeper proxy filter that parses ZooKeeper messages (requests/responses/events). + Refer to ::ref:`ZooKeeper proxy` for more details. 1.9.0 (Dec 20, 2018) ==================== diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index bd3793cc9d16a..f973d63fad0a7 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -77,6 +77,7 @@ EXTENSIONS = { "envoy.filters.network.tcp_proxy": "//source/extensions/filters/network/tcp_proxy:config", "envoy.filters.network.thrift_proxy": "//source/extensions/filters/network/thrift_proxy:config", "envoy.filters.network.sni_cluster": "//source/extensions/filters/network/sni_cluster:config", + "envoy.filters.network.zookeeper_proxy": "//source/extensions/filters/network/zookeeper_proxy:config", # # Resource monitors @@ -194,6 +195,7 @@ WINDOWS_EXTENSIONS = { "envoy.filters.network.tcp_proxy": "//source/extensions/filters/network/tcp_proxy:config", #"envoy.filters.network.thrift_proxy": "//source/extensions/filters/network/thrift_proxy:config", #"envoy.filters.network.sni_cluster": "//source/extensions/filters/network/sni_cluster:config", + #"envoy.filters.network.zookeeper_proxy": "//source/extensions/filters/network/zookeeper_proxy:config", # # Stat sinks diff --git a/source/extensions/filters/network/well_known_names.h b/source/extensions/filters/network/well_known_names.h index f540aab0d36f7..a1d435f4e7b2a 100644 --- a/source/extensions/filters/network/well_known_names.h +++ b/source/extensions/filters/network/well_known_names.h @@ -38,6 +38,8 @@ class NetworkFilterNameValues { const std::string Rbac = "envoy.filters.network.rbac"; // SNI Cluster filter const std::string SniCluster = "envoy.filters.network.sni_cluster"; + // ZooKeeper proxy filter + const std::string ZooKeeperProxy = "envoy.filters.network.zookeeper_proxy"; // Converts names from v1 to v2 const Config::V1Converter v1_converter_; diff --git a/source/extensions/filters/network/zookeeper_proxy/BUILD b/source/extensions/filters/network/zookeeper_proxy/BUILD new file mode 100644 index 0000000000000..4fae6bda72674 --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/BUILD @@ -0,0 +1,47 @@ +licenses(["notice"]) # Apache 2 + +# ZooKeeper proxy L7 network filter. +# Public docs: docs/root/configuration/network_filters/zookeeper_proxy_filter.rst + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "proxy_lib", + srcs = [ + "zookeeper_decoder.cc", + "zookeeper_filter.cc", + "zookeeper_utils.cc", + ], + hdrs = [ + "zookeeper_decoder.h", + "zookeeper_filter.h", + "zookeeper_utils.h", + ], + deps = [ + "//include/envoy/network:filter_interface", + "//include/envoy/server:filter_config_interface", + "//include/envoy/stats:stats_interface", + "//include/envoy/stats:stats_macros", + "//source/common/config:filter_json_lib", + "//source/common/network:filter_lib", + "//source/extensions/filters/network:well_known_names", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["zookeeper_config.cc"], + hdrs = ["zookeeper_config.h"], + deps = [ + ":proxy_lib", + "//source/extensions/filters/network:well_known_names", + "//source/extensions/filters/network/common:factory_base_lib", + "@envoy_api//envoy/config/filter/network/zookeeper_proxy/v1alpha1:zookeeper_proxy_cc", + ], +) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc new file mode 100644 index 0000000000000..ef145cee08387 --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc @@ -0,0 +1,45 @@ +#include "extensions/filters/network/zookeeper_proxy/zookeeper_config.h" + +#include + +#include "envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.pb.validate.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "common/common/logger.h" + +#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +/** + * Config registration for the ZooKeeper proxy filter. @see NamedNetworkFilterConfigFactory. + */ +Network::FilterFactoryCb +NetworkFilters::ZooKeeperProxy::ZooKeeperConfigFactory::createFilterFactoryFromProtoTyped( + const envoy::config::filter::network::zookeeper_proxy::v1alpha1::ZooKeeperProxy& proto_config, + Server::Configuration::FactoryContext& context) { + + ASSERT(!proto_config.stat_prefix().empty()); + + const std::string stat_prefix = fmt::format("{}.zookeeper.", proto_config.stat_prefix()); + + ZooKeeperFilterConfigSharedPtr filter_config( + std::make_shared(stat_prefix, context.scope())); + return [filter_config](Network::FilterManager& filter_manager) -> void { + filter_manager.addFilter(std::make_shared(filter_config)); + }; +} + +/** + * Static registration for the ZooKeeper proxy filter. @see RegisterFactory. + */ +REGISTER_FACTORY(ZooKeeperConfigFactory, Server::Configuration::NamedNetworkFilterConfigFactory); + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h new file mode 100644 index 0000000000000..76e64bd104f75 --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.pb.h" +#include "envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.pb.validate.h" + +#include "extensions/filters/network/common/factory_base.h" +#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" +#include "extensions/filters/network/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +/** + * Config registration for the ZooKeeper proxy filter. + */ +class ZooKeeperConfigFactory + : public Common::FactoryBase< + envoy::config::filter::network::zookeeper_proxy::v1alpha1::ZooKeeperProxy> { +public: + ZooKeeperConfigFactory() : FactoryBase(NetworkFilterNames::get().ZooKeeperProxy) {} + +private: + Network::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::config::filter::network::zookeeper_proxy::v1alpha1::ZooKeeperProxy& proto_config, + Server::Configuration::FactoryContext& context) override; +}; + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc new file mode 100644 index 0000000000000..f42fb4aa5ac4a --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -0,0 +1,199 @@ +#include "extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h" + +#include "extensions/filters/network/zookeeper_proxy/zookeeper_utils.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { + ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); + + // Check message length. + int32_t len; + BufferHelper::peekInt32(data, offset, len); + if (len < 8) { + callbacks_.onDecodeError(); + return; + } + + int32_t xid; + BufferHelper::peekInt32(data, offset, xid); + + // "Special" requests. + switch (xid) { + case CONNECT_XID: + parseConnect(data, offset, len); + return; + case PING_XID: + callbacks_.onPing(); + return; + case AUTH_XID: + parseAuthRequest(data, offset, len); + return; + default: + break; + } + + // "Regular" requests. + int32_t opcode; + BufferHelper::peekInt32(data, offset, opcode); + switch (opcode) { + case enumToInt(Opcodes::GETDATA): + parseGetDataRequest(data, offset, len); + break; + case enumToInt(Opcodes::CREATE): + parseCreateRequest(data, offset, len, false); + break; + case enumToInt(Opcodes::CREATE2): + parseCreateRequest(data, offset, len, true); + break; + case enumToInt(Opcodes::SETDATA): + parseSetRequest(data, offset, len); + break; + case enumToInt(Opcodes::GETCHILDREN): + parseGetChildrenRequest(data, offset, len, false); + break; + case enumToInt(Opcodes::GETCHILDREN2): + parseGetChildrenRequest(data, offset, len, true); + break; + default: + break; + } +} + +void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + bool readonly{}; + + if (len < 28) { + return; + } + + // Read password - skip zxid, timeout, and session id. + std::string passwd; + offset += 20; + BufferHelper::peekString(data, offset, passwd); + + // Read readonly flag, if it's there. + BufferHelper::peekBool(data, offset, readonly); + + callbacks_.onConnect(readonly); +} + +void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + if (len < 20) { + return; + } + + // Skip opcode & type. + std::string scheme; + offset += 8; + BufferHelper::peekString(data, offset, scheme); + std::string credential; + BufferHelper::peekString(data, offset, credential); + + callbacks_.onAuthRequest(scheme); +} + +void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + if (len < 14) { + return; + } + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + bool watch; + BufferHelper::peekBool(data, offset, watch); + + callbacks_.onGetDataRequest(path, watch); +} + +void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { + int32_t count; + BufferHelper::peekInt32(data, offset, count); + + for (int i = 0; i < count; ++i) { + int32_t perms; + BufferHelper::peekInt32(data, offset, perms); + std::string scheme; + BufferHelper::peekString(data, offset, scheme); + std::string credential; + BufferHelper::peekString(data, offset, credential); + } +} + +void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + const bool two) { + if (len < 25) { + return; + } + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + // Skip data. + int32_t datalen; + BufferHelper::peekInt32(data, offset, datalen); + offset += datalen; + // Skip acls. + skipAcls(data, offset); + // Flags. + int32_t flags; + BufferHelper::peekInt32(data, offset, flags); + const bool ephemeral = (flags & 0x1) == 1; + const bool sequence = (flags & 0x2) == 2; + + callbacks_.onCreateRequest(path, ephemeral, sequence, two); +} + +void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + if (len < 20) { + return; + } + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + // Skip data. + int32_t datalen; + BufferHelper::peekInt32(data, offset, datalen); + offset += datalen; + // Version. + int32_t version; + BufferHelper::peekInt32(data, offset, version); + + callbacks_.onSetRequest(path); +} + +void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + const bool two) { + if (len < 14) { + return; + } + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + bool watch; + BufferHelper::peekBool(data, offset, watch); + + callbacks_.onGetChildrenRequest(path, watch, two); +} + +void DecoderImpl::onData(Buffer::Instance& data) { + uint64_t offset = 0; + while (offset < data.length()) { + decode(data, offset); + } +} + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h new file mode 100644 index 0000000000000..749b636f142e1 --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -0,0 +1,99 @@ +#pragma once +#include + +#include "envoy/common/platform.h" + +#include "common/buffer/buffer_impl.h" +#include "common/common/logger.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +const int CONNECT_XID = 0; +const int WATCH_XID = -1; +const int PING_XID = -2; +const int AUTH_XID = -4; +const int SET_WATCHES_XID = -8; + +template constexpr int enumToInt(T val) { return static_cast(val); } + +enum class Opcodes { + CONNECT = 0, + CREATE = 1, + DELETE = 2, + EXISTS = 3, + GETDATA = 4, + SETDATA = 5, + GETACL = 6, + SETACL = 7, + GETCHILDREN = 8, + SYNC = 9, + PING = 11, + GETCHILDREN2 = 12, + CHECK = 13, + MULTI = 14, + CREATE2 = 15, + RECONFIG = 16, + CREATESESSION = -10, + CLOSE = -11, + SETAUTH = 100, + SETWATCHES = 101 +}; + +/** + * General callbacks for dispatching decoded ZooKeeper messages to a sink. + */ +class DecoderCallbacks { +public: + virtual ~DecoderCallbacks() {} + + virtual void onDecodeError() PURE; + virtual void onConnect(const bool readonly) PURE; + virtual void onPing() PURE; + virtual void onAuthRequest(const std::string& scheme) PURE; + virtual void onGetDataRequest(const std::string& path, const bool watch) PURE; + virtual void onCreateRequest(const std::string& path, const bool ephemeral, const bool sequence, + const bool two) PURE; + virtual void onSetRequest(const std::string& path) PURE; + virtual void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) PURE; +}; + +/** + * ZooKeeper message decoder. + */ +class Decoder { +public: + virtual ~Decoder() {} + + virtual void onData(Buffer::Instance& data) PURE; +}; + +typedef std::unique_ptr DecoderPtr; + +class DecoderImpl : public Decoder, Logger::Loggable { +public: + DecoderImpl(DecoderCallbacks& callbacks) : callbacks_(callbacks) {} + + // ZooKeeperProxy::Decoder + void onData(Buffer::Instance& data) override; + +private: + void decode(Buffer::Instance& data, uint64_t& offset); + void parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two); + void skipAcls(Buffer::Instance& data, uint64_t& offset) const; + void parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + const bool two); + + DecoderCallbacks& callbacks_; +}; + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc new file mode 100644 index 0000000000000..b255dd353c8bb --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -0,0 +1,110 @@ +#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" + +#include "common/buffer/buffer_impl.h" +#include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/common/logger.h" + +#include "extensions/filters/network/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, Stats::Scope& scope) + : scope_(scope), stat_prefix_(stat_prefix), stats_(generateStats(stat_prefix, scope)) {} + +ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config) + : config_(std::move(config)) {} + +void ZooKeeperFilter::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) { + read_callbacks_ = &callbacks; +} + +Network::FilterStatus ZooKeeperFilter::onData(Buffer::Instance& data, bool) { + doDecode(data); + return Network::FilterStatus::Continue; +} + +Network::FilterStatus ZooKeeperFilter::onWrite(Buffer::Instance& data, bool) { + doDecode(data); + return Network::FilterStatus::Continue; +} + +Network::FilterStatus ZooKeeperFilter::onNewConnection() { return Network::FilterStatus::Continue; } + +void ZooKeeperFilter::doDecode(Buffer::Instance& buffer) { + // Clear dynamic metadata. + envoy::api::v2::core::Metadata& dynamic_metadata = + read_callbacks_->connection().streamInfo().dynamicMetadata(); + auto& metadata = + (*dynamic_metadata.mutable_filter_metadata())[NetworkFilterNames::get().ZooKeeperProxy]; + metadata.mutable_fields()->clear(); + + if (!decoder_) { + decoder_ = createDecoder(*this); + } + + try { + decoder_->onData(buffer); + } catch (EnvoyException& e) { + ENVOY_LOG(info, "zookeeper_proxy: decoding error: {}", e.what()); + config_->stats_.decoder_error_.inc(); + } +} + +DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks) { + return std::make_unique(callbacks); +} + +void ZooKeeperFilter::onConnect(const bool readonly) { + if (readonly) { + config_->stats_.connect_readonly_.inc(); + } else { + config_->stats_.connect_.inc(); + } + + // TODO: set dynamic metadata. +} + +void ZooKeeperFilter::onDecodeError() { config_->stats_.decoder_error_.inc(); } + +void ZooKeeperFilter::onPing() { config_->stats_.ping_.inc(); } + +void ZooKeeperFilter::onAuthRequest(const std::string& scheme) { + config_->scope_.counter(fmt::format("{}.auth.{}_rq", config_->stat_prefix_, scheme)).inc(); +} + +void ZooKeeperFilter::onGetDataRequest(const std::string&, const bool) { + config_->stats_.getdata_rq_.inc(); + // TODO: set metadata. +} + +void ZooKeeperFilter::onCreateRequest(const std::string&, const bool, const bool, const bool two) { + if (!two) { + config_->stats_.create_rq_.inc(); + } else { + config_->stats_.create2_rq_.inc(); + } + // TODO: set metadata. +} + +void ZooKeeperFilter::onSetRequest(const std::string&) { + config_->stats_.setdata_rq_.inc(); + // TODO: set metadata. +} + +void ZooKeeperFilter::onGetChildrenRequest(const std::string&, const bool, const bool two) { + if (!two) { + config_->stats_.getchildren_rq_.inc(); + } else { + config_->stats_.getchildren2_rq_.inc(); + } + // TODO: set metadata. +} + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h new file mode 100644 index 0000000000000..d037155e20e7f --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -0,0 +1,116 @@ +#pragma once + +#include "envoy/access_log/access_log.h" +#include "envoy/network/connection.h" +#include "envoy/network/filter.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats.h" +#include "envoy/stats/stats_macros.h" + +#include "common/common/logger.h" + +#include "extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +/** + * All ZooKeeper proxy stats. @see stats_macros.h + */ +// clang-format off +#define ALL_ZOOKEEPER_PROXY_STATS(COUNTER) \ + COUNTER(decoder_error) \ + COUNTER(connect) \ + COUNTER(connect_readonly) \ + COUNTER(getdata_rq) \ + COUNTER(create_rq) \ + COUNTER(create2_rq) \ + COUNTER(setdata_rq) \ + COUNTER(getchildren_rq) \ + COUNTER(getchildren2_rq) \ + COUNTER(remove) \ + COUNTER(exists) \ + COUNTER(getacl) \ + COUNTER(setacl) \ + COUNTER(sync) \ + COUNTER(ping) \ + COUNTER(multi) \ + COUNTER(create2) \ + COUNTER(reconfig) \ + COUNTER(createsession) \ + COUNTER(close) \ + COUNTER(setauth) \ + COUNTER(setwatches) \ +// clang-format on + +/** + * Struct definition for all ZooKeeper proxy stats. @see stats_macros.h + */ +struct ZooKeeperProxyStats { + ALL_ZOOKEEPER_PROXY_STATS(GENERATE_COUNTER_STRUCT) +}; + +/** + * Configuration for the ZooKeeper proxy filter. + */ +class ZooKeeperFilterConfig { +public: + ZooKeeperFilterConfig(const std::string &stat_prefix, Stats::Scope& scope); + + const ZooKeeperProxyStats& stats() { return stats_; } + + Stats::Scope& scope_; + const std::string stat_prefix_; + ZooKeeperProxyStats stats_; + +private: + ZooKeeperProxyStats generateStats(const std::string& prefix, + Stats::Scope& scope) { + return ZooKeeperProxyStats{ + ALL_ZOOKEEPER_PROXY_STATS(POOL_COUNTER_PREFIX(scope, prefix))}; + } +}; + +using ZooKeeperFilterConfigSharedPtr = std::shared_ptr; + +/** + * Implementation of ZooKeeper proxy filter. + */ +class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggable { +public: + ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config); + ~ZooKeeperFilter() override = default; + + // Network::ReadFilter + Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; + Network::FilterStatus onNewConnection() override; + void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override; + + // Network::WriteFilter + Network::FilterStatus onWrite(Buffer::Instance& data, bool end_stream) override; + + // ZooKeeperProxy::DecoderCallback + void onDecodeError() override; + void onConnect(const bool readonly) override; + void onPing() override; + void onAuthRequest(const std::string& scheme) override; + void onGetDataRequest(const std::string& path, const bool watch) override; + void onCreateRequest(const std::string& path, const bool ephemeral, const bool sequence, const bool two) override; + void onSetRequest(const std::string& path) override; + void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) override; + + void doDecode(Buffer::Instance& buffer); + DecoderPtr createDecoder(DecoderCallbacks& callbacks); + +private: + Network::ReadFilterCallbacks* read_callbacks_{}; + ZooKeeperFilterConfigSharedPtr config_; + std::unique_ptr decoder_; +}; + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc new file mode 100644 index 0000000000000..32388c7acff2b --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -0,0 +1,53 @@ +#include "extensions/filters/network/zookeeper_proxy/zookeeper_utils.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +bool BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val) { + try { + val = buffer.peekBEInt(offset); + offset += sizeof(uint32_t); + return true; + } catch (EnvoyException& e) { + return false; + } +} + +bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val) { + if (buffer.length() < (offset + 1)) { + return false; + } + + try { + std::string str; + str.assign(std::string(static_cast(buffer.linearize(1 + offset)), 1 + offset)); + str = str.substr(offset); + val = static_cast(str[0] & 255); + offset += 1; + return true; + } catch (EnvoyException& e) { + return false; + } +} + +bool BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str) { + int32_t len = 0; + peekInt32(buffer, offset, len); + + if (buffer.length() < (offset + len) || !len) { + return false; + } + + str.assign(std::string(static_cast(buffer.linearize(len + offset)), len + offset)); + str = str.substr(offset); + offset += len; + + return true; +} + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h new file mode 100644 index 0000000000000..8c0ecb2fb696b --- /dev/null +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -0,0 +1,28 @@ +#pragma once +#include + +#include "envoy/common/platform.h" + +#include "common/buffer/buffer_impl.h" +#include "common/common/byte_order.h" +#include "common/common/logger.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +/** + * IO helpers for reading/writing ZooKeeper data from/to a buffer. + */ +class BufferHelper : public Logger::Loggable { +public: + static bool peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val); + static bool peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str); + static bool peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val); +}; + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/zookeeper_proxy/BUILD b/test/extensions/filters/network/zookeeper_proxy/BUILD new file mode 100644 index 0000000000000..8aee34eec3e00 --- /dev/null +++ b/test/extensions/filters/network/zookeeper_proxy/BUILD @@ -0,0 +1,28 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_mock", + "envoy_extension_cc_test", + "envoy_extension_cc_test_library", +) + +envoy_package() + +envoy_extension_cc_test( + name = "zookeeper_filter_test", + srcs = [ + "zookeeper_filter_test.cc", + ], + extension_name = "envoy.filters.network.zookeeper_proxy", + deps = [ + "//source/extensions/filters/network/zookeeper_proxy:config", + "//test/mocks/network:network_mocks", + ], +) + diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc new file mode 100644 index 0000000000000..f4ec166e15a29 --- /dev/null +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -0,0 +1,271 @@ +#include "common/buffer/buffer_impl.h" + +#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" +#include "extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h" + +#include "test/mocks/network/mocks.h" +#include "test/test_common/test_base.h" + +#include "gmock/gmock.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +class ZooKeeperFilterTest : public TestBase { +public: + ZooKeeperFilterTest() { ENVOY_LOG_MISC(info, "test"); } + + void initialize() { + config_ = std::make_shared(stat_prefix_, scope_); + filter_ = std::make_unique(config_); + filter_->initializeReadFilterCallbacks(filter_callbacks_); + } + + std::string encodeConnect(const bool readonly = false, const uint64_t zxid = 100, + const uint32_t session_timeout = 10, const uint32_t session_id = 200, + const std::string& passwd = "") const { + Buffer::OwnedImpl buffer; + const uint32_t message_size = readonly ? 28 + passwd.length() + 1 : 28 + passwd.length(); + + buffer.writeBEInt(message_size); + buffer.writeBEInt(0); // Protocol version. + buffer.writeBEInt(zxid); + buffer.writeBEInt(session_timeout); + buffer.writeBEInt(session_id); + buffer.writeBEInt(passwd.length()); + buffer.add(passwd); + + if (readonly) { + char readonly_flag = 0b1; + buffer.add(std::string(1, readonly_flag)); + } + + return buffer.toString(); + } + + std::string encodeBadMessage() const { + Buffer::OwnedImpl buffer; + // Bad length. + buffer.writeBEInt(1); + // Trailing int. + buffer.writeBEInt(3); + + return buffer.toString(); + } + + std::string encodePing() const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(8); + buffer.writeBEInt(PING_XID); + + return buffer.toString(); + } + + std::string encodeAuth(const std::string& scheme) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(28 + scheme.length()); + buffer.writeBEInt(AUTH_XID); + // Opcode. + buffer.writeBEInt(100); + // Type. + buffer.writeBEInt(0); + // Scheme. + buffer.writeBEInt(scheme.length()); + buffer.add(scheme); + // Credential. + buffer.writeBEInt(6); + buffer.add("p@sswd"); + + return buffer.toString(); + } + + std::string encodeGetData(const std::string& path, const bool watch, + const bool getchildren = false) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(13 + path.length()); + buffer.writeBEInt(1000); + // Opcode. + const int32_t opcode = + getchildren ? enumToInt(Opcodes::GETCHILDREN) : enumToInt(Opcodes::GETDATA); + buffer.writeBEInt(opcode); + // Path. + buffer.writeBEInt(path.length()); + buffer.add(path); + // Watch. + char watch_flag = watch ? 0b1 : 0b0; + buffer.add(std::string(1, watch_flag)); + + return buffer.toString(); + } + + std::string encodeCreateRequest(const std::string& path, const std::string& data, + const bool ephemeral, const bool sequence) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(24 + path.length() + data.length()); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(enumToInt(Opcodes::CREATE)); + // Path. + buffer.writeBEInt(path.length()); + buffer.add(path); + // Data. + buffer.writeBEInt(data.length()); + buffer.add(data); + // Acls. + buffer.writeBEInt(0); + // Flags. + int flags = 0; + if (ephemeral) { + flags &= 0x1; + } + if (sequence) { + flags &= 0x2; + } + buffer.writeBEInt(flags); + + return buffer.toString(); + } + + std::string encodeSetRequest(const std::string& path, const std::string& data, + const int32_t version) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(20 + path.length() + data.length()); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(enumToInt(Opcodes::SETDATA)); + // Path. + buffer.writeBEInt(path.length()); + buffer.add(path); + // Data. + buffer.writeBEInt(data.length()); + buffer.add(data); + // Version. + buffer.writeBEInt(version); + + return buffer.toString(); + } + + ZooKeeperFilterConfigSharedPtr config_; + std::unique_ptr filter_; + Stats::IsolatedStoreImpl scope_; + std::string stat_prefix_{"test.zookeeper"}; + NiceMock filter_callbacks_; +}; + +// Test Connect counter increment. +TEST_F(ZooKeeperFilterTest, Connect) { + initialize(); + + std::string data = encodeConnect(); + Buffer::InstancePtr connect_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*connect_data, false)); + EXPECT_EQ(1UL, config_->stats().connect_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +// Test Connect readonly counter increment. +TEST_F(ZooKeeperFilterTest, ConnectReadonly) { + initialize(); + + std::string data = encodeConnect(true); + Buffer::InstancePtr connect_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*connect_data, false)); + EXPECT_EQ(0UL, config_->stats().connect_.value()); + EXPECT_EQ(1UL, config_->stats().connect_readonly_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +// Test fallback. +TEST_F(ZooKeeperFilterTest, Fallback) { + initialize(); + + std::string data = encodeBadMessage(); + Buffer::InstancePtr bad_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*bad_data, false)); + EXPECT_EQ(0UL, config_->stats().connect_.value()); + EXPECT_EQ(0UL, config_->stats().connect_readonly_.value()); + EXPECT_EQ(2UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, PingRequest) { + initialize(); + + std::string data = encodePing(); + Buffer::InstancePtr ping_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*ping_data, false)); + EXPECT_EQ(1UL, config_->stats().ping_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, AuthRequest) { + initialize(); + + std::string data = encodeAuth("digest"); + Buffer::InstancePtr auth_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*auth_data, false)); + EXPECT_EQ(scope_.counter("test.zookeeper.auth.digest_rq").value(), 1); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, GetDataRequest) { + initialize(); + + std::string data = encodeGetData("/foo", true); + Buffer::InstancePtr get_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*get_data, false)); + EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, CreateRequest) { + initialize(); + + std::string data = encodeCreateRequest("/foo", "bar", false, false); + Buffer::InstancePtr create_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*create_data, false)); + EXPECT_EQ(1UL, config_->stats().create_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, SetRequest) { + initialize(); + + std::string data = encodeSetRequest("/foo", "bar", -1); + Buffer::InstancePtr set_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*set_data, false)); + EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { + initialize(); + + std::string data = encodeGetData("/foo", false, true); + Buffer::InstancePtr getchildren_data(new Buffer::OwnedImpl(data)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*getchildren_data, false)); + EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy From d2f50eb8c7663a8592efca7a6307ae4fa21e6cb8 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Feb 2019 20:53:38 -0800 Subject: [PATCH 002/114] Fix format issues Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto | 3 +++ .../filters/network/zookeeper_proxy/zookeeper_config.h | 2 +- test/extensions/filters/network/zookeeper_proxy/BUILD | 1 - .../filters/network/zookeeper_proxy/zookeeper_filter_test.cc | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto index 58f9389ea3e99..77fbc96efd086 100644 --- a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto @@ -1,6 +1,9 @@ syntax = "proto3"; package envoy.config.filter.network.zookeeper_proxy.v1alpha1; + +option java_outer_classname = "ZookeeperProxyProto"; +option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.zookeeper_proxy.v1alpha1"; option go_package = "v1alpha1"; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h index 76e64bd104f75..2dc1f86ba332c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.h @@ -4,8 +4,8 @@ #include "envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.pb.validate.h" #include "extensions/filters/network/common/factory_base.h" -#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" #include "extensions/filters/network/well_known_names.h" +#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/zookeeper_proxy/BUILD b/test/extensions/filters/network/zookeeper_proxy/BUILD index 8aee34eec3e00..81af4151cf112 100644 --- a/test/extensions/filters/network/zookeeper_proxy/BUILD +++ b/test/extensions/filters/network/zookeeper_proxy/BUILD @@ -25,4 +25,3 @@ envoy_extension_cc_test( "//test/mocks/network:network_mocks", ], ) - diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index f4ec166e15a29..364258d4045c7 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -1,7 +1,7 @@ #include "common/buffer/buffer_impl.h" -#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" #include "extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h" +#include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" #include "test/mocks/network/mocks.h" #include "test/test_common/test_base.h" From 5a3bfe10c49dd96f19699670efe5f74c9851a2c7 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Feb 2019 20:55:32 -0800 Subject: [PATCH 003/114] Fix docs Signed-off-by: Raul Gutierrez Segales --- .../configuration/network_filters/zookeeper_proxy_filter.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index 9756c5dc26e48..675ef9833f902 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -80,4 +80,4 @@ The ZooKeeper filter emits the following dynamic metadata for each message parse :header: Name, Type, Description :widths: 1, 1, 2 - , string, The path associated with the request, response or event. + , string, The path associated with the request or response or event. From e16f507363a6830f60e1099b95510f9267cc48a3 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Feb 2019 21:01:22 -0800 Subject: [PATCH 004/114] Add missing words to dictionary. Signed-off-by: Raul Gutierrez Segales --- tools/spelling_dictionary.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 7015ac23121c1..e736cbc405e46 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -266,6 +266,7 @@ abcd absl accessor accessors +acls addr agg alloc @@ -747,3 +748,4 @@ xyz zag zig zlib +zxid From ea2c5d6753bee90e0fe369faa5baa4c5d4300865 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Feb 2019 12:20:02 -0800 Subject: [PATCH 005/114] Move var declaration closer to usage. Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index f42fb4aa5ac4a..3024081035ddf 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -64,8 +64,6 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { } void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - bool readonly{}; - if (len < 28) { return; } @@ -76,6 +74,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ BufferHelper::peekString(data, offset, passwd); // Read readonly flag, if it's there. + bool readonly{}; BufferHelper::peekBool(data, offset, readonly); callbacks_.onConnect(readonly); From 71f768673e5a72f3b6459d4f78998c351c84accd Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Feb 2019 12:09:26 -0800 Subject: [PATCH 006/114] Tentatively adding @snowp as sponsor/co-maintainer :-) Signed-off-by: Raul Gutierrez Segales --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index bf61be634b6e0..0f81b447d285f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -23,4 +23,4 @@ # quic extension /*/extensions/quic_listeners/ @alyssawilk @danzh2010 @mattklein123 @mpwarres @wu-bin # zookeeper_proxy extension -/*/extensions/filters/network/zookeeper_proxy @rgs1 +/*/extensions/filters/network/zookeeper_proxy @rgs1 @snowp From ad8f31572d6e573efc2e6b4d737d12343b470945 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Feb 2019 21:30:00 -0800 Subject: [PATCH 007/114] Make DecoderImpl's constructor explicit Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 749b636f142e1..36db1260513b4 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -74,7 +74,7 @@ typedef std::unique_ptr DecoderPtr; class DecoderImpl : public Decoder, Logger::Loggable { public: - DecoderImpl(DecoderCallbacks& callbacks) : callbacks_(callbacks) {} + explicit DecoderImpl(DecoderCallbacks& callbacks) : callbacks_(callbacks) {} // ZooKeeperProxy::Decoder void onData(Buffer::Instance& data) override; From cfa9f3f4bcb2f2c6311976ddffcb65de26da2a6f Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Feb 2019 21:50:39 -0800 Subject: [PATCH 008/114] Use peekInt<>() to extract bool Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 32388c7acff2b..207139a3d1b0f 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -22,9 +22,8 @@ bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& va try { std::string str; - str.assign(std::string(static_cast(buffer.linearize(1 + offset)), 1 + offset)); - str = str.substr(offset); - val = static_cast(str[0] & 255); + char byte = buffer.peekInt(offset); + val = static_cast(byte & 255); offset += 1; return true; } catch (EnvoyException& e) { From 2698f558209f0f343d372e6034b6dbaca815d10d Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Feb 2019 22:11:39 -0800 Subject: [PATCH 009/114] Use copyOut() instead of linearize Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 207139a3d1b0f..8abad961ac57b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -39,8 +39,9 @@ bool BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::s return false; } - str.assign(std::string(static_cast(buffer.linearize(len + offset)), len + offset)); - str = str.substr(offset); + std::unique_ptr data(new char[len]); + buffer.copyOut(offset, len, data.get()); + str.assign(data.get(), len); offset += len; return true; From a17402aa3807e840e76fe90eaaf092bd62d52bb9 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Feb 2019 22:30:41 -0800 Subject: [PATCH 010/114] Simplify test helpers Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 185 +++++++++--------- 1 file changed, 89 insertions(+), 96 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 364258d4045c7..4987662b15df9 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -25,102 +25,104 @@ class ZooKeeperFilterTest : public TestBase { filter_->initializeReadFilterCallbacks(filter_callbacks_); } - std::string encodeConnect(const bool readonly = false, const uint64_t zxid = 100, - const uint32_t session_timeout = 10, const uint32_t session_id = 200, - const std::string& passwd = "") const { - Buffer::OwnedImpl buffer; + Buffer::OwnedImpl* encodeConnect(const bool readonly = false, const uint64_t zxid = 100, + const uint32_t session_timeout = 10, + const uint32_t session_id = 200, + const std::string& passwd = "") const { + Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); const uint32_t message_size = readonly ? 28 + passwd.length() + 1 : 28 + passwd.length(); - buffer.writeBEInt(message_size); - buffer.writeBEInt(0); // Protocol version. - buffer.writeBEInt(zxid); - buffer.writeBEInt(session_timeout); - buffer.writeBEInt(session_id); - buffer.writeBEInt(passwd.length()); - buffer.add(passwd); + buffer->writeBEInt(message_size); + buffer->writeBEInt(0); // Protocol version. + buffer->writeBEInt(zxid); + buffer->writeBEInt(session_timeout); + buffer->writeBEInt(session_id); + buffer->writeBEInt(passwd.length()); + buffer->add(passwd); if (readonly) { char readonly_flag = 0b1; - buffer.add(std::string(1, readonly_flag)); + buffer->add(std::string(1, readonly_flag)); } - return buffer.toString(); + return buffer; } - std::string encodeBadMessage() const { - Buffer::OwnedImpl buffer; + Buffer::OwnedImpl* encodeBadMessage() const { + Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + // Bad length. - buffer.writeBEInt(1); + buffer->writeBEInt(1); // Trailing int. - buffer.writeBEInt(3); + buffer->writeBEInt(3); - return buffer.toString(); + return buffer; } - std::string encodePing() const { - Buffer::OwnedImpl buffer; + Buffer::OwnedImpl* encodePing() const { + Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); - buffer.writeBEInt(8); - buffer.writeBEInt(PING_XID); + buffer->writeBEInt(8); + buffer->writeBEInt(PING_XID); - return buffer.toString(); + return buffer; } - std::string encodeAuth(const std::string& scheme) const { - Buffer::OwnedImpl buffer; + Buffer::OwnedImpl* encodeAuth(const std::string& scheme) const { + Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); - buffer.writeBEInt(28 + scheme.length()); - buffer.writeBEInt(AUTH_XID); + buffer->writeBEInt(28 + scheme.length()); + buffer->writeBEInt(AUTH_XID); // Opcode. - buffer.writeBEInt(100); + buffer->writeBEInt(100); // Type. - buffer.writeBEInt(0); + buffer->writeBEInt(0); // Scheme. - buffer.writeBEInt(scheme.length()); - buffer.add(scheme); + buffer->writeBEInt(scheme.length()); + buffer->add(scheme); // Credential. - buffer.writeBEInt(6); - buffer.add("p@sswd"); + buffer->writeBEInt(6); + buffer->add("p@sswd"); - return buffer.toString(); + return buffer; } - std::string encodeGetData(const std::string& path, const bool watch, - const bool getchildren = false) const { - Buffer::OwnedImpl buffer; + Buffer::OwnedImpl* encodeGetData(const std::string& path, const bool watch, + const bool getchildren = false) const { + Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); - buffer.writeBEInt(13 + path.length()); - buffer.writeBEInt(1000); + buffer->writeBEInt(13 + path.length()); + buffer->writeBEInt(1000); // Opcode. const int32_t opcode = getchildren ? enumToInt(Opcodes::GETCHILDREN) : enumToInt(Opcodes::GETDATA); - buffer.writeBEInt(opcode); + buffer->writeBEInt(opcode); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + buffer->writeBEInt(path.length()); + buffer->add(path); // Watch. char watch_flag = watch ? 0b1 : 0b0; - buffer.add(std::string(1, watch_flag)); + buffer->add(std::string(1, watch_flag)); - return buffer.toString(); + return buffer; } - std::string encodeCreateRequest(const std::string& path, const std::string& data, - const bool ephemeral, const bool sequence) const { - Buffer::OwnedImpl buffer; + Buffer::OwnedImpl* encodeCreateRequest(const std::string& path, const std::string& data, + const bool ephemeral, const bool sequence) const { + Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); - buffer.writeBEInt(24 + path.length() + data.length()); - buffer.writeBEInt(1000); + buffer->writeBEInt(24 + path.length() + data.length()); + buffer->writeBEInt(1000); // Opcode. - buffer.writeBEInt(enumToInt(Opcodes::CREATE)); + buffer->writeBEInt(enumToInt(Opcodes::CREATE)); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + buffer->writeBEInt(path.length()); + buffer->add(path); // Data. - buffer.writeBEInt(data.length()); - buffer.add(data); + buffer->writeBEInt(data.length()); + buffer->add(data); // Acls. - buffer.writeBEInt(0); + buffer->writeBEInt(0); // Flags. int flags = 0; if (ephemeral) { @@ -129,29 +131,29 @@ class ZooKeeperFilterTest : public TestBase { if (sequence) { flags &= 0x2; } - buffer.writeBEInt(flags); + buffer->writeBEInt(flags); - return buffer.toString(); + return buffer; } - std::string encodeSetRequest(const std::string& path, const std::string& data, - const int32_t version) const { - Buffer::OwnedImpl buffer; + Buffer::OwnedImpl* encodeSetRequest(const std::string& path, const std::string& data, + const int32_t version) const { + Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); - buffer.writeBEInt(20 + path.length() + data.length()); - buffer.writeBEInt(1000); + buffer->writeBEInt(20 + path.length() + data.length()); + buffer->writeBEInt(1000); // Opcode. - buffer.writeBEInt(enumToInt(Opcodes::SETDATA)); + buffer->writeBEInt(enumToInt(Opcodes::SETDATA)); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + buffer->writeBEInt(path.length()); + buffer->add(path); // Data. - buffer.writeBEInt(data.length()); - buffer.add(data); + buffer->writeBEInt(data.length()); + buffer->add(data); // Version. - buffer.writeBEInt(version); + buffer->writeBEInt(version); - return buffer.toString(); + return buffer; } ZooKeeperFilterConfigSharedPtr config_; @@ -165,10 +167,9 @@ class ZooKeeperFilterTest : public TestBase { TEST_F(ZooKeeperFilterTest, Connect) { initialize(); - std::string data = encodeConnect(); - Buffer::InstancePtr connect_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeConnect()); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*connect_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().connect_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -177,10 +178,9 @@ TEST_F(ZooKeeperFilterTest, Connect) { TEST_F(ZooKeeperFilterTest, ConnectReadonly) { initialize(); - std::string data = encodeConnect(true); - Buffer::InstancePtr connect_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeConnect(true)); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*connect_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(0UL, config_->stats().connect_.value()); EXPECT_EQ(1UL, config_->stats().connect_readonly_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -190,10 +190,9 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { TEST_F(ZooKeeperFilterTest, Fallback) { initialize(); - std::string data = encodeBadMessage(); - Buffer::InstancePtr bad_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeBadMessage()); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*bad_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(0UL, config_->stats().connect_.value()); EXPECT_EQ(0UL, config_->stats().connect_readonly_.value()); EXPECT_EQ(2UL, config_->stats().decoder_error_.value()); @@ -202,10 +201,9 @@ TEST_F(ZooKeeperFilterTest, Fallback) { TEST_F(ZooKeeperFilterTest, PingRequest) { initialize(); - std::string data = encodePing(); - Buffer::InstancePtr ping_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodePing()); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*ping_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().ping_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -213,10 +211,9 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { TEST_F(ZooKeeperFilterTest, AuthRequest) { initialize(); - std::string data = encodeAuth("digest"); - Buffer::InstancePtr auth_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeAuth("digest")); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*auth_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(scope_.counter("test.zookeeper.auth.digest_rq").value(), 1); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -224,10 +221,9 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { TEST_F(ZooKeeperFilterTest, GetDataRequest) { initialize(); - std::string data = encodeGetData("/foo", true); - Buffer::InstancePtr get_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeGetData("/foo", true)); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*get_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -235,10 +231,9 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { TEST_F(ZooKeeperFilterTest, CreateRequest) { initialize(); - std::string data = encodeCreateRequest("/foo", "bar", false, false); - Buffer::InstancePtr create_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeCreateRequest("/foo", "bar", false, false)); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*create_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -246,10 +241,9 @@ TEST_F(ZooKeeperFilterTest, CreateRequest) { TEST_F(ZooKeeperFilterTest, SetRequest) { initialize(); - std::string data = encodeSetRequest("/foo", "bar", -1); - Buffer::InstancePtr set_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeSetRequest("/foo", "bar", -1)); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*set_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -257,10 +251,9 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { initialize(); - std::string data = encodeGetData("/foo", false, true); - Buffer::InstancePtr getchildren_data(new Buffer::OwnedImpl(data)); + Buffer::InstancePtr data(encodeGetData("/foo", false, true)); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*getchildren_data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } From f5a28777d6e4324acc051dbb8fd6c922f8dabdcf Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 20 Feb 2019 07:59:55 -0800 Subject: [PATCH 011/114] Drop my copy of enumToInt() Signed-off-by: Raul Gutierrez Segales --- source/common/common/enum_to_int.h | 2 +- .../filters/network/zookeeper_proxy/zookeeper_decoder.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/common/enum_to_int.h b/source/common/common/enum_to_int.h index a9c77b59419ee..d975eb8d7d033 100644 --- a/source/common/common/enum_to_int.h +++ b/source/common/common/enum_to_int.h @@ -6,5 +6,5 @@ namespace Envoy { /** * Convert an int based enum to an int. */ -template uint32_t enumToInt(T val) { return static_cast(val); } +template constexpr uint32_t enumToInt(T val) { return static_cast(val); } } // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 36db1260513b4..4c5204c22602d 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -4,6 +4,7 @@ #include "envoy/common/platform.h" #include "common/buffer/buffer_impl.h" +#include "common/common/enum_to_int.h" #include "common/common/logger.h" namespace Envoy { @@ -17,8 +18,6 @@ const int PING_XID = -2; const int AUTH_XID = -4; const int SET_WATCHES_XID = -8; -template constexpr int enumToInt(T val) { return static_cast(val); } - enum class Opcodes { CONNECT = 0, CREATE = 1, From 674687670eadd324129583048f5670a98a9ae43c Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Fri, 22 Feb 2019 12:19:44 -0800 Subject: [PATCH 012/114] Fix order of version history items Signed-off-by: Raul Gutierrez Segales --- docs/root/intro/version_history.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 5e59db43b7ea4..453b6cca024f5 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -48,9 +48,9 @@ Version history * upstream: add hash_function to specify the hash function for :ref:`ring hash` as either xxHash or `murmurHash2 `_. MurmurHash2 is compatible with std::hash in GNU libstdc++ 3.4.20 or above. This is typically the case when compiled on Linux and not macOS. * upstream: added :ref:`degraded health value` which allows routing to certain hosts only when there are insufficient healthy hosts available. +* upstream: added support for host weighting and :ref:`locality weighting ` in the :ref:`ring hash load balancer `, and added a :ref:`maximum_ring_size` config parameter to strictly bound the ring size. * zookeeper: added a ZooKeeper proxy filter that parses ZooKeeper messages (requests/responses/events). Refer to ::ref:`ZooKeeper proxy` for more details. -* upstream: added support for host weighting and :ref:`locality weighting ` in the :ref:`ring hash load balancer `, and added a :ref:`maximum_ring_size` config parameter to strictly bound the ring size. 1.9.0 (Dec 20, 2018) ==================== From 7911d6c61ad00f55d7aa0b3706943d7dd8630f9d Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 13:26:30 -0800 Subject: [PATCH 013/114] Consistently name stats that are requests Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 26 ++++++++--------- .../zookeeper_proxy/zookeeper_filter.cc | 6 ++-- .../zookeeper_proxy/zookeeper_filter.h | 28 +++++++++---------- .../zookeeper_proxy/zookeeper_filter_test.cc | 12 ++++---- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index 675ef9833f902..1e15f6fd5be9c 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -49,25 +49,25 @@ following statistics: :widths: 1, 1, 2 decoder_error, Counter, Number of times a message wasn't decoded - connect, Counter, Number of connect requests - connect_readonly, Counter, Number of connect requests with the readonly flag set - ping, Counter, Number of ping requests + connect_rq, Counter, Number of connect requests + connect_readonly_rq, Counter, Number of connect requests with the readonly flag set + ping_rq, Counter, Number of ping requests auth._rq, Counter, Number of auth requests for a given type getdata_rq, Counter, Number of getdata requests create_rq, Counter, Number of create requests create2_rq, Counter, Number of create2 requests setdata_rq, Counter, Number of setdata requests getchildren_rq, Counter, Number of getchildren requests - getchildren2, Counter, Number of getchildren2 requests - remove, Counter, Number of delete requests - exists, Counter, Number of stat requests - getacl, Counter, Number of getacl requests - setacl, Counter, Number of setacl requests - sync, Counter, Number of sync requests - multi, Counter, Number of multi transaction requests - reconfig, Counter, Number of reconfig requests - close, Counter, Number of close requests - setwatches, Counter, Number of setwatches requests + getchildren2_rq, Counter, Number of getchildren2 requests + remove_rq, Counter, Number of delete requests + exists_rq, Counter, Number of stat requests + getacl_rq, Counter, Number of getacl requests + setacl_rq, Counter, Number of setacl requests + sync_rq, Counter, Number of sync requests + multi_rq, Counter, Number of multi transaction requests + reconfig_rq, Counter, Number of reconfig requests + close_rq, Counter, Number of close requests + setwatches_rq, Counter, Number of setwatches requests .. _config_network_filters_zookeeper_proxy_dynamic_metadata: diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index b255dd353c8bb..27711685a7fd5 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -60,9 +60,9 @@ DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks) { void ZooKeeperFilter::onConnect(const bool readonly) { if (readonly) { - config_->stats_.connect_readonly_.inc(); + config_->stats_.connect_readonly_rq_.inc(); } else { - config_->stats_.connect_.inc(); + config_->stats_.connect_rq_.inc(); } // TODO: set dynamic metadata. @@ -70,7 +70,7 @@ void ZooKeeperFilter::onConnect(const bool readonly) { void ZooKeeperFilter::onDecodeError() { config_->stats_.decoder_error_.inc(); } -void ZooKeeperFilter::onPing() { config_->stats_.ping_.inc(); } +void ZooKeeperFilter::onPing() { config_->stats_.ping_rq_.inc(); } void ZooKeeperFilter::onAuthRequest(const std::string& scheme) { config_->scope_.counter(fmt::format("{}.auth.{}_rq", config_->stat_prefix_, scheme)).inc(); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index d037155e20e7f..775cc61ebc85e 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -22,27 +22,25 @@ namespace ZooKeeperProxy { // clang-format off #define ALL_ZOOKEEPER_PROXY_STATS(COUNTER) \ COUNTER(decoder_error) \ - COUNTER(connect) \ - COUNTER(connect_readonly) \ + COUNTER(connect_rq) \ + COUNTER(connect_readonly_rq) \ COUNTER(getdata_rq) \ COUNTER(create_rq) \ COUNTER(create2_rq) \ COUNTER(setdata_rq) \ COUNTER(getchildren_rq) \ COUNTER(getchildren2_rq) \ - COUNTER(remove) \ - COUNTER(exists) \ - COUNTER(getacl) \ - COUNTER(setacl) \ - COUNTER(sync) \ - COUNTER(ping) \ - COUNTER(multi) \ - COUNTER(create2) \ - COUNTER(reconfig) \ - COUNTER(createsession) \ - COUNTER(close) \ - COUNTER(setauth) \ - COUNTER(setwatches) \ + COUNTER(remove_rq) \ + COUNTER(exists_rq) \ + COUNTER(getacl_rq) \ + COUNTER(setacl_rq) \ + COUNTER(sync_rq) \ + COUNTER(ping_rq) \ + COUNTER(multi_rq) \ + COUNTER(reconfig_rq) \ + COUNTER(close_rq) \ + COUNTER(setauth_rq) \ + COUNTER(setwatches_rq) \ // clang-format on /** diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 4987662b15df9..29e72248268d6 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -170,7 +170,7 @@ TEST_F(ZooKeeperFilterTest, Connect) { Buffer::InstancePtr data(encodeConnect()); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); - EXPECT_EQ(1UL, config_->stats().connect_.value()); + EXPECT_EQ(1UL, config_->stats().connect_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -181,8 +181,8 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { Buffer::InstancePtr data(encodeConnect(true)); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); - EXPECT_EQ(0UL, config_->stats().connect_.value()); - EXPECT_EQ(1UL, config_->stats().connect_readonly_.value()); + EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); + EXPECT_EQ(1UL, config_->stats().connect_readonly_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -193,8 +193,8 @@ TEST_F(ZooKeeperFilterTest, Fallback) { Buffer::InstancePtr data(encodeBadMessage()); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); - EXPECT_EQ(0UL, config_->stats().connect_.value()); - EXPECT_EQ(0UL, config_->stats().connect_readonly_.value()); + EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); + EXPECT_EQ(0UL, config_->stats().connect_readonly_rq_.value()); EXPECT_EQ(2UL, config_->stats().decoder_error_.value()); } @@ -204,7 +204,7 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { Buffer::InstancePtr data(encodePing()); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); - EXPECT_EQ(1UL, config_->stats().ping_.value()); + EXPECT_EQ(1UL, config_->stats().ping_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } From dce2ca71aae668d62ef707ca0d42e1bbdfb2b3b3 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 13:29:48 -0800 Subject: [PATCH 014/114] Fix doc Signed-off-by: Raul Gutierrez Segales --- .../configuration/network_filters/zookeeper_proxy_filter.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index 1e15f6fd5be9c..e760c9c5a36a6 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -80,4 +80,4 @@ The ZooKeeper filter emits the following dynamic metadata for each message parse :header: Name, Type, Description :widths: 1, 1, 2 - , string, The path associated with the request or response or event. + , string, "The path associated with the request, response or event" From 315e6562b7a041dc3e7c32258c43116ab8fdcad0 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 13:38:51 -0800 Subject: [PATCH 015/114] Use enum class for XID codes Also, rename the Opcodes enum for consistency. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 18 +++++++++--------- .../zookeeper_proxy/zookeeper_decoder.h | 14 ++++++++------ .../zookeeper_proxy/zookeeper_filter_test.cc | 12 ++++++------ 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 3024081035ddf..057e16ef14458 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -23,13 +23,13 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // "Special" requests. switch (xid) { - case CONNECT_XID: + case enumToInt(XidCodes::CONNECT_XID): parseConnect(data, offset, len); return; - case PING_XID: + case enumToInt(XidCodes::PING_XID): callbacks_.onPing(); return; - case AUTH_XID: + case enumToInt(XidCodes::AUTH_XID): parseAuthRequest(data, offset, len); return; default: @@ -40,22 +40,22 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { int32_t opcode; BufferHelper::peekInt32(data, offset, opcode); switch (opcode) { - case enumToInt(Opcodes::GETDATA): + case enumToInt(OpCodes::GETDATA): parseGetDataRequest(data, offset, len); break; - case enumToInt(Opcodes::CREATE): + case enumToInt(OpCodes::CREATE): parseCreateRequest(data, offset, len, false); break; - case enumToInt(Opcodes::CREATE2): + case enumToInt(OpCodes::CREATE2): parseCreateRequest(data, offset, len, true); break; - case enumToInt(Opcodes::SETDATA): + case enumToInt(OpCodes::SETDATA): parseSetRequest(data, offset, len); break; - case enumToInt(Opcodes::GETCHILDREN): + case enumToInt(OpCodes::GETCHILDREN): parseGetChildrenRequest(data, offset, len, false); break; - case enumToInt(Opcodes::GETCHILDREN2): + case enumToInt(OpCodes::GETCHILDREN2): parseGetChildrenRequest(data, offset, len, true); break; default: diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 4c5204c22602d..2a690dff3d4af 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -12,13 +12,15 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -const int CONNECT_XID = 0; -const int WATCH_XID = -1; -const int PING_XID = -2; -const int AUTH_XID = -4; -const int SET_WATCHES_XID = -8; +enum class XidCodes { + CONNECT_XID = 0, + WATCH_XID = -1, + PING_XID = -2, + AUTH_XID = -4, + SET_WATCHES_XID = -8 +}; -enum class Opcodes { +enum class OpCodes { CONNECT = 0, CREATE = 1, DELETE = 2, diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 29e72248268d6..aac7edb4f2d24 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -63,7 +63,7 @@ class ZooKeeperFilterTest : public TestBase { Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); buffer->writeBEInt(8); - buffer->writeBEInt(PING_XID); + buffer->writeBEInt(enumToInt(XidCodes::PING_XID)); return buffer; } @@ -72,9 +72,9 @@ class ZooKeeperFilterTest : public TestBase { Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); buffer->writeBEInt(28 + scheme.length()); - buffer->writeBEInt(AUTH_XID); + buffer->writeBEInt(enumToInt(XidCodes::AUTH_XID)); // Opcode. - buffer->writeBEInt(100); + buffer->writeBEInt(enumToInt(OpCodes::SETAUTH)); // Type. buffer->writeBEInt(0); // Scheme. @@ -95,7 +95,7 @@ class ZooKeeperFilterTest : public TestBase { buffer->writeBEInt(1000); // Opcode. const int32_t opcode = - getchildren ? enumToInt(Opcodes::GETCHILDREN) : enumToInt(Opcodes::GETDATA); + getchildren ? enumToInt(OpCodes::GETCHILDREN) : enumToInt(OpCodes::GETDATA); buffer->writeBEInt(opcode); // Path. buffer->writeBEInt(path.length()); @@ -114,7 +114,7 @@ class ZooKeeperFilterTest : public TestBase { buffer->writeBEInt(24 + path.length() + data.length()); buffer->writeBEInt(1000); // Opcode. - buffer->writeBEInt(enumToInt(Opcodes::CREATE)); + buffer->writeBEInt(enumToInt(OpCodes::CREATE)); // Path. buffer->writeBEInt(path.length()); buffer->add(path); @@ -143,7 +143,7 @@ class ZooKeeperFilterTest : public TestBase { buffer->writeBEInt(20 + path.length() + data.length()); buffer->writeBEInt(1000); // Opcode. - buffer->writeBEInt(enumToInt(Opcodes::SETDATA)); + buffer->writeBEInt(enumToInt(OpCodes::SETDATA)); // Path. buffer->writeBEInt(path.length()); buffer->add(path); From 0e2c69866073a489a5877bd10d0b6fad73cd27fc Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 13:55:56 -0800 Subject: [PATCH 016/114] Expand enumToInt() to support signed ints. Signed-off-by: Raul Gutierrez Segales --- source/common/common/enum_to_int.h | 2 +- .../filters/network/zookeeper_proxy/zookeeper_decoder.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/common/common/enum_to_int.h b/source/common/common/enum_to_int.h index d975eb8d7d033..d3fc78d378159 100644 --- a/source/common/common/enum_to_int.h +++ b/source/common/common/enum_to_int.h @@ -6,5 +6,5 @@ namespace Envoy { /** * Convert an int based enum to an int. */ -template constexpr uint32_t enumToInt(T val) { return static_cast(val); } +template constexpr int32_t enumToInt(T val) { return static_cast(val); } } // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 2a690dff3d4af..f6ab28f38a289 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -37,7 +37,6 @@ enum class OpCodes { MULTI = 14, CREATE2 = 15, RECONFIG = 16, - CREATESESSION = -10, CLOSE = -11, SETAUTH = 100, SETWATCHES = 101 From d83923595d4f5cd6ea030b8acd6b0e481c8cbcd8 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 14:10:52 -0800 Subject: [PATCH 017/114] Add enumToIntSigned() Signed-off-by: Raul Gutierrez Segales --- source/common/common/enum_to_int.h | 7 ++++++- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/source/common/common/enum_to_int.h b/source/common/common/enum_to_int.h index d3fc78d378159..57688c9390bca 100644 --- a/source/common/common/enum_to_int.h +++ b/source/common/common/enum_to_int.h @@ -6,5 +6,10 @@ namespace Envoy { /** * Convert an int based enum to an int. */ -template constexpr int32_t enumToInt(T val) { return static_cast(val); } +template constexpr uint32_t enumToInt(T val) { return static_cast(val); } + +/** + * Convert an int based enum to a signed int. + */ +template constexpr int32_t enumToIntSigned(T val) { return static_cast(val); } } // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 057e16ef14458..3d63f5bf1bd20 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -23,13 +23,13 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // "Special" requests. switch (xid) { - case enumToInt(XidCodes::CONNECT_XID): + case enumToIntSigned(XidCodes::CONNECT_XID): parseConnect(data, offset, len); return; - case enumToInt(XidCodes::PING_XID): + case enumToIntSigned(XidCodes::PING_XID): callbacks_.onPing(); return; - case enumToInt(XidCodes::AUTH_XID): + case enumToIntSigned(XidCodes::AUTH_XID): parseAuthRequest(data, offset, len); return; default: From 98e689667ad9a873025b03500d617302b99bae15 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 14:27:10 -0800 Subject: [PATCH 018/114] Move Buffer::OwnedImpl instead of returning pointers Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index aac7edb4f2d24..d688e913319c4 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -25,104 +25,104 @@ class ZooKeeperFilterTest : public TestBase { filter_->initializeReadFilterCallbacks(filter_callbacks_); } - Buffer::OwnedImpl* encodeConnect(const bool readonly = false, const uint64_t zxid = 100, - const uint32_t session_timeout = 10, - const uint32_t session_id = 200, - const std::string& passwd = "") const { - Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + Buffer::OwnedImpl encodeConnect(const bool readonly = false, const uint64_t zxid = 100, + const uint32_t session_timeout = 10, + const uint32_t session_id = 200, + const std::string& passwd = "") const { + Buffer::OwnedImpl buffer; const uint32_t message_size = readonly ? 28 + passwd.length() + 1 : 28 + passwd.length(); - buffer->writeBEInt(message_size); - buffer->writeBEInt(0); // Protocol version. - buffer->writeBEInt(zxid); - buffer->writeBEInt(session_timeout); - buffer->writeBEInt(session_id); - buffer->writeBEInt(passwd.length()); - buffer->add(passwd); + buffer.writeBEInt(message_size); + buffer.writeBEInt(0); // Protocol version. + buffer.writeBEInt(zxid); + buffer.writeBEInt(session_timeout); + buffer.writeBEInt(session_id); + buffer.writeBEInt(passwd.length()); + buffer.add(passwd); if (readonly) { char readonly_flag = 0b1; - buffer->add(std::string(1, readonly_flag)); + buffer.add(std::string(1, readonly_flag)); } return buffer; } - Buffer::OwnedImpl* encodeBadMessage() const { - Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + Buffer::OwnedImpl encodeBadMessage() const { + Buffer::OwnedImpl buffer; // Bad length. - buffer->writeBEInt(1); + buffer.writeBEInt(1); // Trailing int. - buffer->writeBEInt(3); + buffer.writeBEInt(3); return buffer; } - Buffer::OwnedImpl* encodePing() const { - Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + Buffer::OwnedImpl encodePing() const { + Buffer::OwnedImpl buffer; - buffer->writeBEInt(8); - buffer->writeBEInt(enumToInt(XidCodes::PING_XID)); + buffer.writeBEInt(8); + buffer.writeBEInt(enumToInt(XidCodes::PING_XID)); return buffer; } - Buffer::OwnedImpl* encodeAuth(const std::string& scheme) const { - Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + Buffer::OwnedImpl encodeAuth(const std::string& scheme) const { + Buffer::OwnedImpl buffer; - buffer->writeBEInt(28 + scheme.length()); - buffer->writeBEInt(enumToInt(XidCodes::AUTH_XID)); + buffer.writeBEInt(28 + scheme.length()); + buffer.writeBEInt(enumToInt(XidCodes::AUTH_XID)); // Opcode. - buffer->writeBEInt(enumToInt(OpCodes::SETAUTH)); + buffer.writeBEInt(enumToInt(OpCodes::SETAUTH)); // Type. - buffer->writeBEInt(0); + buffer.writeBEInt(0); // Scheme. - buffer->writeBEInt(scheme.length()); - buffer->add(scheme); + buffer.writeBEInt(scheme.length()); + buffer.add(scheme); // Credential. - buffer->writeBEInt(6); - buffer->add("p@sswd"); + buffer.writeBEInt(6); + buffer.add("p@sswd"); return buffer; } - Buffer::OwnedImpl* encodeGetData(const std::string& path, const bool watch, - const bool getchildren = false) const { - Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + Buffer::OwnedImpl encodeGetData(const std::string& path, const bool watch, + const bool getchildren = false) const { + Buffer::OwnedImpl buffer; - buffer->writeBEInt(13 + path.length()); - buffer->writeBEInt(1000); + buffer.writeBEInt(13 + path.length()); + buffer.writeBEInt(1000); // Opcode. const int32_t opcode = getchildren ? enumToInt(OpCodes::GETCHILDREN) : enumToInt(OpCodes::GETDATA); - buffer->writeBEInt(opcode); + buffer.writeBEInt(opcode); // Path. - buffer->writeBEInt(path.length()); - buffer->add(path); + buffer.writeBEInt(path.length()); + buffer.add(path); // Watch. char watch_flag = watch ? 0b1 : 0b0; - buffer->add(std::string(1, watch_flag)); + buffer.add(std::string(1, watch_flag)); return buffer; } - Buffer::OwnedImpl* encodeCreateRequest(const std::string& path, const std::string& data, - const bool ephemeral, const bool sequence) const { - Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + Buffer::OwnedImpl encodeCreateRequest(const std::string& path, const std::string& data, + const bool ephemeral, const bool sequence) const { + Buffer::OwnedImpl buffer; - buffer->writeBEInt(24 + path.length() + data.length()); - buffer->writeBEInt(1000); + buffer.writeBEInt(24 + path.length() + data.length()); + buffer.writeBEInt(1000); // Opcode. - buffer->writeBEInt(enumToInt(OpCodes::CREATE)); + buffer.writeBEInt(enumToInt(OpCodes::CREATE)); // Path. - buffer->writeBEInt(path.length()); - buffer->add(path); + buffer.writeBEInt(path.length()); + buffer.add(path); // Data. - buffer->writeBEInt(data.length()); - buffer->add(data); + buffer.writeBEInt(data.length()); + buffer.add(data); // Acls. - buffer->writeBEInt(0); + buffer.writeBEInt(0); // Flags. int flags = 0; if (ephemeral) { @@ -131,27 +131,27 @@ class ZooKeeperFilterTest : public TestBase { if (sequence) { flags &= 0x2; } - buffer->writeBEInt(flags); + buffer.writeBEInt(flags); return buffer; } - Buffer::OwnedImpl* encodeSetRequest(const std::string& path, const std::string& data, - const int32_t version) const { - Buffer::OwnedImpl* buffer = new Buffer::OwnedImpl(); + Buffer::OwnedImpl encodeSetRequest(const std::string& path, const std::string& data, + const int32_t version) const { + Buffer::OwnedImpl buffer; - buffer->writeBEInt(20 + path.length() + data.length()); - buffer->writeBEInt(1000); + buffer.writeBEInt(20 + path.length() + data.length()); + buffer.writeBEInt(1000); // Opcode. - buffer->writeBEInt(enumToInt(OpCodes::SETDATA)); + buffer.writeBEInt(enumToInt(OpCodes::SETDATA)); // Path. - buffer->writeBEInt(path.length()); - buffer->add(path); + buffer.writeBEInt(path.length()); + buffer.add(path); // Data. - buffer->writeBEInt(data.length()); - buffer->add(data); + buffer.writeBEInt(data.length()); + buffer.add(data); // Version. - buffer->writeBEInt(version); + buffer.writeBEInt(version); return buffer; } @@ -167,7 +167,7 @@ class ZooKeeperFilterTest : public TestBase { TEST_F(ZooKeeperFilterTest, Connect) { initialize(); - Buffer::InstancePtr data(encodeConnect()); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeConnect())); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().connect_rq_.value()); @@ -178,7 +178,7 @@ TEST_F(ZooKeeperFilterTest, Connect) { TEST_F(ZooKeeperFilterTest, ConnectReadonly) { initialize(); - Buffer::InstancePtr data(encodeConnect(true)); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeConnect(true))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); @@ -190,7 +190,7 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { TEST_F(ZooKeeperFilterTest, Fallback) { initialize(); - Buffer::InstancePtr data(encodeBadMessage()); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeBadMessage())); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); @@ -201,7 +201,7 @@ TEST_F(ZooKeeperFilterTest, Fallback) { TEST_F(ZooKeeperFilterTest, PingRequest) { initialize(); - Buffer::InstancePtr data(encodePing()); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePing())); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().ping_rq_.value()); @@ -211,7 +211,7 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { TEST_F(ZooKeeperFilterTest, AuthRequest) { initialize(); - Buffer::InstancePtr data(encodeAuth("digest")); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeAuth("digest"))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(scope_.counter("test.zookeeper.auth.digest_rq").value(), 1); @@ -221,7 +221,7 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { TEST_F(ZooKeeperFilterTest, GetDataRequest) { initialize(); - Buffer::InstancePtr data(encodeGetData("/foo", true)); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeGetData("/foo", true))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); @@ -231,7 +231,7 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { TEST_F(ZooKeeperFilterTest, CreateRequest) { initialize(); - Buffer::InstancePtr data(encodeCreateRequest("/foo", "bar", false, false)); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCreateRequest("/foo", "bar", false, false))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); @@ -241,7 +241,7 @@ TEST_F(ZooKeeperFilterTest, CreateRequest) { TEST_F(ZooKeeperFilterTest, SetRequest) { initialize(); - Buffer::InstancePtr data(encodeSetRequest("/foo", "bar", -1)); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetRequest("/foo", "bar", -1))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); @@ -251,7 +251,7 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { initialize(); - Buffer::InstancePtr data(encodeGetData("/foo", false, true)); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeGetData("/foo", false, true))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); From c371cfbd41ad759bcdd502ca8f06de62f0d47180 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 14:35:49 -0800 Subject: [PATCH 019/114] Emit decoderError() calls when len checks fail Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 3d63f5bf1bd20..6ff297fc5e285 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -63,11 +63,15 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { } } -void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - if (len < 28) { - return; +#define CHECK_LENGTH(LEN, MINL) \ + if (LEN < MINL) { \ + callbacks_.onDecodeError(); \ + return; \ } +void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 28); + // Read password - skip zxid, timeout, and session id. std::string passwd; offset += 20; @@ -81,9 +85,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ } void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - if (len < 20) { - return; - } + CHECK_LENGTH(len, 20); // Skip opcode & type. std::string scheme; @@ -96,9 +98,7 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - if (len < 14) { - return; - } + CHECK_LENGTH(len, 14); // Skip opcode. offset += 4; @@ -126,9 +126,7 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - if (len < 25) { - return; - } + CHECK_LENGTH(len, 25); // Skip opcode. offset += 4; @@ -150,9 +148,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - if (len < 20) { - return; - } + CHECK_LENGTH(len, 20); // Skip opcode. offset += 4; @@ -171,9 +167,7 @@ void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - if (len < 14) { - return; - } + CHECK_LENGTH(len, 14); // Skip opcode. offset += 4; From 73962e4ffd2f71ecabf6963fe07340b678dd6961 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 15:11:22 -0800 Subject: [PATCH 020/114] Parse delete & exists requests Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 49 +++++++++++++++++++ .../zookeeper_proxy/zookeeper_decoder.h | 4 ++ .../zookeeper_proxy/zookeeper_filter.cc | 10 ++++ .../zookeeper_proxy/zookeeper_filter.h | 2 + .../zookeeper_proxy/zookeeper_filter_test.cc | 48 +++++++++++++++--- 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 6ff297fc5e285..e1727dd9096de 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -58,6 +58,28 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { case enumToInt(OpCodes::GETCHILDREN2): parseGetChildrenRequest(data, offset, len, true); break; + case enumToInt(OpCodes::DELETE): + parseDeleteRequest(data, offset, len); + break; + case enumToInt(OpCodes::EXISTS): + parseExistsRequest(data, offset, len); + break; + case enumToInt(OpCodes::GETACL): + break; + case enumToInt(OpCodes::SETACL): + break; + case enumToInt(OpCodes::SYNC): + break; + case enumToInt(OpCodes::CHECK): + break; + case enumToInt(OpCodes::MULTI): + break; + case enumToInt(OpCodes::RECONFIG): + break; + case enumToInt(OpCodes::SETWATCHES): + break; + case enumToIntSigned(OpCodes::CLOSE): + break; default: break; } @@ -179,6 +201,33 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs callbacks_.onGetChildrenRequest(path, watch, two); } +void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 16); + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + // Version. + int32_t version; + BufferHelper::peekInt32(data, offset, version); + + callbacks_.onDeleteRequest(path, version); +} + +void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 13); + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + bool watch; + BufferHelper::peekBool(data, offset, watch); + + callbacks_.onExistsRequest(path, watch); +} + void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index f6ab28f38a289..c6eb3a03d2743 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -58,6 +58,8 @@ class DecoderCallbacks { const bool two) PURE; virtual void onSetRequest(const std::string& path) PURE; virtual void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) PURE; + virtual void onDeleteRequest(const std::string& path, const int32_t version) PURE; + virtual void onExistsRequest(const std::string& path, const bool watch) PURE; }; /** @@ -89,6 +91,8 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two); + void parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); DecoderCallbacks& callbacks_; }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 27711685a7fd5..d2d2d3099f9e7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -104,6 +104,16 @@ void ZooKeeperFilter::onGetChildrenRequest(const std::string&, const bool, const // TODO: set metadata. } +void ZooKeeperFilter::onDeleteRequest(const std::string&, const int32_t) { + config_->stats_.remove_rq_.inc(); + // TODO: set metadata. +} + +void ZooKeeperFilter::onExistsRequest(const std::string&, const bool) { + config_->stats_.exists_rq_.inc(); + // TODO: set metadata. +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 775cc61ebc85e..2c220c581c563 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -98,6 +98,8 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onCreateRequest(const std::string& path, const bool ephemeral, const bool sequence, const bool two) override; void onSetRequest(const std::string& path) override; void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) override; + void onDeleteRequest(const std::string& path, const int32_t version) override; + void onExistsRequest(const std::string& path, const bool watch) override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index d688e913319c4..588ad18c01614 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -87,15 +87,13 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } - Buffer::OwnedImpl encodeGetData(const std::string& path, const bool watch, - const bool getchildren = false) const { + Buffer::OwnedImpl encodePathWatch(const std::string& path, const bool watch, + const int32_t opcode = enumToInt(OpCodes::GETDATA)) const { Buffer::OwnedImpl buffer; buffer.writeBEInt(13 + path.length()); buffer.writeBEInt(1000); // Opcode. - const int32_t opcode = - getchildren ? enumToInt(OpCodes::GETCHILDREN) : enumToInt(OpCodes::GETDATA); buffer.writeBEInt(opcode); // Path. buffer.writeBEInt(path.length()); @@ -156,6 +154,22 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodeDeleteRequest(const std::string& path, const int32_t version) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(16 + path.length()); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(enumToInt(OpCodes::DELETE)); + // Path. + buffer.writeBEInt(path.length()); + buffer.add(path); + // Version. + buffer.writeBEInt(version); + + return buffer; + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -221,7 +235,7 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { TEST_F(ZooKeeperFilterTest, GetDataRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeGetData("/foo", true))); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePathWatch("/foo", true))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); @@ -251,13 +265,35 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeGetData("/foo", false, true))); + Buffer::InstancePtr data( + new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN)))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, DeleteRequest) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeDeleteRequest("/foo", -1))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().remove_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, ExistsRequest) { + initialize(); + + Buffer::InstancePtr data( + new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::EXISTS)))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().exists_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From e19d918fd1d43c155e37a560fa2e24c14a940f21 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 15:31:23 -0800 Subject: [PATCH 021/114] Parse getacl & setacl requests Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 29 +++++++++ .../zookeeper_proxy/zookeeper_decoder.h | 4 ++ .../zookeeper_proxy/zookeeper_filter.cc | 10 +++ .../zookeeper_proxy/zookeeper_filter.h | 2 + .../zookeeper_proxy/zookeeper_filter_test.cc | 65 +++++++++++++++++++ 5 files changed, 110 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index e1727dd9096de..7239e055e8ccc 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -65,8 +65,10 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseExistsRequest(data, offset, len); break; case enumToInt(OpCodes::GETACL): + parseGetAclRequest(data, offset, len); break; case enumToInt(OpCodes::SETACL): + parseSetAclRequest(data, offset, len); break; case enumToInt(OpCodes::SYNC): break; @@ -228,6 +230,33 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u callbacks_.onExistsRequest(path, watch); } +void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 8); + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + + callbacks_.onGetAclRequest(path); +} + +void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 8); + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + // Skip acls. + skipAcls(data, offset); + // Version. + int32_t version; + BufferHelper::peekInt32(data, offset, version); + + callbacks_.onSetAclRequest(path, version); +} + void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index c6eb3a03d2743..af526af301754 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -60,6 +60,8 @@ class DecoderCallbacks { virtual void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) PURE; virtual void onDeleteRequest(const std::string& path, const int32_t version) PURE; virtual void onExistsRequest(const std::string& path, const bool watch) PURE; + virtual void onGetAclRequest(const std::string& path) PURE; + virtual void onSetAclRequest(const std::string& path, const int32_t version) PURE; }; /** @@ -93,6 +95,8 @@ class DecoderImpl : public Decoder, Logger::Loggable { const bool two); void parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); DecoderCallbacks& callbacks_; }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index d2d2d3099f9e7..b51a4eb4db7a1 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -114,6 +114,16 @@ void ZooKeeperFilter::onExistsRequest(const std::string&, const bool) { // TODO: set metadata. } +void ZooKeeperFilter::onGetAclRequest(const std::string&) { + config_->stats_.getacl_rq_.inc(); + // TODO: set metadata. +} + +void ZooKeeperFilter::onSetAclRequest(const std::string&, const int32_t) { + config_->stats_.setacl_rq_.inc(); + // TODO: set metadata. +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 2c220c581c563..98b1653b41432 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -100,6 +100,8 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) override; void onDeleteRequest(const std::string& path, const int32_t version) override; void onExistsRequest(const std::string& path, const bool watch) override; + void onGetAclRequest(const std::string& path) override; + void onSetAclRequest(const std::string& path, const int32_t version) override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 588ad18c01614..69a1083a6a55a 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -105,6 +105,20 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodePath(const std::string& path, const int32_t opcode) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(8 + path.length()); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(opcode); + // Path. + buffer.writeBEInt(path.length()); + buffer.add(path); + + return buffer; + } + Buffer::OwnedImpl encodeCreateRequest(const std::string& path, const std::string& data, const bool ephemeral, const bool sequence) const { Buffer::OwnedImpl buffer; @@ -170,6 +184,36 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodeSetAclRequest(const std::string& path, const std::string& scheme, + const std::string& credential, + const int32_t version) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(32 + path.length() + scheme.length() + credential.length()); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(enumToInt(OpCodes::SETACL)); + // Path. + buffer.writeBEInt(path.length()); + buffer.add(path); + + // Acls. + buffer.writeBEInt(1); + // Type. + buffer.writeBEInt(0); + // Scheme. + buffer.writeBEInt(scheme.length()); + buffer.add(scheme); + // Credential. + buffer.writeBEInt(credential.length()); + buffer.add(credential); + + // Version. + buffer.writeBEInt(version); + + return buffer; + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -294,6 +338,27 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, GetAclRequest) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::GETACL)))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().getacl_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, SetAclRequest) { + initialize(); + + Buffer::InstancePtr data( + new Buffer::OwnedImpl(encodeSetAclRequest("/foo", "digest", "passwd", -1))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().setacl_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From 30546ceed497148dd5d6460073c0c46ddd4477a7 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 15:36:17 -0800 Subject: [PATCH 022/114] Parse sync requests Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/zookeeper_decoder.cc | 12 ++++++++++++ .../network/zookeeper_proxy/zookeeper_decoder.h | 2 ++ .../network/zookeeper_proxy/zookeeper_filter.cc | 5 +++++ .../network/zookeeper_proxy/zookeeper_filter.h | 1 + .../network/zookeeper_proxy/zookeeper_filter_test.cc | 10 ++++++++++ 5 files changed, 30 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 7239e055e8ccc..5867076e89102 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -71,6 +71,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseSetAclRequest(data, offset, len); break; case enumToInt(OpCodes::SYNC): + parseSyncRequest(data, offset, len); break; case enumToInt(OpCodes::CHECK): break; @@ -257,6 +258,17 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u callbacks_.onSetAclRequest(path, version); } +void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 8); + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + + callbacks_.onSyncRequest(path); +} + void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index af526af301754..4e2a774d984f5 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -62,6 +62,7 @@ class DecoderCallbacks { virtual void onExistsRequest(const std::string& path, const bool watch) PURE; virtual void onGetAclRequest(const std::string& path) PURE; virtual void onSetAclRequest(const std::string& path, const int32_t version) PURE; + virtual void onSyncRequest(const std::string& path) PURE; }; /** @@ -97,6 +98,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); DecoderCallbacks& callbacks_; }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index b51a4eb4db7a1..6ebe5a850b581 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -124,6 +124,11 @@ void ZooKeeperFilter::onSetAclRequest(const std::string&, const int32_t) { // TODO: set metadata. } +void ZooKeeperFilter::onSyncRequest(const std::string&) { + config_->stats_.sync_rq_.inc(); + // TODO: set metadata. +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 98b1653b41432..c2fcda3b8d63a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -102,6 +102,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onExistsRequest(const std::string& path, const bool watch) override; void onGetAclRequest(const std::string& path) override; void onSetAclRequest(const std::string& path, const int32_t version) override; + void onSyncRequest(const std::string& path) override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 69a1083a6a55a..34d41fb2b46dc 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -359,6 +359,16 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, SyncRequest) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::SYNC)))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().sync_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From b45467153e74f1c7aa7af1c50c69d637cbeca896 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 23 Feb 2019 15:44:28 -0800 Subject: [PATCH 023/114] Parse check requests Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 1 + .../zookeeper_proxy/zookeeper_decoder.cc | 15 ++++++++++ .../zookeeper_proxy/zookeeper_decoder.h | 2 ++ .../zookeeper_proxy/zookeeper_filter.cc | 5 ++++ .../zookeeper_proxy/zookeeper_filter.h | 2 ++ .../zookeeper_proxy/zookeeper_filter_test.cc | 28 +++++++++++++++++++ 6 files changed, 53 insertions(+) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index e760c9c5a36a6..b4911bf5d9435 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -68,6 +68,7 @@ following statistics: reconfig_rq, Counter, Number of reconfig requests close_rq, Counter, Number of close requests setwatches_rq, Counter, Number of setwatches requests + check_rq, Counter, Number of check requests .. _config_network_filters_zookeeper_proxy_dynamic_metadata: diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 5867076e89102..a3343f3255ce5 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -74,6 +74,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseSyncRequest(data, offset, len); break; case enumToInt(OpCodes::CHECK): + parseCheckRequest(data, offset, len); break; case enumToInt(OpCodes::MULTI): break; @@ -269,6 +270,20 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin callbacks_.onSyncRequest(path); } +void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 8); + + // Skip opcode. + offset += 4; + std::string path; + BufferHelper::peekString(data, offset, path); + // Version. + int32_t version; + BufferHelper::peekInt32(data, offset, version); + + callbacks_.onCheckRequest(path, version); +} + void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 4e2a774d984f5..ec1f9ff47ea8a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -63,6 +63,7 @@ class DecoderCallbacks { virtual void onGetAclRequest(const std::string& path) PURE; virtual void onSetAclRequest(const std::string& path, const int32_t version) PURE; virtual void onSyncRequest(const std::string& path) PURE; + virtual void onCheckRequest(const std::string& path, const int32_t version) PURE; }; /** @@ -99,6 +100,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); DecoderCallbacks& callbacks_; }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 6ebe5a850b581..7ae818c2a3cec 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -129,6 +129,11 @@ void ZooKeeperFilter::onSyncRequest(const std::string&) { // TODO: set metadata. } +void ZooKeeperFilter::onCheckRequest(const std::string&, const int32_t) { + config_->stats_.check_rq_.inc(); + // TODO: set metadata. +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index c2fcda3b8d63a..8072d0dc24578 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -41,6 +41,7 @@ namespace ZooKeeperProxy { COUNTER(close_rq) \ COUNTER(setauth_rq) \ COUNTER(setwatches_rq) \ + COUNTER(check_rq) \ // clang-format on /** @@ -103,6 +104,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onGetAclRequest(const std::string& path) override; void onSetAclRequest(const std::string& path, const int32_t version) override; void onSyncRequest(const std::string& path) override; + void onCheckRequest(const std::string& path, const int32_t version) override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 34d41fb2b46dc..54a56eb071b41 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -105,6 +105,23 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodePathVersion(const std::string& path, const int32_t version, + const int32_t opcode = enumToInt(OpCodes::GETDATA)) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(16 + path.length()); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(opcode); + // Path. + buffer.writeBEInt(path.length()); + buffer.add(path); + // Version + buffer.writeBEInt(version); + + return buffer; + } + Buffer::OwnedImpl encodePath(const std::string& path, const int32_t opcode) const { Buffer::OwnedImpl buffer; @@ -369,6 +386,17 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, CheckRequest) { + initialize(); + + Buffer::InstancePtr data( + new Buffer::OwnedImpl(encodePathVersion("/foo", 100, enumToInt(OpCodes::CHECK)))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().check_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From e2fa765c1e5e6d74598f1b51ac17aab268091231 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 24 Feb 2019 00:29:50 -0800 Subject: [PATCH 024/114] Parse multi requests Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 65 ++++++++++++------- .../zookeeper_proxy/zookeeper_decoder.h | 2 + .../zookeeper_proxy/zookeeper_filter.cc | 5 ++ .../zookeeper_proxy/zookeeper_filter.h | 1 + .../zookeeper_proxy/zookeeper_utils.cc | 2 +- .../zookeeper_proxy/zookeeper_filter_test.cc | 60 +++++++++++++++-- 6 files changed, 105 insertions(+), 30 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index a3343f3255ce5..7f27c5974d051 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -77,6 +77,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseCheckRequest(data, offset, len); break; case enumToInt(OpCodes::MULTI): + parseMultiRequest(data, offset, len); break; case enumToInt(OpCodes::RECONFIG): break; @@ -113,9 +114,9 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 20); - // Skip opcode & type. - std::string scheme; + // Skip opcode + type. offset += 8; + std::string scheme; BufferHelper::peekString(data, offset, scheme); std::string credential; BufferHelper::peekString(data, offset, credential); @@ -124,10 +125,8 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 14); + CHECK_LENGTH(len, 13); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); bool watch; @@ -152,10 +151,8 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - CHECK_LENGTH(len, 25); + CHECK_LENGTH(len, 20); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); // Skip data. @@ -176,8 +173,6 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 20); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); // Skip data. @@ -195,8 +190,6 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs const bool two) { CHECK_LENGTH(len, 14); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); bool watch; @@ -208,8 +201,6 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 16); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); // Version. @@ -222,8 +213,6 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 13); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); bool watch; @@ -235,8 +224,6 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 8); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); @@ -246,8 +233,6 @@ void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 8); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); // Skip acls. @@ -262,8 +247,6 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 8); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); @@ -273,8 +256,6 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { CHECK_LENGTH(len, 8); - // Skip opcode. - offset += 4; std::string path; BufferHelper::peekString(data, offset, path); // Version. @@ -284,6 +265,42 @@ void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, ui callbacks_.onCheckRequest(path, version); } +void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + // Treat empty transactions as a decoding error, there should be at least 1 header. + CHECK_LENGTH(len, 17); + + while (true) { + int32_t type; + BufferHelper::peekInt32(data, offset, type); + bool done{}; + BufferHelper::peekBool(data, offset, done); + int32_t error; + BufferHelper::peekInt32(data, offset, error); + + if (done) { + break; + } + + switch (type) { + case enumToInt(OpCodes::CREATE): + parseCreateRequest(data, offset, len, false); + break; + case enumToInt(OpCodes::SETDATA): + parseSetRequest(data, offset, len); + break; + case enumToInt(OpCodes::CHECK): + parseCheckRequest(data, offset, len); + break; + default: + // Should not happen. + callbacks_.onDecodeError(); + break; + } + } + + callbacks_.onMultiRequest(); +} + void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index ec1f9ff47ea8a..dabfe10c053b3 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -64,6 +64,7 @@ class DecoderCallbacks { virtual void onSetAclRequest(const std::string& path, const int32_t version) PURE; virtual void onSyncRequest(const std::string& path) PURE; virtual void onCheckRequest(const std::string& path, const int32_t version) PURE; + virtual void onMultiRequest() PURE; }; /** @@ -101,6 +102,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); DecoderCallbacks& callbacks_; }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 7ae818c2a3cec..603896a2d7cb8 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -134,6 +134,11 @@ void ZooKeeperFilter::onCheckRequest(const std::string&, const int32_t) { // TODO: set metadata. } +void ZooKeeperFilter::onMultiRequest() { + config_->stats_.multi_rq_.inc(); + // TODO: set metadata. +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 8072d0dc24578..c42266c045271 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -105,6 +105,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onSetAclRequest(const std::string& path, const int32_t version) override; void onSyncRequest(const std::string& path) override; void onCheckRequest(const std::string& path, const int32_t version) override; + void onMultiRequest() override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 8abad961ac57b..bde148453c274 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -8,7 +8,7 @@ namespace ZooKeeperProxy { bool BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val) { try { val = buffer.peekBEInt(offset); - offset += sizeof(uint32_t); + offset += sizeof(int32_t); return true; } catch (EnvoyException& e) { return false; diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 54a56eb071b41..d77b2cd101701 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -137,13 +137,16 @@ class ZooKeeperFilterTest : public TestBase { } Buffer::OwnedImpl encodeCreateRequest(const std::string& path, const std::string& data, - const bool ephemeral, const bool sequence) const { + const bool ephemeral, const bool sequence, + const bool txn = false) const { Buffer::OwnedImpl buffer; - buffer.writeBEInt(24 + path.length() + data.length()); - buffer.writeBEInt(1000); - // Opcode. - buffer.writeBEInt(enumToInt(OpCodes::CREATE)); + if (!txn) { + buffer.writeBEInt(24 + path.length() + data.length()); + buffer.writeBEInt(1000); + buffer.writeBEInt(enumToInt(OpCodes::CREATE)); + } + // Path. buffer.writeBEInt(path.length()); buffer.add(path); @@ -231,6 +234,37 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl + encodeMultiRequest(const std::vector>& ops) const { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl requests; + + for (const auto& op_pair : ops) { + // Header. + requests.writeBEInt(op_pair.first); + requests.add(std::string(1, 0b0)); + requests.writeBEInt(-1); + + // Payload. + requests.add(op_pair.second); + } + + // Done header. + requests.writeBEInt(-1); + requests.add(std::string(1, 0b1)); + requests.writeBEInt(-1); + + // Multi prefix. + buffer.writeBEInt(8 + requests.length()); + buffer.writeBEInt(1000); + buffer.writeBEInt(enumToInt(OpCodes::MULTI)); + + // Requests. + buffer.add(requests); + + return buffer; + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -397,6 +431,22 @@ TEST_F(ZooKeeperFilterTest, CheckRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, MultiRequest) { + initialize(); + + Buffer::OwnedImpl create1 = encodeCreateRequest("/foo", "1", false, false, true); + Buffer::OwnedImpl create2 = encodeCreateRequest("/bar", "1", false, false, true); + std::vector> ops; + ops.push_back(std::make_pair(enumToInt(OpCodes::CREATE), std::move(create1))); + ops.push_back(std::make_pair(enumToInt(OpCodes::CREATE), std::move(create2))); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeMultiRequest(ops))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().multi_rq_.value()); + EXPECT_EQ(2UL, config_->stats().create_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From a21102f60bf763b0b2aa095fada72b76197bcbfb Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 24 Feb 2019 15:26:02 -0800 Subject: [PATCH 025/114] Parse reconfig requests Also, clean up tests a bit and handle bad headers when parsing a Multi request. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 38 +++++++++- .../zookeeper_proxy/zookeeper_decoder.h | 2 + .../zookeeper_proxy/zookeeper_filter.cc | 5 ++ .../zookeeper_proxy/zookeeper_filter.h | 1 + .../zookeeper_proxy/zookeeper_utils.cc | 10 +++ .../network/zookeeper_proxy/zookeeper_utils.h | 1 + .../zookeeper_proxy/zookeeper_filter_test.cc | 76 +++++++++++-------- 7 files changed, 100 insertions(+), 33 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 7f27c5974d051..2b801b455ec8e 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -80,6 +80,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseMultiRequest(data, offset, len); break; case enumToInt(OpCodes::RECONFIG): + parseReconfigRequest(data, offset, len); break; case enumToInt(OpCodes::SETWATCHES): break; @@ -271,11 +272,22 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui while (true) { int32_t type; - BufferHelper::peekInt32(data, offset, type); + if (!BufferHelper::peekInt32(data, offset, type)) { + callbacks_.onDecodeError(); + return; + } + bool done{}; - BufferHelper::peekBool(data, offset, done); + if (!BufferHelper::peekBool(data, offset, done)) { + callbacks_.onDecodeError(); + return; + } + int32_t error; - BufferHelper::peekInt32(data, offset, error); + if (!BufferHelper::peekInt32(data, offset, error)) { + callbacks_.onDecodeError(); + return; + } if (done) { break; @@ -301,6 +313,26 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui callbacks_.onMultiRequest(); } +void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 28); + + // Skip joining. + int32_t datalen; + BufferHelper::peekInt32(data, offset, datalen); + offset += datalen; + // Skip leaving. + BufferHelper::peekInt32(data, offset, datalen); + offset += datalen; + // Skip new members. + BufferHelper::peekInt32(data, offset, datalen); + offset += datalen; + // Read config id. + int64_t config_id; + BufferHelper::peekInt64(data, offset, config_id); + + callbacks_.onReconfigRequest(); +} + void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index dabfe10c053b3..0a2606d239c32 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -65,6 +65,7 @@ class DecoderCallbacks { virtual void onSyncRequest(const std::string& path) PURE; virtual void onCheckRequest(const std::string& path, const int32_t version) PURE; virtual void onMultiRequest() PURE; + virtual void onReconfigRequest() PURE; }; /** @@ -103,6 +104,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); DecoderCallbacks& callbacks_; }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 603896a2d7cb8..d54283e887930 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -139,6 +139,11 @@ void ZooKeeperFilter::onMultiRequest() { // TODO: set metadata. } +void ZooKeeperFilter::onReconfigRequest() { + config_->stats_.reconfig_rq_.inc(); + // TODO: set metadata. +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index c42266c045271..6a644de6c0858 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -106,6 +106,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onSyncRequest(const std::string& path) override; void onCheckRequest(const std::string& path, const int32_t version) override; void onMultiRequest() override; + void onReconfigRequest() override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index bde148453c274..b4b6081c34bde 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -15,6 +15,16 @@ bool BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t } } +bool BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val) { + try { + val = buffer.peekBEInt(offset); + offset += sizeof(int64_t); + return true; + } catch (EnvoyException& e) { + return false; + } +} + bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val) { if (buffer.length() < (offset + 1)) { return false; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h index 8c0ecb2fb696b..77318aba24b06 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -18,6 +18,7 @@ namespace ZooKeeperProxy { class BufferHelper : public Logger::Loggable { public: static bool peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val); + static bool peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val); static bool peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str); static bool peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val); }; diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index d77b2cd101701..9013b49ee4c13 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -37,11 +37,10 @@ class ZooKeeperFilterTest : public TestBase { buffer.writeBEInt(zxid); buffer.writeBEInt(session_timeout); buffer.writeBEInt(session_id); - buffer.writeBEInt(passwd.length()); - buffer.add(passwd); + addString(buffer, passwd); if (readonly) { - char readonly_flag = 0b1; + const char readonly_flag = 0b1; buffer.add(std::string(1, readonly_flag)); } @@ -78,11 +77,9 @@ class ZooKeeperFilterTest : public TestBase { // Type. buffer.writeBEInt(0); // Scheme. - buffer.writeBEInt(scheme.length()); - buffer.add(scheme); + addString(buffer, scheme); // Credential. - buffer.writeBEInt(6); - buffer.add("p@sswd"); + addString(buffer, "p@sswd"); return buffer; } @@ -96,10 +93,9 @@ class ZooKeeperFilterTest : public TestBase { // Opcode. buffer.writeBEInt(opcode); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + addString(buffer, path); // Watch. - char watch_flag = watch ? 0b1 : 0b0; + const char watch_flag = watch ? 0b1 : 0b0; buffer.add(std::string(1, watch_flag)); return buffer; @@ -114,8 +110,7 @@ class ZooKeeperFilterTest : public TestBase { // Opcode. buffer.writeBEInt(opcode); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + addString(buffer, path); // Version buffer.writeBEInt(version); @@ -130,8 +125,7 @@ class ZooKeeperFilterTest : public TestBase { // Opcode. buffer.writeBEInt(opcode); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + addString(buffer, path); return buffer; } @@ -148,11 +142,9 @@ class ZooKeeperFilterTest : public TestBase { } // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + addString(buffer, path); // Data. - buffer.writeBEInt(data.length()); - buffer.add(data); + addString(buffer, data); // Acls. buffer.writeBEInt(0); // Flags. @@ -177,11 +169,9 @@ class ZooKeeperFilterTest : public TestBase { // Opcode. buffer.writeBEInt(enumToInt(OpCodes::SETDATA)); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + addString(buffer, path); // Data. - buffer.writeBEInt(data.length()); - buffer.add(data); + addString(buffer, data); // Version. buffer.writeBEInt(version); @@ -196,8 +186,7 @@ class ZooKeeperFilterTest : public TestBase { // Opcode. buffer.writeBEInt(enumToInt(OpCodes::DELETE)); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + addString(buffer, path); // Version. buffer.writeBEInt(version); @@ -214,19 +203,16 @@ class ZooKeeperFilterTest : public TestBase { // Opcode. buffer.writeBEInt(enumToInt(OpCodes::SETACL)); // Path. - buffer.writeBEInt(path.length()); - buffer.add(path); + addString(buffer, path); // Acls. buffer.writeBEInt(1); // Type. buffer.writeBEInt(0); // Scheme. - buffer.writeBEInt(scheme.length()); - buffer.add(scheme); + addString(buffer, scheme); // Credential. - buffer.writeBEInt(credential.length()); - buffer.add(credential); + addString(buffer, credential); // Version. buffer.writeBEInt(version); @@ -234,6 +220,21 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodeReconfigRequest(const std::string& joining, const std::string& leaving, + const std::string& new_members, int64_t config_id) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(28 + joining.length() + leaving.length() + new_members.length()); + buffer.writeBEInt(1000); + buffer.writeBEInt(enumToInt(OpCodes::RECONFIG)); + addString(buffer, joining); + addString(buffer, leaving); + addString(buffer, new_members); + buffer.writeBEInt(config_id); + + return buffer; + } + Buffer::OwnedImpl encodeMultiRequest(const std::vector>& ops) const { Buffer::OwnedImpl buffer; @@ -265,6 +266,11 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + void addString(Buffer::OwnedImpl& buffer, const std::string& str) const { + buffer.writeBEInt(str.length()); + buffer.add(str); + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -447,6 +453,16 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, ReconfigRequest) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeReconfigRequest("s1", "s2", "s3", 1000))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().reconfig_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From 4635b35c7db551a82ec9948a0ac9d7d15db58f24 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 24 Feb 2019 15:43:35 -0800 Subject: [PATCH 026/114] Parse setWatches requests Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 25 ++++++++++++ .../zookeeper_proxy/zookeeper_decoder.h | 3 ++ .../zookeeper_proxy/zookeeper_filter.cc | 5 +++ .../zookeeper_proxy/zookeeper_filter.h | 1 + .../zookeeper_proxy/zookeeper_filter_test.cc | 40 +++++++++++++++++++ 5 files changed, 74 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 2b801b455ec8e..60f8997671134 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -83,6 +83,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseReconfigRequest(data, offset, len); break; case enumToInt(OpCodes::SETWATCHES): + parseSetWatchesRequest(data, offset, len); break; case enumToIntSigned(OpCodes::CLOSE): break; @@ -333,6 +334,30 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, callbacks_.onReconfigRequest(); } +void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + CHECK_LENGTH(len, 12); + + // Data watches. + skipStrings(data, offset); + // Exist watches. + skipStrings(data, offset); + // Child watches. + skipStrings(data, offset); + + callbacks_.onSetWatchesRequest(); +} + +void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { + int32_t count; + BufferHelper::peekInt32(data, offset, count); + + for (int i = 0; i < count; ++i) { + int32_t len; + BufferHelper::peekInt32(data, offset, len); + offset += len; + } +} + void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 0a2606d239c32..c558be85ed1ed 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -66,6 +66,7 @@ class DecoderCallbacks { virtual void onCheckRequest(const std::string& path, const int32_t version) PURE; virtual void onMultiRequest() PURE; virtual void onReconfigRequest() PURE; + virtual void onSetWatchesRequest() PURE; }; /** @@ -105,6 +106,8 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void skipStrings(Buffer::Instance& data, uint64_t& offset) const; DecoderCallbacks& callbacks_; }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index d54283e887930..02c7e47665307 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -144,6 +144,11 @@ void ZooKeeperFilter::onReconfigRequest() { // TODO: set metadata. } +void ZooKeeperFilter::onSetWatchesRequest() { + config_->stats_.setwatches_rq_.inc(); + // TODO: set metadata. +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 6a644de6c0858..440537ed33215 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -107,6 +107,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onCheckRequest(const std::string& path, const int32_t version) override; void onMultiRequest() override; void onReconfigRequest() override; + void onSetWatchesRequest() override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 9013b49ee4c13..9382b31a5db8d 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -235,6 +235,24 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodeSetWatchesRequest(const std::vector& dataw, + const std::vector& existw, + const std::vector& childw) const { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl watches_buffer; + + addStrings(watches_buffer, dataw); + addStrings(watches_buffer, existw); + addStrings(watches_buffer, childw); + + buffer.writeBEInt(8 + watches_buffer.length()); + buffer.writeBEInt(1000); + buffer.writeBEInt(enumToInt(OpCodes::SETWATCHES)); + buffer.add(watches_buffer); + + return buffer; + } + Buffer::OwnedImpl encodeMultiRequest(const std::vector>& ops) const { Buffer::OwnedImpl buffer; @@ -271,6 +289,14 @@ class ZooKeeperFilterTest : public TestBase { buffer.add(str); } + void addStrings(Buffer::OwnedImpl& buffer, const std::vector& watches) const { + buffer.writeBEInt(watches.size()); + + for (const auto& watch : watches) { + addString(buffer, watch); + } + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -463,6 +489,20 @@ TEST_F(ZooKeeperFilterTest, ReconfigRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { + initialize(); + + const std::vector dataw = {"/foo", "/bar"}; + const std::vector existw = {"/foo1", "/bar1"}; + const std::vector childw = {"/foo2", "/bar2"}; + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetWatchesRequest(dataw, existw, childw))); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From bae0ca2f3392ed3f9c39fa0a65f53a8421588693 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 24 Feb 2019 15:48:41 -0800 Subject: [PATCH 027/114] Parse Close requests Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 1 + .../zookeeper_proxy/zookeeper_decoder.h | 1 + .../zookeeper_proxy/zookeeper_filter.cc | 4 ++++ .../zookeeper_proxy/zookeeper_filter.h | 1 + .../zookeeper_proxy/zookeeper_filter_test.cc | 20 +++++++++++++++++++ 5 files changed, 27 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 60f8997671134..bdd1976ad3c5a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -86,6 +86,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseSetWatchesRequest(data, offset, len); break; case enumToIntSigned(OpCodes::CLOSE): + callbacks_.onCloseRequest(); break; default: break; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index c558be85ed1ed..94681ee97edc7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -67,6 +67,7 @@ class DecoderCallbacks { virtual void onMultiRequest() PURE; virtual void onReconfigRequest() PURE; virtual void onSetWatchesRequest() PURE; + virtual void onCloseRequest() PURE; }; /** diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 02c7e47665307..77c9cf8fcb8d7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -149,6 +149,10 @@ void ZooKeeperFilter::onSetWatchesRequest() { // TODO: set metadata. } +void ZooKeeperFilter::onCloseRequest() { + config_->stats_.close_rq_.inc(); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 440537ed33215..2677c65fa6c7e 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -108,6 +108,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onMultiRequest() override; void onReconfigRequest() override; void onSetWatchesRequest() override; + void onCloseRequest() override; void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 9382b31a5db8d..f9deae7600202 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -67,6 +67,16 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodeCloseRequest() const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(8); + buffer.writeBEInt(1000); + buffer.writeBEInt(enumToInt(OpCodes::CLOSE)); + + return buffer; + } + Buffer::OwnedImpl encodeAuth(const std::string& scheme) const { Buffer::OwnedImpl buffer; @@ -503,6 +513,16 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, CloseRequest) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCloseRequest())); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().close_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions From 24b781c2173700fcba0e294329f893356a56282f Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 24 Feb 2019 15:56:05 -0800 Subject: [PATCH 028/114] Fix format Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 77c9cf8fcb8d7..98568d8ce2ba7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -149,9 +149,7 @@ void ZooKeeperFilter::onSetWatchesRequest() { // TODO: set metadata. } -void ZooKeeperFilter::onCloseRequest() { - config_->stats_.close_rq_.inc(); -} +void ZooKeeperFilter::onCloseRequest() { config_->stats_.close_rq_.inc(); } } // namespace ZooKeeperProxy } // namespace NetworkFilters From 4d3ca2bd922bf58fa42e5f462c6cb1cc9dbb7fd6 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 24 Feb 2019 18:53:04 -0800 Subject: [PATCH 029/114] Set the corresponding metadata for each request Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 5 + .../zookeeper_proxy/zookeeper_filter.cc | 97 +++++++++++++------ .../zookeeper_proxy/zookeeper_filter.h | 2 + .../zookeeper_proxy/zookeeper_filter_test.cc | 50 ++++++++++ 4 files changed, 127 insertions(+), 27 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index b4911bf5d9435..adad6828f08d6 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -82,3 +82,8 @@ The ZooKeeper filter emits the following dynamic metadata for each message parse :widths: 1, 1, 2 , string, "The path associated with the request, response or event" + , string, "The opname for the request, response or event" + , string, "True if a watch is being set, false otherwise" + , string, "True if the create znode will be ephemeral, false otherwise" + , string, "True if the create znode will be sequential, false otherwise" + , string, "The version parameter, if any, given with the request" diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 98568d8ce2ba7..0a582d352004c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -58,98 +58,141 @@ DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks) { return std::make_unique(callbacks); } +void ZooKeeperFilter::setDynamicMetadata(const std::string& key, const std::string& value) { + setDynamicMetadata({{key, value}}); +} + +void ZooKeeperFilter::setDynamicMetadata( + const std::vector>& data) { + envoy::api::v2::core::Metadata& dynamic_metadata = + read_callbacks_->connection().streamInfo().dynamicMetadata(); + ProtobufWkt::Struct metadata( + (*dynamic_metadata.mutable_filter_metadata())[NetworkFilterNames::get().ZooKeeperProxy]); + auto& fields = *metadata.mutable_fields(); + + for (const auto& pair : data) { + auto val = ProtobufWkt::Value(); + val.set_string_value(pair.second); + fields.insert({pair.first, val}); + } + + read_callbacks_->connection().streamInfo().setDynamicMetadata( + NetworkFilterNames::get().ZooKeeperProxy, metadata); +} + void ZooKeeperFilter::onConnect(const bool readonly) { if (readonly) { config_->stats_.connect_readonly_rq_.inc(); + setDynamicMetadata("opname", "connect_readonly"); } else { config_->stats_.connect_rq_.inc(); + setDynamicMetadata("opname", "connect"); } - - // TODO: set dynamic metadata. } -void ZooKeeperFilter::onDecodeError() { config_->stats_.decoder_error_.inc(); } +void ZooKeeperFilter::onDecodeError() { + config_->stats_.decoder_error_.inc(); + setDynamicMetadata("opname", "error"); +} -void ZooKeeperFilter::onPing() { config_->stats_.ping_rq_.inc(); } +void ZooKeeperFilter::onPing() { + config_->stats_.ping_rq_.inc(); + setDynamicMetadata("opname", "ping"); +} void ZooKeeperFilter::onAuthRequest(const std::string& scheme) { config_->scope_.counter(fmt::format("{}.auth.{}_rq", config_->stat_prefix_, scheme)).inc(); + setDynamicMetadata("opname", "auth"); } -void ZooKeeperFilter::onGetDataRequest(const std::string&, const bool) { +void ZooKeeperFilter::onGetDataRequest(const std::string& path, const bool watch) { config_->stats_.getdata_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata({{"opname", "getdata"}, {"path", path}, {"watch", watch ? "true" : "false"}}); } -void ZooKeeperFilter::onCreateRequest(const std::string&, const bool, const bool, const bool two) { +void ZooKeeperFilter::onCreateRequest(const std::string& path, const bool ephemeral, + const bool sequence, const bool two) { if (!two) { config_->stats_.create_rq_.inc(); + setDynamicMetadata({{"opname", "create"}, + {"path", path}, + {"ephemeral", ephemeral ? "true" : "false"}, + {"sequence", sequence ? "true" : "false"}}); } else { config_->stats_.create2_rq_.inc(); + setDynamicMetadata({{"opname", "create2"}, + {"path", path}, + {"ephemeral", ephemeral ? "true" : "false"}, + {"sequence", sequence ? "true" : "false"}}); } - // TODO: set metadata. } -void ZooKeeperFilter::onSetRequest(const std::string&) { +void ZooKeeperFilter::onSetRequest(const std::string& path) { config_->stats_.setdata_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata({{"opname", "setdata"}, {"path", path}}); } -void ZooKeeperFilter::onGetChildrenRequest(const std::string&, const bool, const bool two) { +void ZooKeeperFilter::onGetChildrenRequest(const std::string& path, const bool watch, + const bool two) { if (!two) { config_->stats_.getchildren_rq_.inc(); + setDynamicMetadata( + {{"opname", "getchildren"}, {"path", path}, {"watch", watch ? "true" : "false"}}); } else { config_->stats_.getchildren2_rq_.inc(); + setDynamicMetadata( + {{"opname", "getchildren2"}, {"path", path}, {"watch", watch ? "true" : "false"}}); } - // TODO: set metadata. } -void ZooKeeperFilter::onDeleteRequest(const std::string&, const int32_t) { +void ZooKeeperFilter::onDeleteRequest(const std::string& path, const int32_t version) { config_->stats_.remove_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata({{"opname", "remove"}, {"path", path}, {"version", std::to_string(version)}}); } -void ZooKeeperFilter::onExistsRequest(const std::string&, const bool) { +void ZooKeeperFilter::onExistsRequest(const std::string& path, const bool watch) { config_->stats_.exists_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata({{"opname", "exists"}, {"path", path}, {"watch", watch ? "true" : "false"}}); } -void ZooKeeperFilter::onGetAclRequest(const std::string&) { +void ZooKeeperFilter::onGetAclRequest(const std::string& path) { config_->stats_.getacl_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata({{"opname", "getacl"}, {"path", path}}); } -void ZooKeeperFilter::onSetAclRequest(const std::string&, const int32_t) { +void ZooKeeperFilter::onSetAclRequest(const std::string& path, const int32_t version) { config_->stats_.setacl_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata({{"opname", "setacl"}, {"path", path}, {"version", std::to_string(version)}}); } -void ZooKeeperFilter::onSyncRequest(const std::string&) { +void ZooKeeperFilter::onSyncRequest(const std::string& path) { config_->stats_.sync_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata({{"opname", "sync"}, {"path", path}}); } void ZooKeeperFilter::onCheckRequest(const std::string&, const int32_t) { config_->stats_.check_rq_.inc(); - // TODO: set metadata. } void ZooKeeperFilter::onMultiRequest() { config_->stats_.multi_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata("opname", "multi"); } void ZooKeeperFilter::onReconfigRequest() { config_->stats_.reconfig_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata("opname", "reconfig"); } void ZooKeeperFilter::onSetWatchesRequest() { config_->stats_.setwatches_rq_.inc(); - // TODO: set metadata. + setDynamicMetadata("opname", "setwatches"); } -void ZooKeeperFilter::onCloseRequest() { config_->stats_.close_rq_.inc(); } +void ZooKeeperFilter::onCloseRequest() { + config_->stats_.close_rq_.inc(); + setDynamicMetadata("opname", "close"); +} } // namespace ZooKeeperProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 2677c65fa6c7e..82b59d9be2b88 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -112,6 +112,8 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); + void setDynamicMetadata(const std::string& key, const std::string& value); + void setDynamicMetadata(const std::vector>& data); private: Network::ReadFilterCallbacks* read_callbacks_{}; diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index f9deae7600202..ef5749f50e4f5 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -15,6 +15,15 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { +MATCHER_P(MapEq, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(rhs.size() > 0); + for (auto const& entry : rhs) { + EXPECT_EQ(obj.fields().at(entry.first).string_value(), entry.second); + } + return true; +} + class ZooKeeperFilterTest : public TestBase { public: ZooKeeperFilterTest() { ENVOY_LOG_MISC(info, "test"); } @@ -307,11 +316,19 @@ class ZooKeeperFilterTest : public TestBase { } } + void expectSetDynamicMetadata(const std::map& expected) { + EXPECT_CALL(filter_callbacks_.connection_, streamInfo()) + .WillRepeatedly(ReturnRef(stream_info_)); + EXPECT_CALL(stream_info_, + setDynamicMetadata("envoy.filters.network.zookeeper_proxy", MapEq(expected))); + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; std::string stat_prefix_{"test.zookeeper"}; NiceMock filter_callbacks_; + NiceMock stream_info_; }; // Test Connect counter increment. @@ -320,6 +337,8 @@ TEST_F(ZooKeeperFilterTest, Connect) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeConnect())); + expectSetDynamicMetadata({{"opname", "connect"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().connect_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -331,6 +350,8 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeConnect(true))); + expectSetDynamicMetadata({{"opname", "connect_readonly"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); EXPECT_EQ(1UL, config_->stats().connect_readonly_rq_.value()); @@ -354,6 +375,8 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePing())); + expectSetDynamicMetadata({{"opname", "ping"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().ping_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -364,6 +387,8 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeAuth("digest"))); + expectSetDynamicMetadata({{"opname", "auth"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(scope_.counter("test.zookeeper.auth.digest_rq").value(), 1); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -374,6 +399,8 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePathWatch("/foo", true))); + expectSetDynamicMetadata({{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -384,6 +411,9 @@ TEST_F(ZooKeeperFilterTest, CreateRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCreateRequest("/foo", "bar", false, false))); + expectSetDynamicMetadata( + {{"opname", "create"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -394,6 +424,8 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetRequest("/foo", "bar", -1))); + expectSetDynamicMetadata({{"opname", "setdata"}, {"path", "/foo"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -405,6 +437,8 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN)))); + expectSetDynamicMetadata({{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -415,6 +449,8 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeDeleteRequest("/foo", -1))); + expectSetDynamicMetadata({{"opname", "remove"}, {"path", "/foo"}, {"version", "-1"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().remove_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -426,6 +462,8 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::EXISTS)))); + expectSetDynamicMetadata({{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().exists_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -436,6 +474,8 @@ TEST_F(ZooKeeperFilterTest, GetAclRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::GETACL)))); + expectSetDynamicMetadata({{"opname", "getacl"}, {"path", "/foo"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getacl_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -447,6 +487,8 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodeSetAclRequest("/foo", "digest", "passwd", -1))); + expectSetDynamicMetadata({{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setacl_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -457,6 +499,8 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::SYNC)))); + expectSetDynamicMetadata({{"opname", "sync"}, {"path", "/foo"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().sync_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -494,6 +538,8 @@ TEST_F(ZooKeeperFilterTest, ReconfigRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeReconfigRequest("s1", "s2", "s3", 1000))); + expectSetDynamicMetadata({{"opname", "reconfig"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().reconfig_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -508,6 +554,8 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetWatchesRequest(dataw, existw, childw))); + expectSetDynamicMetadata({{"opname", "setwatches"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -518,6 +566,8 @@ TEST_F(ZooKeeperFilterTest, CloseRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCloseRequest())); + expectSetDynamicMetadata({{"opname", "close"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().close_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); From 6ba1f21c951cea5072a1f6be4c099f7dc3fb13c7 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 27 Feb 2019 11:59:34 -0800 Subject: [PATCH 030/114] Skip obvious comments Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index bdd1976ad3c5a..37681c704ac03 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -162,9 +162,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u int32_t datalen; BufferHelper::peekInt32(data, offset, datalen); offset += datalen; - // Skip acls. skipAcls(data, offset); - // Flags. int32_t flags; BufferHelper::peekInt32(data, offset, flags); const bool ephemeral = (flags & 0x1) == 1; @@ -182,7 +180,6 @@ void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint int32_t datalen; BufferHelper::peekInt32(data, offset, datalen); offset += datalen; - // Version. int32_t version; BufferHelper::peekInt32(data, offset, version); @@ -206,7 +203,6 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u std::string path; BufferHelper::peekString(data, offset, path); - // Version. int32_t version; BufferHelper::peekInt32(data, offset, version); @@ -238,9 +234,7 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u std::string path; BufferHelper::peekString(data, offset, path); - // Skip acls. skipAcls(data, offset); - // Version. int32_t version; BufferHelper::peekInt32(data, offset, version); @@ -261,7 +255,6 @@ void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, ui std::string path; BufferHelper::peekString(data, offset, path); - // Version. int32_t version; BufferHelper::peekInt32(data, offset, version); From e39825d0ba7a875b9630879a891c534bfc5bd58f Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 27 Feb 2019 12:00:46 -0800 Subject: [PATCH 031/114] Few stylistic items * use explicit when possible * drop default destructor Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 82b59d9be2b88..8839159afe500 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -79,8 +79,7 @@ using ZooKeeperFilterConfigSharedPtr = std::shared_ptr; */ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggable { public: - ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config); - ~ZooKeeperFilter() override = default; + explicit ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config); // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; From ac3b3d1b435fb4c8a1e36d80b7198dfcb36ff0e8 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 27 Feb 2019 12:02:32 -0800 Subject: [PATCH 032/114] Describe regular vs readonly connect requests Signed-off-by: Raul Gutierrez Segales --- .../configuration/network_filters/zookeeper_proxy_filter.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index adad6828f08d6..adfb5bce78e38 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -49,7 +49,7 @@ following statistics: :widths: 1, 1, 2 decoder_error, Counter, Number of times a message wasn't decoded - connect_rq, Counter, Number of connect requests + connect_rq, Counter, Number of regular connect (non-readonly) requests connect_readonly_rq, Counter, Number of connect requests with the readonly flag set ping_rq, Counter, Number of ping requests auth._rq, Counter, Number of auth requests for a given type From 83ff78caf987c643b37623334c1db247b8adcad8 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 2 Mar 2019 09:40:53 -0800 Subject: [PATCH 033/114] Add comment noting that ZK uses network ordering Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h index 77318aba24b06..9b450860344d4 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -14,6 +14,8 @@ namespace ZooKeeperProxy { /** * IO helpers for reading/writing ZooKeeper data from/to a buffer. + * + * Note: ZooKeeper's protocol uses network byte ordering (big-endian). */ class BufferHelper : public Logger::Loggable { public: From 2192a9d984d682a51d2cd4e48e7bc340ae51ab13 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 2 Mar 2019 16:51:06 -0800 Subject: [PATCH 034/114] Centralize error handling when parsing Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 35 ++++++-------- .../zookeeper_proxy/zookeeper_utils.cc | 46 +++++-------------- .../network/zookeeper_proxy/zookeeper_utils.h | 8 ++-- 3 files changed, 31 insertions(+), 58 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 37681c704ac03..3129b6c4b5d0c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -95,8 +95,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { #define CHECK_LENGTH(LEN, MINL) \ if (LEN < MINL) { \ - callbacks_.onDecodeError(); \ - return; \ + throw new EnvoyException("Package is too small"); \ } void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { @@ -109,7 +108,11 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ // Read readonly flag, if it's there. bool readonly{}; - BufferHelper::peekBool(data, offset, readonly); + try { + BufferHelper::peekBool(data, offset, readonly); + } catch (EnvoyException& e) { + // Old libraries don't populate the readonly field. + } callbacks_.onConnect(readonly); } @@ -267,22 +270,11 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui while (true) { int32_t type; - if (!BufferHelper::peekInt32(data, offset, type)) { - callbacks_.onDecodeError(); - return; - } - + BufferHelper::peekInt32(data, offset, type); bool done{}; - if (!BufferHelper::peekBool(data, offset, done)) { - callbacks_.onDecodeError(); - return; - } - + BufferHelper::peekBool(data, offset, done); int32_t error; - if (!BufferHelper::peekInt32(data, offset, error)) { - callbacks_.onDecodeError(); - return; - } + BufferHelper::peekInt32(data, offset, error); if (done) { break; @@ -300,8 +292,7 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui break; default: // Should not happen. - callbacks_.onDecodeError(); - break; + throw new EnvoyException("Unknown type within a transaction"); } } @@ -355,7 +346,11 @@ void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; while (offset < data.length()) { - decode(data, offset); + try { + decode(data, offset); + } catch (EnvoyException& e) { + callbacks_.onDecodeError(); + } } } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index b4b6081c34bde..23da880035a8b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -5,56 +5,34 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -bool BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val) { - try { - val = buffer.peekBEInt(offset); - offset += sizeof(int32_t); - return true; - } catch (EnvoyException& e) { - return false; - } +void BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val) { + val = buffer.peekBEInt(offset); + offset += sizeof(int32_t); } -bool BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val) { - try { - val = buffer.peekBEInt(offset); - offset += sizeof(int64_t); - return true; - } catch (EnvoyException& e) { - return false; - } +void BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val) { + val = buffer.peekBEInt(offset); + offset += sizeof(int64_t); } -bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val) { - if (buffer.length() < (offset + 1)) { - return false; - } - - try { - std::string str; - char byte = buffer.peekInt(offset); - val = static_cast(byte & 255); - offset += 1; - return true; - } catch (EnvoyException& e) { - return false; - } +void BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val) { + char byte = buffer.peekInt(offset); + val = static_cast(byte & 255); + offset += 1; } -bool BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str) { +void BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str) { int32_t len = 0; peekInt32(buffer, offset, len); if (buffer.length() < (offset + len) || !len) { - return false; + return; } std::unique_ptr data(new char[len]); buffer.copyOut(offset, len, data.get()); str.assign(data.get(), len); offset += len; - - return true; } } // namespace ZooKeeperProxy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h index 9b450860344d4..8c161980e3ae9 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -19,10 +19,10 @@ namespace ZooKeeperProxy { */ class BufferHelper : public Logger::Loggable { public: - static bool peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val); - static bool peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val); - static bool peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str); - static bool peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val); + static void peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val); + static void peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val); + static void peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str); + static void peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val); }; } // namespace ZooKeeperProxy From 2db08fefdfbb4e8d4903034024cb99d560e154d0 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 19:55:54 -0800 Subject: [PATCH 035/114] Improve coverage This should get us close to 100% coverage, for the most important files. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 65 +++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index ef5749f50e4f5..1f6bd93e55fe1 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -121,13 +121,16 @@ class ZooKeeperFilterTest : public TestBase { } Buffer::OwnedImpl encodePathVersion(const std::string& path, const int32_t version, - const int32_t opcode = enumToInt(OpCodes::GETDATA)) const { + const int32_t opcode = enumToInt(OpCodes::GETDATA), + const bool txn = false) const { Buffer::OwnedImpl buffer; - buffer.writeBEInt(16 + path.length()); - buffer.writeBEInt(1000); - // Opcode. - buffer.writeBEInt(opcode); + if (!txn) { + buffer.writeBEInt(16 + path.length()); + buffer.writeBEInt(1000); + buffer.writeBEInt(opcode); + } + // Path. addString(buffer, path); // Version @@ -151,13 +154,14 @@ class ZooKeeperFilterTest : public TestBase { Buffer::OwnedImpl encodeCreateRequest(const std::string& path, const std::string& data, const bool ephemeral, const bool sequence, - const bool txn = false) const { + const bool txn = false, + const int32_t opcode = enumToInt(OpCodes::CREATE)) const { Buffer::OwnedImpl buffer; if (!txn) { buffer.writeBEInt(24 + path.length() + data.length()); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToInt(OpCodes::CREATE)); + buffer.writeBEInt(opcode); } // Path. @@ -180,13 +184,15 @@ class ZooKeeperFilterTest : public TestBase { } Buffer::OwnedImpl encodeSetRequest(const std::string& path, const std::string& data, - const int32_t version) const { + const int32_t version, const bool txn = false) const { Buffer::OwnedImpl buffer; - buffer.writeBEInt(20 + path.length() + data.length()); - buffer.writeBEInt(1000); - // Opcode. - buffer.writeBEInt(enumToInt(OpCodes::SETDATA)); + if (!txn) { + buffer.writeBEInt(20 + path.length() + data.length()); + buffer.writeBEInt(1000); + buffer.writeBEInt(enumToInt(OpCodes::SETDATA)); + } + // Path. addString(buffer, path); // Data. @@ -419,6 +425,20 @@ TEST_F(ZooKeeperFilterTest, CreateRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, CreateRequest2) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl( + encodeCreateRequest("/foo", "bar", false, false, false, enumToInt(OpCodes::CREATE2)))); + + expectSetDynamicMetadata( + {{"opname", "create2"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().create2_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, SetRequest) { initialize(); @@ -444,6 +464,19 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { + initialize(); + + Buffer::InstancePtr data( + new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN2)))); + + expectSetDynamicMetadata({{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().getchildren2_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, DeleteRequest) { initialize(); @@ -522,14 +555,22 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { Buffer::OwnedImpl create1 = encodeCreateRequest("/foo", "1", false, false, true); Buffer::OwnedImpl create2 = encodeCreateRequest("/bar", "1", false, false, true); + Buffer::OwnedImpl check1 = encodePathVersion("/foo", 100, enumToInt(OpCodes::CHECK), true); + Buffer::OwnedImpl set1 = encodeSetRequest("/bar", "2", -1, true); + std::vector> ops; ops.push_back(std::make_pair(enumToInt(OpCodes::CREATE), std::move(create1))); ops.push_back(std::make_pair(enumToInt(OpCodes::CREATE), std::move(create2))); + ops.push_back(std::make_pair(enumToInt(OpCodes::CHECK), std::move(check1))); + ops.push_back(std::make_pair(enumToInt(OpCodes::SETDATA), std::move(set1))); + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeMultiRequest(ops))); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().multi_rq_.value()); EXPECT_EQ(2UL, config_->stats().create_rq_.value()); + EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); + EXPECT_EQ(1UL, config_->stats().check_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } From e31523399812cf6dd82cc5ba3c816b98c47b1459 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 20:57:41 -0800 Subject: [PATCH 036/114] onWrite() should be used for responses Currently, we only decode requests. So lets not call it from OnWrite() because we'll only get decoding errors. I am runnign a local test using zk-shell and I am seeing some decoding errors that shouldnt happen: ``` ~/src > curl -s 0:8081/stats | grep zookeeper test_using_kazoo.zookeeper.check_rq: 0 test_using_kazoo.zookeeper.close_rq: 0 test_using_kazoo.zookeeper.connect_readonly_rq: 0 test_using_kazoo.zookeeper.connect_rq: 1 test_using_kazoo.zookeeper.create2_rq: 0 test_using_kazoo.zookeeper.create_rq: 1 test_using_kazoo.zookeeper.decoder_error: 8 test_using_kazoo.zookeeper.exists_rq: 6 test_using_kazoo.zookeeper.getacl_rq: 0 test_using_kazoo.zookeeper.getchildren2_rq: 0 test_using_kazoo.zookeeper.getchildren_rq: 8 test_using_kazoo.zookeeper.getdata_rq: 1 test_using_kazoo.zookeeper.multi_rq: 0 test_using_kazoo.zookeeper.ping_rq: 8 test_using_kazoo.zookeeper.reconfig_rq: 0 test_using_kazoo.zookeeper.remove_rq: 0 test_using_kazoo.zookeeper.setacl_rq: 0 test_using_kazoo.zookeeper.setauth_rq: 0 test_using_kazoo.zookeeper.setdata_rq: 1 test_using_kazoo.zookeeper.setwatches_rq: 0 test_using_kazoo.zookeeper.sync_rq: 0 ``` Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 0a582d352004c..d1a0e2ac2201e 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -27,8 +27,7 @@ Network::FilterStatus ZooKeeperFilter::onData(Buffer::Instance& data, bool) { return Network::FilterStatus::Continue; } -Network::FilterStatus ZooKeeperFilter::onWrite(Buffer::Instance& data, bool) { - doDecode(data); +Network::FilterStatus ZooKeeperFilter::onWrite(Buffer::Instance&, bool) { return Network::FilterStatus::Continue; } From edf37a8b24e2de53e50b058b55094ac65aaeed45 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 21:02:46 -0800 Subject: [PATCH 037/114] Fix mem leak Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 3129b6c4b5d0c..3d30903f60c84 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -95,7 +95,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { #define CHECK_LENGTH(LEN, MINL) \ if (LEN < MINL) { \ - throw new EnvoyException("Package is too small"); \ + throw EnvoyException("Package is too small"); \ } void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { @@ -292,7 +292,7 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui break; default: // Should not happen. - throw new EnvoyException("Unknown type within a transaction"); + throw EnvoyException("Unknown type within a transaction"); } } From 5cac7fa06c0eda957c425ee35cf171d8e984a63a Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 21:18:33 -0800 Subject: [PATCH 038/114] Fix decoding error with Pings I forgot that Ping requests had opcodes, too. Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 3d30903f60c84..f27989627c4b3 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -27,6 +27,8 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseConnect(data, offset, len); return; case enumToIntSigned(XidCodes::PING_XID): + // Skip opcode. + offset += 4; callbacks_.onPing(); return; case enumToIntSigned(XidCodes::AUTH_XID): From 020d46af916e20096b0b56cd0f76c61dbde02a2c Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 21:22:07 -0800 Subject: [PATCH 039/114] Address nit Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 23da880035a8b..5e1cfd9d688bd 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -25,7 +25,7 @@ void BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::s int32_t len = 0; peekInt32(buffer, offset, len); - if (buffer.length() < (offset + len) || !len) { + if (buffer.length() < (offset + len) || len == 0) { return; } From c241f2505478c95b26dead14ea208364901cef79 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 21:33:07 -0800 Subject: [PATCH 040/114] Don't use exceptions to check if a bool is avail Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index f27989627c4b3..01667fdec581a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -110,10 +110,8 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ // Read readonly flag, if it's there. bool readonly{}; - try { + if (data.length() >= offset + 1) { BufferHelper::peekBool(data, offset, readonly); - } catch (EnvoyException& e) { - // Old libraries don't populate the readonly field. } callbacks_.onConnect(readonly); From 6cc7279be82eefbaba0d67bc58d50f6e728a9a57 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 21:36:40 -0800 Subject: [PATCH 041/114] Make check length a helper and not a macro Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 37 ++++++++++--------- .../zookeeper_proxy/zookeeper_decoder.h | 1 + 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 01667fdec581a..5561ee2c522d9 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -95,13 +95,14 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { } } -#define CHECK_LENGTH(LEN, MINL) \ - if (LEN < MINL) { \ - throw EnvoyException("Package is too small"); \ +void DecoderImpl::checkLength(const int32_t len, const int32_t minlen) const { + if (len < minlen) { + throw EnvoyException("Package is too small"); } +} void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 28); + checkLength(len, 28); // Read password - skip zxid, timeout, and session id. std::string passwd; @@ -118,7 +119,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ } void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 20); + checkLength(len, 20); // Skip opcode + type. offset += 8; @@ -131,7 +132,7 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 13); + checkLength(len, 13); std::string path; BufferHelper::peekString(data, offset, path); @@ -157,7 +158,7 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - CHECK_LENGTH(len, 20); + checkLength(len, 20); std::string path; BufferHelper::peekString(data, offset, path); @@ -175,7 +176,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 20); + checkLength(len, 20); std::string path; BufferHelper::peekString(data, offset, path); @@ -191,7 +192,7 @@ void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - CHECK_LENGTH(len, 14); + checkLength(len, 14); std::string path; BufferHelper::peekString(data, offset, path); @@ -202,7 +203,7 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs } void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 16); + checkLength(len, 16); std::string path; BufferHelper::peekString(data, offset, path); @@ -213,7 +214,7 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 13); + checkLength(len, 13); std::string path; BufferHelper::peekString(data, offset, path); @@ -224,7 +225,7 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 8); + checkLength(len, 8); std::string path; BufferHelper::peekString(data, offset, path); @@ -233,7 +234,7 @@ void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 8); + checkLength(len, 8); std::string path; BufferHelper::peekString(data, offset, path); @@ -245,7 +246,7 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 8); + checkLength(len, 8); std::string path; BufferHelper::peekString(data, offset, path); @@ -254,7 +255,7 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 8); + checkLength(len, 8); std::string path; BufferHelper::peekString(data, offset, path); @@ -266,7 +267,7 @@ void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, ui void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { // Treat empty transactions as a decoding error, there should be at least 1 header. - CHECK_LENGTH(len, 17); + checkLength(len, 17); while (true) { int32_t type; @@ -300,7 +301,7 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui } void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 28); + checkLength(len, 28); // Skip joining. int32_t datalen; @@ -320,7 +321,7 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, } void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - CHECK_LENGTH(len, 12); + checkLength(len, 12); // Data watches. skipStrings(data, offset); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 94681ee97edc7..3a715587f58fc 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -109,6 +109,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void skipStrings(Buffer::Instance& data, uint64_t& offset) const; + void checkLength(const int32_t len, const int32_t minlen) const; DecoderCallbacks& callbacks_; }; From 701cce0ed8f542015555b66ea999e047d6c99343 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 21:37:50 -0800 Subject: [PATCH 042/114] Avoid potential infinite loops Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 5561ee2c522d9..f05e0831b2306 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -346,12 +346,12 @@ void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; - while (offset < data.length()) { - try { + try { + while (offset < data.length()) { decode(data, offset); - } catch (EnvoyException& e) { - callbacks_.onDecodeError(); } + } catch (EnvoyException& e) { + callbacks_.onDecodeError(); } } From 354bd32afd7a6349b016af8cfe68fe78dd6bcc43 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 21:46:24 -0800 Subject: [PATCH 043/114] Split empty string vs not big enough buffer When a string is empty, it's fine to silently return - this is valid. When the length of a string is bigger than the available buffer, we throw an exception. Though this potentially could cause issues depending on how buffering happens for network filters. I'll need more testing here. Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 5e1cfd9d688bd..da7b5ad3d2d3c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -25,10 +25,14 @@ void BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::s int32_t len = 0; peekInt32(buffer, offset, len); - if (buffer.length() < (offset + len) || len == 0) { + if (len == 0) { return; } + if (buffer.length() < (offset + len)) { + throw EnvoyException("buffer is too small"); + } + std::unique_ptr data(new char[len]); buffer.copyOut(offset, len, data.get()); str.assign(data.get(), len); From 4eb510f9e47bee6438a2efa1561f0f4fa35e3fa9 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 4 Mar 2019 22:02:55 -0800 Subject: [PATCH 044/114] Use return values instead of out param references Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 121 +++++++----------- .../zookeeper_proxy/zookeeper_utils.cc | 27 ++-- .../network/zookeeper_proxy/zookeeper_utils.h | 8 +- 3 files changed, 63 insertions(+), 93 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index f05e0831b2306..4a953d5bd920b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -11,17 +11,14 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); // Check message length. - int32_t len; - BufferHelper::peekInt32(data, offset, len); + int32_t len = BufferHelper::peekInt32(data, offset); if (len < 8) { callbacks_.onDecodeError(); return; } - int32_t xid; - BufferHelper::peekInt32(data, offset, xid); - // "Special" requests. + int32_t xid = BufferHelper::peekInt32(data, offset); switch (xid) { case enumToIntSigned(XidCodes::CONNECT_XID): parseConnect(data, offset, len); @@ -39,8 +36,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { } // "Regular" requests. - int32_t opcode; - BufferHelper::peekInt32(data, offset, opcode); + int32_t opcode = BufferHelper::peekInt32(data, offset); switch (opcode) { case enumToInt(OpCodes::GETDATA): parseGetDataRequest(data, offset, len); @@ -105,14 +101,13 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ checkLength(len, 28); // Read password - skip zxid, timeout, and session id. - std::string passwd; offset += 20; - BufferHelper::peekString(data, offset, passwd); + std::string passwd = BufferHelper::peekString(data, offset); // Read readonly flag, if it's there. bool readonly{}; if (data.length() >= offset + 1) { - BufferHelper::peekBool(data, offset, readonly); + readonly = BufferHelper::peekBool(data, offset); } callbacks_.onConnect(readonly); @@ -123,10 +118,8 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin // Skip opcode + type. offset += 8; - std::string scheme; - BufferHelper::peekString(data, offset, scheme); - std::string credential; - BufferHelper::peekString(data, offset, credential); + std::string scheme = BufferHelper::peekString(data, offset); + std::string credential = BufferHelper::peekString(data, offset); callbacks_.onAuthRequest(scheme); } @@ -134,25 +127,19 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 13); - std::string path; - BufferHelper::peekString(data, offset, path); - bool watch; - BufferHelper::peekBool(data, offset, watch); + std::string path = BufferHelper::peekString(data, offset); + bool watch = BufferHelper::peekBool(data, offset); callbacks_.onGetDataRequest(path, watch); } void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { - int32_t count; - BufferHelper::peekInt32(data, offset, count); + int32_t count = BufferHelper::peekInt32(data, offset); for (int i = 0; i < count; ++i) { - int32_t perms; - BufferHelper::peekInt32(data, offset, perms); - std::string scheme; - BufferHelper::peekString(data, offset, scheme); - std::string credential; - BufferHelper::peekString(data, offset, credential); + BufferHelper::peekInt32(data, offset); + BufferHelper::peekString(data, offset); + BufferHelper::peekString(data, offset); } } @@ -160,15 +147,13 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u const bool two) { checkLength(len, 20); - std::string path; - BufferHelper::peekString(data, offset, path); + std::string path = BufferHelper::peekString(data, offset); + // Skip data. - int32_t datalen; - BufferHelper::peekInt32(data, offset, datalen); + int32_t datalen = BufferHelper::peekInt32(data, offset); offset += datalen; skipAcls(data, offset); - int32_t flags; - BufferHelper::peekInt32(data, offset, flags); + int32_t flags = BufferHelper::peekInt32(data, offset); const bool ephemeral = (flags & 0x1) == 1; const bool sequence = (flags & 0x2) == 2; @@ -178,14 +163,12 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 20); - std::string path; - BufferHelper::peekString(data, offset, path); + std::string path = BufferHelper::peekString(data, offset); // Skip data. - int32_t datalen; - BufferHelper::peekInt32(data, offset, datalen); + int32_t datalen = BufferHelper::peekInt32(data, offset); offset += datalen; - int32_t version; - BufferHelper::peekInt32(data, offset, version); + // Ignore version. + BufferHelper::peekInt32(data, offset); callbacks_.onSetRequest(path); } @@ -194,10 +177,8 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs const bool two) { checkLength(len, 14); - std::string path; - BufferHelper::peekString(data, offset, path); - bool watch; - BufferHelper::peekBool(data, offset, watch); + std::string path = BufferHelper::peekString(data, offset); + bool watch = BufferHelper::peekBool(data, offset); callbacks_.onGetChildrenRequest(path, watch, two); } @@ -205,10 +186,8 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 16); - std::string path; - BufferHelper::peekString(data, offset, path); - int32_t version; - BufferHelper::peekInt32(data, offset, version); + std::string path = BufferHelper::peekString(data, offset); + int32_t version = BufferHelper::peekInt32(data, offset); callbacks_.onDeleteRequest(path, version); } @@ -216,10 +195,8 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 13); - std::string path; - BufferHelper::peekString(data, offset, path); - bool watch; - BufferHelper::peekBool(data, offset, watch); + std::string path = BufferHelper::peekString(data, offset); + bool watch = BufferHelper::peekBool(data, offset); callbacks_.onExistsRequest(path, watch); } @@ -227,8 +204,7 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path; - BufferHelper::peekString(data, offset, path); + std::string path = BufferHelper::peekString(data, offset); callbacks_.onGetAclRequest(path); } @@ -236,11 +212,9 @@ void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path; - BufferHelper::peekString(data, offset, path); + std::string path = BufferHelper::peekString(data, offset); skipAcls(data, offset); - int32_t version; - BufferHelper::peekInt32(data, offset, version); + int32_t version = BufferHelper::peekInt32(data, offset); callbacks_.onSetAclRequest(path, version); } @@ -248,8 +222,7 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path; - BufferHelper::peekString(data, offset, path); + std::string path = BufferHelper::peekString(data, offset); callbacks_.onSyncRequest(path); } @@ -257,10 +230,8 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path; - BufferHelper::peekString(data, offset, path); - int32_t version; - BufferHelper::peekInt32(data, offset, version); + std::string path = BufferHelper::peekString(data, offset); + int32_t version = BufferHelper::peekInt32(data, offset); callbacks_.onCheckRequest(path, version); } @@ -270,12 +241,10 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui checkLength(len, 17); while (true) { - int32_t type; - BufferHelper::peekInt32(data, offset, type); - bool done{}; - BufferHelper::peekBool(data, offset, done); - int32_t error; - BufferHelper::peekInt32(data, offset, error); + int32_t type = BufferHelper::peekInt32(data, offset); + bool done = BufferHelper::peekBool(data, offset); + // Ignore error field. + BufferHelper::peekInt32(data, offset); if (done) { break; @@ -304,18 +273,16 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, checkLength(len, 28); // Skip joining. - int32_t datalen; - BufferHelper::peekInt32(data, offset, datalen); + int32_t datalen = BufferHelper::peekInt32(data, offset); offset += datalen; // Skip leaving. - BufferHelper::peekInt32(data, offset, datalen); + datalen = BufferHelper::peekInt32(data, offset); offset += datalen; // Skip new members. - BufferHelper::peekInt32(data, offset, datalen); + datalen = BufferHelper::peekInt32(data, offset); offset += datalen; // Read config id. - int64_t config_id; - BufferHelper::peekInt64(data, offset, config_id); + BufferHelper::peekInt64(data, offset); callbacks_.onReconfigRequest(); } @@ -334,12 +301,10 @@ void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offse } void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { - int32_t count; - BufferHelper::peekInt32(data, offset, count); + int32_t count = BufferHelper::peekInt32(data, offset); for (int i = 0; i < count; ++i) { - int32_t len; - BufferHelper::peekInt32(data, offset, len); + int32_t len = BufferHelper::peekInt32(data, offset); offset += len; } } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index da7b5ad3d2d3c..f90504bb0f6ab 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -5,28 +5,31 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -void BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val) { - val = buffer.peekBEInt(offset); +int32_t BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset) { + int32_t val = buffer.peekBEInt(offset); offset += sizeof(int32_t); + return val; } -void BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val) { - val = buffer.peekBEInt(offset); +int64_t BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset) { + int64_t val = buffer.peekBEInt(offset); offset += sizeof(int64_t); + return val; } -void BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val) { +bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { char byte = buffer.peekInt(offset); - val = static_cast(byte & 255); + bool val = static_cast(byte & 255); offset += 1; + return val; } -void BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str) { - int32_t len = 0; - peekInt32(buffer, offset, len); +std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) { + std::string val; + int32_t len = peekInt32(buffer, offset); if (len == 0) { - return; + return val; } if (buffer.length() < (offset + len)) { @@ -35,8 +38,10 @@ void BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset, std::s std::unique_ptr data(new char[len]); buffer.copyOut(offset, len, data.get()); - str.assign(data.get(), len); + val.assign(data.get(), len); offset += len; + + return val; } } // namespace ZooKeeperProxy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h index 8c161980e3ae9..87b53e165508e 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -19,10 +19,10 @@ namespace ZooKeeperProxy { */ class BufferHelper : public Logger::Loggable { public: - static void peekInt32(Buffer::Instance& buffer, uint64_t& offset, int32_t& val); - static void peekInt64(Buffer::Instance& buffer, uint64_t& offset, int64_t& val); - static void peekString(Buffer::Instance& buffer, uint64_t& offset, std::string& str); - static void peekBool(Buffer::Instance& buffer, uint64_t& offset, bool& val); + static int32_t peekInt32(Buffer::Instance& buffer, uint64_t& offset); + static int64_t peekInt64(Buffer::Instance& buffer, uint64_t& offset); + static std::string peekString(Buffer::Instance& buffer, uint64_t& offset); + static bool peekBool(Buffer::Instance& buffer, uint64_t& offset); }; } // namespace ZooKeeperProxy From 4c8204db540db018a3ccfcb4658152c83f42df5e Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 5 Mar 2019 19:49:20 -0800 Subject: [PATCH 045/114] Track request bytes This is a useful counter, plus it makes tests more robust. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 2 + .../zookeeper_proxy/zookeeper_decoder.cc | 7 +- .../zookeeper_proxy/zookeeper_decoder.h | 1 + .../zookeeper_proxy/zookeeper_filter.cc | 5 + .../zookeeper_proxy/zookeeper_filter.h | 2 + .../zookeeper_proxy/zookeeper_filter_test.cc | 97 ++++++++++++++----- 6 files changed, 84 insertions(+), 30 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index adfb5bce78e38..bae0ad41ef3bd 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -49,6 +49,7 @@ following statistics: :widths: 1, 1, 2 decoder_error, Counter, Number of times a message wasn't decoded + request_bytes, Counter, Number of bytes in decoded request messages connect_rq, Counter, Number of regular connect (non-readonly) requests connect_readonly_rq, Counter, Number of connect requests with the readonly flag set ping_rq, Counter, Number of ping requests @@ -83,6 +84,7 @@ The ZooKeeper filter emits the following dynamic metadata for each message parse , string, "The path associated with the request, response or event" , string, "The opname for the request, response or event" + , string, "The size of the request message in bytes" , string, "True if a watch is being set, false otherwise" , string, "True if the create znode will be ephemeral, false otherwise" , string, "True if the create znode will be sequential, false otherwise" diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 4a953d5bd920b..0496d8d22b937 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -12,10 +12,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // Check message length. int32_t len = BufferHelper::peekInt32(data, offset); - if (len < 8) { - callbacks_.onDecodeError(); - return; - } + checkLength(len, 8); // "Special" requests. int32_t xid = BufferHelper::peekInt32(data, offset); @@ -313,7 +310,9 @@ void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; try { while (offset < data.length()) { + uint64_t current = offset; decode(data, offset); + callbacks_.onRequestBytes(offset - current); } } catch (EnvoyException& e) { callbacks_.onDecodeError(); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 3a715587f58fc..cf1856025e534 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -50,6 +50,7 @@ class DecoderCallbacks { virtual ~DecoderCallbacks() {} virtual void onDecodeError() PURE; + virtual void onRequestBytes(const uint64_t bytes) PURE; virtual void onConnect(const bool readonly) PURE; virtual void onPing() PURE; virtual void onAuthRequest(const std::string& scheme) PURE; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index d1a0e2ac2201e..b4a2301f5d7e0 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -94,6 +94,11 @@ void ZooKeeperFilter::onDecodeError() { setDynamicMetadata("opname", "error"); } +void ZooKeeperFilter::onRequestBytes(const uint64_t bytes) { + config_->stats_.request_bytes_.add(bytes); + setDynamicMetadata("bytes", std::to_string(bytes)); +} + void ZooKeeperFilter::onPing() { config_->stats_.ping_rq_.inc(); setDynamicMetadata("opname", "ping"); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 8839159afe500..b60fbd4738e3b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -22,6 +22,7 @@ namespace ZooKeeperProxy { // clang-format off #define ALL_ZOOKEEPER_PROXY_STATS(COUNTER) \ COUNTER(decoder_error) \ + COUNTER(request_bytes) \ COUNTER(connect_rq) \ COUNTER(connect_readonly_rq) \ COUNTER(getdata_rq) \ @@ -91,6 +92,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab // ZooKeeperProxy::DecoderCallback void onDecodeError() override; + void onRequestBytes(const uint64_t bytes) override; void onConnect(const bool readonly) override; void onPing() override; void onAuthRequest(const std::string& scheme) override; diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 1f6bd93e55fe1..e42a5a150a149 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -15,8 +15,7 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -MATCHER_P(MapEq, rhs, "") { - const ProtobufWkt::Struct& obj = arg; +bool protoMapEq(const ProtobufWkt::Struct& obj, const std::map& rhs) { EXPECT_TRUE(rhs.size() > 0); for (auto const& entry : rhs) { EXPECT_EQ(obj.fields().at(entry.first).string_value(), entry.second); @@ -24,6 +23,8 @@ MATCHER_P(MapEq, rhs, "") { return true; } +MATCHER_P(MapEq, rhs, "") { return protoMapEq(arg, rhs); } + class ZooKeeperFilterTest : public TestBase { public: ZooKeeperFilterTest() { ENVOY_LOG_MISC(info, "test"); } @@ -72,6 +73,7 @@ class ZooKeeperFilterTest : public TestBase { buffer.writeBEInt(8); buffer.writeBEInt(enumToInt(XidCodes::PING_XID)); + buffer.writeBEInt(enumToInt(OpCodes::PING)); return buffer; } @@ -87,18 +89,16 @@ class ZooKeeperFilterTest : public TestBase { } Buffer::OwnedImpl encodeAuth(const std::string& scheme) const { + const std::string credential = "p@sswd"; Buffer::OwnedImpl buffer; - buffer.writeBEInt(28 + scheme.length()); + buffer.writeBEInt(28 + scheme.length() + credential.length()); buffer.writeBEInt(enumToInt(XidCodes::AUTH_XID)); - // Opcode. buffer.writeBEInt(enumToInt(OpCodes::SETAUTH)); // Type. buffer.writeBEInt(0); - // Scheme. addString(buffer, scheme); - // Credential. - addString(buffer, "p@sswd"); + addString(buffer, credential); return buffer; } @@ -329,6 +329,21 @@ class ZooKeeperFilterTest : public TestBase { setDynamicMetadata("envoy.filters.network.zookeeper_proxy", MapEq(expected))); } + void expectSetDynamicMetadata(const std::map& first, + const std::map& second) { + EXPECT_CALL(filter_callbacks_.connection_, streamInfo()) + .WillRepeatedly(ReturnRef(stream_info_)); + EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)) + .WillOnce(Invoke([first](const std::string& key, const ProtobufWkt::Struct& obj) -> void { + EXPECT_STREQ(key.c_str(), "envoy.filters.network.zookeeper_proxy"); + protoMapEq(obj, first); + })) + .WillOnce(Invoke([second](const std::string& key, const ProtobufWkt::Struct& obj) -> void { + EXPECT_STREQ(key.c_str(), "envoy.filters.network.zookeeper_proxy"); + protoMapEq(obj, second); + })); + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -343,10 +358,11 @@ TEST_F(ZooKeeperFilterTest, Connect) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeConnect())); - expectSetDynamicMetadata({{"opname", "connect"}}); + expectSetDynamicMetadata({{"opname", "connect"}}, {{"bytes", "32"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().connect_rq_.value()); + EXPECT_EQ(32UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -356,11 +372,12 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeConnect(true))); - expectSetDynamicMetadata({{"opname", "connect_readonly"}}); + expectSetDynamicMetadata({{"opname", "connect_readonly"}}, {{"bytes", "33"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); EXPECT_EQ(1UL, config_->stats().connect_readonly_rq_.value()); + EXPECT_EQ(33UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -373,7 +390,7 @@ TEST_F(ZooKeeperFilterTest, Fallback) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); EXPECT_EQ(0UL, config_->stats().connect_readonly_rq_.value()); - EXPECT_EQ(2UL, config_->stats().decoder_error_.value()); + EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); } TEST_F(ZooKeeperFilterTest, PingRequest) { @@ -381,10 +398,11 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePing())); - expectSetDynamicMetadata({{"opname", "ping"}}); + expectSetDynamicMetadata({{"opname", "ping"}}, {{"bytes", "12"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().ping_rq_.value()); + EXPECT_EQ(12UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -393,10 +411,11 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeAuth("digest"))); - expectSetDynamicMetadata({{"opname", "auth"}}); + expectSetDynamicMetadata({{"opname", "auth"}}, {{"bytes", "36"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(scope_.counter("test.zookeeper.auth.digest_rq").value(), 1); + EXPECT_EQ(36UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -405,9 +424,11 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePathWatch("/foo", true))); - expectSetDynamicMetadata({{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}); + expectSetDynamicMetadata({{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}, + {{"bytes", "21"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -418,10 +439,12 @@ TEST_F(ZooKeeperFilterTest, CreateRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCreateRequest("/foo", "bar", false, false))); expectSetDynamicMetadata( - {{"opname", "create"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}); + {{"opname", "create"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}, + {{"bytes", "35"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); + EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -432,10 +455,12 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { encodeCreateRequest("/foo", "bar", false, false, false, enumToInt(OpCodes::CREATE2)))); expectSetDynamicMetadata( - {{"opname", "create2"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}); + {{"opname", "create2"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}, + {{"bytes", "35"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().create2_rq_.value()); + EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -444,10 +469,11 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetRequest("/foo", "bar", -1))); - expectSetDynamicMetadata({{"opname", "setdata"}, {"path", "/foo"}}); + expectSetDynamicMetadata({{"opname", "setdata"}, {"path", "/foo"}}, {{"bytes", "31"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); + EXPECT_EQ(31UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -457,10 +483,12 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN)))); - expectSetDynamicMetadata({{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}); + expectSetDynamicMetadata({{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, + {{"bytes", "21"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); + EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -470,10 +498,12 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN2)))); - expectSetDynamicMetadata({{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}); + expectSetDynamicMetadata({{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, + {{"bytes", "21"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getchildren2_rq_.value()); + EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -482,10 +512,12 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeDeleteRequest("/foo", -1))); - expectSetDynamicMetadata({{"opname", "remove"}, {"path", "/foo"}, {"version", "-1"}}); + expectSetDynamicMetadata({{"opname", "remove"}, {"path", "/foo"}, {"version", "-1"}}, + {{"bytes", "24"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().remove_rq_.value()); + EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -495,10 +527,12 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::EXISTS)))); - expectSetDynamicMetadata({{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}); + expectSetDynamicMetadata({{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, + {{"bytes", "21"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().exists_rq_.value()); + EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -507,10 +541,11 @@ TEST_F(ZooKeeperFilterTest, GetAclRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::GETACL)))); - expectSetDynamicMetadata({{"opname", "getacl"}, {"path", "/foo"}}); + expectSetDynamicMetadata({{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().getacl_rq_.value()); + EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -520,10 +555,12 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodeSetAclRequest("/foo", "digest", "passwd", -1))); - expectSetDynamicMetadata({{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}); + expectSetDynamicMetadata({{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}, + {{"bytes", "52"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setacl_rq_.value()); + EXPECT_EQ(52UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -532,10 +569,11 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::SYNC)))); - expectSetDynamicMetadata({{"opname", "sync"}, {"path", "/foo"}}); + expectSetDynamicMetadata({{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().sync_rq_.value()); + EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -545,8 +583,11 @@ TEST_F(ZooKeeperFilterTest, CheckRequest) { Buffer::InstancePtr data( new Buffer::OwnedImpl(encodePathVersion("/foo", 100, enumToInt(OpCodes::CHECK)))); + expectSetDynamicMetadata({{"bytes", "24"}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().check_rq_.value()); + EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -568,6 +609,7 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().multi_rq_.value()); + EXPECT_EQ(128UL, config_->stats().request_bytes_.value()); EXPECT_EQ(2UL, config_->stats().create_rq_.value()); EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); EXPECT_EQ(1UL, config_->stats().check_rq_.value()); @@ -579,10 +621,11 @@ TEST_F(ZooKeeperFilterTest, ReconfigRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeReconfigRequest("s1", "s2", "s3", 1000))); - expectSetDynamicMetadata({{"opname", "reconfig"}}); + expectSetDynamicMetadata({{"opname", "reconfig"}}, {{"bytes", "38"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().reconfig_rq_.value()); + EXPECT_EQ(38UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -595,10 +638,11 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetWatchesRequest(dataw, existw, childw))); - expectSetDynamicMetadata({{"opname", "setwatches"}}); + expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); + EXPECT_EQ(76UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -607,10 +651,11 @@ TEST_F(ZooKeeperFilterTest, CloseRequest) { Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCloseRequest())); - expectSetDynamicMetadata({{"opname", "close"}}); + expectSetDynamicMetadata({{"opname", "close"}}, {{"bytes", "12"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); EXPECT_EQ(1UL, config_->stats().close_rq_.value()); + EXPECT_EQ(12UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } From 44a179f5a816abc8cc15b9bc5419815bca46a95c Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 6 Mar 2019 23:39:14 -0800 Subject: [PATCH 046/114] Naming & link for ZooKeeper Per @phunt's feedback, use Apache ZooKeeper to refer to the project as a whole and add a link too. Signed-off-by: Raul Gutierrez Segales --- .../network_filters/zookeeper_proxy_filter.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index bae0ad41ef3bd..96ff81a49851d 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -3,11 +3,12 @@ ZooKeeper proxy =============== -The ZooKeeper proxy filter decodes the wire protocol between the ZooKeeper client -and server. It decodes the requests, responses and events in the payload. -The decoded info is emitted as dynamic metadata that can be combined with -access log filters to get detailed information on paths that are accessed -as well as the requests that are performed on each path. +The ZooKeeper proxy filter decodes the client protocol for +`Apache ZooKeeper `_. It decodes the requests, +responses and events in the payload. The decoded info is emitted as dynamic +metadata that can be combined with access log filters to get detailed +information on paths that are accessed as well as the requests that are +performed on each path. .. attention:: From c9cadb79ad5cd2246fc68c677a69b56be32f39e3 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 6 Mar 2019 23:45:35 -0800 Subject: [PATCH 047/114] Mention supported ZK version & opcodes handling Signed-off-by: Raul Gutierrez Segales --- .../configuration/network_filters/zookeeper_proxy_filter.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index 96ff81a49851d..5a9d596e30e5c 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -8,7 +8,8 @@ The ZooKeeper proxy filter decodes the client protocol for responses and events in the payload. The decoded info is emitted as dynamic metadata that can be combined with access log filters to get detailed information on paths that are accessed as well as the requests that are -performed on each path. +performed on each path. Most opcodes known in ZooKeeper version 3.5.4 are +supported, while unknown ones will be treated as decoding errors. .. attention:: From 70827ce4a9504135c7672f2f423e6a0fc190e09f Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 6 Mar 2019 23:57:20 -0800 Subject: [PATCH 048/114] Handle unknown opcodes Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 5 ++--- .../zookeeper_proxy/zookeeper_filter_test.cc | 22 ++++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 0496d8d22b937..4cd8a7ff3a1d5 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -28,8 +28,6 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { case enumToIntSigned(XidCodes::AUTH_XID): parseAuthRequest(data, offset, len); return; - default: - break; } // "Regular" requests. @@ -84,7 +82,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { callbacks_.onCloseRequest(); break; default: - break; + throw EnvoyException(fmt::format("Unknown opcode: {}", opcode)); } } @@ -315,6 +313,7 @@ void DecoderImpl::onData(Buffer::Instance& data) { callbacks_.onRequestBytes(offset - current); } } catch (EnvoyException& e) { + ENVOY_LOG(debug, "zookeeper_proxy: decoding exception {}", e.what()); callbacks_.onDecodeError(); } } diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index e42a5a150a149..9b032acf1d1d8 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -78,6 +78,16 @@ class ZooKeeperFilterTest : public TestBase { return buffer; } + Buffer::OwnedImpl encodeUnknownOpcode() const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(8); + buffer.writeBEInt(1000); + buffer.writeBEInt(200); + + return buffer; + } + Buffer::OwnedImpl encodeCloseRequest() const { Buffer::OwnedImpl buffer; @@ -352,7 +362,6 @@ class ZooKeeperFilterTest : public TestBase { NiceMock stream_info_; }; -// Test Connect counter increment. TEST_F(ZooKeeperFilterTest, Connect) { initialize(); @@ -366,7 +375,6 @@ TEST_F(ZooKeeperFilterTest, Connect) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } -// Test Connect readonly counter increment. TEST_F(ZooKeeperFilterTest, ConnectReadonly) { initialize(); @@ -381,7 +389,6 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } -// Test fallback. TEST_F(ZooKeeperFilterTest, Fallback) { initialize(); @@ -393,6 +400,15 @@ TEST_F(ZooKeeperFilterTest, Fallback) { EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, UnknownOpcode) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeUnknownOpcode())); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, PingRequest) { initialize(); From 028d7e91ba8795b0a3f37796110aa12289096739 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 7 Mar 2019 08:44:11 -0800 Subject: [PATCH 049/114] Add support for {check, remove}watches Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 2 ++ .../zookeeper_proxy/zookeeper_decoder.cc | 20 +++++++++++++ .../zookeeper_proxy/zookeeper_decoder.h | 7 +++++ .../zookeeper_proxy/zookeeper_filter.cc | 10 +++++++ .../zookeeper_proxy/zookeeper_filter.h | 4 +++ .../zookeeper_proxy/zookeeper_filter_test.cc | 28 +++++++++++++++++++ 6 files changed, 71 insertions(+) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index 5a9d596e30e5c..4bbfbe32f4410 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -71,6 +71,8 @@ following statistics: reconfig_rq, Counter, Number of reconfig requests close_rq, Counter, Number of close requests setwatches_rq, Counter, Number of setwatches requests + checkwatches_rq, Counter, Number of checkwatches requests + removewatches_rq, Counter, Number of removewatches requests check_rq, Counter, Number of check requests .. _config_network_filters_zookeeper_proxy_dynamic_metadata: diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 4cd8a7ff3a1d5..12ef4d1d3c150 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -78,6 +78,12 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { case enumToInt(OpCodes::SETWATCHES): parseSetWatchesRequest(data, offset, len); break; + case enumToInt(OpCodes::CHECKWATCHES): + parseXWatchesRequest(data, offset, len, OpCodes::CHECKWATCHES); + break; + case enumToInt(OpCodes::REMOVEWATCHES): + parseXWatchesRequest(data, offset, len, OpCodes::REMOVEWATCHES); + break; case enumToIntSigned(OpCodes::CLOSE): callbacks_.onCloseRequest(); break; @@ -295,6 +301,20 @@ void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offse callbacks_.onSetWatchesRequest(); } +void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + OpCodes opcode) { + checkLength(len, 8); + + std::string path = BufferHelper::peekString(data, offset); + int32_t type = BufferHelper::peekInt32(data, offset); + + if (opcode == OpCodes::CHECKWATCHES) { + callbacks_.onCheckWatchesRequest(path, type); + } else { + callbacks_.onRemoveWatchesRequest(path, type); + } +} + void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { int32_t count = BufferHelper::peekInt32(data, offset); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index cf1856025e534..fca9739d3b7e2 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -37,11 +37,15 @@ enum class OpCodes { MULTI = 14, CREATE2 = 15, RECONFIG = 16, + CHECKWATCHES = 17, + REMOVEWATCHES = 18, CLOSE = -11, SETAUTH = 100, SETWATCHES = 101 }; +enum class WatcherType { CHILDREN = 1, DATA = 2, ANY = 3 }; + /** * General callbacks for dispatching decoded ZooKeeper messages to a sink. */ @@ -68,6 +72,8 @@ class DecoderCallbacks { virtual void onMultiRequest() PURE; virtual void onReconfigRequest() PURE; virtual void onSetWatchesRequest() PURE; + virtual void onCheckWatchesRequest(const std::string& path, const int32_t type) PURE; + virtual void onRemoveWatchesRequest(const std::string& path, const int32_t type) PURE; virtual void onCloseRequest() PURE; }; @@ -109,6 +115,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); void skipStrings(Buffer::Instance& data, uint64_t& offset) const; void checkLength(const int32_t len, const int32_t minlen) const; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index b4a2301f5d7e0..b46638c8dc565 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -178,6 +178,16 @@ void ZooKeeperFilter::onCheckRequest(const std::string&, const int32_t) { config_->stats_.check_rq_.inc(); } +void ZooKeeperFilter::onCheckWatchesRequest(const std::string& path, const int32_t) { + config_->stats_.checkwatches_rq_.inc(); + setDynamicMetadata({{"opname", "checkwatches"}, {"path", path}}); +} + +void ZooKeeperFilter::onRemoveWatchesRequest(const std::string& path, const int32_t) { + config_->stats_.removewatches_rq_.inc(); + setDynamicMetadata({{"opname", "removewatches"}, {"path", path}}); +} + void ZooKeeperFilter::onMultiRequest() { config_->stats_.multi_rq_.inc(); setDynamicMetadata("opname", "multi"); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index b60fbd4738e3b..23a86388bd535 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -42,6 +42,8 @@ namespace ZooKeeperProxy { COUNTER(close_rq) \ COUNTER(setauth_rq) \ COUNTER(setwatches_rq) \ + COUNTER(checkwatches_rq) \ + COUNTER(removewatches_rq) \ COUNTER(check_rq) \ // clang-format on @@ -109,6 +111,8 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onMultiRequest() override; void onReconfigRequest() override; void onSetWatchesRequest() override; + void onCheckWatchesRequest(const std::string& path, const int32_t type) override; + void onRemoveWatchesRequest(const std::string& path, const int32_t type) override; void onCloseRequest() override; void doDecode(Buffer::Instance& buffer); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 9b032acf1d1d8..1d12d3e0e471c 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -662,6 +662,34 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePathVersion( + "/foo", enumToInt(WatcherType::CHILDREN), enumToInt(OpCodes::CHECKWATCHES)))); + + expectSetDynamicMetadata({{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().checkwatches_rq_.value()); + EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { + initialize(); + + Buffer::InstancePtr data(new Buffer::OwnedImpl( + encodePathVersion("/foo", enumToInt(WatcherType::DATA), enumToInt(OpCodes::REMOVEWATCHES)))); + + expectSetDynamicMetadata({{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().removewatches_rq_.value()); + EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, CloseRequest) { initialize(); From fcf42bbfc550c74e5722fd36de768f66a635776f Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 7 Mar 2019 11:53:36 -0800 Subject: [PATCH 050/114] Kick CI Signed-off-by: Raul Gutierrez Segales From 90c58da083a42c3ceb22947c9821ece0e17a6700 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sat, 9 Mar 2019 14:45:15 -0800 Subject: [PATCH 051/114] Document the different between XIDs <=0 and > 0. This is an attempt at documenting what control requests look like, versus data requests. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 25 +++++++++++++++++-- .../zookeeper_proxy/zookeeper_filter_test.cc | 23 +++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 12ef4d1d3c150..dc7551a4ec811 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -14,7 +14,18 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { int32_t len = BufferHelper::peekInt32(data, offset); checkLength(len, 8); - // "Special" requests. + // Control requests, with XIDs <= 0. + // + // These are meant to control the state of a session: + // connect, keep-alive, authenticate and set initial watches. + // + // Note: setWatches is a command historically used to + // set watches right after connecting, typically + // used when roaming from one ZK server to the next. + // Thus, the special XID. There has beeen talks in + // severeal client implementations to expose setWatches + // as a reegular data request as well. So we support + // it without a special XID as well. int32_t xid = BufferHelper::peekInt32(data, offset); switch (xid) { case enumToIntSigned(XidCodes::CONNECT_XID): @@ -28,9 +39,19 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { case enumToIntSigned(XidCodes::AUTH_XID): parseAuthRequest(data, offset, len); return; + case enumToIntSigned(XidCodes::SET_WATCHES_XID): + // Skip opcode. + offset += 4; + parseSetWatchesRequest(data, offset, len); + return; } - // "Regular" requests. + // Data requests, with XIDs > 0. + // + // These are meant to happen after a successful control request, except + // for two cases: auth requests can happen at any time and ping requests + // must happen every 1/3 of the negotiated session timeout, to keep + // the session alive. int32_t opcode = BufferHelper::peekInt32(data, offset); switch (opcode) { case enumToInt(OpCodes::GETDATA): diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 1d12d3e0e471c..ca69d5c8ff831 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -272,7 +272,8 @@ class ZooKeeperFilterTest : public TestBase { Buffer::OwnedImpl encodeSetWatchesRequest(const std::vector& dataw, const std::vector& existw, - const std::vector& childw) const { + const std::vector& childw, + int32_t xid = 1000) const { Buffer::OwnedImpl buffer; Buffer::OwnedImpl watches_buffer; @@ -281,7 +282,7 @@ class ZooKeeperFilterTest : public TestBase { addStrings(watches_buffer, childw); buffer.writeBEInt(8 + watches_buffer.length()); - buffer.writeBEInt(1000); + buffer.writeBEInt(xid); buffer.writeBEInt(enumToInt(OpCodes::SETWATCHES)); buffer.add(watches_buffer); @@ -645,6 +646,24 @@ TEST_F(ZooKeeperFilterTest, ReconfigRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { + initialize(); + + const std::vector dataw = {"/foo", "/bar"}; + const std::vector existw = {"/foo1", "/bar1"}; + const std::vector childw = {"/foo2", "/bar2"}; + + Buffer::InstancePtr data(new Buffer::OwnedImpl( + encodeSetWatchesRequest(dataw, existw, childw, enumToInt(XidCodes::SET_WATCHES_XID)))); + + expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); + EXPECT_EQ(76UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { initialize(); From 3dc1f72d42add330714f55098ffa30f4e55aef30 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 10 Mar 2019 11:48:44 -0700 Subject: [PATCH 052/114] Fix spelling Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index dc7551a4ec811..4478d09658809 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -23,7 +23,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // set watches right after connecting, typically // used when roaming from one ZK server to the next. // Thus, the special XID. There has beeen talks in - // severeal client implementations to expose setWatches + // several client implementations to expose setWatches // as a reegular data request as well. So we support // it without a special XID as well. int32_t xid = BufferHelper::peekInt32(data, offset); From 2b67b7ab635c433df85cf07eece4a2ab8a41dc02 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 10 Mar 2019 11:53:58 -0700 Subject: [PATCH 053/114] Use const for skipping offsets Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/zookeeper_decoder.cc | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 4478d09658809..47522287572c7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -7,6 +7,12 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { +const uint32_t INT_LENGTH = 4; +const uint32_t OPCODE_LENGTH = 4; +const uint32_t ZXID_LENGTH = 8; +const uint32_t TIMEOUT_LENGTH = 4; +const uint32_t SESSION_LENGTH = 8; + void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); @@ -32,16 +38,14 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseConnect(data, offset, len); return; case enumToIntSigned(XidCodes::PING_XID): - // Skip opcode. - offset += 4; + offset += OPCODE_LENGTH; callbacks_.onPing(); return; case enumToIntSigned(XidCodes::AUTH_XID): parseAuthRequest(data, offset, len); return; case enumToIntSigned(XidCodes::SET_WATCHES_XID): - // Skip opcode. - offset += 4; + offset += OPCODE_LENGTH; parseSetWatchesRequest(data, offset, len); return; } @@ -123,7 +127,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ checkLength(len, 28); // Read password - skip zxid, timeout, and session id. - offset += 20; + offset += ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH; std::string passwd = BufferHelper::peekString(data, offset); // Read readonly flag, if it's there. @@ -139,7 +143,7 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin checkLength(len, 20); // Skip opcode + type. - offset += 8; + offset += OPCODE_LENGTH + INT_LENGTH; std::string scheme = BufferHelper::peekString(data, offset); std::string credential = BufferHelper::peekString(data, offset); From 1854c331662bc7df57eb7bd5bc832a2cae657f62 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 10 Mar 2019 11:55:53 -0700 Subject: [PATCH 054/114] Skip reading credential, since we don't use it. Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 47522287572c7..8c3c6448ed7b9 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -145,7 +145,9 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin // Skip opcode + type. offset += OPCODE_LENGTH + INT_LENGTH; std::string scheme = BufferHelper::peekString(data, offset); - std::string credential = BufferHelper::peekString(data, offset); + // Skip credential. + int32_t credlen = BufferHelper::peekInt32(data, offset); + offset += credlen; callbacks_.onAuthRequest(scheme); } From 70e4852b332739c0d196d20978867cd7d7865c48 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 10 Mar 2019 12:57:49 -0700 Subject: [PATCH 055/114] More spelling Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/zookeeper_decoder.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 8c3c6448ed7b9..cb4a6addbfa96 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -25,13 +25,11 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // These are meant to control the state of a session: // connect, keep-alive, authenticate and set initial watches. // - // Note: setWatches is a command historically used to - // set watches right after connecting, typically - // used when roaming from one ZK server to the next. - // Thus, the special XID. There has beeen talks in - // several client implementations to expose setWatches - // as a reegular data request as well. So we support - // it without a special XID as well. + // Note: setWatches is a command historically used to set watches + // right after connecting, typically used when roaming from one + // ZooKeeper server to the next. Thus, the special xid. + // However, some client implementations might expose setWatches + // as a regular data request, so we support that as well. int32_t xid = BufferHelper::peekInt32(data, offset); switch (xid) { case enumToIntSigned(XidCodes::CONNECT_XID): From bdad78553ee396c527216f5f28a00092abb18690 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 10 Mar 2019 13:02:37 -0700 Subject: [PATCH 056/114] Add xid to the dictionary Signed-off-by: Raul Gutierrez Segales --- tools/spelling_dictionary.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 15820e8967cc3..4fd4c307e763b 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -757,6 +757,7 @@ xDS xeon xform xhtml +xid xxhash xxs xyz From fcc62ef34f9ae4c837eaf93182e87972e45e7e21 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 03:45:48 -0700 Subject: [PATCH 057/114] Kick CI Signed-off-by: Raul Gutierrez Segales From 3180d02916cfcd69462512da121d56ef435f1a45 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 03:50:40 -0700 Subject: [PATCH 058/114] Make local vars const when possible Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index cb4a6addbfa96..9f3305e985946 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -17,7 +17,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); // Check message length. - int32_t len = BufferHelper::peekInt32(data, offset); + const int32_t len = BufferHelper::peekInt32(data, offset); checkLength(len, 8); // Control requests, with XIDs <= 0. @@ -30,7 +30,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // ZooKeeper server to the next. Thus, the special xid. // However, some client implementations might expose setWatches // as a regular data request, so we support that as well. - int32_t xid = BufferHelper::peekInt32(data, offset); + const int32_t xid = BufferHelper::peekInt32(data, offset); switch (xid) { case enumToIntSigned(XidCodes::CONNECT_XID): parseConnect(data, offset, len); @@ -54,7 +54,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // for two cases: auth requests can happen at any time and ping requests // must happen every 1/3 of the negotiated session timeout, to keep // the session alive. - int32_t opcode = BufferHelper::peekInt32(data, offset); + const int32_t opcode = BufferHelper::peekInt32(data, offset); switch (opcode) { case enumToInt(OpCodes::GETDATA): parseGetDataRequest(data, offset, len); @@ -126,10 +126,10 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ // Read password - skip zxid, timeout, and session id. offset += ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH; - std::string passwd = BufferHelper::peekString(data, offset); + const std::string passwd = BufferHelper::peekString(data, offset); // Read readonly flag, if it's there. - bool readonly{}; + const bool readonly{}; if (data.length() >= offset + 1) { readonly = BufferHelper::peekBool(data, offset); } @@ -142,9 +142,9 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin // Skip opcode + type. offset += OPCODE_LENGTH + INT_LENGTH; - std::string scheme = BufferHelper::peekString(data, offset); + const std::string scheme = BufferHelper::peekString(data, offset); // Skip credential. - int32_t credlen = BufferHelper::peekInt32(data, offset); + const int32_t credlen = BufferHelper::peekInt32(data, offset); offset += credlen; callbacks_.onAuthRequest(scheme); @@ -153,14 +153,14 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 13); - std::string path = BufferHelper::peekString(data, offset); - bool watch = BufferHelper::peekBool(data, offset); + const std::string path = BufferHelper::peekString(data, offset); + const bool watch = BufferHelper::peekBool(data, offset); callbacks_.onGetDataRequest(path, watch); } void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { - int32_t count = BufferHelper::peekInt32(data, offset); + const int32_t count = BufferHelper::peekInt32(data, offset); for (int i = 0; i < count; ++i) { BufferHelper::peekInt32(data, offset); @@ -173,13 +173,13 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u const bool two) { checkLength(len, 20); - std::string path = BufferHelper::peekString(data, offset); + const std::string path = BufferHelper::peekString(data, offset); // Skip data. - int32_t datalen = BufferHelper::peekInt32(data, offset); + const int32_t datalen = BufferHelper::peekInt32(data, offset); offset += datalen; skipAcls(data, offset); - int32_t flags = BufferHelper::peekInt32(data, offset); + const int32_t flags = BufferHelper::peekInt32(data, offset); const bool ephemeral = (flags & 0x1) == 1; const bool sequence = (flags & 0x2) == 2; @@ -189,9 +189,9 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 20); - std::string path = BufferHelper::peekString(data, offset); + const std::string path = BufferHelper::peekString(data, offset); // Skip data. - int32_t datalen = BufferHelper::peekInt32(data, offset); + const int32_t datalen = BufferHelper::peekInt32(data, offset); offset += datalen; // Ignore version. BufferHelper::peekInt32(data, offset); @@ -203,8 +203,8 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs const bool two) { checkLength(len, 14); - std::string path = BufferHelper::peekString(data, offset); - bool watch = BufferHelper::peekBool(data, offset); + const std::string path = BufferHelper::peekString(data, offset); + const bool watch = BufferHelper::peekBool(data, offset); callbacks_.onGetChildrenRequest(path, watch, two); } @@ -212,8 +212,8 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 16); - std::string path = BufferHelper::peekString(data, offset); - int32_t version = BufferHelper::peekInt32(data, offset); + const std::string path = BufferHelper::peekString(data, offset); + const int32_t version = BufferHelper::peekInt32(data, offset); callbacks_.onDeleteRequest(path, version); } @@ -221,8 +221,8 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 13); - std::string path = BufferHelper::peekString(data, offset); - bool watch = BufferHelper::peekBool(data, offset); + const std::string path = BufferHelper::peekString(data, offset); + const bool watch = BufferHelper::peekBool(data, offset); callbacks_.onExistsRequest(path, watch); } @@ -230,7 +230,7 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path = BufferHelper::peekString(data, offset); + const std::string path = BufferHelper::peekString(data, offset); callbacks_.onGetAclRequest(path); } @@ -238,9 +238,9 @@ void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path = BufferHelper::peekString(data, offset); + const std::string path = BufferHelper::peekString(data, offset); skipAcls(data, offset); - int32_t version = BufferHelper::peekInt32(data, offset); + const int32_t version = BufferHelper::peekInt32(data, offset); callbacks_.onSetAclRequest(path, version); } @@ -248,7 +248,7 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path = BufferHelper::peekString(data, offset); + const std::string path = BufferHelper::peekString(data, offset); callbacks_.onSyncRequest(path); } @@ -256,8 +256,8 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { checkLength(len, 8); - std::string path = BufferHelper::peekString(data, offset); - int32_t version = BufferHelper::peekInt32(data, offset); + const std::string path = BufferHelper::peekString(data, offset); + const int32_t version = BufferHelper::peekInt32(data, offset); callbacks_.onCheckRequest(path, version); } @@ -267,8 +267,8 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui checkLength(len, 17); while (true) { - int32_t type = BufferHelper::peekInt32(data, offset); - bool done = BufferHelper::peekBool(data, offset); + const int32_t type = BufferHelper::peekInt32(data, offset); + const bool done = BufferHelper::peekBool(data, offset); // Ignore error field. BufferHelper::peekInt32(data, offset); @@ -299,7 +299,7 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, checkLength(len, 28); // Skip joining. - int32_t datalen = BufferHelper::peekInt32(data, offset); + const int32_t datalen = BufferHelper::peekInt32(data, offset); offset += datalen; // Skip leaving. datalen = BufferHelper::peekInt32(data, offset); @@ -330,8 +330,8 @@ void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, OpCodes opcode) { checkLength(len, 8); - std::string path = BufferHelper::peekString(data, offset); - int32_t type = BufferHelper::peekInt32(data, offset); + const std::string path = BufferHelper::peekString(data, offset); + const int32_t type = BufferHelper::peekInt32(data, offset); if (opcode == OpCodes::CHECKWATCHES) { callbacks_.onCheckWatchesRequest(path, type); @@ -341,10 +341,10 @@ void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, } void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { - int32_t count = BufferHelper::peekInt32(data, offset); + const int32_t count = BufferHelper::peekInt32(data, offset); for (int i = 0; i < count; ++i) { - int32_t len = BufferHelper::peekInt32(data, offset); + const int32_t len = BufferHelper::peekInt32(data, offset); offset += len; } } @@ -353,11 +353,11 @@ void DecoderImpl::onData(Buffer::Instance& data) { uint64_t offset = 0; try { while (offset < data.length()) { - uint64_t current = offset; + const uint64_t current = offset; decode(data, offset); callbacks_.onRequestBytes(offset - current); } - } catch (EnvoyException& e) { + } catch (const EnvoyException& e) { ENVOY_LOG(debug, "zookeeper_proxy: decoding exception {}", e.what()); callbacks_.onDecodeError(); } From 057550288bf1f7760ada48d9b77573fcbc37c4e6 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 03:55:59 -0700 Subject: [PATCH 059/114] Use consts to explain how the min length is derived Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 9f3305e985946..c5f634095cba0 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -8,10 +8,12 @@ namespace NetworkFilters { namespace ZooKeeperProxy { const uint32_t INT_LENGTH = 4; +const uint32_t XID_LENGTH = 4; const uint32_t OPCODE_LENGTH = 4; const uint32_t ZXID_LENGTH = 8; const uint32_t TIMEOUT_LENGTH = 4; const uint32_t SESSION_LENGTH = 8; +const uint32_t MULTI_HEADER_LENGTH = 9; void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); @@ -264,7 +266,7 @@ void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, ui void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { // Treat empty transactions as a decoding error, there should be at least 1 header. - checkLength(len, 17); + checkLength(len, XID_LENGTH + OPCODE_LENGTH + MULTI_HEADER_LENGTH); while (true) { const int32_t type = BufferHelper::peekInt32(data, offset); From 93caa9b67ffd3800ab74449c2b97f1fc2f5df1c5 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 03:56:50 -0700 Subject: [PATCH 060/114] Fix bool that shouldn't be const Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index c5f634095cba0..acdcc08b23384 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -131,7 +131,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ const std::string passwd = BufferHelper::peekString(data, offset); // Read readonly flag, if it's there. - const bool readonly{}; + bool readonly{}; if (data.length() >= offset + 1) { readonly = BufferHelper::peekBool(data, offset); } From d55144b673db7009a002d642eeb5b103246f995b Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 03:58:21 -0700 Subject: [PATCH 061/114] Rename checkLength to ensureMinLength Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 36 +++++++++---------- .../zookeeper_proxy/zookeeper_decoder.h | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index acdcc08b23384..3c3df86a80417 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -20,7 +20,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // Check message length. const int32_t len = BufferHelper::peekInt32(data, offset); - checkLength(len, 8); + ensureMinLength(len, 8); // Control requests, with XIDs <= 0. // @@ -117,14 +117,14 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { } } -void DecoderImpl::checkLength(const int32_t len, const int32_t minlen) const { +void DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const { if (len < minlen) { throw EnvoyException("Package is too small"); } } void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 28); + ensureMinLength(len, 28); // Read password - skip zxid, timeout, and session id. offset += ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH; @@ -140,7 +140,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ } void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 20); + ensureMinLength(len, 20); // Skip opcode + type. offset += OPCODE_LENGTH + INT_LENGTH; @@ -153,7 +153,7 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 13); + ensureMinLength(len, 13); const std::string path = BufferHelper::peekString(data, offset); const bool watch = BufferHelper::peekBool(data, offset); @@ -173,7 +173,7 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - checkLength(len, 20); + ensureMinLength(len, 20); const std::string path = BufferHelper::peekString(data, offset); @@ -189,7 +189,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 20); + ensureMinLength(len, 20); const std::string path = BufferHelper::peekString(data, offset); // Skip data. @@ -203,7 +203,7 @@ void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - checkLength(len, 14); + ensureMinLength(len, 14); const std::string path = BufferHelper::peekString(data, offset); const bool watch = BufferHelper::peekBool(data, offset); @@ -212,7 +212,7 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs } void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 16); + ensureMinLength(len, 16); const std::string path = BufferHelper::peekString(data, offset); const int32_t version = BufferHelper::peekInt32(data, offset); @@ -221,7 +221,7 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 13); + ensureMinLength(len, 13); const std::string path = BufferHelper::peekString(data, offset); const bool watch = BufferHelper::peekBool(data, offset); @@ -230,7 +230,7 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 8); + ensureMinLength(len, 8); const std::string path = BufferHelper::peekString(data, offset); @@ -238,7 +238,7 @@ void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 8); + ensureMinLength(len, 8); const std::string path = BufferHelper::peekString(data, offset); skipAcls(data, offset); @@ -248,7 +248,7 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 8); + ensureMinLength(len, 8); const std::string path = BufferHelper::peekString(data, offset); @@ -256,7 +256,7 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 8); + ensureMinLength(len, 8); const std::string path = BufferHelper::peekString(data, offset); const int32_t version = BufferHelper::peekInt32(data, offset); @@ -266,7 +266,7 @@ void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, ui void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { // Treat empty transactions as a decoding error, there should be at least 1 header. - checkLength(len, XID_LENGTH + OPCODE_LENGTH + MULTI_HEADER_LENGTH); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + MULTI_HEADER_LENGTH); while (true) { const int32_t type = BufferHelper::peekInt32(data, offset); @@ -298,7 +298,7 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui } void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 28); + ensureMinLength(len, 28); // Skip joining. const int32_t datalen = BufferHelper::peekInt32(data, offset); @@ -316,7 +316,7 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, } void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - checkLength(len, 12); + ensureMinLength(len, 12); // Data watches. skipStrings(data, offset); @@ -330,7 +330,7 @@ void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offse void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode) { - checkLength(len, 8); + ensureMinLength(len, 8); const std::string path = BufferHelper::peekString(data, offset); const int32_t type = BufferHelper::peekInt32(data, offset); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index fca9739d3b7e2..97e03b7014def 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -117,7 +117,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); void skipStrings(Buffer::Instance& data, uint64_t& offset) const; - void checkLength(const int32_t len, const int32_t minlen) const; + void ensureMinLength(const int32_t len, const int32_t minlen) const; DecoderCallbacks& callbacks_; }; From dfd64fe35ae3970d054547492e7c98b2a6456900 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 04:18:24 -0700 Subject: [PATCH 062/114] Use consts to express what the min lengths mean Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 3c3df86a80417..a95c82400e81c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -7,7 +7,9 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { +const uint32_t BOOL_LENGTH = 1; const uint32_t INT_LENGTH = 4; +const uint32_t LONG_LENGTH = 8; const uint32_t XID_LENGTH = 4; const uint32_t OPCODE_LENGTH = 4; const uint32_t ZXID_LENGTH = 8; @@ -20,7 +22,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // Check message length. const int32_t len = BufferHelper::peekInt32(data, offset); - ensureMinLength(len, 8); + ensureMinLength(len, INT_LENGTH + XID_LENGTH); // Control requests, with XIDs <= 0. // @@ -124,7 +126,7 @@ void DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const } void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 28); + ensureMinLength(len, XID_LENGTH + ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); // Read password - skip zxid, timeout, and session id. offset += ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH; @@ -140,7 +142,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ } void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 20); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + INT_LENGTH + INT_LENGTH); // Skip opcode + type. offset += OPCODE_LENGTH + INT_LENGTH; @@ -153,7 +155,7 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 13); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); const std::string path = BufferHelper::peekString(data, offset); const bool watch = BufferHelper::peekBool(data, offset); @@ -173,7 +175,7 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - ensureMinLength(len, 20); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); const std::string path = BufferHelper::peekString(data, offset); @@ -189,7 +191,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 20); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); const std::string path = BufferHelper::peekString(data, offset); // Skip data. @@ -203,7 +205,7 @@ void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - ensureMinLength(len, 14); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); const std::string path = BufferHelper::peekString(data, offset); const bool watch = BufferHelper::peekBool(data, offset); @@ -212,7 +214,7 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs } void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 16); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); const std::string path = BufferHelper::peekString(data, offset); const int32_t version = BufferHelper::peekInt32(data, offset); @@ -221,7 +223,7 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 13); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); const std::string path = BufferHelper::peekString(data, offset); const bool watch = BufferHelper::peekBool(data, offset); @@ -230,7 +232,7 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 8); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); const std::string path = BufferHelper::peekString(data, offset); @@ -238,7 +240,7 @@ void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 8); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); const std::string path = BufferHelper::peekString(data, offset); skipAcls(data, offset); @@ -248,7 +250,7 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u } void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 8); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); const std::string path = BufferHelper::peekString(data, offset); @@ -256,7 +258,7 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin } void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 8); + ensureMinLength(len, (2 * INT_LENGTH)); const std::string path = BufferHelper::peekString(data, offset); const int32_t version = BufferHelper::peekInt32(data, offset); @@ -298,10 +300,10 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui } void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 28); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH) + LONG_LENGTH); // Skip joining. - const int32_t datalen = BufferHelper::peekInt32(data, offset); + int32_t datalen = BufferHelper::peekInt32(data, offset); offset += datalen; // Skip leaving. datalen = BufferHelper::peekInt32(data, offset); @@ -316,7 +318,7 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, } void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, 12); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); // Data watches. skipStrings(data, offset); @@ -330,7 +332,7 @@ void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offse void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode) { - ensureMinLength(len, 8); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); const std::string path = BufferHelper::peekString(data, offset); const int32_t type = BufferHelper::peekInt32(data, offset); From 0d1102631fe9b4df98b5a303e1461569bab5d618 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 04:23:24 -0700 Subject: [PATCH 063/114] Fix format Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index a95c82400e81c..d71618237841d 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -205,7 +205,7 @@ void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); const std::string path = BufferHelper::peekString(data, offset); const bool watch = BufferHelper::peekBool(data, offset); From 6ce575239591ea32d574fbe60e172c30e4d0cc66 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 04:31:26 -0700 Subject: [PATCH 064/114] Avoid const for values in declarations Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.h | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 97e03b7014def..25a96bf01d282 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -54,26 +54,26 @@ class DecoderCallbacks { virtual ~DecoderCallbacks() {} virtual void onDecodeError() PURE; - virtual void onRequestBytes(const uint64_t bytes) PURE; - virtual void onConnect(const bool readonly) PURE; + virtual void onRequestBytes(uint64_t bytes) PURE; + virtual void onConnect(bool readonly) PURE; virtual void onPing() PURE; virtual void onAuthRequest(const std::string& scheme) PURE; - virtual void onGetDataRequest(const std::string& path, const bool watch) PURE; - virtual void onCreateRequest(const std::string& path, const bool ephemeral, const bool sequence, - const bool two) PURE; + virtual void onGetDataRequest(const std::string& path, bool watch) PURE; + virtual void onCreateRequest(const std::string& path, bool ephemeral, bool sequence, + bool two) PURE; virtual void onSetRequest(const std::string& path) PURE; - virtual void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) PURE; - virtual void onDeleteRequest(const std::string& path, const int32_t version) PURE; - virtual void onExistsRequest(const std::string& path, const bool watch) PURE; + virtual void onGetChildrenRequest(const std::string& path, bool watch, bool two) PURE; + virtual void onDeleteRequest(const std::string& path, int32_t version) PURE; + virtual void onExistsRequest(const std::string& path, bool watch) PURE; virtual void onGetAclRequest(const std::string& path) PURE; - virtual void onSetAclRequest(const std::string& path, const int32_t version) PURE; + virtual void onSetAclRequest(const std::string& path, int32_t version) PURE; virtual void onSyncRequest(const std::string& path) PURE; - virtual void onCheckRequest(const std::string& path, const int32_t version) PURE; + virtual void onCheckRequest(const std::string& path, int32_t version) PURE; virtual void onMultiRequest() PURE; virtual void onReconfigRequest() PURE; virtual void onSetWatchesRequest() PURE; - virtual void onCheckWatchesRequest(const std::string& path, const int32_t type) PURE; - virtual void onRemoveWatchesRequest(const std::string& path, const int32_t type) PURE; + virtual void onCheckWatchesRequest(const std::string& path, int32_t type) PURE; + virtual void onRemoveWatchesRequest(const std::string& path, int32_t type) PURE; virtual void onCloseRequest() PURE; }; @@ -101,11 +101,10 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, const bool two); + void parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, bool two); void skipAcls(Buffer::Instance& data, uint64_t& offset) const; void parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, - const bool two); + void parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, bool two); void parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); @@ -117,7 +116,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); void skipStrings(Buffer::Instance& data, uint64_t& offset) const; - void ensureMinLength(const int32_t len, const int32_t minlen) const; + void ensureMinLength(int32_t len, int32_t minlen) const; DecoderCallbacks& callbacks_; }; From b9490e33a213b8d4f2fa8e9f46cc2403ba8bbd03 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 04:34:41 -0700 Subject: [PATCH 065/114] Simplify peekBool() Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index f90504bb0f6ab..c6724c06a5f24 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -19,7 +19,7 @@ int64_t BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset) { bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { char byte = buffer.peekInt(offset); - bool val = static_cast(byte & 255); + bool val = static_cast(byte); offset += 1; return val; } From 9389ef1ab3c42f96ab550e3c20a542dbcaba5982 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 04:36:33 -0700 Subject: [PATCH 066/114] More context when throwing exception Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index c6724c06a5f24..0f08ee0d3b126 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -33,7 +33,7 @@ std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) } if (buffer.length() < (offset + len)) { - throw EnvoyException("buffer is too small"); + throw EnvoyException("peekString: buffer is smaller than string length"); } std::unique_ptr data(new char[len]); From 9a744d7e9c55bd59d2674f0c4ff19b94c33d4149 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 04:52:56 -0700 Subject: [PATCH 067/114] Update tests * use buffer references, instead of pointers * test_base.h is gone Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 117 +++++++++--------- 1 file changed, 56 insertions(+), 61 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index ca69d5c8ff831..d89ec47699f7a 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -4,9 +4,9 @@ #include "extensions/filters/network/zookeeper_proxy/zookeeper_filter.h" #include "test/mocks/network/mocks.h" -#include "test/test_common/test_base.h" #include "gmock/gmock.h" +#include "gtest/gtest.h" using testing::NiceMock; @@ -25,7 +25,7 @@ bool protoMapEq(const ProtobufWkt::Struct& obj, const std::maponData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().connect_rq_.value()); EXPECT_EQ(32UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -379,11 +379,11 @@ TEST_F(ZooKeeperFilterTest, Connect) { TEST_F(ZooKeeperFilterTest, ConnectReadonly) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeConnect(true))); + Buffer::OwnedImpl data = encodeConnect(true); expectSetDynamicMetadata({{"opname", "connect_readonly"}}, {{"bytes", "33"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); EXPECT_EQ(1UL, config_->stats().connect_readonly_rq_.value()); EXPECT_EQ(33UL, config_->stats().request_bytes_.value()); @@ -393,9 +393,9 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { TEST_F(ZooKeeperFilterTest, Fallback) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeBadMessage())); + Buffer::OwnedImpl data = encodeBadMessage(); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); EXPECT_EQ(0UL, config_->stats().connect_readonly_rq_.value()); EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); @@ -404,20 +404,20 @@ TEST_F(ZooKeeperFilterTest, Fallback) { TEST_F(ZooKeeperFilterTest, UnknownOpcode) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeUnknownOpcode())); + Buffer::OwnedImpl data = encodeUnknownOpcode(); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); } TEST_F(ZooKeeperFilterTest, PingRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePing())); + Buffer::OwnedImpl data = encodePing(); expectSetDynamicMetadata({{"opname", "ping"}}, {{"bytes", "12"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().ping_rq_.value()); EXPECT_EQ(12UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -426,11 +426,11 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { TEST_F(ZooKeeperFilterTest, AuthRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeAuth("digest"))); + Buffer::OwnedImpl data = encodeAuth("digest"); expectSetDynamicMetadata({{"opname", "auth"}}, {{"bytes", "36"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(scope_.counter("test.zookeeper.auth.digest_rq").value(), 1); EXPECT_EQ(36UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -439,12 +439,12 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { TEST_F(ZooKeeperFilterTest, GetDataRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePathWatch("/foo", true))); + Buffer::OwnedImpl data = encodePathWatch("/foo", true); expectSetDynamicMetadata({{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}, {{"bytes", "21"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -453,13 +453,13 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { TEST_F(ZooKeeperFilterTest, CreateRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCreateRequest("/foo", "bar", false, false))); + Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", false, false); expectSetDynamicMetadata( {{"opname", "create"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}, {{"bytes", "35"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -468,14 +468,14 @@ TEST_F(ZooKeeperFilterTest, CreateRequest) { TEST_F(ZooKeeperFilterTest, CreateRequest2) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl( - encodeCreateRequest("/foo", "bar", false, false, false, enumToInt(OpCodes::CREATE2)))); + Buffer::OwnedImpl data = + encodeCreateRequest("/foo", "bar", false, false, false, enumToInt(OpCodes::CREATE2)); expectSetDynamicMetadata( {{"opname", "create2"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}, {{"bytes", "35"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create2_rq_.value()); EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -484,11 +484,11 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { TEST_F(ZooKeeperFilterTest, SetRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetRequest("/foo", "bar", -1))); + Buffer::OwnedImpl data = encodeSetRequest("/foo", "bar", -1); expectSetDynamicMetadata({{"opname", "setdata"}, {"path", "/foo"}}, {{"bytes", "31"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); EXPECT_EQ(31UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -497,13 +497,12 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { initialize(); - Buffer::InstancePtr data( - new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN)))); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN)); expectSetDynamicMetadata({{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -512,13 +511,12 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { initialize(); - Buffer::InstancePtr data( - new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN2)))); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN2)); expectSetDynamicMetadata({{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().getchildren2_rq_.value()); EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -527,12 +525,12 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { TEST_F(ZooKeeperFilterTest, DeleteRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeDeleteRequest("/foo", -1))); + Buffer::OwnedImpl data = encodeDeleteRequest("/foo", -1); expectSetDynamicMetadata({{"opname", "remove"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "24"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().remove_rq_.value()); EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -541,13 +539,12 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { TEST_F(ZooKeeperFilterTest, ExistsRequest) { initialize(); - Buffer::InstancePtr data( - new Buffer::OwnedImpl(encodePathWatch("/foo", false, enumToInt(OpCodes::EXISTS)))); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToInt(OpCodes::EXISTS)); expectSetDynamicMetadata({{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().exists_rq_.value()); EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -556,11 +553,11 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { TEST_F(ZooKeeperFilterTest, GetAclRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::GETACL)))); + Buffer::OwnedImpl data = encodePath("/foo", enumToInt(OpCodes::GETACL)); expectSetDynamicMetadata({{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().getacl_rq_.value()); EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -569,13 +566,12 @@ TEST_F(ZooKeeperFilterTest, GetAclRequest) { TEST_F(ZooKeeperFilterTest, SetAclRequest) { initialize(); - Buffer::InstancePtr data( - new Buffer::OwnedImpl(encodeSetAclRequest("/foo", "digest", "passwd", -1))); + Buffer::OwnedImpl data = encodeSetAclRequest("/foo", "digest", "passwd", -1); expectSetDynamicMetadata({{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "52"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().setacl_rq_.value()); EXPECT_EQ(52UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -584,11 +580,11 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { TEST_F(ZooKeeperFilterTest, SyncRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePath("/foo", enumToInt(OpCodes::SYNC)))); + Buffer::OwnedImpl data = encodePath("/foo", enumToInt(OpCodes::SYNC)); expectSetDynamicMetadata({{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().sync_rq_.value()); EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -597,12 +593,11 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { TEST_F(ZooKeeperFilterTest, CheckRequest) { initialize(); - Buffer::InstancePtr data( - new Buffer::OwnedImpl(encodePathVersion("/foo", 100, enumToInt(OpCodes::CHECK)))); + Buffer::OwnedImpl data = encodePathVersion("/foo", 100, enumToInt(OpCodes::CHECK)); expectSetDynamicMetadata({{"bytes", "24"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().check_rq_.value()); EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -622,9 +617,9 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { ops.push_back(std::make_pair(enumToInt(OpCodes::CHECK), std::move(check1))); ops.push_back(std::make_pair(enumToInt(OpCodes::SETDATA), std::move(set1))); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeMultiRequest(ops))); + Buffer::OwnedImpl data = encodeMultiRequest(ops); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().multi_rq_.value()); EXPECT_EQ(128UL, config_->stats().request_bytes_.value()); EXPECT_EQ(2UL, config_->stats().create_rq_.value()); @@ -636,11 +631,11 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { TEST_F(ZooKeeperFilterTest, ReconfigRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeReconfigRequest("s1", "s2", "s3", 1000))); + Buffer::OwnedImpl data = encodeReconfigRequest("s1", "s2", "s3", 1000); expectSetDynamicMetadata({{"opname", "reconfig"}}, {{"bytes", "38"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().reconfig_rq_.value()); EXPECT_EQ(38UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -653,12 +648,12 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { const std::vector existw = {"/foo1", "/bar1"}; const std::vector childw = {"/foo2", "/bar2"}; - Buffer::InstancePtr data(new Buffer::OwnedImpl( - encodeSetWatchesRequest(dataw, existw, childw, enumToInt(XidCodes::SET_WATCHES_XID)))); + Buffer::OwnedImpl data = + encodeSetWatchesRequest(dataw, existw, childw, enumToInt(XidCodes::SET_WATCHES_XID)); expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); EXPECT_EQ(76UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -671,11 +666,11 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { const std::vector existw = {"/foo1", "/bar1"}; const std::vector childw = {"/foo2", "/bar2"}; - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeSetWatchesRequest(dataw, existw, childw))); + Buffer::OwnedImpl data = encodeSetWatchesRequest(dataw, existw, childw); expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); EXPECT_EQ(76UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -684,12 +679,12 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodePathVersion( - "/foo", enumToInt(WatcherType::CHILDREN), enumToInt(OpCodes::CHECKWATCHES)))); + Buffer::OwnedImpl data = + encodePathVersion("/foo", enumToInt(WatcherType::CHILDREN), enumToInt(OpCodes::CHECKWATCHES)); expectSetDynamicMetadata({{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().checkwatches_rq_.value()); EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -698,12 +693,12 @@ TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl( - encodePathVersion("/foo", enumToInt(WatcherType::DATA), enumToInt(OpCodes::REMOVEWATCHES)))); + Buffer::OwnedImpl data = + encodePathVersion("/foo", enumToInt(WatcherType::DATA), enumToInt(OpCodes::REMOVEWATCHES)); expectSetDynamicMetadata({{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().removewatches_rq_.value()); EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); @@ -712,11 +707,11 @@ TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { TEST_F(ZooKeeperFilterTest, CloseRequest) { initialize(); - Buffer::InstancePtr data(new Buffer::OwnedImpl(encodeCloseRequest())); + Buffer::OwnedImpl data = encodeCloseRequest(); expectSetDynamicMetadata({{"opname", "close"}}, {{"bytes", "12"}}); - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*data, false)); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().close_rq_.value()); EXPECT_EQ(12UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); From cbc618a0ef50ea8ff4d1d0141e7abe9463cf6963 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 06:29:42 -0700 Subject: [PATCH 068/114] Enforce max bytes check This is a starting point, we look at a packet length and reject it if it's too big. However, we still need to enforce not reading beyond the given length. Signed-off-by: Raul Gutierrez Segales --- .../v1alpha1/zookeeper_proxy.proto | 10 ++++++++++ .../zookeeper_proxy/zookeeper_config.cc | 5 ++++- .../zookeeper_proxy/zookeeper_decoder.cc | 18 +++++++++++++++--- .../zookeeper_proxy/zookeeper_decoder.h | 5 ++++- .../zookeeper_proxy/zookeeper_filter.cc | 8 +++++--- .../network/zookeeper_proxy/zookeeper_filter.h | 4 +++- .../zookeeper_proxy/zookeeper_filter_test.cc | 17 +++++++++++++++++ 7 files changed, 58 insertions(+), 9 deletions(-) diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto index 77fbc96efd086..8f896aba76d4d 100644 --- a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto @@ -19,4 +19,14 @@ message ZooKeeperProxy { // [#not-implemented-hide:] The optional path to use for writing ZooKeeper access logs. // If the access log field is empty, access logs will not be written. string access_log = 2; + + // Messages — requests, responses and events — that are bigger than this value will + // be ignored. If it is not set, the default value is 1Mb. + // + // The value here should match the jute.maxbuffer property in your cluster configuration: + // + // https://zookeeper.apache.org/doc/r3.4.10/zookeeperAdmin.html#Unsafe+Options + // + // if that is set. If it isn't, ZooKeeper's default is also 1Mb. + uint32 max_packet_bytes = 3; } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc index ef145cee08387..4cef70251da69 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc @@ -26,9 +26,12 @@ NetworkFilters::ZooKeeperProxy::ZooKeeperConfigFactory::createFilterFactoryFromP ASSERT(!proto_config.stat_prefix().empty()); const std::string stat_prefix = fmt::format("{}.zookeeper.", proto_config.stat_prefix()); + const uint32_t default_max_bytes = 1048576; + const uint32_t max_packet_bytes = + proto_config.max_packet_bytes() == 0 ? default_max_bytes : proto_config.max_packet_bytes(); ZooKeeperFilterConfigSharedPtr filter_config( - std::make_shared(stat_prefix, context.scope())); + std::make_shared(stat_prefix, max_packet_bytes, context.scope())); return [filter_config](Network::FilterManager& filter_manager) -> void { filter_manager.addFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index d71618237841d..804d3ce92fa6b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -23,6 +23,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // Check message length. const int32_t len = BufferHelper::peekInt32(data, offset); ensureMinLength(len, INT_LENGTH + XID_LENGTH); + ensureMaxLength(len); // Control requests, with XIDs <= 0. // @@ -121,7 +122,13 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { void DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const { if (len < minlen) { - throw EnvoyException("Package is too small"); + throw EnvoyException("Packet is too small"); + } +} + +void DecoderImpl::ensureMaxLength(const int32_t len) const { + if (len > max_packet_bytes_) { + throw EnvoyException("Packet is too big"); } } @@ -167,9 +174,14 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { const int32_t count = BufferHelper::peekInt32(data, offset); for (int i = 0; i < count; ++i) { + // Perms. BufferHelper::peekInt32(data, offset); - BufferHelper::peekString(data, offset); - BufferHelper::peekString(data, offset); + // Skip scheme. + int32_t datalen = BufferHelper::peekInt32(data, offset); + offset += datalen; + // Skip cred. + datalen = BufferHelper::peekInt32(data, offset); + offset += datalen; } } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 25a96bf01d282..c7b3c0512ebb7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -91,7 +91,8 @@ typedef std::unique_ptr DecoderPtr; class DecoderImpl : public Decoder, Logger::Loggable { public: - explicit DecoderImpl(DecoderCallbacks& callbacks) : callbacks_(callbacks) {} + explicit DecoderImpl(DecoderCallbacks& callbacks, uint32_t max_packet_bytes) + : callbacks_(callbacks), max_packet_bytes_(max_packet_bytes) {} // ZooKeeperProxy::Decoder void onData(Buffer::Instance& data) override; @@ -117,8 +118,10 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); void skipStrings(Buffer::Instance& data, uint64_t& offset) const; void ensureMinLength(int32_t len, int32_t minlen) const; + void ensureMaxLength(int32_t len) const; DecoderCallbacks& callbacks_; + const uint32_t max_packet_bytes_; }; } // namespace ZooKeeperProxy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index b46638c8dc565..9fffae49e85c5 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -12,8 +12,10 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, Stats::Scope& scope) - : scope_(scope), stat_prefix_(stat_prefix), stats_(generateStats(stat_prefix, scope)) {} +ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, + const uint32_t max_packet_bytes, Stats::Scope& scope) + : scope_(scope), max_packet_bytes_(max_packet_bytes), stat_prefix_(stat_prefix), + stats_(generateStats(stat_prefix, scope)) {} ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config) : config_(std::move(config)) {} @@ -54,7 +56,7 @@ void ZooKeeperFilter::doDecode(Buffer::Instance& buffer) { } DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks) { - return std::make_unique(callbacks); + return std::make_unique(callbacks, config_->maxPacketBytes()); } void ZooKeeperFilter::setDynamicMetadata(const std::string& key, const std::string& value) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 23a86388bd535..55b8649ec2742 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -59,11 +59,13 @@ struct ZooKeeperProxyStats { */ class ZooKeeperFilterConfig { public: - ZooKeeperFilterConfig(const std::string &stat_prefix, Stats::Scope& scope); + ZooKeeperFilterConfig(const std::string &stat_prefix, uint32_t max_packet_bytes, Stats::Scope& scope); const ZooKeeperProxyStats& stats() { return stats_; } + const uint32_t maxPacketBytes() const { return max_packet_bytes_; } Stats::Scope& scope_; + const uint32_t max_packet_bytes_; const std::string stat_prefix_; ZooKeeperProxyStats stats_; diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index d89ec47699f7a..d558e0bcf4c62 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -68,6 +68,14 @@ class ZooKeeperFilterTest : public testing::Test { return buffer; } + Buffer::OwnedImpl encodeTooBigMessage() const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(1048577); + + return buffer; + } + Buffer::OwnedImpl encodePing() const { Buffer::OwnedImpl buffer; @@ -401,6 +409,15 @@ TEST_F(ZooKeeperFilterTest, Fallback) { EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, PacketTooBig) { + initialize(); + + Buffer::OwnedImpl data = encodeTooBigMessage(); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, UnknownOpcode) { initialize(); From 980207e31f5da3fad54f567c8901d8f79e5be9c5 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 06:48:31 -0700 Subject: [PATCH 069/114] Fix signature Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 55b8649ec2742..71d90a7911d8b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -62,7 +62,7 @@ class ZooKeeperFilterConfig { ZooKeeperFilterConfig(const std::string &stat_prefix, uint32_t max_packet_bytes, Stats::Scope& scope); const ZooKeeperProxyStats& stats() { return stats_; } - const uint32_t maxPacketBytes() const { return max_packet_bytes_; } + uint32_t maxPacketBytes() const { return max_packet_bytes_; } Stats::Scope& scope_; const uint32_t max_packet_bytes_; From df93faafa083491732a7241831abbd838a4e0473 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 06:51:20 -0700 Subject: [PATCH 070/114] Cast len to uint Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 804d3ce92fa6b..3b0690fa62202 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -127,7 +127,7 @@ void DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const } void DecoderImpl::ensureMaxLength(const int32_t len) const { - if (len > max_packet_bytes_) { + if (static_cast(len) > max_packet_bytes_) { throw EnvoyException("Packet is too big"); } } From 0ba76959e6dc5d0a05503e4820a1f898d2f8e217 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 11 Mar 2019 07:01:42 -0700 Subject: [PATCH 071/114] Fix test Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index d558e0bcf4c62..779c7fbcea09d 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -30,7 +30,7 @@ class ZooKeeperFilterTest : public testing::Test { ZooKeeperFilterTest() { ENVOY_LOG_MISC(info, "test"); } void initialize() { - config_ = std::make_shared(stat_prefix_, scope_); + config_ = std::make_shared(stat_prefix_, 1048576, scope_); filter_ = std::make_unique(config_); filter_->initializeReadFilterCallbacks(filter_callbacks_); } From ceecefb03a386111fdcd8631ca5bfded1014b218 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 07:47:46 -0700 Subject: [PATCH 072/114] Use google.protobuf.UInt32Value Per @snowp, this is the right datatype when the default value doesn't match what the description says. Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto index 8f896aba76d4d..61ecef08f1fea 100644 --- a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto @@ -28,5 +28,5 @@ message ZooKeeperProxy { // https://zookeeper.apache.org/doc/r3.4.10/zookeeperAdmin.html#Unsafe+Options // // if that is set. If it isn't, ZooKeeper's default is also 1Mb. - uint32 max_packet_bytes = 3; + google.protobuf.UInt32Value max_packet_bytes = 3; } From 804b83c86cfd52cf720f4bbcb04bc6176de3958f Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 07:56:39 -0700 Subject: [PATCH 073/114] Enumerate unsupported opcodes. Signed-off-by: Raul Gutierrez Segales --- .../network_filters/zookeeper_proxy_filter.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index 4bbfbe32f4410..e39397073f0fa 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -5,11 +5,18 @@ ZooKeeper proxy The ZooKeeper proxy filter decodes the client protocol for `Apache ZooKeeper `_. It decodes the requests, -responses and events in the payload. The decoded info is emitted as dynamic -metadata that can be combined with access log filters to get detailed -information on paths that are accessed as well as the requests that are -performed on each path. Most opcodes known in ZooKeeper version 3.5.4 are -supported, while unknown ones will be treated as decoding errors. +responses and events in the payload. Most opcodes known in +`ZooKeeper 3.5 `_ +are supported. + +The unsupported ones are: + +* createContainer +* deleteContainer +* createTTL +* sasl +* getEphemerals +* getAllChildrenNumber .. attention:: From 55293f2cdbc670c527a368a7e455d6b29841ab45 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 08:01:21 -0700 Subject: [PATCH 074/114] Add missing import for UInt32Value Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto index 61ecef08f1fea..6a8afdd12ec07 100644 --- a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto @@ -8,6 +8,7 @@ option java_package = "io.envoyproxy.envoy.config.filter.network.zookeeper_proxy option go_package = "v1alpha1"; import "validate/validate.proto"; +import "google/protobuf/wrappers.proto"; // [#protodoc-title: ZooKeeper proxy] // ZooKeeper Proxy :ref:`configuration overview `. From 6c714150ee45f5948a55635a9fac0b42ba2da080 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 08:05:36 -0700 Subject: [PATCH 075/114] Use PROTOBUF_GET_WRAPPED_OR_DEFAULT Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_config.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc index 4cef70251da69..7a2bda7a7bcbd 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_config.cc @@ -26,9 +26,8 @@ NetworkFilters::ZooKeeperProxy::ZooKeeperConfigFactory::createFilterFactoryFromP ASSERT(!proto_config.stat_prefix().empty()); const std::string stat_prefix = fmt::format("{}.zookeeper.", proto_config.stat_prefix()); - const uint32_t default_max_bytes = 1048576; const uint32_t max_packet_bytes = - proto_config.max_packet_bytes() == 0 ? default_max_bytes : proto_config.max_packet_bytes(); + PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, max_packet_bytes, 1024 * 1024); ZooKeeperFilterConfigSharedPtr filter_config( std::make_shared(stat_prefix, max_packet_bytes, context.scope())); From c121ceec1d42dc33b59ab626e6e353817bea4819 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 08:06:25 -0700 Subject: [PATCH 076/114] Use constexpr Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 3b0690fa62202..e05c4468ec265 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -7,15 +7,15 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -const uint32_t BOOL_LENGTH = 1; -const uint32_t INT_LENGTH = 4; -const uint32_t LONG_LENGTH = 8; -const uint32_t XID_LENGTH = 4; -const uint32_t OPCODE_LENGTH = 4; -const uint32_t ZXID_LENGTH = 8; -const uint32_t TIMEOUT_LENGTH = 4; -const uint32_t SESSION_LENGTH = 8; -const uint32_t MULTI_HEADER_LENGTH = 9; +constexpr uint32_t BOOL_LENGTH = 1; +constexpr uint32_t INT_LENGTH = 4; +constexpr uint32_t LONG_LENGTH = 8; +constexpr uint32_t XID_LENGTH = 4; +constexpr uint32_t OPCODE_LENGTH = 4; +constexpr uint32_t ZXID_LENGTH = 8; +constexpr uint32_t TIMEOUT_LENGTH = 4; +constexpr uint32_t SESSION_LENGTH = 8; +constexpr uint32_t MULTI_HEADER_LENGTH = 9; void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); From 6c8c9776dff3e7fb06ecd92b830bbfba033af6d3 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 08:12:36 -0700 Subject: [PATCH 077/114] Use helper to skip individual strings Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 36 +++++++++---------- .../zookeeper_proxy/zookeeper_decoder.h | 1 + 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index e05c4468ec265..63909d9971c86 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -137,7 +137,9 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ // Read password - skip zxid, timeout, and session id. offset += ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH; - const std::string passwd = BufferHelper::peekString(data, offset); + + // Skip passwd. + skipString(data, offset); // Read readonly flag, if it's there. bool readonly{}; @@ -155,8 +157,7 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin offset += OPCODE_LENGTH + INT_LENGTH; const std::string scheme = BufferHelper::peekString(data, offset); // Skip credential. - const int32_t credlen = BufferHelper::peekInt32(data, offset); - offset += credlen; + skipString(data, offset); callbacks_.onAuthRequest(scheme); } @@ -177,11 +178,9 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { // Perms. BufferHelper::peekInt32(data, offset); // Skip scheme. - int32_t datalen = BufferHelper::peekInt32(data, offset); - offset += datalen; + skipString(data, offset); // Skip cred. - datalen = BufferHelper::peekInt32(data, offset); - offset += datalen; + skipString(data, offset); } } @@ -192,8 +191,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u const std::string path = BufferHelper::peekString(data, offset); // Skip data. - const int32_t datalen = BufferHelper::peekInt32(data, offset); - offset += datalen; + skipString(data, offset); skipAcls(data, offset); const int32_t flags = BufferHelper::peekInt32(data, offset); const bool ephemeral = (flags & 0x1) == 1; @@ -207,8 +205,7 @@ void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint const std::string path = BufferHelper::peekString(data, offset); // Skip data. - const int32_t datalen = BufferHelper::peekInt32(data, offset); - offset += datalen; + skipString(data, offset); // Ignore version. BufferHelper::peekInt32(data, offset); @@ -315,14 +312,11 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH) + LONG_LENGTH); // Skip joining. - int32_t datalen = BufferHelper::peekInt32(data, offset); - offset += datalen; + skipString(data, offset); // Skip leaving. - datalen = BufferHelper::peekInt32(data, offset); - offset += datalen; + skipString(data, offset); // Skip new members. - datalen = BufferHelper::peekInt32(data, offset); - offset += datalen; + skipString(data, offset); // Read config id. BufferHelper::peekInt64(data, offset); @@ -356,12 +350,16 @@ void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, } } +void DecoderImpl::skipString(Buffer::Instance& data, uint64_t& offset) const { + const int32_t slen = BufferHelper::peekInt32(data, offset); + offset += slen; +} + void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { const int32_t count = BufferHelper::peekInt32(data, offset); for (int i = 0; i < count; ++i) { - const int32_t len = BufferHelper::peekInt32(data, offset); - offset += len; + skipString(data, offset); } } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index c7b3c0512ebb7..3fd876ae85809 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -116,6 +116,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); + void skipString(Buffer::Instance& data, uint64_t& offset) const; void skipStrings(Buffer::Instance& data, uint64_t& offset) const; void ensureMinLength(int32_t len, int32_t minlen) const; void ensureMaxLength(int32_t len) const; From e3ee7f2bec0b06fea37aa58bfe2a25053e0fa85b Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 08:15:20 -0700 Subject: [PATCH 078/114] Remove const from values in decls Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter.h | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 71d90a7911d8b..8b3dd77624ba1 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -96,25 +96,25 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab // ZooKeeperProxy::DecoderCallback void onDecodeError() override; - void onRequestBytes(const uint64_t bytes) override; - void onConnect(const bool readonly) override; + void onRequestBytes(uint64_t bytes) override; + void onConnect(bool readonly) override; void onPing() override; void onAuthRequest(const std::string& scheme) override; - void onGetDataRequest(const std::string& path, const bool watch) override; - void onCreateRequest(const std::string& path, const bool ephemeral, const bool sequence, const bool two) override; + void onGetDataRequest(const std::string& path, bool watch) override; + void onCreateRequest(const std::string& path, bool ephemeral, bool sequence, bool two) override; void onSetRequest(const std::string& path) override; - void onGetChildrenRequest(const std::string& path, const bool watch, const bool two) override; - void onDeleteRequest(const std::string& path, const int32_t version) override; - void onExistsRequest(const std::string& path, const bool watch) override; + void onGetChildrenRequest(const std::string& path, bool watch, bool two) override; + void onDeleteRequest(const std::string& path, int32_t version) override; + void onExistsRequest(const std::string& path, bool watch) override; void onGetAclRequest(const std::string& path) override; - void onSetAclRequest(const std::string& path, const int32_t version) override; + void onSetAclRequest(const std::string& path, int32_t version) override; void onSyncRequest(const std::string& path) override; - void onCheckRequest(const std::string& path, const int32_t version) override; + void onCheckRequest(const std::string& path, int32_t version) override; void onMultiRequest() override; void onReconfigRequest() override; void onSetWatchesRequest() override; - void onCheckWatchesRequest(const std::string& path, const int32_t type) override; - void onRemoveWatchesRequest(const std::string& path, const int32_t type) override; + void onCheckWatchesRequest(const std::string& path, int32_t type) override; + void onRemoveWatchesRequest(const std::string& path, int32_t type) override; void onCloseRequest() override; void doDecode(Buffer::Instance& buffer); From d845a5a1aa46fbd5edb277028e096e5ea7a51f61 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 08:16:31 -0700 Subject: [PATCH 079/114] Clarify that helpers aren't for IO Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h index 87b53e165508e..191b55e0da285 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -13,7 +13,7 @@ namespace NetworkFilters { namespace ZooKeeperProxy { /** - * IO helpers for reading/writing ZooKeeper data from/to a buffer. + * Helpers for reading/writing ZooKeeper data from/to a buffer. * * Note: ZooKeeper's protocol uses network byte ordering (big-endian). */ From a115c66027062aeea3a6e7d345408916d2ed2271 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Thu, 14 Mar 2019 08:25:20 -0700 Subject: [PATCH 080/114] Fix spelling Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 63909d9971c86..ef5c1a27dc352 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -135,10 +135,10 @@ void DecoderImpl::ensureMaxLength(const int32_t len) const { void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); - // Read password - skip zxid, timeout, and session id. + // Skip zxid, timeout, and session id. offset += ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH; - // Skip passwd. + // Skip password. skipString(data, offset); // Read readonly flag, if it's there. From f0016644a76c27415e81c580d63aeeb0745d0045 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 07:04:53 -0700 Subject: [PATCH 081/114] Support all of create()'s flags Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 6 +--- .../zookeeper_proxy/zookeeper_decoder.cc | 27 +++++++++++++--- .../zookeeper_proxy/zookeeper_decoder.h | 32 +++++++++++++++++-- .../zookeeper_proxy/zookeeper_filter.cc | 16 ++++------ .../zookeeper_proxy/zookeeper_filter.h | 2 +- .../zookeeper_proxy/zookeeper_filter_test.cc | 10 +++--- 6 files changed, 65 insertions(+), 28 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index e39397073f0fa..facdaf4fce826 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -11,9 +11,6 @@ are supported. The unsupported ones are: -* createContainer -* deleteContainer -* createTTL * sasl * getEphemerals * getAllChildrenNumber @@ -95,8 +92,7 @@ The ZooKeeper filter emits the following dynamic metadata for each message parse , string, "The path associated with the request, response or event" , string, "The opname for the request, response or event" + , string, "The string representation of the flags applied to the znode" , string, "The size of the request message in bytes" , string, "True if a watch is being set, false otherwise" - , string, "True if the create znode will be ephemeral, false otherwise" - , string, "True if the create znode will be sequential, false otherwise" , string, "The version parameter, if any, given with the request" diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index ef5c1a27dc352..50d1e43c2edc3 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -193,11 +193,30 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u // Skip data. skipString(data, offset); skipAcls(data, offset); - const int32_t flags = BufferHelper::peekInt32(data, offset); - const bool ephemeral = (flags & 0x1) == 1; - const bool sequence = (flags & 0x2) == 2; - callbacks_.onCreateRequest(path, ephemeral, sequence, two); + CreateFlags flags = CreateFlags.PERSISTENT; + switch (BufferHelper::peekInt32(data, offset)) { + case 6: + flags = CreateFlags.PERSISTENT_SEQUENTIAL_WITH_TTL; + break; + case 5: + flags = CreateFlags.PERSISTENT_WITH_TTL; + break; + case 4: + flags = CreateFlags.CONTAINER; + break; + case 3: + flags = CreateFlags.EPHEMERAL_SEQUENTIAL; + break; + case 2: + flags = CreateFlags.SEQUENTIAL; + break; + case 1: + flags = CreateFlags.EPHEMERAL; + break; + } + + callbacks_.onCreateRequest(path, flags, two); } void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 3fd876ae85809..52645d0cc62ee 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -46,6 +46,35 @@ enum class OpCodes { enum class WatcherType { CHILDREN = 1, DATA = 2, ANY = 3 }; +enum class CreateFlags { + PERSISTENT, + PERSISTENT_SEQUENTIAL, + EPHEMERAL, + EPHEMERAL_SEQUENTIAL, + CONTAINER, + PESISTENT_WITH_TTL, + PERSISTENT_SEQUENTIAL_WITH_TTL +}; + +const char* createFlagsToString(CreateFlags flags) { + switch (flags) { + case CreateFlags.PERSISTENT: + return "persistent"; + case CreateFlags.PERSISTENT_SEQUENTIAL: + return "persistent_sequential"; + case CreateFlags.EPHEMERAL: + return "ephemeral"; + case CreateFlags.EPHEMERAL_SEQUENTIAL: + return "ephemeral_sequential"; + case CreateFlags.CONTAINER: + return "container"; + case CreateFlags.PESISTENT_WITH_TTL: + return "persistent_with_ttl"; + case CreateFlags.PERSISTENT_SEQUENTIAL_WITH_TTL: + return "persistent_sequential_with_ttl"; + } +} + /** * General callbacks for dispatching decoded ZooKeeper messages to a sink. */ @@ -59,8 +88,7 @@ class DecoderCallbacks { virtual void onPing() PURE; virtual void onAuthRequest(const std::string& scheme) PURE; virtual void onGetDataRequest(const std::string& path, bool watch) PURE; - virtual void onCreateRequest(const std::string& path, bool ephemeral, bool sequence, - bool two) PURE; + virtual void onCreateRequest(const std::string& path, CreateFlags flags, bool two) PURE; virtual void onSetRequest(const std::string& path) PURE; virtual void onGetChildrenRequest(const std::string& path, bool watch, bool two) PURE; virtual void onDeleteRequest(const std::string& path, int32_t version) PURE; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 9fffae49e85c5..947ba04df02cd 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -116,20 +116,16 @@ void ZooKeeperFilter::onGetDataRequest(const std::string& path, const bool watch setDynamicMetadata({{"opname", "getdata"}, {"path", path}, {"watch", watch ? "true" : "false"}}); } -void ZooKeeperFilter::onCreateRequest(const std::string& path, const bool ephemeral, - const bool sequence, const bool two) { +void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags flags, + const bool two) { if (!two) { config_->stats_.create_rq_.inc(); - setDynamicMetadata({{"opname", "create"}, - {"path", path}, - {"ephemeral", ephemeral ? "true" : "false"}, - {"sequence", sequence ? "true" : "false"}}); + setDynamicMetadata( + {{"opname", "create"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); } else { config_->stats_.create2_rq_.inc(); - setDynamicMetadata({{"opname", "create2"}, - {"path", path}, - {"ephemeral", ephemeral ? "true" : "false"}, - {"sequence", sequence ? "true" : "false"}}); + setDynamicMetadata( + {{"opname", "create2"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); } } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 8b3dd77624ba1..aeb59188e2bee 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -101,7 +101,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onPing() override; void onAuthRequest(const std::string& scheme) override; void onGetDataRequest(const std::string& path, bool watch) override; - void onCreateRequest(const std::string& path, bool ephemeral, bool sequence, bool two) override; + void onCreateRequest(const std::string& path, CreateFlags flags, bool two) override; void onSetRequest(const std::string& path) override; void onGetChildrenRequest(const std::string& path, bool watch, bool two) override; void onDeleteRequest(const std::string& path, int32_t version) override; diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 779c7fbcea09d..4441150ce350a 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -472,9 +472,8 @@ TEST_F(ZooKeeperFilterTest, CreateRequest) { Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", false, false); - expectSetDynamicMetadata( - {{"opname", "create"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}, - {{"bytes", "35"}}); + expectSetDynamicMetadata({{"opname", "create"}, {"path", "/foo"}, {"create_type", "persistent"}}, + {{"bytes", "35"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); @@ -488,9 +487,8 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", false, false, false, enumToInt(OpCodes::CREATE2)); - expectSetDynamicMetadata( - {{"opname", "create2"}, {"path", "/foo"}, {"ephemeral", "false"}, {"sequence", "false"}}, - {{"bytes", "35"}}); + expectSetDynamicMetadata({{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, + {{"bytes", "35"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create2_rq_.value()); From 36d1c396e7856be1db2753fd2033f6823435a6d4 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 07:13:35 -0700 Subject: [PATCH 082/114] Fix build Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/zookeeper_decoder.cc | 14 +++++++------- .../network/zookeeper_proxy/zookeeper_decoder.h | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 50d1e43c2edc3..a3894565107d4 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -194,25 +194,25 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u skipString(data, offset); skipAcls(data, offset); - CreateFlags flags = CreateFlags.PERSISTENT; + CreateFlags flags = CreateFlags::PERSISTENT; switch (BufferHelper::peekInt32(data, offset)) { case 6: - flags = CreateFlags.PERSISTENT_SEQUENTIAL_WITH_TTL; + flags = CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL; break; case 5: - flags = CreateFlags.PERSISTENT_WITH_TTL; + flags = CreateFlags::PERSISTENT_WITH_TTL; break; case 4: - flags = CreateFlags.CONTAINER; + flags = CreateFlags::CONTAINER; break; case 3: - flags = CreateFlags.EPHEMERAL_SEQUENTIAL; + flags = CreateFlags::EPHEMERAL_SEQUENTIAL; break; case 2: - flags = CreateFlags.SEQUENTIAL; + flags = CreateFlags::SEQUENTIAL; break; case 1: - flags = CreateFlags.EPHEMERAL; + flags = CreateFlags::EPHEMERAL; break; } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 52645d0cc62ee..f2f7ff0c4c376 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -58,19 +58,19 @@ enum class CreateFlags { const char* createFlagsToString(CreateFlags flags) { switch (flags) { - case CreateFlags.PERSISTENT: + case CreateFlags::PERSISTENT: return "persistent"; - case CreateFlags.PERSISTENT_SEQUENTIAL: + case CreateFlags::PERSISTENT_SEQUENTIAL: return "persistent_sequential"; - case CreateFlags.EPHEMERAL: + case CreateFlags::EPHEMERAL: return "ephemeral"; - case CreateFlags.EPHEMERAL_SEQUENTIAL: + case CreateFlags::EPHEMERAL_SEQUENTIAL: return "ephemeral_sequential"; - case CreateFlags.CONTAINER: + case CreateFlags::CONTAINER: return "container"; - case CreateFlags.PESISTENT_WITH_TTL: + case CreateFlags::PESISTENT_WITH_TTL: return "persistent_with_ttl"; - case CreateFlags.PERSISTENT_SEQUENTIAL_WITH_TTL: + case CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL: return "persistent_sequential_with_ttl"; } } From 7ef7405a31a8ae5a871b85a204593f9b9c8c5697 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 07:22:40 -0700 Subject: [PATCH 083/114] Move createFlagsToString out of header Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 19 +++++++++++++++++++ .../zookeeper_proxy/zookeeper_decoder.h | 19 +------------------ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index a3894565107d4..68cff5fad5b22 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -17,6 +17,25 @@ constexpr uint32_t TIMEOUT_LENGTH = 4; constexpr uint32_t SESSION_LENGTH = 8; constexpr uint32_t MULTI_HEADER_LENGTH = 9; +const char* createFlagsToString(CreateFlags flags) { + switch (flags) { + case CreateFlags::PERSISTENT: + return "persistent"; + case CreateFlags::PERSISTENT_SEQUENTIAL: + return "persistent_sequential"; + case CreateFlags::EPHEMERAL: + return "ephemeral"; + case CreateFlags::EPHEMERAL_SEQUENTIAL: + return "ephemeral_sequential"; + case CreateFlags::CONTAINER: + return "container"; + case CreateFlags::PESISTENT_WITH_TTL: + return "persistent_with_ttl"; + case CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL: + return "persistent_sequential_with_ttl"; + } +} + void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index f2f7ff0c4c376..b746c5981678a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -56,24 +56,7 @@ enum class CreateFlags { PERSISTENT_SEQUENTIAL_WITH_TTL }; -const char* createFlagsToString(CreateFlags flags) { - switch (flags) { - case CreateFlags::PERSISTENT: - return "persistent"; - case CreateFlags::PERSISTENT_SEQUENTIAL: - return "persistent_sequential"; - case CreateFlags::EPHEMERAL: - return "ephemeral"; - case CreateFlags::EPHEMERAL_SEQUENTIAL: - return "ephemeral_sequential"; - case CreateFlags::CONTAINER: - return "container"; - case CreateFlags::PESISTENT_WITH_TTL: - return "persistent_with_ttl"; - case CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL: - return "persistent_sequential_with_ttl"; - } -} +const char* createFlagsToString(CreateFlags flags); /** * General callbacks for dispatching decoded ZooKeeper messages to a sink. From be08bfa117c018405939e7991eb7c1a3618fc454 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 07:31:28 -0700 Subject: [PATCH 084/114] Fix name Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 68cff5fad5b22..11266f758c452 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -228,7 +228,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u flags = CreateFlags::EPHEMERAL_SEQUENTIAL; break; case 2: - flags = CreateFlags::SEQUENTIAL; + flags = CreateFlags::PERSISTENT_SEQUENTIAL; break; case 1: flags = CreateFlags::EPHEMERAL; From 4695977c6c9b0fd3d33a89906aa2b140e677b6de Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 07:32:28 -0700 Subject: [PATCH 085/114] Fix more typos Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- .../filters/network/zookeeper_proxy/zookeeper_decoder.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 11266f758c452..2a446e312e01f 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -29,7 +29,7 @@ const char* createFlagsToString(CreateFlags flags) { return "ephemeral_sequential"; case CreateFlags::CONTAINER: return "container"; - case CreateFlags::PESISTENT_WITH_TTL: + case CreateFlags::PERSISTENT_WITH_TTL: return "persistent_with_ttl"; case CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL: return "persistent_sequential_with_ttl"; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index b746c5981678a..df368269c8d90 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -52,7 +52,7 @@ enum class CreateFlags { EPHEMERAL, EPHEMERAL_SEQUENTIAL, CONTAINER, - PESISTENT_WITH_TTL, + PERSISTENT_WITH_TTL, PERSISTENT_SEQUENTIAL_WITH_TTL }; From e301b74b9623468d0edc70b77bb9f1ec54dbfa0c Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 07:36:25 -0700 Subject: [PATCH 086/114] Handle unknown flags. Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 2a446e312e01f..f39ee166e8dc7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -34,6 +34,8 @@ const char* createFlagsToString(CreateFlags flags) { case CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL: return "persistent_sequential_with_ttl"; } + + return "unknown"; } void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { From eb2cebde0e4a991fec08dc2b94a29df71d962404 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 08:23:36 -0700 Subject: [PATCH 087/114] Handle container & TTL znode creation Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 10 ++++---- .../zookeeper_proxy/zookeeper_decoder.h | 4 +++- .../zookeeper_proxy/zookeeper_filter.cc | 23 ++++++++++++++++--- .../zookeeper_proxy/zookeeper_filter.h | 4 +++- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index f39ee166e8dc7..33db28e827cb2 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -86,10 +86,10 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseGetDataRequest(data, offset, len); break; case enumToInt(OpCodes::CREATE): - parseCreateRequest(data, offset, len, false); - break; case enumToInt(OpCodes::CREATE2): - parseCreateRequest(data, offset, len, true); + case enumToInt(OpCodes::CREATECONTAINER): + case enumToInt(OpCodes::CREATETTL): + parseCreateRequest(data, offset, len, opcode); break; case enumToInt(OpCodes::SETDATA): parseSetRequest(data, offset, len); @@ -206,7 +206,7 @@ void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { } void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, - const bool two) { + OpCodes opcode) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); const std::string path = BufferHelper::peekString(data, offset); @@ -237,7 +237,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u break; } - callbacks_.onCreateRequest(path, flags, two); + callbacks_.onCreateRequest(path, flags, opcode); } void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index df368269c8d90..899a414e7f397 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -39,6 +39,8 @@ enum class OpCodes { RECONFIG = 16, CHECKWATCHES = 17, REMOVEWATCHES = 18, + CREATECONTAINER = 19, + CREATETTL = 21, CLOSE = -11, SETAUTH = 100, SETWATCHES = 101 @@ -71,7 +73,7 @@ class DecoderCallbacks { virtual void onPing() PURE; virtual void onAuthRequest(const std::string& scheme) PURE; virtual void onGetDataRequest(const std::string& path, bool watch) PURE; - virtual void onCreateRequest(const std::string& path, CreateFlags flags, bool two) PURE; + virtual void onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) PURE; virtual void onSetRequest(const std::string& path) PURE; virtual void onGetChildrenRequest(const std::string& path, bool watch, bool two) PURE; virtual void onDeleteRequest(const std::string& path, int32_t version) PURE; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 947ba04df02cd..849d87b1895b1 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -117,15 +117,32 @@ void ZooKeeperFilter::onGetDataRequest(const std::string& path, const bool watch } void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags flags, - const bool two) { - if (!two) { + const OpCodes opcode) { + switch (opcode) { + case OpCodes::CREATE: config_->stats_.create_rq_.inc(); setDynamicMetadata( {{"opname", "create"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); - } else { + break; + case OpCodes::CREATE2: config_->stats_.create2_rq_.inc(); setDynamicMetadata( {{"opname", "create2"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); + break; + case OpCodes::CREATECONTAINER: + config_->stats_.createcontainer_rq_.inc(); + setDynamicMetadata({{"opname", "createcontainer"}, + {"path", path}, + {"create_type", createFlagsToString(flags)}}); + break; + case OpCodes::CREATETTL: + config_->stats_.createttl_rq_.inc(); + setDynamicMetadata( + {{"opname", "createttl"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); + break; + default: + throw EnvoyException(fmt::format("Unknown opcode: {}", opcode)); + break; } } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index aeb59188e2bee..fd1406b35d08a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -28,6 +28,8 @@ namespace ZooKeeperProxy { COUNTER(getdata_rq) \ COUNTER(create_rq) \ COUNTER(create2_rq) \ + COUNTER(createcontainer_rq) \ + COUNTER(createttl_rq) \ COUNTER(setdata_rq) \ COUNTER(getchildren_rq) \ COUNTER(getchildren2_rq) \ @@ -101,7 +103,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onPing() override; void onAuthRequest(const std::string& scheme) override; void onGetDataRequest(const std::string& path, bool watch) override; - void onCreateRequest(const std::string& path, CreateFlags flags, bool two) override; + void onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) override; void onSetRequest(const std::string& path) override; void onGetChildrenRequest(const std::string& path, bool watch, bool two) override; void onDeleteRequest(const std::string& path, int32_t version) override; From 2a0374df39f5f02ce48aaf7723ec2220a817c7ae Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 10:53:37 -0700 Subject: [PATCH 088/114] Fix Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 849d87b1895b1..a0092e3c37190 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -141,7 +141,7 @@ void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags {{"opname", "createttl"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); break; default: - throw EnvoyException(fmt::format("Unknown opcode: {}", opcode)); + throw EnvoyException(fmt::format("Unknown opcode: {}", enumToInt(opcode))); break; } } From 2f1b994f835799f27d0766a3cd0157b3aae2b724 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 10:55:37 -0700 Subject: [PATCH 089/114] Fix signature Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 899a414e7f397..dbb8f6b5db59c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -115,7 +115,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, bool two); + void parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); void skipAcls(Buffer::Instance& data, uint64_t& offset) const; void parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, bool two); From 8b705e687edacb3bca3918225bdf4ca0112641ba Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 13:12:01 -0700 Subject: [PATCH 090/114] Fix type Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 33db28e827cb2..d099c2c1eeffb 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -89,7 +89,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { case enumToInt(OpCodes::CREATE2): case enumToInt(OpCodes::CREATECONTAINER): case enumToInt(OpCodes::CREATETTL): - parseCreateRequest(data, offset, len, opcode); + parseCreateRequest(data, offset, len, static_cast(opcode)); break; case enumToInt(OpCodes::SETDATA): parseSetRequest(data, offset, len); From 3043ab2c9a6c101046b17b702429a2c7d13533d9 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 13:19:41 -0700 Subject: [PATCH 091/114] Fix parseCreateRequest call Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index d099c2c1eeffb..54f2b93ae082a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -331,7 +331,7 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui switch (type) { case enumToInt(OpCodes::CREATE): - parseCreateRequest(data, offset, len, false); + parseCreateRequest(data, offset, len, OpCodes::CREATE); break; case enumToInt(OpCodes::SETDATA): parseSetRequest(data, offset, len); From b9d48833f9029c286035752036a00e1663620416 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 19:32:10 -0700 Subject: [PATCH 092/114] Reduce calls to enumToInt() Also, in the tests I had a bunch of calls to enumToInt() when what was needed was enumToIntSigned(). Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 65 +++++++++-------- .../zookeeper_proxy/zookeeper_decoder.h | 1 - .../zookeeper_proxy/zookeeper_filter.cc | 3 +- .../zookeeper_proxy/zookeeper_filter_test.cc | 69 ++++++++++--------- 4 files changed, 69 insertions(+), 69 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 54f2b93ae082a..0ba172eb0d571 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -57,18 +57,18 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // However, some client implementations might expose setWatches // as a regular data request, so we support that as well. const int32_t xid = BufferHelper::peekInt32(data, offset); - switch (xid) { - case enumToIntSigned(XidCodes::CONNECT_XID): + switch (static_cast(xid)) { + case XidCodes::CONNECT_XID: parseConnect(data, offset, len); return; - case enumToIntSigned(XidCodes::PING_XID): + case XidCodes::PING_XID: offset += OPCODE_LENGTH; callbacks_.onPing(); return; - case enumToIntSigned(XidCodes::AUTH_XID): + case XidCodes::AUTH_XID: parseAuthRequest(data, offset, len); return; - case enumToIntSigned(XidCodes::SET_WATCHES_XID): + case XidCodes::SET_WATCHES_XID: offset += OPCODE_LENGTH; parseSetWatchesRequest(data, offset, len); return; @@ -81,59 +81,59 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // must happen every 1/3 of the negotiated session timeout, to keep // the session alive. const int32_t opcode = BufferHelper::peekInt32(data, offset); - switch (opcode) { - case enumToInt(OpCodes::GETDATA): + switch (static_cast(opcode)) { + case OpCodes::GETDATA: parseGetDataRequest(data, offset, len); break; - case enumToInt(OpCodes::CREATE): - case enumToInt(OpCodes::CREATE2): - case enumToInt(OpCodes::CREATECONTAINER): - case enumToInt(OpCodes::CREATETTL): + case OpCodes::CREATE: + case OpCodes::CREATE2: + case OpCodes::CREATECONTAINER: + case OpCodes::CREATETTL: parseCreateRequest(data, offset, len, static_cast(opcode)); break; - case enumToInt(OpCodes::SETDATA): + case OpCodes::SETDATA: parseSetRequest(data, offset, len); break; - case enumToInt(OpCodes::GETCHILDREN): + case OpCodes::GETCHILDREN: parseGetChildrenRequest(data, offset, len, false); break; - case enumToInt(OpCodes::GETCHILDREN2): + case OpCodes::GETCHILDREN2: parseGetChildrenRequest(data, offset, len, true); break; - case enumToInt(OpCodes::DELETE): + case OpCodes::DELETE: parseDeleteRequest(data, offset, len); break; - case enumToInt(OpCodes::EXISTS): + case OpCodes::EXISTS: parseExistsRequest(data, offset, len); break; - case enumToInt(OpCodes::GETACL): + case OpCodes::GETACL: parseGetAclRequest(data, offset, len); break; - case enumToInt(OpCodes::SETACL): + case OpCodes::SETACL: parseSetAclRequest(data, offset, len); break; - case enumToInt(OpCodes::SYNC): + case OpCodes::SYNC: parseSyncRequest(data, offset, len); break; - case enumToInt(OpCodes::CHECK): + case OpCodes::CHECK: parseCheckRequest(data, offset, len); break; - case enumToInt(OpCodes::MULTI): + case OpCodes::MULTI: parseMultiRequest(data, offset, len); break; - case enumToInt(OpCodes::RECONFIG): + case OpCodes::RECONFIG: parseReconfigRequest(data, offset, len); break; - case enumToInt(OpCodes::SETWATCHES): + case OpCodes::SETWATCHES: parseSetWatchesRequest(data, offset, len); break; - case enumToInt(OpCodes::CHECKWATCHES): + case OpCodes::CHECKWATCHES: parseXWatchesRequest(data, offset, len, OpCodes::CHECKWATCHES); break; - case enumToInt(OpCodes::REMOVEWATCHES): + case OpCodes::REMOVEWATCHES: parseXWatchesRequest(data, offset, len, OpCodes::REMOVEWATCHES); break; - case enumToIntSigned(OpCodes::CLOSE): + case OpCodes::CLOSE: callbacks_.onCloseRequest(); break; default: @@ -320,7 +320,7 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + MULTI_HEADER_LENGTH); while (true) { - const int32_t type = BufferHelper::peekInt32(data, offset); + const int32_t opcode = BufferHelper::peekInt32(data, offset); const bool done = BufferHelper::peekBool(data, offset); // Ignore error field. BufferHelper::peekInt32(data, offset); @@ -329,19 +329,18 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui break; } - switch (type) { - case enumToInt(OpCodes::CREATE): + switch (static_cast(opcode)) { + case OpCodes::CREATE: parseCreateRequest(data, offset, len, OpCodes::CREATE); break; - case enumToInt(OpCodes::SETDATA): + case OpCodes::SETDATA: parseSetRequest(data, offset, len); break; - case enumToInt(OpCodes::CHECK): + case OpCodes::CHECK: parseCheckRequest(data, offset, len); break; default: - // Should not happen. - throw EnvoyException("Unknown type within a transaction"); + throw EnvoyException(fmt::format("Unknown opcode within a transaction: {}", opcode)); } } diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index dbb8f6b5db59c..6a7829734767e 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -4,7 +4,6 @@ #include "envoy/common/platform.h" #include "common/buffer/buffer_impl.h" -#include "common/common/enum_to_int.h" #include "common/common/logger.h" namespace Envoy { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index a0092e3c37190..07a0b9c009756 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -2,6 +2,7 @@ #include "common/buffer/buffer_impl.h" #include "common/common/assert.h" +#include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/logger.h" @@ -141,7 +142,7 @@ void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags {{"opname", "createttl"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); break; default: - throw EnvoyException(fmt::format("Unknown opcode: {}", enumToInt(opcode))); + throw EnvoyException(fmt::format("Unknown opcode: {}", enumToIntSigned(opcode))); break; } } diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 4441150ce350a..b8c20a3541cb5 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -80,7 +80,7 @@ class ZooKeeperFilterTest : public testing::Test { Buffer::OwnedImpl buffer; buffer.writeBEInt(8); - buffer.writeBEInt(enumToInt(XidCodes::PING_XID)); + buffer.writeBEInt(enumToIntSigned(XidCodes::PING_XID)); buffer.writeBEInt(enumToInt(OpCodes::PING)); return buffer; @@ -101,7 +101,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(8); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToInt(OpCodes::CLOSE)); + buffer.writeBEInt(enumToIntSigned(OpCodes::CLOSE)); return buffer; } @@ -111,8 +111,8 @@ class ZooKeeperFilterTest : public testing::Test { Buffer::OwnedImpl buffer; buffer.writeBEInt(28 + scheme.length() + credential.length()); - buffer.writeBEInt(enumToInt(XidCodes::AUTH_XID)); - buffer.writeBEInt(enumToInt(OpCodes::SETAUTH)); + buffer.writeBEInt(enumToIntSigned(XidCodes::AUTH_XID)); + buffer.writeBEInt(enumToIntSigned(OpCodes::SETAUTH)); // Type. buffer.writeBEInt(0); addString(buffer, scheme); @@ -121,8 +121,9 @@ class ZooKeeperFilterTest : public testing::Test { return buffer; } - Buffer::OwnedImpl encodePathWatch(const std::string& path, const bool watch, - const int32_t opcode = enumToInt(OpCodes::GETDATA)) const { + Buffer::OwnedImpl + encodePathWatch(const std::string& path, const bool watch, + const int32_t opcode = enumToIntSigned(OpCodes::GETDATA)) const { Buffer::OwnedImpl buffer; buffer.writeBEInt(13 + path.length()); @@ -139,7 +140,7 @@ class ZooKeeperFilterTest : public testing::Test { } Buffer::OwnedImpl encodePathVersion(const std::string& path, const int32_t version, - const int32_t opcode = enumToInt(OpCodes::GETDATA), + const int32_t opcode = enumToIntSigned(OpCodes::GETDATA), const bool txn = false) const { Buffer::OwnedImpl buffer; @@ -170,10 +171,10 @@ class ZooKeeperFilterTest : public testing::Test { return buffer; } - Buffer::OwnedImpl encodeCreateRequest(const std::string& path, const std::string& data, - const bool ephemeral, const bool sequence, - const bool txn = false, - const int32_t opcode = enumToInt(OpCodes::CREATE)) const { + Buffer::OwnedImpl + encodeCreateRequest(const std::string& path, const std::string& data, const bool ephemeral, + const bool sequence, const bool txn = false, + const int32_t opcode = enumToIntSigned(OpCodes::CREATE)) const { Buffer::OwnedImpl buffer; if (!txn) { @@ -208,7 +209,7 @@ class ZooKeeperFilterTest : public testing::Test { if (!txn) { buffer.writeBEInt(20 + path.length() + data.length()); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToInt(OpCodes::SETDATA)); + buffer.writeBEInt(enumToIntSigned(OpCodes::SETDATA)); } // Path. @@ -227,7 +228,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(16 + path.length()); buffer.writeBEInt(1000); // Opcode. - buffer.writeBEInt(enumToInt(OpCodes::DELETE)); + buffer.writeBEInt(enumToIntSigned(OpCodes::DELETE)); // Path. addString(buffer, path); // Version. @@ -244,7 +245,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(32 + path.length() + scheme.length() + credential.length()); buffer.writeBEInt(1000); // Opcode. - buffer.writeBEInt(enumToInt(OpCodes::SETACL)); + buffer.writeBEInt(enumToIntSigned(OpCodes::SETACL)); // Path. addString(buffer, path); @@ -269,7 +270,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(28 + joining.length() + leaving.length() + new_members.length()); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToInt(OpCodes::RECONFIG)); + buffer.writeBEInt(enumToIntSigned(OpCodes::RECONFIG)); addString(buffer, joining); addString(buffer, leaving); addString(buffer, new_members); @@ -291,7 +292,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(8 + watches_buffer.length()); buffer.writeBEInt(xid); - buffer.writeBEInt(enumToInt(OpCodes::SETWATCHES)); + buffer.writeBEInt(enumToIntSigned(OpCodes::SETWATCHES)); buffer.add(watches_buffer); return buffer; @@ -320,7 +321,7 @@ class ZooKeeperFilterTest : public testing::Test { // Multi prefix. buffer.writeBEInt(8 + requests.length()); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToInt(OpCodes::MULTI)); + buffer.writeBEInt(enumToIntSigned(OpCodes::MULTI)); // Requests. buffer.add(requests); @@ -485,7 +486,7 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { initialize(); Buffer::OwnedImpl data = - encodeCreateRequest("/foo", "bar", false, false, false, enumToInt(OpCodes::CREATE2)); + encodeCreateRequest("/foo", "bar", false, false, false, enumToIntSigned(OpCodes::CREATE2)); expectSetDynamicMetadata({{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, {{"bytes", "35"}}); @@ -512,7 +513,7 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { initialize(); - Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN)); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToIntSigned(OpCodes::GETCHILDREN)); expectSetDynamicMetadata({{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); @@ -526,7 +527,7 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { initialize(); - Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToInt(OpCodes::GETCHILDREN2)); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToIntSigned(OpCodes::GETCHILDREN2)); expectSetDynamicMetadata({{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); @@ -554,7 +555,7 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { TEST_F(ZooKeeperFilterTest, ExistsRequest) { initialize(); - Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToInt(OpCodes::EXISTS)); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToIntSigned(OpCodes::EXISTS)); expectSetDynamicMetadata({{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); @@ -568,7 +569,7 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { TEST_F(ZooKeeperFilterTest, GetAclRequest) { initialize(); - Buffer::OwnedImpl data = encodePath("/foo", enumToInt(OpCodes::GETACL)); + Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::GETACL)); expectSetDynamicMetadata({{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}); @@ -595,7 +596,7 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { TEST_F(ZooKeeperFilterTest, SyncRequest) { initialize(); - Buffer::OwnedImpl data = encodePath("/foo", enumToInt(OpCodes::SYNC)); + Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::SYNC)); expectSetDynamicMetadata({{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}); @@ -608,7 +609,7 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { TEST_F(ZooKeeperFilterTest, CheckRequest) { initialize(); - Buffer::OwnedImpl data = encodePathVersion("/foo", 100, enumToInt(OpCodes::CHECK)); + Buffer::OwnedImpl data = encodePathVersion("/foo", 100, enumToIntSigned(OpCodes::CHECK)); expectSetDynamicMetadata({{"bytes", "24"}}); @@ -623,14 +624,14 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { Buffer::OwnedImpl create1 = encodeCreateRequest("/foo", "1", false, false, true); Buffer::OwnedImpl create2 = encodeCreateRequest("/bar", "1", false, false, true); - Buffer::OwnedImpl check1 = encodePathVersion("/foo", 100, enumToInt(OpCodes::CHECK), true); + Buffer::OwnedImpl check1 = encodePathVersion("/foo", 100, enumToIntSigned(OpCodes::CHECK), true); Buffer::OwnedImpl set1 = encodeSetRequest("/bar", "2", -1, true); std::vector> ops; - ops.push_back(std::make_pair(enumToInt(OpCodes::CREATE), std::move(create1))); - ops.push_back(std::make_pair(enumToInt(OpCodes::CREATE), std::move(create2))); - ops.push_back(std::make_pair(enumToInt(OpCodes::CHECK), std::move(check1))); - ops.push_back(std::make_pair(enumToInt(OpCodes::SETDATA), std::move(set1))); + ops.push_back(std::make_pair(enumToIntSigned(OpCodes::CREATE), std::move(create1))); + ops.push_back(std::make_pair(enumToIntSigned(OpCodes::CREATE), std::move(create2))); + ops.push_back(std::make_pair(enumToIntSigned(OpCodes::CHECK), std::move(check1))); + ops.push_back(std::make_pair(enumToIntSigned(OpCodes::SETDATA), std::move(set1))); Buffer::OwnedImpl data = encodeMultiRequest(ops); @@ -664,7 +665,7 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { const std::vector childw = {"/foo2", "/bar2"}; Buffer::OwnedImpl data = - encodeSetWatchesRequest(dataw, existw, childw, enumToInt(XidCodes::SET_WATCHES_XID)); + encodeSetWatchesRequest(dataw, existw, childw, enumToIntSigned(XidCodes::SET_WATCHES_XID)); expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); @@ -694,8 +695,8 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { initialize(); - Buffer::OwnedImpl data = - encodePathVersion("/foo", enumToInt(WatcherType::CHILDREN), enumToInt(OpCodes::CHECKWATCHES)); + Buffer::OwnedImpl data = encodePathVersion("/foo", enumToIntSigned(WatcherType::CHILDREN), + enumToIntSigned(OpCodes::CHECKWATCHES)); expectSetDynamicMetadata({{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); @@ -708,8 +709,8 @@ TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { initialize(); - Buffer::OwnedImpl data = - encodePathVersion("/foo", enumToInt(WatcherType::DATA), enumToInt(OpCodes::REMOVEWATCHES)); + Buffer::OwnedImpl data = encodePathVersion("/foo", enumToIntSigned(WatcherType::DATA), + enumToIntSigned(OpCodes::REMOVEWATCHES)); expectSetDynamicMetadata({{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); From ff5284e304150db11766e80a5d87db1c04be0f73 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 19:39:24 -0700 Subject: [PATCH 093/114] Handle enum value that wasn't being handled Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 0ba172eb0d571..86ab5e622cf9b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -72,6 +72,9 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { offset += OPCODE_LENGTH; parseSetWatchesRequest(data, offset, len); return; + default: + // WATCH_XID is generated by the server, so it can be ignored here. + break; } // Data requests, with XIDs > 0. From f91a5dbad7b45bb0a1b95db776587e6a6062be98 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Sun, 17 Mar 2019 19:46:13 -0700 Subject: [PATCH 094/114] Improve comment on handling XIDs. Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 86ab5e622cf9b..40bf8473e2dd4 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -73,7 +73,8 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseSetWatchesRequest(data, offset, len); return; default: - // WATCH_XID is generated by the server, so it can be ignored here. + // WATCH_XID is generated by the server, so that and everything + // else can be ignored here. break; } From 823ccc50ea5de53a3ec32fb7f8dcf44f2c645237 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 06:17:38 -0700 Subject: [PATCH 095/114] Guard against longer than length messages It's possible for a message (request) to be longer than its declared length; so lets do checks in between each peek call to ensure we don't read beyond than max_length. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 69 ++++++++++--------- .../zookeeper_proxy/zookeeper_decoder.h | 3 +- .../zookeeper_proxy/zookeeper_utils.cc | 18 ++++- .../network/zookeeper_proxy/zookeeper_utils.h | 24 +++++-- .../zookeeper_proxy/zookeeper_filter_test.cc | 26 +++++++ 5 files changed, 100 insertions(+), 40 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 40bf8473e2dd4..92e230b58306f 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -41,8 +41,10 @@ const char* createFlagsToString(CreateFlags flags) { void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); + helper_.reset(); + // Check message length. - const int32_t len = BufferHelper::peekInt32(data, offset); + const int32_t len = helper_.peekInt32(data, offset); ensureMinLength(len, INT_LENGTH + XID_LENGTH); ensureMaxLength(len); @@ -56,7 +58,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // ZooKeeper server to the next. Thus, the special xid. // However, some client implementations might expose setWatches // as a regular data request, so we support that as well. - const int32_t xid = BufferHelper::peekInt32(data, offset); + const int32_t xid = helper_.peekInt32(data, offset); switch (static_cast(xid)) { case XidCodes::CONNECT_XID: parseConnect(data, offset, len); @@ -84,7 +86,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // for two cases: auth requests can happen at any time and ping requests // must happen every 1/3 of the negotiated session timeout, to keep // the session alive. - const int32_t opcode = BufferHelper::peekInt32(data, offset); + const int32_t opcode = helper_.peekInt32(data, offset); switch (static_cast(opcode)) { case OpCodes::GETDATA: parseGetDataRequest(data, offset, len); @@ -169,7 +171,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ // Read readonly flag, if it's there. bool readonly{}; if (data.length() >= offset + 1) { - readonly = BufferHelper::peekBool(data, offset); + readonly = helper_.peekBool(data, offset); } callbacks_.onConnect(readonly); @@ -180,7 +182,7 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin // Skip opcode + type. offset += OPCODE_LENGTH + INT_LENGTH; - const std::string scheme = BufferHelper::peekString(data, offset); + const std::string scheme = helper_.peekString(data, offset); // Skip credential. skipString(data, offset); @@ -190,18 +192,18 @@ void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uin void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); - const std::string path = BufferHelper::peekString(data, offset); - const bool watch = BufferHelper::peekBool(data, offset); + const std::string path = helper_.peekString(data, offset); + const bool watch = helper_.peekBool(data, offset); callbacks_.onGetDataRequest(path, watch); } void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { - const int32_t count = BufferHelper::peekInt32(data, offset); + const int32_t count = helper_.peekInt32(data, offset); for (int i = 0; i < count; ++i) { // Perms. - BufferHelper::peekInt32(data, offset); + helper_.peekInt32(data, offset); // Skip scheme. skipString(data, offset); // Skip cred. @@ -213,14 +215,14 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u OpCodes opcode) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); - const std::string path = BufferHelper::peekString(data, offset); + const std::string path = helper_.peekString(data, offset); // Skip data. skipString(data, offset); skipAcls(data, offset); CreateFlags flags = CreateFlags::PERSISTENT; - switch (BufferHelper::peekInt32(data, offset)) { + switch (helper_.peekInt32(data, offset)) { case 6: flags = CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL; break; @@ -247,11 +249,11 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); - const std::string path = BufferHelper::peekString(data, offset); + const std::string path = helper_.peekString(data, offset); // Skip data. skipString(data, offset); // Ignore version. - BufferHelper::peekInt32(data, offset); + helper_.peekInt32(data, offset); callbacks_.onSetRequest(path); } @@ -260,8 +262,8 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs const bool two) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); - const std::string path = BufferHelper::peekString(data, offset); - const bool watch = BufferHelper::peekBool(data, offset); + const std::string path = helper_.peekString(data, offset); + const bool watch = helper_.peekBool(data, offset); callbacks_.onGetChildrenRequest(path, watch, two); } @@ -269,8 +271,8 @@ void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offs void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); - const std::string path = BufferHelper::peekString(data, offset); - const int32_t version = BufferHelper::peekInt32(data, offset); + const std::string path = helper_.peekString(data, offset); + const int32_t version = helper_.peekInt32(data, offset); callbacks_.onDeleteRequest(path, version); } @@ -278,8 +280,8 @@ void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); - const std::string path = BufferHelper::peekString(data, offset); - const bool watch = BufferHelper::peekBool(data, offset); + const std::string path = helper_.peekString(data, offset); + const bool watch = helper_.peekBool(data, offset); callbacks_.onExistsRequest(path, watch); } @@ -287,7 +289,7 @@ void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); - const std::string path = BufferHelper::peekString(data, offset); + const std::string path = helper_.peekString(data, offset); callbacks_.onGetAclRequest(path); } @@ -295,9 +297,9 @@ void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); - const std::string path = BufferHelper::peekString(data, offset); + const std::string path = helper_.peekString(data, offset); skipAcls(data, offset); - const int32_t version = BufferHelper::peekInt32(data, offset); + const int32_t version = helper_.peekInt32(data, offset); callbacks_.onSetAclRequest(path, version); } @@ -305,7 +307,7 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); - const std::string path = BufferHelper::peekString(data, offset); + const std::string path = helper_.peekString(data, offset); callbacks_.onSyncRequest(path); } @@ -313,8 +315,8 @@ void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uin void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, (2 * INT_LENGTH)); - const std::string path = BufferHelper::peekString(data, offset); - const int32_t version = BufferHelper::peekInt32(data, offset); + const std::string path = helper_.peekString(data, offset); + const int32_t version = helper_.peekInt32(data, offset); callbacks_.onCheckRequest(path, version); } @@ -324,10 +326,10 @@ void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, ui ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + MULTI_HEADER_LENGTH); while (true) { - const int32_t opcode = BufferHelper::peekInt32(data, offset); - const bool done = BufferHelper::peekBool(data, offset); + const int32_t opcode = helper_.peekInt32(data, offset); + const bool done = helper_.peekBool(data, offset); // Ignore error field. - BufferHelper::peekInt32(data, offset); + helper_.peekInt32(data, offset); if (done) { break; @@ -361,7 +363,7 @@ void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, // Skip new members. skipString(data, offset); // Read config id. - BufferHelper::peekInt64(data, offset); + helper_.peekInt64(data, offset); callbacks_.onReconfigRequest(); } @@ -383,8 +385,8 @@ void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, OpCodes opcode) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); - const std::string path = BufferHelper::peekString(data, offset); - const int32_t type = BufferHelper::peekInt32(data, offset); + const std::string path = helper_.peekString(data, offset); + const int32_t type = helper_.peekInt32(data, offset); if (opcode == OpCodes::CHECKWATCHES) { callbacks_.onCheckWatchesRequest(path, type); @@ -394,12 +396,13 @@ void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, } void DecoderImpl::skipString(Buffer::Instance& data, uint64_t& offset) const { - const int32_t slen = BufferHelper::peekInt32(data, offset); + const int32_t slen = helper_.peekInt32(data, offset); offset += slen; + helper_.add(slen); } void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { - const int32_t count = BufferHelper::peekInt32(data, offset); + const int32_t count = helper_.peekInt32(data, offset); for (int i = 0; i < count; ++i) { skipString(data, offset); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 6a7829734767e..d1eb5ecaf0525 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -104,7 +104,7 @@ typedef std::unique_ptr DecoderPtr; class DecoderImpl : public Decoder, Logger::Loggable { public: explicit DecoderImpl(DecoderCallbacks& callbacks, uint32_t max_packet_bytes) - : callbacks_(callbacks), max_packet_bytes_(max_packet_bytes) {} + : callbacks_(callbacks), max_packet_bytes_(max_packet_bytes), helper_(max_packet_bytes) {} // ZooKeeperProxy::Decoder void onData(Buffer::Instance& data) override; @@ -135,6 +135,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { DecoderCallbacks& callbacks_; const uint32_t max_packet_bytes_; + BufferHelper helper_; }; } // namespace ZooKeeperProxy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 0f08ee0d3b126..6856b4d17e246 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -6,18 +6,24 @@ namespace NetworkFilters { namespace ZooKeeperProxy { int32_t BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset) { + ensureMaxLen(sizeof(int32_t)); + int32_t val = buffer.peekBEInt(offset); offset += sizeof(int32_t); return val; } int64_t BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset) { + ensureMaxLen(sizeof(int64_t)); + int64_t val = buffer.peekBEInt(offset); offset += sizeof(int64_t); return val; } bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { + ensureMaxLen(1); + char byte = buffer.peekInt(offset); bool val = static_cast(byte); offset += 1; @@ -26,7 +32,7 @@ bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) { std::string val; - int32_t len = peekInt32(buffer, offset); + uint32_t len = peekInt32(buffer, offset); if (len == 0) { return val; @@ -36,6 +42,8 @@ std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) throw EnvoyException("peekString: buffer is smaller than string length"); } + ensureMaxLen(len); + std::unique_ptr data(new char[len]); buffer.copyOut(offset, len, data.get()); val.assign(data.get(), len); @@ -44,6 +52,14 @@ std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) return val; } +void ensureMaxLen(const uint32_t size) { + current_ += size; + + if (current_ > max_len_) { + throw EnvoyException("read beyond max length"); + } +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h index 191b55e0da285..c9d7039aa7f57 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -13,16 +13,30 @@ namespace NetworkFilters { namespace ZooKeeperProxy { /** - * Helpers for reading/writing ZooKeeper data from/to a buffer. + * Helper for extracting ZooKeeper data from a buffer. + * + * If at any point a peek is tried beyond max_len, an EnvoyException + * will be thrown. This is important to protect Envoy against malformed + * requests (e.g.: when the declared and actual length don't match). * * Note: ZooKeeper's protocol uses network byte ordering (big-endian). */ class BufferHelper : public Logger::Loggable { public: - static int32_t peekInt32(Buffer::Instance& buffer, uint64_t& offset); - static int64_t peekInt64(Buffer::Instance& buffer, uint64_t& offset); - static std::string peekString(Buffer::Instance& buffer, uint64_t& offset); - static bool peekBool(Buffer::Instance& buffer, uint64_t& offset); + BufferHelper(uint32_t max_len) : max_len_(max_len) {} + + int32_t peekInt32(Buffer::Instance& buffer, uint64_t& offset); + int64_t peekInt64(Buffer::Instance& buffer, uint64_t& offset); + std::string peekString(Buffer::Instance& buffer, uint64_t& offset); + bool peekBool(Buffer::Instance& buffer, uint64_t& offset); + void add(uint32_t size) { current_ += size; } + void reset() { current_ = 0; } + +private: + void ensureMaxLen(uint32_t size); + + uint32_t max_len_; + uint32_t current_{}; }; } // namespace ZooKeeperProxy diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index b8c20a3541cb5..c497bdc51fd73 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -76,6 +76,23 @@ class ZooKeeperFilterTest : public testing::Test { return buffer; } + Buffer::OwnedImpl encodeBiggerThanLengthMessage const { + Buffer::OwnedImpl buffer; + + // We craft a delete request with a path that's longer than + // the declared message length. + buffer.writeBEInt(50); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(enumToIntSigned(OpCodes::DELETE)); + // Path. + addString(buffer, std::string(2 * 1024 * 1024, '*')); + // Version. + buffer.writeBEInt(version); + + return buffer; + } + Buffer::OwnedImpl encodePing() const { Buffer::OwnedImpl buffer; @@ -419,6 +436,15 @@ TEST_F(ZooKeeperFilterTest, PacketTooBig) { EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, PacketBiggerThanLength) { + initialize(); + + Buffer::OwnedImpl data = encodeBiggerThanLengthMessage(); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, UnknownOpcode) { initialize(); From a2ae4ecadb31be0ce87c0e1d430cae0a887dbc10 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 06:23:25 -0700 Subject: [PATCH 096/114] Properly namespace method Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 6856b4d17e246..26a0b38bd6d86 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -52,7 +52,7 @@ std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) return val; } -void ensureMaxLen(const uint32_t size) { +void BufferHelper::ensureMaxLen(const uint32_t size) { current_ += size; if (current_ > max_len_) { From aceaca46f71a582bcfc618b90fbc0303651acd0b Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 06:30:10 -0700 Subject: [PATCH 097/114] Move missing header to the right place Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 2 -- .../filters/network/zookeeper_proxy/zookeeper_decoder.h | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 92e230b58306f..7490b5bc3bdcb 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -1,7 +1,5 @@ #include "extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h" -#include "extensions/filters/network/zookeeper_proxy/zookeeper_utils.h" - namespace Envoy { namespace Extensions { namespace NetworkFilters { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index d1eb5ecaf0525..a66ab12b0a82b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -6,6 +6,8 @@ #include "common/buffer/buffer_impl.h" #include "common/common/logger.h" +#include "extensions/filters/network/zookeeper_proxy/zookeeper_utils.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { From f21580fa2b04c8919331d3c102acf2f9150de19a Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 06:50:41 -0700 Subject: [PATCH 098/114] Relax erstwhile const methods Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 6 +++--- .../filters/network/zookeeper_proxy/zookeeper_decoder.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 7490b5bc3bdcb..a4e42930d4506 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -196,7 +196,7 @@ void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, callbacks_.onGetDataRequest(path, watch); } -void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) const { +void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) { const int32_t count = helper_.peekInt32(data, offset); for (int i = 0; i < count; ++i) { @@ -393,13 +393,13 @@ void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, } } -void DecoderImpl::skipString(Buffer::Instance& data, uint64_t& offset) const { +void DecoderImpl::skipString(Buffer::Instance& data, uint64_t& offset) { const int32_t slen = helper_.peekInt32(data, offset); offset += slen; helper_.add(slen); } -void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) const { +void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) { const int32_t count = helper_.peekInt32(data, offset); for (int i = 0; i < count; ++i) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index a66ab12b0a82b..9479ce4d5e641 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -117,7 +117,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); - void skipAcls(Buffer::Instance& data, uint64_t& offset) const; + void skipAcls(Buffer::Instance& data, uint64_t& offset); void parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, bool two); void parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); @@ -130,8 +130,8 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); - void skipString(Buffer::Instance& data, uint64_t& offset) const; - void skipStrings(Buffer::Instance& data, uint64_t& offset) const; + void skipString(Buffer::Instance& data, uint64_t& offset); + void skipStrings(Buffer::Instance& data, uint64_t& offset); void ensureMinLength(int32_t len, int32_t minlen) const; void ensureMaxLength(int32_t len) const; From 9284fc121754070bfbae8f230a825d0ebb687f77 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 07:01:49 -0700 Subject: [PATCH 099/114] Fix syntax Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index c497bdc51fd73..84af734ef174c 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -76,7 +76,7 @@ class ZooKeeperFilterTest : public testing::Test { return buffer; } - Buffer::OwnedImpl encodeBiggerThanLengthMessage const { + Buffer::OwnedImpl encodeBiggerThanLengthMessage() const { Buffer::OwnedImpl buffer; // We craft a delete request with a path that's longer than From 514e92a4b24755e3ed562be8bde0462bb3b038fa Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 07:34:34 -0700 Subject: [PATCH 100/114] Fix test Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 84af734ef174c..37cfe845faab2 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -88,7 +88,7 @@ class ZooKeeperFilterTest : public testing::Test { // Path. addString(buffer, std::string(2 * 1024 * 1024, '*')); // Version. - buffer.writeBEInt(version); + buffer.writeBEInt(-1); return buffer; } From a1a28bcde3814349c2ce5d9dc05df8fed29c13ff Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 12:01:01 -0700 Subject: [PATCH 101/114] Support getEphemerals & getAllChildrenNumber Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 8 +---- .../zookeeper_proxy/zookeeper_decoder.cc | 15 ++++++---- .../zookeeper_proxy/zookeeper_decoder.h | 8 +++-- .../zookeeper_proxy/zookeeper_filter.cc | 10 +++++++ .../zookeeper_proxy/zookeeper_filter.h | 4 +++ .../zookeeper_proxy/zookeeper_filter_test.cc | 29 ++++++++++++++++++- 6 files changed, 58 insertions(+), 16 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index facdaf4fce826..cf8e1c9716a72 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -7,13 +7,7 @@ The ZooKeeper proxy filter decodes the client protocol for `Apache ZooKeeper `_. It decodes the requests, responses and events in the payload. Most opcodes known in `ZooKeeper 3.5 `_ -are supported. - -The unsupported ones are: - -* sasl -* getEphemerals -* getAllChildrenNumber +are supported. The unsupported ones are related to SASL authentication. .. attention:: diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index a4e42930d4506..69a35f8f8e531 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -117,7 +117,7 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { parseSetAclRequest(data, offset, len); break; case OpCodes::SYNC: - parseSyncRequest(data, offset, len); + callbacks_.onSyncRequest(pathOnlyRequest(data, offset, len)); break; case OpCodes::CHECK: parseCheckRequest(data, offset, len); @@ -137,6 +137,12 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { case OpCodes::REMOVEWATCHES: parseXWatchesRequest(data, offset, len, OpCodes::REMOVEWATCHES); break; + case OpCodes::GETEPHEMERALS: + callbacks_.onGetEphemeralsRequest(pathOnlyRequest(data, offset, len)); + break; + case OpCodes::GETALLCHILDRENNUMBER: + callbacks_.onGetAllChildrenNumberRequest(pathOnlyRequest(data, offset, len)); + break; case OpCodes::CLOSE: callbacks_.onCloseRequest(); break; @@ -302,12 +308,9 @@ void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, u callbacks_.onSetAclRequest(path, version); } -void DecoderImpl::parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { +std::string DecoderImpl::pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); - - const std::string path = helper_.peekString(data, offset); - - callbacks_.onSyncRequest(path); + return helper_.peekString(data, offset); } void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 9479ce4d5e641..638b5766680a6 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -44,7 +44,9 @@ enum class OpCodes { CREATETTL = 21, CLOSE = -11, SETAUTH = 100, - SETWATCHES = 101 + SETWATCHES = 101, + GETEPHEMERALS = 103, + GETALLCHILDRENNUMBER = 104 }; enum class WatcherType { CHILDREN = 1, DATA = 2, ANY = 3 }; @@ -77,6 +79,8 @@ class DecoderCallbacks { virtual void onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) PURE; virtual void onSetRequest(const std::string& path) PURE; virtual void onGetChildrenRequest(const std::string& path, bool watch, bool two) PURE; + virtual void onGetEphemeralsRequest(const std::string& path) PURE; + virtual void onGetAllChildrenNumberRequest(const std::string& path) PURE; virtual void onDeleteRequest(const std::string& path, int32_t version) PURE; virtual void onExistsRequest(const std::string& path, bool watch) PURE; virtual void onGetAclRequest(const std::string& path) PURE; @@ -124,7 +128,6 @@ class DecoderImpl : public Decoder, Logger::Loggable { void parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseSyncRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); @@ -134,6 +137,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void skipStrings(Buffer::Instance& data, uint64_t& offset); void ensureMinLength(int32_t len, int32_t minlen) const; void ensureMaxLength(int32_t len) const; + std::string pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); DecoderCallbacks& callbacks_; const uint32_t max_packet_bytes_; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 07a0b9c009756..f4940da40c274 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -219,6 +219,16 @@ void ZooKeeperFilter::onSetWatchesRequest() { setDynamicMetadata("opname", "setwatches"); } +void ZooKeeperFilter::onGetEphemeralsRequest(const std::string& path) { + config_->stats_.getephemerals_rq_.inc(); + setDynamicMetadata({{"opname", "getephemerals"}, {"path", path}}); +} + +void ZooKeeperFilter::onGetAllChildrenNumberRequest(const std::string& path) { + config_->stats_.getallchildrennumber_rq_.inc(); + setDynamicMetadata({{"opname", "getallchildrennumber"}, {"path", path}}); +} + void ZooKeeperFilter::onCloseRequest() { config_->stats_.close_rq_.inc(); setDynamicMetadata("opname", "close"); diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index fd1406b35d08a..87892cd24420c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -33,6 +33,8 @@ namespace ZooKeeperProxy { COUNTER(setdata_rq) \ COUNTER(getchildren_rq) \ COUNTER(getchildren2_rq) \ + COUNTER(getephemerals_rq) \ + COUNTER(getallchildrennumber_rq) \ COUNTER(remove_rq) \ COUNTER(exists_rq) \ COUNTER(getacl_rq) \ @@ -117,6 +119,8 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onSetWatchesRequest() override; void onCheckWatchesRequest(const std::string& path, int32_t type) override; void onRemoveWatchesRequest(const std::string& path, int32_t type) override; + void onGetEphemeralsRequest(const std::string& path) override; + void onGetAllChildrenNumberRequest(const std::string& path) override; void onCloseRequest() override; void doDecode(Buffer::Instance& buffer); diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 37cfe845faab2..32e59205b15aa 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -79,7 +79,7 @@ class ZooKeeperFilterTest : public testing::Test { Buffer::OwnedImpl encodeBiggerThanLengthMessage() const { Buffer::OwnedImpl buffer; - // We craft a delete request with a path that's longer than + // Craft a delete request with a path that's longer than // the declared message length. buffer.writeBEInt(50); buffer.writeBEInt(1000); @@ -632,6 +632,33 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { + initialize(); + + Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::GETEPHEMERALS)); + + expectSetDynamicMetadata({{"opname", "getephemerals"}, {"path", "/foo"}}, {{"bytes", "20"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, config_->stats().getephemerals_rq_.value()); + EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + +TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { + initialize(); + + Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::GETALLCHILDRENNUMBER)); + + expectSetDynamicMetadata({{"opname", "getallchildrennumber"}, {"path", "/foo"}}, + {{"bytes", "20"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, config_->stats().getallchildrennumber_rq_.value()); + EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, CheckRequest) { initialize(); From 4cd3f6efbe91465886985488709d8825b709c24d Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 18 Mar 2019 13:40:52 -0700 Subject: [PATCH 102/114] Simplify int to enum conversion Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.cc | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 69a35f8f8e531..2ce0679a4640a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -225,28 +225,7 @@ void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, u skipString(data, offset); skipAcls(data, offset); - CreateFlags flags = CreateFlags::PERSISTENT; - switch (helper_.peekInt32(data, offset)) { - case 6: - flags = CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL; - break; - case 5: - flags = CreateFlags::PERSISTENT_WITH_TTL; - break; - case 4: - flags = CreateFlags::CONTAINER; - break; - case 3: - flags = CreateFlags::EPHEMERAL_SEQUENTIAL; - break; - case 2: - flags = CreateFlags::PERSISTENT_SEQUENTIAL; - break; - case 1: - flags = CreateFlags::EPHEMERAL; - break; - } - + const CreateFlags flags = static_cast(helper_.peekInt32(data, offset)); callbacks_.onCreateRequest(path, flags, opcode); } From e43cd1ec43a39c956d09eb8688d5c7eb310a29be Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Mar 2019 09:41:06 -0700 Subject: [PATCH 103/114] Encapsulate skip within the BufferHelper Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 3 +-- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 5 +++++ .../filters/network/zookeeper_proxy/zookeeper_utils.h | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 2ce0679a4640a..2a685644b01c7 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -377,8 +377,7 @@ void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, void DecoderImpl::skipString(Buffer::Instance& data, uint64_t& offset) { const int32_t slen = helper_.peekInt32(data, offset); - offset += slen; - helper_.add(slen); + helper_.skip(slen, offset); } void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 26a0b38bd6d86..15af93b87c874 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -52,6 +52,11 @@ std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) return val; } +void BufferHelper::skip(const uint32_t len, uint64_t& offset) { + offset += len; + current_ += len; +} + void BufferHelper::ensureMaxLen(const uint32_t size) { current_ += size; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h index c9d7039aa7f57..559ef0f63093d 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.h @@ -29,7 +29,7 @@ class BufferHelper : public Logger::Loggable { int64_t peekInt64(Buffer::Instance& buffer, uint64_t& offset); std::string peekString(Buffer::Instance& buffer, uint64_t& offset); bool peekBool(Buffer::Instance& buffer, uint64_t& offset); - void add(uint32_t size) { current_ += size; } + void skip(uint32_t len, uint64_t& offset); void reset() { current_ = 0; } private: From fe401f4145b75017d7b4152c61fdcea613807285 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Mar 2019 09:43:43 -0700 Subject: [PATCH 104/114] Explain how offset and cursor_ are different. Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_decoder.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc index 2a685644b01c7..dddd22a0ef634 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.cc @@ -39,6 +39,13 @@ const char* createFlagsToString(CreateFlags flags) { void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); + // Reset the helper's cursor, to ensure the current message stays within the + // allowed max length, even when it's different than the declared length + // by the message. + // + // Note: we need to keep two cursors — offset and helper_'s internal one — because + // a buffer may contain multiple messages, so offset is global and helper_'s + // internal cursor is reset for each individual message. helper_.reset(); // Check message length. From 6e0f226467f8caed1180c67898c0a53d26e5f200 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Mar 2019 17:25:28 -0700 Subject: [PATCH 105/114] Test all possible create flags Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 59 +++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 32e59205b15aa..686c1420d74ea 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -189,8 +189,8 @@ class ZooKeeperFilterTest : public testing::Test { } Buffer::OwnedImpl - encodeCreateRequest(const std::string& path, const std::string& data, const bool ephemeral, - const bool sequence, const bool txn = false, + encodeCreateRequest(const std::string& path, const std::string& data, const CreateFlags flags, + const bool txn = false, const int32_t opcode = enumToIntSigned(OpCodes::CREATE)) const { Buffer::OwnedImpl buffer; @@ -207,14 +207,7 @@ class ZooKeeperFilterTest : public testing::Test { // Acls. buffer.writeBEInt(0); // Flags. - int flags = 0; - if (ephemeral) { - flags &= 0x1; - } - if (sequence) { - flags &= 0x2; - } - buffer.writeBEInt(flags); + buffer.writeBEInt(static_cast(flags)); return buffer; } @@ -381,6 +374,20 @@ class ZooKeeperFilterTest : public testing::Test { })); } + void testCreate(CreateFlags flags) { + initialize(); + Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", flags); + + expectSetDynamicMetadata( + {{"opname", "create"}, {"path", "/foo"}, {"create_type", createFlagsToString(flags)}}, + {{"bytes", "35"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, config_->stats().create_rq_.value()); + EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -494,25 +501,31 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } -TEST_F(ZooKeeperFilterTest, CreateRequest) { - initialize(); +TEST_F(ZooKeeperFilterTest, CreateRequestPersistent) { testCreate(CreateFlags::PERSISTENT); } - Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", false, false); +TEST_F(ZooKeeperFilterTest, CreateRequestPersistentSequential) { + testCreate(CreateFlags::PERSISTENT_SEQUENTIAL); +} - expectSetDynamicMetadata({{"opname", "create"}, {"path", "/foo"}, {"create_type", "persistent"}}, - {{"bytes", "35"}}); +TEST_F(ZooKeeperFilterTest, CreateRequestEphemeral) { testCreate(CreateFlags::EPHEMERAL); } - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().create_rq_.value()); - EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +TEST_F(ZooKeeperFilterTest, CreateRequestEphemeralSequential) { + testCreate(CreateFlags::EPHEMERAL_SEQUENTIAL); +} + +TEST_F(ZooKeeperFilterTest, CreateRequestContainer) { testCreate(CreateFlags::CONTAINER); } + +TEST_F(ZooKeeperFilterTest, CreateRequestTTL) { testCreate(CreateFlags::PERSISTENT_WITH_TTL); } + +TEST_F(ZooKeeperFilterTest, CreateRequestTTLSequential) { + testCreate(CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL); } TEST_F(ZooKeeperFilterTest, CreateRequest2) { initialize(); - Buffer::OwnedImpl data = - encodeCreateRequest("/foo", "bar", false, false, false, enumToIntSigned(OpCodes::CREATE2)); + Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", CreateFlags::PERSISTENT, false, + enumToIntSigned(OpCodes::CREATE2)); expectSetDynamicMetadata({{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, {{"bytes", "35"}}); @@ -675,8 +688,8 @@ TEST_F(ZooKeeperFilterTest, CheckRequest) { TEST_F(ZooKeeperFilterTest, MultiRequest) { initialize(); - Buffer::OwnedImpl create1 = encodeCreateRequest("/foo", "1", false, false, true); - Buffer::OwnedImpl create2 = encodeCreateRequest("/bar", "1", false, false, true); + Buffer::OwnedImpl create1 = encodeCreateRequest("/foo", "1", CreateFlags::PERSISTENT, true); + Buffer::OwnedImpl create2 = encodeCreateRequest("/bar", "1", CreateFlags::PERSISTENT, true); Buffer::OwnedImpl check1 = encodePathVersion("/foo", 100, enumToIntSigned(OpCodes::CHECK), true); Buffer::OwnedImpl set1 = encodeSetRequest("/bar", "2", -1, true); From a80279da0bd4c16542e1cfad7537a06c6ccc7745 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Mar 2019 19:45:12 -0700 Subject: [PATCH 106/114] Improve coverage fo ZooKeeperFilter::onCreateRequest Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 686c1420d74ea..b20c9744cfb67 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -374,16 +374,39 @@ class ZooKeeperFilterTest : public testing::Test { })); } - void testCreate(CreateFlags flags) { + void testCreate(CreateFlags flags, const OpCodes opcode = OpCodes::CREATE) { initialize(); - Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", flags); + Buffer::OwnedImpl data = + encodeCreateRequest("/foo", "bar", flags, false, enumToIntSigned(opcode)); + std::string opname = "create"; + + switch (opcode) { + case OpCodes::CREATECONTAINER: + opname = "createcontainer"; + break; + case OpCodes::CREATETTL: + opname = "createttl"; + break; + } expectSetDynamicMetadata( - {{"opname", "create"}, {"path", "/foo"}, {"create_type", createFlagsToString(flags)}}, + {{"opname", opname}, {"path", "/foo"}, {"create_type", createFlagsToString(flags)}}, {{"bytes", "35"}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().create_rq_.value()); + + switch (opcode) { + case OpCodes::CREATE: + EXPECT_EQ(1UL, config_->stats().create_rq_.value()); + break; + case OpCodes::CREATECONTAINER: + EXPECT_EQ(1UL, config_->stats().createcontainer_rq_.value()); + break; + case OpCodes::CREATETTL: + EXPECT_EQ(1UL, config_->stats().createttl_rq_.value()); + break; + } + EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -513,9 +536,13 @@ TEST_F(ZooKeeperFilterTest, CreateRequestEphemeralSequential) { testCreate(CreateFlags::EPHEMERAL_SEQUENTIAL); } -TEST_F(ZooKeeperFilterTest, CreateRequestContainer) { testCreate(CreateFlags::CONTAINER); } +TEST_F(ZooKeeperFilterTest, CreateRequestContainer) { + testCreate(CreateFlags::CONTAINER, OpCodes::CREATECONTAINER); +} -TEST_F(ZooKeeperFilterTest, CreateRequestTTL) { testCreate(CreateFlags::PERSISTENT_WITH_TTL); } +TEST_F(ZooKeeperFilterTest, CreateRequestTTL) { + testCreate(CreateFlags::PERSISTENT_WITH_TTL, OpCodes::CREATETTL); +} TEST_F(ZooKeeperFilterTest, CreateRequestTTLSequential) { testCreate(CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL); From 67227d71e3911d7d2503850bb91c3d001a41b027 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Mar 2019 20:03:25 -0700 Subject: [PATCH 107/114] Add test for empty string as path This is valid in the client protocol, so lets handle that. Also, this improves coverage for peekString(), since it tests the branch for zero-length strings. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index b20c9744cfb67..8eb28a80e766d 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -387,6 +387,8 @@ class ZooKeeperFilterTest : public testing::Test { case OpCodes::CREATETTL: opname = "createttl"; break; + default: + break; } expectSetDynamicMetadata( @@ -405,6 +407,8 @@ class ZooKeeperFilterTest : public testing::Test { case OpCodes::CREATETTL: EXPECT_EQ(1UL, config_->stats().createttl_rq_.value()); break; + default: + break; } EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); @@ -524,6 +528,22 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { + initialize(); + + // It's valid in ZK to send an empty string as the path, which + // gets treated as /. + Buffer::OwnedImpl data = encodePathWatch("", true); + + expectSetDynamicMetadata({{"opname", "getdata"}, {"path", ""}, {"watch", "true"}}, + {{"bytes", "17"}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(17UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, CreateRequestPersistent) { testCreate(CreateFlags::PERSISTENT); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistentSequential) { From 2533893dba7d1833c546ee83488494533455de5a Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Tue, 19 Mar 2019 20:09:27 -0700 Subject: [PATCH 108/114] Fix comment Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_filter_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 8eb28a80e766d..c295e1f6b45be 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -531,8 +531,8 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { initialize(); - // It's valid in ZK to send an empty string as the path, which - // gets treated as /. + // It's valid to see an empty string as the path, which gets treated as / + // by the server. Buffer::OwnedImpl data = encodePathWatch("", true); expectSetDynamicMetadata({{"opname", "getdata"}, {"path", ""}, {"watch", "true"}}, From f18cbfe355742278fd0a7048acbf481e887c647c Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 20 Mar 2019 06:22:27 -0700 Subject: [PATCH 109/114] DRY things up a bit Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_decoder.h | 2 +- .../zookeeper_proxy/zookeeper_filter.cc | 35 ++++++++++--------- .../zookeeper_proxy/zookeeper_filter.h | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h index 638b5766680a6..62144ef91006f 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_decoder.h @@ -78,7 +78,7 @@ class DecoderCallbacks { virtual void onGetDataRequest(const std::string& path, bool watch) PURE; virtual void onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) PURE; virtual void onSetRequest(const std::string& path) PURE; - virtual void onGetChildrenRequest(const std::string& path, bool watch, bool two) PURE; + virtual void onGetChildrenRequest(const std::string& path, bool watch, bool v2) PURE; virtual void onGetEphemeralsRequest(const std::string& path) PURE; virtual void onGetAllChildrenNumberRequest(const std::string& path) PURE; virtual void onDeleteRequest(const std::string& path, int32_t version) PURE; diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index f4940da40c274..843c3eb98cbb1 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -119,32 +119,32 @@ void ZooKeeperFilter::onGetDataRequest(const std::string& path, const bool watch void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags flags, const OpCodes opcode) { + std::string opname; + switch (opcode) { case OpCodes::CREATE: + opname = "create"; config_->stats_.create_rq_.inc(); - setDynamicMetadata( - {{"opname", "create"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); break; case OpCodes::CREATE2: + opname = "create2"; config_->stats_.create2_rq_.inc(); - setDynamicMetadata( - {{"opname", "create2"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); break; case OpCodes::CREATECONTAINER: + opname = "createcontainer"; config_->stats_.createcontainer_rq_.inc(); - setDynamicMetadata({{"opname", "createcontainer"}, - {"path", path}, - {"create_type", createFlagsToString(flags)}}); break; case OpCodes::CREATETTL: + opname = "createttl"; config_->stats_.createttl_rq_.inc(); - setDynamicMetadata( - {{"opname", "createttl"}, {"path", path}, {"create_type", createFlagsToString(flags)}}); break; default: throw EnvoyException(fmt::format("Unknown opcode: {}", enumToIntSigned(opcode))); break; } + + setDynamicMetadata( + {{"opname", opname}, {"path", path}, {"create_type", createFlagsToString(flags)}}); } void ZooKeeperFilter::onSetRequest(const std::string& path) { @@ -153,16 +153,17 @@ void ZooKeeperFilter::onSetRequest(const std::string& path) { } void ZooKeeperFilter::onGetChildrenRequest(const std::string& path, const bool watch, - const bool two) { - if (!two) { - config_->stats_.getchildren_rq_.inc(); - setDynamicMetadata( - {{"opname", "getchildren"}, {"path", path}, {"watch", watch ? "true" : "false"}}); - } else { + const bool v2) { + std::string opname = "getchildren"; + + if (v2) { config_->stats_.getchildren2_rq_.inc(); - setDynamicMetadata( - {{"opname", "getchildren2"}, {"path", path}, {"watch", watch ? "true" : "false"}}); + opname = "getchildren2"; + } else { + config_->stats_.getchildren_rq_.inc(); } + + setDynamicMetadata({{"opname", opname}, {"path", path}, {"watch", watch ? "true" : "false"}}); } void ZooKeeperFilter::onDeleteRequest(const std::string& path, const int32_t version) { diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index 87892cd24420c..af5985d9da4fd 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -107,7 +107,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab void onGetDataRequest(const std::string& path, bool watch) override; void onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) override; void onSetRequest(const std::string& path) override; - void onGetChildrenRequest(const std::string& path, bool watch, bool two) override; + void onGetChildrenRequest(const std::string& path, bool watch, bool v2) override; void onDeleteRequest(const std::string& path, int32_t version) override; void onExistsRequest(const std::string& path, bool watch) override; void onGetAclRequest(const std::string& path) override; From 0a1e14f4d41bd93b8c8541baacfa5465622f0129 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 20 Mar 2019 08:30:49 -0700 Subject: [PATCH 110/114] Tidy up ZooKeeperFilter::doDecode * the try/catch block was redundant (happens in the decoder already) * move clear dynamic metadata into its own method Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter.cc | 22 +++++++++---------- .../zookeeper_proxy/zookeeper_filter.h | 1 + 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 843c3eb98cbb1..9a5729be05c20 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -37,23 +37,13 @@ Network::FilterStatus ZooKeeperFilter::onWrite(Buffer::Instance&, bool) { Network::FilterStatus ZooKeeperFilter::onNewConnection() { return Network::FilterStatus::Continue; } void ZooKeeperFilter::doDecode(Buffer::Instance& buffer) { - // Clear dynamic metadata. - envoy::api::v2::core::Metadata& dynamic_metadata = - read_callbacks_->connection().streamInfo().dynamicMetadata(); - auto& metadata = - (*dynamic_metadata.mutable_filter_metadata())[NetworkFilterNames::get().ZooKeeperProxy]; - metadata.mutable_fields()->clear(); + clearDynamicMetadata(); if (!decoder_) { decoder_ = createDecoder(*this); } - try { - decoder_->onData(buffer); - } catch (EnvoyException& e) { - ENVOY_LOG(info, "zookeeper_proxy: decoding error: {}", e.what()); - config_->stats_.decoder_error_.inc(); - } + decoder_->onData(buffer); } DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks) { @@ -64,6 +54,14 @@ void ZooKeeperFilter::setDynamicMetadata(const std::string& key, const std::stri setDynamicMetadata({{key, value}}); } +void ZooKeeperFilter::clearDynamicMetadata() { + envoy::api::v2::core::Metadata& dynamic_metadata = + read_callbacks_->connection().streamInfo().dynamicMetadata(); + auto& metadata = + (*dynamic_metadata.mutable_filter_metadata())[NetworkFilterNames::get().ZooKeeperProxy]; + metadata.mutable_fields()->clear(); +} + void ZooKeeperFilter::setDynamicMetadata( const std::vector>& data) { envoy::api::v2::core::Metadata& dynamic_metadata = diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h index af5985d9da4fd..20cdfec0a8f40 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.h @@ -127,6 +127,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggab DecoderPtr createDecoder(DecoderCallbacks& callbacks); void setDynamicMetadata(const std::string& key, const std::string& value); void setDynamicMetadata(const std::vector>& data); + void clearDynamicMetadata(); private: Network::ReadFilterCallbacks* read_callbacks_{}; From d68979403c16e8c9ee2d471a615c6071ae9a9e48 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 20 Mar 2019 08:39:01 -0700 Subject: [PATCH 111/114] Add missing coverage for BufferHelper::peekString Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy/zookeeper_filter_test.cc | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index c295e1f6b45be..4dca3ceb516a5 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -188,6 +188,21 @@ class ZooKeeperFilterTest : public testing::Test { return buffer; } + Buffer::OwnedImpl encodePathLongerThanBuffer(const std::string& path, + const int32_t opcode) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(8 + path.length()); + buffer.writeBEInt(1000); + // Opcode. + buffer.writeBEInt(opcode); + // Path. + buffer.writeBEInt(str.length() * 2); + buffer.add(str); + + return buffer; + } + Buffer::OwnedImpl encodeCreateRequest(const std::string& path, const std::string& data, const CreateFlags flags, const bool txn = false, @@ -488,6 +503,15 @@ TEST_F(ZooKeeperFilterTest, UnknownOpcode) { EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); } +TEST_F(ZooKeeperFilterTest, BufferSmallerThanStringLength) { + initialize(); + + Buffer::OwnedImpl data = encodePathLongerThanBuffer("/foo", enumToIntSigned(OpCodes::SYNC)); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); +} + TEST_F(ZooKeeperFilterTest, PingRequest) { initialize(); From bda8df1f640f51edcf56b21015e7ce7e440b0cc4 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Wed, 20 Mar 2019 09:05:34 -0700 Subject: [PATCH 112/114] Fix Signed-off-by: Raul Gutierrez Segales --- .../network/zookeeper_proxy/zookeeper_filter_test.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 4dca3ceb516a5..339c6c8e94060 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -194,11 +194,9 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(8 + path.length()); buffer.writeBEInt(1000); - // Opcode. buffer.writeBEInt(opcode); - // Path. - buffer.writeBEInt(str.length() * 2); - buffer.add(str); + buffer.writeBEInt(path.length() * 2); + buffer.add(path); return buffer; } From 3f3f46617eb519394c971ea65b2e997e0c5129da Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Fri, 22 Mar 2019 15:09:58 -0700 Subject: [PATCH 113/114] More constness Signed-off-by: Raul Gutierrez Segales --- .../filters/network/zookeeper_proxy/zookeeper_utils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc index 15af93b87c874..1a4ad1c7af4d2 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_utils.cc @@ -24,8 +24,8 @@ int64_t BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset) { bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { ensureMaxLen(1); - char byte = buffer.peekInt(offset); - bool val = static_cast(byte); + const char byte = buffer.peekInt(offset); + const bool val = static_cast(byte); offset += 1; return val; } From 3ec7cb4abdc75b680fffeb431cee225486775a30 Mon Sep 17 00:00:00 2001 From: Raul Gutierrez Segales Date: Mon, 25 Mar 2019 08:28:48 -0700 Subject: [PATCH 114/114] Rename enumToIntSigned -> enumToSignedInt Signed-off-by: Raul Gutierrez Segales --- source/common/common/enum_to_int.h | 2 +- .../zookeeper_proxy/zookeeper_filter.cc | 2 +- .../zookeeper_proxy/zookeeper_filter_test.cc | 70 +++++++++---------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/source/common/common/enum_to_int.h b/source/common/common/enum_to_int.h index 57688c9390bca..ec613ef67df9b 100644 --- a/source/common/common/enum_to_int.h +++ b/source/common/common/enum_to_int.h @@ -11,5 +11,5 @@ template constexpr uint32_t enumToInt(T val) { return static_cast constexpr int32_t enumToIntSigned(T val) { return static_cast(val); } +template constexpr int32_t enumToSignedInt(T val) { return static_cast(val); } } // namespace Envoy diff --git a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc index 9a5729be05c20..ac78aad9c7b71 100644 --- a/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/zookeeper_filter.cc @@ -137,7 +137,7 @@ void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags config_->stats_.createttl_rq_.inc(); break; default: - throw EnvoyException(fmt::format("Unknown opcode: {}", enumToIntSigned(opcode))); + throw EnvoyException(fmt::format("Unknown opcode: {}", enumToSignedInt(opcode))); break; } diff --git a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc index 339c6c8e94060..182ea1bba5700 100644 --- a/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/zookeeper_filter_test.cc @@ -84,7 +84,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(50); buffer.writeBEInt(1000); // Opcode. - buffer.writeBEInt(enumToIntSigned(OpCodes::DELETE)); + buffer.writeBEInt(enumToSignedInt(OpCodes::DELETE)); // Path. addString(buffer, std::string(2 * 1024 * 1024, '*')); // Version. @@ -97,7 +97,7 @@ class ZooKeeperFilterTest : public testing::Test { Buffer::OwnedImpl buffer; buffer.writeBEInt(8); - buffer.writeBEInt(enumToIntSigned(XidCodes::PING_XID)); + buffer.writeBEInt(enumToSignedInt(XidCodes::PING_XID)); buffer.writeBEInt(enumToInt(OpCodes::PING)); return buffer; @@ -118,7 +118,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(8); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToIntSigned(OpCodes::CLOSE)); + buffer.writeBEInt(enumToSignedInt(OpCodes::CLOSE)); return buffer; } @@ -128,8 +128,8 @@ class ZooKeeperFilterTest : public testing::Test { Buffer::OwnedImpl buffer; buffer.writeBEInt(28 + scheme.length() + credential.length()); - buffer.writeBEInt(enumToIntSigned(XidCodes::AUTH_XID)); - buffer.writeBEInt(enumToIntSigned(OpCodes::SETAUTH)); + buffer.writeBEInt(enumToSignedInt(XidCodes::AUTH_XID)); + buffer.writeBEInt(enumToSignedInt(OpCodes::SETAUTH)); // Type. buffer.writeBEInt(0); addString(buffer, scheme); @@ -140,7 +140,7 @@ class ZooKeeperFilterTest : public testing::Test { Buffer::OwnedImpl encodePathWatch(const std::string& path, const bool watch, - const int32_t opcode = enumToIntSigned(OpCodes::GETDATA)) const { + const int32_t opcode = enumToSignedInt(OpCodes::GETDATA)) const { Buffer::OwnedImpl buffer; buffer.writeBEInt(13 + path.length()); @@ -157,7 +157,7 @@ class ZooKeeperFilterTest : public testing::Test { } Buffer::OwnedImpl encodePathVersion(const std::string& path, const int32_t version, - const int32_t opcode = enumToIntSigned(OpCodes::GETDATA), + const int32_t opcode = enumToSignedInt(OpCodes::GETDATA), const bool txn = false) const { Buffer::OwnedImpl buffer; @@ -204,7 +204,7 @@ class ZooKeeperFilterTest : public testing::Test { Buffer::OwnedImpl encodeCreateRequest(const std::string& path, const std::string& data, const CreateFlags flags, const bool txn = false, - const int32_t opcode = enumToIntSigned(OpCodes::CREATE)) const { + const int32_t opcode = enumToSignedInt(OpCodes::CREATE)) const { Buffer::OwnedImpl buffer; if (!txn) { @@ -232,7 +232,7 @@ class ZooKeeperFilterTest : public testing::Test { if (!txn) { buffer.writeBEInt(20 + path.length() + data.length()); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToIntSigned(OpCodes::SETDATA)); + buffer.writeBEInt(enumToSignedInt(OpCodes::SETDATA)); } // Path. @@ -251,7 +251,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(16 + path.length()); buffer.writeBEInt(1000); // Opcode. - buffer.writeBEInt(enumToIntSigned(OpCodes::DELETE)); + buffer.writeBEInt(enumToSignedInt(OpCodes::DELETE)); // Path. addString(buffer, path); // Version. @@ -268,7 +268,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(32 + path.length() + scheme.length() + credential.length()); buffer.writeBEInt(1000); // Opcode. - buffer.writeBEInt(enumToIntSigned(OpCodes::SETACL)); + buffer.writeBEInt(enumToSignedInt(OpCodes::SETACL)); // Path. addString(buffer, path); @@ -293,7 +293,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(28 + joining.length() + leaving.length() + new_members.length()); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToIntSigned(OpCodes::RECONFIG)); + buffer.writeBEInt(enumToSignedInt(OpCodes::RECONFIG)); addString(buffer, joining); addString(buffer, leaving); addString(buffer, new_members); @@ -315,7 +315,7 @@ class ZooKeeperFilterTest : public testing::Test { buffer.writeBEInt(8 + watches_buffer.length()); buffer.writeBEInt(xid); - buffer.writeBEInt(enumToIntSigned(OpCodes::SETWATCHES)); + buffer.writeBEInt(enumToSignedInt(OpCodes::SETWATCHES)); buffer.add(watches_buffer); return buffer; @@ -344,7 +344,7 @@ class ZooKeeperFilterTest : public testing::Test { // Multi prefix. buffer.writeBEInt(8 + requests.length()); buffer.writeBEInt(1000); - buffer.writeBEInt(enumToIntSigned(OpCodes::MULTI)); + buffer.writeBEInt(enumToSignedInt(OpCodes::MULTI)); // Requests. buffer.add(requests); @@ -390,7 +390,7 @@ class ZooKeeperFilterTest : public testing::Test { void testCreate(CreateFlags flags, const OpCodes opcode = OpCodes::CREATE) { initialize(); Buffer::OwnedImpl data = - encodeCreateRequest("/foo", "bar", flags, false, enumToIntSigned(opcode)); + encodeCreateRequest("/foo", "bar", flags, false, enumToSignedInt(opcode)); std::string opname = "create"; switch (opcode) { @@ -504,7 +504,7 @@ TEST_F(ZooKeeperFilterTest, UnknownOpcode) { TEST_F(ZooKeeperFilterTest, BufferSmallerThanStringLength) { initialize(); - Buffer::OwnedImpl data = encodePathLongerThanBuffer("/foo", enumToIntSigned(OpCodes::SYNC)); + Buffer::OwnedImpl data = encodePathLongerThanBuffer("/foo", enumToSignedInt(OpCodes::SYNC)); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); @@ -594,7 +594,7 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { initialize(); Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", CreateFlags::PERSISTENT, false, - enumToIntSigned(OpCodes::CREATE2)); + enumToSignedInt(OpCodes::CREATE2)); expectSetDynamicMetadata({{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, {{"bytes", "35"}}); @@ -621,7 +621,7 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { initialize(); - Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToIntSigned(OpCodes::GETCHILDREN)); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToSignedInt(OpCodes::GETCHILDREN)); expectSetDynamicMetadata({{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); @@ -635,7 +635,7 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { initialize(); - Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToIntSigned(OpCodes::GETCHILDREN2)); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToSignedInt(OpCodes::GETCHILDREN2)); expectSetDynamicMetadata({{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); @@ -663,7 +663,7 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { TEST_F(ZooKeeperFilterTest, ExistsRequest) { initialize(); - Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToIntSigned(OpCodes::EXISTS)); + Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToSignedInt(OpCodes::EXISTS)); expectSetDynamicMetadata({{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}); @@ -677,7 +677,7 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { TEST_F(ZooKeeperFilterTest, GetAclRequest) { initialize(); - Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::GETACL)); + Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GETACL)); expectSetDynamicMetadata({{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}); @@ -704,7 +704,7 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { TEST_F(ZooKeeperFilterTest, SyncRequest) { initialize(); - Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::SYNC)); + Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::SYNC)); expectSetDynamicMetadata({{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}); @@ -717,7 +717,7 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { initialize(); - Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::GETEPHEMERALS)); + Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GETEPHEMERALS)); expectSetDynamicMetadata({{"opname", "getephemerals"}, {"path", "/foo"}}, {{"bytes", "20"}}); @@ -730,7 +730,7 @@ TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { initialize(); - Buffer::OwnedImpl data = encodePath("/foo", enumToIntSigned(OpCodes::GETALLCHILDRENNUMBER)); + Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GETALLCHILDRENNUMBER)); expectSetDynamicMetadata({{"opname", "getallchildrennumber"}, {"path", "/foo"}}, {{"bytes", "20"}}); @@ -744,7 +744,7 @@ TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { TEST_F(ZooKeeperFilterTest, CheckRequest) { initialize(); - Buffer::OwnedImpl data = encodePathVersion("/foo", 100, enumToIntSigned(OpCodes::CHECK)); + Buffer::OwnedImpl data = encodePathVersion("/foo", 100, enumToSignedInt(OpCodes::CHECK)); expectSetDynamicMetadata({{"bytes", "24"}}); @@ -759,14 +759,14 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { Buffer::OwnedImpl create1 = encodeCreateRequest("/foo", "1", CreateFlags::PERSISTENT, true); Buffer::OwnedImpl create2 = encodeCreateRequest("/bar", "1", CreateFlags::PERSISTENT, true); - Buffer::OwnedImpl check1 = encodePathVersion("/foo", 100, enumToIntSigned(OpCodes::CHECK), true); + Buffer::OwnedImpl check1 = encodePathVersion("/foo", 100, enumToSignedInt(OpCodes::CHECK), true); Buffer::OwnedImpl set1 = encodeSetRequest("/bar", "2", -1, true); std::vector> ops; - ops.push_back(std::make_pair(enumToIntSigned(OpCodes::CREATE), std::move(create1))); - ops.push_back(std::make_pair(enumToIntSigned(OpCodes::CREATE), std::move(create2))); - ops.push_back(std::make_pair(enumToIntSigned(OpCodes::CHECK), std::move(check1))); - ops.push_back(std::make_pair(enumToIntSigned(OpCodes::SETDATA), std::move(set1))); + ops.push_back(std::make_pair(enumToSignedInt(OpCodes::CREATE), std::move(create1))); + ops.push_back(std::make_pair(enumToSignedInt(OpCodes::CREATE), std::move(create2))); + ops.push_back(std::make_pair(enumToSignedInt(OpCodes::CHECK), std::move(check1))); + ops.push_back(std::make_pair(enumToSignedInt(OpCodes::SETDATA), std::move(set1))); Buffer::OwnedImpl data = encodeMultiRequest(ops); @@ -800,7 +800,7 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { const std::vector childw = {"/foo2", "/bar2"}; Buffer::OwnedImpl data = - encodeSetWatchesRequest(dataw, existw, childw, enumToIntSigned(XidCodes::SET_WATCHES_XID)); + encodeSetWatchesRequest(dataw, existw, childw, enumToSignedInt(XidCodes::SET_WATCHES_XID)); expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); @@ -830,8 +830,8 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { initialize(); - Buffer::OwnedImpl data = encodePathVersion("/foo", enumToIntSigned(WatcherType::CHILDREN), - enumToIntSigned(OpCodes::CHECKWATCHES)); + Buffer::OwnedImpl data = encodePathVersion("/foo", enumToSignedInt(WatcherType::CHILDREN), + enumToSignedInt(OpCodes::CHECKWATCHES)); expectSetDynamicMetadata({{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); @@ -844,8 +844,8 @@ TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { initialize(); - Buffer::OwnedImpl data = encodePathVersion("/foo", enumToIntSigned(WatcherType::DATA), - enumToIntSigned(OpCodes::REMOVEWATCHES)); + Buffer::OwnedImpl data = encodePathVersion("/foo", enumToSignedInt(WatcherType::DATA), + enumToSignedInt(OpCodes::REMOVEWATCHES)); expectSetDynamicMetadata({{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}});