stats: convert tag extractor regexs to Re2

source/common/config/well_known_names.cc

@@ -1,8 +1,31 @@
 #include "common/config/well_known_names.h"
 
+#include "absl/strings/str_replace.h"
+
 namespace Envoy {
 namespace Config {
 
+namespace {
+
+// To allow for more readable regular expressions to be declared below, and to
+// reduce duplication, define a few common pattern substitutions for regex
+// segments.
+std::string expandRegex(const std::string& regex) {
+  return absl::StrReplaceAll(
+      regex, {// Regex to look for either IPv4 or IPv6 addresses plus port number after underscore.
+              {"<ADDRESS>", R"((?:(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|\[[a-fA-F_\d]+\])_\d+))"},
+              // Cipher names can contain alphanumerics with dashes and
+              // underscores.
+              {"<CIPHER>", R"([\w-]+)"},
+              // A generic name can contain any character except dots.
+              {"<NAME>", R"([^\.]+)"},
+              // Route names may contain dots in addition to alphanumerics and
+              // dashes with underscores.
+              {"<ROUTE_CONFIG_NAME>", R"([\w-\.]+)"}});
+}
+
+} // namespace
+
 TagNameValues::TagNameValues() {
   // Note: the default regexes are defined below in the order that they will typically be matched
   // (see the TagExtractor class definition for an explanation of the iterative matching process).
@@ -24,107 +47,101 @@ TagNameValues::TagNameValues() {
   // - Typical * notation will be used to denote an arbitrary set of characters.
 
   // *_rq(_<response_code>)
-  addRegex(RESPONSE_CODE, "_rq(_(\\d{3}))$", "_rq_");
+  addRe2(RESPONSE_CODE, R"(_rq(_(\d{3}))$)", "_rq_");
 
   // *_rq_(<response_code_class>)xx
-  addRegex(RESPONSE_CODE_CLASS, "_rq_(\\d)xx$", "_rq_");
+  addRe2(RESPONSE_CODE_CLASS, R"(_rq_((\d))xx$)", "_rq_");
 
   // http.[<stat_prefix>.]dynamodb.table.[<table_name>.]capacity.[<operation_name>.](__partition_id=<last_seven_characters_from_partition_id>)
-  addRegex(DYNAMO_PARTITION_ID,
-           "^http(?=\\.).*?\\.dynamodb\\.table(?=\\.).*?\\."
-           "capacity(?=\\.).*?(\\.__partition_id=(\\w{7}))$",
-           ".dynamodb.table.");
+  addRe2(DYNAMO_PARTITION_ID,
+         R"(^http\.<NAME>\.dynamodb\.table\.<NAME>\.capacity\.<NAME>(\.__partition_id=(\w{7}))$)",
+         ".dynamodb.table.");
 
-  // http.[<stat_prefix>.]dynamodb.operation.(<operation_name>.)<base_stat> or
+  // http.[<stat_prefix>.]dynamodb.operation.(<operation_name>.)* or
   // http.[<stat_prefix>.]dynamodb.table.[<table_name>.]capacity.(<operation_name>.)[<partition_id>]
-  addRegex(DYNAMO_OPERATION,
-           "^http(?=\\.).*?\\.dynamodb.(?:operation|table(?="
-           "\\.).*?\\.capacity)(\\.(.*?))(?:\\.|$)",
-           ".dynamodb.");
+  addRe2(DYNAMO_OPERATION,
+         R"(^http\.<NAME>\.dynamodb.(?:operation|table\.<NAME>\.capacity)(\.(<NAME>))(?:\.|$))",
+         ".dynamodb.");
 
-  // mongo.[<stat_prefix>.]collection.[<collection>.]callsite.(<callsite>.)query.<base_stat>
-  addRegex(MONGO_CALLSITE,
-           R"(^mongo(?=\.).*?\.collection(?=\.).*?\.callsite\.((.*?)\.).*?query.\w+?$)",
-           ".collection.");
+  // mongo.[<stat_prefix>.]collection.[<collection>.]callsite.(<callsite>.)query.*
+  addRe2(MONGO_CALLSITE, R"(^mongo\.<NAME>\.collection\.<NAME>\.callsite\.((<NAME>)\.)query\.)",
+         ".collection.");
 
-  // http.[<stat_prefix>.]dynamodb.table.(<table_name>.) or
+  // http.[<stat_prefix>.]dynamodb.table.(<table_name>.)* or
   // http.[<stat_prefix>.]dynamodb.error.(<table_name>.)*
-  addRegex(DYNAMO_TABLE, R"(^http(?=\.).*?\.dynamodb.(?:table|error)\.((.*?)\.))", ".dynamodb.");
+  addRe2(DYNAMO_TABLE, R"(^http\.<NAME>\.dynamodb.(?:table|error)\.((<NAME>)\.))", ".dynamodb.");
 
-  // mongo.[<stat_prefix>.]collection.(<collection>.)query.<base_stat>
-  addRegex(MONGO_COLLECTION, R"(^mongo(?=\.).*?\.collection\.((.*?)\.).*?query.\w+?$)",
-           ".collection.");
+  // mongo.[<stat_prefix>.]collection.(<collection>.)query.*
+  addRe2(MONGO_COLLECTION, R"(^mongo\.<NAME>\.collection\.((<NAME>)\.).*?query\.)", ".collection.");
 
-  // mongo.[<stat_prefix>.]cmd.(<cmd>.)<base_stat>
-  addRegex(MONGO_CMD, R"(^mongo(?=\.).*?\.cmd\.((.*?)\.)\w+?$)", ".cmd.");
+  // mongo.[<stat_prefix>.]cmd.(<cmd>.)*
+  addRe2(MONGO_CMD, R"(^mongo\.<NAME>\.cmd\.((<NAME>)\.))", ".cmd.");
 
-  // cluster.[<route_target_cluster>.]grpc.[<grpc_service>.](<grpc_method>.)<base_stat>
-  addRegex(GRPC_BRIDGE_METHOD, R"(^cluster(?=\.).*?\.grpc(?=\.).*\.((.*?)\.)\w+?$)", ".grpc.");
+  // cluster.[<route_target_cluster>.]grpc.[<grpc_service>.](<grpc_method>.)*
+  addRe2(GRPC_BRIDGE_METHOD, R"(^cluster\.<NAME>\.grpc\.<NAME>\.((<NAME>)\.))", ".grpc.");
 
-  // http.[<stat_prefix>.]user_agent.(<user_agent>.)<base_stat>
-  addRegex(HTTP_USER_AGENT, R"(^http(?=\.).*?\.user_agent\.((.*?)\.)\w+?$)", ".user_agent.");
+  // http.[<stat_prefix>.]user_agent.(<user_agent>.)*
+  addRe2(HTTP_USER_AGENT, R"(^http\.<NAME>\.user_agent\.((<NAME>)\.))", ".user_agent.");
 
-  // vhost.[<virtual host name>.]vcluster.(<virtual_cluster_name>.)<base_stat>
-  addRegex(VIRTUAL_CLUSTER, R"(^vhost(?=\.).*?\.vcluster\.((.*?)\.)\w+?$)", ".vcluster.");
+  // vhost.[<virtual host name>.]vcluster.(<virtual_cluster_name>.)*
+  addRe2(VIRTUAL_CLUSTER, R"(^vhost\.<NAME>\.vcluster\.((<NAME>)\.))", ".vcluster.");
 
-  // http.[<stat_prefix>.]fault.(<downstream_cluster>.)<base_stat>
-  addRegex(FAULT_DOWNSTREAM_CLUSTER, R"(^http(?=\.).*?\.fault\.((.*?)\.)\w+?$)", ".fault.");
+  // http.[<stat_prefix>.]fault.(<downstream_cluster>.)*
+  addRe2(FAULT_DOWNSTREAM_CLUSTER, R"(^http\.<NAME>\.fault\.((<NAME>)\.))", ".fault.");
 
   // listener.[<address>.]ssl.cipher.(<cipher>)
-  addRegex(SSL_CIPHER, R"(^listener(?=\.).*?\.ssl\.cipher(\.(.*?))$)");
+  addRe2(SSL_CIPHER, R"(^listener\..*?\.ssl\.cipher(\.(<CIPHER>))$)");
 
   // cluster.[<cluster_name>.]ssl.ciphers.(<cipher>)
-  addRegex(SSL_CIPHER_SUITE, R"(^cluster(?=\.).*?\.ssl\.ciphers(\.(.*?))$)", ".ssl.ciphers.");
+  addRe2(SSL_CIPHER_SUITE, R"(^cluster\.<NAME>\.ssl\.ciphers(\.(<CIPHER>))$)", ".ssl.ciphers.");
 
   // cluster.[<route_target_cluster>.]grpc.(<grpc_service>.)*
-  addRegex(GRPC_BRIDGE_SERVICE, R"(^cluster(?=\.).*?\.grpc\.((.*?)\.))", ".grpc.");
+  addRe2(GRPC_BRIDGE_SERVICE, R"(^cluster\.<NAME>\.grpc\.((<NAME>)\.))", ".grpc.");
 
-  // tcp.(<stat_prefix>.)<base_stat>
-  addRegex(TCP_PREFIX, R"(^tcp\.((.*?)\.)\w+?$)");
+  // tcp.(<stat_prefix>.)*
+  addRe2(TCP_PREFIX, R"(^tcp\.((<NAME>)\.))");
 
-  // udp.(<stat_prefix>.)<base_stat>
-  addRegex(UDP_PREFIX, R"(^udp\.((.*?)\.)\w+?$)");
+  // udp.(<stat_prefix>.)*
+  addRe2(UDP_PREFIX, R"(^udp\.((<NAME>)\.))");
 
-  // auth.clientssl.(<stat_prefix>.)<base_stat>
-  addRegex(CLIENTSSL_PREFIX, R"(^auth\.clientssl\.((.*?)\.)\w+?$)");
+  // auth.clientssl.(<stat_prefix>.)*
+  addRe2(CLIENTSSL_PREFIX, R"(^auth\.clientssl\.((<NAME>)\.))");
 
-  // ratelimit.(<stat_prefix>.)<base_stat>
-  addRegex(RATELIMIT_PREFIX, R"(^ratelimit\.((.*?)\.)\w+?$)");
+  // ratelimit.(<stat_prefix>.)*
+  addRe2(RATELIMIT_PREFIX, R"(^ratelimit\.((<NAME>)\.))");
 
   // cluster.(<cluster_name>.)*
-  addRe2(CLUSTER_NAME, "^cluster\\.(([^\\.]+)\\.).*");
+  addRe2(CLUSTER_NAME, R"(^cluster\.((<NAME>)\.))");
 
   // listener.[<address>.]http.(<stat_prefix>.)*
-  addRegex(HTTP_CONN_MANAGER_PREFIX, R"(^listener(?=\.).*?\.http\.((.*?)\.))", ".http.");
+  // The <address> part can be anything here (.*?) for the sake of a simpler
+  // internal state of the regex which performs better.
+  addRe2(HTTP_CONN_MANAGER_PREFIX, R"(^listener\..*?\.http\.((<NAME>)\.))", ".http.");
 
   // http.(<stat_prefix>.)*
-  addRegex(HTTP_CONN_MANAGER_PREFIX, "^http\\.((.*?)\\.)");
+  addRe2(HTTP_CONN_MANAGER_PREFIX, R"(^http\.((<NAME>)\.))");
 
   // listener.(<address>.)*
-  addRegex(LISTENER_ADDRESS,
-           R"(^listener\.(((?:[_.[:digit:]]*|[_\[\]aAbBcCdDeEfF[:digit:]]*))\.))");
+  addRe2(LISTENER_ADDRESS, R"(^listener\.((<ADDRESS>)\.))");
 
   // vhost.(<virtual host name>.)*
-  addRegex(VIRTUAL_HOST, "^vhost\\.((.*?)\\.)");
+  addRe2(VIRTUAL_HOST, R"(^vhost\.((<NAME>)\.))");
 
   // mongo.(<stat_prefix>.)*
-  addRegex(MONGO_PREFIX, "^mongo\\.((.*?)\\.)");
+  addRe2(MONGO_PREFIX, R"(^mongo\.((<NAME>)\.))");
 
   // http.[<stat_prefix>.]rds.(<route_config_name>.)<base_stat>
-  addRegex(RDS_ROUTE_CONFIG, R"(^http(?=\.).*?\.rds\.((.*?)\.)\w+?$)", ".rds.");
+  // Note: <route_config_name> can contain dots thus we have to maintain full
+  // match.
+  addRe2(RDS_ROUTE_CONFIG, R"(^http\.<NAME>\.rds\.((<ROUTE_CONFIG_NAME>)\.)\w+?$)", ".rds.");
 
   // listener_manager.(worker_<id>.)*
-  addRegex(WORKER_ID, R"(^listener_manager\.((worker_\d+)\.))", "listener_manager.worker_");
-}
-
-void TagNameValues::addRegex(const std::string& name, const std::string& regex,
-                             const std::string& substr) {
-  descriptor_vec_.emplace_back(Descriptor{name, regex, substr, Regex::Type::StdRegex});
+  addRe2(WORKER_ID, R"(^listener_manager\.((worker_\d+)\.))", "listener_manager.worker_");
 }
 
 void TagNameValues::addRe2(const std::string& name, const std::string& regex,
                            const std::string& substr) {
-  descriptor_vec_.emplace_back(Descriptor{name, regex, substr, Regex::Type::Re2});
+  descriptor_vec_.emplace_back(Descriptor{name, expandRegex(regex), substr, Regex::Type::Re2});
 }
 
 } // namespace Config

source/common/config/well_known_names.h

@@ -129,7 +129,6 @@ class TagNameValues {
   const std::vector<Descriptor>& descriptorVec() const { return descriptor_vec_; }
 
 private:
-  void addRegex(const std::string& name, const std::string& regex, const std::string& substr = "");
   void addRe2(const std::string& name, const std::string& regex, const std::string& substr = "");
 
   // Collection of tag descriptors.

source/common/stats/tag_extractor_impl.cc

@@ -26,7 +26,9 @@ bool regexStartsWithDot(absl::string_view regex) {
 
 TagExtractorImplBase::TagExtractorImplBase(absl::string_view name, absl::string_view regex,
                                            absl::string_view substr)
-    : name_(name), prefix_(std::string(extractRegexPrefix(regex))), substr_(substr) {}
+    : name_(name), prefix_(std::string(extractRegexPrefix(regex))), substr_(substr) {
+  PERF_TAG_INIT;
+}
 
 std::string TagExtractorImplBase::extractRegexPrefix(absl::string_view regex) {
   std::string prefix;
@@ -90,6 +92,7 @@ bool TagExtractorStdRegexImpl::extractTag(absl::string_view stat_name, std::vect
 
   if (substrMismatch(stat_name)) {
     PERF_RECORD(perf, "re-skip", name_);
+    PERF_TAG_INC(skipped_);
     return false;
   }
 
@@ -113,9 +116,11 @@ bool TagExtractorStdRegexImpl::extractTag(absl::string_view stat_name, std::vect
     std::string::size_type end = remove_subexpr.second - stat_name.begin();
     remove_characters.insert(start, end);
     PERF_RECORD(perf, "re-match", name_);
+    PERF_TAG_INC(matched_);
     return true;
   }
   PERF_RECORD(perf, "re-miss", name_);
+  PERF_TAG_INC(missed_);
   return false;
 }
 
@@ -129,15 +134,16 @@ bool TagExtractorRe2Impl::extractTag(absl::string_view stat_name, std::vector<Ta
 
   if (substrMismatch(stat_name)) {
     PERF_RECORD(perf, "re2-skip", name_);
+    PERF_TAG_INC(skipped_);
     return false;
   }
 
   // remove_subexpr is the first submatch. It represents the portion of the string to be removed.
   re2::StringPiece remove_subexpr, value_subexpr;
 
   // The regex must match and contain one or more subexpressions (all after the first are ignored).
-  if (re2::RE2::FullMatch(re2::StringPiece(stat_name.data(), stat_name.size()), regex_,
-                          &remove_subexpr, &value_subexpr) &&
+  if (re2::RE2::PartialMatch(re2::StringPiece(stat_name.data(), stat_name.size()), regex_,
+                             &remove_subexpr, &value_subexpr) &&
       !remove_subexpr.empty()) {
 
     // value_subexpr is the optional second submatch. It is usually inside the first submatch
@@ -155,9 +161,11 @@ bool TagExtractorRe2Impl::extractTag(absl::string_view stat_name, std::vector<Ta
     remove_characters.insert(start, end);
 
     PERF_RECORD(perf, "re2-match", name_);
+    PERF_TAG_INC(matched_);
     return true;
   }
   PERF_RECORD(perf, "re2-miss", name_);
+  PERF_TAG_INC(missed_);
   return false;
 }
 

source/common/stats/tag_extractor_impl.h

@@ -4,6 +4,10 @@
 #include <regex>
 #include <string>
 
+#ifdef ENVOY_PERF_ANNOTATION
+#include <fmt/core.h>
+#endif
+
 #include "envoy/stats/tag_extractor.h"
 
 #include "common/common/regex.h"
@@ -14,6 +18,29 @@
 namespace Envoy {
 namespace Stats {
 
+// To check if a tag extractor is actually used you can run
+// bazel test //test/... --test_output=streamed --define=perf_annotation=enabled
+#ifdef ENVOY_PERF_ANNOTATION
+
+struct Counters {
+  uint32_t skipped_{};
+  uint32_t matched_{};
+  uint32_t missed_{};
+};
+
+#define PERF_TAG_COUNTERS std::unique_ptr<Counters> counters_
+
+#define PERF_TAG_INIT counters_ = std::make_unique<Counters>()
+#define PERF_TAG_INC(member) ++(counters_->member)
+
+#else
+
+#define PERF_TAG_COUNTERS
+#define PERF_TAG_INIT
+#define PERF_TAG_INC(member)
+
+#endif
+
 class TagExtractorImplBase : public TagExtractor {
 public:
   /**
@@ -32,6 +59,13 @@ class TagExtractorImplBase : public TagExtractor {
 
   TagExtractorImplBase(absl::string_view name, absl::string_view regex,
                        absl::string_view substr = "");
+#ifdef ENVOY_PERF_ANNOTATION
+  ~TagExtractorImplBase() override {
+    std::cout << fmt::format("TagStats for {} tag extractor: skipped {}, matched {}, missing {}",
+                             name_, counters_->skipped_, counters_->matched_, counters_->missed_)
+              << std::endl;
+  }
+#endif
   std::string name() const override { return name_; }
   absl::string_view prefixToken() const override { return prefix_; }
 
@@ -62,6 +96,8 @@ class TagExtractorImplBase : public TagExtractor {
   const std::string name_;
   const std::string prefix_;
   const std::string substr_;
+
+  PERF_TAG_COUNTERS;
 };
 
 class TagExtractorStdRegexImpl : public TagExtractorImplBase {

test/common/stats/BUILD

@@ -231,6 +231,25 @@ envoy_cc_test(
     ],
 )
 
+envoy_cc_benchmark_binary(
+    name = "tag_extractor_impl_benchmark",
+    srcs = [
+        "tag_extractor_impl_speed_test.cc",
+    ],
+    external_deps = [
+        "benchmark",
+    ],
+    deps = [
+        "//source/common/stats:tag_producer_lib",
+        "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_benchmark_test(
+    name = "tag_extractor_impl_benchmark_test",
+    benchmark_binary = "tag_extractor_impl_benchmark",
+)
+
 envoy_cc_test(
     name = "thread_local_store_test",
     srcs = ["thread_local_store_test.cc"],