stats: use RE2 and a better pattern to accelerate a single stats tag-extraction RE

source/common/config/well_known_names.cc

@@ -89,7 +89,8 @@ TagNameValues::TagNameValues() {
   addRegex(RATELIMIT_PREFIX, R"(^ratelimit\.((.*?)\.)\w+?$)");
 
   // cluster.(<cluster_name>.)*
-  addRegex(CLUSTER_NAME, "^cluster\\.((.*?)\\.)");
+  // addRegex(CLUSTER_NAME, "^cluster\\.((.*?)\\.)");
+  addRe2(CLUSTER_NAME, "^cluster\\.(([^\\.]+)\\.).*");
 
   // listener.[<address>.]http.(<stat_prefix>.)*
   addRegex(HTTP_CONN_MANAGER_PREFIX, R"(^listener(?=\.).*?\.http\.((.*?)\.))", ".http.");
@@ -116,7 +117,12 @@ TagNameValues::TagNameValues() {
 
 void TagNameValues::addRegex(const std::string& name, const std::string& regex,
                              const std::string& substr) {
-  descriptor_vec_.emplace_back(Descriptor(name, regex, substr));
+  descriptor_vec_.emplace_back(Descriptor{name, regex, substr, false});
+}
+
+void TagNameValues::addRe2(const std::string& name, const std::string& regex,
+                           const std::string& substr) {
+  descriptor_vec_.emplace_back(Descriptor{name, regex, substr, true});
 }
 
 } // namespace Config

source/common/config/well_known_names.h

@@ -63,11 +63,10 @@ class TagNameValues {
    * tags, such as "_rq_(\\d)xx$", will probably stay as regexes.
    */
   struct Descriptor {
-    Descriptor(const std::string& name, const std::string& regex, const std::string& substr = "")
-        : name_(name), regex_(regex), substr_(substr) {}
     const std::string name_;
     const std::string regex_;
     const std::string substr_;
+    bool use_re2_;
   };
 
   // Cluster name tag
@@ -129,6 +128,7 @@ class TagNameValues {
 
 private:
   void addRegex(const std::string& name, const std::string& regex, const std::string& substr = "");
+  void addRe2(const std::string& name, const std::string& regex, const std::string& substr = "");
 
   // Collection of tag descriptors.
   std::vector<Descriptor> descriptor_vec_;

source/common/stats/tag_extractor_impl.cc

@@ -25,8 +25,7 @@ bool regexStartsWithDot(absl::string_view regex) {
 
 TagExtractorImpl::TagExtractorImpl(const std::string& name, const std::string& regex,
                                    const std::string& substr)
-    : name_(name), prefix_(std::string(extractRegexPrefix(regex))), substr_(substr),
-      regex_(Regex::Utility::parseStdRegex(regex)) {}
+    : name_(name), prefix_(std::string(extractRegexPrefix(regex))), substr_(substr) {}
 
 std::string TagExtractorImpl::extractRegexPrefix(absl::string_view regex) {
   std::string prefix;
@@ -49,7 +48,7 @@ std::string TagExtractorImpl::extractRegexPrefix(absl::string_view regex) {
 
 TagExtractorPtr TagExtractorImpl::createTagExtractor(const std::string& name,
                                                      const std::string& regex,
-                                                     const std::string& substr) {
+                                                     const std::string& substr, bool use_re2) {
 
   if (name.empty()) {
     throw EnvoyException("tag_name cannot be empty");
@@ -59,19 +58,26 @@ TagExtractorPtr TagExtractorImpl::createTagExtractor(const std::string& name,
     throw EnvoyException(fmt::format(
         "No regex specified for tag specifier and no default regex for name: '{}'", name));
   }
-  return TagExtractorPtr{new TagExtractorImpl(name, regex, substr)};
+  if (use_re2) {
+    return std::make_unique<TagExtractorRe2Impl>(name, regex, substr);
+  }
+  return std::make_unique<TagExtractorStdRegexImpl>(name, regex, substr);
 }
 
 bool TagExtractorImpl::substrMismatch(absl::string_view stat_name) const {
   return !substr_.empty() && stat_name.find(substr_) == absl::string_view::npos;
 }
 
-bool TagExtractorImpl::extractTag(absl::string_view stat_name, std::vector<Tag>& tags,
-                                  IntervalSet<size_t>& remove_characters) const {
+TagExtractorStdRegexImpl::TagExtractorStdRegexImpl(const std::string& name, const std::string regex,
+                                                   const std::string& substr)
+    : TagExtractorImpl(name, regex, substr), regex_(Regex::Utility::parseStdRegex(regex)) {}
+
+bool TagExtractorStdRegexImpl::extractTag(absl::string_view stat_name, std::vector<Tag>& tags,
+                                          IntervalSet<size_t>& remove_characters) const {
   PERF_OPERATION(perf);
 
   if (substrMismatch(stat_name)) {
-    PERF_RECORD(perf, "re-skip-substr", name_);
+    PERF_RECORD(perf, "re-skip", name_);
     return false;
   }
 
@@ -105,5 +111,50 @@ bool TagExtractorImpl::extractTag(absl::string_view stat_name, std::vector<Tag>&
   return false;
 }
 
+TagExtractorRe2Impl::TagExtractorRe2Impl(const std::string& name, const std::string regex,
+                                         const std::string& substr)
+    : TagExtractorImpl(name, regex, substr), regex_(regex) {}
+
+bool TagExtractorRe2Impl::extractTag(absl::string_view stat_name, std::vector<Tag>& tags,
+                                     IntervalSet<size_t>& remove_characters) const {
+  PERF_OPERATION(perf);
+
+  if (substrMismatch(stat_name)) {
+    PERF_RECORD(perf, "re2-skip", name_);
+    return false;
+  }
+
+  // remove_subexpr is the first submatch. It represents the portion of the string to be removed.
+  re2::StringPiece remove_subexpr, value_subexpr;
+
+  // The regex must match and contain one or more subexpressions (all after the first are ignored).
+  if (re2::RE2::FullMatch(re2::StringPiece(stat_name.data(), stat_name.size()), regex_,
+                          &remove_subexpr, &value_subexpr) &&
+      !remove_subexpr.empty()) {
+
+    // value_subexpr is the optional second submatch. It is usually inside the first submatch
+    // (remove_subexpr) to allow the expression to strip off extra characters that should be removed
+    // from the string but also not necessary in the tag value ("." for example). If there is no
+    // second submatch, then the value_subexpr is the same as the remove_subexpr.
+    if (value_subexpr.empty()) {
+      value_subexpr = remove_subexpr;
+    }
+
+    tags.emplace_back();
+    Tag& tag = tags.back();
+    tag.name_ = name_;
+    tag.value_ = std::string(value_subexpr);
+
+    // Determines which characters to remove from stat_name to elide remove_subexpr.
+    std::string::size_type start = remove_subexpr.data() - stat_name.data();
+    std::string::size_type end = remove_subexpr.data() + remove_subexpr.size() - stat_name.data();
+    remove_characters.insert(start, end);
+    PERF_RECORD(perf, "re2-match", name_);
+    return true;
+  }
+  PERF_RECORD(perf, "re2-miss", name_);
+  return false;
+}
+
 } // namespace Stats
 } // namespace Envoy

source/common/stats/tag_extractor_impl.h

@@ -7,6 +7,7 @@
 #include "envoy/stats/tag_extractor.h"
 
 #include "absl/strings/string_view.h"
+#include "re2/re2.h"
 
 namespace Envoy {
 namespace Stats {
@@ -23,13 +24,11 @@ class TagExtractorImpl : public TagExtractor {
    * @return TagExtractorPtr newly constructed TagExtractor.
    */
   static TagExtractorPtr createTagExtractor(const std::string& name, const std::string& regex,
-                                            const std::string& substr = "");
+                                            const std::string& substr = "", bool is_re2 = false);
 
   TagExtractorImpl(const std::string& name, const std::string& regex,
                    const std::string& substr = "");
   std::string name() const override { return name_; }
-  bool extractTag(absl::string_view tag_extracted_name, std::vector<Tag>& tags,
-                  IntervalSet<size_t>& remove_characters) const override;
   absl::string_view prefixToken() const override { return prefix_; }
 
   /**
@@ -39,7 +38,7 @@ class TagExtractorImpl : public TagExtractor {
    */
   bool substrMismatch(absl::string_view stat_name) const;
 
-private:
+protected:
   /**
    * Examines a regex string, looking for the pattern: ^alphanumerics_with_underscores\.
    * Returns "alphanumerics_with_underscores" if that pattern is found, empty-string otherwise.
@@ -50,8 +49,31 @@ class TagExtractorImpl : public TagExtractor {
   const std::string name_;
   const std::string prefix_;
   const std::string substr_;
+};
+
+class TagExtractorStdRegexImpl : public TagExtractorImpl {
+public:
+  TagExtractorStdRegexImpl(const std::string& name, const std::string regex,
+                           const std::string& substr = "");
+
+  bool extractTag(absl::string_view tag_extracted_name, std::vector<Tag>& tags,
+                  IntervalSet<size_t>& remove_characters) const override;
+
+private:
   const std::regex regex_;
 };
 
+class TagExtractorRe2Impl : public TagExtractorImpl {
+public:
+  TagExtractorRe2Impl(const std::string& name, const std::string regex,
+                      const std::string& substr = "");
+
+  bool extractTag(absl::string_view tag_extracted_name, std::vector<Tag>& tags,
+                  IntervalSet<size_t>& remove_characters) const override;
+
+private:
+  re2::RE2 regex_;
+};
+
 } // namespace Stats
 } // namespace Envoy

source/common/stats/tag_producer_impl.cc

@@ -45,8 +45,8 @@ int TagProducerImpl::addExtractorsMatching(absl::string_view name) {
   int num_found = 0;
   for (const auto& desc : Config::TagNames::get().descriptorVec()) {
     if (desc.name_ == name) {
-      addExtractor(
-          Stats::TagExtractorImpl::createTagExtractor(desc.name_, desc.regex_, desc.substr_));
+      addExtractor(Stats::TagExtractorImpl::createTagExtractor(desc.name_, desc.regex_,
+                                                               desc.substr_, desc.use_re2_));
       ++num_found;
     }
   }
@@ -101,8 +101,8 @@ TagProducerImpl::addDefaultExtractors(const envoy::config::metrics::v2::StatsCon
   if (!config.has_use_all_default_tags() || config.use_all_default_tags().value()) {
     for (const auto& desc : Config::TagNames::get().descriptorVec()) {
       names.emplace(desc.name_);
-      addExtractor(
-          Stats::TagExtractorImpl::createTagExtractor(desc.name_, desc.regex_, desc.substr_));
+      addExtractor(Stats::TagExtractorImpl::createTagExtractor(desc.name_, desc.regex_,
+                                                               desc.substr_, desc.use_re2_));
     }
   }
   return names;

test/common/common/BUILD

@@ -196,6 +196,17 @@ envoy_cc_test(
     deps = ["//source/common/common:callback_impl_lib"],
 )
 
+envoy_cc_binary(
+    name = "re_speed_test",
+    srcs = ["re_speed_test.cc"],
+    external_deps = ["benchmark"],
+    deps = [
+        "//source/common/common:assert_lib",
+        "//source/common/common:utility_lib",
+        "@com_googlesource_code_re2//:re2",
+    ],
+)
+
 envoy_cc_binary(
     name = "utility_speed_test",
     srcs = ["utility_speed_test.cc"],

test/common/common/re_speed_test.cc

@@ -0,0 +1,132 @@
+// Note: this should be run with --compilation_mode=opt, and would benefit from
+// a quiescent system with disabled cstate power management.
+
+#include <regex>
+
+#include "common/common/assert.h"
+
+#include "absl/strings/string_view.h"
+#include "benchmark/benchmark.h"
+#include "re2/re2.h"
+
+// NOLINT(namespace-envoy)
+
+static const char* cluster_inputs[] = {
+    "cluster.no_trailing_dot",
+    "cluster.match.",
+    "cluster.match.normal",
+    "cluster.match.and.a.whole.lot.of.things.coming.after.the.matches.really.too.much.stuff",
+};
+
+static const char cluster_re_pattern[] = "^cluster\\.((.*?)\\.)";
+static const char cluster_re_alt_pattern[] = "^cluster\\.(([^\\.]+)\\.).*";
+
+static void BM_StdRegex(benchmark::State& state) {
+  std::regex re(cluster_re_pattern);
+  uint32_t passes = 0;
+  std::vector<std::string> inputs;
+  for (const char* cluster_input : cluster_inputs) {
+    inputs.push_back(cluster_input);
+  }
+
+  for (auto _ : state) {
+    for (const std::string& cluster_input : inputs) {
+      std::smatch match;
+      if (std::regex_search(cluster_input, match, re)) {
+        ASSERT(match.size() >= 3);
+        ASSERT(match[1] == "match.");
+        ASSERT(match[2] == "match");
+        ++passes;
+      }
+    }
+  }
+  RELEASE_ASSERT(passes > 0, "");
+}
+BENCHMARK(BM_StdRegex);
+
+static void BM_StdRegexStringView(benchmark::State& state) {
+  std::regex re(cluster_re_pattern);
+  std::vector<absl::string_view> inputs;
+  for (const char* cluster_input : cluster_inputs) {
+    inputs.push_back(cluster_input);
+  }
+  uint32_t passes = 0;
+  for (auto _ : state) {
+    for (absl::string_view cluster_input : inputs) {
+      std::match_results<absl::string_view::iterator> smatch;
+      if (std::regex_search(cluster_input.begin(), cluster_input.end(), smatch, re)) {
+        ASSERT(smatch.size() >= 3);
+        ASSERT(smatch[1] == "match.");
+        ASSERT(smatch[2] == "match");
+        ++passes;
+      }
+    }
+  }
+  RELEASE_ASSERT(passes > 0, "");
+}
+BENCHMARK(BM_StdRegexStringView);
+
+static void BM_StdRegexStringViewAltPattern(benchmark::State& state) {
+  std::regex re(cluster_re_alt_pattern);
+  std::vector<absl::string_view> inputs;
+  for (const char* cluster_input : cluster_inputs) {
+    inputs.push_back(cluster_input);
+  }
+  uint32_t passes = 0;
+  for (auto _ : state) {
+    for (absl::string_view cluster_input : inputs) {
+      std::match_results<absl::string_view::iterator> smatch;
+      if (std::regex_search(cluster_input.begin(), cluster_input.end(), smatch, re)) {
+        ASSERT(smatch.size() >= 3);
+        ASSERT(smatch[1] == "match.");
+        ASSERT(smatch[2] == "match");
+        ++passes;
+      }
+    }
+  }
+  RELEASE_ASSERT(passes > 0, "");
+}
+BENCHMARK(BM_StdRegexStringViewAltPattern);
+
+static void BM_RE2(benchmark::State& state) {
+  re2::RE2 re(cluster_re_pattern);
+  uint32_t passes = 0;
+  for (auto _ : state) {
+    for (const char* cluster_input : cluster_inputs) {
+      re2::StringPiece match1, match2;
+      if (re2::RE2::PartialMatch(cluster_input, re, &match1, &match2)) {
+        ASSERT(match1 == "match.");
+        ASSERT(match2 == "match");
+        ++passes;
+      }
+    }
+  }
+  RELEASE_ASSERT(passes > 0, "");
+}
+BENCHMARK(BM_RE2);
+
+static void BM_RE2_AltPattern(benchmark::State& state) {
+  re2::RE2 re(cluster_re_alt_pattern);
+  uint32_t passes = 0;
+  for (auto _ : state) {
+    for (const char* cluster_input : cluster_inputs) {
+      re2::StringPiece match1, match2;
+      if (re2::RE2::PartialMatch(cluster_input, re, &match1, &match2)) {
+        ASSERT(match1 == "match.");
+        ASSERT(match2 == "match");
+        ++passes;
+      }
+    }
+  }
+  RELEASE_ASSERT(passes > 0, "");
+}
+BENCHMARK(BM_RE2_AltPattern);
+
+// Boilerplate main(), which discovers benchmarks in the same file and runs them.
+int main(int argc, char** argv) {
+  benchmark::Initialize(&argc, argv);
+  if (benchmark::ReportUnrecognizedArguments(argc, argv)) {
+    return 1;
+  }
+  benchmark::RunSpecifiedBenchmarks();
+}

test/common/stats/BUILD

@@ -160,6 +160,7 @@ envoy_cc_test(
         "//source/common/stats:tag_extractor_lib",
         "//source/common/stats:tag_producer_lib",
         "//test/test_common:utility_lib",
+        "@com_googlesource_code_re2//:re2",
         "@envoy_api//envoy/config/metrics/v2:pkg_cc_proto",
     ],
 )