Add tags for stats extracted by configurable regexes.

bazel/repositories.bzl

@@ -112,7 +112,7 @@ def envoy_api_deps(skip_targets):
     native.git_repository(
         name = "envoy_api",
         remote = REPO_LOCATIONS["envoy_api"],
-        commit = "9be6aff6da46e024af56cce20cb5d5d3184f19c5",
+        commit = "ccc1ed57bae22673c003ed18ee23a173b1bf6b5c",
     )
     api_bind_targets = [
         "address",

include/envoy/stats/stats.h

@@ -5,6 +5,7 @@
 #include <list>
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "envoy/common/pure.h"
 
@@ -19,40 +20,91 @@ class Instance;
 
 namespace Stats {
 
+/**
+ * General representation of a tag.
+ */
+struct Tag {
+  std::string name_;
+  std::string value_;
+};
+
+class TagExtractor {
+public:
+  virtual ~TagExtractor() {}
+
+  /**
+   * Identifier for this tag.
+   */
+  virtual std::string name() const PURE;
+
+  /**
+   * Updates the tag extracted name and the set of tags by extracting the tag represented by this
+   * TagExtractor. If the tag is not represented in the current tag_extracted_name, nothing will be
+   * modified.
+   * @param name name from which the tag will be removed if found to exist.
+   * @param tags list of tags updated with the tag name and value if found in the
+   * name.
+   * @returns modified tag_extracted_name with the tag removed.
+   */
+  virtual std::string updateTags(const std::string& name, std::vector<Tag>& tags) const PURE;
+};
+
+typedef std::unique_ptr<TagExtractor> TagExtractorPtr;
+
+/**
+ * General interface for all stats objects.
+ */
+class Metric {
+public:
+  virtual ~Metric() {}
+  /**
+   * Returns the full name of the Metric.
+   */
+  virtual std::string name() const PURE;
+
+  /**
+   * Returns a vector of configurable tags to identify this Metric.
+   */
+  virtual const std::vector<Tag>& tags() const PURE;
+
+  /**
+   * Returns the name of the Metric with the portions designated as tags removed.
+   */
+  virtual const std::string& tagExtractedName() const PURE;
+};
+
 /**
  * An always incrementing counter with latching capability. Each increment is added both to a
  * global counter as well as periodic counter. Calling latch() returns the periodic counter and
  * clears it.
  */
-class Counter {
+class Counter : public Metric {
 public:
   virtual ~Counter() {}
   virtual void add(uint64_t amount) PURE;
   virtual void inc() PURE;
   virtual uint64_t latch() PURE;
-  virtual std::string name() PURE;
   virtual void reset() PURE;
-  virtual bool used() PURE;
-  virtual uint64_t value() PURE;
+  virtual bool used() const PURE;
+  virtual uint64_t value() const PURE;
 };
 
 typedef std::shared_ptr<Counter> CounterSharedPtr;
 
 /**
  * A gauge that can both increment and decrement.
  */
-class Gauge {
+class Gauge : public Metric {
 public:
   virtual ~Gauge() {}
 
   virtual void add(uint64_t amount) PURE;
   virtual void dec() PURE;
   virtual void inc() PURE;
-  virtual std::string name() PURE;
   virtual void set(uint64_t value) PURE;
   virtual void sub(uint64_t amount) PURE;
-  virtual bool used() PURE;
-  virtual uint64_t value() PURE;
+  virtual bool used() const PURE;
+  virtual uint64_t value() const PURE;
 };
 
 typedef std::shared_ptr<Gauge> GaugeSharedPtr;
@@ -83,16 +135,28 @@ typedef std::unique_ptr<Timespan> TimespanPtr;
 /**
  * A timer that can capture timespans.
  */
-class Timer {
+class Timer : public Metric {
 public:
   virtual ~Timer() {}
 
   virtual TimespanPtr allocateSpan() PURE;
-  virtual std::string name() PURE;
+  virtual void recordDuration(std::chrono::milliseconds ms) PURE;
 };
 
 typedef std::shared_ptr<Timer> TimerSharedPtr;
 
+/**
+ * A histogram that captures values one at a time.
+ */
+class Histogram : public Metric {
+public:
+  virtual ~Histogram() {}
+
+  virtual void recordValue(uint64_t value) PURE;
+};
+
+typedef std::shared_ptr<Histogram> HistogramSharedPtr;
+
 /**
  * A sink for stats. Each sink is responsible for writing stats to a backing store.
  */
@@ -109,12 +173,12 @@ class Sink {
   /**
    * Flush a counter delta.
    */
-  virtual void flushCounter(const std::string& name, uint64_t delta) PURE;
+  virtual void flushCounter(const Metric& counter, uint64_t delta) PURE;
 
   /**
    * Flush a gauge value.
    */
-  virtual void flushGauge(const std::string& name, uint64_t value) PURE;
+  virtual void flushGauge(const Metric& gauge, uint64_t value) PURE;
 
   /**
    * This will be called after beginFlush(), some number of flushCounter(), and some number of
@@ -125,12 +189,12 @@ class Sink {
   /**
    * Flush a histogram value.
    */
-  virtual void onHistogramComplete(const std::string& name, uint64_t value) PURE;
+  virtual void onHistogramComplete(const Metric& histogram, uint64_t value) PURE;
 
   /**
    * Flush a timespan value.
    */
-  virtual void onTimespanComplete(const std::string& name, std::chrono::milliseconds ms) PURE;
+  virtual void onTimespanComplete(const Metric& timespan, std::chrono::milliseconds ms) PURE;
 };
 
 typedef std::unique_ptr<Sink> SinkPtr;
@@ -157,12 +221,12 @@ class Scope {
   /**
    * Deliver an individual histogram value to all registered sinks.
    */
-  virtual void deliverHistogramToSinks(const std::string& name, uint64_t value) PURE;
+  virtual void deliverHistogramToSinks(const Metric& histogram, uint64_t value) PURE;
 
   /**
    * Deliver an individual timespan completion to all registered sinks.
    */
-  virtual void deliverTimingToSinks(const std::string& name, std::chrono::milliseconds ms) PURE;
+  virtual void deliverTimingToSinks(const Metric& timer, std::chrono::milliseconds ms) PURE;
 
   /**
    * @return a counter within the scope's namespace.
@@ -178,6 +242,11 @@ class Scope {
    * @return a timer within the scope's namespace.
    */
   virtual Timer& timer(const std::string& name) PURE;
+
+  /**
+   * @return a histogram within the scope's namespace.
+   */
+  virtual Histogram& histogram(const std::string& name) PURE;
 };
 
 /**
@@ -196,6 +265,8 @@ class Store : public Scope {
   virtual std::list<GaugeSharedPtr> gauges() const PURE;
 };
 
+typedef std::unique_ptr<Store> StorePtr;
+
 /**
  * The root of the stat store.
  */
@@ -206,6 +277,11 @@ class StoreRoot : public Store {
    */
   virtual void addSink(Sink& sink) PURE;
 
+  /**
+   * Add an extractor to extract a portion of stats names as a tag.
+   */
+  virtual void setTagExtractors(const std::vector<TagExtractorPtr>& tag_extractor) PURE;
+
   /**
    * Initialize the store for threading. This will be called once after all worker threads have
    * been initialized. At this point the store can initialize itself for multi-threaded operation.

include/envoy/stats/stats_macros.h

@@ -39,4 +39,5 @@ namespace Envoy {
 #define POOL_COUNTER(POOL) POOL_COUNTER_PREFIX(POOL, "")
 #define POOL_GAUGE(POOL) POOL_GAUGE_PREFIX(POOL, "")
 #define POOL_TIMER(POOL) POOL_TIMER_PREFIX(POOL, "")
+
 } // Envoy

source/common/config/BUILD

@@ -295,6 +295,7 @@ envoy_cc_library(
 
 envoy_cc_library(
     name = "well_known_names",
+    srcs = ["well_known_names.cc"],
     hdrs = ["well_known_names.h"],
     deps = ["//source/common/common:singleton"],
 )

source/common/config/well_known_names.cc

@@ -0,0 +1,82 @@
+#include "common/config/well_known_names.h"
+
+namespace Envoy {
+namespace Config {
+
+std::unordered_map<std::string, std::string> TagNameValues::getRegexMapping() {
+  std::unordered_map<std::string, std::string> regex_mapping;
+
+  // cluster.<cluster_name>.
+  regex_mapping[CLUSTER_NAME] = "^cluster\\.(([^.]*)\\.)";
+
+  // listener.<port>.
+  regex_mapping[LISTENER_PORT] = "^listener\\.((\\d+?)\\.)";
+
+  // http.<stat_prefix>.
+  regex_mapping[HTTP_CONN_MANAGER_PREFIX] = "^http\\.(([^.]*)\\.)";
+
+  // http.<stat_prefix>.user_agent.<user_agent>.[base_stat] =
+  regex_mapping[HTTP_USER_AGENT] = "^http(?=\\.).*?\\.user_agent\\.(([^.]*)\\.)\\w+?$";
+
+  // listener.<port>.ssl.cipher.<cipher>
+  regex_mapping[SSL_CIPHER] = "^listener(?=\\.).*?\\.ssl\\.cipher(\\.([^.]*))$";
+
+  // auth.clientssl.<stat_prefix>.
+  regex_mapping[CLIENTSSL_PREFIX] = "^auth\\.clientssl\\.(([^.]*)\\.)";
+
+  // mongo.<stat_prefix>.
+  regex_mapping[MONGO_PREFIX] = "^mongo\\.(([^.]*)\\.)";
+
+  // mongo.<stat_prefix>.cmd.<cmd>.[base_stat] =
+  regex_mapping[MONGO_CMD] = "^mongo(?=\\.).*?\\.cmd\\.(([^.]*)\\.)\\w+?$";
+
+  // mongo.<stat_prefix>.collection.<collection>.query.[base_stat] =
+  regex_mapping[MONGO_COLLECTION] = "^mongo(?=\\.).*?\\.collection\\.(([^.]*)\\.).*?query.\\w+?$";
+
+  // mongo.<stat_prefix>.collection.<collection>.callsite.<callsite>.query.[base_stat] =
+  regex_mapping[MONGO_CALLSITE] = "^mongo(?=\\.).*?\\.callsite\\.(([^.]*)\\.).*?query.\\w+?$";
+
+  // ratelimit.<stat_prefix>.
+  regex_mapping[RATELIMIT_PREFIX] = "^ratelimit\\.(([^.]*)\\.)";
+
+  // tcp.<stat_prefix>.
+  regex_mapping[TCP_PREFIX] = "^tcp\\.(([^.]*)\\.)";
+
+  // http.<stat_prefix>.fault.<downstream-cluster>.
+  regex_mapping[FAULT_DOWNSTREAM_CLUSTER] = "^http(?=\\.).*?\\.fault\\.(([^.]*)\\.)";
+
+  // http.<stat_prefix>.dynamodb.(operation or table.<table_name>.capacity).<operation_name>.
+  regex_mapping[DYNAMO_OPERATION] =
+      "^http(?=\\.).*?\\.dynamodb.(?:operation|table(?=\\.).*?\\.capacity)(\\.([^.]*))(?:\\."
+      "|$)";
+
+  // http.<stat_prefix>.dynamodb.(table or error).<table_name>.
+  regex_mapping[DYNAMO_TABLE] = "^http(?=\\.).*?\\.dynamodb.(?:table|error)\\.(([^.]*)\\.)";
+
+  // http.<stat_prefix>.dynamodb.table.<table_name>.capacity.<operation_name>.__partition_id=<last_seven_characters_from_partition_id>
+  regex_mapping[DYNAMO_PARTITION_ID] =
+      "^http(?=\\.).*?\\.dynamodb\\..+?(\\.__partition_id=(\\w{7}))$";
+
+  // cluster.<route target cluster>.grpc.<grpc service>.
+  regex_mapping[GRPC_BRIDGE_SERVICE] = "^cluster(?=\\.).*?\\.grpc\\.(([^.]*)\\.)";
+
+  // cluster.<route target cluster>.grpc.<grpc service>.<grpc method>.[base_stat] =
+  regex_mapping[GRPC_BRIDGE_METHOD] = "^cluster(?=\\.).*?\\.grpc(?=\\.).*\\.(([^.]*)\\.)\\w+?$";
+
+  // vhost.<virtual host name>.
+  regex_mapping[VIRTUAL_HOST] = "^vhost\\.(([^.]*)\\.)";
+
+  // vhost.<virtual host name>.vcluster.<virtual cluster name>.[base_stat] =
+  regex_mapping[VIRTUAL_CLUSTER] = "^vhost(?=\\.).*?\\.vcluster\\.(([^.]*)\\.)\\w+?$";
+
+  // *_rq_<response_code>
+  regex_mapping[RESPONSE_CODE] = "_rq(_(\\d{3}))$";
+
+  // *_rq_<response_code_class>xx
+  regex_mapping[RESPONSE_CODE_CLASS] = "_rq(_(\\dxx))$";
+
+  return regex_mapping;
+}
+
+} // namespace Config
+} // namespace Envoy

source/common/config/well_known_names.h

@@ -172,5 +172,70 @@ class MetadataEnvoyLbKeyValues {
 
 typedef ConstSingleton<MetadataEnvoyLbKeyValues> MetadataEnvoyLbKeys;
 
+/**
+ * Well known tags values and a mapping from these names to the regexes they
+ * represent. Note: when names are added to the list, they also must be added to
+ * the regex map by adding an entry in the getRegexMapping function.
+ */
+class TagNameValues {
+public:
+  // Cluster name tag
+  const std::string CLUSTER_NAME = "envoy.cluster_name";
+  // Listener port tag
+  const std::string LISTENER_PORT = "envoy.listener_port";
+  // Stats prefix for HttpConnectionManager
+  const std::string HTTP_CONN_MANAGER_PREFIX = "envoy.http_conn_manager_prefix";
+  // User agent for a connection
+  const std::string HTTP_USER_AGENT = "envoy.http_user_agent";
+  // SSL Cipher for a connection
+  const std::string SSL_CIPHER = "envoy.ssl_cipher";
+  // Stats prefix for the Client SSL Auth network filter
+  const std::string CLIENTSSL_PREFIX = "envoy.clientssl_prefix";
+  // Stats prefix for the Mongo Proxy network filter
+  const std::string MONGO_PREFIX = "envoy.mongo_prefix";
+  // Request command for the Mongo Proxy network filter
+  const std::string MONGO_CMD = "envoy.mongo_cmd";
+  // Request collection for the Mongo Proxy network filter
+  const std::string MONGO_COLLECTION = "envoy.mongo_collection";
+  // Request callsite for the Mongo Proxy network filter
+  const std::string MONGO_CALLSITE = "envoy.mongo_callsite";
+  // Stats prefix for the Ratelimit network filter
+  const std::string RATELIMIT_PREFIX = "envoy.ratelimit_prefix";
+  // Stats prefix for the TCP Proxy network filter
+  const std::string TCP_PREFIX = "envoy.tcp_prefix";
+  // Downstream cluster for the Fault http filter
+  const std::string FAULT_DOWNSTREAM_CLUSTER = "envoy.fault_downstream_cluster";
+  // Operation name for the Dynamo http filter
+  const std::string DYNAMO_OPERATION = "envoy.dynamo_operation";
+  // Table name for the Dynamo http filter
+  const std::string DYNAMO_TABLE = "envoy.dyanmo_table";
+  // Partition ID for the Dynamo http filter
+  const std::string DYNAMO_PARTITION_ID = "envoy.dynamo_partition_id";
+  // Request service name GRPC Bridge http filter
+  const std::string GRPC_BRIDGE_SERVICE = "envoy.grpc_bridge_service";
+  // Request method name for the GRPC Bridge http filter
+  const std::string GRPC_BRIDGE_METHOD = "envoy.grpc_bridge_method";
+  // Request virtual host given by the Router http filter
+  const std::string VIRTUAL_HOST = "envoy.virtual_host";
+  // Request virtual cluster given by the Router http filter
+  const std::string VIRTUAL_CLUSTER = "envoy.virtual_cluster";
+  // Request response code
+  const std::string RESPONSE_CODE = "envoy.response_code";
+  // Request response code class
+  const std::string RESPONSE_CODE_CLASS = "envoy.response_code_class";
+
+  // Mapping from the names above to their respective regex strings.
+  const std::unordered_map<std::string, std::string> regex_map_;
+
+  // Constructor to fill map.
+  TagNameValues() : regex_map_(getRegexMapping()) {}
+
+private:
+  // Creates a regex mapping for all tag names.
+  std::unordered_map<std::string, std::string> getRegexMapping();
+};
+
+typedef ConstSingleton<TagNameValues> TagNames;
+
 } // namespace Config
 } // namespace Envoy