tools: add coverage reporting & enforcement to router check

docs/root/configuration/tools/router_check.rst

@@ -3,9 +3,11 @@
 Route table check tool
 ======================
 
-**NOTE: The following configuration is for the route table check tool only and is not part of the Envoy binary.
-The route table check tool is a standalone binary that can be used to verify Envoy's routing for a given configuration
-file.**
+.. note::
+
+  The following configuration is for the route table check tool only and is not part of the Envoy binary.
+  The route table check tool is a standalone binary that can be used to verify Envoy's routing for a given configuration
+  file.
 
 The following specifies input to the route table check tool. The route table check tool checks if
 the route returned by a :ref:`router <envoy_api_msg_RouteConfiguration>` matches what is expected.
@@ -148,3 +150,23 @@ validate
 
     value
       *(required, string)* The value of the header field to match.
+
+Coverage
+--------
+
+The router check tool will report route coverage at the end of a successful test run.
+
+.. code:: bash
+
+  > bazel-bin/test/tools/router_check/router_check_tool --config-path ... --test-path ... --useproto
+  Current route coverage: 0.0744863
+
+This reporting can be leveraged to enforce a minimum coverage percentage by using
+the `-f` or `--fail-under` flag. If coverage falls below this percentage the test
+run will fail.
+
+.. code:: bash
+
+  > bazel-bin/test/tools/router_check/router_check_tool --config-path ... --test-path ... --useproto --fail-under 0.08
+  Current route coverage: 0.0744863
+  Failed to meet coverage requirement: 0.08

docs/root/intro/version_history.rst

@@ -4,13 +4,14 @@ Version history
 1.12.0 (pending)
 ================
 * admin: added ability to configure listener :ref:`socket options <envoy_api_field_config.bootstrap.v2.Admin.socket_options>`.
-* admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump <envoy_api_msg_admin.v2alpha.SecretsConfigDump>`. 
+* admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump <envoy_api_msg_admin.v2alpha.SecretsConfigDump>`.
 * config: async data access for local and remote data source.
 * config: changed the default value of :ref:`initial_fetch_timeout <envoy_api_field_core.ConfigSource.initial_fetch_timeout>` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process <arch_overview_initialization>` for more details.
 * fault: added overrides for default runtime keys in :ref:`HTTPFault <envoy_api_msg_config.filter.http.fault.v2.HTTPFault>` filter.
 * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`.
 * listeners: added :ref:`HTTP inspector listener filter <config_listener_filters_http_inspector>`.
 * router: added :ref:`rq_retry_skipped_request_not_complete <config_http_filters_router_stats>` counter stat to router stats.
+* router check tool: add coverage reporting & enforcement.
 * tls: added verification of IP address SAN fields in certificates against configured SANs in the
   certificate validation context.
 

test/tools/router_check/BUILD

@@ -19,6 +19,8 @@ envoy_cc_test_binary(
 envoy_cc_test_library(
     name = "router_check_main_lib",
     srcs = [
+        "coverage.cc",
+        "coverage.h",
         "router.cc",
         "router.h",
     ],

test/tools/router_check/coverage.cc

@@ -0,0 +1,23 @@
+#include "test/tools/router_check/coverage.h"
+
+#include <algorithm>
+
+#include "envoy/api/v2/core/base.pb.h"
+
+namespace Envoy {
+void Coverage::markCovered(const Envoy::Router::RouteEntry* route) {
+  const bool seen =
+      std::find(seen_routes_.begin(), seen_routes_.end(), route) != seen_routes_.end();
+  if (!seen) {
+    seen_routes_.push_back(route);
+  }
+}
+
+double Coverage::report() {
+  int size_t = 0;
+  for (const auto& host : route_config_.virtual_hosts()) {
+    size_t += host.routes_size();
+  }
+  return 100 * static_cast<double>(seen_routes_.size()) / size_t;
+}
+} // namespace Envoy

test/tools/router_check/coverage.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "envoy/api/v2/core/base.pb.h"
+#include "envoy/router/router.h"
+
+#include "common/common/empty_string.h"
+#include "common/common/logger.h"
+#include "common/stats/fake_symbol_table_impl.h"
+
+#include "test/mocks/server/mocks.h"
+#include "test/test_common/utility.h"
+
+namespace Envoy {
+class Coverage : Logger::Loggable<Logger::Id::testing> {
+public:
+  Coverage(envoy::api::v2::RouteConfiguration config) : route_config_(config){};
+  void markCovered(const Envoy::Router::RouteEntry* route);
+  double report();
+
+private:
+  std::vector<const Envoy::Router::RouteEntry*> seen_routes_;
+  envoy::api::v2::RouteConfiguration route_config_;
+};
+} // namespace Envoy

test/tools/router_check/router.cc

@@ -73,16 +73,17 @@ RouterCheckTool RouterCheckTool::create(const std::string& router_config_file) {
   auto factory_context = std::make_unique<NiceMock<Server::Configuration::MockFactoryContext>>();
   auto config = std::make_unique<Router::ConfigImpl>(route_config, *factory_context, false);
 
+  Coverage coverage_report = Coverage(route_config);
   return RouterCheckTool(std::move(factory_context), std::move(config), std::move(stats),
-                         std::move(api));
+                         std::move(api), std::move(coverage_report));
 }
 
 RouterCheckTool::RouterCheckTool(
     std::unique_ptr<NiceMock<Server::Configuration::MockFactoryContext>> factory_context,
     std::unique_ptr<Router::ConfigImpl> config, std::unique_ptr<Stats::IsolatedStoreImpl> stats,
-    Api::ApiPtr api)
+    Api::ApiPtr api, Coverage coverage)
     : factory_context_(std::move(factory_context)), config_(std::move(config)),
-      stats_(std::move(stats)), api_(std::move(api)) {
+      stats_(std::move(stats)), api_(std::move(api)), coverage_(std::move(coverage)) {
   ON_CALL(factory_context_->runtime_loader_.snapshot_,
           featureEnabled(_, testing::An<const envoy::type::FractionalPercent&>(),
                          testing::An<uint64_t>()))
@@ -281,7 +282,11 @@ bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::str
 
     actual = tool_config.headers_->get_(Http::Headers::get().Path);
   }
-  return compareResults(actual, expected, "path_rewrite");
+  const bool matches = compareResults(actual, expected, "path_rewrite");
+  if (matches) {
+    coverage_.markCovered(tool_config.route_->routeEntry());
+  }
+  return matches;
 }
 
 bool RouterCheckTool::compareRewritePath(
@@ -415,6 +420,9 @@ Options::Options(int argc, char** argv) {
   TCLAP::CmdLine cmd("router_check_tool", ' ', "none", true);
   TCLAP::SwitchArg is_proto("p", "useproto", "Use Proto test file schema", cmd, false);
   TCLAP::SwitchArg is_detailed("d", "details", "Show detailed test execution results", cmd, false);
+  TCLAP::ValueArg<double> fail_under("f", "fail-under",
+                                     "Fail if test coverage is under a specified amount", false,
+                                     0.0, "float", cmd);
   TCLAP::ValueArg<std::string> config_path("c", "config-path", "Path to configuration file.", false,
                                            "", "string", cmd);
   TCLAP::ValueArg<std::string> test_path("t", "test-path", "Path to test file.", false, "",
@@ -430,6 +438,7 @@ Options::Options(int argc, char** argv) {
 
   is_proto_ = is_proto.getValue();
   is_detailed_ = is_detailed.getValue();
+  fail_under_ = fail_under.getValue();
 
   if (is_proto_) {
     config_path_ = config_path.getValue();

test/tools/router_check/router.h

@@ -17,6 +17,7 @@
 #include "test/test_common/global.h"
 #include "test/test_common/printers.h"
 #include "test/test_common/utility.h"
+#include "test/tools/router_check/coverage.h"
 #include "test/tools/router_check/json/tool_config_schemas.h"
 #include "test/tools/router_check/validation.pb.h"
 #include "test/tools/router_check/validation.pb.validate.h"
@@ -88,11 +89,13 @@ class RouterCheckTool : Logger::Loggable<Logger::Id::testing> {
    */
   void setShowDetails() { details_ = true; }
 
+  float coverage() { return coverage_.report(); }
+
 private:
   RouterCheckTool(
       std::unique_ptr<NiceMock<Server::Configuration::MockFactoryContext>> factory_context,
       std::unique_ptr<Router::ConfigImpl> config, std::unique_ptr<Stats::IsolatedStoreImpl> stats,
-      Api::ApiPtr api);
+      Api::ApiPtr api, Coverage coverage);
 
   bool compareCluster(ToolConfig& tool_config, const std::string& expected);
   bool compareCluster(ToolConfig& tool_config,
@@ -143,6 +146,7 @@ class RouterCheckTool : Logger::Loggable<Logger::Id::testing> {
   std::unique_ptr<Stats::IsolatedStoreImpl> stats_;
   Api::ApiPtr api_;
   std::string active_runtime;
+  Coverage coverage_;
 };
 
 /**
@@ -172,6 +176,11 @@ class Options {
    */
   const std::string& unlabelledTestPath() const { return unlabelled_test_path_; }
 
+  /**
+   * @return the minimum required percentage of routes coverage.
+   */
+  double failUnder() const { return fail_under_; }
+
   /**
    * @return true if proto schema test is used.
    */
@@ -187,6 +196,7 @@ class Options {
   std::string config_path_;
   std::string unlabelled_test_path_;
   std::string unlabelled_config_path_;
+  float fail_under_;
   bool is_proto_;
   bool is_detailed_;
 };

test/tools/router_check/router_check.cc

@@ -7,6 +7,7 @@
 int main(int argc, char* argv[]) {
   Envoy::Options options(argc, argv);
 
+  const bool enforceCoverage = options.failUnder() != 0.0;
   try {
     Envoy::RouterCheckTool checktool =
         options.isProto() ? Envoy::RouterCheckTool::create(options.configPath())
@@ -23,6 +24,16 @@ int main(int argc, char* argv[]) {
     if (!is_equal) {
       return EXIT_FAILURE;
     }
+
+    const double current_coverage = checktool.coverage();
+    std::cerr << "Current route coverage: " << current_coverage << "%" << std::endl;
+    if (enforceCoverage) {
+      if (current_coverage < options.failUnder()) {
+        std::cerr << "Failed to meet coverage requirement: " << options.failUnder() << "%"
+                  << std::endl;
+        return EXIT_FAILURE;
+      }
+    }
   } catch (const Envoy::EnvoyException& ex) {
     std::cerr << ex.what() << std::endl;
     return EXIT_FAILURE;