Skip to content

test: Adds an integration test for SDS #2395

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alyssawilk merged 7 commits into from
Jun 29, 2022
Merged

test: Adds an integration test for SDS #2395

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ae2407f
test: Adds an integration test for SDS
abeyad Jun 15, 2022
90ebfef
remove commented out line
abeyad Jun 28, 2022
d20d4a7
run check_format.sh fix
abeyad Jun 28, 2022
21f863f
sds_cluster.lyft.com to xds_cluster.lyft.com
abeyad Jun 28, 2022
769c57d
fix formatting
abeyad Jun 28, 2022
4af4806
address Alyssa's feedback
abeyad Jun 29, 2022
34521cb
address review feedback
abeyad Jun 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 50 additions & 6 deletions test/common/integration/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,45 @@ envoy_cc_test(
# TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses.
"sandboxNetwork": "standard",
},
external_deps = [
"abseil_strings",
],
repository = "@envoy",
deps = [
":base_client_integration_test_lib",
"@envoy//test/common/grpc:grpc_client_integration_lib",
"@envoy//test/server:utility_lib",
":xds_integration_test_lib",
"@envoy//test/test_common:utility_lib",
"@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
"@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
"@envoy_api//envoy/service/runtime/v3:pkg_cc_proto",
],
)

envoy_cc_test(
name = "sds_integration_test",
srcs = [
"sds_integration_test.cc",
],
data = [
"@envoy//test/config/integration/certs",
],
exec_properties = {
# TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses.
"sandboxNetwork": "standard",
},
repository = "@envoy",
deps = [
":xds_integration_test_lib",
"@envoy//source/common/config:protobuf_link_hacks",
"@envoy//source/extensions/transport_sockets/tls:config",
"@envoy//source/extensions/transport_sockets/tls:context_config_lib",
"@envoy//source/extensions/transport_sockets/tls:context_lib",
"@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
"@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
"@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto",
"@envoy_api//envoy/service/secret/v3:pkg_cc_proto",
],
)

envoy_cc_test_library(
name = "base_client_integration_test_lib",
srcs = [
Expand All @@ -54,13 +82,29 @@ envoy_cc_test_library(
"//library/common/http:client_lib",
"//library/common/http:header_utility_lib",
"//library/common/types:c_types_lib",
"@envoy//source/extensions/http/header_formatters/preserve_case:config",
"@envoy//source/extensions/http/header_formatters/preserve_case:preserve_case_formatter",
"@envoy//test/common/http:common_lib",
"@envoy//test/integration:http_integration_lib",
],
)

envoy_cc_test_library(
name = "xds_integration_test_lib",
srcs = [
"xds_integration_test.cc",
],
hdrs = [
"xds_integration_test.h",
],
repository = "@envoy",
deps = [
":base_client_integration_test_lib",
"@envoy//source/common/config:api_version_lib",
"@envoy//test/common/grpc:grpc_client_integration_lib",
"@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
"@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
],
)

# interface libs for quic test server's jni implementation
envoy_cc_test_library(
name = "quic_test_server_interface_lib",
Expand Down
23 changes: 11 additions & 12 deletions test/common/integration/base_client_integration_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,25 @@
#include "test/common/integration/base_client_integration_test.h"

#include "source/extensions/http/header_formatters/preserve_case/config.h"
#include "source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h"

#include "test/common/http/common.h"

#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "library/common/config/internal.h"
#include "library/common/http/header_utility.h"

namespace Envoy {
namespace {

void validateStreamIntel(const envoy_final_stream_intel& final_intel, bool expect_dns) {
void validateStreamIntel(const envoy_final_stream_intel& final_intel, bool expect_dns,
bool tls_upstream) {
if (expect_dns) {
EXPECT_NE(-1, final_intel.dns_start_ms);
EXPECT_NE(-1, final_intel.dns_end_ms);
}
// This test doesn't do TLS.
EXPECT_EQ(-1, final_intel.ssl_start_ms);
EXPECT_EQ(-1, final_intel.ssl_end_ms);

if (tls_upstream) {
EXPECT_GT(final_intel.ssl_start_ms, 0);
Comment thread
abeyad marked this conversation as resolved.
EXPECT_GT(final_intel.ssl_end_ms, 0);
} else {
EXPECT_EQ(-1, final_intel.ssl_start_ms);
EXPECT_EQ(-1, final_intel.ssl_end_ms);
}

ASSERT_NE(-1, final_intel.stream_start_ms);
ASSERT_NE(-1, final_intel.connect_start_ms);
Expand Down Expand Up @@ -72,7 +71,7 @@ void BaseClientIntegrationTest::initialize() {
});
stream_prototype_->setOnComplete(
[this](envoy_stream_intel, envoy_final_stream_intel final_intel) {
validateStreamIntel(final_intel, expect_dns_);
validateStreamIntel(final_intel, expect_dns_, tls_upstream_);
cc_.on_complete_received_byte_count = final_intel.received_byte_count;
cc_.on_complete_calls++;
cc_.terminal_callback->setReady();
Expand Down
1 change: 1 addition & 0 deletions test/common/integration/base_client_integration_test.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class BaseClientIntegrationTest : public BaseIntegrationTest, public Platform::E
bool explicit_flow_control_ = false;
bool expect_dns_ = true;
bool override_builder_config_ = false;
bool tls_upstream_ = false;
Comment thread
abeyad marked this conversation as resolved.
Outdated
};

} // namespace Envoy
179 changes: 9 additions & 170 deletions test/common/integration/rtds_integration_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,50 +1,14 @@
#include "envoy/config/bootstrap/v3/bootstrap.pb.h"
#include "envoy/config/cluster/v3/cluster.pb.h"
#include "envoy/service/runtime/v3/rtds.pb.h"

#include "source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h"

#include "test/common/grpc/grpc_client_integration.h"
#include "test/common/http/common.h"
#include "test/common/integration/base_client_integration_test.h"
#include "test/integration/autonomous_upstream.h"
#include "test/integration/http_integration.h"
#include "test/server/utility.h"
#include "test/test_common/environment.h"
#include "test/common/integration/xds_integration_test.h"
#include "test/test_common/utility.h"

#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "library/common/data/utility.h"
#include "library/common/http/client.h"
#include "library/common/http/header_utility.h"
#include "library/common/types/c_types.h"

namespace Envoy {
namespace {

envoy::config::cluster::v3::Cluster
createSingleEndpointClusterConfig(const std::string& cluster_name,
const std::string& loopbackAddr) {
envoy::config::cluster::v3::Cluster config;
config.set_name(cluster_name);

// Set the endpoint.
auto* load_assignment = config.mutable_load_assignment();
load_assignment->set_cluster_name(cluster_name);
auto* endpoint = load_assignment->add_endpoints()->add_lb_endpoints()->mutable_endpoint();
endpoint->mutable_address()->mutable_socket_address()->set_address(loopbackAddr);
endpoint->mutable_address()->mutable_socket_address()->set_port_value(0);

// Set the protocol options.
envoy::extensions::upstreams::http::v3::HttpProtocolOptions options;
options.mutable_explicit_http_config()->mutable_http2_protocol_options();
(*config.mutable_typed_extension_protocol_options())
["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"]
.PackFrom(options);
return config;
}

envoy::config::bootstrap::v3::LayeredRuntime layeredRuntimeConfig(const std::string& api_type) {
const std::string yaml = fmt::format(R"EOF(
layers:
Expand All @@ -62,166 +26,41 @@ envoy::config::bootstrap::v3::LayeredRuntime layeredRuntimeConfig(const std::str
transport_api_version: V3
grpc_services:
envoy_grpc:
cluster_name: rtds_cluster
cluster_name: {}
set_node_on_first_message_only: true
- name: some_admin_layer
admin_layer: {{}}
)EOF",
api_type);
api_type, XDS_CLUSTER);

envoy::config::bootstrap::v3::LayeredRuntime config;
TestUtility::loadFromYaml(yaml, config);
return config;
}

envoy::config::bootstrap::v3::Admin adminConfig(const std::string& loopbackAddr) {
const std::string yaml = fmt::format(R"EOF(
access_log:
- name: envoy.access_loggers.file
typed_config:
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
path: "{}"
address:
socket_address:
address: {}
port_value: 0
)EOF",
::Platform::null_device_path, loopbackAddr);

envoy::config::bootstrap::v3::Admin config;
TestUtility::loadFromYaml(yaml, config);
return config;
}

class RtdsIntegrationTest : public BaseClientIntegrationTest,
public Grpc::DeltaSotwIntegrationParamTest {
class RtdsIntegrationTest : public XdsIntegrationTest {
public:
RtdsIntegrationTest() : BaseClientIntegrationTest(ipVersion()) {
override_builder_config_ = true; // The builder does not yet have RTDS support.
expect_dns_ = false; // TODO(alyssawilk) debug.
create_xds_upstream_ = true;
sotw_or_delta_ = sotwOrDelta();
scheme_ = "https";

if (sotw_or_delta_ == Grpc::SotwOrDelta::UnifiedSotw ||
sotw_or_delta_ == Grpc::SotwOrDelta::UnifiedDelta) {
config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true");
}

// Set up the cluster config.
//
// For now, we clear the default cluster configs and add just two clusters:
// - a cluster named "base_h2" because that's what the api_listener is configured to talk to
// - an RTDS cluster, for sending and receiving RTDS config
//
// The reason we must clear the default cluster configs is because ConfigHelper::setPorts
// requires that the number of fake upstream ports equal the number of clusters in the config
// that have dynamic port configuration (i.e. port is 0). In other words, either all fake
// upstreams must be configured with a dynamic port or none of them (can't mix and match).
//
// TODO(abeyad): fix the ConfigHelper::setPorts logic to enable a subset of clusters to have
// dynamic port configuration.

config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
const std::string loopback = loopbackAddr();
bootstrap.mutable_static_resources()->clear_clusters();
bootstrap.mutable_static_resources()->add_clusters()->MergeFrom(
createSingleEndpointClusterConfig("base_h2", loopback));
bootstrap.mutable_static_resources()->add_clusters()->MergeFrom(
createSingleEndpointClusterConfig("rtds_cluster", loopback));
});

// xDS upstream is created separately in the test infra, and there's only one non-xDS cluster.
setUpstreamCount(1);

// Add the Admin config.
config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
bootstrap.mutable_admin()->MergeFrom(adminConfig(loopbackAddr()));
});
admin_filename_ = TestEnvironment::temporaryPath("admin_address.txt");
setAdminAddressPathForTests(admin_filename_);
}

void SetUp() override {
// TODO(abeyad): Add paramaterized tests for HTTP1, HTTP2, and HTTP3.
setUpstreamProtocol(Http::CodecType::HTTP2);
}

void TearDown() override {
cleanup();
BaseClientIntegrationTest::TearDown();
}

void createEnvoy() override {
BaseClientIntegrationTest::createEnvoy();
std::string admin_str = TestEnvironment::readFileToStringForTest(admin_filename_);
auto addr = Network::Utility::parseInternetAddressAndPort(admin_str);
registerPort("admin", addr->ip()->port());
}

void initialize() override {
BaseClientIntegrationTest::initialize();
// Register admin port.
// registerTestServerPorts({});

acceptXdsConnection();
}

void addRuntimeRtdsConfig() {
// Add the layered runtime config, which includes the RTDS layer.
RtdsIntegrationTest() : XdsIntegrationTest() {

@RyanTheOptimist RyanTheOptimist Jun 28, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Is the ": XdsIntegrationTest()" required, or does C++ magically do the right thing without it?

@abeyad abeyad Jun 29, 2022

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

C++ does the right thing since it's a zero arg constructor so it automatically calls the parent class' zero-arg constructor. I removed the explicit call.

config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
// Add the layered runtime config, which includes the RTDS layer.
const std::string api_type = sotw_or_delta_ == Grpc::SotwOrDelta::Sotw ||
sotw_or_delta_ == Grpc::SotwOrDelta::UnifiedSotw
? "GRPC"
: "DELTA_GRPC";

bootstrap.mutable_layered_runtime()->MergeFrom(layeredRuntimeConfig(api_type));
});
}

void acceptXdsConnection() {
// Initial RTDS connection.
createXdsConnection();
AssertionResult result =
xds_connection_->waitForNewStream(*BaseIntegrationTest::dispatcher_, xds_stream_);
RELEASE_ASSERT(result, result.message());
xds_stream_->startGrpcStream();
}

Network::Address::IpVersion ipVersion() const override { return std::get<0>(GetParam()); }
Grpc::ClientType clientType() const override { return std::get<1>(GetParam()); }
Grpc::SotwOrDelta sotwOrDelta() const { return std::get<2>(GetParam()); }

std::string loopbackAddr() const {
if (ipVersion() == Network::Address::IpVersion::v6) {
return "::1";
}
return "127.0.0.1";
}

protected:
std::string getRuntimeKey(const std::string& key) {
auto response = IntegrationUtil::makeSingleRequest(
lookupPort("admin"), "GET", "/runtime?format=json", "", Http::CodecType::HTTP2, version_);
EXPECT_TRUE(response->complete());
EXPECT_EQ("200", response->headers().getStatusValue());
Json::ObjectSharedPtr loader = TestEnvironment::jsonLoadFromString(response->body());
auto entries = loader->getObject("entries");
if (entries->hasObject(key)) {
return entries->getObject(key)->getString("final_value");
}
return "";
void initialize() override {
BaseClientIntegrationTest::initialize();
initializeXdsStream();
}

uint32_t initial_load_success_{};
std::string admin_filename_;
};

INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest,
DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS);

TEST_P(RtdsIntegrationTest, RtdsReload) {
addRuntimeRtdsConfig();
initialize();

// Send a request on the data plane.
Expand Down
Loading