Skip to content
Merged
Show file tree
Hide file tree
Changes from 57 commits
Commits
Show all changes
61 commits
Select commit Hold shift + click to select a range
a24ecb3
codec split
asraa Mar 30, 2020
8a80bb9
more integration test splits
asraa Mar 31, 2020
53d5d22
Revert "more integration test splits"
asraa Mar 31, 2020
3084c49
Revert "codec split"
asraa Mar 31, 2020
11d7da4
add runtime and test plumbing
asraa Mar 31, 2020
8eadbf1
add switch in fake upstream
asraa Mar 31, 2020
321985a
enable runtime flag for integration tests with asan in ci
asraa Apr 2, 2020
a25c204
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Apr 3, 2020
4ff37a6
make legacy namespace in test
asraa Apr 3, 2020
65ecdd9
backport codec change to legacy codec
asraa Apr 3, 2020
c6714f9
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Apr 6, 2020
86416d4
patch in changes to legacy codecs
asraa Apr 6, 2020
c0967d3
patch H/1 and parametrize H/1 tests
asraa Apr 6, 2020
41cbed2
fix h/2 tests
asraa Apr 6, 2020
fc497e8
add golden diffs
asraa Apr 6, 2020
6f5b4dd
add version history
asraa Apr 6, 2020
e57e759
fix python format
asraa Apr 7, 2020
2d7cebe
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Apr 7, 2020
44b80d9
patch h/1
asraa Apr 7, 2020
32da690
use readLine utility for opening files
asraa Apr 7, 2020
22e9e21
add diff to error message
asraa Apr 7, 2020
d0b7f79
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Apr 7, 2020
1b3262b
yet another patch
asraa Apr 7, 2020
14aad1f
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Apr 9, 2020
18432dd
address comments
asraa Apr 9, 2020
4be1184
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jun 23, 2020
23507ff
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jun 23, 2020
7987381
debug format
asraa Jun 24, 2020
efa71af
Fix format
asraa Jun 24, 2020
5a38663
fix format
asraa Jun 24, 2020
e036bc8
fix format
asraa Jun 30, 2020
b790c38
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 7, 2020
486ef19
remove extra ./
asraa Jul 7, 2020
220d33a
fix format test helper
asraa Jul 7, 2020
5c0fb12
fix
asraa Jul 7, 2020
43a7cc4
fix tests
asraa Jul 8, 2020
7cc9497
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 8, 2020
2506ab2
manual fix release note merge
asraa Jul 8, 2020
1e1a884
fix coverage and runtime test
asraa Jul 8, 2020
d6e7571
attempt to fix windows ci
asraa Jul 8, 2020
8bc999a
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 8, 2020
e0c6183
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 9, 2020
2c3153d
fix legacy tests after merge
asraa Jul 9, 2020
2ae285e
fails on windows
asraa Jul 9, 2020
632758e
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 9, 2020
0bd9774
fix up
asraa Jul 9, 2020
de11d7a
fix build
asraa Jul 9, 2020
33708be
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 10, 2020
2049662
fix format
asraa Jul 10, 2020
d7ea2ac
fix deps, make it harder to break
asraa Jul 10, 2020
9d75493
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 13, 2020
e6aaf3a
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 13, 2020
2e2514d
fix format
asraa Jul 13, 2020
e9be89a
address all but h2 test comment
asraa Jul 13, 2020
071d2f7
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 15, 2020
f5eff5e
fixup and parametrize h2
asraa Jul 15, 2020
3b550a7
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 15, 2020
3eeb952
fix affected tests by name change
asraa Jul 16, 2020
f7f14c7
Merge remote-tracking branch 'upstream/master' into codec-split
asraa Jul 16, 2020
0a65360
fix fuzz test
asraa Jul 16, 2020
7c5484d
fix order of members bc of use-after-free
asraa Jul 16, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions bazel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,11 @@ config_setting(
values = {"define": "path_normalization_by_default=true"},
)

config_setting(
name = "enable_legacy_codecs_in_integration_tests",
values = {"define": "use_legacy_codecs_in_integration_tests=true"},
)

cc_proto_library(
name = "grpc_health_proto",
deps = ["@com_github_grpc_grpc//src/proto/grpc/health/v1:_health_proto_only"],
Expand Down
2 changes: 2 additions & 0 deletions bazel/envoy_build_system.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ load(
_envoy_select_boringssl = "envoy_select_boringssl",
_envoy_select_google_grpc = "envoy_select_google_grpc",
_envoy_select_hot_restart = "envoy_select_hot_restart",
_envoy_select_legacy_codecs_in_integration_tests = "envoy_select_legacy_codecs_in_integration_tests",
)
load(
":envoy_test.bzl",
Expand Down Expand Up @@ -168,6 +169,7 @@ def envoy_google_grpc_external_deps():
envoy_select_boringssl = _envoy_select_boringssl
envoy_select_google_grpc = _envoy_select_google_grpc
envoy_select_hot_restart = _envoy_select_hot_restart
envoy_select_legacy_codecs_in_integration_tests = _envoy_select_legacy_codecs_in_integration_tests

# Binary wrappers (from envoy_binary.bzl)
envoy_cc_binary = _envoy_cc_binary
Expand Down
7 changes: 7 additions & 0 deletions bazel/envoy_select.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,10 @@ def envoy_select_hot_restart(xs, repository = ""):
repository + "//bazel:disable_hot_restart_or_apple": [],
"//conditions:default": xs,
})

# Select the given values if use legacy codecs in test is on in the current build.
def envoy_select_legacy_codecs_in_integration_tests(xs, repository = ""):
return select({
repository + "//bazel:enable_legacy_codecs_in_integration_tests": xs,
"//conditions:default": [],
})
5 changes: 5 additions & 0 deletions ci/do_ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
--define quiche=enabled \
--define path_normalization_by_default=true \
--define deprecated_features=disabled \
--define use_legacy_codecs_in_integration_tests=true \
"
ENVOY_STDLIB="${ENVOY_STDLIB:-libstdc++}"
setup_clang_toolchain
Expand All @@ -236,6 +237,10 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
echo "Building and testing ${TEST_TARGETS}"
bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg ${TEST_TARGETS} --test_tag_filters=-nofips --build_tests_only

# Legacy codecs "--define legacy_codecs_in_integration_tests=true" should also be tested in
# integration tests with asan.
bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg @envoy//test/integration/... --config=clang-asan --build_tests_only
Comment thread
htuch marked this conversation as resolved.

# "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only
# these tests under "-c opt" to save time in CI.
bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c opt @envoy//test/common/common:assert_test @envoy//test/server:server_test
Expand Down
1 change: 1 addition & 0 deletions docs/root/version_history/current.rst
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ Removed Config or Runtime
New Features
------------
* grpc-json: support specifying `response_body` field in for `google.api.HttpBody` message.
* http: introduced new HTTP/1 and HTTP/2 codec implementations that will remove the use of exceptions for control flow due to high risk factors and instead use error statuses. The old behavior is deprecated, but can be used during the removal period by setting the runtime feature `envoy.reloadable_features.new_codec_behavior` to false. The removal period will be one month.
* load balancer: added a :ref:`configuration<envoy_v3_api_msg_config.cluster.v3.Cluster.LeastRequestLbConfig>` option to specify the active request bias used by the least request load balancer.
* tap: added :ref:`generic body matcher<envoy_v3_api_msg_config.tap.v3.HttpGenericBodyMatch>` to scan http requests and responses for text or hex patterns.

Expand Down
7 changes: 7 additions & 0 deletions source/common/http/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,21 @@ envoy_cc_library(
"//include/envoy/http:codec_interface",
"//include/envoy/network:connection_interface",
"//include/envoy/network:filter_interface",
"//include/envoy/runtime:runtime_interface",
"//source/common/common:assert_lib",
"//source/common/common:enum_to_int",
"//source/common/common:linked_object",
"//source/common/common:minimal_logger_lib",
"//source/common/config:utility_lib",
"//source/common/http/http1:codec_legacy_lib",
"//source/common/http/http1:codec_lib",
"//source/common/http/http2:codec_legacy_lib",
"//source/common/http/http2:codec_lib",
"//source/common/http/http3:quic_codec_factory_lib",
"//source/common/http/http3:well_known_names",
"//source/common/network:filter_lib",
"//source/common/runtime:runtime_features_lib",
"//source/common/runtime:runtime_lib",
],
)

Expand Down Expand Up @@ -207,7 +212,9 @@ envoy_cc_library(
"//source/common/common:scope_tracker",
"//source/common/common:utility_lib",
"//source/common/config:utility_lib",
"//source/common/http/http1:codec_legacy_lib",
"//source/common/http/http1:codec_lib",
"//source/common/http/http2:codec_legacy_lib",
"//source/common/http/http2:codec_lib",
"//source/common/http/http3:quic_codec_factory_lib",
"//source/common/http/http3:well_known_names",
Expand Down
32 changes: 25 additions & 7 deletions source/common/http/codec_client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@
#include "common/config/utility.h"
#include "common/http/exception.h"
#include "common/http/http1/codec_impl.h"
#include "common/http/http1/codec_impl_legacy.h"
#include "common/http/http2/codec_impl.h"
#include "common/http/http2/codec_impl_legacy.h"
#include "common/http/http3/quic_codec_factory.h"
#include "common/http/http3/well_known_names.h"
#include "common/http/status.h"
#include "common/http/utility.h"
#include "common/runtime/runtime_features.h"
#include "common/runtime/runtime_impl.h"

namespace Envoy {
namespace Http {
Expand Down Expand Up @@ -150,23 +154,37 @@ CodecClientProd::CodecClientProd(Type type, Network::ClientConnectionPtr&& conne

switch (type) {
case Type::HTTP1: {
codec_ = std::make_unique<Http1::ClientConnectionImpl>(
*connection_, host->cluster().http1CodecStats(), *this, host->cluster().http1Settings(),
host->cluster().maxResponseHeadersCount());
if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.new_codec_behavior")) {
codec_ = std::make_unique<Http1::ClientConnectionImpl>(
*connection_, host->cluster().http1CodecStats(), *this, host->cluster().http1Settings(),
host->cluster().maxResponseHeadersCount());
} else {
codec_ = std::make_unique<Legacy::Http1::ClientConnectionImpl>(
*connection_, host->cluster().http1CodecStats(), *this, host->cluster().http1Settings(),
host->cluster().maxResponseHeadersCount());
}
break;
}
case Type::HTTP2: {
codec_ = std::make_unique<Http2::ClientConnectionImpl>(
*connection_, *this, host->cluster().http2CodecStats(), host->cluster().http2Options(),
Http::DEFAULT_MAX_REQUEST_HEADERS_KB, host->cluster().maxResponseHeadersCount(),
Http2::ProdNghttp2SessionFactory::get());
if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.new_codec_behavior")) {
codec_ = std::make_unique<Http2::ClientConnectionImpl>(
*connection_, *this, host->cluster().http2CodecStats(), host->cluster().http2Options(),
Http::DEFAULT_MAX_REQUEST_HEADERS_KB, host->cluster().maxResponseHeadersCount(),
Http2::ProdNghttp2SessionFactory::get());
} else {
codec_ = std::make_unique<Http2::ClientConnectionImpl>(
*connection_, *this, host->cluster().http2CodecStats(), host->cluster().http2Options(),
Http::DEFAULT_MAX_REQUEST_HEADERS_KB, host->cluster().maxResponseHeadersCount(),
Http2::ProdNghttp2SessionFactory::get());
}
break;
}
case Type::HTTP3: {
codec_ = std::unique_ptr<ClientConnection>(
Config::Utility::getAndCheckFactoryByName<Http::QuicHttpClientConnectionFactory>(
Http::QuicCodecNames::get().Quiche)
.createQuicClientConnection(*connection_, *this));
break;
}
}
}
Expand Down
26 changes: 20 additions & 6 deletions source/common/http/conn_manager_utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
#include "common/http/header_utility.h"
#include "common/http/headers.h"
#include "common/http/http1/codec_impl.h"
#include "common/http/http1/codec_impl_legacy.h"
#include "common/http/http2/codec_impl.h"
#include "common/http/http2/codec_impl_legacy.h"
#include "common/http/path_utility.h"
#include "common/http/utility.h"
#include "common/network/utility.h"
Expand Down Expand Up @@ -51,14 +53,26 @@ ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec(
headers_with_underscores_action) {
if (determineNextProtocol(connection, data) == Utility::AlpnNames::get().Http2) {
Http2::CodecStats& stats = Http2::CodecStats::atomicGet(http2_codec_stats, scope);
return std::make_unique<Http2::ServerConnectionImpl>(
connection, callbacks, stats, http2_options, max_request_headers_kb,
max_request_headers_count, headers_with_underscores_action);
if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.new_codec_behavior")) {
return std::make_unique<Http2::ServerConnectionImpl>(
connection, callbacks, stats, http2_options, max_request_headers_kb,
max_request_headers_count, headers_with_underscores_action);
} else {
return std::make_unique<Legacy::Http2::ServerConnectionImpl>(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Question - given the overhead we're going to have for edits during this period, would it make sense to tackle these one at a time? I don't know how long you estimate the work on each codec to be - if it's a week or so it's not worth splitting, but if it's going to be a month on HTTP/1 it'd be nice to not incur extra work on HTTP/2 during that time period. What are your thoughts here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yanavlasov Possibly. I only just finished merging edits in source files and tests for both codecs and parametrizing the H/1 codec unit test so it's easier to spot when it's out of sync.

For H/2: It's very messy to parametrize the H/2 codec because it uses another test class. If we're okay not parametrizing the H/2 codec_impl_test and relying on just diffing the test files for the two H/2 codec implementation in fix_format (working on right now), then the H/2 work is done as well. Otherwise if we really want to parametrize the H/2 codec_impl test like I did for H/1 I'll split this out.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. Yes parameterizing H/2 codec test would be quite a bit of surgery. I think it would be very good to have H/2 test parameterized and I'm going to see if it can be done reasonably easy. It is probably not a blocker for this PR.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thinking more about this, I think the way it is done now it totally fine for this PR. We have coverage for both H/2 codecs through separate tests. Yes, it is more overhead and a bit inconvenient. Note however that test files are exactly the same with the exception of the namespace declaration, so it should just be cut and paste of the test.

connection, callbacks, stats, http2_options, max_request_headers_kb,
max_request_headers_count, headers_with_underscores_action);
}
} else {
Http1::CodecStats& stats = Http1::CodecStats::atomicGet(http1_codec_stats, scope);
return std::make_unique<Http1::ServerConnectionImpl>(
connection, stats, callbacks, http1_settings, max_request_headers_kb,
max_request_headers_count, headers_with_underscores_action);
if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.new_codec_behavior")) {
return std::make_unique<Http1::ServerConnectionImpl>(
connection, stats, callbacks, http1_settings, max_request_headers_kb,
max_request_headers_count, headers_with_underscores_action);
} else {
return std::make_unique<Legacy::Http1::ServerConnectionImpl>(
connection, stats, callbacks, http1_settings, max_request_headers_kb,
max_request_headers_count, headers_with_underscores_action);
}
}
}

Expand Down
60 changes: 35 additions & 25 deletions source/common/http/http1/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -24,36 +24,46 @@ envoy_cc_library(
],
)

CODEC_LIB_DEPS = [
":codec_stats_lib",
":header_formatter_lib",
"//include/envoy/buffer:buffer_interface",
"//include/envoy/http:codec_interface",
"//include/envoy/http:header_map_interface",
"//include/envoy/network:connection_interface",
"//source/common/buffer:buffer_lib",
"//source/common/buffer:watermark_buffer_lib",
"//source/common/common:assert_lib",
"//source/common/common:statusor_lib",
"//source/common/common:utility_lib",
"//source/common/grpc:common_lib",
"//source/common/http:codec_helper_lib",
"//source/common/http:codes_lib",
"//source/common/http:exception_lib",
"//source/common/http:header_map_lib",
"//source/common/http:header_utility_lib",
"//source/common/http:headers_lib",
"//source/common/http:status_lib",
"//source/common/http:url_utility_lib",
"//source/common/http:utility_lib",
"//source/common/runtime:runtime_features_lib",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
]

envoy_cc_library(
name = "codec_lib",
srcs = ["codec_impl.cc"],
hdrs = ["codec_impl.h"],
external_deps = ["http_parser"],
deps = [
":codec_stats_lib",
":header_formatter_lib",
"//include/envoy/buffer:buffer_interface",
"//include/envoy/http:codec_interface",
"//include/envoy/http:header_map_interface",
"//include/envoy/network:connection_interface",
"//source/common/buffer:buffer_lib",
"//source/common/buffer:watermark_buffer_lib",
"//source/common/common:assert_lib",
"//source/common/common:statusor_lib",
"//source/common/common:utility_lib",
"//source/common/grpc:common_lib",
"//source/common/http:codec_helper_lib",
"//source/common/http:codes_lib",
"//source/common/http:exception_lib",
"//source/common/http:header_map_lib",
"//source/common/http:header_utility_lib",
"//source/common/http:headers_lib",
"//source/common/http:status_lib",
"//source/common/http:url_utility_lib",
"//source/common/http:utility_lib",
"//source/common/runtime:runtime_features_lib",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
],
deps = CODEC_LIB_DEPS,
)

envoy_cc_library(
name = "codec_legacy_lib",
srcs = ["codec_impl_legacy.cc"],
hdrs = ["codec_impl_legacy.h"],
external_deps = ["http_parser"],
deps = CODEC_LIB_DEPS,
)

envoy_cc_library(
Expand Down
Loading