From 88756b353e64b6997742f2a2abc4df413e0aa34c Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Wed, 15 Apr 2020 13:14:41 -0700 Subject: [PATCH 01/13] Add support to fuzz proto data in uber filter fuzzer Signed-off-by: Teju Nareddy --- test/fuzz/common.proto | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/fuzz/common.proto b/test/fuzz/common.proto index b32db65c98e09..55db964d8670a 100644 --- a/test/fuzz/common.proto +++ b/test/fuzz/common.proto @@ -5,6 +5,7 @@ package test.fuzz; import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/address.proto"; +import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; @@ -15,9 +16,29 @@ message Headers { repeated envoy.config.core.v3.HeaderValue headers = 1; } +message HttpBody { + // The bytes that will be used as the request body. + repeated string data = 1 [(validate.rules).repeated.min_items = 1]; +} + +// HttpBody cannot efficiently create serialized protos. +// Use ProtoBody instead to test grpc data. +message ProtoBody { + // The proto message that will be serialized and used as the request body. + google.protobuf.Any message = 1 [(validate.rules).any.required = true]; + + // The size (in bytes) of each buffer when forming the requests. + uint64 chuck_size = 2 [(validate.rules).uint64.gt = 0]; +} + message HttpData { Headers headers = 1; - repeated string data = 2; + + oneof body { + HttpBody http_body = 2; + ProtoBody proto_body = 4; + } + Headers trailers = 3; } From 8118f8418da247536ff787c5a0ca8db7d5758925 Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Wed, 15 Apr 2020 13:18:20 -0700 Subject: [PATCH 02/13] Fix format Signed-off-by: Teju Nareddy --- test/fuzz/common.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fuzz/common.proto b/test/fuzz/common.proto index 55db964d8670a..c45521a766a8d 100644 --- a/test/fuzz/common.proto +++ b/test/fuzz/common.proto @@ -18,7 +18,7 @@ message Headers { message HttpBody { // The bytes that will be used as the request body. - repeated string data = 1 [(validate.rules).repeated.min_items = 1]; + repeated string data = 1 [(validate.rules).repeated .min_items = 1]; } // HttpBody cannot efficiently create serialized protos. From 6267c6892591d6af63e5488221c9029abb8c4832 Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Thu, 23 Apr 2020 12:37:19 -0700 Subject: [PATCH 03/13] Update libprotobufmutator and fix corpus Signed-off-by: Teju Nareddy --- .../filters/http/common/fuzz/filter_corpus/buffer1 | 4 +++- .../crash-3014465358f0947e73ac12ccb40b299d5b0646b3 | 6 ++++-- tools/code_format/check_format.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/buffer1 b/test/extensions/filters/http/common/fuzz/filter_corpus/buffer1 index a1bf00f67a61c..9b8bf63c7ea9c 100644 --- a/test/extensions/filters/http/common/fuzz/filter_corpus/buffer1 +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/buffer1 @@ -15,5 +15,7 @@ data { "a" value : "b" } } - data: "hello" + http_body { + data: "hello" + } } diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/crash-3014465358f0947e73ac12ccb40b299d5b0646b3 b/test/extensions/filters/http/common/fuzz/filter_corpus/crash-3014465358f0947e73ac12ccb40b299d5b0646b3 index fb63866ea5a51..72bcfa0b0baed 100644 --- a/test/extensions/filters/http/common/fuzz/filter_corpus/crash-3014465358f0947e73ac12ccb40b299d5b0646b3 +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/crash-3014465358f0947e73ac12ccb40b299d5b0646b3 @@ -7,8 +7,10 @@ data { value: "\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\360\240\240\240\314\255" } } - data: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" - data: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + http_body { + data: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + data: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } trailers { headers { key: "6" diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 27bc32dbe9267..d54d2a36abc4b 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -58,6 +58,7 @@ "./test/common/config/version_converter_test.cc", "./test/common/grpc/codec_test.cc", "./test/common/grpc/codec_fuzz_test.cc", + "./test/extensions/filters/http/common/fuzz/uber_filter.h", ) # Files in these paths can use Protobuf::util::JsonStringToMessage From 2c93844a6fae066b1d22d4835c5d349756ce7322 Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Wed, 6 May 2020 10:03:53 -0700 Subject: [PATCH 04/13] Review comments Signed-off-by: Teju Nareddy --- .../filters/http/common/fuzz/uber_filter.cc | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index 7ec9b022b99a5..7a171ee5d4bb8 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -42,7 +42,7 @@ void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuz headers.setHost("foo.com"); } - if (data.data().empty() && !data.has_trailers()) { + if (data.body_case() == test::fuzz::HttpData::BODY_NOT_SET && !data.has_trailers()) { end_stream = true; } ENVOY_LOG_MISC(debug, "Decoding headers: {} ", data.headers().DebugString()); @@ -52,14 +52,30 @@ void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuz return; } - for (int i = 0; i < data.data().size(); i++) { - if (i == data.data().size() - 1 && !data.has_trailers()) { - end_stream = true; + if (data.has_http_body()) { + for (int i = 0; i < data.http_body().data_size(); i++) { + if (i == data.http_body().data_size() - 1 && !data.has_trailers()) { + end_stream = true; + } + Buffer::OwnedImpl buffer(data.http_body().data(i)); + ENVOY_LOG_MISC(debug, "Decoding http data: {} ", buffer.toString()); + if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { + return; + } } - Buffer::OwnedImpl buffer(data.data().Get(i)); - ENVOY_LOG_MISC(debug, "Decoding data: {} ", buffer.toString()); - if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { - return; + } else if (data.has_proto_body()) { + const std::string serialized = data.proto_body().message().SerializeAsString(); + const std::vector serialized_chunks = absl::StrSplit(serialized, absl::ByLength(data.proto_body().chuck_size())); + + for (size_t i = 0; i < serialized_chunks.size(); i++) { + if (!data.has_trailers() && i == serialized_chunks.size() - 1) { + end_stream = true; + } + Buffer::OwnedImpl buffer(serialized_chunks[i]); + ENVOY_LOG_MISC(debug, "Decoding proto http data: {} ", buffer.toString()); + if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { + return; + } } } From 36c643d7b3eccd0e887c14d74c188ffbf636eeda Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Wed, 6 May 2020 11:08:13 -0700 Subject: [PATCH 05/13] Fix corpus and any serialization Signed-off-by: Teju Nareddy --- ...h-bb74d7280823776808e881b20c0a9c87f7a2163b | 4 ++- .../http/common/fuzz/filter_corpus/grpc_json | 23 ------------- .../filter_corpus/grpc_transcoding_http_data | 24 ++++++++++++++ .../filter_corpus/grpc_transcoding_proto_data | 32 +++++++++++++++++++ .../filters/http/common/fuzz/uber_filter.cc | 10 +++--- test/fuzz/common.proto | 2 +- 6 files changed, 65 insertions(+), 30 deletions(-) delete mode 100644 test/extensions/filters/http/common/fuzz/filter_corpus/grpc_json create mode 100644 test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_http_data create mode 100644 test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_proto_data diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/crash-bb74d7280823776808e881b20c0a9c87f7a2163b b/test/extensions/filters/http/common/fuzz/filter_corpus/crash-bb74d7280823776808e881b20c0a9c87f7a2163b index 3eea853ad21eb..7c2bbfbae7f7e 100644 --- a/test/extensions/filters/http/common/fuzz/filter_corpus/crash-bb74d7280823776808e881b20c0a9c87f7a2163b +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/crash-bb74d7280823776808e881b20c0a9c87f7a2163b @@ -6,7 +6,9 @@ config { } } data { - data: "\001\000\000\t" + http_body { + data: "\001\000\000\t" + } trailers { headers { key: "0" diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_json b/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_json deleted file mode 100644 index 3846826fa9d84..0000000000000 --- a/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_json +++ /dev/null @@ -1,23 +0,0 @@ -config { -name: "envoy.filters.http.grpc_json_transcoder" -typed_config: { -} -} - -data { -headers { -headers { - key: "content-type" - value: "application/json" -} -headers { - key: ":method" - value: "POST" -} -headers { - key: ":path" - value: "/bookstore.Bookstore/CreateShelfWithPackageServiceAndMethod" -} -} -data: "{\"theme\": \"Children\"}" -} \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_http_data b/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_http_data new file mode 100644 index 0000000000000..cf0e8282a0830 --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_http_data @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.http.grpc_json_transcoder" + typed_config: {} +} + +data { + headers { + headers { + key: "content-type" + value: "application/json" + } + headers { + key: ":method" + value: "POST" + } + headers { + key: ":path" + value: "/bookstore.Bookstore/CreateShelfWithPackageServiceAndMethod" + } + } + http_body { + data: "{\"theme\": \"Children\"}" + } +} \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_proto_data b/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_proto_data new file mode 100644 index 0000000000000..711ea9f66ec5d --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/grpc_transcoding_proto_data @@ -0,0 +1,32 @@ +config { + name: "envoy.filters.http.grpc_json_transcoder" + typed_config: {} +} + +data { + headers { + headers { + key: "content-type" + value: "application/json" + } + headers { + key: ":method" + value: "POST" + } + headers { + key: ":path" + value: "/bookstore.Bookstore/CreateShelf" + } + } + proto_body { + message { + [type.googleapis.com/bookstore.CreateShelfRequest] { + shelf: { + id: 32 + theme: "Children" + } + } + } + chunk_size: 3 + } +} \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index 7a171ee5d4bb8..080e9e0d7a1cf 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -45,7 +45,7 @@ void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuz if (data.body_case() == test::fuzz::HttpData::BODY_NOT_SET && !data.has_trailers()) { end_stream = true; } - ENVOY_LOG_MISC(debug, "Decoding headers: {} ", data.headers().DebugString()); + ENVOY_LOG_MISC(debug, "Decoding headers (end_stream={}): {} ", end_stream, data.headers().DebugString()); const auto& headersStatus = filter->decodeHeaders(headers, end_stream); if (headersStatus != Http::FilterHeadersStatus::Continue && headersStatus != Http::FilterHeadersStatus::StopIteration) { @@ -58,21 +58,21 @@ void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuz end_stream = true; } Buffer::OwnedImpl buffer(data.http_body().data(i)); - ENVOY_LOG_MISC(debug, "Decoding http data: {} ", buffer.toString()); + ENVOY_LOG_MISC(debug, "Decoding http data (end_stream={}): {} ", end_stream, buffer.toString()); if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { return; } } } else if (data.has_proto_body()) { - const std::string serialized = data.proto_body().message().SerializeAsString(); - const std::vector serialized_chunks = absl::StrSplit(serialized, absl::ByLength(data.proto_body().chuck_size())); + const std::string serialized = data.proto_body().message().value(); + const std::vector serialized_chunks = absl::StrSplit(serialized, absl::ByLength(data.proto_body().chunk_size())); for (size_t i = 0; i < serialized_chunks.size(); i++) { if (!data.has_trailers() && i == serialized_chunks.size() - 1) { end_stream = true; } Buffer::OwnedImpl buffer(serialized_chunks[i]); - ENVOY_LOG_MISC(debug, "Decoding proto http data: {} ", buffer.toString()); + ENVOY_LOG_MISC(debug, "Decoding serialized proto data (end_stream={}): {} ", end_stream, buffer.toString()); if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { return; } diff --git a/test/fuzz/common.proto b/test/fuzz/common.proto index c45521a766a8d..92df9a1b40219 100644 --- a/test/fuzz/common.proto +++ b/test/fuzz/common.proto @@ -28,7 +28,7 @@ message ProtoBody { google.protobuf.Any message = 1 [(validate.rules).any.required = true]; // The size (in bytes) of each buffer when forming the requests. - uint64 chuck_size = 2 [(validate.rules).uint64.gt = 0]; + uint64 chunk_size = 2 [(validate.rules).uint64.gt = 0]; } message HttpData { From 07f9a9fde83fbd9586d6f897be59596b55deb66b Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Wed, 6 May 2020 11:57:06 -0700 Subject: [PATCH 06/13] Format Signed-off-by: Teju Nareddy --- .../filters/http/common/fuzz/uber_filter.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index 080e9e0d7a1cf..ade5b377ce06c 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -45,7 +45,8 @@ void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuz if (data.body_case() == test::fuzz::HttpData::BODY_NOT_SET && !data.has_trailers()) { end_stream = true; } - ENVOY_LOG_MISC(debug, "Decoding headers (end_stream={}): {} ", end_stream, data.headers().DebugString()); + ENVOY_LOG_MISC(debug, "Decoding headers (end_stream={}): {} ", end_stream, + data.headers().DebugString()); const auto& headersStatus = filter->decodeHeaders(headers, end_stream); if (headersStatus != Http::FilterHeadersStatus::Continue && headersStatus != Http::FilterHeadersStatus::StopIteration) { @@ -58,21 +59,24 @@ void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuz end_stream = true; } Buffer::OwnedImpl buffer(data.http_body().data(i)); - ENVOY_LOG_MISC(debug, "Decoding http data (end_stream={}): {} ", end_stream, buffer.toString()); + ENVOY_LOG_MISC(debug, "Decoding http data (end_stream={}): {} ", end_stream, + buffer.toString()); if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { return; } } } else if (data.has_proto_body()) { const std::string serialized = data.proto_body().message().value(); - const std::vector serialized_chunks = absl::StrSplit(serialized, absl::ByLength(data.proto_body().chunk_size())); + const std::vector serialized_chunks = + absl::StrSplit(serialized, absl::ByLength(data.proto_body().chunk_size())); for (size_t i = 0; i < serialized_chunks.size(); i++) { if (!data.has_trailers() && i == serialized_chunks.size() - 1) { end_stream = true; } Buffer::OwnedImpl buffer(serialized_chunks[i]); - ENVOY_LOG_MISC(debug, "Decoding serialized proto data (end_stream={}): {} ", end_stream, buffer.toString()); + ENVOY_LOG_MISC(debug, "Decoding serialized proto data (end_stream={}): {} ", end_stream, + buffer.toString()); if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { return; } From 9c469dcc2159cdde2c0b2509bb136eb2007c4a9c Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Wed, 6 May 2020 16:16:51 -0700 Subject: [PATCH 07/13] Implement any guidance Signed-off-by: Teju Nareddy --- .../http/common/fuzz/filter_fuzz_test.cc | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc index 8a20604d7a03f..2309a89137da9 100644 --- a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc +++ b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc @@ -36,6 +36,36 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::http::FilterFuzzTestCase& i input->mutable_config()->mutable_typed_config()->set_type_url( absl::StrCat("type.googleapis.com/", factory->createEmptyConfigProto()->GetDescriptor()->full_name())); + + // For fuzzing proto data, guide the mutator to useful 'Any' types half + // the time. The other half the time, let the fuzzing engine choose + // any message to serialize. + if (seed % 2 == 0 && input->data().has_proto_body()) { + // These types are request/response from the test Bookstore service + // for the gRPC Transcoding filter. + static const std::vector expected_types = { + "type.googleapis.com/bookstore.ListShelvesResponse", + "type.googleapis.com/bookstore.CreateShelfRequest", + "type.googleapis.com/bookstore.GetShelfRequest", + "type.googleapis.com/bookstore.DeleteShelfRequest", + "type.googleapis.com/bookstore.ListBooksRequest", + "type.googleapis.com/bookstore.CreateBookRequest", + "type.googleapis.com/bookstore.GetBookRequest", + "type.googleapis.com/bookstore.UpdateBookRequest", + "type.googleapis.com/bookstore.DeleteBookRequest", + "type.googleapis.com/bookstore.GetAuthorRequest", + "type.googleapis.com/bookstore.EchoBodyRequest", + "type.googleapis.com/bookstore.EchoStructReqResp", + "type.googleapis.com/bookstore.Shelf", + "type.googleapis.com/bookstore.Book", + "type.googleapis.com/google.protobuf.Empty", + "type.googleapis.com/google.api.HttpBody", + }; + ProtobufWkt::Any* mutable_any = + input->mutable_data()->mutable_proto_body()->mutable_message(); + const std::string& type_url = expected_types[(seed / 2) % expected_types.size()]; + mutable_any->set_type_url(type_url); + } }}; try { From 248db5fecdff237dc691b15edc0438eebe0a3ea4 Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Tue, 12 May 2020 10:59:41 -0700 Subject: [PATCH 08/13] Review comments Signed-off-by: Teju Nareddy --- .../http/common/fuzz/filter_fuzz_test.cc | 25 +------ .../filters/http/common/fuzz/uber_filter.cc | 75 ++++++++++++------- .../filters/http/common/fuzz/uber_filter.h | 5 ++ 3 files changed, 55 insertions(+), 50 deletions(-) diff --git a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc index 2309a89137da9..246388f7fa582 100644 --- a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc +++ b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc @@ -41,30 +41,7 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::http::FilterFuzzTestCase& i // the time. The other half the time, let the fuzzing engine choose // any message to serialize. if (seed % 2 == 0 && input->data().has_proto_body()) { - // These types are request/response from the test Bookstore service - // for the gRPC Transcoding filter. - static const std::vector expected_types = { - "type.googleapis.com/bookstore.ListShelvesResponse", - "type.googleapis.com/bookstore.CreateShelfRequest", - "type.googleapis.com/bookstore.GetShelfRequest", - "type.googleapis.com/bookstore.DeleteShelfRequest", - "type.googleapis.com/bookstore.ListBooksRequest", - "type.googleapis.com/bookstore.CreateBookRequest", - "type.googleapis.com/bookstore.GetBookRequest", - "type.googleapis.com/bookstore.UpdateBookRequest", - "type.googleapis.com/bookstore.DeleteBookRequest", - "type.googleapis.com/bookstore.GetAuthorRequest", - "type.googleapis.com/bookstore.EchoBodyRequest", - "type.googleapis.com/bookstore.EchoStructReqResp", - "type.googleapis.com/bookstore.Shelf", - "type.googleapis.com/bookstore.Book", - "type.googleapis.com/google.protobuf.Empty", - "type.googleapis.com/google.api.HttpBody", - }; - ProtobufWkt::Any* mutable_any = - input->mutable_data()->mutable_proto_body()->mutable_message(); - const std::string& type_url = expected_types[(seed / 2) % expected_types.size()]; - mutable_any->set_type_url(type_url); + UberFilterFuzzer::guideAnyProtoType(input->mutable_data(), seed); } }}; diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index ade5b377ce06c..e1f4215c892b2 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -28,6 +28,21 @@ UberFilterFuzzer::UberFilterFuzzer() { perFilterSetup(); } +std::vector UberFilterFuzzer::parseHttpData(const test::fuzz::HttpData& data) { + std::vector data_chunks; + + if (data.has_http_body()) { + for (const auto& http_data : data.http_body().data()) { + data_chunks.push_back(http_data.data()); + } + } else if (data.has_proto_body()) { + const std::string serialized = data.proto_body().message().value(); + data_chunks = absl::StrSplit(serialized, absl::ByLength(data.proto_body().chunk_size())); + } + + return data_chunks; +} + void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuzz::HttpData& data) { bool end_stream = false; @@ -53,33 +68,15 @@ void UberFilterFuzzer::decode(Http::StreamDecoderFilter* filter, const test::fuz return; } - if (data.has_http_body()) { - for (int i = 0; i < data.http_body().data_size(); i++) { - if (i == data.http_body().data_size() - 1 && !data.has_trailers()) { - end_stream = true; - } - Buffer::OwnedImpl buffer(data.http_body().data(i)); - ENVOY_LOG_MISC(debug, "Decoding http data (end_stream={}): {} ", end_stream, - buffer.toString()); - if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { - return; - } + const std::vector data_chunks = parseHttpData(data); + for (size_t i = 0; i < data_chunks.size(); i++) { + if (!data.has_trailers() && i == data_chunks.size() - 1) { + end_stream = true; } - } else if (data.has_proto_body()) { - const std::string serialized = data.proto_body().message().value(); - const std::vector serialized_chunks = - absl::StrSplit(serialized, absl::ByLength(data.proto_body().chunk_size())); - - for (size_t i = 0; i < serialized_chunks.size(); i++) { - if (!data.has_trailers() && i == serialized_chunks.size() - 1) { - end_stream = true; - } - Buffer::OwnedImpl buffer(serialized_chunks[i]); - ENVOY_LOG_MISC(debug, "Decoding serialized proto data (end_stream={}): {} ", end_stream, - buffer.toString()); - if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { - return; - } + Buffer::OwnedImpl buffer(data_chunks[i]); + ENVOY_LOG_MISC(debug, "Decoding data (end_stream={}): {} ", end_stream, buffer.toString()); + if (filter->decodeData(buffer, end_stream) != Http::FilterDataStatus::Continue) { + return; } } @@ -114,6 +111,32 @@ void UberFilterFuzzer::fuzz( reset(); } +void UberFilterFuzzer::guideAnyProtoType(test::fuzz::HttpData* mutable_data, int seed) { + // These types are request/response from the test Bookstore service + // for the gRPC Transcoding filter. + static const std::vector expected_types = { + "type.googleapis.com/bookstore.ListShelvesResponse", + "type.googleapis.com/bookstore.CreateShelfRequest", + "type.googleapis.com/bookstore.GetShelfRequest", + "type.googleapis.com/bookstore.DeleteShelfRequest", + "type.googleapis.com/bookstore.ListBooksRequest", + "type.googleapis.com/bookstore.CreateBookRequest", + "type.googleapis.com/bookstore.GetBookRequest", + "type.googleapis.com/bookstore.UpdateBookRequest", + "type.googleapis.com/bookstore.DeleteBookRequest", + "type.googleapis.com/bookstore.GetAuthorRequest", + "type.googleapis.com/bookstore.EchoBodyRequest", + "type.googleapis.com/bookstore.EchoStructReqResp", + "type.googleapis.com/bookstore.Shelf", + "type.googleapis.com/bookstore.Book", + "type.googleapis.com/google.protobuf.Empty", + "type.googleapis.com/google.api.HttpBody", + }; + ProtobufWkt::Any* mutable_any = mutable_data->mutable_proto_body()->mutable_message(); + const std::string& type_url = expected_types[(seed / 2) % expected_types.size()]; + mutable_any->set_type_url(type_url); +} + void UberFilterFuzzer::reset() { if (filter_ != nullptr) { filter_->onDestroy(); diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.h b/test/extensions/filters/http/common/fuzz/uber_filter.h index a18d1ae8057ba..57e033076556c 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.h +++ b/test/extensions/filters/http/common/fuzz/uber_filter.h @@ -16,6 +16,9 @@ class UberFilterFuzzer { proto_config, const test::fuzz::HttpData& data); + // For fuzzing proto data, guide the mutator to useful 'Any' types. + static void guideAnyProtoType(test::fuzz::HttpData* mutable_data, int seed); + protected: // Set-up filter specific mock expectations in constructor. void perFilterSetup(); @@ -23,7 +26,9 @@ class UberFilterFuzzer { void cleanFuzzedConfig(absl::string_view filter_name, Protobuf::Message* message); // This executes the decode methods to be fuzzed. + std::vector parseHttpData(const test::fuzz::HttpData& data); void decode(Http::StreamDecoderFilter* filter, const test::fuzz::HttpData& data); + void reset(); private: From c8de25c0f4c905b6717790b2979cc21cd53f6311 Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Tue, 12 May 2020 11:04:40 -0700 Subject: [PATCH 09/13] Fix comment Signed-off-by: Teju Nareddy --- test/extensions/filters/http/common/fuzz/uber_filter.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.h b/test/extensions/filters/http/common/fuzz/uber_filter.h index 57e033076556c..fc4b15ce55f51 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.h +++ b/test/extensions/filters/http/common/fuzz/uber_filter.h @@ -25,8 +25,10 @@ class UberFilterFuzzer { // Filter specific input cleanup. void cleanFuzzedConfig(absl::string_view filter_name, Protobuf::Message* message); - // This executes the decode methods to be fuzzed. + // Parses http or proto body into chunks. std::vector parseHttpData(const test::fuzz::HttpData& data); + + // This executes the decode methods to be fuzzed. void decode(Http::StreamDecoderFilter* filter, const test::fuzz::HttpData& data); void reset(); From 28a058195af2691cc5b5eda4c7adba97342c8d0a Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Tue, 12 May 2020 12:34:24 -0700 Subject: [PATCH 10/13] Review comments Signed-off-by: Teju Nareddy --- .../http/common/fuzz/filter_fuzz_test.cc | 2 +- .../filters/http/common/fuzz/uber_filter.cc | 26 ------------------- .../filters/http/common/fuzz/uber_filter.h | 2 +- .../http/common/fuzz/uber_per_filter.cc | 26 +++++++++++++++++++ 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc index 246388f7fa582..edfa89f917c77 100644 --- a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc +++ b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc @@ -41,7 +41,7 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::http::FilterFuzzTestCase& i // the time. The other half the time, let the fuzzing engine choose // any message to serialize. if (seed % 2 == 0 && input->data().has_proto_body()) { - UberFilterFuzzer::guideAnyProtoType(input->mutable_data(), seed); + UberFilterFuzzer::guideAnyProtoType(input->mutable_data(), seed / 2); } }}; diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index e1f4215c892b2..508bd18ae390f 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -111,32 +111,6 @@ void UberFilterFuzzer::fuzz( reset(); } -void UberFilterFuzzer::guideAnyProtoType(test::fuzz::HttpData* mutable_data, int seed) { - // These types are request/response from the test Bookstore service - // for the gRPC Transcoding filter. - static const std::vector expected_types = { - "type.googleapis.com/bookstore.ListShelvesResponse", - "type.googleapis.com/bookstore.CreateShelfRequest", - "type.googleapis.com/bookstore.GetShelfRequest", - "type.googleapis.com/bookstore.DeleteShelfRequest", - "type.googleapis.com/bookstore.ListBooksRequest", - "type.googleapis.com/bookstore.CreateBookRequest", - "type.googleapis.com/bookstore.GetBookRequest", - "type.googleapis.com/bookstore.UpdateBookRequest", - "type.googleapis.com/bookstore.DeleteBookRequest", - "type.googleapis.com/bookstore.GetAuthorRequest", - "type.googleapis.com/bookstore.EchoBodyRequest", - "type.googleapis.com/bookstore.EchoStructReqResp", - "type.googleapis.com/bookstore.Shelf", - "type.googleapis.com/bookstore.Book", - "type.googleapis.com/google.protobuf.Empty", - "type.googleapis.com/google.api.HttpBody", - }; - ProtobufWkt::Any* mutable_any = mutable_data->mutable_proto_body()->mutable_message(); - const std::string& type_url = expected_types[(seed / 2) % expected_types.size()]; - mutable_any->set_type_url(type_url); -} - void UberFilterFuzzer::reset() { if (filter_ != nullptr) { filter_->onDestroy(); diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.h b/test/extensions/filters/http/common/fuzz/uber_filter.h index fc4b15ce55f51..511c587a6e625 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.h +++ b/test/extensions/filters/http/common/fuzz/uber_filter.h @@ -17,7 +17,7 @@ class UberFilterFuzzer { const test::fuzz::HttpData& data); // For fuzzing proto data, guide the mutator to useful 'Any' types. - static void guideAnyProtoType(test::fuzz::HttpData* mutable_data, int seed); + static void guideAnyProtoType(test::fuzz::HttpData* mutable_data, uint choice); protected: // Set-up filter specific mock expectations in constructor. diff --git a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc index 55ba9d253eae8..89b34da13a916 100644 --- a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc @@ -44,6 +44,32 @@ void addBookstoreProtoDescriptor(Protobuf::Message* message) { } } // namespace +void UberFilterFuzzer::guideAnyProtoType(test::fuzz::HttpData* mutable_data, uint choice) { + // These types are request/response from the test Bookstore service + // for the gRPC Transcoding filter. + static const std::vector expected_types = { + "type.googleapis.com/bookstore.ListShelvesResponse", + "type.googleapis.com/bookstore.CreateShelfRequest", + "type.googleapis.com/bookstore.GetShelfRequest", + "type.googleapis.com/bookstore.DeleteShelfRequest", + "type.googleapis.com/bookstore.ListBooksRequest", + "type.googleapis.com/bookstore.CreateBookRequest", + "type.googleapis.com/bookstore.GetBookRequest", + "type.googleapis.com/bookstore.UpdateBookRequest", + "type.googleapis.com/bookstore.DeleteBookRequest", + "type.googleapis.com/bookstore.GetAuthorRequest", + "type.googleapis.com/bookstore.EchoBodyRequest", + "type.googleapis.com/bookstore.EchoStructReqResp", + "type.googleapis.com/bookstore.Shelf", + "type.googleapis.com/bookstore.Book", + "type.googleapis.com/google.protobuf.Empty", + "type.googleapis.com/google.api.HttpBody", + }; + ProtobufWkt::Any* mutable_any = mutable_data->mutable_proto_body()->mutable_message(); + const std::string& type_url = expected_types[(choice / 2) % expected_types.size()]; + mutable_any->set_type_url(type_url); +} + void UberFilterFuzzer::cleanFuzzedConfig(absl::string_view filter_name, Protobuf::Message* message) { // Map filter name to clean-up function. From 0836ca598a5cf7481eb8b42106e32dba8b0a3878 Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Tue, 12 May 2020 12:36:16 -0700 Subject: [PATCH 11/13] Fix clang tidy Signed-off-by: Teju Nareddy --- test/extensions/filters/http/common/fuzz/uber_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index 508bd18ae390f..c6c0ead5401d1 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -33,7 +33,7 @@ std::vector UberFilterFuzzer::parseHttpData(const test::fuzz::HttpD if (data.has_http_body()) { for (const auto& http_data : data.http_body().data()) { - data_chunks.push_back(http_data.data()); + data_chunks.push_back(http_data); } } else if (data.has_proto_body()) { const std::string serialized = data.proto_body().message().value(); From 7e553ade66d46efeb5b67a63a66065b5a713179c Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Tue, 12 May 2020 12:39:53 -0700 Subject: [PATCH 12/13] Fix clang tidy Signed-off-by: Teju Nareddy --- test/extensions/filters/http/common/fuzz/uber_filter.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index c6c0ead5401d1..a88cc585a72ef 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -32,7 +32,8 @@ std::vector UberFilterFuzzer::parseHttpData(const test::fuzz::HttpD std::vector data_chunks; if (data.has_http_body()) { - for (const auto& http_data : data.http_body().data()) { + data_chunks.reserve(data.http_body().data_size()); + for (const std::string& http_data : data.http_body().data()) { data_chunks.push_back(http_data); } } else if (data.has_proto_body()) { From 3c31248e76c45c98eeaf518dfb9932e1fa5fdf9c Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Tue, 12 May 2020 13:35:03 -0700 Subject: [PATCH 13/13] Fix index Signed-off-by: Teju Nareddy --- test/extensions/filters/http/common/fuzz/uber_per_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc index 89b34da13a916..353eea56f0bee 100644 --- a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc @@ -66,7 +66,7 @@ void UberFilterFuzzer::guideAnyProtoType(test::fuzz::HttpData* mutable_data, uin "type.googleapis.com/google.api.HttpBody", }; ProtobufWkt::Any* mutable_any = mutable_data->mutable_proto_body()->mutable_message(); - const std::string& type_url = expected_types[(choice / 2) % expected_types.size()]; + const std::string& type_url = expected_types[choice % expected_types.size()]; mutable_any->set_type_url(type_url); }