diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index 6bb2baf6dbd6e..b82d3130563cd 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -258,14 +258,14 @@ def _proto_header(proto_path): return None # Envoy proto targets should be specified with this function. -def envoy_proto_library(name, srcs = [], deps = []): +def envoy_proto_library(name, srcs = [], deps = [], external_deps = []): internal_name = name + "_internal" cc_proto_library( name = internal_name, srcs = srcs, default_runtime = "//external:protobuf", protoc = "//external:protoc", - deps = deps, + deps = deps + [envoy_external_dep_path(dep) for dep in external_deps], linkstatic = 1, ) # We can't use include_prefix directly in cc_proto_library, since it @@ -278,3 +278,30 @@ def envoy_proto_library(name, srcs = [], deps = []): deps = [internal_name], linkstatic = 1, ) + +# Envoy proto descriptor targets should be specified with this function. +# This is used for testing only. +def envoy_proto_descriptor(name, out, srcs = [], protocopts = [], external_deps = []): + input_files = ["$(location " + src + ")" for src in srcs] + include_paths = [".", PACKAGE_NAME] + + if "http_api_protos" in external_deps: + srcs.append("@googleapis//:http_api_protos_src") + include_paths.append("external/googleapis") + + if "well_known_protos" in external_deps: + srcs.append("@protobuf_bzl//:well_known_protos") + include_paths.append("external/protobuf_bzl/src") + + options = protocopts[:] + options.extend(["-I" + include_path for include_path in include_paths]) + options.append("--descriptor_set_out=$@") + + cmd = "$(location //external:protoc) " + " ".join(options + input_files) + native.genrule( + name = name, + srcs = srcs, + outs = [out], + cmd = cmd, + tools = ["//external:protoc"], + ) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 0a29cea8f87c0..f769d21380bc9 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -116,6 +116,14 @@ def envoy_api_deps(skip_targets): name = "envoy_eds", actual = "@envoy_api//api:eds", ) + native.bind( + name = "http_api_protos", + actual = "@googleapis//:http_api_protos", + ) + native.bind( + name = "http_api_protos_genproto", + actual = "@googleapis//:http_api_protos_genproto", + ) def envoy_dependencies(path = "@envoy_deps//", skip_protobuf_bzl = False, skip_targets = []): if not skip_protobuf_bzl: diff --git a/docs/configuration/http_filters/grpc_json_transcoder_filter.rst b/docs/configuration/http_filters/grpc_json_transcoder_filter.rst new file mode 100644 index 0000000000000..60ad09c065cb4 --- /dev/null +++ b/docs/configuration/http_filters/grpc_json_transcoder_filter.rst @@ -0,0 +1,56 @@ +.. _config_http_filters_grpc_json_transcoder: + +gRPC-JSON transcoder filter +=========================== + +gRPC :ref:`architecture overview `. + +This is a filter which allows a RESTful JSON API client to send requests to Envoy over HTTP +and get proxied to a gRPC service. The HTTP mapping for the gRPC service has to be defined by +`custom options `_. + +Configure gRPC-JSON transcoder +------------------------------ + +The filter config for the filter requires the descriptor file as well as a list of the gRPC +services to be transcoded. + +.. code-block:: json + + { + "type": "both", + "name": "grpc_json_transcoder", + "config": { + "proto_descriptors": "proto.pb", + "services": ["grpc.service.Service"] + } + } + +proto_descriptors + *(required, string)* Supplies the binary protobuf descriptor set for the gRPC services. + The descriptor set has to include all of the types that are used in the services. Make sure + to use the ``--include_import`` option for ``protoc``. + + To generate a protobuf descriptor set for the gRPC service, you'll also need to clone the + googleapis repository from Github before running protoc, as you'll need annotations.proto + in your include path. + + .. code-block:: bash + + git clone https://github.com/googleapis/googleapis + GOOGLEAPIS_DIR= + + Then run protoc to generate the descriptor set from bookstore.proto: + + .. code-block:: bash + + protoc -I$(GOOGLEAPIS_DIR) -I. --include_imports --include_source_info \ + --descriptor_set_out=proto.pb test/proto/bookstore.proto + + If you have more than one proto source files, you can pass all of them in one command. + +services + *(required, array)* A list of strings that supplies the service names that the + transcoder will translate. If the service name doesn't exist in ``proto_descriptors``, Envoy + will fail at startup. The ``proto_descriptors`` may contain more services than the service names + specified here, but they won't be translated. diff --git a/docs/configuration/http_filters/http_filters.rst b/docs/configuration/http_filters/http_filters.rst index 8834b1cbbeb3a..2aa840a4fd4b6 100644 --- a/docs/configuration/http_filters/http_filters.rst +++ b/docs/configuration/http_filters/http_filters.rst @@ -10,6 +10,7 @@ HTTP filters fault_filter dynamodb_filter grpc_http1_bridge_filter + grpc_json_transcoder_filter grpc_web_filter health_check_filter ip_tagging_filter diff --git a/docs/intro/arch_overview/grpc.rst b/docs/intro/arch_overview/grpc.rst index 4ef1dc600a474..4915dd116f640 100644 --- a/docs/intro/arch_overview/grpc.rst +++ b/docs/intro/arch_overview/grpc.rst @@ -21,3 +21,6 @@ application layer: client to send requests to Envoy over HTTP/1.1 and get proxied to a gRPC server. It's under active development and is expected to be the successor to the gRPC :ref:`bridge filter `. +* gRPC-JSON transcoder is supported by a :ref:`filter ` + that allows a RESTful JSON API client to send requests to Envoy over HTTP and get proxied to a + gRPC service. diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index e115046199d30..51a85c9b35199 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -71,6 +71,25 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "json_transcoder_filter_lib", + srcs = ["json_transcoder_filter.cc"], + hdrs = ["json_transcoder_filter.h"], + external_deps = [ + "path_matcher", + "grpc_transcoding", + "http_api_protos", + ], + deps = [ + ":codec_lib", + ":common_lib", + ":transcoder_input_stream_lib", + "//include/envoy/http:filter_interface", + "//source/common/common:base64_lib", + "//source/common/http:headers_lib", + ], +) + envoy_cc_library( name = "grpc_web_filter_lib", srcs = ["grpc_web_filter.cc"], diff --git a/source/common/grpc/json_transcoder_filter.cc b/source/common/grpc/json_transcoder_filter.cc new file mode 100644 index 0000000000000..ee9d7bf9249d8 --- /dev/null +++ b/source/common/grpc/json_transcoder_filter.cc @@ -0,0 +1,393 @@ +#include "common/grpc/json_transcoder_filter.h" + +#include "envoy/common/exception.h" +#include "envoy/http/filter.h" + +#include "common/common/assert.h" +#include "common/common/enum_to_int.h" +#include "common/common/utility.h" +#include "common/filesystem/filesystem_impl.h" +#include "common/http/headers.h" +#include "common/http/utility.h" + +#include "google/api/annotations.pb.h" +#include "google/api/http.pb.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "google/protobuf/util/type_resolver.h" +#include "google/protobuf/util/type_resolver_util.h" +#include "grpc_transcoding/json_request_translator.h" +#include "grpc_transcoding/response_to_json_translator.h" + +using google::grpc::transcoding::JsonRequestTranslator; +using google::grpc::transcoding::RequestInfo; +using google::grpc::transcoding::ResponseToJsonTranslator; +using google::grpc::transcoding::Transcoder; +using google::grpc::transcoding::TranscoderInputStream; +using google::protobuf::DescriptorPool; +using google::protobuf::FileDescriptor; +using google::protobuf::FileDescriptorSet; +using google::protobuf::io::ZeroCopyInputStream; +using google::protobuf::util::error::Code; +using google::protobuf::util::Status; + +namespace Envoy { +namespace Grpc { + +namespace { + +const std::string TYPE_URL_PREFIX{"type.googleapis.com"}; + +// Transcoder: +// https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/blob/master/src/include/grpc_transcoding/transcoder.h +// implementation based on JsonRequestTranslator & ResponseToJsonTranslator +class TranscoderImpl : public Transcoder { +public: + /** + * Construct a transcoder implementation + * @param request_translator a JsonRequestTranslator that does the request translation + * @param response_translator a ResponseToJsonTranslator that does the response translation + */ + TranscoderImpl(std::unique_ptr request_translator, + std::unique_ptr response_translator) + : request_translator_(std::move(request_translator)), + response_translator_(std::move(response_translator)), + request_stream_(request_translator_->Output().CreateInputStream()), + response_stream_(response_translator_->CreateInputStream()) {} + + // Transcoder + ::google::grpc::transcoding::TranscoderInputStream* RequestOutput() { + return request_stream_.get(); + } + Status RequestStatus() { return request_translator_->Output().Status(); } + + ZeroCopyInputStream* ResponseOutput() { return response_stream_.get(); } + Status ResponseStatus() { return response_translator_->Status(); } + +private: + std::unique_ptr request_translator_; + std::unique_ptr response_translator_; + std::unique_ptr request_stream_; + std::unique_ptr response_stream_; +}; + +} // namespace + +JsonTranscoderConfig::JsonTranscoderConfig(const Json::Object& config) { + const std::string proto_descriptor_file = config.getString("proto_descriptor"); + FileDescriptorSet descriptor_set; + if (!descriptor_set.ParseFromString(Filesystem::fileReadToEnd(proto_descriptor_file))) { + throw EnvoyException("transcoding_filter: Unable to parse proto descriptor"); + } + + for (const auto& file : descriptor_set.file()) { + if (descriptor_pool_.BuildFile(file) == nullptr) { + throw EnvoyException("transcoding_filter: Unable to build proto descriptor pool"); + } + } + + // TODO(lizan): Consider factor out building PathMatcher building. + google::grpc::transcoding::PathMatcherBuilder pmb; + + for (const auto& service_name : config.getStringArray("services")) { + auto service = descriptor_pool_.FindServiceByName(service_name); + if (service == nullptr) { + throw EnvoyException("transcoding_filter: Could not find '" + service_name + + "' in the proto descriptor"); + } + for (int i = 0; i < service->method_count(); ++i) { + auto method = service->method(i); + + auto http_rule = method->options().GetExtension(google::api::http); + + switch (http_rule.pattern_case()) { + case ::google::api::HttpRule::kGet: + pmb.Register("GET", http_rule.get(), http_rule.body(), method); + break; + case ::google::api::HttpRule::kPut: + pmb.Register("PUT", http_rule.put(), http_rule.body(), method); + break; + case ::google::api::HttpRule::kPost: + pmb.Register("POST", http_rule.post(), http_rule.body(), method); + break; + case ::google::api::HttpRule::kDelete: + pmb.Register("DELETE", http_rule.delete_(), http_rule.body(), method); + break; + case ::google::api::HttpRule::kPatch: + pmb.Register("PATCH", http_rule.patch(), http_rule.body(), method); + break; + case ::google::api::HttpRule::kCustom: + pmb.Register(http_rule.custom().kind(), http_rule.custom().path(), http_rule.body(), + method); + break; + default: // ::google::api::HttpRule::PATTEN_NOT_SET + break; + } + } + } + + path_matcher_ = pmb.Build(); + + type_helper_.reset(new google::grpc::transcoding::TypeHelper( + google::protobuf::util::NewTypeResolverForDescriptorPool(TYPE_URL_PREFIX, + &descriptor_pool_))); +} + +Status JsonTranscoderConfig::createTranscoder( + const Http::HeaderMap& headers, ZeroCopyInputStream& request_input, + google::grpc::transcoding::TranscoderInputStream& response_input, + std::unique_ptr& transcoder, + const google::protobuf::MethodDescriptor*& method_descriptor) { + const std::string method = headers.Method()->value().c_str(); + std::string path = headers.Path()->value().c_str(); + std::string args; + + const size_t pos = path.find('?'); + if (pos != std::string::npos) { + args = path.substr(pos + 1); + path = path.substr(0, pos); + } + + RequestInfo request_info; + std::vector variable_bindings; + method_descriptor = + path_matcher_->Lookup(method, path, args, &variable_bindings, &request_info.body_field_path); + if (!method_descriptor) { + return Status(Code::NOT_FOUND, "Could not resolve " + path + " to a method"); + } + + Status status = methodToRequestInfo(method_descriptor, &request_info); + if (!status.ok()) { + return status; + } + + for (const auto& binding : variable_bindings) { + google::grpc::transcoding::RequestWeaver::BindingInfo resolved_binding; + status = type_helper_->ResolveFieldPath(*request_info.message_type, binding.field_path, + &resolved_binding.field_path); + if (!status.ok()) { + return status; + } + + resolved_binding.value = binding.value; + + request_info.variable_bindings.emplace_back(std::move(resolved_binding)); + } + + std::unique_ptr request_translator{ + new JsonRequestTranslator(type_helper_->Resolver(), &request_input, request_info, + method_descriptor->client_streaming(), true)}; + + const auto response_type_url = + TYPE_URL_PREFIX + "/" + method_descriptor->output_type()->full_name(); + std::unique_ptr response_translator{ + new ResponseToJsonTranslator(type_helper_->Resolver(), response_type_url, + method_descriptor->server_streaming(), &response_input)}; + + transcoder.reset( + new TranscoderImpl(std::move(request_translator), std::move(response_translator))); + return Status::OK; +} + +Status JsonTranscoderConfig::methodToRequestInfo(const google::protobuf::MethodDescriptor* method, + google::grpc::transcoding::RequestInfo* info) { + auto request_type_url = TYPE_URL_PREFIX + "/" + method->input_type()->full_name(); + info->message_type = type_helper_->Info()->GetTypeByTypeUrl(request_type_url); + if (info->message_type == nullptr) { + ENVOY_LOG(debug, "Cannot resolve input-type: {}", method->input_type()->full_name()); + return Status(Code::NOT_FOUND, "Could not resolve type: " + method->input_type()->full_name()); + } + + return Status::OK; +} + +JsonTranscoderFilter::JsonTranscoderFilter(JsonTranscoderConfig& config) : config_(config) {} + +Http::FilterHeadersStatus JsonTranscoderFilter::decodeHeaders(Http::HeaderMap& headers, + bool end_stream) { + const auto status = + config_.createTranscoder(headers, request_in_, response_in_, transcoder_, method_); + + if (!status.ok()) { + // If transcoder couldn't be created, it might be a normal gRPC request, so the filter will + // just pass-through the request to upstream. + return Http::FilterHeadersStatus::Continue; + } + + headers.removeContentLength(); + headers.insertContentType().value(Http::Headers::get().ContentTypeValues.Grpc); + headers.insertPath().value("/" + method_->service()->full_name() + "/" + method_->name()); + + headers.insertMethod().value(Http::Headers::get().MethodValues.Post); + + headers.insertTE().value(Http::Headers::get().TEValues.Trailers); + + if (end_stream) { + request_in_.finish(); + + const auto& request_status = transcoder_->RequestStatus(); + if (!request_status.ok()) { + ENVOY_LOG(debug, "Transcoding request error " + request_status.ToString()); + error_ = true; + Http::Utility::sendLocalReply(*decoder_callbacks_, Http::Code::BadRequest, + request_status.error_message().ToString()); + + return Http::FilterHeadersStatus::StopIteration; + } + + Buffer::OwnedImpl data; + readToBuffer(*transcoder_->RequestOutput(), data); + + if (data.length() > 0) { + decoder_callbacks_->addDecodedData(data); + } + } + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterDataStatus JsonTranscoderFilter::decodeData(Buffer::Instance& data, bool end_stream) { + ASSERT(!error_); + + if (!transcoder_) { + return Http::FilterDataStatus::Continue; + } + + request_in_.move(data); + + if (end_stream) { + request_in_.finish(); + } + + readToBuffer(*transcoder_->RequestOutput(), data); + + const auto& request_status = transcoder_->RequestStatus(); + + if (!request_status.ok()) { + ENVOY_LOG(debug, "Transcoding request error " + request_status.ToString()); + error_ = true; + Http::Utility::sendLocalReply(*decoder_callbacks_, Http::Code::BadRequest, + request_status.error_message().ToString()); + + return Http::FilterDataStatus::StopIterationNoBuffer; + } + return Http::FilterDataStatus::Continue; +} + +Http::FilterTrailersStatus JsonTranscoderFilter::decodeTrailers(Http::HeaderMap&) { + ASSERT(!error_); + + if (!transcoder_) { + return Http::FilterTrailersStatus::Continue; + } + + request_in_.finish(); + + Buffer::OwnedImpl data; + readToBuffer(*transcoder_->RequestOutput(), data); + + if (data.length()) { + decoder_callbacks_->addDecodedData(data); + } + return Http::FilterTrailersStatus::Continue; +} + +void JsonTranscoderFilter::setDecoderFilterCallbacks( + Http::StreamDecoderFilterCallbacks& callbacks) { + decoder_callbacks_ = &callbacks; +} + +Http::FilterHeadersStatus JsonTranscoderFilter::encodeHeaders(Http::HeaderMap& headers, + bool end_stream) { + if (error_ || !transcoder_) { + return Http::FilterHeadersStatus::Continue; + } + + response_headers_ = &headers; + headers.insertContentType().value(Http::Headers::get().ContentTypeValues.Json); + if (!method_->server_streaming() && !end_stream) { + return Http::FilterHeadersStatus::StopIteration; + } + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterDataStatus JsonTranscoderFilter::encodeData(Buffer::Instance& data, bool end_stream) { + if (error_ || !transcoder_) { + return Http::FilterDataStatus::Continue; + } + + response_in_.move(data); + + if (end_stream) { + response_in_.finish(); + } + + readToBuffer(*transcoder_->ResponseOutput(), data); + + if (!method_->server_streaming()) { + return Http::FilterDataStatus::StopIterationAndBuffer; + } + // TODO(lizan): Check ResponseStatus + + return Http::FilterDataStatus::Continue; +} + +Http::FilterTrailersStatus JsonTranscoderFilter::encodeTrailers(Http::HeaderMap& trailers) { + if (error_ || !transcoder_) { + return Http::FilterTrailersStatus::Continue; + } + + response_in_.finish(); + + Buffer::OwnedImpl data; + readToBuffer(*transcoder_->ResponseOutput(), data); + + if (data.length()) { + encoder_callbacks_->addEncodedData(data); + } + + if (method_->server_streaming()) { + // For streaming case, the headers are already sent, so just continue here. + return Http::FilterTrailersStatus::Continue; + } + + const Http::HeaderEntry* grpc_status_header = trailers.GrpcStatus(); + if (grpc_status_header) { + uint64_t grpc_status_code; + if (!StringUtil::atoul(grpc_status_header->value().c_str(), grpc_status_code)) { + response_headers_->Status()->value(enumToInt(Http::Code::ServiceUnavailable)); + } + response_headers_->insertGrpcStatus().value(*grpc_status_header); + } + + const Http::HeaderEntry* grpc_message_header = trailers.GrpcMessage(); + if (grpc_message_header) { + response_headers_->insertGrpcMessage().value(*grpc_message_header); + } + + response_headers_->insertContentLength().value( + encoder_callbacks_->encodingBuffer() ? encoder_callbacks_->encodingBuffer()->length() : 0); + return Http::FilterTrailersStatus::Continue; +} + +void JsonTranscoderFilter::setEncoderFilterCallbacks( + Http::StreamEncoderFilterCallbacks& callbacks) { + encoder_callbacks_ = &callbacks; +} + +// TODO(lizan): Incorporate watermarks to bound buffer sizes +bool JsonTranscoderFilter::readToBuffer(google::protobuf::io::ZeroCopyInputStream& stream, + Buffer::Instance& data) { + const void* out; + int size; + while (stream.Next(&out, &size)) { + data.add(out, size); + + if (size == 0) { + return true; + } + } + return false; +} + +} // namespace Grpc +} // namespace Envoy diff --git a/source/common/grpc/json_transcoder_filter.h b/source/common/grpc/json_transcoder_filter.h new file mode 100644 index 0000000000000..009add8feb54a --- /dev/null +++ b/source/common/grpc/json_transcoder_filter.h @@ -0,0 +1,121 @@ +#pragma once + +#include "envoy/buffer/buffer.h" +#include "envoy/http/filter.h" +#include "envoy/http/header_map.h" +#include "envoy/json/json_object.h" + +#include "common/common/logger.h" +#include "common/grpc/transcoder_input_stream_impl.h" + +#include "google/protobuf/descriptor.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/util/internal/type_info.h" +#include "google/protobuf/util/type_resolver.h" +#include "grpc_transcoding/path_matcher.h" +#include "grpc_transcoding/request_message_translator.h" +#include "grpc_transcoding/transcoder.h" +#include "grpc_transcoding/type_helper.h" + +namespace Envoy { +namespace Grpc { + +/** + * VariableBinding specifies a value for a single field in the request message. + * When transcoding HTTP/REST/JSON to gRPC/proto the request message is + * constructed using the HTTP body and the variable bindings (specified through + * request url). + * See https://github.com/googleapis/googleapis/blob/master/google/api/http.proto + * for details of variable binding. + */ +struct VariableBinding { + // The location of the field in the protobuf message, where the value + // needs to be inserted, e.g. "shelf.theme" would mean the "theme" field + // of the nested "shelf" message of the request protobuf message. + std::vector field_path; + // The value to be inserted. + std::string value; +}; + +/** + * Global configuration for the gRPC JSON transcoder filter. Factory for the Transcoder interface. + */ +class JsonTranscoderConfig : public Logger::Loggable { +public: + /** + * constructor that loads protobuf descriptors from the file specified in the JSON config. + * and construct a path matcher for HTTP path bindings. + */ + JsonTranscoderConfig(const Json::Object& config); + + /** + * Create an instance of Transcoder interface based on incoming request + * @param headers headers received from decoder + * @param request_input a ZeroCopyInputStream reading from downstream request body + * @param response_input a TranscoderInputStream reading from upstream response body + * @param transcoder output parameter for the instance of Transcoder interface + * @param method_descriptor output parameter for the method looked up from config + * @return status whether the Transcoder instance are successfully created or not + */ + google::protobuf::util::Status + createTranscoder(const Http::HeaderMap& headers, + google::protobuf::io::ZeroCopyInputStream& request_input, + google::grpc::transcoding::TranscoderInputStream& response_input, + std::unique_ptr& transcoder, + const google::protobuf::MethodDescriptor*& method_descriptor); + + /** + * Convert method descriptor to RequestInfo that needed for transcoding library + */ + google::protobuf::util::Status + methodToRequestInfo(const google::protobuf::MethodDescriptor* method, + google::grpc::transcoding::RequestInfo* info); + +private: + google::protobuf::DescriptorPool descriptor_pool_; + google::grpc::transcoding::PathMatcherPtr + path_matcher_; + std::unique_ptr type_helper_; +}; + +typedef std::shared_ptr TranscodingConfigSharedPtr; + +/** + * The filter instance for gRPC JSON transcoder. + */ +class JsonTranscoderFilter : public Http::StreamFilter, public Logger::Loggable { +public: + JsonTranscoderFilter(JsonTranscoderConfig& config); + + // Http::StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap& headers, bool end_stream) override; + Http::FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus decodeTrailers(Http::HeaderMap& trailers) override; + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override; + + // Http::StreamEncoderFilter + Http::FilterHeadersStatus encodeHeaders(Http::HeaderMap& headers, bool end_stream) override; + Http::FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus encodeTrailers(Http::HeaderMap& trailers) override; + void setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callbacks) override; + + // Http::StreamFilterBase + void onDestroy() override {} + +private: + bool readToBuffer(google::protobuf::io::ZeroCopyInputStream& stream, Buffer::Instance& data); + + JsonTranscoderConfig& config_; + std::unique_ptr transcoder_; + TranscoderInputStreamImpl request_in_; + TranscoderInputStreamImpl response_in_; + Http::StreamDecoderFilterCallbacks* decoder_callbacks_{nullptr}; + Http::StreamEncoderFilterCallbacks* encoder_callbacks_{nullptr}; + const google::protobuf::MethodDescriptor* method_{nullptr}; + Http::HeaderMap* response_headers_{nullptr}; + + bool error_{false}; +}; + +} // namespace Grpc +} // namespace Envoy diff --git a/source/common/http/headers.h b/source/common/http/headers.h index 6d255ba0bac5b..941e29122d872 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -81,6 +81,7 @@ class HeaderValues { const std::string GrpcWebProto{"application/grpc-web+proto"}; const std::string GrpcWebText{"application/grpc-web-text"}; const std::string GrpcWebTextProto{"application/grpc-web-text+proto"}; + const std::string Json{"application/json"}; } ContentTypeValues; struct { diff --git a/source/exe/BUILD b/source/exe/BUILD index b61c3ddf915d3..5c837ea3533e5 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -36,6 +36,7 @@ envoy_cc_library( "//source/server/config/http:dynamo_lib", "//source/server/config/http:fault_lib", "//source/server/config/http:grpc_http1_bridge_lib", + "//source/server/config/http:grpc_json_transcoder_lib", "//source/server/config/http:grpc_web_lib", "//source/server/config/http:ratelimit_lib", "//source/server/config/http:router_lib", diff --git a/source/server/config/http/BUILD b/source/server/config/http/BUILD index d1dc3f27afce3..961c39a4302e4 100644 --- a/source/server/config/http/BUILD +++ b/source/server/config/http/BUILD @@ -54,6 +54,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "grpc_json_transcoder_lib", + srcs = ["grpc_json_transcoder.cc"], + hdrs = ["grpc_json_transcoder.h"], + deps = [ + "//include/envoy/server:instance_interface", + "//source/common/grpc:json_transcoder_filter_lib", + "//source/server/config/network:http_connection_manager_lib", + ], +) + envoy_cc_library( name = "grpc_web_lib", srcs = ["grpc_web.cc"], diff --git a/source/server/config/http/grpc_json_transcoder.cc b/source/server/config/http/grpc_json_transcoder.cc new file mode 100644 index 0000000000000..52cfd00011815 --- /dev/null +++ b/source/server/config/http/grpc_json_transcoder.cc @@ -0,0 +1,30 @@ +#include "server/config/http/grpc_json_transcoder.h" + +#include "envoy/registry/registry.h" + +#include "common/grpc/json_transcoder_filter.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +HttpFilterFactoryCb +GrpcJsonTranscoderFilterConfig::createFilterFactory(const Json::Object& config_json, + const std::string&, FactoryContext&) { + Grpc::TranscodingConfigSharedPtr config = + std::make_shared(config_json); + + return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(Http::StreamFilterSharedPtr{new Grpc::JsonTranscoderFilter(*config)}); + }; +} + +/** + * Static registration for the grpc transcoding filter. @see RegisterNamedHttpFilterConfigFactory. + */ +static Registry::RegisterFactory + register_; + +} // Configuration +} // Server +} // Envoy diff --git a/source/server/config/http/grpc_json_transcoder.h b/source/server/config/http/grpc_json_transcoder.h new file mode 100644 index 0000000000000..c1fb19e596634 --- /dev/null +++ b/source/server/config/http/grpc_json_transcoder.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include "envoy/server/instance.h" + +#include "server/config/network/http_connection_manager.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +/** + * Config registration for the gRPC JSON transcoder filter. @see NamedHttpFilterConfigFactory. + */ +class GrpcJsonTranscoderFilterConfig : public NamedHttpFilterConfigFactory { +public: + HttpFilterFactoryCb createFilterFactory(const Json::Object&, const std::string&, + FactoryContext& context) override; + std::string name() override { return "grpc_json_transcoder"; }; + HttpFilterType type() override { return HttpFilterType::Both; } +}; + +} // Configuration +} // Server +} // Envoy diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index 2a4005d67afba..9930434108fcf 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -68,6 +68,24 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "json_transcoder_filter_test", + srcs = ["json_transcoder_filter_test.cc"], + data = [ + "//test/proto:bookstore.proto", + "//test/proto:bookstore_proto_descriptor", + "//test/proto:bookstore_proto_descriptor_bad", + ], + deps = [ + "//source/common/grpc:json_transcoder_filter_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/upstream:upstream_mocks", + "//test/proto:bookstore_proto", + "//test/test_common:environment_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "rpc_channel_impl_test", srcs = ["rpc_channel_impl_test.cc"], diff --git a/test/common/grpc/json_transcoder_filter_test.cc b/test/common/grpc/json_transcoder_filter_test.cc new file mode 100644 index 0000000000000..02ec3ccb7291f --- /dev/null +++ b/test/common/grpc/json_transcoder_filter_test.cc @@ -0,0 +1,192 @@ +#include "common/buffer/buffer_impl.h" +#include "common/grpc/codec.h" +#include "common/grpc/common.h" +#include "common/grpc/json_transcoder_filter.h" +#include "common/http/header_map_impl.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/upstream/mocks.h" +#include "test/proto/bookstore.pb.h" +#include "test/test_common/environment.h" +#include "test/test_common/printers.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "google/protobuf/util/message_differencer.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; +using testing::ReturnPointee; +using testing::ReturnRef; + +using google::protobuf::util::MessageDifferencer; +using google::protobuf::util::Status; +using google::protobuf::util::error::Code; + +namespace Envoy { +namespace Grpc { + +class GrpcJsonTranscoderFilterTest : public testing::Test { +public: + GrpcJsonTranscoderFilterTest() : config_(*bookstoreJson()), filter_(config_) { + filter_.setDecoderFilterCallbacks(decoder_callbacks_); + filter_.setEncoderFilterCallbacks(encoder_callbacks_); + } + + const Json::ObjectSharedPtr bookstoreJson() { + std::string json_string = "{\"proto_descriptor\": \"" + bookstoreDescriptorPath() + + "\",\"services\": [\"bookstore.Bookstore\"]}"; + return Json::Factory::loadFromString(json_string); + } + + const std::string bookstoreDescriptorPath() { + return TestEnvironment::runfilesPath("test/proto/bookstore.descriptor"); + } + + // TODO(lizan): Add a mock of JsonTranscoderConfig and test more error cases. + JsonTranscoderConfig config_; + JsonTranscoderFilter filter_; + NiceMock decoder_callbacks_; + NiceMock encoder_callbacks_; +}; + +TEST_F(GrpcJsonTranscoderFilterTest, BadConfig) { + + const Json::ObjectSharedPtr unknown_service = + Json::Factory::loadFromString("{\"proto_descriptor\": \"" + bookstoreDescriptorPath() + + "\",\"services\": [\"grpc.service.UnknownService\"]}"); + + EXPECT_THROW_WITH_MESSAGE( + JsonTranscoderConfig config(*unknown_service), EnvoyException, + "transcoding_filter: Could not find 'grpc.service.UnknownService' in the proto descriptor"); + + const Json::ObjectSharedPtr bad_descriptor = Json::Factory::loadFromString( + "{\"proto_descriptor\": \"" + + TestEnvironment::runfilesPath("test/proto/bookstore_bad.descriptor") + + "\",\"services\": [\"bookstore.Bookstore\"]}"); + + EXPECT_THROW_WITH_MESSAGE(JsonTranscoderConfig config(*bad_descriptor), EnvoyException, + "transcoding_filter: Unable to build proto descriptor pool"); + + const Json::ObjectSharedPtr not_descriptor = Json::Factory::loadFromString( + "{\"proto_descriptor\": \"" + TestEnvironment::runfilesPath("test/proto/bookstore.proto") + + "\",\"services\": [\"bookstore.Bookstore\"]}"); + + EXPECT_THROW_WITH_MESSAGE(JsonTranscoderConfig config(*not_descriptor), EnvoyException, + "transcoding_filter: Unable to parse proto descriptor"); +} + +TEST_F(GrpcJsonTranscoderFilterTest, NoTranscoding) { + Http::TestHeaderMapImpl request_headers{{"content-type", "application/grpc"}, + {":method", "POST"}, + {":path", "/grpc.service/UnknownGrpcMethod"}}; + + Http::TestHeaderMapImpl expected_request_headers{{"content-type", "application/grpc"}, + {":method", "POST"}, + {":path", "/grpc.service/UnknownGrpcMethod"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ(expected_request_headers, request_headers); + + Buffer::OwnedImpl request_data{"{}"}; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(request_data, false)); + EXPECT_EQ(2, request_data.length()); + + Http::TestHeaderMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_trailers)); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.encodeHeaders(request_headers, false)); + EXPECT_EQ(expected_request_headers, request_headers); + + Buffer::OwnedImpl response_data{"{}"}; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.encodeData(response_data, false)); + EXPECT_EQ(2, response_data.length()); + + Http::TestHeaderMapImpl response_trailers{{"grpc-status", "0"}}; + Http::TestHeaderMapImpl expected_response_trailers{{"grpc-status", "0"}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.encodeTrailers(response_trailers)); + EXPECT_EQ(expected_response_trailers, response_trailers); +} + +TEST_F(GrpcJsonTranscoderFilterTest, TranscodingUnaryPost) { + Http::TestHeaderMapImpl request_headers{ + {"content-type", "application/json"}, {":method", "POST"}, {":path", "/shelf"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ("application/grpc", request_headers.get_("content-type")); + EXPECT_EQ("/bookstore.Bookstore/CreateShelf", request_headers.get_(":path")); + EXPECT_EQ("trailers", request_headers.get_("te")); + + Buffer::OwnedImpl request_data{"{\"theme\": \"Children\"}"}; + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(request_data, true)); + + Decoder decoder; + std::vector frames; + decoder.decode(request_data, frames); + + EXPECT_EQ(1, frames.size()); + + bookstore::CreateShelfRequest expected_request; + expected_request.mutable_shelf()->set_theme("Children"); + + bookstore::CreateShelfRequest request; + request.ParseFromArray(frames[0].data_->linearize(frames[0].length_), frames[0].length_); + + EXPECT_EQ(expected_request.ByteSize(), frames[0].length_); + EXPECT_TRUE(MessageDifferencer::Equals(expected_request, request)); + + Http::TestHeaderMapImpl response_headers{{"content-type", "application/grpc"}, + {":status", "200"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_.encodeHeaders(response_headers, false)); + EXPECT_EQ("application/json", response_headers.get_("content-type")); + + bookstore::Shelf response; + response.set_id(20); + response.set_theme("Children"); + + auto response_data = Common::serializeBody(response); + + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, + filter_.encodeData(*response_data, false)); + + std::string response_json( + reinterpret_cast(response_data->linearize(response_data->length())), + response_data->length()); + + EXPECT_EQ("{\"id\":\"20\",\"theme\":\"Children\"}", response_json); + + Http::TestHeaderMapImpl response_trailers{{"grpc-status", "0"}, {"grpc-message", ""}}; + + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(response_trailers)); +} + +TEST_F(GrpcJsonTranscoderFilterTest, TranscodingUnaryError) { + Http::TestHeaderMapImpl request_headers{ + {"content-type", "application/json"}, {":method", "POST"}, {":path", "/shelf"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ("application/grpc", request_headers.get_("content-type")); + EXPECT_EQ("/bookstore.Bookstore/CreateShelf", request_headers.get_(":path")); + EXPECT_EQ("trailers", request_headers.get_("te")); + + Buffer::OwnedImpl request_data{"{\"theme\": \"Children\""}; + + EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)) + .WillOnce(Invoke([](Http::HeaderMap& headers, bool end_stream) { + EXPECT_STREQ("400", headers.Status()->value().c_str()); + EXPECT_FALSE(end_stream); + })); + EXPECT_CALL(decoder_callbacks_, encodeData(_, true)); + + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_.decodeData(request_data, true)); + EXPECT_EQ(0, request_data.length()); +} + +} // namespace Grpc +} // namespace Envoy \ No newline at end of file diff --git a/test/config/integration/BUILD b/test/config/integration/BUILD index ffd5e2fd08a0d..bf7e32ef71112 100644 --- a/test/config/integration/BUILD +++ b/test/config/integration/BUILD @@ -10,6 +10,7 @@ envoy_package() exports_files([ "echo_server.json", "server.json", + "server_grpc_json_transcoder.json", "server_http2.json", "server_http2_upstream.json", "server_proxy_proto.json", diff --git a/test/config/integration/server_grpc_json_transcoder.json b/test/config/integration/server_grpc_json_transcoder.json new file mode 100644 index 0000000000000..4110cfb39b824 --- /dev/null +++ b/test/config/integration/server_grpc_json_transcoder.json @@ -0,0 +1,76 @@ +{ + "listeners": [ + { + "address": "tcp://{{ ip_loopback_address }}:0", + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "http1", + "access_log": [ + { + "path": "/dev/null", + "filter" : { + "type": "logical_or", + "filters": [ + { + "type": "status_code", + "op": ">=", + "value": 500 + }, + { + "type": "duration", + "op": ">=", + "value": 1000000 + } + ] + } + }, + { + "path": "/dev/null" + }], + "stat_prefix": "router", + "route_config": + { + "virtual_hosts": [ + { + "name": "integration", + "domains": [ "*" ], + "routes": [ + { + "prefix": "/", + "cluster": "cluster_1" + } + ] + } + ] + }, + "filters": [ + { "type": "both", "name": "grpc_json_transcoder", + "config": { + "proto_descriptor": "{{ test_rundir }}/test/proto/bookstore.descriptor", + "services": ["bookstore.Bookstore"] + } + }, + { "type": "decoder", "name": "router", "config": {} } + ] + } + }] + }], + + "admin": { "access_log_path": "/dev/null", "address": "tcp://{{ ip_loopback_address }}:0" }, + "statsd_local_udp_port": 8125, + + "cluster_manager": { + "clusters": [ + { + "name": "cluster_1", + "features": "http2", + "connect_timeout_ms": 5000, + "type": "static", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://{{ ip_loopback_address }}:{{ upstream_0 }}"}] + }] + } +} diff --git a/test/integration/BUILD b/test/integration/BUILD index aaba00a38bb22..4f292ebce63df 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -62,6 +62,24 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "grpc_json_transcoder_integration_test", + srcs = [ + "grpc_json_transcoder_integration_test.cc", + ], + data = [ + "//test/config/integration:server_grpc_json_transcoder.json", + "//test/proto:bookstore_proto_descriptor", + ], + deps = [ + ":integration_lib", + "//source/common/grpc:codec_lib", + "//source/common/http:header_map_lib", + "//test/proto:bookstore_proto", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "integration_admin_test", srcs = [ @@ -128,6 +146,7 @@ envoy_cc_test_library( "//source/server/config/http:dynamo_lib", "//source/server/config/http:fault_lib", "//source/server/config/http:grpc_http1_bridge_lib", + "//source/server/config/http:grpc_json_transcoder_lib", "//source/server/config/http:lightstep_lib", "//source/server/config/http:ratelimit_lib", "//source/server/config/http:router_lib", diff --git a/test/integration/grpc_json_transcoder_integration_test.cc b/test/integration/grpc_json_transcoder_integration_test.cc new file mode 100644 index 0000000000000..448d04b2c05b6 --- /dev/null +++ b/test/integration/grpc_json_transcoder_integration_test.cc @@ -0,0 +1,292 @@ +#include "common/grpc/codec.h" +#include "common/grpc/common.h" +#include "common/http/message_impl.h" + +#include "test/integration/integration.h" +#include "test/mocks/http/mocks.h" +#include "test/proto/bookstore.pb.h" + +#include "google/protobuf/stubs/status.h" +#include "google/protobuf/text_format.h" +#include "google/protobuf/util/message_differencer.h" +#include "gtest/gtest.h" + +using google::protobuf::util::MessageDifferencer; +using google::protobuf::util::Status; +using google::protobuf::util::error::Code; +using google::protobuf::Empty; +using google::protobuf::Message; +using google::protobuf::TextFormat; + +namespace Envoy { + +class GrpcJsonTranscoderIntegrationTest + : public BaseIntegrationTest, + public testing::TestWithParam { +public: + GrpcJsonTranscoderIntegrationTest() : BaseIntegrationTest(GetParam()) {} + /** + * Global initializer for all integration tests. + */ + void SetUp() override { + fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_)); + registerPort("upstream_0", fake_upstreams_.back()->localAddress()->ip()->port()); + createTestServer("test/config/integration/server_grpc_json_transcoder.json", {"http"}); + } + + /** + * Global destructor for all integration tests. + */ + void TearDown() override { + test_server_.reset(); + fake_upstreams_.clear(); + } + +protected: + template + void testTranscoding(Http::HeaderMap&& request_headers, const std::string& request_body, + const std::vector& grpc_request_messages, + const std::vector& grpc_response_messages, + const Status& grpc_status, Http::HeaderMap&& response_headers, + const std::string& response_body) { + IntegrationCodecClientPtr codec_client; + FakeHttpConnectionPtr fake_upstream_connection; + IntegrationStreamDecoderPtr response(new IntegrationStreamDecoder(*dispatcher_)); + FakeStreamPtr request_stream; + + codec_client = makeHttpConnection(lookupPort("http"), Http::CodecClient::Type::HTTP1); + + if (!request_body.empty()) { + Http::StreamEncoder& encoder = codec_client->startRequest(request_headers, *response); + Buffer::OwnedImpl body(request_body); + codec_client->sendData(encoder, body, true); + } else { + codec_client->makeHeaderOnlyRequest(request_headers, *response); + } + + fake_upstream_connection = fake_upstreams_[0]->waitForHttpConnection(*dispatcher_); + request_stream = fake_upstream_connection->waitForNewStream(); + if (!grpc_request_messages.empty()) { + request_stream->waitForEndStream(*dispatcher_); + + Grpc::Decoder grpc_decoder; + std::vector frames; + EXPECT_TRUE(grpc_decoder.decode(request_stream->body(), frames)); + EXPECT_EQ(grpc_request_messages.size(), frames.size()); + + for (size_t i = 0; i < grpc_request_messages.size(); ++i) { + RequestType actual_message; + if (frames[i].length_ > 0) { + EXPECT_TRUE(actual_message.ParseFromArray(frames[i].data_->linearize(frames[i].length_), + frames[i].length_)); + } + RequestType expected_message; + EXPECT_TRUE(TextFormat::ParseFromString(grpc_request_messages[i], &expected_message)); + + EXPECT_TRUE(MessageDifferencer::Equivalent(expected_message, actual_message)); + } + + Http::TestHeaderMapImpl response_headers; + response_headers.insertStatus().value(200); + response_headers.insertContentType().value(std::string("application/grpc")); + if (grpc_response_messages.empty()) { + response_headers.insertGrpcStatus().value(grpc_status.error_code()); + response_headers.insertGrpcMessage().value(grpc_status.error_message()); + request_stream->encodeHeaders(response_headers, true); + } else { + request_stream->encodeHeaders(response_headers, false); + for (const auto& response_message_str : grpc_response_messages) { + ResponseType response_message; + EXPECT_TRUE(TextFormat::ParseFromString(response_message_str, &response_message)); + auto buffer = Grpc::Common::serializeBody(response_message); + request_stream->encodeData(*buffer, false); + } + Http::TestHeaderMapImpl response_trailers; + response_trailers.insertGrpcStatus().value(grpc_status.error_code()); + response_trailers.insertGrpcMessage().value(grpc_status.error_message()); + request_stream->encodeTrailers(response_trailers); + } + EXPECT_TRUE(request_stream->complete()); + } else { + request_stream->waitForReset(); + } + + response->waitForEndStream(); + EXPECT_TRUE(response->complete()); + response_headers.iterate([](const Http::HeaderEntry& entry, void* context) -> void { + IntegrationStreamDecoder* response = static_cast(context); + Http::LowerCaseString lower_key{entry.key().c_str()}; + EXPECT_STREQ(entry.value().c_str(), response->headers().get(lower_key)->value().c_str()); + }, response.get()); + if (!response_body.empty()) { + EXPECT_EQ(response_body, response->body()); + } + + codec_client->close(); + fake_upstream_connection->close(); + fake_upstream_connection->waitForDisconnect(); + } +}; + +INSTANTIATE_TEST_CASE_P(IpVersions, GrpcJsonTranscoderIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); + +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryPost) { + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/shelf"}, + {":authority", "host"}, + {"content-type", "application/json"}}, + R"({"theme": "Children"})", {R"(shelf { theme: "Children" })"}, + {R"(id: 20 theme: "Children" )"}, Status::OK, + Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "application/json"}, + {"content-length", "30"}, + {"grpc-status", "0"}}, + R"({"id":"20","theme":"Children"})"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGet) { + testTranscoding( + Http::TestHeaderMapImpl{{":method", "GET"}, {":path", "/shelves"}, {":authority", "host"}}, + "", {""}, {R"(shelves { id: 20 theme: "Children" } + shelves { id: 1 theme: "Foo" } )"}, + Status::OK, Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "application/json"}, + {"content-length", "69"}, + {"grpc-status", "0"}}, + R"({"shelves":[{"id":"20","theme":"Children"},{"id":"1","theme":"Foo"}]})"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetError) { + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "GET"}, {":path", "/shelves/100?"}, {":authority", "host"}}, + "", {"shelf: 100"}, {}, Status(Code::NOT_FOUND, "Shelf 100 Not Found"), + Http::TestHeaderMapImpl{ + {":status", "200"}, {"grpc-status", "5"}, {"grpc-message", "Shelf 100 Not Found"}}, + ""); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryDelete) { + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "DELETE"}, {":path", "/shelves/456/books/123"}, {":authority", "host"}}, + "", {"shelf: 456 book: 123"}, {""}, Status::OK, + Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "application/json"}, + {"content-length", "2"}, + {"grpc-status", "0"}}, + "{}"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryPatch) { + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "PATCH"}, {":path", "/shelves/456/books/123"}, {":authority", "host"}}, + R"({"author" : "Leo Tolstoy", "title" : "War and Peace"})", + {R"(shelf: 456 book { id: 123 author: "Leo Tolstoy" title: "War and Peace" })"}, + {R"(id: 123 author: "Leo Tolstoy" title: "War and Peace")"}, Status::OK, + Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "application/json"}, + {"content-length", "59"}, + {"grpc-status", "0"}}, + R"({"id":"123","author":"Leo Tolstoy","title":"War and Peace"})"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryCustom) { + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "OPTIONS"}, {":path", "/shelves/456"}, {":authority", "host"}}, + "", {"shelf: 456"}, {""}, Status::OK, + Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "application/json"}, + {"content-length", "2"}, + {"grpc-status", "0"}}, + "{}"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, BindingAndBody) { + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "PUT"}, {":path", "/shelves/1/books"}, {":authority", "host"}}, + R"({"author" : "Leo Tolstoy", "title" : "War and Peace"})", + {R"(shelf: 1 book { author: "Leo Tolstoy" title: "War and Peace" })"}, + {R"(id: 3 author: "Leo Tolstoy" title: "War and Peace")"}, Status::OK, + Http::TestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, + R"({"id":"3","author":"Leo Tolstoy","title":"War and Peace"})"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, ServerStreamingGet) { + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "GET"}, {":path", "/shelves/1/books"}, {":authority", "host"}}, + "", {"shelf: 1"}, {R"(id: 1 author: "Neal Stephenson" title: "Readme")", + R"(id: 2 author: "George R.R. Martin" title: "A Game of Thrones")"}, + Status::OK, Http::TestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, + R"([{"id":"1","author":"Neal Stephenson","title":"Readme"})" + R"(,{"id":"2","author":"George R.R. Martin","title":"A Game of Thrones"}])"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, StreamingPost) { + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "POST"}, {":path", "/bulk/shelves"}, {":authority", "host"}}, + R"([ + { "theme" : "Classics" }, + { "theme" : "Satire" }, + { "theme" : "Russian" }, + { "theme" : "Children" }, + { "theme" : "Documentary" }, + { "theme" : "Mystery" }, + ])", + {R"(shelf { theme: "Classics" })", + R"(shelf { theme: "Satire" })", + R"(shelf { theme: "Russian" })", + R"(shelf { theme: "Children" })", + R"(shelf { theme: "Documentary" })", + R"(shelf { theme: "Mystery" })"}, + {R"(id: 3 theme: "Classics")", + R"(id: 4 theme: "Satire")", + R"(id: 5 theme: "Russian")", + R"(id: 6 theme: "Children")", + R"(id: 7 theme: "Documentary")", + R"(id: 8 theme: "Mystery")"}, + Status::OK, Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "application/json"}, + {"transfer-encoding", "chunked"}}, + R"([{"id":"3","theme":"Classics"})" + R"(,{"id":"4","theme":"Satire"})" + R"(,{"id":"5","theme":"Russian"})" + R"(,{"id":"6","theme":"Children"})" + R"(,{"id":"7","theme":"Documentary"})" + R"(,{"id":"8","theme":"Mystery"}])"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, InvalidJson) { + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, {":path", "/shelf"}, {":authority", "host"}}, + R"(INVALID_JSON)", {}, {}, Status::OK, + Http::TestHeaderMapImpl{{":status", "400"}, {"content-type", "text/plain"}}, + "Unexpected token.\n" + "INVALID_JSON\n" + "^"); + + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, {":path", "/shelf"}, {":authority", "host"}}, + R"({ "theme" : "Children")", {}, {}, Status::OK, + Http::TestHeaderMapImpl{{":status", "400"}, {"content-type", "text/plain"}}, + "Unexpected end of string. Expected , or } after key:value pair.\n" + "\n" + "^"); + + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, {":path", "/shelf"}, {":authority", "host"}}, + R"({ "theme" "Children" })", {}, {}, Status::OK, + Http::TestHeaderMapImpl{{":status", "400"}, {"content-type", "text/plain"}}, + "Expected : between key:value pair.\n" + "{ \"theme\" \"Children\" }\n" + " ^"); +} + +} // namespace Envoy \ No newline at end of file diff --git a/test/proto/BUILD b/test/proto/BUILD index 551b549b5f4e8..4515b26cc97f4 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -4,11 +4,48 @@ load( "//bazel:envoy_build_system.bzl", "envoy_package", "envoy_proto_library", + "envoy_proto_descriptor", ) envoy_package() +exports_files(["bookstore.proto"]) + envoy_proto_library( name = "helloworld_proto", srcs = [":helloworld.proto"], ) + +envoy_proto_library( + name = "bookstore_proto", + srcs = [":bookstore.proto"], + external_deps = [ + "http_api_protos", + "cc_wkt_protos", + ], +) + +envoy_proto_descriptor( + name = "bookstore_proto_descriptor", + srcs = [ + "bookstore.proto", + ], + out = "bookstore.descriptor", + external_deps = [ + "http_api_protos", + "well_known_protos", + ], + protocopts = ["--include_imports"], +) + +envoy_proto_descriptor( + name = "bookstore_proto_descriptor_bad", + srcs = [ + "bookstore.proto", + ], + out = "bookstore_bad.descriptor", + external_deps = [ + "http_api_protos", + "well_known_protos", + ], +) diff --git a/test/proto/bookstore.proto b/test/proto/bookstore.proto new file mode 100644 index 0000000000000..b2d153ada4097 --- /dev/null +++ b/test/proto/bookstore.proto @@ -0,0 +1,159 @@ +syntax = "proto3"; + +package bookstore; + +import "google/api/annotations.proto"; +import "google/protobuf/empty.proto"; + +// A simple Bookstore API. +// +// The API manages shelves and books resources. Shelves contain books. +service Bookstore { + // Returns a list of all shelves in the bookstore. + rpc ListShelves(google.protobuf.Empty) returns (ListShelvesResponse) { + option (google.api.http) = { + get: "/shelves" + }; + } + // Creates a new shelf in the bookstore. + rpc CreateShelf(CreateShelfRequest) returns (Shelf) { + option (google.api.http) = { + post: "/shelf" + body: "shelf" + }; + } + // Creates multiple shelves with one streaming call + rpc BulkCreateShelf(stream CreateShelfRequest) returns (stream Shelf) { + option (google.api.http) = { + post: "/bulk/shelves" + body: "shelf" + }; + } + // Returns a specific bookstore shelf. + rpc GetShelf(GetShelfRequest) returns (Shelf) { + option (google.api.http) = { + get: "/shelves/{shelf}" + }; + } + // Deletes a shelf, including all books that are stored on the shelf. + rpc DeleteShelf(DeleteShelfRequest) returns (google.protobuf.Empty) { + } + // Returns a list of books on a shelf. + rpc ListBooks(ListBooksRequest) returns (stream Book) { + option (google.api.http) = { + get: "/shelves/{shelf}/books" + }; + } + // Creates a new book. + rpc CreateBook(CreateBookRequest) returns (Book) { + option (google.api.http) = { + put: "/shelves/{shelf}/books" + body: "book" + }; + } + // Returns a specific book. + rpc GetBook(GetBookRequest) returns (Book) { + } + // Deletes a book from a shelf. + rpc DeleteBook(DeleteBookRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/shelves/{shelf}/books/{book}" + }; + } + rpc UpdateBook(UpdateBookRequest) returns (Book) { + option (google.api.http) = { + patch: "/shelves/{shelf}/books/{book.id}" + body: "book" + }; + } + rpc BookstoreOptions(GetShelfRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + custom { + kind: "OPTIONS" + path: "/shelves/{shelf}" + } + }; + } +} + +// A shelf resource. +message Shelf { + // A unique shelf id. + int64 id = 1; + // A theme of the shelf (fiction, poetry, etc). + string theme = 2; +} + +// A book resource. +message Book { + // A unique book id. + int64 id = 1; + // An author of the book. + string author = 2; + // A book title. + string title = 3; + // Quotes from the book. + repeated string quotes = 4; +} + +// Response to ListShelves call. +message ListShelvesResponse { + // Shelves in the bookstore. + repeated Shelf shelves = 1; +} + +// Request message for CreateShelf method. +message CreateShelfRequest { + // The shelf resource to create. + Shelf shelf = 1; +} + +// Request message for GetShelf method. +message GetShelfRequest { + // The ID of the shelf resource to retrieve. + int64 shelf = 1; +} + +// Request message for DeleteShelf method. +message DeleteShelfRequest { + // The ID of the shelf to delete. + int64 shelf = 1; +} + +// Request message for ListBooks method. +message ListBooksRequest { + // ID of the shelf which books to list. + int64 shelf = 1; +} + +// Request message for CreateBook method. +message CreateBookRequest { + // The ID of the shelf on which to create a book. + int64 shelf = 1; + // A book resource to create on the shelf. + Book book = 2; +} + +// Request message for GetBook method. +message GetBookRequest { + // The ID of the shelf from which to retrieve a book. + int64 shelf = 1; + // The ID of the book to retrieve. + int64 book = 2; +} + +// Request message for UpdateBook method +message UpdateBookRequest { + // The ID of the shelf from which to retrieve a book. + int64 shelf = 1; + // A book resource to update on the shelf. + Book book = 2; +} + +// Request message for DeleteBook method. +message DeleteBookRequest { + // The ID of the shelf from which to delete a book. + int64 shelf = 1; + // The ID of the book to delete. + int64 book = 2; +} diff --git a/test/server/config/http/BUILD b/test/server/config/http/BUILD index f288f74fdc533..e0a6a5eb42062 100644 --- a/test/server/config/http/BUILD +++ b/test/server/config/http/BUILD @@ -16,6 +16,7 @@ envoy_cc_test( "//source/server/config/http:dynamo_lib", "//source/server/config/http:fault_lib", "//source/server/config/http:grpc_http1_bridge_lib", + "//source/server/config/http:grpc_json_transcoder_lib", "//source/server/config/http:grpc_web_lib", "//source/server/config/http:ip_tagging_lib", "//source/server/config/http:lightstep_lib",