diff --git a/contrib/endpoints/src/api_manager/proto/api_manager_status.proto b/contrib/endpoints/src/api_manager/proto/api_manager_status.proto index fb9a43aa3eb..d4945b6daf1 100644 --- a/contrib/endpoints/src/api_manager/proto/api_manager_status.proto +++ b/contrib/endpoints/src/api_manager/proto/api_manager_status.proto @@ -1,26 +1,16 @@ -// Copyright (C) Extensible Service Proxy Authors -// All rights reserved. +// Copyright 2016 Google Inc. All Rights Reserved. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -// SUCH DAMAGE. +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // //////////////////////////////////////////////////////////////////////////////// // diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 8680199847b..1fe04af6d07 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -1,26 +1,16 @@ -// Copyright (C) Extensible Service Proxy Authors -// All rights reserved. +// Copyright 2016 Google Inc. All Rights Reserved. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -// SUCH DAMAGE. +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // //////////////////////////////////////////////////////////////////////////////// // diff --git a/contrib/endpoints/src/grpc/transcoding/bookstore.proto b/contrib/endpoints/src/grpc/transcoding/bookstore.proto new file mode 100644 index 00000000000..e33cd487d39 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/bookstore.proto @@ -0,0 +1,67 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +// bookstore.proto +// Test proto for transcoding +syntax = "proto3"; +package google.api_manager.transcoding; +message Biography { + int64 year_born = 1; + int64 year_died = 2; + string text = 3; +} +message AuthorInfo { + string first_name = 1; + string last_name = 2; + Biography bio = 3; +} +message Book { + string author = 1; + string name = 2; + string title = 3; + AuthorInfo author_info = 4; +} +message Shelf { + string name = 1; + string theme = 2; +} +message ListShelvesResponse { + repeated Shelf shelves = 1; +} +message CreateShelfRequest { + Shelf shelf = 1; +} +message GetShelfRequest { + int64 shelf = 1; +} +message DeleteShelfRequest { + int64 shelf = 1; +} +message ListBooksRequest { + int64 shelf = 1; +} +message CreateBookRequest { + int64 shelf = 1; + Book book = 2; +} +message GetBookRequest { + int64 shelf = 1; + int64 book = 2; +} +message DeleteBookRequest { + int64 shelf = 1; + int64 book = 2; +} diff --git a/contrib/endpoints/src/grpc/transcoding/json_request_translator.cc b/contrib/endpoints/src/grpc/transcoding/json_request_translator.cc new file mode 100644 index 00000000000..3ab1591d116 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/json_request_translator.cc @@ -0,0 +1,164 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/json_request_translator.h" + +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/status.h" +#include "google/protobuf/util/internal/json_stream_parser.h" +#include "google/protobuf/util/internal/object_writer.h" +#include "src/grpc/transcoding/message_stream.h" +#include "src/grpc/transcoding/request_message_translator.h" +#include "src/grpc/transcoding/request_stream_translator.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace { + +namespace pb = ::google::protobuf; +namespace pbio = ::google::protobuf::io; +namespace pbutil = ::google::protobuf::util; +namespace pbconv = ::google::protobuf::util::converter; + +// An on-demand request translation implementation where the reading of the +// input and translation happen only as needed when the caller asks for an +// output message. +// +// LazyRequestTranslator is given +// - a ZeroCopyInputStream (json_input) to read the input JSON from, +// - a JsonStreamParser (parser) - the input end of the translation +// pipeline, i.e. that takes the input JSON, +// - a MessageStream (translated), the output end of the translation +// pipeline, i.e. where the output proto messages appear. +// When asked for a message it reads chunks from the input stream and passes +// to the json parser until a message appears in the output (translated) +// stream, or until the input JSON stream runs out of data (in this case, caller +// will call NextMessage again in the future when more data is available). +class LazyRequestTranslator : public MessageStream { + public: + LazyRequestTranslator(pbio::ZeroCopyInputStream* json_input, + pbconv::JsonStreamParser* json_parser, + MessageStream* translated) + : input_json_(json_input), + json_parser_(json_parser), + translated_(translated), + seen_input_(false) {} + + // MessageStream implementation + bool NextMessage(std::string* message) { + // Keep translating chunks until a message appears in the translated stream. + while (!translated_->NextMessage(message)) { + if (!TranslateChunk()) { + // Error or no more input to translate. + return false; + } + } + return true; + } + bool Finished() const { return translated_->Finished() || !status_.ok(); } + pbutil::Status Status() const { return status_; } + + private: + // Translates one chunk of data. Returns true, if there was input to + // translate; otherwise or in case of an error returns false. + bool TranslateChunk() { + if (Finished()) { + return false; + } + // Read the next chunk of data from input_json_ + const void* data = nullptr; + int size = 0; + if (!input_json_->Next(&data, &size)) { + // End of input + if (!seen_input_) { + // If there was no input at all translate an empty JSON object ("{}"). + status_ = json_parser_->Parse("{}"); + return status_.ok(); + } + // No more data to translate, finish the parser and return false. + status_ = json_parser_->FinishParse(); + return false; + } else if (0 == size) { + // No data at this point, but there might be more input later. + return false; + } + seen_input_ = true; + + // Feed the chunk to the parser & check the status. + status_ = json_parser_->Parse( + pb::StringPiece(reinterpret_cast(data), size)); + if (!status_.ok()) { + return false; + } + // Check the translation status + status_ = translated_->Status(); + if (!status_.ok()) { + return false; + } + return true; + } + + // The input JSON stream + pbio::ZeroCopyInputStream* input_json_; + + // The JSON parser that is the starting point of the translation pipeline + pbconv::JsonStreamParser* json_parser_; + + // The stream where the translated messages appear + MessageStream* translated_; + + // Whether we have seen any input or not + bool seen_input_; + + // Translation status + pbutil::Status status_; +}; + +} // namespace + +JsonRequestTranslator::JsonRequestTranslator( + pbutil::TypeResolver* type_resolver, pbio::ZeroCopyInputStream* json_input, + RequestInfo request_info, bool streaming, bool output_delimiters) { + // A writer that accepts input ObjectWriter events for translation + pbconv::ObjectWriter* writer = nullptr; + // The stream where translated messages appear + MessageStream* translated = nullptr; + if (streaming) { + // Streaming - we'll need a RequestStreamTranslator + stream_translator_.reset(new RequestStreamTranslator( + *type_resolver, output_delimiters, std::move(request_info))); + writer = stream_translator_.get(); + translated = stream_translator_.get(); + } else { + // No streaming - use a RequestMessageTranslator + message_translator_.reset(new RequestMessageTranslator( + *type_resolver, output_delimiters, std::move(request_info))); + writer = &message_translator_->Input(); + translated = message_translator_.get(); + } + parser_.reset(new pbconv::JsonStreamParser(writer)); + output_.reset( + new LazyRequestTranslator(json_input, parser_.get(), translated)); +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/json_request_translator.h b/contrib/endpoints/src/grpc/transcoding/json_request_translator.h new file mode 100644 index 00000000000..bc1490f2ada --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/json_request_translator.h @@ -0,0 +1,111 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define GRPC_TRANSCODING_JSON_REQUEST_TRANSLATOR_H_ + +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/util/internal/json_stream_parser.h" +#include "google/protobuf/util/type_resolver.h" +#include "src/grpc/transcoding/message_stream.h" +#include "src/grpc/transcoding/request_message_translator.h" +#include "src/grpc/transcoding/request_stream_translator.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// JsonRequestTranslator translates HTTP JSON request into gRPC message(s) +// according to the http rules defined in the service config (see +// third_party/config/google/api/http.proto). +// +// It supports streaming, weaving variable bindings and prefixing the messages +// with GRPC message delimiters (see http://www.grpc.io/docs/guides/wire.html). +// Also to support flow-control JsonRequestTranslator does the translation in a +// lazy fashion, i.e. reads the input stream only as needed. +// +// Example: +// JsonRequestTranslator translator(type_resolver, input_json, request_info, +// /*streaming*/true, +// /*output_delimiters*/true); +// +// MessageStream& out = translator.Output(); +// +// if (!out.Status().ok()) { +// printf("Error: %s\n", out.Status().ErrorMessage().as_string().c_str()); +// return; +// } +// +// std::string message; +// while (out.NextMessage(&message)) { +// printf("Message=%s\n", message.c_str()); +// } +// +// The implementation uses JsonStreamParser to parse the incoming JSON and +// RequestMessageTranslator or RequestStreamTranslator to translate it into +// protobuf message(s). +// - JsonStreamParser converts the incoming JSON into ObjectWriter events, +// - in a non-streaming case RequestMessageTranslator translates these +// events into a protobuf message, +// - in a streaming case RequestStreamTranslator translates these events +// into a stream of protobuf messages. +class JsonRequestTranslator { + public: + // type_resolver - provides type information necessary for translation ( + // passed to the underlying RequestMessageTranslator or + // RequestStreamTranslator). Note that JsonRequestTranslator + // doesn't maintain the ownership of type_resolver. + // json_input - the input JSON stream representated through + // a ZeroCopyInputStream. Note that JsonRequestTranslator does + // not maintain the ownership of json_input. + // request_info - information about the request being translated (passed to + // the underlying RequestMessageTranslator or + // RequestStreamTranslator). + // streaming - whether this is a streaming call or not + // output_delimiters - whether to ouptut gRPC message delimiters or not + JsonRequestTranslator(::google::protobuf::util::TypeResolver* type_resolver, + ::google::protobuf::io::ZeroCopyInputStream* json_input, + RequestInfo request_info, bool streaming, + bool output_delimiters); + + // The translated output stream + MessageStream& Output() { return *output_; } + + private: + // The JSON parser + std::unique_ptr<::google::protobuf::util::converter::JsonStreamParser> + parser_; + + // The output stream + std::unique_ptr output_; + + // A single message translator (empty unique_ptr if this is a streaming call) + std::unique_ptr message_translator_; + + // A message stream translator (empty unique_ptr if this is a non-streaming + // call) + std::unique_ptr stream_translator_; + + JsonRequestTranslator(const JsonRequestTranslator&) = delete; + JsonRequestTranslator& operator=(const JsonRequestTranslator&) = delete; +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_REQUEST_TRANSLATOR_H diff --git a/contrib/endpoints/src/grpc/transcoding/json_request_translator_test.cc b/contrib/endpoints/src/grpc/transcoding/json_request_translator_test.cc new file mode 100644 index 00000000000..9d05181223b --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/json_request_translator_test.cc @@ -0,0 +1,746 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/json_request_translator.h" + +#include +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/bookstore.pb.h" +#include "src/grpc/transcoding/proto_stream_tester.h" +#include "src/grpc/transcoding/request_translator_test_base.h" +#include "src/grpc/transcoding/test_common.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +namespace pberr = google::protobuf::util::error; + +// TranslationTestCase helps us build translation test cases and validate the +// translation output. +// +// The tests construct the test case using AddMessage() and Build(). With +// AddMessage() the JSON message as well as the expected proto are specified. +// TranslationTestCase builds the input JSON and for each expected proto message +// remembers the position in the input JSON where it's expected. +// +// After building the test case, the tests can use TestAt() method to validate +// the output translation stream (using the given ProtoStreamTester) at the +// specified position in JSON input. +class TranslationTestCase { + public: + TranslationTestCase(bool streaming) + : input_json_(streaming ? "[" : ""), streaming_(streaming) {} + + bool Streaming() const { return streaming_; } + + // Add an input message. + void AddMessage(const std::string& json, const std::string& expected_proto) { + input_json_ += json; + // The message expected_proto is expected at position input_json_.size() + expected_.emplace_back(ExpectedAt{input_json_.size(), expected_proto}); + if (streaming_) { + input_json_ += ", "; + } + } + + // Build the test case. + void Build() { + if (streaming_) { + input_json_ += "]"; + } + Reset(); + } + + // Resets the test case to start (re-)testing + void Reset() { next_expected_ = std::begin(expected_); } + + // Return the input JSON to be passed to the translator. + const std::string& InputJson() const { return input_json_; } + + // Test the output translation stream (using the specified ProtoStreamTester), + // after [0, pos) part of the input JSON has been fed to the translator. + template + bool TestAt(ProtoStreamTester& tester, size_t pos) { + // While we have proto messages that are expected before or at pos, try to + // match them. + while (next_expected_ != std::end(expected_) && pos >= next_expected_->at) { + // Must not be finished + if (!tester.ExpectFinishedEq(false)) { + ADD_FAILURE() << "Finished unexpectedly at " + << input_json_.substr(0, pos) << std::endl; + return false; + } + // Match the message + if (!tester.ExpectNextEq(next_expected_->proto)) { + ADD_FAILURE() << "Failed matching the message at " + << input_json_.substr(0, pos) << std::endl; + return false; + } + ++next_expected_; + } + if (!tester.ExpectNone()) { + // We have processed all expected messages, the stream must not have + // any messages left. + ADD_FAILURE() << "Unexpected message at " << input_json_.substr(0, pos) + << std::endl; + return false; + } + if (pos < input_json_.size()) { + // There is still input left, so the stream must not be finished yet. + if (!tester.ExpectFinishedEq(false)) { + ADD_FAILURE() << "Finished unexpectedly at " + << input_json_.substr(0, pos) << std::endl; + return false; + } + } else { // pos >= input_json_.size() + // There is no input left, so the stream must be finished. + if (!tester.ExpectFinishedEq(true)) { + ADD_FAILURE() << "Not finished after all input has been translated\n"; + return false; + } + } + return true; + } + + private: + std::string input_json_; + bool streaming_; + + struct ExpectedAt { + // The position in the input_json_, after which this proto is expected + size_t at; + // The expected proto message in a proto-text format + std::string proto; + }; + std::vector expected_; + + // An iterator to the next expected message. + std::vector::const_iterator next_expected_; +}; + +class JsonRequestTranslatorTest : public RequestTranslatorTestBase { + protected: + JsonRequestTranslatorTest() : streaming_(false) {} + + // Sets whether this is a streaming call or not. Use it before calling + // Build(). Default is non-streaming + void SetStreaming(bool streaming) { streaming_ = streaming; } + + // Add an input chunk + void AddChunk(const std::string& json) { input_->AddChunk(json); } + + // End the input + void Finish() { input_->Finish(); } + + // Test the translation test case with different partitions of the input + // chunk_count - the number of chunks (parts) per partition + // partitioning_coefficient - defines how exhaustive the test should be. See + // the comment on RunTestForInputPartitions() in + // test_common.h for more details. + // test_case - the test case to run, + // delimiters - whether to output GRPC delimiters when translating or not. + template + bool RunTest(size_t chunk_count, double partitioning_coefficient, + TranslationTestCase* test_case, bool delimiters) { + // Set streaming flag + SetStreaming(test_case->Streaming()); + // Run the test for different m-partitions of the input JSON. + return RunTestForInputPartitions( + chunk_count, partitioning_coefficient, test_case->InputJson(), + [this, test_case](const std::vector& t) { + // Rebuild the translator & reset the test case to reset its state. + Build(); + test_case->Reset(); + + // Feed the chunks according to the partition defined by tuple t and + // test along the way. + const std::string& input = test_case->InputJson(); + size_t pos = 0; + for (size_t i = 0; i < t.size(); ++i) { + AddChunk(input.substr(pos, t[i] - pos)); + pos = t[i]; + if (!test_case->TestAt(Tester(), pos)) { + return false; + } + } + // Feed the last chunk, finish & test. + AddChunk(input.substr(pos)); + Finish(); + return test_case->TestAt(Tester(), input.size()); + }); + } + + // Calls the above function both with delimiters=true and delimiters=false. + template + bool RunTest(size_t chunk_count, double partitioning_coefficient, + TranslationTestCase* test_case) { + return RunTest(chunk_count, partitioning_coefficient, + test_case, false) && + RunTest(chunk_count, partitioning_coefficient, + test_case, true); + } + + private: + // RequestTranslatorTestBase::Create() + virtual MessageStream* Create( + google::protobuf::util::TypeResolver& type_resolver, bool delimiters, + RequestInfo request_info) { + input_.reset(new TestZeroCopyInputStream()); + translator_.reset(new JsonRequestTranslator(&type_resolver, input_.get(), + std::move(request_info), + streaming_, delimiters)); + return &translator_->Output(); + } + + bool streaming_; + std::unique_ptr input_; + std::unique_ptr translator_; +}; + +TEST_F(JsonRequestTranslatorTest, Simple) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + TranslationTestCase tc(false); + tc.AddMessage(R"({ "name" : "1", "theme" : "Russian" })", + R"(name : "1" theme : "Russian")"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); + EXPECT_TRUE((RunTest(3, 1.0, &tc))); + EXPECT_TRUE((RunTest(4, 0.5, &tc))); + EXPECT_TRUE((RunTest(5, 0.4, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, Nested) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Book"); + TranslationTestCase tc(false); + tc.AddMessage( + R"({ + "name" : "8", + "author" : "Leo Tolstoy", + "title" : "War and Peace", + "authorInfo" : { + "firstName" : "Leo", + "lastName" : "Tolstoy", + "bio" : { + "yearBorn" : 1830, + "yearDied" : 1910, + "text" : "some text" + } + } + })", + R"( + name : "8" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + bio { + year_born : 1830 + year_died : 1910 + text : "some text" + } + } + )"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 0.5, &tc))); + EXPECT_TRUE((RunTest(3, 0.05, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, Prefix) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book"); + TranslationTestCase tc(false); + tc.AddMessage( + R"({ + "name" : "9", + "author" : "Leo Tolstoy", + "title" : "War and Peace", + })", + R"( + book { + name : "9" + author : "Leo Tolstoy" + title : "War and Peace" + } + )"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); + EXPECT_TRUE((RunTest(3, 0.1, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, Bindings) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book"); + AddVariableBinding("book.authorInfo.firstName", "Leo"); + AddVariableBinding("book.authorInfo.lastName", "Tolstoy"); + TranslationTestCase tc(false); + tc.AddMessage( + R"({ + "name" : "11", + "author" : "Leo Tolstoy", + "title" : "Anna Karenina", + })", + R"( + book { + name : "11" + author : "Leo Tolstoy" + title : "Anna Karenina" + author_info : { + first_name : "Leo" + last_name : "Tolstoy" + } + } + )"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); + EXPECT_TRUE((RunTest(3, 0.1, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, MorePrefixAndBindings) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book.authorInfo.bio"); + AddVariableBinding("book.name", "7"); + AddVariableBinding("book.author", "Fyodor Dostoevski"); + AddVariableBinding("book.title", "Idiot"); + AddVariableBinding("book.author_info.first_name", "Fyodor"); + AddVariableBinding("book.author_info.last_name", "Dostoevski"); + TranslationTestCase tc(false); + tc.AddMessage( + R"({ + "yearBorn" : "1840", + "yearDied" : "1920", + "text" : "bio text", + })", + R"( + book { + name : "7" + author : "Fyodor Dostoevski" + title : "Idiot" + author_info : { + first_name : "Fyodor" + last_name : "Dostoevski" + bio { + year_born : 1840 + year_died : 1920 + text : "bio text" + } + } + } + )"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); + EXPECT_TRUE((RunTest(3, 0.1, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, OnlyBindings) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + AddVariableBinding("name", "1"); + AddVariableBinding("theme", "Fiction"); + TranslationTestCase tc(false); + tc.AddMessage("", R"( name : "1" theme : "Fiction" )"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, ScalarBody) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + SetBodyPrefix("theme"); + TranslationTestCase tc(false); + tc.AddMessage(R"("History")", R"(theme : "History")"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); + EXPECT_TRUE((RunTest(3, 0.1, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, Empty) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Finish(); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + EXPECT_TRUE(Tester().ExpectNextEq("")); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(JsonRequestTranslatorTest, Large) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + + auto sizes = {1, 256, 1024, 1234, 4096, 65537}; + for (auto size : sizes) { + auto theme = GenerateInput("0123456789abcdefgh", size); + + TranslationTestCase tc(false); + tc.AddMessage( + R"({ "name" : "1", "theme" : ")" + theme + R"("})", + R"(name : "1" theme : ")" + theme + R"(")"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + } +} + +TEST_F(JsonRequestTranslatorTest, OneByteChunks) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + + std::string json = R"({ "name" : "99", "theme" : "Fiction" })"; + + for (auto c : json) { + EXPECT_TRUE(Tester().ExpectNone()); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + AddChunk(std::string(1, c)); + } + Finish(); + + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + EXPECT_TRUE(Tester().ExpectNextEq(R"(name : "99" theme : "Fiction")")); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(JsonRequestTranslatorTest, UnknownsIgnored) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + TranslationTestCase tc(false); + tc.AddMessage( + R"({ + "name" : "1", + "theme" : "Russian", + "unknownField" : "ignored", + "unknownObject" : { + "name" : "value" + }, + "unknownArray" : [1, 2, 3, 4] + })", + R"(name : "1" theme : "Russian")"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, ErrorInvalidJson) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + + auto invalids = { + "Invalid", + R"({"name")", + R"({"name" : 1)", + R"({"theme" , "name" : 1})", + R"({"name" : 1])", + R"(["theme" , "name" : 1})", + R"({"name" : 1, { "a" : [ "c", "d"} ]})", + R"(["name" : 1 ]})", + R"([{"name" : 1 ]})", + " ", + "\r \t\n", + }; + + for (auto streaming : {false, true}) { + for (auto invalid : invalids) { + SetStreaming(streaming); + Build(); + AddChunk(invalid); + Finish(); + EXPECT_TRUE(Tester().ExpectNone()); + EXPECT_TRUE(Tester().ExpectStatusEq(pberr::INVALID_ARGUMENT)); + } + } +} + +TEST_F(JsonRequestTranslatorTest, StreamingSimple) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + TranslationTestCase tc(/*streaming*/ true); + tc.AddMessage(R"({ "name" : "1", "theme" : "Russian" })", + R"(name : "1" theme : "Russian")"); + tc.AddMessage(R"({ "name" : "2", "theme" : "History" })", + R"(name : "2" theme : "History")"); + tc.AddMessage(R"({ "name" : "3", "theme" : "Mistery" })", + R"(name : "3" theme : "Mistery")"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); + EXPECT_TRUE((RunTest(3, 0.2, &tc))); + EXPECT_TRUE((RunTest(4, 0.1, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, StreamingNested) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Book"); + TranslationTestCase tc(/*streaming*/ true); + tc.AddMessage( + R"({ + "name" : "1", + "author" : "Leo Tolstoy", + "title" : "War and Peace", + "authorInfo" : { + "firstName" : "Leo", + "lastName" : "Tolstoy", + } + })", + R"( + name : "1" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + } + )"); + tc.AddMessage( + R"({ + "name" : "2", + "author" : "Leo Tolstoy", + "title" : "Anna Karenina", + "authorInfo" : { + "firstName" : "Leo", + "lastName" : "Tolstoy", + } + })", + R"( + name : "2" + author : "Leo Tolstoy" + title : "Anna Karenina" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + } + )"); + tc.AddMessage( + R"({ + "name" : "3", + "author" : "Fyodor Dostoevski", + "title" : "Crime and Punishment", + "authorInfo" : { + "firstName" : "Fyodor", + "lastName" : "Dostoevski", + } + })", + R"( + name : "3" + author : "Fyodor Dostoevski" + title : "Crime and Punishment" + author_info { + first_name : "Fyodor" + last_name : "Dostoevski" + } + )"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, StreamingPrefixAndBindings) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book.authorInfo"); + AddVariableBinding("book.name", "100"); + AddVariableBinding("book.title", "The Old Man And The Sea"); + AddVariableBinding("book.author", "Ernest Hemingway"); + TranslationTestCase tc(/*streaming*/ true); + tc.AddMessage( + R"({ + "firstName" : "Ernest", + "lastName" : "Hemingway", + })", + R"( + book { + name : "100" + author : "Ernest Hemingway" + title : "The Old Man And The Sea" + author_info { + first_name : "Ernest" + last_name : "Hemingway" + } + } + )"); + // Note that variable bindings apply only to the first message. So "name", + // "title" and "author" won't be presend in 2nd and 3rd messages + tc.AddMessage( + R"({ + "firstName" : "Ernest-2", + "lastName" : "Hemingway-2", + })", + R"( + book { + author_info { + first_name : "Ernest-2" + last_name : "Hemingway-2" + } + } + )"); + tc.AddMessage( + R"({ + "firstName" : "Ernest-3", + "lastName" : "Hemingway-3", + })", + R"( + book { + author_info { + first_name : "Ernest-3" + last_name : "Hemingway-3" + } + } + )"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, Streaming1KMessages) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + TranslationTestCase tc(/*streaming*/ true); + + for (size_t i = 0; i < 1000; ++i) { + std::string no = std::to_string(i + 1); + tc.AddMessage(R"({ "name" : ")" + no + R"(", "theme" : "th)" + no + R"(" + })", + R"(name : ")" + no + R"(" theme : "th)" + no + R"(")"); + } + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, StreamingScalars) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + SetBodyPrefix("theme"); + TranslationTestCase tc(/*streaming*/ true); + // ["Classic", "Fiction", "Documentary"] + tc.AddMessage(R"("Classic")", R"(theme : "Classic")"); + tc.AddMessage(R"("Fiction")", R"(theme : "Fiction")"); + tc.AddMessage(R"("Documentary")", R"(theme : "Documentary")"); + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, StreamingArrays) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("ListShelvesResponse"); + // "shelves" is a repeated field in "ListShelvesResponse" + SetBodyPrefix("shelves"); + TranslationTestCase tc(/*streaming*/ true); + // The input becomes an array of arrays: + // [ + // [ {shelf1}, {shelf2} ], + // [ {shelf3}, {shelf4} ], + // [ {shelf5}, {shelf6} ], + // ] + // The output is a stream of messages of type ListShelvesResponse: + // 1 - "shelves {shelf1} shelves {shelf 2}" + // 2 - "shelves {shelf3} shelves {shelf 4}" + // 3 - "shelves {shelf5} shelves {shelf 6}" + tc.AddMessage( + R"([ + {"name" : "1", "theme" : "Classic"}, + {"name" : "2", "theme" : "Fiction"}, + {"name" : "3", "theme" : "Documentary"}, + ])", + R"( + shelves : { name : "1" theme : "Classic" } + shelves : { name : "2" theme : "Fiction" } + shelves : { name : "3" theme : "Documentary" } + )"); + tc.AddMessage( + R"([ + {"name" : "5", "theme" : "Drama"}, + {"name" : "6", "theme" : "Russian"}, + ])", + R"( + shelves : { name : "5" theme : "Drama" } + shelves : { name : "6" theme : "Russian" } + )"); + + tc.Build(); + + EXPECT_TRUE((RunTest(1, 1.0, &tc))); + EXPECT_TRUE((RunTest(2, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, StreamingEmptyStream) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + TranslationTestCase tc(/*streaming*/ true); + tc.Build(); + EXPECT_TRUE((RunTest(1, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, StreamingEmptyMessages) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + TranslationTestCase tc(/*streaming*/ true); + tc.AddMessage("{}", ""); + tc.AddMessage(R"({"theme" : "Classic"})", R"(theme : "Classic")"); + tc.AddMessage("{}", ""); + tc.Build(); + EXPECT_TRUE((RunTest(1, 1.0, &tc))); +} + +TEST_F(JsonRequestTranslatorTest, StreamingErrorNotAnArray) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + SetStreaming(true); + Build(); + AddChunk(R"({"name" : "1"})"); + Finish(); + EXPECT_TRUE(Tester().ExpectNone()); + EXPECT_TRUE(Tester().ExpectStatusEq(pberr::INVALID_ARGUMENT)); +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/message_reader.cc b/contrib/endpoints/src/grpc/transcoding/message_reader.cc new file mode 100644 index 00000000000..5568b160d47 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/message_reader.cc @@ -0,0 +1,142 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/message_reader.h" + +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/zero_copy_stream_impl.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +namespace pb = ::google::protobuf; +namespace pbio = ::google::protobuf::io; + +MessageReader::MessageReader(pbio::ZeroCopyInputStream* in) + : in_(in), + current_message_size_(0), + have_current_message_size_(false), + finished_(false) {} + +namespace { + +// A helper function that reads the given number of bytes from a +// ZeroCopyInputStream and copies it to the given buffer +bool ReadStream(pbio::ZeroCopyInputStream* stream, unsigned char* buffer, + int size) { + int size_in = 0; + const void* data_in = nullptr; + // While we have bytes to read + while (size > 0) { + if (!stream->Next(&data_in, &size_in)) { + return false; + } + int to_copy = std::min(size, size_in); + memcpy(buffer, data_in, to_copy); + // Advance buffer and subtract the size to reflect the number of bytes left + buffer += to_copy; + size -= to_copy; + // Keep track of uncopied bytes + size_in -= to_copy; + } + // Return the uncopied bytes + stream->BackUp(size_in); + return true; +} + +// Determines whether the stream is finished or not. +bool IsStreamFinished(pbio::ZeroCopyInputStream* stream) { + int size = 0; + const void* data = nullptr; + if (!stream->Next(&data, &size)) { + return true; + } else { + stream->BackUp(size); + return false; + } +} + +// A helper function to extract the size from a gRPC wire format message +// delimiter - see http://www.grpc.io/docs/guides/wire.html. +unsigned DelimiterToSize(const unsigned char* delimiter) { + unsigned size = 0; + // Bytes 1-4 are big-endian 32-bit message size + size = size | static_cast(delimiter[1]); + size <<= 8; + size = size | static_cast(delimiter[2]); + size <<= 8; + size = size | static_cast(delimiter[3]); + size <<= 8; + size = size | static_cast(delimiter[4]); + return size; +} + +} // namespace + +std::unique_ptr MessageReader::NextMessage() { + if (finished_) { + // The stream has ended + return std::unique_ptr(); + } + + // Check if we have the current message size. If not try to read it. + if (!have_current_message_size_) { + const size_t kDelimiterSize = 5; + if (in_->ByteCount() < static_cast(kDelimiterSize)) { + // We don't have 5 bytes available to read the length of the message. + // Find out whether the stream is finished and return false. + finished_ = IsStreamFinished(in_); + return std::unique_ptr(); + } + + // Try to read the delimiter + unsigned char delimiter[kDelimiterSize] = {0}; + if (!ReadStream(in_, delimiter, sizeof(delimiter))) { + finished_ = true; + return std::unique_ptr(); + } + + current_message_size_ = DelimiterToSize(delimiter); + have_current_message_size_ = true; + } + + // We interpret ZeroCopyInputStream::ByteCount() as the number of bytes + // available for reading at the moment. Check if we have the full message + // available to read. + if (in_->ByteCount() < static_cast(current_message_size_)) { + // We don't have a full message + return std::unique_ptr(); + } + + // We have a message! Use LimitingInputStream to wrap the input stream and + // limit it to current_message_size_ bytes to cover only the current message. + auto result = std::unique_ptr( + new pbio::LimitingInputStream(in_, current_message_size_)); + + // Reset the have_current_message_size_ for the next message + have_current_message_size_ = false; + + return result; +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/message_reader.h b/contrib/endpoints/src/grpc/transcoding/message_reader.h new file mode 100644 index 00000000000..df175561f47 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/message_reader.h @@ -0,0 +1,102 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_MESSAGE_READER_H_ +#define GRPC_TRANSCODING_MESSAGE_READER_H_ + +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/status.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// MessageReader helps extract full messages from a ZeroCopyInputStream of +// messages in gRPC wire format (http://www.grpc.io/docs/guides/wire.html). Each +// message is returned in a ZeroCopyInputStream. MessageReader doesn't advance +// the underlying ZeroCopyInputStream unless there is a full message available. +// This is done to avoid copying while buffering. +// +// Example: +// MessageReader reader(&input); +// +// while (!reader.Finished()) { +// auto message = reader.NextMessage(); +// if (!message) { +// // No message is available at this moment. +// break; +// } +// +// const void* buffer = nullptr; +// int size = 0; +// while (message.Next(&buffer, &size)) { +// // Process the message data. +// ... +// } +// } +// +// NOTE: MesssageReader assumes that ZeroCopyInputStream::ByteCount() returns +// the number of bytes available to read at the moment. That's what +// MessageReader uses to determine whether there is a complete message +// available or not. +// +// NOTE: MessageReader is unable to recognize the case when there is an +// incomplete message at the end of the input. The callers will need to +// detect it and act appropriately. +// This is because the MessageReader doesn't call Next() on the input +// stream until there is a full message available. So, if there is an +// incomplete message at the end of the input, MessageReader won't call +// Next() and won't know that the stream has finished. +// +class MessageReader { + public: + MessageReader(::google::protobuf::io::ZeroCopyInputStream* in); + + // If a full message is available, NextMessage() returns a ZeroCopyInputStream + // over the message. Otherwise returns nullptr - this might be temporary, the + // caller can call NextMessage() again later to check. + // NOTE: the caller must consume the entire message before calling + // NextMessage() again. + // That's because the returned ZeroCopyInputStream is a wrapper on top + // of the original ZeroCopyInputStream and the MessageReader relies on + // the caller to advance the stream to the next message before calling + // NextMessage() again. + std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> NextMessage(); + + // Returns true if the stream has ended (this is permanent); otherwise returns + // false. + bool Finished() const { return finished_; } + + private: + ::google::protobuf::io::ZeroCopyInputStream* in_; + // The size of the current message. + unsigned int current_message_size_; + // Whether we have read the current message size or not + bool have_current_message_size_; + // Are we all done? + bool finished_; + + MessageReader(const MessageReader&) = delete; + MessageReader& operator=(const MessageReader&) = delete; +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_MESSAGE_READER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/message_reader_test.cc b/contrib/endpoints/src/grpc/transcoding/message_reader_test.cc new file mode 100644 index 00000000000..40a2f6eafb7 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/message_reader_test.cc @@ -0,0 +1,365 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/message_reader.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "src/grpc/transcoding/test_common.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +std::string ReadAllFromStream( + ::google::protobuf::io::ZeroCopyInputStream* stream) { + std::ostringstream all; + const void* data = nullptr; + int size = 0; + while (stream->Next(&data, &size) && size != 0) { + all << std::string(static_cast(data), size); + } + return all.str(); +} + +// A helper structure to store a single expected message and its position +struct ExpectedAt { + // The position in the input, after which this message is expected + size_t at; + std::string message; +}; + +// MessageReaderTestRun tests a single MessageReader processing the input as +// expected. +// It allows feeding chunks of the input (AddChunk()) to the MessageReader and +// testing that the MessageReader yields the expected messages at any point +// (Test()). +class MessageReaderTestRun { + public: + // input - the input to be passed to the MessageReader + // expected - the expected messages as the input is processed + // NOTE: Both input and expected are stored as references in the + // MessageReaderTestRun, so the caller must make sure they exist + // throughout the lifetime of MessageReaderTestRun. + MessageReaderTestRun(const std::string& input, + const std::vector& expected) + : input_(input), + expected_(expected), + input_stream_(new TestZeroCopyInputStream()), + reader_(new MessageReader(input_stream_.get())), + position_(0), + next_expected_(std::begin(expected_)) {} + + // Returns the total size of the input including the message delimiters + size_t TotalInputSize() const { return input_.size(); } + + // Adds a chunk of a given size to be processed + void AddChunk(size_t size) { + input_stream_->AddChunk(input_.substr(position_, size)); + position_ += size; + } + + // Finishes the input stream + void FinishInputStream() { input_stream_->Finish(); } + + // Tests the MessageReader at the current position of the input. + bool Test() { + // While we still have expected messages before or at the current position + // try to match. + while (next_expected_ != std::end(expected_) && + next_expected_->at <= position_) { + // Must not be finished as we expect a message + if (reader_->Finished()) { + ADD_FAILURE() << "Finished unexpectedly" << std::endl; + return false; + } + // Read the message + auto stream = reader_->NextMessage(); + if (!stream) { + ADD_FAILURE() << "No message available" << std::endl; + return false; + } + // Match the message with the expected message + auto message = ReadAllFromStream(stream.get()); + if (next_expected_->message != message) { + EXPECT_EQ(next_expected_->message, message); + return false; + } + // Move to the next expected message + ++next_expected_; + } + // We have read all the expected messages, so NextMessage() must return + // nullptr + if (reader_->NextMessage().get()) { + ADD_FAILURE() << "Unexpected message" << std::endl; + return false; + } + // Expect the reader to be finished iff we have called Finish() on + // input_stream_. + if (input_stream_->Finished() != reader_->Finished()) { + EXPECT_EQ(input_stream_->Finished(), reader_->Finished()); + return false; + } + return true; + } + + private: + const std::string& input_; + const std::vector& expected_; + + std::unique_ptr input_stream_; + std::unique_ptr reader_; + + // The current position in the input stream. + size_t position_; + + // An iterator that points the next expected message. + std::vector::const_iterator next_expected_; +}; + +// MessageReaderTestCase tests a single input test case with different +// partitions of the input. +class MessageReaderTestCase { + public: + MessageReaderTestCase(std::vector messages) { + for (const auto& message : messages) { + // First add the delimiter + input_ += SizeToDelimiter(message.size()); + // Then add the message itself + input_ += message; + // Remember that we should expect this message after input_.size() bytes + // are processed. + expected_.emplace_back(ExpectedAt{input_.size(), message}); + } + } + + std::unique_ptr NewRun() { + return std::unique_ptr( + new MessageReaderTestRun(input_, expected_)); + } + + // Runs the test for different partitions of the input. + // chunk_count - the number of chunks (parts) per partition + // partitioning_coefficient - defines how exhaustive the test should be. See + // the comment on RunTestForInputPartitions() in + // test_common.h for more details. + bool Test(size_t chunk_count, double partitioning_coefficient) { + return RunTestForInputPartitions(chunk_count, partitioning_coefficient, + input_, + [this](const std::vector& t) { + auto run = NewRun(); + + // Feed the chunks according to the + // partition defined by tuple t and + // test along the way. + size_t pos = 0; + for (size_t i = 0; i < t.size(); ++i) { + run->AddChunk(t[i] - pos); + pos = t[i]; + if (!run->Test()) { + return false; + } + } + // Feed the last chunk, finish & test. + run->AddChunk(input_.size() - pos); + run->FinishInputStream(); + return run->Test(); + }); + } + + private: + std::string input_; + std::vector expected_; +}; + +class MessageReaderTest : public ::testing::Test { + protected: + MessageReaderTest() {} + + // Add an input message + void AddMessage(std::string message) { + messages_.emplace_back(std::move(message)); + } + + // Builds a test case using the added messages and clears the messages in case + // the test needs to add messages & build another test case. + std::unique_ptr Build() { + std::vector messages; + messages.swap(messages_); + return std::unique_ptr( + new MessageReaderTestCase(std::move(messages))); + } + + private: + std::vector messages_; +}; + +TEST_F(MessageReaderTest, OneMessage) { + AddMessage("SimpleMessage"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 1.0)); + EXPECT_TRUE(tc->Test(4, 0.1)); +} + +TEST_F(MessageReaderTest, ThreeMessages) { + AddMessage("Message1"); + AddMessage("Message2"); + AddMessage("Message3"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 1.0)); + EXPECT_TRUE(tc->Test(4, 0.2)); +} + +TEST_F(MessageReaderTest, TenKMessages) { + for (size_t i = 1; i < 10000; ++i) { + AddMessage("Message" + std::to_string(i)); + } + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); +} + +TEST_F(MessageReaderTest, DifferentSizesAllInOneStream) { + auto sizes = {0, 1, 2, 3, 4, 5, 6, 10, 12, 100, 128, 256, 1024, 4096, 65537}; + + for (auto size : sizes) { + AddMessage(GenerateInput("abcdefg", size)); + } + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); +} + +TEST_F(MessageReaderTest, DifferentSizesEachInItsOwnStream) { + auto sizes = {0, 1, 2, 3, 4, 5, 6, 10, 12, 100, 128, 256, 1024, 4096, 65537}; + + for (auto size : sizes) { + AddMessage(GenerateInput("abcdefg", size)); + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + if (size < 50) { + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 1.0)); + } + } +} + +TEST_F(MessageReaderTest, OneByteChunks) { + AddMessage("Message1"); + AddMessage("Message2"); + AddMessage("Message3"); + + auto tc = Build(); + auto tr = tc->NewRun(); + + for (size_t i = 0; i < tr->TotalInputSize(); ++i) { + tr->AddChunk(1); + EXPECT_TRUE(tr->Test()); + } + tr->FinishInputStream(); + EXPECT_TRUE(tr->Test()); +} + +TEST_F(MessageReaderTest, DirectTest) { + TestZeroCopyInputStream input_stream; + MessageReader reader(&input_stream); + + std::string message1 = "This is a message"; + std::string message2 = "This is a different message"; + std::string message3 = "This is another message"; + std::string message4 = "This is yet another message"; + + // Nothing yet + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + + // Add the delimiter for the first message + input_stream.AddChunk(SizeToDelimiter(message1.size())); + + // Still nothing + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + + // Add the message itself + input_stream.AddChunk(message1); + + // Now message1 must be available + auto message1_stream = reader.NextMessage(); + ASSERT_NE(nullptr, message1_stream.get()); + EXPECT_EQ(message1, ReadAllFromStream(message1_stream.get())); + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + + // Add part of the message2 + input_stream.AddChunk(SizeToDelimiter(message2.size())); + input_stream.AddChunk(message2.substr(0, 10)); + + // No message should be available + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + + // Add the rest of the second message + input_stream.AddChunk(message2.substr(10)); + + // Now the message2 is available + auto message2_stream = reader.NextMessage(); + ASSERT_NE(nullptr, message2_stream.get()); + EXPECT_EQ(message2, ReadAllFromStream(message2_stream.get())); + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + + // Adding both message3 & message4 in one shot and Finish the stream + input_stream.AddChunk(SizeToDelimiter(message3.size())); + input_stream.AddChunk(message3); + input_stream.AddChunk(SizeToDelimiter(message4.size())); + input_stream.AddChunk(message4); + input_stream.Finish(); + + // Not finished yet as we still have messages to read + EXPECT_FALSE(reader.Finished()); + + // Now both message3 & message4 must be available + auto message3_stream = reader.NextMessage(); + ASSERT_NE(nullptr, message3_stream.get()); + EXPECT_EQ(message3, ReadAllFromStream(message3_stream.get())); + EXPECT_FALSE(reader.Finished()); + + auto message4_stream = reader.NextMessage(); + ASSERT_NE(nullptr, message4_stream.get()); + EXPECT_EQ(message4, ReadAllFromStream(message4_stream.get())); + + // All done, the reader must be finished + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_TRUE(reader.Finished()); +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream.cc b/contrib/endpoints/src/grpc/transcoding/message_stream.cc new file mode 100644 index 00000000000..ce3e62e8a99 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/message_stream.cc @@ -0,0 +1,121 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/message_stream.h" + +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +namespace pbio = ::google::protobuf::io; + +namespace { + +// a ZeroCopyInputStream implementation over a MessageStream implementation +class ZeroCopyStreamOverMessageStream : public pbio::ZeroCopyInputStream { + public: + // src - the underlying MessageStream. ZeroCopyStreamOverMessageStream doesn't + // maintain the ownership of src, the caller must make sure it exists + // throughtout the lifetime of ZeroCopyStreamOverMessageStream. + ZeroCopyStreamOverMessageStream(MessageStream* src) + : src_(src), message_(), position_(0) {} + + // ZeroCopyInputStream implementation + bool Next(const void** data, int* size) { + // Done with the current message, try to get another one. + if (position_ >= message_.size()) { + ReadNextMessage(); + } + + if (position_ < message_.size()) { + *data = static_cast(&message_[position_]); + // Assuming message_.size() - position_ < INT_MAX + *size = static_cast(message_.size() - position_); + // Advance the position + position_ = message_.size(); + return true; + } else { + // No data at this point. + *size = 0; + // Return false if the source stream has finished as this is the end + // of the data; otherwise return true. + return !src_->Finished(); + } + } + + void BackUp(int count) { + if (count > 0 && static_cast(count) <= position_) { + position_ -= static_cast(count); + } + // Otherwise, BackUp has been called illegaly, so we ignore it. + } + + bool Skip(int) { return false; } // Not implemented (no need) + + ::google::protobuf::int64 ByteCount() const { + // NOTE: we are changing the ByteCount() interpretation. In our case + // ByteCount() returns the number of bytes available for reading at this + // moment. In the original interpretation it is supposed to be the number + // of bytes read so far. + // We need this such that the consumers are able to read the gRPC delimited + // message stream only if there is a full message available. + if (position_ >= message_.size()) { + // If the current message is all done, try to read the next message + // to make sure we return the correct byte count. + const_cast(this)->ReadNextMessage(); + } + return static_cast<::google::protobuf::int64>(message_.size() - position_); + } + + private: + // Updates the current message and creates an ArrayInputStream over it. + void ReadNextMessage() { + message_.clear(); + position_ = 0; + // Try to find the next non-empty message in the stream + while (message_.empty() && src_->NextMessage(&message_)) { + } + } + + // The source MessageStream + MessageStream* src_; + + // The current message being read + std::string message_; + + // The current position in the current message + size_t position_; +}; + +} // namespace + +std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> +MessageStream::CreateZeroCopyInputStream() { + return std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream>( + new ZeroCopyStreamOverMessageStream(this)); +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream.h b/contrib/endpoints/src/grpc/transcoding/message_stream.h new file mode 100644 index 00000000000..435040332a7 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/message_stream.h @@ -0,0 +1,85 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_MESSAGE_STREAM_H_ +#define GRPC_TRANSCODING_MESSAGE_STREAM_H_ + +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/status.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// MessageStream abstracts a stream of std::string represented messages. Among +// other things MessageStream helps us to reuse some code for streaming and +// non-streaming implementations of request translation. +// We'll use this simple interface internally in transcoding (ESP) and will +// implement ZeroCopyInputStream based on this for exposing it to Nginx +// integration code (or potentially other consumers of transcoding). +// +// Example: +// MessageStream& stream = GetTranslatorStream(); +// +// if (!stream.Status().ok()) { +// printf("Error - %s", stream.Status().error_message().c_str()); +// return; +// } +// +// std::string message; +// while (stream.Message(&message)) { +// printf("Message=%s\n", message.c_str()); +// } +// +// if (stream.Finished()) { +// printf("Finished\n"); +// } else { +// printf("No messages at this time. Try again later \n"); +// } +// +// Unlike ZeroCopyInputStream this interface doesn't support features like +// BackUp(), Skip() and is easier to implement. At the same time the +// implementations can achieve "zero copy" by moving the std::string messages. +// However, this assumes a particular implementation (std::string based), so +// we use it only internally. +// +class MessageStream { + public: + // Retrieves the next message from the stream if there is one available. + // The implementation can use move assignment operator to avoid a copy. + // If no message is available at this time, returns false (this might be + // temporary); otherwise returns true. + virtual bool NextMessage(std::string* message) = 0; + // Returns true if no messages are left (this is permanent); otherwise return + // false. + virtual bool Finished() const = 0; + // Stream status to report errors + virtual ::google::protobuf::util::Status Status() const = 0; + // Virtual destructor + virtual ~MessageStream() {} + // Creates ZeroCopyInputStream implementation based on this stream + std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> + CreateZeroCopyInputStream(); +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_MESSAGE_STREAM_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc b/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc new file mode 100644 index 00000000000..ab3c24f8e14 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc @@ -0,0 +1,280 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/message_stream.h" + +#include +#include + +#include "gtest/gtest.h" +#include "src/grpc/transcoding/test_common.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +namespace pbio = ::google::protobuf::io; +namespace pbutil = ::google::protobuf::util; + +// A test MessageStream implementation for testing ZeroCopyInputStream over +// MessageStream implementation. +class TestMessageStream : public MessageStream { + public: + TestMessageStream() : finished_(false) {} + + void AddMessage(std::string message) { + messages_.emplace_back(std::move(message)); + } + + void Finish() { finished_ = true; } + + // MessageStream implementation + bool NextMessage(std::string* message) { + if (messages_.empty()) { + return false; + } else { + *message = std::move(messages_.front()); + messages_.pop_front(); + return true; + } + } + bool Finished() const { return messages_.empty() && finished_; } + pbutil::Status Status() const { return pbutil::Status::OK; } + + private: + bool finished_; + std::deque messages_; +}; + +class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { + protected: + ZeroCopyInputStreamOverMessageStreamTest() {} + + typedef std::vector Messages; + + bool Test(const Messages& messages) { + TestMessageStream test_message_stream; + auto zero_copy_stream = test_message_stream.CreateZeroCopyInputStream(); + + const void* data = nullptr; + int size = 0; + + // Check that Next() returns true and a 0-sized buffer meaning that + // nothing is available at the moment. + if (!zero_copy_stream->Next(&data, &size)) { + ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; + return false; + } + if (0 != size) { + EXPECT_EQ(0, size); + return false; + } + + for (auto message : messages) { + // Add the message to the MessageStream + test_message_stream.AddMessage(message); + + // message.size() bytes must be available for reading + if (static_cast(message.size()) != zero_copy_stream->ByteCount()) { + EXPECT_EQ(message.size(), zero_copy_stream->ByteCount()); + return false; + } + + // Now try to read & match the message + if (!zero_copy_stream->Next(&data, &size)) { + ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; + return false; + } + auto actual = std::string(reinterpret_cast(data), size); + if (message != actual) { + EXPECT_EQ(message, actual); + return false; + } + + // Try backing up & reading again different sizes + auto backup_sizes = {1ul, + 2ul, + 10ul, + message.size() / 2ul, + 3ul * message.size() / 4ul, + message.size()}; + + for (auto backup_size : backup_sizes) { + if (0 == backup_size || message.size() < backup_size) { + // Not a valid test case + continue; + } + zero_copy_stream->BackUp(backup_size); + + // backup_size bytes must be available for reading again + if (static_cast(backup_size) != zero_copy_stream->ByteCount()) { + EXPECT_EQ(message.size(), zero_copy_stream->ByteCount()); + return false; + } + + // Now Next() must return the backed up data again. + if (!zero_copy_stream->Next(&data, &size)) { + ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; + return false; + } + auto actual = std::string(reinterpret_cast(data), size); + // We expect the last backup_size bytes of the message. + auto expected = message.substr(message.size() - backup_size); + if (expected != actual) { + EXPECT_EQ(expected, actual); + return false; + } + } + + // At this point no data should be available + if (!zero_copy_stream->Next(&data, &size)) { + ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; + return false; + } + if (0 != size) { + EXPECT_EQ(0, size); + return false; + } + } + + // Now finish the MessageStream & make sure the ZeroCopyInputStream has + // ended. + test_message_stream.Finish(); + if (zero_copy_stream->Next(&data, &size)) { + ADD_FAILURE() << "The stream still hasn't finished" << std::endl; + return false; + } + + return true; + } +}; + +TEST_F(ZeroCopyInputStreamOverMessageStreamTest, OneMessage) { + EXPECT_TRUE(Test(Messages{1, "This is a test message"})); +} + +TEST_F(ZeroCopyInputStreamOverMessageStreamTest, ThreeMessages) { + EXPECT_TRUE(Test(Messages{"Message One", "Message Two", "Message Three"})); +} + +TEST_F(ZeroCopyInputStreamOverMessageStreamTest, TenKMessages) { + Messages messages; + for (int i = 1; i <= 10000; ++i) { + messages.emplace_back("Message " + std::to_string(i)); + } + EXPECT_TRUE(Test(messages)); +} + +TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DifferenteSizes) { + auto sizes = {0, 1, 2, 3, 4, 5, 6, 10, 12, 100, 128, 256, 1024, 4096, 65537}; + + for (auto size : sizes) { + EXPECT_TRUE(Test(Messages{1, GenerateInput("abcdefg12345", size)})); + } +} + +TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DifferenteSizesOneStream) { + auto sizes = {0, 1, 2, 3, 4, 5, 6, 10, 12, 100, 128, 256, 1024, 4096, 65537}; + + Messages messages; + for (auto size : sizes) { + messages.emplace_back(GenerateInput("abcdefg12345", size)); + } + EXPECT_TRUE(Test(messages)); +} + +TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { + TestMessageStream test_message_stream; + auto zero_copy_stream = test_message_stream.CreateZeroCopyInputStream(); + + const void* data = nullptr; + int size = 0; + + // Check that Next() returns true and a 0-sized buffer meaning that + // nothing is available at the moment. + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(0, size); + + // Test messages + std::string message1 = "This is a message"; + std::string message2 = "This is a message too"; + std::string message3 = "Another message"; + std::string message4 = "Yet another message"; + + // Add message1 to the MessageStream + test_message_stream.AddMessage(message1); + + // message1 is available for reading + EXPECT_EQ(message1.size(), zero_copy_stream->ByteCount()); + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message1, std::string(reinterpret_cast(data), size)); + + // Back up a bit + zero_copy_stream->BackUp(5); + + // Now read the backed up data again + EXPECT_EQ(5, zero_copy_stream->ByteCount()); + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message1.substr(message1.size() - 5), + std::string(reinterpret_cast(data), size)); + + // Add message2 to the MessageStream + test_message_stream.AddMessage(message2); + + // message2 is available for reading + EXPECT_EQ(message2.size(), zero_copy_stream->ByteCount()); + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message2, std::string(reinterpret_cast(data), size)); + + // Back up all of message2 + zero_copy_stream->BackUp(message2.size()); + + // Now read message2 again + EXPECT_EQ(message2.size(), zero_copy_stream->ByteCount()); + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message2, std::string(reinterpret_cast(data), size)); + + // At this point no data should be available + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(0, size); + + // Add both message3 & message4 & finish the MessageStream afterwards + test_message_stream.AddMessage(message3); + test_message_stream.AddMessage(message4); + test_message_stream.Finish(); + + // Read & match both message3 & message4 + EXPECT_EQ(message3.size(), zero_copy_stream->ByteCount()); + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message3, std::string(reinterpret_cast(data), size)); + + EXPECT_EQ(message4.size(), zero_copy_stream->ByteCount()); + EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message4, std::string(reinterpret_cast(data), size)); + + // All done! + EXPECT_FALSE(zero_copy_stream->Next(&data, &size)); +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/prefix_writer.cc b/contrib/endpoints/src/grpc/transcoding/prefix_writer.cc new file mode 100644 index 00000000000..e04cfcbfc65 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/prefix_writer.cc @@ -0,0 +1,216 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/prefix_writer.h" + +#include + +#include "google/protobuf/stubs/stringpiece.h" +#include "google/protobuf/stubs/strutil.h" +#include "google/protobuf/util/internal/object_writer.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +PrefixWriter::PrefixWriter(const std::string& prefix, + google::protobuf::util::converter::ObjectWriter* ow) + : prefix_(google::protobuf::Split(prefix, ".", true)), + non_actionable_depth_(0), + writer_(ow) {} + +PrefixWriter* PrefixWriter::StartObject(google::protobuf::StringPiece name) { + if (++non_actionable_depth_ == 1) { + name = StartPrefix(name); + } + writer_->StartObject(name); + return this; +} + +PrefixWriter* PrefixWriter::EndObject() { + writer_->EndObject(); + if (--non_actionable_depth_ == 0) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::StartList(google::protobuf::StringPiece name) { + if (++non_actionable_depth_ == 1) { + name = StartPrefix(name); + } + writer_->StartList(name); + return this; +} + +PrefixWriter* PrefixWriter::EndList() { + writer_->EndList(); + if (--non_actionable_depth_ == 0) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderBool(google::protobuf::StringPiece name, + bool value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderBool(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderInt32(google::protobuf::StringPiece name, + google::protobuf::int32 value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderInt32(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderUint32(google::protobuf::StringPiece name, + google::protobuf::uint32 value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderUint32(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderInt64(google::protobuf::StringPiece name, + google::protobuf::int64 value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderInt64(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderUint64(google::protobuf::StringPiece name, + google::protobuf::uint64 value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderUint64(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderDouble(google::protobuf::StringPiece name, + double value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderDouble(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderFloat(google::protobuf::StringPiece name, + float value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderFloat(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderString(google::protobuf::StringPiece name, + google::protobuf::StringPiece value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderString(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderBytes(google::protobuf::StringPiece name, + google::protobuf::StringPiece value) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + writer_->RenderBytes(name, value); + if (root) { + EndPrefix(); + } + return this; +} + +PrefixWriter* PrefixWriter::RenderNull(google::protobuf::StringPiece name) { + bool root = non_actionable_depth_ == 0; + if (root) { + name = StartPrefix(name); + } + + writer_->RenderNull(name); + if (root) { + EndPrefix(); + } + return this; +} + +google::protobuf::StringPiece PrefixWriter::StartPrefix( + google::protobuf::StringPiece name) { + for (const auto& prefix : prefix_) { + writer_->StartObject(name); + name = prefix; + } + return name; +} + +void PrefixWriter::EndPrefix() { + for (size_t i = 0; i < prefix_.size(); ++i) { + writer_->EndObject(); + } +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/prefix_writer.h b/contrib/endpoints/src/grpc/transcoding/prefix_writer.h new file mode 100644 index 00000000000..0ce228c92b8 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/prefix_writer.h @@ -0,0 +1,113 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_PREFIX_WRITER_H_ +#define GRPC_TRANSCODING_PREFIX_WRITER_H_ + +#include +#include + +#include "google/protobuf/stubs/stringpiece.h" +#include "google/protobuf/util/internal/object_writer.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// PrefixWriter is helper ObjectWriter implementation that for each incoming +// object +// 1) writes the given prefix path by starting objects to the output +// ObjectWriter, +// 2) forwards the writer events for a single object, +// 3) unwinds the prefix, by closing objects in the reverse order. +// +// E.g. +// +// PrefixWriter pw("A.B.C", out); +// pw.StartObject("Root"); +// ... +// pw.RenderString("x", "value"); +// ... +// pw.EndObject("Root"); +// +// is equivalent to +// +// out.StartObject("Root"); +// out.StartObject("A"); +// out.StartObject("B"); +// out.StartObject("C"); +// ... +// pw.RenderString("x", "value"); +// ... +// out.EndObject("C"); +// out.EndObject("B"); +// out.EndObject("A"); +// out.EndObject("Root"); +// +class PrefixWriter : public google::protobuf::util::converter::ObjectWriter { + public: + // prefix is a '.' delimited prefix path to be added + PrefixWriter(const std::string& prefix, + google::protobuf::util::converter::ObjectWriter* ow); + + // ObjectWriter methods. + PrefixWriter* StartObject(google::protobuf::StringPiece name); + PrefixWriter* EndObject(); + PrefixWriter* StartList(google::protobuf::StringPiece name); + PrefixWriter* EndList(); + PrefixWriter* RenderBool(google::protobuf::StringPiece name, bool value); + PrefixWriter* RenderInt32(google::protobuf::StringPiece name, + google::protobuf::int32 value); + PrefixWriter* RenderUint32(google::protobuf::StringPiece name, + google::protobuf::uint32 value); + PrefixWriter* RenderInt64(google::protobuf::StringPiece name, + google::protobuf::int64 value); + PrefixWriter* RenderUint64(google::protobuf::StringPiece name, + google::protobuf::uint64 value); + PrefixWriter* RenderDouble(google::protobuf::StringPiece name, double value); + PrefixWriter* RenderFloat(google::protobuf::StringPiece name, float value); + PrefixWriter* RenderString(google::protobuf::StringPiece name, + google::protobuf::StringPiece value); + PrefixWriter* RenderBytes(google::protobuf::StringPiece name, + google::protobuf::StringPiece value); + PrefixWriter* RenderNull(google::protobuf::StringPiece name); + + private: + // Helper method to start the prefix and return the name to use for the value. + google::protobuf::StringPiece StartPrefix(google::protobuf::StringPiece name); + + // Helper method to end the prefix. + void EndPrefix(); + + // The path prefix if the HTTP body maps to a nested message in the proto. + std::vector prefix_; + + // Tracks the depth within the output, so we know when to write the prefix + // and when to close it off. + int non_actionable_depth_; + + // The output object writer to forward the writer events. + google::protobuf::util::converter::ObjectWriter* writer_; + + PrefixWriter(const PrefixWriter&) = delete; + PrefixWriter& operator=(const PrefixWriter&) = delete; +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_PREFIX_WRITER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/prefix_writer_test.cc b/contrib/endpoints/src/grpc/transcoding/prefix_writer_test.cc new file mode 100644 index 00000000000..7727367274b --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/prefix_writer_test.cc @@ -0,0 +1,209 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/prefix_writer.h" + +#include +#include + +#include "google/protobuf/util/internal/expecting_objectwriter.h" +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +using ::testing::InSequence; + +class PrefixWriterTest : public ::testing::Test { + protected: + PrefixWriterTest() : mock_(), expect_(&mock_) {} + + std::unique_ptr Create(const std::string& prefix) { + return std::unique_ptr(new PrefixWriter(prefix, &mock_)); + } + + google::protobuf::util::converter::MockObjectWriter mock_; + google::protobuf::util::converter::ExpectingObjectWriter expect_; + InSequence seq_; // all our expectations must be ordered +}; + +TEST_F(PrefixWriterTest, EmptyPrefix) { + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "a"); + expect_.RenderBytes("by", "b"); + expect_.RenderInt32("i", google::protobuf::int32(1)); + expect_.RenderUint32("ui", google::protobuf::uint32(2)); + expect_.RenderInt64("i64", google::protobuf::int64(3)); + expect_.RenderUint64("ui64", google::protobuf::uint64(4)); + expect_.RenderBool("b", true); + expect_.RenderNull("null"); + expect_.StartObject("B"); + expect_.RenderString("y", "b"); + expect_.EndObject(); // B + expect_.EndObject(); // A + expect_.EndObject(); // "" + + auto w = Create(""); + + w->StartObject(""); + w->StartObject("A"); + w->RenderString("x", "a"); + w->RenderBytes("by", "b"); + w->RenderInt32("i", google::protobuf::int32(1)); + w->RenderUint32("ui", google::protobuf::uint32(2)); + w->RenderInt64("i64", google::protobuf::int64(3)); + w->RenderUint64("ui64", google::protobuf::uint64(4)); + w->RenderBool("b", true); + w->RenderNull("null"); + w->StartObject("B"); + w->RenderString("y", "b"); + w->EndObject(); // B + w->EndObject(); // A + w->EndObject(); // "" +} + +TEST_F(PrefixWriterTest, OneLevelPrefix1) { + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "a"); + expect_.StartObject("B"); + expect_.RenderString("y", "b"); + expect_.EndObject(); // B + expect_.EndObject(); // A + expect_.EndObject(); // "" + + expect_.StartObject("C"); + expect_.StartObject("A"); + expect_.RenderString("z", "c"); + expect_.EndObject(); // C + expect_.EndObject(); // A + + auto w = Create("A"); + + w->StartObject(""); + w->RenderString("x", "a"); + w->StartObject("B"); + w->RenderString("y", "b"); + w->EndObject(); // B + w->EndObject(); // A, "" + + w->StartObject("C"); + w->RenderString("z", "c"); + w->EndObject(); // C, A +} + +TEST_F(PrefixWriterTest, OneLevelPrefix2) { + expect_.StartObject("x"); + expect_.RenderString("A", "a"); + expect_.EndObject(); // "A" + + expect_.StartObject("by"); + expect_.RenderBytes("A", "b"); + expect_.EndObject(); // "A" + + expect_.StartObject("i32"); + expect_.RenderInt32("A", google::protobuf::int32(-32)); + expect_.EndObject(); // "A" + + expect_.StartObject("ui32"); + expect_.RenderUint32("A", google::protobuf::uint32(32)); + expect_.EndObject(); // "A" + + expect_.StartObject("i64"); + expect_.RenderInt64("A", google::protobuf::int64(-64)); + expect_.EndObject(); // "A" + + expect_.StartObject("ui64"); + expect_.RenderUint64("A", google::protobuf::uint64(64)); + expect_.EndObject(); // "A" + + expect_.StartObject("b"); + expect_.RenderBool("A", false); + expect_.EndObject(); // "A" + + expect_.StartObject("nil"); + expect_.RenderNull("A"); + expect_.EndObject(); // "A" + + auto w = Create("A"); + + w->RenderString("x", "a"); + w->RenderBytes("by", "b"); + w->RenderInt32("i32", google::protobuf::int32(-32)); + w->RenderUint32("ui32", google::protobuf::uint32(32)); + w->RenderInt64("i64", google::protobuf::int64(-64)); + w->RenderUint64("ui64", google::protobuf::uint64(64)); + w->RenderBool("b", false); + w->RenderNull("nil"); +} + +TEST_F(PrefixWriterTest, TwoLevelPrefix) { + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.StartObject("B"); + expect_.RenderString("x", "a"); + expect_.EndObject(); // B + expect_.EndObject(); // A + expect_.EndObject(); // "" + + expect_.StartObject("C"); + expect_.StartObject("A"); + expect_.StartObject("B"); + expect_.RenderString("y", "b"); + expect_.EndObject(); // B + expect_.EndObject(); // A + expect_.EndObject(); // C + + auto w = Create("A.B"); + + w->StartObject(""); + w->RenderString("x", "a"); + w->EndObject(); // B, A, "" + + w->StartObject("C"); + w->RenderString("y", "b"); + w->EndObject(); // B, A, C +} + +TEST_F(PrefixWriterTest, ThreeLevelPrefix) { + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.StartObject("B"); + expect_.StartObject("C"); + expect_.RenderString("x", "a"); + expect_.EndObject(); // C + expect_.EndObject(); // B + expect_.EndObject(); // A + expect_.EndObject(); // "" + + auto w = Create("A.B.C"); + + w->StartObject(""); + w->RenderString("x", "a"); + w->EndObject(); // C, B, A, "" +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/proto_stream_tester.cc b/contrib/endpoints/src/grpc/transcoding/proto_stream_tester.cc new file mode 100644 index 00000000000..f45d223ac18 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/proto_stream_tester.cc @@ -0,0 +1,117 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/proto_stream_tester.h" + +#include + +#include "google/protobuf/stubs/status.h" +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { + +ProtoStreamTester::ProtoStreamTester(MessageStream& stream, bool delimiters) + : stream_(stream), delimiters_(delimiters) {} + +bool ProtoStreamTester::ExpectNone() { + // First check the status of the stream + if (!ExpectStatusEq(google::protobuf::util::error::OK)) { + return false; + } + std::string message; + if (stream_.NextMessage(&message)) { + ADD_FAILURE() << "ProtoStreamTester::ValidateNone: NextMessage() returned " + "true; expected false.\n"; + return false; + } + return true; +} + +bool ProtoStreamTester::ExpectFinishedEq(bool expected) { + // First check the status of the stream + if (!ExpectStatusEq(google::protobuf::util::error::OK)) { + return false; + } + if (expected != stream_.Finished()) { + ADD_FAILURE() + << (expected + ? "The stream was expected to be finished, but it's not.\n" + : "The stream was not expected to be finished, but it is.\n"); + EXPECT_EQ(expected, stream_.Finished()); + return false; + } + return true; +} + +namespace { + +unsigned DelimiterToSize(const unsigned char* delimiter) { + unsigned size = 0; + size = size | static_cast(delimiter[1]); + size <<= 8; + size = size | static_cast(delimiter[2]); + size <<= 8; + size = size | static_cast(delimiter[3]); + size <<= 8; + size = size | static_cast(delimiter[4]); + return size; +} + +} // namespace + +bool ProtoStreamTester::ValidateDelimiter(const std::string& message) { + // First check the status of the stream + if (!ExpectStatusEq(google::protobuf::util::error::OK)) { + return false; + } + if (message.size() < kDelimiterSize) { + ADD_FAILURE() << "ProtoStreamTester::ValidateSizeDelimiter: message size " + "is less than a delimiter size (5).\n"; + } + int size_actual = + DelimiterToSize(reinterpret_cast(&message[0])); + int size_expected = static_cast(message.size() - kDelimiterSize); + if (size_expected != size_actual) { + ADD_FAILURE() << "The size extracted from the message delimiter: " + << size_actual + << " doesn't match the size of the message: " << size_expected + << std::endl; + return false; + } + return true; +} + +bool ProtoStreamTester::ExpectStatusEq(int error_code) { + if (error_code != stream_.Status().error_code()) { + ADD_FAILURE() + << "ObjectTranslatorTest::ValidateStatus: Status doesn't match " + "expected: " + << error_code << " actual: " << stream_.Status().error_code() << " - " + << stream_.Status().error_message() << std::endl; + return false; + } + return true; +} + +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/proto_stream_tester.h b/contrib/endpoints/src/grpc/transcoding/proto_stream_tester.h new file mode 100644 index 00000000000..73529d877f1 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/proto_stream_tester.h @@ -0,0 +1,118 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_PROTO_STREAM_TESTER_H_ +#define GRPC_TRANSCODING_PROTO_STREAM_TESTER_H_ + +#include + +#include "google/protobuf/stubs/status.h" +#include "google/protobuf/text_format.h" +#include "google/protobuf/util/message_differencer.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/message_stream.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { + +// A helper that makes it easy to test a stream of protobuf messages +// represented through a MessageStream interface. It handles matching +// proto messages, validating the GRPC message delimiter (see +// http://www.grpc.io/docs/guides/wire.html) and automatically checking the +// stream status. +class ProtoStreamTester { + public: + // stream - the stream to be tested + // delimiters - whether the messages have delimiters or not + ProtoStreamTester(MessageStream& stream, bool delimiters); + + // Validation methods + template + bool ExpectNextEq(const std::string& expected_proto_text); + bool ExpectNone(); + bool ExpectFinishedEq(bool expected); + bool ExpectStatusEq(int error_code); + + private: + // Validates the GRPC message delimiter at the beginning + // of the message. + bool ValidateDelimiter(const std::string& message); + + MessageStream& stream_; + bool delimiters_; + + static const int kDelimiterSize = 5; +}; + +template +bool ProtoStreamTester::ExpectNextEq(const std::string& expected_proto_text) { + // First check the status of the stream + if (!ExpectStatusEq(google::protobuf::util::error::OK)) { + return false; + } + // Try to get a message + std::string message; + if (!stream_.NextMessage(&message)) { + ADD_FAILURE() << "ProtoStreamTester::ValidateNext: NextMessage() " + "returned false\n"; + // Use ExpectStatusEq() to output the status if it's not OK. + ExpectStatusEq(google::protobuf::util::error::OK); + return false; + } + // Validate the delimiter if it's expected + if (delimiters_) { + if (!ValidateDelimiter(message)) { + return false; + } else { + // Strip the delimiter + message = message.substr(kDelimiterSize); + } + } + // Parse the actual message + MessageType actual; + if (!actual.ParseFromString(message)) { + ADD_FAILURE() << "ProtoStreamTester::ValidateNext: couldn't parse " + "the actual message:\n" + << message << std::endl; + return false; + } + // Parse the expected message + MessageType expected; + if (!google::protobuf::TextFormat::ParseFromString(expected_proto_text, + &expected)) { + ADD_FAILURE() << "ProtoStreamTester::ValidateNext: couldn't parse " + "the expected message:\n" + << expected_proto_text << std::endl; + return false; + } + // Now try matching the protos + if (!google::protobuf::util::MessageDifferencer::Equivalent(expected, + actual)) { + // Use EXPECT_EQ on debug strings to output the diff + EXPECT_EQ(expected.DebugString(), actual.DebugString()); + return false; + } + return true; +} + +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_PROTO_STREAM_TESTER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/request_message_translator.cc b/contrib/endpoints/src/grpc/transcoding/request_message_translator.cc new file mode 100644 index 00000000000..79b6e79c80e --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_message_translator.cc @@ -0,0 +1,161 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/request_message_translator.h" + +#include + +#include "google/protobuf/stubs/bytestream.h" +#include "google/protobuf/util/internal/error_listener.h" +#include "google/protobuf/util/internal/protostream_objectwriter.h" +#include "src/grpc/transcoding/prefix_writer.h" +#include "src/grpc/transcoding/request_weaver.h" + +namespace pb = ::google::protobuf; +namespace pbutil = ::google::protobuf::util; +namespace pbconv = ::google::protobuf::util::converter; + +namespace google { +namespace api_manager { + +namespace transcoding { + +namespace { + +pbconv::ProtoStreamObjectWriter::Options GetProtoWriterOptions() { + auto options = pbconv::ProtoStreamObjectWriter::Options::Defaults(); + // Don't fail the translation if there are unknown fields in JSON. + // This will make sure that we allow backward and forward compatible APIs. + options.ignore_unknown_fields = true; + return options; +} + +} // namespace + +RequestMessageTranslator::RequestMessageTranslator( + google::protobuf::util::TypeResolver& type_resolver, bool output_delimiter, + RequestInfo request_info) + : message_(), + sink_(&message_), + error_listener_(), + proto_writer_(&type_resolver, *request_info.message_type, &sink_, + &error_listener_, GetProtoWriterOptions()), + request_weaver_(), + prefix_writer_(), + writer_pipeline_(&proto_writer_), + output_delimiter_(output_delimiter), + finished_(false) { + // Create a RequestWeaver if we have variable bindings to weave + if (!request_info.variable_bindings.empty()) { + request_weaver_.reset(new RequestWeaver( + std::move(request_info.variable_bindings), writer_pipeline_)); + writer_pipeline_ = request_weaver_.get(); + } + + // Create a PrefixWriter if there is a prefix to write + if (!request_info.body_field_path.empty() && + "*" != request_info.body_field_path) { + prefix_writer_.reset( + new PrefixWriter(request_info.body_field_path, writer_pipeline_)); + writer_pipeline_ = prefix_writer_.get(); + } + + if (output_delimiter_) { + // Reserve space for the delimiter at the begining of the message_ + ReserveDelimiterSpace(); + } +} + +RequestMessageTranslator::~RequestMessageTranslator() {} + +bool RequestMessageTranslator::Finished() const { return finished_; } + +bool RequestMessageTranslator::NextMessage(std::string* message) { + if (Finished()) { + // Finished reading + return false; + } + if (!proto_writer_.done()) { + // No full message yet + return false; + } + if (output_delimiter_) { + WriteDelimiter(); + } + *message = std::move(message_); + finished_ = true; + return true; +} + +void RequestMessageTranslator::ReserveDelimiterSpace() { + static char reserved[kDelimiterSize] = {0}; + sink_.Append(reserved, sizeof(reserved)); +} + +namespace { + +void SizeToDelimiter(unsigned size, unsigned char* delimiter) { + delimiter[0] = 0; // compression bit + + // big-endian 32-bit length + delimiter[4] = 0xFF & size; + size >>= 8; + delimiter[3] = 0xFF & size; + size >>= 8; + delimiter[2] = 0xFF & size; + size >>= 8; + delimiter[1] = 0xFF & size; +} + +} // namespace + +void RequestMessageTranslator::WriteDelimiter() { + // Asumming that the message_.size() - kDelimiterSize is less than UINT_MAX + SizeToDelimiter(static_cast(message_.size() - kDelimiterSize), + reinterpret_cast(&message_[0])); +} + +void RequestMessageTranslator::StatusErrorListener::InvalidName( + const ::google::protobuf::util::converter::LocationTrackerInterface& loc, + ::google::protobuf::StringPiece unknown_name, + ::google::protobuf::StringPiece message) { + status_ = ::google::protobuf::util::Status( + ::google::protobuf::util::error::INVALID_ARGUMENT, + loc.ToString() + ": " + message.ToString()); +} + +void RequestMessageTranslator::StatusErrorListener::InvalidValue( + const ::google::protobuf::util::converter::LocationTrackerInterface& loc, + ::google::protobuf::StringPiece type_name, + ::google::protobuf::StringPiece value) { + status_ = ::google::protobuf::util::Status( + ::google::protobuf::util::error::INVALID_ARGUMENT, + loc.ToString() + ": invalid value " + value.ToString() + " for type " + + type_name.ToString()); +} + +void RequestMessageTranslator::StatusErrorListener::MissingField( + const ::google::protobuf::util::converter::LocationTrackerInterface& loc, + ::google::protobuf::StringPiece missing_name) { + status_ = ::google::protobuf::util::Status( + ::google::protobuf::util::error::INVALID_ARGUMENT, + loc.ToString() + ": missing field " + missing_name.ToString()); +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/request_message_translator.h b/contrib/endpoints/src/grpc/transcoding/request_message_translator.h new file mode 100644 index 00000000000..dfe802246b5 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_message_translator.h @@ -0,0 +1,202 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_REQUEST_MESSAGE_TRANSLATOR_H_ +#define GRPC_TRANSCODING_REQUEST_MESSAGE_TRANSLATOR_H_ + +#include +#include + +#include "google/protobuf/stubs/bytestream.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/internal/error_listener.h" +#include "google/protobuf/util/internal/protostream_objectwriter.h" +#include "google/protobuf/util/type_resolver.h" +#include "src/grpc/transcoding/message_stream.h" +#include "src/grpc/transcoding/prefix_writer.h" +#include "src/grpc/transcoding/request_weaver.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// RequestInfo contains the information needed for request translation. +struct RequestInfo { + // The protobuf type that we are translating to. + const google::protobuf::Type* message_type; + + // body_field_path is a dot-delimited chain of protobuf field names that + // defines the (potentially nested) location in the message, where the + // translated HTTP body must be inserted. E.g. "shelf.theme" means that the + // translated HTTP body must be inserted into the "theme" field of the "shelf" + // field of the request message. + std::string body_field_path; + + // A collection of variable bindings extracted from the HTTP url or other + // sources that must be injected into certain fields of the translated + // message. + std::vector variable_bindings; +}; + +// RequestMessageTranslator translates ObjectWriter events into a single +// protobuf message. The protobuf message is built based on the input +// ObjectWriter events and a RequestInfo. +// If output_delimiter is true, RequestMessageTranslator will prepend the output +// message with a GRPC message delimiter - a 1-byte compression flag and a +// 4-byte message length (see http://www.grpc.io/docs/guides/wire.html). +// The translated message is exposed through MessageStream interface. +// +// The implementation uses a pipeline of ObjectWriters to do the job: +// PrefixWriter -> RequestWeaver -> ProtoStreamObjectWriter +// +// - PrefixWriter writes the body prefix making sure that the body goes to the +// right place and forwards the writer events to the RequestWeaver. This link +// will be absent if the prefix is empty. +// - RequestWeaver injects the variable bindings and forwards the writer events +// to the ProtoStreamObjectWriter. This link will be absent if there are no +// variable bindings to weave. +// - ProtoStreamObjectWriter does the actual proto writing. +// +// Example: +// RequestMessageTranslator t(type_resolver, true, std::move(request_info)); +// +// ObjectWriter& input = t.Input(); +// +// input.StartObject(""); +// ... +// write the request body using input ObjectWriter +// ... +// input.EndObject(); +// +// if (!t.Status().ok()) { +// printf("Error: %s\n", t->Status().ErrorMessage().as_string().c_str()); +// return; +// } +// +// std::string message; +// if (t.NextMessage(&message)) { +// printf("Message=%s\n", message.c_str()); +// } +// +class RequestMessageTranslator : public MessageStream { + public: + // type_resolver is forwarded to the ProtoStreamObjectWriter that does the + // actual proto writing. + // output_delimiter specifies whether to output the GRPC 5 byte message + // delimiter before the message or not. + RequestMessageTranslator(google::protobuf::util::TypeResolver& type_resolver, + bool output_delimiter, RequestInfo request_info); + + ~RequestMessageTranslator(); + + // An ObjectWriter that takes the input object to translate + google::protobuf::util::converter::ObjectWriter& Input() { + return *writer_pipeline_; + } + + // MessageStream methods + bool NextMessage(std::string* message); + bool Finished() const; + google::protobuf::util::Status Status() const { + return error_listener_.status(); + } + + private: + // Reserves space (5 bytes) for the GRPC delimiter to be written later. As it + // requires the length of the message, we can't write it before the message + // itself. + void ReserveDelimiterSpace(); + + // Writes the wire delimiter into the reserved delimiter space at the begining + // of this->message_. + void WriteDelimiter(); + + // The message being written + std::string message_; + + // StringByteSink instance that appends the bytes to this->message_. We pass + // this to the ProtoStreamObjectWriter for writing the translated message. + google::protobuf::strings::StringByteSink sink_; + + // StatusErrorListener converts the error events into a Status + class StatusErrorListener + : public ::google::protobuf::util::converter::ErrorListener { + public: + StatusErrorListener() : status_(::google::protobuf::util::Status::OK) {} + virtual ~StatusErrorListener() {} + + ::google::protobuf::util::Status status() const { return status_; } + + // ErrorListener implementation + void InvalidName( + const ::google::protobuf::util::converter::LocationTrackerInterface& + loc, + ::google::protobuf::StringPiece unknown_name, + ::google::protobuf::StringPiece message); + void InvalidValue( + const ::google::protobuf::util::converter::LocationTrackerInterface& + loc, + ::google::protobuf::StringPiece type_name, + ::google::protobuf::StringPiece value); + void MissingField( + const ::google::protobuf::util::converter::LocationTrackerInterface& + loc, + ::google::protobuf::StringPiece missing_name); + + private: + ::google::protobuf::util::Status status_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StatusErrorListener); + }; + + // ErrorListener implementation that converts the error events into + // a status. + StatusErrorListener error_listener_; + + // The proto writer for writing the actual proto bytes + google::protobuf::util::converter::ProtoStreamObjectWriter proto_writer_; + + // A RequestWeaver for writing the variable bindings + std::unique_ptr request_weaver_; + + // A PrefixWriter for writing the body prefix + std::unique_ptr prefix_writer_; + + // The ObjectWriter that will receive the events + // This is either &proto_writer_, request_weaver_.get() or + // prefix_writer_.get() + google::protobuf::util::converter::ObjectWriter* writer_pipeline_; + + // Whether to ouput a delimiter before the message or not + bool output_delimiter_; + + // A flag that indicates whether the message has been already read or not + // This helps with the MessageStream implementation. + bool finished_; + + // GRPC delimiter size = 1 + 4 - 1-byte compression flag and 4-byte message + // length. + static const int kDelimiterSize = 5; + + RequestMessageTranslator(const RequestMessageTranslator&) = delete; + RequestMessageTranslator& operator=(const RequestMessageTranslator&) = delete; +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_REQUEST_MESSAGE_TRANSLATOR_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/request_message_translator_test.cc b/contrib/endpoints/src/grpc/transcoding/request_message_translator_test.cc new file mode 100644 index 00000000000..50940c21535 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_message_translator_test.cc @@ -0,0 +1,501 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/request_message_translator.h" + +#include +#include + +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/type.pb.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/bookstore.pb.h" +#include "src/grpc/transcoding/request_translator_test_base.h" +#include "src/grpc/transcoding/test_common.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +class RequestMessageTranslatorTest : public RequestTranslatorTestBase { + protected: + RequestMessageTranslatorTest() : RequestTranslatorTestBase() {} + + template + bool ExpectMessageEq(const std::string& expected_proto_text) { + // We expect only one message + return Tester().ExpectFinishedEq(false) && + Tester().ExpectNextEq(expected_proto_text) && + Tester().ExpectFinishedEq(true); + } + + google::protobuf::util::converter::ObjectWriter& Input() { + return translator_->Input(); + } + + private: + // RequestTranslatorTestBase::Create() + virtual MessageStream* Create( + google::protobuf::util::TypeResolver& type_resolver, + bool output_delimiters, RequestInfo request_info) { + translator_.reset(new RequestMessageTranslator( + type_resolver, output_delimiters, std::move(request_info))); + return translator_.get(); + } + + std::unique_ptr translator_; +}; + +TEST_F(RequestMessageTranslatorTest, Simple) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input() + .StartObject("") + ->RenderString("name", "1") + ->RenderString("theme", "History") + ->EndObject(); + + auto expected = R"( + name : "1" + theme : "History" + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, Nested) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateShelfRequest"); + Build(); + Input() + .StartObject("") + ->StartObject("shelf") + ->RenderString("name", "2") + ->RenderString("theme", "Russian") + ->EndObject() + ->EndObject(); + + auto expected = R"( + shelf : { + name : "2" + theme : "Russian" + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, MultipleLevelNested) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + Build(); + Input() + .StartObject("") + ->RenderString("shelf", "99") + ->StartObject("book") + ->RenderString("name", "999") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "War and Peace") + ->StartObject("authorInfo") + ->RenderString("firstName", "Leo") + ->RenderString("lastName", "Tolstoy") + ->StartObject("bio") + ->RenderString("yearBorn", "1830") + ->RenderString("yearDied", "1910") + ->RenderString("text", "bio text") + ->EndObject() // bio + ->EndObject() // authorInfo + ->EndObject() // book + ->EndObject(); // "" + + auto expected = R"( + shelf : 99 + book { + name : "999" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + bio { + year_born : 1830 + year_died : 1910 + text : "bio text" + } + } + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, Empty) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input().StartObject("")->EndObject(); + + EXPECT_TRUE(ExpectMessageEq("")); +} + +TEST_F(RequestMessageTranslatorTest, Delimiter) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetOutputDelimiters(true); + Build(); + Input() + .StartObject("") + ->RenderString("shelf", "7") + ->StartObject("book") + ->RenderString("name", "77") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "Anna Karenina") + ->EndObject() // book + ->EndObject(); // "" + + auto expected = R"( + shelf : 7 + book { + name : "77" + author : "Leo Tolstoy" + title : "Anna Karenina" + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, DelimiterDifferentSizes) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetOutputDelimiters(true); + + auto sizes = {1, 256, 1024, 1234, 4096, 65537}; + for (auto size : sizes) { + Build(); + + auto title = GenerateInput("0123456789abcdefgh", size); + Input() + .StartObject("") + ->RenderString("shelf", "7") + ->StartObject("book") + ->RenderString("name", "77") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", title) + ->EndObject() // book + ->EndObject(); // "" + + auto expected = R"( + shelf : 7 + book { + name : "77" + author : "Leo Tolstoy" + title : ")" + + title + R"(" + })"; + + EXPECT_TRUE(ExpectMessageEq(expected)) + << "Delimiter test failed for size " << size << std::endl; + } +} + +TEST_F(RequestMessageTranslatorTest, DelimiterEmpty) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetOutputDelimiters(true); + Build(); + Input().StartObject("")->EndObject(); // "" + + EXPECT_TRUE(ExpectMessageEq("")); +} + +TEST_F(RequestMessageTranslatorTest, Bindings) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + AddVariableBinding("shelf", "99"); + AddVariableBinding("book.author", "Leo Tolstoy"); + AddVariableBinding("book.authorInfo.firstName", "Leo"); + AddVariableBinding("book.authorInfo.lastName", "Tolstoy"); + Build(); + Input() + .StartObject("") + ->StartObject("book") + ->RenderString("name", "999") + ->RenderString("title", "War and Peace") + // authorInfo { + // first_name : "Leo" <-- weaved + // last_name : "Tolstoy" <-- weaved + // } + // author : "Leo Tolstoy" <-- weaved + ->EndObject() // book + // weaved: shelf : 99 <-- weaved + ->EndObject(); // "" + + auto expected = R"( + shelf : 99 + book { + name : "999" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + } + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, Prefix) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book"); + Build(); + Input() + .StartObject("") + // book { <-- prefix + ->RenderString("name", "777") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "War and Peace") + // } <-- end of prefix + ->EndObject(); // "" + + auto expected = R"( + book { + name : "777" + author : "Leo Tolstoy" + title : "War and Peace" + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, NestedPrefix) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book.authorInfo.bio"); + Build(); + Input() + .StartObject("") + // book { authorInfo { bio { <-- prefix + ->RenderString("yearBorn", "1830") + ->RenderString("yearDied", "1910") + ->RenderString("text", "bio text") + // }}} <-- end of prefix + ->EndObject(); // "" + + auto expected = R"( + book { + author_info { + bio { + year_born : 1830 + year_died : 1910 + text : "bio text" + } + } + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, PrefixAndBinding) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book"); + AddVariableBinding("shelf", "99"); + SetOutputDelimiters(true); + Build(); + Input() + .StartObject("") + // book { <-- prefix + ->RenderString("name", "999") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "War and Peace") + // } <-- end of prefix + // shelf : 99 <-- weaved + ->EndObject(); // "" + + auto expected = R"( + shelf : 99 + book { + name : "999" + author : "Leo Tolstoy" + title : "War and Peace" + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, ScalarBody) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateShelfRequest"); + SetBodyPrefix("shelf.theme"); + Build(); + Input().RenderString("", "History"); + + auto expected = R"( + shelf { + theme : "History" + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, ListBody) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("ListShelvesResponse"); + SetBodyPrefix("shelves"); + Build(); + Input() + .StartList("") + ->StartObject("") + ->RenderString("name", "1") + ->RenderString("theme", "History") + ->EndObject() // "" + ->StartObject("") + ->RenderString("name", "2") + ->RenderString("theme", "Mystery") + ->EndObject() // "" + ->EndList(); // "" + + auto expected = R"( + shelves { + name : "1" + theme : "History" + } + shelves { + name : "2" + theme : "Mystery" + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, PartialObject) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + Build(); + EXPECT_EQ(true, Tester().ExpectNone()); + + Input().StartObject(""); + EXPECT_EQ(true, Tester().ExpectNone()); + + Input().RenderString("shelf", "99"); + EXPECT_EQ(true, Tester().ExpectNone()); + + Input() + .StartObject("book") + ->RenderString("name", "999") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "War and Peace"); + EXPECT_EQ(true, Tester().ExpectNone()); + + Input() + .StartObject("authorInfo") + ->RenderString("firstName", "Leo") + ->RenderString("lastName", "Tolstoy") + ->EndObject(); // authorInfo + EXPECT_EQ(true, Tester().ExpectNone()); + + Input() + .EndObject() // book + ->EndObject(); // "" + + auto expected = R"( + shelf : 99 + book { + name : "999" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + } + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +TEST_F(RequestMessageTranslatorTest, UnexpectedScalarBody) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Book"); + Build(); + Input().RenderString("", "History"); + + EXPECT_TRUE(Tester().ExpectStatusEq( + ::google::protobuf::util::error::INVALID_ARGUMENT)); +} + +TEST_F(RequestMessageTranslatorTest, UnexpectedList) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Book"); + Build(); + Input().StartList("")->EndList(); + + EXPECT_TRUE(Tester().ExpectStatusEq( + ::google::protobuf::util::error::INVALID_ARGUMENT)); +} + +TEST_F(RequestMessageTranslatorTest, IgnoreUnkownFields) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateShelfRequest"); + Build(); + Input() + .StartObject("") + ->StartObject("shelf") + ->RenderString("name", "3") + ->RenderString("theme", "Classics") + // Unkown field + ->RenderString("unknownField", "value") + ->EndObject() + // Unkown object + ->StartObject("unknownObject") + ->RenderString("field", "value") + ->EndObject() + // Unkown list + ->StartList("unknownList") + ->RenderString("field1", "value1") + ->RenderString("field2", "value2") + ->EndList() + ->EndObject(); + + auto expected = R"( + shelf : { + name : "3" + theme : "Classics" + } + )"; + + EXPECT_TRUE(ExpectMessageEq(expected)); +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/request_stream_translator.cc b/contrib/endpoints/src/grpc/transcoding/request_stream_translator.cc new file mode 100644 index 00000000000..a6963eef52d --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_stream_translator.cc @@ -0,0 +1,288 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/request_stream_translator.h" + +#include +#include + +#include "google/protobuf/stubs/stringpiece.h" +#include "src/grpc/transcoding/request_message_translator.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +namespace pb = google::protobuf; +namespace pbutil = google::protobuf::util; +namespace pberr = google::protobuf::util::error; +namespace pbconv = google::protobuf::util::converter; + +RequestStreamTranslator::RequestStreamTranslator( + google::protobuf::util::TypeResolver& type_resolver, bool output_delimiters, + RequestInfo request_info) + : type_resolver_(type_resolver), + status_(), + request_info_(std::move(request_info)), + output_delimiters_(output_delimiters), + translator_(), + messages_(), + depth_(0), + done_(false) {} + +RequestStreamTranslator::~RequestStreamTranslator() {} + +bool RequestStreamTranslator::NextMessage(std::string* message) { + if (!messages_.empty()) { + *message = std::move(messages_.front()); + messages_.pop_front(); + return true; + } else { + return false; + } +} + +bool RequestStreamTranslator::Finished() const { + return (messages_.empty() && done_) || !status_.ok(); +} + +RequestStreamTranslator* RequestStreamTranslator::StartObject( + pb::StringPiece name) { + if (!status_.ok()) { + // In error state - return right away + return this; + } + if (depth_ == 0) { + // In depth_ == 0 case we expect only StartList() + status_ = pbutil::Status(pberr::INVALID_ARGUMENT, + "Expected an array instead of an object"); + return this; + } + if (depth_ == 1) { + // An element of the outermost array - start the ProtoMessageTranslator to + // to translate the array. + StartMessageTranslator(); + } + translator_->Input().StartObject(name); + ++depth_; + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::EndObject() { + if (!status_.ok()) { + // In error state - return right away + return this; + } + --depth_; + if (depth_ < 1) { + status_ = + pbutil::Status(pberr::INVALID_ARGUMENT, "Mismatched end of object."); + return this; + } + translator_->Input().EndObject(); + if (depth_ == 1) { + // An element of the outermost array was closed - end the + // ProtoMessageTranslator to save the translated message. + EndMessageTranslator(); + } + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::StartList( + pb::StringPiece name) { + if (!status_.ok()) { + // In error state - return right away + return this; + } + if (depth_ == 0) { + // Started the outermost array - do nothing + ++depth_; + return this; + } + if (depth_ == 1) { + // This means we have an array of arrays. This can happen if the HTTP body + // is mapped to a repeated field. + // Start the ProtoMessageTranslator to translate the array. + StartMessageTranslator(); + } + translator_->Input().StartList(name); + ++depth_; + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::EndList() { + if (!status_.ok()) { + // In error state - return right away + return this; + } + --depth_; + if (depth_ < 0) { + status_ = + pbutil::Status(pberr::INVALID_ARGUMENT, "Mismatched end of array."); + return this; + } + if (depth_ == 0) { + // We ended the root list, we're all done! + done_ = true; + return this; + } + translator_->Input().EndList(); + if (depth_ == 1) { + // An element of the outermost array was closed - end the + // ProtoMessageTranslator to save the translated message. + EndMessageTranslator(); + } + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderBool( + pb::StringPiece name, bool value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderBool(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderInt32( + pb::StringPiece name, pb::int32 value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderInt32(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderUint32( + pb::StringPiece name, pb::uint32 value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderUint32(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderInt64( + pb::StringPiece name, pb::int64 value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderInt64(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderUint64( + pb::StringPiece name, pb::uint64 value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderUint64(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderDouble( + pb::StringPiece name, double value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderDouble(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderFloat( + pb::StringPiece name, float value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderFloat(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderString( + pb::StringPiece name, pb::StringPiece value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderString(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderBytes( + pb::StringPiece name, pb::StringPiece value) { + RenderData(name, [this, name, value]() { + translator_->Input().RenderBytes(name, value); + }); + return this; +} + +RequestStreamTranslator* RequestStreamTranslator::RenderNull( + pb::StringPiece name) { + RenderData(name, [this, name]() { translator_->Input().RenderNull(name); }); + return this; +} + +void RequestStreamTranslator::StartMessageTranslator() { + RequestInfo request_info; + request_info.message_type = request_info_.message_type; + request_info.body_field_path = request_info_.body_field_path; + // As we need to weave the variable bindings only for the first message, we + // can use vector::swap() to avoid copying and to clear the bindings from + // request_info_, s.t. the subsequent messages don't use them. + request_info.variable_bindings.swap(request_info_.variable_bindings); + // Create a RequestMessageTranslator to handle the events in a single message + translator_.reset(new RequestMessageTranslator( + type_resolver_, output_delimiters_, std::move(request_info))); +} + +void RequestStreamTranslator::EndMessageTranslator() { + if (!translator_->Status().ok()) { + // Translation wasn't successful + status_ = translator_->Status(); + return; + } + // Save the translated message and reset our state for the next one. + std::string message; + if (translator_->NextMessage(&message)) { + messages_.emplace_back(std::move(message)); + } else { + // This shouldn't happen unless something like StartList(), StartObject(), + // EndList() has been called + status_ = pbutil::Status(pberr::INVALID_ARGUMENT, "Invalid object"); + } + translator_.reset(); +} + +void RequestStreamTranslator::RenderData(pb::StringPiece name, + std::function renderer) { + if (!status_.ok()) { + // In error state - ignore + return; + } + if (depth_ == 0) { + // In depth_ == 0 case we expect only a StartList() + status_ = pbutil::Status(pberr::INVALID_ARGUMENT, + "Expected an array instead of a scalar value."); + } else if (depth_ == 1) { + // This means we have an array of scalar values. This can happen if the HTTP + // body is mapped to a scalar field. + // We need to start the ProtoMessageTranslator, render the scalar value to + // translate it and end the ProtoMessageTranslator to save the translated + // message. + StartMessageTranslator(); + renderer(); + EndMessageTranslator(); + } else { // depth_ > 1 + renderer(); + } +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/request_stream_translator.h b/contrib/endpoints/src/grpc/transcoding/request_stream_translator.h new file mode 100644 index 00000000000..1f512d2002f --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_stream_translator.h @@ -0,0 +1,145 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_REQUEST_STREAM_TRANSLATOR_H_ +#define GRPC_TRANSCODING_REQUEST_STREAM_TRANSLATOR_H_ + +#include +#include +#include + +#include "google/protobuf/stubs/stringpiece.h" +#include "google/protobuf/util/internal/object_writer.h" +#include "google/protobuf/util/type_resolver.h" +#include "src/grpc/transcoding/message_stream.h" +#include "src/grpc/transcoding/request_message_translator.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// Translates ObjectWriter events into protobuf messages for streaming requests. +// RequestStreamTranslator handles the outermost array and for each element uses +// a RequestMessageTranslator to translate it to a proto message. Collects the +// translated messages into a deque and exposes those through MessageStream +// interface. +// Example: +// RequestMessageTranslator t(type_resolver, true, std::move(request_info)); +// +// t.StartList(""); +// ... +// t.StartObject(""); +// write object 1 +// t.EndObject(); +// ... +// t.StartObject(""); +// write object 2 +// t.EndObject(); +// ... +// t.EndList(); +// +// if (!t.Status().ok()) { +// printf("Error: %s\n", t->Status().ErrorMessage().as_string().c_str()); +// return; +// } +// +// std::string message; +// while (t.NextMessage(&message)) { +// printf("Message=%s\n", message.c_str()); +// } +// +class RequestStreamTranslator + : public google::protobuf::util::converter::ObjectWriter, + public MessageStream { + public: + RequestStreamTranslator(google::protobuf::util::TypeResolver& type_resolver, + bool output_delimiters, RequestInfo request_info); + ~RequestStreamTranslator(); + + // MessageStream methods + bool NextMessage(std::string* message); + bool Finished() const; + google::protobuf::util::Status Status() const { return status_; } + + private: + // ObjectWriter methods. + RequestStreamTranslator* StartObject(google::protobuf::StringPiece name); + RequestStreamTranslator* EndObject(); + RequestStreamTranslator* StartList(google::protobuf::StringPiece name); + RequestStreamTranslator* EndList(); + RequestStreamTranslator* RenderBool(google::protobuf::StringPiece name, + bool value); + RequestStreamTranslator* RenderInt32(google::protobuf::StringPiece name, + google::protobuf::int32 value); + RequestStreamTranslator* RenderUint32(google::protobuf::StringPiece name, + google::protobuf::uint32 value); + RequestStreamTranslator* RenderInt64(google::protobuf::StringPiece name, + google::protobuf::int64 value); + RequestStreamTranslator* RenderUint64(google::protobuf::StringPiece name, + google::protobuf::uint64 value); + RequestStreamTranslator* RenderDouble(google::protobuf::StringPiece name, + double value); + RequestStreamTranslator* RenderFloat(google::protobuf::StringPiece name, + float value); + RequestStreamTranslator* RenderString(google::protobuf::StringPiece name, + google::protobuf::StringPiece value); + RequestStreamTranslator* RenderBytes(google::protobuf::StringPiece name, + google::protobuf::StringPiece value); + RequestStreamTranslator* RenderNull(google::protobuf::StringPiece name); + + // Sets up the ProtoMessageHelper to handle writing data. + void StartMessageTranslator(); + + // Closes down the ProtoMessageHelper and stores its message. + void EndMessageTranslator(); + + // Helper method to render a single piece of data, to reuse code. + void RenderData(google::protobuf::StringPiece name, + std::function renderer); + + // TypeResolver to be passed to the RequestMessageTranslator + google::protobuf::util::TypeResolver& type_resolver_; + + // The status of the translation + google::protobuf::util::Status status_; + + // The request info + RequestInfo request_info_; + + // Whether to prefix each message with a delimiter or not + bool output_delimiters_; + + // The ProtoMessageWriter that is currently writing a message, or null if we + // are at the root or have invalid input. + std::unique_ptr translator_; + + // Holds the messages we've translated so far. + std::deque messages_; + + // Depth within the object tree. We special case the root level. + int depth_; + + // Done with the translation (i.e., have seen the last EndList()) + bool done_; + + RequestStreamTranslator(const RequestStreamTranslator&) = delete; + RequestStreamTranslator& operator=(const RequestStreamTranslator&) = delete; +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google +#endif // API_MANAGER_TRANSCODING_REQUEST_STREAM_TRANSLATOR_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/request_stream_translator_test.cc b/contrib/endpoints/src/grpc/transcoding/request_stream_translator_test.cc new file mode 100644 index 00000000000..f086d5259fc --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_stream_translator_test.cc @@ -0,0 +1,582 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/request_stream_translator.h" + +#include +#include + +#include "google/protobuf/type.pb.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/bookstore.pb.h" +#include "src/grpc/transcoding/request_translator_test_base.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +class RequestStreamTranslatorTest : public RequestTranslatorTestBase { + protected: + RequestStreamTranslatorTest() : RequestTranslatorTestBase() {} + + google::protobuf::util::converter::ObjectWriter& Input() { + return *translator_; + } + + private: + // RequestTranslatorTestBase::Create() + virtual MessageStream* Create( + google::protobuf::util::TypeResolver& type_resolver, + bool output_delimiters, RequestInfo request_info) { + translator_.reset(new RequestStreamTranslator( + type_resolver, output_delimiters, std::move(request_info))); + return translator_.get(); + } + + std::unique_ptr translator_; +}; + +TEST_F(RequestStreamTranslatorTest, One) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input() + .StartList("") + ->StartObject("") + ->RenderString("name", "1") + ->RenderString("theme", "History"); + EXPECT_TRUE(Tester().ExpectNone()); + + // EndObject() should produce one message + Input().EndObject(); // "" + EXPECT_TRUE(Tester().ExpectNextEq(R"(name : "1" theme : "History")")); + + // We're not finished yet (still expecting an end of list) + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input().EndList(); // "" + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, Two) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input() + .StartList("") + ->StartObject("") + ->RenderString("name", "1") + ->RenderString("theme", "History") + ->EndObject(); // "" + + EXPECT_TRUE(Tester().ExpectNextEq(R"(name : "1" theme : "History")")); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input() + .StartObject("") + ->RenderString("name", "2") + ->RenderString("theme", "Mistery") + ->EndObject(); // "" + + EXPECT_TRUE(Tester().ExpectNextEq(R"(name : "2" theme : "Mistery")")); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input().EndList(); // "" + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, OneHundred) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + + Input().StartList(""); + for (int i = 0; i < 100; ++i) { + auto n = std::to_string(i + 1); + + Input().StartObject(""); + EXPECT_TRUE(Tester().ExpectNone()); + + Input().RenderString("name", n); + EXPECT_TRUE(Tester().ExpectNone()); + + Input().RenderString("theme", "Theme" + n); + EXPECT_TRUE(Tester().ExpectNone()); + + Input().EndObject(); // "" + + EXPECT_TRUE(Tester().ExpectNextEq("name : \"" + n + + "\" theme : \"Theme" + n + "\"")); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + } + Input().EndList(); // "" + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, Nested) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + Build(); + Input().StartList(""); + + Input() + .StartObject("") + ->RenderString("shelf", "99") + ->StartObject("book") + ->RenderString("name", "999") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "War and Peace") + ->StartObject("authorInfo") + ->RenderString("firstName", "Leo") + ->RenderString("lastName", "Tolstoy") + ->StartObject("bio") + ->RenderString("yearBorn", "1830") + ->RenderString("yearDied", "1910") + ->RenderString("text", "bio text") + ->EndObject() // bio + ->EndObject() // authorInfo + ->EndObject() // book + ->EndObject(); // "" + + auto expected1 = R"( + shelf : 99 + book { + name : "999" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + bio { + year_born : 1830 + year_died : 1910 + text : "bio text" + } + } + } + )"; + + EXPECT_TRUE(Tester().ExpectNextEq(expected1)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input() + .StartObject("") + ->RenderString("shelf", "77") + ->StartObject("book") + ->RenderString("name", "777") + ->RenderString("author", "Fyodor Dostoevski") + ->RenderString("title", "Crime & Punishment") + ->StartObject("authorInfo") + ->RenderString("firstName", "Fyodor") + ->RenderString("lastName", "Dostoevski") + ->StartObject("bio") + ->RenderString("yearBorn", "1850") + ->RenderString("yearDied", "1920") + ->RenderString("text", "bio text") + ->EndObject() // bio + ->EndObject() // authorInfo + ->EndObject() // book + ->EndObject(); // "" + + auto expected2 = R"( + shelf : 77 + book { + name : "777" + author : "Fyodor Dostoevski" + title : "Crime & Punishment" + author_info { + first_name : "Fyodor" + last_name : "Dostoevski" + bio { + year_born : 1850 + year_died : 1920 + text : "bio text" + } + } + } + )"; + + EXPECT_TRUE(Tester().ExpectNextEq(expected2)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input().EndList(); // "" + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, Empty) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + Build(); + + Input().StartList(""); + EXPECT_TRUE(Tester().ExpectNone()); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input().EndList(); // "" + EXPECT_TRUE(Tester().ExpectNone()); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, DelimiterEmpty) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetOutputDelimiters(true); + Build(); + + Input().StartList(""); + EXPECT_TRUE(Tester().ExpectNone()); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input().EndList(); // "" + EXPECT_TRUE(Tester().ExpectNone()); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, Delimiter) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetOutputDelimiters(true); + Build(); + Input().StartList(""); + + Input() + .StartObject("") + ->RenderString("shelf", "99") + ->StartObject("book") + ->RenderString("name", "999") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "War and Peace") + ->EndObject() // book + ->EndObject(); // "" + + auto expected1 = R"( + shelf : 99 + book { + name : "999" + author : "Leo Tolstoy" + title : "War and Peace" + } + )"; + + EXPECT_TRUE(Tester().ExpectNextEq(expected1)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input() + .StartObject("") + ->RenderString("shelf", "77") + ->StartObject("book") + ->RenderString("name", "777") + ->RenderString("author", "Fyodor Dostoevski") + ->RenderString("title", "Crime & Punishment") + ->EndObject() // book + ->EndObject(); // "" + + auto expected2 = R"( + shelf : 77 + book { + name : "777" + author : "Fyodor Dostoevski" + title : "Crime & Punishment" + } + )"; + + EXPECT_TRUE(Tester().ExpectNextEq(expected2)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + Input().EndList(); // "" + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, Prefix) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + SetBodyPrefix("book.authorInfo.bio"); + Build(); + Input() + .StartList("") + ->StartObject("") + // book { authorInfo { biod { <-- prefix + ->RenderString("yearBorn", "1830") + ->RenderString("yearDied", "1910") + ->RenderString("text", "bio text 1") + // } } } <-- end of prefix + ->EndObject() // "" + ->StartObject("") + // book { authorInfo { biod { <-- prefix + ->RenderString("yearBorn", "1840") + ->RenderString("yearDied", "1920") + ->RenderString("text", "bio text 2") + // } } } <-- end of prefix + ->EndObject() // "" + ->EndList(); // "" + + auto expected1 = R"( + book { + author_info { + bio { + year_born : 1830 + year_died : 1910 + text : "bio text 1" + } + } + } + )"; + + auto expected2 = R"( + book { + author_info { + bio { + year_born : 1840 + year_died : 1920 + text : "bio text 2" + } + } + } + )"; + + EXPECT_TRUE(Tester().ExpectNextEq(expected1)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + EXPECT_TRUE(Tester().ExpectNextEq(expected2)); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, Bindings) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("CreateBookRequest"); + AddVariableBinding("shelf", "8"); + AddVariableBinding("book.authorInfo.firstName", "Leo"); + AddVariableBinding("book.authorInfo.lastName", "Tolstoy"); + SetBodyPrefix("book"); + Build(); + Input() + .StartList("") + ->StartObject("") + // book { <-- prefix + ->RenderString("name", "1") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "War and Peace") + // authorInfo { + // first_name : "Leo" <-- weaved + // last_name : "Tolstoy" <-- weaved + // } + // } <-- prefix + // shelf : 8 <-- weaved + ->EndObject() // "" + ->StartObject("") + // book { <-- prefix + ->RenderString("name", "2") + ->RenderString("author", "Leo Tolstoy") + ->RenderString("title", "Anna Karenina") + // } + ->EndObject() // "" + ->StartObject("") + // book { <-- prefix + ->RenderString("name", "3") + ->RenderString("author", "Fyodor Dostoevski") + ->RenderString("title", "Crime and Punishment") + // } + ->EndObject() // "" + ->EndList(); // "" + + // The bindings have effect only on the first message + auto expected1 = R"( + shelf : 8 + book { + name : "1" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + } + } + )"; + + auto expected2 = R"( + book { + name : "2" + author : "Leo Tolstoy" + title : "Anna Karenina" + } + )"; + + auto expected3 = R"( + book { + name : "3" + author : "Fyodor Dostoevski" + title : "Crime and Punishment" + } + )"; + + EXPECT_TRUE(Tester().ExpectNextEq(expected1)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + EXPECT_TRUE(Tester().ExpectNextEq(expected2)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + + EXPECT_TRUE(Tester().ExpectNextEq(expected3)); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, ListOfScalars) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + SetBodyPrefix("theme"); + Build(); + Input() + .StartList("") + ->RenderString("", "History") + ->RenderString("", "Mistery") + ->RenderString("", "Russian") + ->EndList(); + + EXPECT_TRUE(Tester().ExpectNextEq("theme : \"History\"")); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + EXPECT_TRUE(Tester().ExpectNextEq("theme : \"Mistery\"")); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + EXPECT_TRUE(Tester().ExpectNextEq("theme : \"Russian\"")); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, ListOfLists) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("ListShelvesResponse"); + SetBodyPrefix("shelves"); + Build(); + Input().StartList(""); + + Input() + .StartList("") + ->StartObject("") + ->RenderString("name", "1") + ->RenderString("theme", "A") + ->EndObject() + ->StartObject("") + ->RenderString("name", "2") + ->RenderString("theme", "B") + ->EndObject() + ->EndList(); + + Input() + .StartList("") + ->StartObject("") + ->RenderString("name", "3") + ->RenderString("theme", "C") + ->EndObject() + ->StartObject("") + ->RenderString("name", "4") + ->RenderString("theme", "D") + ->EndObject() + ->EndList(); + + Input().EndList(); + + auto expected1 = R"( + shelves { + name : "1" + theme : "A" + } + shelves { + name : "2" + theme : "B" + } + )"; + + auto expected2 = R"( + shelves { + name : "3" + theme : "C" + } + shelves { + name : "4" + theme : "D" + } + )"; + + EXPECT_TRUE(Tester().ExpectNextEq(expected1)); + EXPECT_TRUE(Tester().ExpectFinishedEq(false)); + EXPECT_TRUE(Tester().ExpectNextEq(expected2)); + EXPECT_TRUE(Tester().ExpectFinishedEq(true)); +} + +TEST_F(RequestStreamTranslatorTest, Error1) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input().StartObject(""); + Tester().ExpectStatusEq(google::protobuf::util::error::INVALID_ARGUMENT); +} + +TEST_F(RequestStreamTranslatorTest, Error2) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input().StartList(""); + Input().EndObject(); + Tester().ExpectStatusEq(google::protobuf::util::error::INVALID_ARGUMENT); +} + +TEST_F(RequestStreamTranslatorTest, Error3) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input().EndList(); + Tester().ExpectStatusEq(google::protobuf::util::error::INVALID_ARGUMENT); +} + +TEST_F(RequestStreamTranslatorTest, Error4) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input().EndObject(); + Tester().ExpectStatusEq(google::protobuf::util::error::INVALID_ARGUMENT); +} + +TEST_F(RequestStreamTranslatorTest, Error5) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input().StartList(""); + Input().StartList(""); + Input().EndObject(); + Tester().ExpectStatusEq(google::protobuf::util::error::INVALID_ARGUMENT); +} + +TEST_F(RequestStreamTranslatorTest, Error6) { + LoadService("bookstore_service.pb.txt"); + SetMessageType("Shelf"); + Build(); + Input().StartList(""); + Input().StartObject(""); + Input().RenderString("theme", "Russian"); + Input().EndList(); + Input().EndList(); + // google::protobuf::ProtoStreamObjectWriter for some reason accepts EndList() + // instead of EndObject(). Should be an error instead. + // Tester().ExpectStatusEq(google::protobuf::util::error::INVALID_ARGUMENT); + Tester().ExpectNextEq(R"( theme : "Russian" )"); +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/request_translator_test_base.cc b/contrib/endpoints/src/grpc/transcoding/request_translator_test_base.cc new file mode 100644 index 00000000000..38254142e4e --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_translator_test_base.cc @@ -0,0 +1,127 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/request_translator_test_base.h" + +#include +#include +#include +#include +#include + +#include "google/api/service.pb.h" +#include "google/protobuf/stubs/strutil.h" +#include "google/protobuf/text_format.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/internal/type_info.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/message_stream.h" +#include "src/grpc/transcoding/test_common.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +// Parse a dot delimited field path string into a vector of actual field +// pointers +std::vector ParseFieldPath( + const google::protobuf::Type& type, + google::protobuf::util::converter::TypeInfo& type_info, + const std::string& field_path_str) { + // First, split the field names + auto field_names = google::protobuf::Split(field_path_str, "."); + + auto current_type = &type; + std::vector field_path; + for (size_t i = 0; i < field_names.size(); ++i) { + // Find the field by name + auto field = type_info.FindField(current_type, field_names[i]); + EXPECT_NE(nullptr, field) << "Could not find field " << field_names[i] + << " in type " << current_type->name() + << " while parsing field path " << field_path_str + << std::endl; + field_path.push_back(field); + + if (i < field_names.size() - 1) { + // If it's not the last one in the path, it must be a message + EXPECT_EQ(google::protobuf::Field::TYPE_MESSAGE, field->kind()) + << "Encountered a non-leaf field " << field->name() + << " that is not a message while parsing field path" << field_path_str + << std::endl; + + // Update the type of the current field + current_type = type_info.GetTypeByTypeUrl(field->type_url()); + EXPECT_NE(nullptr, current_type) + << "Could not resolve type url " << field->type_url() + << " of the field " << field_names[i] << " while parsing field path " + << field_path_str << std::endl; + } + } + return field_path; +} + +} // namespace + +RequestTranslatorTestBase::RequestTranslatorTestBase() + : type_(), + body_prefix_(), + bindings_(), + output_delimiters_(false), + tester_() {} + +RequestTranslatorTestBase::~RequestTranslatorTestBase() {} + +void RequestTranslatorTestBase::LoadService( + const std::string& config_pb_txt_file) { + EXPECT_TRUE(transcoding::testing::LoadService(config_pb_txt_file, &service_)); + type_helper_.reset(new TypeHelper(service_.types(), service_.enums())); +} + +void RequestTranslatorTestBase::SetMessageType(const std::string& type_name) { + type_ = type_helper_->Info()->GetTypeByTypeUrl("type.googleapis.com/" + + type_name); + EXPECT_NE(nullptr, type_) << "Could not resolve the message type " + << type_name << std::endl; +} + +void RequestTranslatorTestBase::AddVariableBinding( + const std::string& field_path_str, std::string value) { + auto field_path = + ParseFieldPath(*type_, *type_helper_->Info(), field_path_str); + bindings_.emplace_back( + RequestWeaver::BindingInfo{field_path, std::move(value)}); +} + +void RequestTranslatorTestBase::Build() { + RequestInfo request_info; + request_info.message_type = type_; + request_info.body_field_path = body_prefix_; + request_info.variable_bindings = bindings_; + + auto output_stream = Create(*type_helper_->Resolver(), output_delimiters_, + std::move(request_info)); + + tester_.reset(new ProtoStreamTester(*output_stream, output_delimiters_)); +} + +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/request_translator_test_base.h b/contrib/endpoints/src/grpc/transcoding/request_translator_test_base.h new file mode 100644 index 00000000000..05d2c7a5fc3 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_translator_test_base.h @@ -0,0 +1,91 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_REQUEST_TRANSLATOR_TEST_BASE_H_ +#define GRPC_TRANSCODING_REQUEST_TRANSLATOR_TEST_BASE_H_ + +#include +#include +#include + +#include "google/api/service.pb.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/type_resolver.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/message_stream.h" +#include "src/grpc/transcoding/proto_stream_tester.h" +#include "src/grpc/transcoding/request_message_translator.h" +#include "src/grpc/transcoding/type_helper.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { + +// A base that provides common functionality for streaming and non-streaming +// translator tests. +class RequestTranslatorTestBase : public ::testing::Test { + protected: + RequestTranslatorTestBase(); + ~RequestTranslatorTestBase(); + + // Loads the test service from a config file. Must be the first call of each + // test. + void LoadService(const std::string& config_pb_txt_file); + + // Methods that tests can use to build the translator + void SetMessageType(const std::string& type_name); + void SetBodyPrefix(const std::string& body_prefix) { + body_prefix_ = body_prefix; + } + void AddVariableBinding(const std::string& field_path_str, std::string value); + void SetOutputDelimiters(bool output_delimiters) { + output_delimiters_ = output_delimiters; + } + void Build(); + + // ProtoStreamTester that the tests can use to validate the output + ProtoStreamTester& Tester() { return *tester_; } + + private: + // Virtual Create() function that each test class must override to create the + // translator and return the output MessageStream. + virtual MessageStream* Create( + google::protobuf::util::TypeResolver& type_resolver, + bool output_delimiters, RequestInfo info) = 0; + + // The test service config + google::api::Service service_; + + // TypeHelper for the service types (helps with resolving/navigating service + // type information) + std::unique_ptr type_helper_; + + // Input for building the translator + const google::protobuf::Type* type_; + std::string body_prefix_; + std::vector bindings_; + bool output_delimiters_; + + std::unique_ptr tester_; +}; + +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_REQUEST_TRANSLATOR_TEST_BASE_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/request_weaver.cc b/contrib/endpoints/src/grpc/transcoding/request_weaver.cc new file mode 100644 index 00000000000..b380c526513 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_weaver.cc @@ -0,0 +1,255 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/request_weaver.h" + +#include +#include + +#include "google/protobuf/stubs/stringpiece.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/internal/datapiece.h" +#include "google/protobuf/util/internal/object_writer.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +namespace pb = google::protobuf; +namespace pbconv = google::protobuf::util::converter; + +RequestWeaver::RequestWeaver(std::vector bindings, + pbconv::ObjectWriter* ow) + : root_(), current_(), ow_(ow), non_actionable_depth_(0) { + for (const auto& b : bindings) { + Bind(std::move(b.field_path), std::move(b.value)); + } +} + +RequestWeaver* RequestWeaver::StartObject(pb::StringPiece name) { + ow_->StartObject(name); + if (current_.empty()) { + // The outermost StartObject(""); + current_.push(&root_); + return this; + } + if (non_actionable_depth_ == 0) { + WeaveInfo* info = current_.top()->FindWeaveMsg(name); + if (info != nullptr) { + current_.push(info); + return this; + } + } + // At this point, we don't match any messages we need to weave into, so + // we won't need to do any matching until we leave this object. + ++non_actionable_depth_; + return this; +} + +RequestWeaver* RequestWeaver::EndObject() { + if (non_actionable_depth_ > 0) { + --non_actionable_depth_; + } else { + WeaveTree(current_.top()); + current_.pop(); + } + ow_->EndObject(); + return this; +} + +RequestWeaver* RequestWeaver::StartList(google::protobuf::StringPiece name) { + ow_->StartList(name); + // We don't support weaving inside lists, so we won't need to do any matching + // until we leave this list. + ++non_actionable_depth_; + return this; +} + +RequestWeaver* RequestWeaver::EndList() { + ow_->EndList(); + --non_actionable_depth_; + return this; +} + +RequestWeaver* RequestWeaver::RenderBool(google::protobuf::StringPiece name, + bool value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderBool(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderInt32(google::protobuf::StringPiece name, + google::protobuf::int32 value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderInt32(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderUint32(google::protobuf::StringPiece name, + google::protobuf::uint32 value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderUint32(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderInt64(google::protobuf::StringPiece name, + google::protobuf::int64 value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderInt64(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderUint64(google::protobuf::StringPiece name, + google::protobuf::uint64 value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderUint64(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderDouble(google::protobuf::StringPiece name, + double value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderDouble(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderFloat(google::protobuf::StringPiece name, + float value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderFloat(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderString( + google::protobuf::StringPiece name, google::protobuf::StringPiece value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderString(name, value); + return this; +} + +RequestWeaver* RequestWeaver::RenderNull(google::protobuf::StringPiece name) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderNull(name); + return this; +} + +RequestWeaver* RequestWeaver::RenderBytes(google::protobuf::StringPiece name, + google::protobuf::StringPiece value) { + if (non_actionable_depth_ == 0) { + CollisionCheck(name); + } + ow_->RenderBytes(name, value); + return this; +} + +void RequestWeaver::Bind(std::vector field_path, + std::string value) { + WeaveInfo* current = &root_; + + // Find or create the path from the root to the leaf message, where the value + // should be injected. + for (size_t i = 0; i < field_path.size() - 1; ++i) { + current = current->FindOrCreateWeaveMsg(field_path[i]); + } + + if (!field_path.empty()) { + current->bindings.emplace_back(field_path.back(), std::move(value)); + } +} + +void RequestWeaver::WeaveTree(RequestWeaver::WeaveInfo* info) { + for (const auto& data : info->bindings) { + pbconv::ObjectWriter::RenderDataPieceTo( + pbconv::DataPiece(pb::StringPiece(data.second), true), + pb::StringPiece(data.first->name()), ow_); + } + info->bindings.clear(); + for (auto& msg : info->messages) { + // Enter into the message only if there are bindings or submessages left + if (!msg.second.bindings.empty() || !msg.second.messages.empty()) { + ow_->StartObject(msg.first->name()); + WeaveTree(&msg.second); + ow_->EndObject(); + } + } + info->messages.clear(); +} + +void RequestWeaver::CollisionCheck(pb::StringPiece name) { + if (current_.empty()) return; + + for (auto it = current_.top()->bindings.begin(); + it != current_.top()->bindings.end();) { + if (name == it->first->name()) { + if (it->first->cardinality() == pb::Field::CARDINALITY_REPEATED) { + pbconv::ObjectWriter::RenderDataPieceTo( + pbconv::DataPiece(pb::StringPiece(it->second), true), name, ow_); + } else { + // TODO: Report collision error. For now we just ignore + // the conflicting binding. + } + it = current_.top()->bindings.erase(it); + continue; + } + ++it; + } +} + +RequestWeaver::WeaveInfo* RequestWeaver::WeaveInfo::FindWeaveMsg( + const pb::StringPiece field_name) { + for (auto& msg : messages) { + if (field_name == msg.first->name()) { + return &msg.second; + } + } + return nullptr; +} + +RequestWeaver::WeaveInfo* RequestWeaver::WeaveInfo::CreateWeaveMsg( + const pb::Field* field) { + messages.emplace_back(field, WeaveInfo()); + return &messages.back().second; +} + +RequestWeaver::WeaveInfo* RequestWeaver::WeaveInfo::FindOrCreateWeaveMsg( + const pb::Field* field) { + WeaveInfo* found = FindWeaveMsg(field->name()); + return found == nullptr ? CreateWeaveMsg(field) : found; +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/request_weaver.h b/contrib/endpoints/src/grpc/transcoding/request_weaver.h new file mode 100644 index 00000000000..70d1956290b --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_weaver.h @@ -0,0 +1,157 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_REQUEST_WEAVER_H_ +#define GRPC_TRANSCODING_REQUEST_WEAVER_H_ + +#include +#include +#include +#include + +#include "google/protobuf/stubs/stringpiece.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/internal/object_writer.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// RequestWeaver is an ObjectWriter implementation that weaves-in given variable +// bindings together with the input ObjectWriter events and forwards it to the +// output ObjectWriter specified in the constructor. +// +// E.g., assume we have the {"shelf.theme" -> "Russian Classics"} binding and +// the caller is "writing" an object calling the weaver methods as follows: +// +// weaver.StartObject(""); +// ... +// weaver.StartObject("shelf"); +// weaver.RenderString("name", "1"); +// weaver.EndObject(); +// ... +// weaver.EndObject(); +// +// The request weaver will forward all these events to the output ObjectWriter +// and will also inject the "shelf.theme" value: +// +// out.StartObject(""); +// ... +// out.StartObject("shelf"); +// out.RenderString("name", "1"); +// out.RenderString("theme", "Russian Classics"); <-- weaved value +// out.EndObject(); +// ... +// out.EndObject(); +// +class RequestWeaver : public google::protobuf::util::converter::ObjectWriter { + public: + // a single binding to be weaved-in into the message + struct BindingInfo { + // field_path is a chain of protobuf fields that defines the (potentially + // nested) location in the message, where the value should be weaved-in. + // E.g. {"shelf", "theme"} field_path means that the value should be + // inserted into the "theme" field of the "shelf" field of the request + // message. + std::vector field_path; + std::string value; + }; + + // We accept 'bindings' by value to enable moving if the caller doesn't need + // the passed object anymore. + // RequestWeaver does not take the ownership of 'ow'. The caller must make + // sure that it exists throughout the lifetime of the RequestWeaver. + RequestWeaver(std::vector bindings, + google::protobuf::util::converter::ObjectWriter* ow); + + // ObjectWriter methods + RequestWeaver* StartObject(google::protobuf::StringPiece name); + RequestWeaver* EndObject(); + RequestWeaver* StartList(google::protobuf::StringPiece name); + RequestWeaver* EndList(); + RequestWeaver* RenderBool(google::protobuf::StringPiece name, bool value); + RequestWeaver* RenderInt32(google::protobuf::StringPiece name, + google::protobuf::int32 value); + RequestWeaver* RenderUint32(google::protobuf::StringPiece name, + google::protobuf::uint32 value); + RequestWeaver* RenderInt64(google::protobuf::StringPiece name, + google::protobuf::int64 value); + RequestWeaver* RenderUint64(google::protobuf::StringPiece name, + google::protobuf::uint64 value); + RequestWeaver* RenderDouble(google::protobuf::StringPiece name, double value); + RequestWeaver* RenderFloat(google::protobuf::StringPiece name, float value); + RequestWeaver* RenderString(google::protobuf::StringPiece name, + google::protobuf::StringPiece value); + RequestWeaver* RenderNull(google::protobuf::StringPiece name); + RequestWeaver* RenderBytes(google::protobuf::StringPiece name, + google::protobuf::StringPiece value); + + private: + // Container for information to be weaved. + // WeaveInfo represents an internal node in the weave tree. + // messages: list of non-leaf children nodes. + // bindings: list of binding values (leaf nodes) in this node. + struct WeaveInfo { + // NOTE: using list instead of map/unordered_map as the number of keys is + // going to be small. + std::list> messages; + std::list> bindings; + + // Find the entry for the speciied field in messages list . + WeaveInfo* FindWeaveMsg(google::protobuf::StringPiece field_name); + + // Create an entry in messages for the given field. The caller must make + // sure that there is no existing entry for the same field before calling. + WeaveInfo* CreateWeaveMsg(const google::protobuf::Field* field); + + // Ensure that there is an entry for the given field and return it. + WeaveInfo* FindOrCreateWeaveMsg(const google::protobuf::Field* field); + }; + + // Bind value to location indicated by fields. + void Bind(std::vector field_path, + std::string value); + + // Write out the whole subtree rooted at info to the ProtoStreamObjectWriter. + void WeaveTree(WeaveInfo* info); + + // Checks if any repeated fields with the same field name are in the current + // node of the weave tree. Output them if there are any. + void CollisionCheck(google::protobuf::StringPiece name); + + // All the headers, variable bindings and parameter bindings to be weaved in. + // root_ : root of the tree to be weaved in. + // current_: stack of nodes in the current visit path from the root. + // NOTE: current_ points to the nodes owned by root_. It doesn't maintain the + // ownership itself. + WeaveInfo root_; + std::stack current_; + + // Destination ObjectWriter for final output. + google::protobuf::util::converter::ObjectWriter* ow_; + + // Counter for number of uninteresting nested messages. + int non_actionable_depth_; + + RequestWeaver(const RequestWeaver&) = delete; + RequestWeaver& operator=(const RequestWeaver&) = delete; +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_REQUEST_WEAVER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/request_weaver_test.cc b/contrib/endpoints/src/grpc/transcoding/request_weaver_test.cc new file mode 100644 index 00000000000..2c09324a4a9 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/request_weaver_test.cc @@ -0,0 +1,518 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/request_weaver.h" + +#include +#include +#include + +#include "google/protobuf/stubs/strutil.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/internal/expecting_objectwriter.h" +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +using google::protobuf::Field; +using ::testing::InSequence; + +class RequestWeaverTest : public ::testing::Test { + protected: + RequestWeaverTest() : mock_(), expect_(&mock_) {} + + void Bind(std::string field_path_str, std::string value) { + auto field_names = google::protobuf::Split(field_path_str, "."); + std::vector field_path; + for (const auto& n : field_names) { + fields_.emplace_back(CreateField(n)); + field_path.emplace_back(&fields_.back()); + } + bindings_.emplace_back( + RequestWeaver::BindingInfo{field_path, std::move(value)}); + } + + std::unique_ptr Create() { + return std::unique_ptr( + new RequestWeaver(std::move(bindings_), &mock_)); + } + + google::protobuf::util::converter::MockObjectWriter mock_; + google::protobuf::util::converter::ExpectingObjectWriter expect_; + InSequence seq_; // all our expectations must be ordered + + private: + std::vector bindings_; + std::list fields_; + + Field CreateField(google::protobuf::StringPiece name) { + Field::Cardinality card; + if (name.ends_with("*")) { + // we use "*" at the end of the field name to denote a repeated field. + card = Field::CARDINALITY_REPEATED; + name.remove_suffix(1); + } else { + card = Field::CARDINALITY_OPTIONAL; + } + Field field; + field.set_name(name); + field.set_kind(Field::TYPE_STRING); + field.set_cardinality(card); + field.set_number(1); // dummy number + return field; + } +}; + +TEST_F(RequestWeaverTest, PassThrough) { + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "a"); + expect_.RenderBytes("by", "b"); + expect_.RenderInt32("i", 1); + expect_.RenderUint32("ui", 2); + expect_.RenderInt64("i64", 3); + expect_.RenderUint64("ui64", 4); + expect_.RenderBool("b", true); + expect_.RenderNull("null"); + expect_.StartObject("B"); + expect_.RenderString("y", "b"); + expect_.EndObject(); // B + expect_.EndObject(); // A + expect_.EndObject(); // "" + + auto w = Create(); + w->StartObject(""); + w->StartObject("A"); + w->RenderString("x", "a"); + w->RenderBytes("by", "b"); + w->RenderInt32("i", google::protobuf::int32(1)); + w->RenderUint32("ui", google::protobuf::uint32(2)); + w->RenderInt64("i64", google::protobuf::int64(3)); + w->RenderUint64("ui64", google::protobuf::uint64(4)); + w->RenderBool("b", true); + w->RenderNull("null"); + w->StartObject("B"); + w->RenderString("y", "b"); + w->EndObject(); + w->EndObject(); + w->EndObject(); +} + +TEST_F(RequestWeaverTest, Level0Bindings) { + Bind("_x", "a"); + Bind("_y", "b"); + Bind("_z", "c"); + + // { + // "i" : "10", + // "x" : "d", + // ("_x" : "a",) + // ("_y" : "b",) + // ("_z" : "c",) + // } + + expect_.StartObject(""); + expect_.RenderInt32("i", 10); + expect_.RenderString("x", "d"); + expect_.RenderString("_x", "a"); + expect_.RenderString("_y", "b"); + expect_.RenderString("_z", "c"); + expect_.EndObject(); + + auto w = Create(); + + w->StartObject(""); + w->RenderInt32("i", 10); + w->RenderString("x", "d"); + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, Level1Bindings) { + Bind("A._x", "a"); + Bind("A._y", "b"); + Bind("B._x", "c"); + + // { + // "x" : "d", + // "A" : { + // "y" : "e", + // ("_x" : "a"), + // ("_y" : "b",) + // } + // "B" : { + // "z" : "f", + // ("_x" : "c", ) + // } + // } + + expect_.StartObject(""); + expect_.RenderString("x", "d"); + expect_.StartObject("A"); + expect_.RenderString("y", "e"); + expect_.RenderString("_x", "a"); + expect_.RenderString("_y", "b"); + expect_.EndObject(); // A + expect_.StartObject("B"); + expect_.RenderString("z", "f"); + expect_.RenderString("_x", "c"); + expect_.EndObject(); // B + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->RenderString("x", "d"); + w->StartObject("A"); + w->RenderString("y", "e"); + w->EndObject(); // A + w->StartObject("B"); + w->RenderString("z", "f"); + w->EndObject(); // B + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, Level2Bindings) { + Bind("A.B._x", "a"); + Bind("A.C._y", "b"); + Bind("D.E._x", "c"); + + // { + // "A" : { + // "B" : { + // "x" : "d", + // ("_x" : "a",) + // }, + // "y" : "e", + // "C" : { + // ("_y" : "b",) + // } + // } + // "D" : { + // "z" : "f", + // "E" : { + // "u" : "g", + // ("_x" : "c",) + // }, + // } + // } + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.StartObject("B"); + expect_.RenderString("x", "d"); + expect_.RenderString("_x", "a"); + expect_.EndObject(); // "B" + expect_.RenderString("y", "e"); + expect_.StartObject("C"); + expect_.RenderString("_y", "b"); + expect_.EndObject(); // "C" + expect_.EndObject(); // "A" + expect_.StartObject("D"); + expect_.RenderString("z", "f"); + expect_.StartObject("E"); + expect_.RenderString("u", "g"); + expect_.RenderString("_x", "c"); + expect_.EndObject(); // "E" + expect_.EndObject(); // "D" + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->StartObject("A"); + w->StartObject("B"); + w->RenderString("x", "d"); + w->EndObject(); // "B" + w->RenderString("y", "e"); + w->StartObject("C"); + w->EndObject(); // "C" + w->EndObject(); // "A" + w->StartObject("D"); + w->RenderString("z", "f"); + w->StartObject("E"); + w->RenderString("u", "g"); + w->EndObject(); // "E" + w->EndObject(); // "D" + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, Level2WeaveNewSubTree) { + Bind("A.B._x", "a"); + + // { + // "x" : "b", + // "C" : { + // "y" : "c", + // "D" : { + // "z" : "c", + // } + // }, + // ( + // "A" { + // "B" { + // "_x" : "a" + // } + // } + // ) + // } + + expect_.StartObject(""); + expect_.RenderString("x", "b"); + expect_.StartObject("C"); + expect_.RenderString("y", "c"); + expect_.StartObject("D"); + expect_.RenderString("z", "d"); + expect_.EndObject(); // "C" + expect_.EndObject(); // "D" + expect_.StartObject("A"); + expect_.StartObject("B"); + expect_.RenderString("_x", "a"); + expect_.EndObject(); // "B" + expect_.EndObject(); // "A" + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->RenderString("x", "b"); + w->StartObject("C"); + w->RenderString("y", "c"); + w->StartObject("D"); + w->RenderString("z", "d"); + w->EndObject(); // "C" + w->EndObject(); // "D" + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, MixedBindings) { + Bind("_x", "a"); + Bind("A.B._y", "b"); + Bind("A._z", "c"); + + // { + // "A" : { + // "x" : "d", + // "B" : { + // "y" : "e", + // ("_y" : "b",) + // }, + // ("_z" : "c",) + // }, + // ("_x" : "a",) + // } + + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "d"); + expect_.StartObject("B"); + expect_.RenderString("y", "e"); + expect_.RenderString("_y", "b"); + expect_.EndObject(); // "B" + expect_.RenderString("_z", "c"); + expect_.EndObject(); // "A" + expect_.RenderString("_x", "a"); + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->StartObject("A"); + w->RenderString("x", "d"); + w->StartObject("B"); + w->RenderString("y", "e"); + w->EndObject(); // "B" + w->EndObject(); // "A" + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, MoreMixedBindings) { + Bind("_x", "a"); + Bind("A._y", "b"); + Bind("B._z", "c"); + Bind("C.D._u", "d"); + + // { + // "A" : { + // "x" : "d", + // ("_y" : "b",) + // }, + // "B" : { + // "y" : "e", + // ("_z" : "c",) + // }, + // ("_x" : "a",) + // ( + // "C" : { + // "D" : { + // ("_u" : "d",) + // }, + // }, + // ) + // } + + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "d"); + expect_.RenderString("_y", "b"); + expect_.EndObject(); // "A" + expect_.StartObject("B"); + expect_.RenderString("y", "e"); + expect_.RenderString("_z", "c"); + expect_.EndObject(); // "B" + expect_.RenderString("_x", "a"); + expect_.StartObject("C"); + expect_.StartObject("D"); + expect_.RenderString("_u", "d"); + expect_.EndObject(); // "D" + expect_.EndObject(); // "C" + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->StartObject("A"); + w->RenderString("x", "d"); + w->EndObject(); // "A" + w->StartObject("B"); + w->RenderString("y", "e"); + w->EndObject(); // "B" + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, CollisionIgnored) { + Bind("A.x", "a"); + + // { + // "A" : { + // "x" : "b", + // ("x" : "a") -- ignored + // } + // } + + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "b"); + expect_.EndObject(); // "A" + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->StartObject("A"); + w->RenderString("x", "b"); + w->EndObject(); // "A" + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, CollisionRepeated) { + // "x*" means a repeated field with the name "x" + Bind("A.x*", "b"); + Bind("A.x*", "c"); + Bind("A.x*", "d"); + + // { + // "A" : { + // "x" : "a", + // ("x" : "b") + // ("x" : "c") + // ("x" : "d") + // } + // } + + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "b"); + expect_.RenderString("x", "c"); + expect_.RenderString("x", "d"); + expect_.RenderString("x", "a"); + expect_.EndObject(); // "A" + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->StartObject("A"); + w->RenderString("x", "a"); + w->EndObject(); // "A" + w->EndObject(); // "" +} + +TEST_F(RequestWeaverTest, IgnoreListTest) { + Bind("A._x", "a"); + + // { + // "L" : [ + // { + // "A" : { + // "x" : "b" + // }, + // }, + // ], + // "A" : ["c", "d"] + // "A" : { + // "y" : "e", + // ("_x" : "a"), + // }, + // } + + expect_.StartObject(""); + expect_.StartList("L"); + expect_.StartObject(""); + expect_.StartObject("A"); + expect_.RenderString("x", "b"); + expect_.EndObject(); // "A" + expect_.EndObject(); // "" + expect_.EndList(); // "L" + expect_.StartList("A"); + expect_.RenderString("", "c"); + expect_.RenderString("", "d"); + expect_.EndList(); // "A" + expect_.StartObject("A"); + expect_.RenderString("y", "e"); + expect_.RenderString("_x", "a"); + expect_.EndObject(); // "A" + expect_.EndObject(); // "" + + auto w = Create(); + + w->StartObject(""); + w->StartList("L"); + w->StartObject(""); + w->StartObject("A"); + w->RenderString("x", "b"); + w->EndObject(); // "A" + w->EndObject(); // "" + w->EndList(); // "L" + w->StartList("A"); + w->RenderString("", "c"); + w->RenderString("", "d"); + w->EndList(); // "A" + w->StartObject("A"); + w->RenderString("y", "e"); + w->EndObject(); // "A" + w->EndObject(); // "" +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc new file mode 100644 index 00000000000..3a58d507034 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc @@ -0,0 +1,133 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/response_to_json_translator.h" + +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/stubs/status.h" +#include "google/protobuf/util/json_util.h" +#include "google/protobuf/util/type_resolver.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +ResponseToJsonTranslator::ResponseToJsonTranslator( + ::google::protobuf::util::TypeResolver* type_resolver, std::string type_url, + bool streaming, ::google::protobuf::io::ZeroCopyInputStream* in) + : type_resolver_(type_resolver), + type_url_(std::move(type_url)), + streaming_(streaming), + reader_(in), + first_(true), + finished_(false) {} + +bool ResponseToJsonTranslator::NextMessage(std::string* message) { + if (Finished()) { + // All done + return false; + } + // Try to read a message + auto proto_in = reader_.NextMessage(); + if (proto_in) { + std::string json_out; + if (TranslateMessage(proto_in.get(), &json_out)) { + *message = std::move(json_out); + if (!streaming_) { + // This is a non-streaming call, so we don't expect more messages. + finished_ = true; + } + return true; + } else { + // TranslateMessage() failed - return false. The error details are stored + // in status_. + return false; + } + } else if (streaming_ && reader_.Finished()) { + // This is a streaming call and the input is finished. Return the final ']' + // or "[]" in case this was an empty stream. + *message = first_ ? "[]" : "]"; + finished_ = true; + return true; + } else { + // Don't have an input message + return false; + } +} + +namespace { + +// A helper to write a single char to a ZeroCopyOutputStream +bool WriteChar(::google::protobuf::io::ZeroCopyOutputStream* stream, char c) { + int size = 0; + void* data = 0; + if (!stream->Next(&data, &size) || 0 == size) { + return false; + } + // Write the char to the first byte of the buffer and return the rest size-1 + // bytes to the stream. + *reinterpret_cast(data) = c; + stream->BackUp(size - 1); + return true; +} + +} // namespace + +bool ResponseToJsonTranslator::TranslateMessage( + ::google::protobuf::io::ZeroCopyInputStream* proto_in, + std::string* json_out) { + ::google::protobuf::io::StringOutputStream json_stream(json_out); + + if (streaming_) { + if (first_) { + // This is a streaming call and this is the first message, so prepend the + // output JSON with a '['. + if (!WriteChar(&json_stream, '[')) { + status_ = ::google::protobuf::util::Status( + ::google::protobuf::util::error::INTERNAL, + "Failed to build the response message."); + return false; + } + first_ = false; + } else { + // For streaming calls add a ',' before each message except the first. + if (!WriteChar(&json_stream, ',')) { + status_ = ::google::protobuf::util::Status( + ::google::protobuf::util::error::INTERNAL, + "Failed to build the response message."); + return false; + } + } + } + + // Do the actual translation. + status_ = ::google::protobuf::util::BinaryToJsonStream( + type_resolver_, type_url_, proto_in, &json_stream); + if (!status_.ok()) { + return false; + } + + return true; +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h new file mode 100644 index 00000000000..c88b56362e4 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h @@ -0,0 +1,102 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_RESPONSE_TO_JSON_TRANSLATOR_H_ +#define GRPC_TRANSCODING_RESPONSE_TO_JSON_TRANSLATOR_H_ + +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/status.h" +#include "google/protobuf/util/type_resolver.h" +#include "src/grpc/transcoding/message_reader.h" +#include "src/grpc/transcoding/message_stream.h" + +namespace google { +namespace api_manager { + +namespace transcoding { + +// ResponseToJsonTranslator translates gRPC response message(s) into JSON. It +// accepts the input from a ZeroCopyInputStream and exposes the output through a +// MessageStream implementation. Supports streaming calls. +// +// The implementation uses a MessageReader to extract complete messages from the +// input stream and ::google::protobuf::util::BinaryToJsonStream() to do the +// actual translation. For streaming calls emits '[', ',' and ']' in appropriate +// locations to construct a JSON array. +// +// Example: +// ResponseToJsonTranslator translator(type_resolver, +// "type.googleapis.com/Shelf", +// true, input_stream); +// +// std::string message; +// while (translator.NextMessage(&message)) { +// printf("Message=%s\n", message.c_str()); +// } +// +// if (!translator.Status().ok()) { +// printf("Error: %s\n", +// translator.Status().error_message().as_string().c_str()); +// return; +// } +// +// NOTE: ResponseToJsonTranslator is unable to recognize the case when there is +// an incomplete message at the end of the input. The callers will need to +// detect it and act appropriately. +// +class ResponseToJsonTranslator : public MessageStream { + public: + // type_resolver - passed to BinaryToJsonStream() to do the translation + // type_url - the type of input proto message(s) + // streaming - whether this is a streaming call or not + // in - the input stream of delimited proto message(s) as in the gRPC wire + // format (http://www.grpc.io/docs/guides/wire.html) + ResponseToJsonTranslator( + ::google::protobuf::util::TypeResolver* type_resolver, + std::string type_url, bool streaming, + ::google::protobuf::io::ZeroCopyInputStream* in); + + // MessageStream implementation + bool NextMessage(std::string* message); + bool Finished() const { return finished_ || !status_.ok(); } + ::google::protobuf::util::Status Status() const { return status_; } + + private: + // Translates a single message + bool TranslateMessage(::google::protobuf::io::ZeroCopyInputStream* proto_in, + std::string* json_out); + + ::google::protobuf::util::TypeResolver* type_resolver_; + std::string type_url_; + bool streaming_; + + // A MessageReader to extract full messages + MessageReader reader_; + + // Whether this is the first message of a streaming call or not. Used to emit + // the opening '['. + bool first_; + + bool finished_; + ::google::protobuf::util::Status status_; +}; + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_RESPONSE_TO_JSON_TRANSLATOR_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator_test.cc b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator_test.cc new file mode 100644 index 00000000000..f012a7ce1e6 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator_test.cc @@ -0,0 +1,745 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/response_to_json_translator.h" + +#include +#include +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/text_format.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/bookstore.pb.h" +#include "src/grpc/transcoding/test_common.h" +#include "src/grpc/transcoding/type_helper.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { +namespace { + +namespace pbutil = ::google::protobuf::util; +namespace pberr = ::google::protobuf::util::error; + +// A helper structure to store a single expected chunk of json and its position +struct ExpectedAt { + // The position in the input, at which this json is expected + size_t at; + std::string json; +}; + +// ResponseToJsonTranslatorTestRun tests a single ResponseToJsonTranslator +// processing the input as expected. +// It allows feeding chunks of the input (AddChunk()) to the translator and +// testing that the translated messages are generated correctly (Test()). +class ResponseToJsonTranslatorTestRun { + public: + // type_resolver - the TypeResolver to be passed to the translator, + // streaming - whether this is a streaming call or not, + // type_url - type url of messages being translated, + // input - the input to be passed to the MessageReader, + // expected - the expected translated json chunks as the input is processed, + ResponseToJsonTranslatorTestRun(pbutil::TypeResolver* type_resolver, + bool streaming, const std::string& type_url, + const std::string& input, + const std::vector& expected) + : input_(input), + expected_(expected), + streaming_(streaming), + input_stream_(new TestZeroCopyInputStream()), + translator_(new ResponseToJsonTranslator( + type_resolver, type_url, streaming_, input_stream_.get())), + position_(0), + next_expected_(std::begin(expected_)) {} + + // Returns the total input size including the delimiters. + size_t TotalInputSize() const { return input_.size(); } + + // Adds the next size bytes of input chunk to the input stream, such that the + // translator can process. + void AddChunk(size_t size) { + input_stream_->AddChunk(input_.substr(position_, size)); + position_ += size; + } + + // Marks the input stream as finished. + void FinishInputStream() { input_stream_->Finish(); } + + // Tests the ResponseToJsonTranslator at the current position of the input. + bool Test() { + // While we still have expected messages before or at the current position + // try to match. + while (next_expected_ != std::end(expected_) && + next_expected_->at <= position_) { + // Check the status first + if (!translator_->Status().ok()) { + ADD_FAILURE() << "Error: " << translator_->Status().error_message() + << std::endl; + return false; + } + + // Read the message + std::string actual; + if (!translator_->NextMessage(&actual)) { + ADD_FAILURE() << "No message available" << std::endl; + return false; + } + + // Match the message + if (streaming_) { + if (!json_array_tester_.TestElement(next_expected_->json, actual)) { + return false; + } + } else { + if (!ExpectJsonObjectEq(next_expected_->json, actual)) { + return false; + } + } + + // Advance to the next expected message + ++next_expected_; + } + if (input_stream_->Finished() && streaming_) { + // In case of streaming calls if the input is finished, we expect the + // final ']' at the end of the stream. + + // Read the message + std::string actual; + if (!translator_->NextMessage(&actual)) { + ADD_FAILURE() << "No message available. Missing final ']'" << std::endl; + return false; + } + + // Test that it closes the array + if (!json_array_tester_.TestClosed(actual)) { + return false; + } + } + + // At this point we don't expect any more messages as we read all the ones + // that must have been available + std::string actual; + if (translator_->NextMessage(&actual)) { + ADD_FAILURE() << "Unexpected message: \"" << actual << "\"" << std::endl; + return false; + } + + // Check the status + if (!translator_->Status().ok()) { + ADD_FAILURE() << "Error: " << translator_->Status().error_message() + << std::endl; + return false; + } + + // Now check that Finished() returns as expected. + if (translator_->Finished() != input_stream_->Finished()) { + EXPECT_EQ(input_stream_->Finished(), translator_->Finished()); + return false; + } + + return true; + } + + private: + std::string input_; + std::vector expected_; + bool streaming_; + + std::unique_ptr input_stream_; + std::unique_ptr translator_; + + // The position in the input string that indicates the part of the input that + // has already been processed. + size_t position_; + + // An iterator that points to the next expected message. + std::vector::const_iterator next_expected_; + + // JsonArrayTester for testing the output JSON array in streaming case + JsonArrayTester json_array_tester_; +}; + +// ResponseToJsonTranslatorTestCase tests a single input test case with +// different partitions of the input. +class ResponseToJsonTranslatorTestCase { + public: + // type_resolver - the TypeResolver to be passed to the translator, + // streaming - whether this is a streaming call or not, + // type_url - type url of messages being translated, + // input - the input to be passed to the MessageReader, + // expected - the expected translated json chunks as the input is processed, + ResponseToJsonTranslatorTestCase(pbutil::TypeResolver* type_resolver, + bool streaming, const std::string& type_url, + std::string input, + std::vector expected) + : type_resolver_(type_resolver), + streaming_(streaming), + type_url_(type_url), + input_(std::move(input)), + expected_(std::move(expected)) {} + + std::unique_ptr NewRun() { + return std::unique_ptr( + new ResponseToJsonTranslatorTestRun(type_resolver_, streaming_, + type_url_, input_, expected_)); + } + + // Runs the test for different partitions of the input. + // chunk_count - the number of chunks (parts) per partition + // partitioning_coefficient - defines how exhaustive the test should be. See + // the comment on RunTestForInputPartitions() in + // test_common.h for more details. + bool Test(size_t chunk_count, double partitioning_coefficient) { + return RunTestForInputPartitions(chunk_count, partitioning_coefficient, + input_, + [this](const std::vector& t) { + auto run = NewRun(); + + // Feed the chunks according to the + // partition defined by tuple t and + // test along the way. + size_t pos = 0; + for (size_t i = 0; i < t.size(); ++i) { + run->AddChunk(t[i] - pos); + pos = t[i]; + if (!run->Test()) { + return false; + } + } + // Feed the last chunk, finish & test. + run->AddChunk(input_.size() - pos); + run->FinishInputStream(); + return run->Test(); + }); + } + + private: + pbutil::TypeResolver* type_resolver_; + bool streaming_; + std::string type_url_; + + // The entire input including message delimiters + std::string input_; + + // Expected JSON chunks + std::vector expected_; +}; + +class ResponseToJsonTranslatorTest : public ::testing::Test { + protected: + ResponseToJsonTranslatorTest() : streaming_(false) {} + + // Load the service config to be used for testing. This must be the first call + // in a test. + bool LoadService(const std::string& config_pb_txt_file) { + if (!transcoding::testing::LoadService(config_pb_txt_file, &service_)) { + return false; + } + type_helper_.reset(new TypeHelper(service_.types(), service_.enums())); + return true; + } + + // Sets the message type for used in this test. Must be used before Build(). + void SetMessageType(const std::string& type_name) { + type_url_ = "type.googleapis.com/" + type_name; + } + + // Sets whether this is a streaming call or not. Must be used before Build(). + // The default is non-streaming. + void SetStreaming(bool streaming) { streaming_ = streaming; } + + // Adds a message to be tested and the corresponding expected JSON. Must be + // used before Build(). + template + void AddMessage(const std::string& proto_text, std::string expected_json) { + // Generate a gRPC message and add it to the input + input_ += GenerateGrpcMessage(proto_text); + // We will expect expected_json after input.size() bytes are processed. + expected_.emplace_back(ExpectedAt{input_.size(), expected_json}); + } + + // Builds a ResponseToJsonTranslatorTestCase and resets the input messages in + // case the test needs to build another one. + std::unique_ptr Build() { + std::string input; + std::vector expected; + input.swap(input_); + expected.swap(expected_); + + return std::unique_ptr( + new ResponseToJsonTranslatorTestCase( + type_helper_->Resolver(), streaming_, type_url_, std::move(input), + std::move(expected))); + } + + private: + ::google::api::Service service_; + std::unique_ptr type_helper_; + + std::string type_url_; + bool streaming_; + + // The entire input + std::string input_; + + // Expected JSON chunks + std::vector expected_; +}; + +TEST_F(ResponseToJsonTranslatorTest, Simple) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMessageType("Shelf"); + AddMessage(R"(name : "1" theme : "History")", + R"({ "name" : "1", "theme" : "History"})"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 1.0)); + EXPECT_TRUE(tc->Test(4, 0.5)); +} + +TEST_F(ResponseToJsonTranslatorTest, Nested) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMessageType("Book"); + AddMessage( + R"( + name : "8" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + bio { + year_born : 1830 + year_died : 1910 + text : "some text" + } + } + )", + R"({ + "author" : "Leo Tolstoy", + "name" : "8", + "title" : "War and Peace", + "authorInfo" : { + "firstName" : "Leo", + "lastName" : "Tolstoy", + "bio" : { + "yearBorn" : "1830", + "yearDied" : "1910", + "text" : "some text" + } + } + })"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 0.2)); +} + +TEST_F(ResponseToJsonTranslatorTest, Empty) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMessageType("Shelf"); + AddMessage("", "{}"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); +} + +TEST_F(ResponseToJsonTranslatorTest, DifferentSizes) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMessageType("Shelf"); + + auto sizes = {1, 2, 3, 4, 5, 6, 10, 12, 100, 128, 256, 1024, 4096, 65537}; + for (auto size : sizes) { + auto theme = GenerateInput("abcdefgh12345", size); + AddMessage(R"(name : "1" theme : ")" + theme + R"(")", + R"({ "name" : "1", "theme" : ")" + theme + R"("})"); + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + } +} + +TEST_F(ResponseToJsonTranslatorTest, StreamingOneMessage) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetStreaming(true); + SetMessageType("Shelf"); + AddMessage(R"(name : "1" theme : "History")", + R"({ "name" : "1", "theme" : "History"})"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 0.5)); + EXPECT_TRUE(tc->Test(4, 0.1)); +} + +TEST_F(ResponseToJsonTranslatorTest, StreamingThreeMessages) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetStreaming(true); + SetMessageType("Shelf"); + AddMessage(R"(name : "1" theme : "History")", + R"({ "name" : "1", "theme" : "History"})"); + AddMessage(R"(name : "2" theme : "Mistery")", + R"({ "name" : "2", "theme" : "Mistery"})"); + AddMessage(R"(name : "3" theme : "Russian")", + R"({ "name" : "3", "theme" : "Russian"})"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 0.2)); + EXPECT_TRUE(tc->Test(4, 0.1)); +} + +TEST_F(ResponseToJsonTranslatorTest, StreamingNoMessages) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetStreaming(true); + SetMessageType("Shelf"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); +} + +TEST_F(ResponseToJsonTranslatorTest, StreamingEmptyMessage) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetStreaming(true); + SetMessageType("Shelf"); + AddMessage("", "{}"); + AddMessage(R"(name : "1" theme : "History")", + R"({ "name" : "1", "theme" : "History"})"); + AddMessage("", "{}"); + AddMessage(R"(name : "2" theme : "Classics")", + R"({ "name" : "2", "theme" : "Classics"})"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 1.0)); + EXPECT_TRUE(tc->Test(3, 0.2)); +} + +TEST_F(ResponseToJsonTranslatorTest, Streaming50Messages) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetStreaming(true); + SetMessageType("Shelf"); + + for (size_t i = 1; i <= 50; ++i) { + auto no = std::to_string(i); + AddMessage(R"(name : ")" + no + + R"(" theme : "th-)" + no + R"(")", + R"({ "name" : ")" + no + + R"(", "theme" : "th-)" + no + R"("})"); + } + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); +} + +TEST_F(ResponseToJsonTranslatorTest, StreamingNested) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetStreaming(true); + SetMessageType("Book"); + AddMessage( + R"( + name : "8" + author : "Leo Tolstoy" + title : "War and Peace" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + bio { + year_born : 1830 + year_died : 1910 + text : "some text" + } + } + )", + R"({ + "author" : "Leo Tolstoy", + "name" : "8", + "title" : "War and Peace", + "authorInfo" : { + "firstName" : "Leo", + "lastName" : "Tolstoy", + "bio" : { + "yearBorn" : "1830", + "yearDied" : "1910", + "text" : "some text" + } + } + })"); + AddMessage( + R"( + name : "88" + author : "Fyodor Dostoevski" + title : "Crime & Punishment" + author_info { + first_name : "Fyodor" + last_name : "Dostoevski" + bio { + year_born : 1840 + year_died : 1920 + text : "some text" + } + } + )", + R"({ + "author" : "Fyodor Dostoevski", + "name" : "88", + "title" : "Crime & Punishment", + "authorInfo" : { + "firstName" : "Fyodor", + "lastName" : "Dostoevski", + "bio" : { + "yearBorn" : "1840", + "yearDied" : "1920", + "text" : "some text" + } + } + })"); + + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); + EXPECT_TRUE(tc->Test(2, 0.3)); + EXPECT_TRUE(tc->Test(3, 0.05)); +} + +TEST_F(ResponseToJsonTranslatorTest, StreamingDifferentSizes) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMessageType("Shelf"); + SetStreaming(true); + + auto sizes = {1, 2, 3, 4, 5, 6, 10, 12, 100, 128, 256, 1024, 4096, 65537}; + for (auto size : sizes) { + auto theme = GenerateInput("abcdefgh12345", size); + AddMessage(R"(name : "1" theme : ")" + theme + R"(")", + R"({ "name" : "1", "theme" : ")" + theme + R"("})"); + } + auto tc = Build(); + EXPECT_TRUE(tc->Test(1, 1.0)); +} + +TEST_F(ResponseToJsonTranslatorTest, ErrorInvalidType) { + // Load the service config + ::google::api::Service service; + ASSERT_TRUE( + transcoding::testing::LoadService("bookstore_service.pb.txt", &service)); + + // Create a TypeHelper using the service config + TypeHelper type_helper(service.types(), service.enums()); + + TestZeroCopyInputStream input_stream; + ResponseToJsonTranslator translator(type_helper.Resolver(), + "type.googleapis.com/InvalidType", false, + &input_stream); + + input_stream.AddChunk( + GenerateGrpcMessage(R"( name : "1" theme : "Fiction" )")); + + // Call NextMessage() to trigger the error + std::string message; + EXPECT_FALSE(translator.NextMessage(&message)); + EXPECT_EQ(pberr::NOT_FOUND, translator.Status().error_code()); +} + +TEST_F(ResponseToJsonTranslatorTest, DirectTest) { + // Load the service config + ::google::api::Service service; + ASSERT_TRUE( + transcoding::testing::LoadService("bookstore_service.pb.txt", &service)); + + // Create a TypeHelper using the service config + TypeHelper type_helper(service.types(), service.enums()); + + // A message to test + auto test_message = + GenerateGrpcMessage(R"(name : "1" theme : "Fiction")"); + + TestZeroCopyInputStream input_stream; + ResponseToJsonTranslator translator(type_helper.Resolver(), + "type.googleapis.com/Shelf", false, + &input_stream); + + std::string message; + // There is nothing translated + EXPECT_FALSE(translator.NextMessage(&message)); + + // Add the first 10 bytes of the message to the stream + input_stream.AddChunk(test_message.substr(0, 10)); + + // Still nothing + EXPECT_FALSE(translator.NextMessage(&message)); + + // Add the rest of the message to the stream + input_stream.AddChunk(test_message.substr(10)); + + // Now we should have a message + EXPECT_TRUE(translator.NextMessage(&message)); + EXPECT_TRUE( + ExpectJsonObjectEq(R"({ "name":"1", "theme":"Fiction" })", message)); +} + +TEST_F(ResponseToJsonTranslatorTest, StreamingDirectTest) { + // Load the service config + ::google::api::Service service; + ASSERT_TRUE( + transcoding::testing::LoadService("bookstore_service.pb.txt", &service)); + + // Create a TypeHelper using the service config + TypeHelper type_helper(service.types(), service.enums()); + + // Messages to test + auto test_message1 = + GenerateGrpcMessage(R"(name : "1" theme : "Fiction")"); + auto test_message2 = + GenerateGrpcMessage(R"(name : "2" theme : "Fantasy")"); + auto test_message3 = + GenerateGrpcMessage(R"(name : "3" theme : "Children")"); + auto test_message4 = + GenerateGrpcMessage(R"(name : "4" theme : "Classics")"); + + TestZeroCopyInputStream input_stream; + ResponseToJsonTranslator translator( + type_helper.Resolver(), "type.googleapis.com/Shelf", true, &input_stream); + + std::string message; + // There is nothing translated + EXPECT_FALSE(translator.NextMessage(&message)); + + // Add test_message1 to the stream + input_stream.AddChunk(test_message1); + + JsonArrayTester tester; + + // Now we should have the test_message1 translated + EXPECT_TRUE(translator.NextMessage(&message)); + EXPECT_TRUE( + tester.TestElement(R"({ "name":"1", "theme":"Fiction" })", message)); + + // No more messages, but not finished yet + EXPECT_FALSE(translator.NextMessage(&message)); + EXPECT_FALSE(translator.Finished()); + + // Add the test_message2, test_message3 and part of test_message4 + input_stream.AddChunk(test_message2); + input_stream.AddChunk(test_message3); + input_stream.AddChunk(test_message4.substr(0, 10)); + + // Now we should have test_message2 & test_message3 translated + EXPECT_TRUE(translator.NextMessage(&message)); + EXPECT_TRUE( + tester.TestElement(R"({ "name":"2", "theme":"Fantasy" })", message)); + + EXPECT_TRUE(translator.NextMessage(&message)); + EXPECT_TRUE( + tester.TestElement(R"({ "name":"3", "theme":"Children" })", message)); + + // No more messages, but not finished yet + EXPECT_FALSE(translator.NextMessage(&message)); + EXPECT_FALSE(translator.Finished()); + + // Add the rest of test_message4 + input_stream.AddChunk(test_message4.substr(10)); + + // Now we should have the test_message4 translated + EXPECT_TRUE(translator.NextMessage(&message)); + EXPECT_TRUE( + tester.TestElement(R"({ "name":"4", "theme":"Classics" })", message)); + + // No more messages, but not finished yet + EXPECT_FALSE(translator.NextMessage(&message)); + EXPECT_FALSE(translator.Finished()); + + // Now finish the stream + input_stream.Finish(); + + // Expect the final ']' + EXPECT_TRUE(translator.NextMessage(&message)); + EXPECT_TRUE(tester.TestClosed(message)); + + // All done! + EXPECT_FALSE(translator.NextMessage(&message)); + EXPECT_TRUE(translator.Finished()); +} + +TEST_F(ResponseToJsonTranslatorTest, Streaming5KMessages) { + // Load the service config + ::google::api::Service service; + ASSERT_TRUE( + transcoding::testing::LoadService("bookstore_service.pb.txt", &service)); + + // Create a TypeHelper using the service config + TypeHelper type_helper(service.types(), service.enums()); + + TestZeroCopyInputStream input_stream; + ResponseToJsonTranslator translator( + type_helper.Resolver(), "type.googleapis.com/Shelf", true, &input_stream); + + // Add all messages to the input stream & construct the expected output json + // array + std::string expected_json_array = "["; + std::string actual_json_array; + for (size_t i = 1; i <= 5000; ++i) { + auto no = std::to_string(i); + + // Add the message to the input + input_stream.AddChunk(GenerateGrpcMessage( + R"(name : ")" + no + R"(" theme : "th-)" + no + R"(")")); + + // Read the translated message + std::string actual; + EXPECT_TRUE(translator.NextMessage(&actual)); + actual_json_array += actual; + + // Append the corresponding JSON to the expected array + if (i > 1) { + expected_json_array += ","; + } + expected_json_array += + R"({ "name" : ")" + no + R"(", "theme" : "th-)" + no + R"("})"; + } + + // Close the input stream + input_stream.Finish(); + + // Read the closing ']' + std::string actual; + EXPECT_TRUE(translator.NextMessage(&actual)); + actual_json_array += actual; + + // Close the expected array + expected_json_array += "]"; + + // Check the status + EXPECT_TRUE(translator.Status().ok()) + << "Error " << translator.Status().error_message() << std::endl; + + // Match the output array + EXPECT_TRUE(ExpectJsonArrayEq(expected_json_array, actual_json_array)); +} + +} // namespace +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/test_common.cc b/contrib/endpoints/src/grpc/transcoding/test_common.cc new file mode 100644 index 00000000000..3550cb766c1 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/test_common.cc @@ -0,0 +1,346 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "test_common.h" + +#include +#include +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/text_format.h" +#include "google/protobuf/util/json_util.h" +#include "google/protobuf/util/message_differencer.h" +#include "google/protobuf/util/type_resolver_util.h" +#include "google/protobuf/util/type_resolver_util.h" +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { + +namespace pb = ::google::protobuf; +namespace pbutil = ::google::protobuf::util; + +TestZeroCopyInputStream::TestZeroCopyInputStream() + : finished_(false), position_(0) {} + +void TestZeroCopyInputStream::AddChunk(std::string chunk) { + // Some of our tests generate 0-sized chunks. Let's not add them to the flow + // here because the tests explicitly handle 'out of input' and 'there is now + // new input' situations explicitly and empty chunks in the input would + // interfere with the test driver logic. + if (!chunk.empty()) { + chunks_.emplace_back(std::move(chunk)); + } +} + +bool TestZeroCopyInputStream::Next(const void** data, int* size) { + if (position_ < static_cast(current_.size())) { + // Still have some data in the current buffer + *data = current_.data() + position_; + *size = current_.size() - position_; + position_ = current_.size(); + return true; + } else if (!chunks_.empty()) { + // Return the next buffer + current_ = std::move(chunks_.front()); + chunks_.pop_front(); + *data = current_.data(); + *size = current_.size(); + position_ = current_.size(); + return true; + } else { + *size = 0; + return !finished_; + } +} + +void TestZeroCopyInputStream::BackUp(int count) { + EXPECT_TRUE(count <= position_) << "Out of bounds while trying to back up " + "the input stream. position: " + << position_ << " count: " << count + << std::endl; + position_ -= count; +} + +pb::int64 TestZeroCopyInputStream::ByteCount() const { + auto total = current_.size() - position_; + for (auto chunk : chunks_) { + total += chunk.size(); + } + return total; +} + +namespace { + +typedef std::vector Tuple; + +// Generates all distinct m-tuples of {1..n} numbers and calls f on each one. +// If f returns false, the enumeration stops; otherwise it continues. +// The numbers within each generated tuple are in increasing order. +bool ForAllDistinctTuples(size_t m, size_t n, + std::function f, Tuple& t) { + if (0 == m) { + // The base case + return f(t); + } + for (unsigned i = (t.empty() ? 1 : t.back() + 1); i <= n; ++i) { + t.emplace_back(i); + if (!ForAllDistinctTuples(m - 1, n, f, t)) { + // Somewhere f returned false - early termination + return false; + } + t.pop_back(); + } + return true; +} + +// Convenience overload without the initial tuple +bool ForAllDistinctTuples(size_t m, size_t n, + std::function f) { + Tuple t; + return ForAllDistinctTuples(m, n, f, t); +} + +// Calls f for partitioning_coefficient^m-th part of all the partitions (0 < +// partitioning_coefficient <= 1). It calls ForAllDistinctTuples() for +// partitioning_coefficient*n and multiplies each tuple by +// 1/partitioning_coefficient to achieve uniformity (more or less). +bool ForSomeDistinctTuples(size_t m, size_t n, double partitioning_coefficient, + std::function f) { + return ForAllDistinctTuples( + m, static_cast(partitioning_coefficient * n), + [f, partitioning_coefficient](const Tuple& t) { + auto tt = t; + for (auto& i : tt) { + i = static_cast(i / partitioning_coefficient); + } + return f(tt); + }); +} + +// Returns a display string for a partition of str defined by tuple t. Used for +// displaying errors. +std::string PartitionToDisplayString(const std::string& str, const Tuple& t) { + std::string result; + size_t pos = 0; + for (size_t i = 0; i < t.size(); ++i) { + if (i > 0) { + result += " | "; + } + result += str.substr(pos, t[i] - pos); + pos = t[i]; + } + result += " | "; + result += str.substr(pos); + return result; +} + +} // namespace + +bool RunTestForInputPartitions(size_t chunk_count, + double partitioning_coefficient, + const std::string& input, + std::function test) { + // To choose a m-partition of input of size n, we need to choose m-1 breakdown + // points between 1 and n-1. + return ForSomeDistinctTuples( + chunk_count - 1, input.size() - 1, partitioning_coefficient, + [&input, test](const Tuple& t) { + if (!test(t)) { + ADD_FAILURE() << "Failed for the following partition \"" + << PartitionToDisplayString(input, t) << "\"\n"; + return false; + } else { + return true; + } + }); +} + +std::string GenerateInput(const std::string& seed, size_t size) { + std::string result = seed; + while (result.size() < size) { + result += seed; + } + result.resize(size); + return result; +} + +namespace { + +std::string LoadFile(const std::string& input_file_name) { + const char kTestdata[] = "src/grpc/transcoding/testdata/"; + std::string file_name = std::string(kTestdata) + input_file_name; + + std::ifstream ifs(file_name); + if (!ifs) { + ADD_FAILURE() << "Could not open " << file_name.c_str() << std::endl; + return std::string(); + } + std::ostringstream ss; + ss << ifs.rdbuf(); + return ss.str(); +} + +} // namespace + +bool LoadService(const std::string& config_pb_txt_file, + ::google::api::Service* service) { + auto config = LoadFile(config_pb_txt_file); + if (config.empty()) { + return false; + } + + if (!pb::TextFormat::ParseFromString(config, service)) { + ADD_FAILURE() << "Could not parse service config from " + << config_pb_txt_file.c_str() << std::endl; + return false; + } else { + return true; + } +} + +unsigned DelimiterToSize(const unsigned char* delimiter) { + unsigned size = 0; + // Bytes 1-4 are big-endian 32-bit message size + size = size | static_cast(delimiter[1]); + size <<= 8; + size = size | static_cast(delimiter[2]); + size <<= 8; + size = size | static_cast(delimiter[3]); + size <<= 8; + size = size | static_cast(delimiter[4]); + return size; +} + +std::string SizeToDelimiter(unsigned size) { + unsigned char delimiter[5]; + // Byte 0 is the compression bit - set to 0 (no compression) + delimiter[0] = 0; + // Bytes 1-4 are big-endian 32-bit message size + delimiter[4] = 0xFF & size; + size >>= 8; + delimiter[3] = 0xFF & size; + size >>= 8; + delimiter[2] = 0xFF & size; + size >>= 8; + delimiter[1] = 0xFF & size; + + return std::string(reinterpret_cast(delimiter), + sizeof(delimiter)); +} + +namespace { + +bool JsonToStruct(const std::string& json, pb::Struct* message) { + static std::unique_ptr type_resolver( + pbutil::NewTypeResolverForDescriptorPool( + "type.googleapis.com", pb::Struct::descriptor()->file()->pool())); + + std::string binary; + auto status = pbutil::JsonToBinaryString( + type_resolver.get(), "type.googleapis.com/google.protobuf.Struct", json, + &binary); + if (!status.ok()) { + ADD_FAILURE() << "Error: " << status.error_message() << std::endl + << "Failed to parse \"" << json << "\"." << std::endl; + return false; + } + + if (!message->ParseFromString(binary)) { + ADD_FAILURE() << "Failed to create a struct message for \"" << json << "\"." + << std::endl; + return false; + } + + return true; +} + +} // namespace + +bool ExpectJsonObjectEq(const std::string& expected_json, + const std::string& actual_json) { + // Convert expected_json and actual_json to google.protobuf.Struct messages + pb::Struct expected_proto, actual_proto; + if (!JsonToStruct(expected_json, &expected_proto) || + !JsonToStruct(actual_json, &actual_proto)) { + return false; + } + + // Now try matching the protobuf messages + if (!pbutil::MessageDifferencer::Equivalent(expected_proto, actual_proto)) { + // Use EXPECT_EQ on debug strings to output the diff + EXPECT_EQ(expected_proto.DebugString(), actual_proto.DebugString()); + return false; + } + + return true; +} + +bool ExpectJsonArrayEq(const std::string& expected, const std::string& actual) { + // Wrap the JSON arrays into JSON objects and compare as object + return ExpectJsonObjectEq(R"( { "array" : )" + expected + "}", + R"( { "array" : )" + actual + "}"); +} + +bool JsonArrayTester::TestElement(const std::string& expected, + const std::string& actual) { + if (expected_so_far_.empty()) { + // First element - open the array and add the element + expected_so_far_ = "[" + expected; + } else { + // Had elements before - add a comma and then the element + expected_so_far_ += "," + expected; + } + actual_so_far_ += actual; + + // Add the closing "]" to the partial arrays and compare + return ExpectJsonArrayEq(expected_so_far_ + "]", actual_so_far_ + "]"); +} + +bool JsonArrayTester::TestChunk(const std::string& expected, + const std::string& actual, bool closes) { + expected_so_far_ += expected; + actual_so_far_ += actual; + + // Add the closing "]" to the partial arrays if needed and compare + return ExpectJsonArrayEq(expected_so_far_ + (closes ? "" : "]"), + actual_so_far_ + (closes ? "" : "]")); +} + +bool JsonArrayTester::TestClosed(const std::string& actual) { + if (expected_so_far_.empty()) { + // Empty array case + expected_so_far_ = "["; + } + // Close the finish + expected_so_far_ += "]"; + actual_so_far_ += actual; + + // Compare the closed arrays + return ExpectJsonArrayEq(expected_so_far_, actual_so_far_); +} + +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/test_common.h b/contrib/endpoints/src/grpc/transcoding/test_common.h new file mode 100644 index 00000000000..bc452e052fc --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/test_common.h @@ -0,0 +1,151 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_TEST_COMMON_H_ +#define GRPC_TRANSCODING_TEST_COMMON_H_ + +#include +#include +#include +#include + +#include "google/api/service.pb.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/text_format.h" +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { + +// An implementation of ZeroCopyInputStream for testing. +// The tests define the chunks that TestZeroCopyInputStream produces. +class TestZeroCopyInputStream + : public ::google::protobuf::io::ZeroCopyInputStream { + public: + TestZeroCopyInputStream(); + + // Add an input chunk + void AddChunk(std::string chunk); + + // Ends the stream + void Finish() { finished_ = true; } + + // Returns whether Finish() has been called on this stream or not. + bool Finished() const { return finished_; } + + // ZeroCopyInputStream methods + bool Next(const void** data, int* size); + void BackUp(int count); + ::google::protobuf::int64 ByteCount() const; + bool Skip(int) { return false; } // Not implemented + + private: + std::deque chunks_; + bool finished_; + + int position_; + std::string current_; +}; + +// Test the translation test case with different partitions of the input. The +// test will generate combinations of partitioning input into specified number +// of chunks (chunk_count). For each of the input partition, test assertion is +// verified. The partition is passed to the test assertion as a std::vector of +// partitioning points in the input. +// Because the number of partitionings is O(N^chunk_count) we use a coefficient +// which controls which fraction of partitionings is generated and tested. +// The process of generating partitionings is deterministic. +// +// chunk_count - the number of parts (chunks) in each partition +// partitioning_coefficient - a real number in (0, 1] interval that defines how +// exhaustive the test should be, i.e. what part of +// all partitions of the input string should be +// tested (1.0 means all partitions). +// input - the input string +// test - the test to run +bool RunTestForInputPartitions( + size_t chunk_count, double partitioning_coefficient, + const std::string& input, + std::function& t)> test); + +// Generate an input string of the specified size using the specified seed. +std::string GenerateInput(const std::string& seed, size_t size); + +// Load service from a proto text file. Returns true if loading succeeds; +// otherwise returns false. +bool LoadService(const std::string& config_pb_txt_file, + ::google::api::Service* service); + +// Parses the gRPC message delimiter and returns the size of the message. +unsigned DelimiterToSize(const unsigned char* delimiter); + +// Generates a gRPC message delimiter with the given message size. +std::string SizeToDelimiter(unsigned size); + +// Genereate a proto message with the gRPC delimiter from proto text +template +std::string GenerateGrpcMessage(const std::string& proto_text) { + // Parse the message from text & serialize to binary + MessageType message; + EXPECT_TRUE( + ::google::protobuf::TextFormat::ParseFromString(proto_text, &message)); + std::string binary; + EXPECT_TRUE(message.SerializeToString(&binary)); + + // Now prefix the binary with a delimiter and return + return SizeToDelimiter(binary.size()) + binary; +} + +// Compares JSON objects +bool ExpectJsonObjectEq(const std::string& expected, const std::string& actual); + +// Compares JSON arrays +bool ExpectJsonArrayEq(const std::string& expected, const std::string& actual); + +// JSON array tester that supports matching partial arrays. +class JsonArrayTester { + public: + // Tests a new element of the array. + // expected - the expected new element of the array to match + // actual - the actual JSON chunk (which will include "[", "]" or "," if + // needed) + bool TestElement(const std::string& expected, const std::string& actual); + + // Tests a new chunk of the array (potentially multiple elements). + // expected - the expected new chunk of the array to match (including "[", "]" + // or "," if needed) + // actual - the actual JSON chunk (including "[", "]" or "," if needed) + // closes - indicates whether the chunk closes the array or not. + bool TestChunk(const std::string& expected, const std::string& actual, + bool closes); + + // Test that the array is closed after adding the given JSON chunk (i.e. must + // be "]" modulo whitespace) + bool TestClosed(const std::string& actual); + + private: + std::string expected_so_far_; + std::string actual_so_far_; +}; + +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_MESSAGE_READER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/testdata/bookstore_service.pb.txt b/contrib/endpoints/src/grpc/transcoding/testdata/bookstore_service.pb.txt new file mode 100644 index 00000000000..ebed1e5ab66 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/testdata/bookstore_service.pb.txt @@ -0,0 +1,458 @@ +name: "esp-test.appspot.com" +apis { + methods { + name: "ListShelves" + request_type_url: "type.googleapis.com/google.protobuf.Empty" + response_type_url: "type.googleapis.com/ListShelvesResponse" + } + methods { + name: "CreateShelf" + request_type_url: "type.googleapis.com/CreateShelfRequest" + response_type_url: "type.googleapis.com/Shelf" + } + methods { + name: "GetShelf" + request_type_url: "type.googleapis.com/GetShelfRequest" + response_type_url: "type.googleapis.com/Shelf" + } + methods { + name: "DeleteShelf" + request_type_url: "type.googleapis.com/DeleteShelfRequest" + response_type_url: "type.googleapis.com/google.protobuf.Value" + } + methods { + name: "ListBooks" + request_type_url: "type.googleapis.com/ListBooksRequest" + response_type_url: "type.googleapis.com/ListBooksResponse" + } + methods { + name: "CreateBook" + request_type_url: "type.googleapis.com/CreateBookRequest" + response_type_url: "type.googleapis.com/Book" + } + methods { + name: "CreateBookWithAuthorInfo" + request_type_url: "type.googleapis.com/CreateBookRequest" + response_type_url: "type.googleapis.com/Book" + } + methods { + name: "GetBook" + request_type_url: "type.googleapis.com/GetBookRequest" + response_type_url: "type.googleapis.com/Book" + } + methods { + name: "DeleteBook" + request_type_url: "type.googleapis.com/DeleteBookRequest" + response_type_url: "type.googleapis.com/google.protobuf.Value" + } + methods { + name: "BulkCreateShelves" + request_type_url: "type.googleapis.com/CreateShelfRequest" + response_type_url: "type.googleapis.com/Shelf" + request_streaming: true + response_streaming: true + } + version: "v1" + source_context { + } +} +types { + name: "Biography" + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "year_born" + json_name: "yearBorn" + } + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "year_died" + json_name: "yearDied" + } + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 3 + name: "text" + json_name: "text" + } + source_context { + } +} +types { + name: "AuthorInfo" + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "first_name" + json_name: "firstName" + } + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "last_name" + json_name: "lastName" + } + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_OPTIONAL + number: 3 + name: "bio" + type_url: "type.googleapis.com/Biography" + json_name: "bio" + } + source_context { + } +} +types { + name: "Book" + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "author" + json_name: "author" + } + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "name" + json_name: "name" + } + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 3 + name: "title" + json_name: "title" + } + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_OPTIONAL + number: 4 + name: "author_info" + type_url: "type.googleapis.com/AuthorInfo" + json_name: "authorInfo" + } + source_context { + } +} +types { + name: "ListBooksResponse" + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_REPEATED + number: 1 + name: "books" + type_url: "type.googleapis.com/Book" + json_name: "books" + } + source_context { + } +} +types { + name: "Shelf" + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "name" + json_name: "name" + } + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "theme" + json_name: "theme" + } + source_context { + } +} +types { + name: "ListShelvesResponse" + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_REPEATED + number: 1 + name: "shelves" + type_url: "type.googleapis.com/Shelf" + json_name: "shelves" + } + source_context { + } +} +types { + name: "CreateShelfRequest" + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "shelf" + type_url: "type.googleapis.com/Shelf" + json_name: "shelf" + } + source_context { + } +} +types { + name: "GetShelfRequest" + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "shelf" + json_name: "shelf" + } + source_context { + } +} +types { + name: "DeleteShelfRequest" + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "shelf" + json_name: "shelf" + } + source_context { + } +} +types { + name: "ListBooksRequest" + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "shelf" + json_name: "shelf" + } + source_context { + } +} +types { + name: "CreateBookRequest" + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "shelf" + json_name: "shelf" + } + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "book" + type_url: "type.googleapis.com/Book" + json_name: "book" + } + source_context { + } +} +types { + name: "GetBookRequest" + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "shelf" + json_name: "shelf" + } + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "book" + json_name: "book" + } + source_context { + } +} +types { + name: "DeleteBookRequest" + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "shelf" + json_name: "shelf" + } + fields { + kind: TYPE_INT64 + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "book" + json_name: "book" + } + source_context { + } +} +types { + name: "google.protobuf.ListValue" + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_REPEATED + number: 1 + name: "values" + type_url: "type.googleapis.com/google.protobuf.Value" + json_name: "values" + } + source_context { + file_name: "struct.proto" + } +} +types { + name: "google.protobuf.Struct" + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_REPEATED + number: 1 + name: "fields" + type_url: "type.googleapis.com/google.protobuf.Struct.FieldsEntry" + json_name: "fields" + } + source_context { + file_name: "struct.proto" + } +} +types { + name: "google.protobuf.Struct.FieldsEntry" + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "key" + json_name: "key" + } + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "value" + type_url: "type.googleapis.com/google.protobuf.Value" + json_name: "value" + } + source_context { + file_name: "struct.proto" + } +} +types { + name: "google.protobuf.Empty" + source_context { + file_name: "struct.proto" + } +} +types { + name: "google.protobuf.Value" + fields { + kind: TYPE_ENUM + cardinality: CARDINALITY_OPTIONAL + number: 1 + name: "null_value" + type_url: "type.googleapis.com/google.protobuf.NullValue" + json_name: "nullValue" + } + fields { + kind: TYPE_DOUBLE + cardinality: CARDINALITY_OPTIONAL + number: 2 + name: "number_value" + json_name: "numberValue" + } + fields { + kind: TYPE_STRING + cardinality: CARDINALITY_OPTIONAL + number: 3 + name: "string_value" + json_name: "stringValue" + } + fields { + kind: TYPE_BOOL + cardinality: CARDINALITY_OPTIONAL + number: 4 + name: "bool_value" + json_name: "boolValue" + } + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_OPTIONAL + number: 5 + name: "struct_value" + type_url: "type.googleapis.com/google.protobuf.Struct" + json_name: "structValue" + } + fields { + kind: TYPE_MESSAGE + cardinality: CARDINALITY_OPTIONAL + number: 6 + name: "list_value" + type_url: "type.googleapis.com/google.protobuf.ListValue" + json_name: "listValue" + } + source_context { + file_name: "struct.proto" + } +} +enums { + name: "google.protobuf.NullValue" + enumvalue { + name: "NULL_VALUE" + } + source_context { + file_name: "struct.proto" + } +} +http { + rules { + selector: "ListShelves" + get: "/shelves" + } + rules { + selector: "CreateShelf" + post: "/shelves" + body: "shelf" + } + rules { + selector: "GetShelf" + get: "/shelves/{shelf}" + } + rules { + selector: "DeleteShelf" + delete: "/shelves/{shelf}" + } + rules { + selector: "ListBooks" + get: "/shelves/{shelf}/books" + } + rules { + selector: "CreateBook" + post: "/shelves/{shelf}/books" + body: "book" + } + rules { + selector: "CreateBookWithAuthorInfo" + post: "/shelves/{shelf}/books/{book.authorInfo.firstName}/{book.authorInfo.lastName}" + body: "book" + } + rules { + selector: "GetBook" + get: "/shelves/{shelf}/books/{book}" + } + rules { + selector: "DeleteBook" + delete: "/shelves/{shelf}/books/{book}" + } + rules { + selector: "BulkCreateShelves" + post: "/bulk/shelves" + body: "shelf" + } +} diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder.h b/contrib/endpoints/src/grpc/transcoding/transcoder.h new file mode 100644 index 00000000000..da58386089d --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/transcoder.h @@ -0,0 +1,155 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_TRANSCODER_H_ +#define GRPC_TRANSCODING_TRANSCODER_H_ + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/status.h" + +namespace google { +namespace api_manager { +namespace transcoding { + +// Transcoder interface that transcodes a single request. It holds +// - translated request stream, +// - status of request translation, +// - translated response stream, +// - status of response translation. +// +// NOTE: Transcoder uses ::google::protobuf::io::ZeroCopyInputStream for +// carrying the payloads both for input and output. It assumes the +// following interpretation of the ZeroCopyInputStream interface: +// +// bool ZeroCopyInputStream::Next(const void** data, int* size); +// +// Obtains a chunk of data from the stream. +// +// Preconditions: +// * "size" and "data" are not NULL. +// +// Postconditions: +// * If the returned value is false, there is no more data to return or an error +// occurred. This is permanent. +// * Otherwise, "size" points to the actual number of bytes read and "data" +// points to a pointer to a buffer containing these bytes. +// * Ownership of this buffer remains with the stream, and the buffer remains +// valid only until some other method of the stream is called or the stream is +// destroyed. +// * It is legal for the returned buffer to have zero size. That means there is +// no data available at this point. This is temporary. The caller needs to try +// again later. +// +// +// void ZeroCopyInputStream::BackUp(int count); +// +// Backs up a number of bytes, so that the next call to Next() returns +// data again that was already returned by the last call to Next(). This +// is useful when writing procedures that are only supposed to read up +// to a certain point in the input, then return. If Next() returns a +// buffer that goes beyond what you wanted to read, you can use BackUp() +// to return to the point where you intended to finish. +// +// Preconditions: +// * The last method called must have been Next(). +// * count must be less than or equal to the size of the last buffer +// returned by Next(). +// +// Postconditions: +// * The last "count" bytes of the last buffer returned by Next() will be +// pushed back into the stream. Subsequent calls to Next() will return +// the same data again before producing new data. +// +// +// bool ZeroCopyInputStream::Skip(int count); +// +// Not used and not implemented by the Transcoder. +// +// +// int64 ZeroCopyInputStream::ByteCount() const; +// +// Returns the number of bytes available for reading at this moment +// +// +// NOTE: To support flow-control the translation & reading the input stream +// happens on-demand in both directions. I.e. Transcoder doesn't call +// Next() on the input stream unless Next() is called on the output stream +// and it ran out of input to translate. +// +// EXAMPLE: +// Transcoder* t = transcoder_factory->Create(...); +// +// const void* buffer = nullptr; +// int size = 0; +// while (backend can accept request) { +// if (!t->RequestOutput()->Next(&buffer, &size)) { +// // end of input or error +// if (t->RequestStatus().ok()) { +// // half-close the request +// } else { +// // error +// } +// } else if (size == 0) { +// // no transcoded request data available at this point; wait for more +// // request data to arrive and run this loop again later. +// break; +// } else { +// // send the buffer to the backend +// ... +// } +// } +// +// const void* buffer = nullptr; +// int size = 0; +// while (client can accept response) { +// if (!t->ResponseOutput()->Next(&buffer, &size)) { +// // end of input or error +// if (t->ResponseStatus().ok()) { +// // close the request +// } else { +// // error +// } +// } else if (size == 0) { +// // no transcoded response data available at this point; wait for more +// // response data to arrive and run this loop again later. +// break; +// } else { +// // send the buffer to the client +// ... +// } +// } +// +class Transcoder { + public: + // ZeroCopyInputStream to read the transcoded request. + virtual ::google::protobuf::io::ZeroCopyInputStream* RequestOutput() = 0; + + // The status of request transcoding + virtual ::google::protobuf::util::Status RequestStatus() = 0; + + // ZeroCopyInputStream to read the transcoded response. + virtual ::google::protobuf::io::ZeroCopyInputStream* ResponseOutput() = 0; + + // The status of response transcoding + virtual ::google::protobuf::util::Status ResponseStatus() = 0; + + // Virtual destructor + virtual ~Transcoder() {} +}; + +} // namespace transcoding +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc new file mode 100644 index 00000000000..5b245666916 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc @@ -0,0 +1,169 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/transcoder_factory.h" + +#include +#include +#include + +#include "google/api/service.pb.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/common.h" +#include "google/protobuf/stubs/status.h" +#include "include/api_manager/method_call_info.h" +#include "src/grpc/transcoding/json_request_translator.h" +#include "src/grpc/transcoding/message_stream.h" +#include "src/grpc/transcoding/response_to_json_translator.h" +#include "src/grpc/transcoding/type_helper.h" + +namespace google { +namespace api_manager { +namespace transcoding { +namespace { + +namespace pb = ::google::protobuf; +namespace pbio = ::google::protobuf::io; +namespace pbutil = ::google::protobuf::util; +namespace pberr = ::google::protobuf::util::error; + +// Transcoder implementation based on JsonRequestTranslator & +// ResponseToJsonTranslator +class TranscoderImpl : public Transcoder { + public: + // request_translator - a JsonRequestTranslator that does the request + // translation + // response_translator - a ResponseToJsonTranslator that does the response + // translation + TranscoderImpl(std::unique_ptr request_translator, + std::unique_ptr response_translator) + : request_translator_(std::move(request_translator)), + response_translator_(std::move(response_translator)), + request_zero_copy_stream_( + request_translator_->Output().CreateZeroCopyInputStream()), + response_zero_copy_stream_( + response_translator_->CreateZeroCopyInputStream()) {} + + // Transcoder implementation + pbio::ZeroCopyInputStream* RequestOutput() { + return request_zero_copy_stream_.get(); + } + pbutil::Status RequestStatus() { + return request_translator_->Output().Status(); + } + + pbio::ZeroCopyInputStream* ResponseOutput() { + return response_zero_copy_stream_.get(); + } + pbutil::Status ResponseStatus() { return response_translator_->Status(); } + + private: + std::unique_ptr request_translator_; + std::unique_ptr response_translator_; + std::unique_ptr request_zero_copy_stream_; + std::unique_ptr response_zero_copy_stream_; +}; + +// Converts MethodCallInfo into a RequestInfo structure needed by the +// JsonRequestTranslator. +pbutil::Status MethodCallInfoToRequestInfo(TypeHelper* type_helper, + const MethodCallInfo& call_info, + RequestInfo* request_info) { + // Try to resolve the request type + const auto& request_type_url = call_info.method_info->request_type_url(); + request_info->message_type = + type_helper->Info()->GetTypeByTypeUrl(request_type_url); + if (nullptr == request_info->message_type) { + return pbutil::Status(pberr::NOT_FOUND, + "Could not resolve the type \"" + request_type_url + + "\". Invalid service configuration."); + } + + // Copy the body field path + request_info->body_field_path = call_info.body_field_path; + + // Resolve the field paths of the bindings and add to the request_info + for (const auto& unresolved_binding : call_info.variable_bindings) { + RequestWeaver::BindingInfo resolved_binding; + + // Verify that the value is valid UTF8 before continuing + if (!pb::internal::IsStructurallyValidUTF8( + unresolved_binding.value.c_str(), + unresolved_binding.value.size())) { + return pbutil::Status(pberr::INVALID_ARGUMENT, + "Encountered non UTF-8 code points."); + } + + resolved_binding.value = unresolved_binding.value; + + // Try to resolve the field path + auto status = type_helper->ResolveFieldPath(*request_info->message_type, + unresolved_binding.field_path, + &resolved_binding.field_path); + if (!status.ok()) { + // Field path could not be resolved (usually a config error) - return + // the error. + return status; + } + + request_info->variable_bindings.emplace_back(std::move(resolved_binding)); + } + + return pbutil::Status::OK; +} + +} // namespace + +TranscoderFactory::TranscoderFactory(const ::google::api::Service& service) + : type_helper_(service.types(), service.enums()) {} + +pbutil::Status TranscoderFactory::Create( + const MethodCallInfo& call_info, pbio::ZeroCopyInputStream* request_input, + pbio::ZeroCopyInputStream* response_input, + std::unique_ptr* transcoder) { + // Convert MethodCallInfo into RequestInfo + RequestInfo request_info; + auto status = + MethodCallInfoToRequestInfo(&type_helper_, call_info, &request_info); + if (!status.ok()) { + return status; + } + + // For now we support only HTTP/JSON <=> gRPC transcoding. + + // Create a JsonRequestTranslator for translating the request + std::unique_ptr request_translator( + new JsonRequestTranslator(type_helper_.Resolver(), request_input, + request_info, + call_info.method_info->request_streaming(), + /*output_delimiters*/ true)); + + // Create a ResponseToJsonTranslator for translating the response + std::unique_ptr response_translator( + new ResponseToJsonTranslator( + type_helper_.Resolver(), call_info.method_info->response_type_url(), + call_info.method_info->response_streaming(), response_input)); + + // Create the Transcoder + transcoder->reset(new TranscoderImpl(std::move(request_translator), + std::move(response_translator))); + + return pbutil::Status::OK; +} + +} // namespace transcoding +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h new file mode 100644 index 00000000000..bbe7b96a021 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h @@ -0,0 +1,85 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_TRANSODER_FACTORY_H_ +#define GRPC_TRANSCODING_TRANSODER_FACTORY_H_ + +#include + +#include "google/api/service.pb.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/status.h" +#include "include/api_manager/method_call_info.h" +#include "src/grpc/transcoding/transcoder.h" +#include "src/grpc/transcoding/type_helper.h" + +namespace google { +namespace api_manager { +namespace transcoding { + +// Transcoder factory for a specific service config. Holds the preprocessed +// service config and creates a Transcoder per each client request using the +// following information: +// - method call information +// - RPC method info +// - request & response message types +// - request & response streaming flags +// - HTTP body field path +// - request variable bindings +// - Values for certain fields to be injected into the request message. +// - request input stream for the request JSON coming from the client, +// - response input stream for the response proto coming from the backend. +// +// EXAMPLE: +// TranscoderFactory factory(service_config); +// +// ::google::protobuf::io::ZeroCopyInputStream *request_downstream = +// CreateClientJsonRequestStream(); +// +// ::google::protobuf::io::ZeroCopyInputStream *response_upstream = +// CreateBackendProtoResponseStream(); +// +// unique_ptr transcoder; +// status = factory.Create(request_info, +// request_downstream, +// response_upstream, +// &transcoder); +// +class TranscoderFactory { + public: + // service - The service config for which the factory is created + TranscoderFactory(const ::google::api::Service& service_config); + + // Creates a Transcoder object to transcode a single client request + // call_info - contains all the necessary info for setting up transcoding + // request_input - ZeroCopyInputStream that carries the JSON request coming + // from the client + // response_input - ZeroCopyInputStream that carries the proto response + // coming from the backend + // transcoder - the output Transcoder object + ::google::protobuf::util::Status Create( + const MethodCallInfo& call_info, + ::google::protobuf::io::ZeroCopyInputStream* request_input, + ::google::protobuf::io::ZeroCopyInputStream* response_input, + std::unique_ptr* transcoder); + + private: + TypeHelper type_helper_; +}; + +} // namespace transcoding +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_TRANSODER_FACTORY_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc new file mode 100644 index 00000000000..569e7b4b404 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc @@ -0,0 +1,513 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include +#include +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/stubs/strutil.h" +#include "google/protobuf/util/message_differencer.h" +#include "gtest/gtest.h" +#include "include/api_manager/method.h" +#include "include/api_manager/method_call_info.h" +#include "src/grpc/transcoding/bookstore.pb.h" +#include "src/grpc/transcoding/message_reader.h" +#include "src/grpc/transcoding/test_common.h" +#include "src/grpc/transcoding/transcoder.h" +#include "src/grpc/transcoding/transcoder_factory.h" + +namespace google { +namespace api_manager { +namespace transcoding { +namespace testing { +namespace { + +namespace pb = google::protobuf; +namespace pbio = google::protobuf::io; +namespace pbutil = google::protobuf::util; +namespace pberr = google::protobuf::util::error; + +// MethodInfo implementation for testing. Only implements the methods that +// the TranscoderFactory needs. +class TestMethodInfo : public MethodInfo { + public: + TestMethodInfo() {} + TestMethodInfo(const std::string &request_type_url, + const std::string &response_type_url, bool request_streaming, + bool response_streaming, const std::string &body_field_path) + : request_type_url_(request_type_url), + response_type_url_(response_type_url), + request_streaming_(request_streaming), + response_streaming_(response_streaming), + body_field_path_(body_field_path) {} + + // MethodInfo implementation + // Methods that the Transcoder doesn't use + const std::string &name() const { return empty_; } + const std::string &api_name() const { return empty_; } + const std::string &api_version() const { return empty_; } + const std::string &selector() const { return empty_; } + bool auth() const { return false; } + bool allow_unregistered_calls() const { return false; } + bool isIssuerAllowed(const std::string &issuer) const { return false; } + bool isAudienceAllowed(const std::string &issuer, + const std::set &jwt_audiences) const { + return false; + } + const std::vector *http_header_parameters( + const std::string &name) const { + return nullptr; + } + const std::vector *url_query_parameters( + const std::string &name) const { + return nullptr; + } + const std::vector *api_key_http_headers() const { + return nullptr; + } + const std::vector *api_key_url_query_parameters() const { + return nullptr; + } + const std::string &backend_address() const { return empty_; } + const std::string &rpc_method_full_name() const { return empty_; } + const std::set &system_query_parameter_names() const { + static std::set dummy; + return dummy; + }; + + // Methods that the Transcoder does use + const std::string &request_type_url() const { return request_type_url_; } + bool request_streaming() const { return request_streaming_; } + const std::string &response_type_url() const { return response_type_url_; } + bool response_streaming() const { return response_streaming_; } + const std::string &body_field_path() const { return body_field_path_; } + + private: + std::string request_type_url_; + std::string response_type_url_; + bool request_streaming_; + bool response_streaming_; + std::string body_field_path_; + std::string empty_; +}; + +class TranscoderTest : public ::testing::Test { + public: + // Load the service config to be used for testing. This must be the first call + // in a test. + bool LoadService(const std::string &config_pb_txt_file) { + if (!::google::api_manager::transcoding::testing::LoadService( + config_pb_txt_file, &service_)) { + return false; + } + transcoder_factory_.reset(new TranscoderFactory(service_)); + return true; + } + + // Provide the method info + void SetMethodInfo(const std::string &request_type_url, + const std::string &response_type_url, + bool request_streaming = false, + bool response_streaming = false, + const std::string &body_field_path = "") { + method_info_.reset(new TestMethodInfo(request_type_url, response_type_url, + request_streaming, response_streaming, + body_field_path)); + } + + void AddVariableBinding(const std::string &field_path, + const std::string &value) { + VariableBinding binding; + binding.field_path = pb::Split(field_path, ".", /*skip_empty*/ true); + binding.value = value; + variable_bindings_.emplace_back(binding); + } + + pbutil::Status Build(pbio::ZeroCopyInputStream *request_input, + pbio::ZeroCopyInputStream *response_input, + std::unique_ptr *transcoder) { + MethodCallInfo call_info; + call_info.method_info = method_info_.get(); + call_info.variable_bindings = std::move(variable_bindings_); + call_info.body_field_path = method_info_->body_field_path(); + + return transcoder_factory_->Create(call_info, request_input, response_input, + transcoder); + } + + private: + ::google::api::Service service_; + std::unique_ptr transcoder_factory_; + + std::unique_ptr method_info_; + std::vector variable_bindings_; +}; + +// A helper function that determines whether the ZeroCopyInputStream has +// finished or not. +bool IsFinished(pbio::ZeroCopyInputStream *stream) { + const void *data = nullptr; + int size = 0; + if (stream->Next(&data, &size)) { + stream->BackUp(size); + return false; + } else { + return true; + } +} + +// A helper function to read all available data from a ZeroCopyInputStream +std::string ReadAll(pbio::ZeroCopyInputStream *stream) { + std::ostringstream all; + const void *data = nullptr; + int size = 0; + while (stream->Next(&data, &size) && 0 != size) { + all << std::string(reinterpret_cast(data), + static_cast(size)); + } + return all.str(); +} + +TEST_F(TranscoderTest, SimpleRequestAndResponse) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMethodInfo(/*request_type_url*/ "type.googleapis.com/Shelf", + /*response_type_url*/ "type.googleapis.com/Shelf"); + + // Build the Transcoder + std::unique_ptr t; + TestZeroCopyInputStream request_in, response_in; + auto status = Build(&request_in, &response_in, &t); + ASSERT_TRUE(status.ok()) << "Error building Transcoder - " + << status.error_message() << std::endl; + + // Create a MessageReader for reading & testing the request output + MessageReader reader(t->RequestOutput()); + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + + // Add a JSON chunk of a partial message to the request input + request_in.AddChunk(R"({"name" : "1")"); + + // Nothing yet + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + + // Add the rest of the message + request_in.AddChunk(R"(, "theme" : "Fiction"})"); + request_in.Finish(); + + Shelf expected; + ASSERT_TRUE(pb::TextFormat::ParseFromString(R"(name : "1" theme : "Fiction")", + &expected)); + + // Read the message + EXPECT_FALSE(reader.Finished()); + auto actual_proto = reader.NextMessage(); + ASSERT_NE(nullptr, actual_proto.get()); + + // Parse & match + Shelf actual; + ASSERT_TRUE(actual.ParseFromZeroCopyStream(actual_proto.get())); + EXPECT_TRUE(pbutil::MessageDifferencer::Equivalent(expected, actual)); + + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_TRUE(reader.Finished()); + EXPECT_TRUE(t->RequestStatus().ok()) << "Error while translating - " + << t->RequestStatus().error_message() + << std::endl; + + // Now test the response translation + EXPECT_TRUE(ReadAll(t->ResponseOutput()).empty()); + EXPECT_FALSE(IsFinished(t->ResponseOutput())); + + // Add a partial message + auto message = GenerateGrpcMessage(R"(name : "2" theme : "Mystery")"); + response_in.AddChunk(message.substr(0, 10)); + + // Nothing yet + EXPECT_TRUE(ReadAll(t->ResponseOutput()).empty()); + EXPECT_FALSE(IsFinished(t->ResponseOutput())); + + // Add the rest of the message + response_in.AddChunk(message.substr(10)); + response_in.Finish(); + + // Read & test the JSON message + auto json = ReadAll(t->ResponseOutput()); + EXPECT_TRUE( + ExpectJsonObjectEq(R"({"name" : "2", "theme" : "Mystery"})", json)); + EXPECT_TRUE(IsFinished(t->ResponseOutput())); +} + +TEST_F(TranscoderTest, RequestBindingsAndPrefix) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMethodInfo(/*request_type_url*/ "type.googleapis.com/CreateBookRequest", + /*response_type_url*/ "type.googleapis.com/Book", + /*request_streaming*/ false, + /*response_streaming*/ false, + /*body_field_path*/ "book"); + AddVariableBinding("shelf", "99"); + AddVariableBinding("book.author", "Leo Tolstoy"); + AddVariableBinding("book.authorInfo.firstName", "Leo"); + AddVariableBinding("book.authorInfo.lastName", "Tolstoy"); + + auto json = R"({ + "name" : "1", + "title" : "War and Peace" + } + )"; + + auto expected_proto_text = + R"( + shelf : 99 + book { + name : "1" + title : "War and Peace" + author : "Leo Tolstoy" + author_info { + first_name : "Leo" + last_name : "Tolstoy" + } + } + )"; + + CreateBookRequest expected; + ASSERT_TRUE(pb::TextFormat::ParseFromString(expected_proto_text, &expected)); + + // Build the Transcoder + std::unique_ptr t; + TestZeroCopyInputStream request_in, response_in; + auto status = Build(&request_in, &response_in, &t); + ASSERT_TRUE(status.ok()) << "Error building Transcoder - " + << status.error_message() << std::endl; + + // Add input the json + request_in.AddChunk(json); + + // Read the message + MessageReader reader(t->RequestOutput()); + auto actual_proto = reader.NextMessage(); + ASSERT_NE(nullptr, actual_proto.get()); + + // Parse & match + CreateBookRequest actual; + ASSERT_TRUE(actual.ParseFromZeroCopyStream(actual_proto.get())); + EXPECT_TRUE(pbutil::MessageDifferencer::Equivalent(expected, actual)); + + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_TRUE(reader.Finished()); + EXPECT_TRUE(t->RequestStatus().ok()) << "Error while translating - " + << t->RequestStatus().error_message() + << std::endl; +} + +TEST_F(TranscoderTest, StreamingRequestAndResponse) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMethodInfo(/*request_type_url*/ "type.googleapis.com/Shelf", + /*response_type_url*/ "type.googleapis.com/Shelf", + /*request_streaming*/ true, + /*response_streaming*/ true); + + // Build the Transcoder + std::unique_ptr t; + TestZeroCopyInputStream request_in, response_in; + auto status = Build(&request_in, &response_in, &t); + ASSERT_TRUE(status.ok()) << "Error building Transcoder - " + << status.error_message() << std::endl; + + // Add 2 complete messages and a partial one + request_in.AddChunk( + R"( + [ + {"name" : "1", "theme" : "Fiction"}, + {"name" : "2", "theme" : "Satire"}, + {"name" : "3", + )"); + + // Read & test the 2 translated messages + MessageReader reader(t->RequestOutput()); + + auto actual_proto1 = reader.NextMessage(); + ASSERT_NE(nullptr, actual_proto1.get()); + Shelf actual1; + ASSERT_TRUE(actual1.ParseFromZeroCopyStream(actual_proto1.get())); + + auto actual_proto2 = reader.NextMessage(); + ASSERT_NE(nullptr, actual_proto2.get()); + Shelf actual2; + ASSERT_TRUE(actual2.ParseFromZeroCopyStream(actual_proto2.get())); + + Shelf expected1; + ASSERT_TRUE(pb::TextFormat::ParseFromString( + R"(name : "1" theme : "Fiction")", &expected1)); + EXPECT_TRUE(pbutil::MessageDifferencer::Equivalent(expected1, actual1)); + + Shelf expected2; + ASSERT_TRUE(pb::TextFormat::ParseFromString( + R"(name : "2" theme : "Satire")", &expected2)); + EXPECT_TRUE(pbutil::MessageDifferencer::Equivalent(expected2, actual2)); + + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_FALSE(reader.Finished()); + EXPECT_TRUE(t->RequestStatus().ok()) << "Error while translating - " + << t->RequestStatus().error_message() + << std::endl; + + // Add the rest of the 3rd message, the 4th message and close the array + request_in.AddChunk( + R"( + "theme" : "Classic"}, + {"name" : "4", "theme" : "Russian"} + ] + )"); + + auto actual_proto3 = reader.NextMessage(); + ASSERT_NE(nullptr, actual_proto3.get()); + Shelf actual3; + ASSERT_TRUE(actual3.ParseFromZeroCopyStream(actual_proto3.get())); + + auto actual_proto4 = reader.NextMessage(); + ASSERT_NE(nullptr, actual_proto4.get()); + Shelf actual4; + ASSERT_TRUE(actual4.ParseFromZeroCopyStream(actual_proto4.get())); + + Shelf expected3; + ASSERT_TRUE(pb::TextFormat::ParseFromString( + R"(name : "3" theme : "Classic")", &expected3)); + EXPECT_TRUE(pbutil::MessageDifferencer::Equivalent(expected3, actual3)); + + Shelf expected4; + ASSERT_TRUE(pb::TextFormat::ParseFromString( + R"(name : "4" theme : "Russian")", &expected4)); + EXPECT_TRUE(pbutil::MessageDifferencer::Equivalent(expected4, actual4)); + + EXPECT_EQ(nullptr, reader.NextMessage().get()); + EXPECT_TRUE(reader.Finished()); + EXPECT_TRUE(t->RequestStatus().ok()) << "Error while translating - " + << t->RequestStatus().error_message() + << std::endl; + + // Test the response translation + + // Add two full messages and one partial + auto message1 = GenerateGrpcMessage(R"(name : "1" theme : "Fiction")"); + auto message2 = GenerateGrpcMessage(R"(name : "2" theme : "Satire")"); + auto message3 = GenerateGrpcMessage(R"(name : "3" theme : "Classic")"); + response_in.AddChunk(message1); + response_in.AddChunk(message2); + response_in.AddChunk(message3.substr(0, 10)); + + // Read & test the translated JSON + + auto expected12 = R"( + [ + {"name" : "1", "theme" : "Fiction"}, + {"name" : "2", "theme" : "Satire"}, + )"; + + auto actual12 = ReadAll(t->ResponseOutput()); + + JsonArrayTester array_tester; + EXPECT_TRUE(array_tester.TestChunk(expected12, actual12, false)); + EXPECT_FALSE(IsFinished(t->ResponseOutput())); + EXPECT_TRUE(t->ResponseStatus().ok()) << "Error while translating - " + << t->ResponseStatus().error_message() + << std::endl; + + // Add the rest of the third message, the fourth message and finish + response_in.AddChunk(message3.substr(10)); + auto message4 = GenerateGrpcMessage(R"(name : "4" theme : "Russian")"); + response_in.AddChunk(message4); + response_in.Finish(); + + // Read & test + + auto expected34 = R"( + {"name" : "3", "theme" : "Classic"}, + {"name" : "4", "theme" : "Russian"} + ])"; + + auto actual34 = ReadAll(t->ResponseOutput()); + + EXPECT_TRUE(array_tester.TestChunk(expected34, actual34, true)); + EXPECT_TRUE(IsFinished(t->ResponseOutput())); + EXPECT_TRUE(t->ResponseStatus().ok()) << "Error while translating - " + << t->ResponseStatus().error_message() + << std::endl; +} + +TEST_F(TranscoderTest, ErrorResolvingVariableBinding) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMethodInfo(/*request_type_url*/ "type.googleapis.com/Shelf", + /*response_type_url*/ "type.googleapis.com/Shelf"); + AddVariableBinding("invalid.binding", "value"); + + std::unique_ptr t; + TestZeroCopyInputStream request_in, response_in; + EXPECT_EQ(pberr::INVALID_ARGUMENT, + Build(&request_in, &response_in, &t).error_code()); +} + +TEST_F(TranscoderTest, TranslationError) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + // Using an invalid type for simulating response translation error as it is + // hard to generate an invalid protobuf messsage. Here we mainly test whether + // the error is propagated correctly or not. + SetMethodInfo(/*request_type_url*/ "type.googleapis.com/Shelf", + /*response_type_url*/ "type.googleapis.com/InvalidType"); + + // Build the Transcoder + std::unique_ptr t; + TestZeroCopyInputStream request_in, response_in; + auto status = Build(&request_in, &response_in, &t); + ASSERT_TRUE(status.ok()) << "Error building Transcoder - " + << status.error_message() << std::endl; + + // Request error + request_in.AddChunk(R"(Invalid JSON)"); + + // Read the stream to trigger the error + const void *buffer = nullptr; + int size = 0; + EXPECT_FALSE(t->RequestOutput()->Next(&buffer, &size)); + EXPECT_EQ(pberr::INVALID_ARGUMENT, t->RequestStatus().error_code()); + + // Response error + response_in.AddChunk( + GenerateGrpcMessage(R"(name : "1" theme : "Fiction")")); + EXPECT_FALSE(t->ResponseOutput()->Next(&buffer, &size)); + EXPECT_EQ(pberr::NOT_FOUND, t->ResponseStatus().error_code()); +} + +TEST_F(TranscoderTest, InvalidUTF8InVariableBinding) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + SetMethodInfo(/*request_type_url*/ "type.googleapis.com/Shelf", + /*response_type_url*/ "type.googleapis.com/Shelf"); + AddVariableBinding("theme", "\xC2\xE2\x98"); + + std::unique_ptr t; + TestZeroCopyInputStream request_in, response_in; + EXPECT_EQ(pberr::INVALID_ARGUMENT, + Build(&request_in, &response_in, &t).error_code()); +} + +} // namespace +} // namespace testing +} // namespace transcoding +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/type_helper.cc b/contrib/endpoints/src/grpc/transcoding/type_helper.cc new file mode 100644 index 00000000000..af0cac5a48e --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/type_helper.cc @@ -0,0 +1,182 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/type_helper.h" + +#include "google/protobuf/stubs/strutil.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/internal/type_info.h" +#include "google/protobuf/util/type_resolver.h" + +#include +#include + +namespace pb = ::google::protobuf; +namespace pbutil = ::google::protobuf::util; +namespace pbconv = ::google::protobuf::util::converter; +namespace pberr = ::google::protobuf::util::error; + +namespace google { +namespace api_manager { + +namespace transcoding { + +const char DEFAULT_URL_PREFIX[] = "type.googleapis.com/"; + +class SimpleTypeResolver : public pbutil::TypeResolver { + public: + SimpleTypeResolver() : url_prefix_(DEFAULT_URL_PREFIX) {} + + void AddType(const pb::Type& t) { + type_map_.emplace(url_prefix_ + t.name(), &t); + // A temporary workaround for service configs that use + // "proto2.MessageOptions.*" options. + ReplaceProto2WithGoogleProtobufInOptionNames(const_cast(&t)); + } + + void AddEnum(const pb::Enum& e) { + enum_map_.emplace(url_prefix_ + e.name(), &e); + } + + // TypeResolver implementation + // Resolves a type url for a message type. + virtual pbutil::Status ResolveMessageType(const std::string& type_url, + pb::Type* type) { + auto i = type_map_.find(type_url); + if (end(type_map_) != i) { + if (nullptr != type) { + *type = *i->second; + } + return pbutil::Status(); + } else { + return pbutil::Status(pberr::NOT_FOUND, + "Type '" + type_url + "' cannot be found."); + } + } + + // Resolves a type url for an enum type. + virtual pbutil::Status ResolveEnumType(const std::string& type_url, + pb::Enum* enum_type) override { + auto i = enum_map_.find(type_url); + if (end(enum_map_) != i) { + if (nullptr != enum_type) { + *enum_type = *i->second; + } + return pbutil::Status(); + } else { + return pbutil::Status(pberr::NOT_FOUND, + "Enum '" + type_url + "' cannot be found."); + } + } + + private: + void ReplaceProto2WithGoogleProtobufInOptionNames(pb::Type* type) { + // As a temporary workaround for service configs that use + // "proto2.MessageOptions.*" options instead of + // "google.protobuf.MessageOptions.*", we replace the option names to make + // protobuf library recognize them. + for (auto& option : *type->mutable_options()) { + if (option.name() == "proto2.MessageOptions.map_entry") { + option.set_name("google.protobuf.MessageOptions.map_entry"); + } else if (option.name() == + "proto2.MessageOptions.message_set_wire_format") { + option.set_name( + "google.protobuf.MessageOptions.message_set_wire_format"); + } + } + } + + std::string url_prefix_; + std::unordered_map type_map_; + std::unordered_map enum_map_; + + SimpleTypeResolver(const SimpleTypeResolver&) = delete; + SimpleTypeResolver& operator=(const SimpleTypeResolver&) = delete; +}; + +TypeHelper::~TypeHelper() { + type_info_.reset(); + delete type_resolver_; +} + +pbutil::TypeResolver* TypeHelper::Resolver() const { return type_resolver_; } + +pbconv::TypeInfo* TypeHelper::Info() const { return type_info_.get(); } + +void TypeHelper::Initialize() { + type_resolver_ = new SimpleTypeResolver(); + type_info_.reset(pbconv::TypeInfo::NewTypeInfo(type_resolver_)); +} + +void TypeHelper::AddType(const pb::Type& t) { type_resolver_->AddType(t); } + +void TypeHelper::AddEnum(const pb::Enum& e) { type_resolver_->AddEnum(e); } + +pbutil::Status TypeHelper::ResolveFieldPath( + const pb::Type& type, const std::string& field_path_str, + std::vector* field_path_out) const { + // Split the field names & call ResolveFieldPath() + auto field_names = pb::Split(field_path_str, "."); + return ResolveFieldPath(type, field_names, field_path_out); +} + +pbutil::Status TypeHelper::ResolveFieldPath( + const pb::Type& type, const std::vector& field_names, + std::vector* field_path_out) const { + // The type of the current message being processed (initially the type of the + // top level message) + auto current_type = &type; + + // The resulting field path + std::vector field_path; + + for (size_t i = 0; i < field_names.size(); ++i) { + // Find the field by name in the current type + auto field = Info()->FindField(current_type, field_names[i]); + if (nullptr == field) { + return pbutil::Status(pberr::INVALID_ARGUMENT, + "Could not find field \"" + field_names[i] + + "\" in the type \"" + current_type->name() + + "\"."); + } + field_path.push_back(field); + + if (i < field_names.size() - 1) { + // If this is not the last field in the path, it must be a message + if (pb::Field::TYPE_MESSAGE != field->kind()) { + return pbutil::Status( + pberr::INVALID_ARGUMENT, + "Encountered a non-leaf field \"" + field->name() + + "\" that is not a message while parsing a field path"); + } + + // Update the type of the current message + current_type = Info()->GetTypeByTypeUrl(field->type_url()); + if (nullptr == current_type) { + return pbutil::Status(pberr::INVALID_ARGUMENT, + "Cannot find the type \"" + field->type_url() + + "\" while parsing a field path."); + } + } + } + *field_path_out = std::move(field_path); + return pbutil::Status::OK; +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/grpc/transcoding/type_helper.h b/contrib/endpoints/src/grpc/transcoding/type_helper.h new file mode 100644 index 00000000000..33509988e78 --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/type_helper.h @@ -0,0 +1,105 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_TYPE_HELPER_H_ +#define GRPC_TRANSCODING_TYPE_HELPER_H_ + +#include "google/protobuf/type.pb.h" +#include "google/protobuf/util/internal/type_info.h" +#include "google/protobuf/util/type_resolver.h" + +#include +#include + +namespace google { +namespace api_manager { + +namespace transcoding { + +class SimpleTypeResolver; + +// Provides ::google::protobuf::util::TypeResolver and +// ::google::protobuf::util::converter::TypeInfo implementations based on a +// collection of types and a collection of enums. +class TypeHelper { + public: + template + TypeHelper(const Types& types, const Enums& enums); + ~TypeHelper(); + + ::google::protobuf::util::TypeResolver* Resolver() const; + ::google::protobuf::util::converter::TypeInfo* Info() const; + + // Takes a string representation of a field path & resolves it into actual + // protobuf Field pointers. + // + // A field path is a sequence of fields that identifies a potentially nested + // field in the message. It can be empty as well, which identifies the entire + // message. + // E.g. "shelf.theme" field path would correspond to the "theme" field of the + // "shelf" field of the top-level message. The type of the top-level message + // is passed to ResolveFieldPath(). + // + // The string representation of the field path is just the dot-delimited + // list of the field names or empty: + // FieldPath = "" | Field {"." Field}; + // Field = ; + ::google::protobuf::util::Status ResolveFieldPath( + const ::google::protobuf::Type& type, const std::string& field_path_str, + std::vector* field_path) const; + + // Resolve a field path specified through a vector of field names into a + // vector of actual protobuf Field pointers. + // Similiar to the above method but accepts the field path as a vector of + // names instead of one dot-delimited string. + ::google::protobuf::util::Status ResolveFieldPath( + const ::google::protobuf::Type& type, + const std::vector& field_path_unresolved, + std::vector* field_path_resolved) const; + + private: + void Initialize(); + void AddType(const ::google::protobuf::Type& t); + void AddEnum(const ::google::protobuf::Enum& e); + + // We can't use a unique_ptr as the default deleter of + // unique_ptr requires the type to be defined when the unique_ptr destructor + // is called. In our case it's called from the template constructor below + // (most likely as a part of stack unwinding when an exception occurs). + SimpleTypeResolver* type_resolver_; + std::unique_ptr<::google::protobuf::util::converter::TypeInfo> type_info_; + + TypeHelper() = delete; + TypeHelper(const TypeHelper&) = delete; + TypeHelper& operator=(const TypeHelper&) = delete; +}; + +template +TypeHelper::TypeHelper(const Types& types, const Enums& enums) + : type_resolver_(nullptr), type_info_() { + Initialize(); + for (const auto& t : types) { + AddType(t); + } + for (const auto& e : enums) { + AddEnum(e); + } +} + +} // namespace transcoding + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODING_TYPE_HELPER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/type_helper_test.cc b/contrib/endpoints/src/grpc/transcoding/type_helper_test.cc new file mode 100644 index 00000000000..d943bf2f71c --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/type_helper_test.cc @@ -0,0 +1,344 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/grpc/transcoding/type_helper.h" + +#include +#include +#include +#include + +#include "google/api/service.pb.h" +#include "google/protobuf/text_format.h" +#include "google/protobuf/type.pb.h" +#include "gtest/gtest.h" +#include "src/grpc/transcoding/test_common.h" + +namespace pb = ::google::protobuf; + +namespace google { +namespace api_manager { + +namespace transcoding { +namespace testing { + +namespace { + +class TypeHelperTest : public ::testing::Test { + protected: + TypeHelperTest() : types_(), enums_() {} + + void AddType(const std::string& n) { + pb::Type t; + t.set_name(n); + types_.emplace_back(std::move(t)); + } + + void AddEnum(const std::string& n) { + pb::Enum e; + e.set_name(n); + enums_.emplace_back(std::move(e)); + } + + void Build() { helper_.reset(new TypeHelper(types_, enums_)); } + + const pb::Type* GetType(const std::string& url) { + return helper_->Info()->GetTypeByTypeUrl(url); + } + + const pb::Enum* GetEnum(const std::string& url) { + return helper_->Info()->GetEnumByTypeUrl(url); + } + + private: + std::vector types_; + std::vector enums_; + std::unique_ptr helper_; +}; + +TEST_F(TypeHelperTest, OneType) { + AddType("Shelf"); + Build(); + + ASSERT_NE(nullptr, GetType("type.googleapis.com/Shelf")); + EXPECT_EQ("Shelf", GetType("type.googleapis.com/Shelf")->name()); + EXPECT_EQ(nullptr, GetEnum("type.googleapis.com/Shelf")); +} + +TEST_F(TypeHelperTest, OneEnum) { + AddEnum("ShelfType"); + Build(); + + ASSERT_NE(nullptr, GetEnum("type.googleapis.com/ShelfType")); + EXPECT_EQ("ShelfType", GetEnum("type.googleapis.com/ShelfType")->name()); + EXPECT_EQ(nullptr, GetType("type.googleapis.com/ShelfType")); +} + +TEST_F(TypeHelperTest, MultipleTypesAndEnums) { + AddType("Shelf"); + AddType("ShelfList"); + AddType("Book"); + AddType("BookList"); + AddEnum("ShelfType"); + AddEnum("BookType"); + Build(); + + ASSERT_NE(nullptr, GetType("type.googleapis.com/Shelf")); + EXPECT_EQ("Shelf", GetType("type.googleapis.com/Shelf")->name()); + + ASSERT_NE(nullptr, GetType("type.googleapis.com/ShelfList")); + EXPECT_EQ("ShelfList", GetType("type.googleapis.com/ShelfList")->name()); + + ASSERT_NE(nullptr, GetType("type.googleapis.com/Book")); + EXPECT_EQ("Book", GetType("type.googleapis.com/Book")->name()); + + ASSERT_NE(nullptr, GetType("type.googleapis.com/BookList")); + EXPECT_EQ("BookList", GetType("type.googleapis.com/BookList")->name()); + + ASSERT_NE(nullptr, GetEnum("type.googleapis.com/ShelfType")); + EXPECT_EQ("ShelfType", GetEnum("type.googleapis.com/ShelfType")->name()); + + ASSERT_NE(nullptr, GetEnum("type.googleapis.com/BookType")); + EXPECT_EQ("BookType", GetEnum("type.googleapis.com/BookType")->name()); +} + +TEST_F(TypeHelperTest, UrlMismatch) { + AddType("Shelf"); + AddEnum("ShelfType"); + Build(); + + EXPECT_EQ(nullptr, GetType("type.other.com/Shelf")); + EXPECT_EQ(nullptr, GetEnum("type.other.com/ShelfType")); +} + +class ServiceConfigBasedTypeHelperTest : public ::testing::Test { + protected: + ServiceConfigBasedTypeHelperTest() {} + + bool LoadService(const std::string& config_pb_txt) { + if (!transcoding::testing::LoadService(config_pb_txt, &service_)) { + return false; + } + helper_.reset(new TypeHelper(service_.types(), service_.enums())); + return true; + } + + const pb::Type* GetType(const std::string& url) { + return helper_->Info()->GetTypeByTypeUrl(url); + } + + const pb::Enum* GetEnum(const std::string& url) { + return helper_->Info()->GetEnumByTypeUrl(url); + } + + const pb::Field* GetField(const std::string& type_url, + const std::string& field_name) { + auto t = GetType(type_url); + if (nullptr == t) { + return nullptr; + } + return helper_->Info()->FindField(t, field_name); + } + + bool ResolveFieldPath(const std::string& type_name, + const std::string& field_path_str, + std::vector* field_path) { + auto type = GetType("type.googleapis.com/" + type_name); + if (nullptr == type) { + ADD_FAILURE() << "Could not find top level type \"" + type_name + "\"" + << std::endl; + return false; + } + + auto status = helper_->ResolveFieldPath(*type, field_path_str, field_path); + if (!status.ok()) { + ADD_FAILURE() << "Error " << status.error_code() << " - " + << status.error_message() << std::endl; + return false; + } + + return true; + } + + private: + ::google::api::Service service_; + std::unique_ptr helper_; +}; + +TEST_F(ServiceConfigBasedTypeHelperTest, FullTypeTests) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + + auto t = GetType("type.googleapis.com/CreateShelfRequest"); + + ASSERT_NE(nullptr, t); + EXPECT_EQ("CreateShelfRequest", t->name()); + EXPECT_EQ(1, t->fields_size()); + EXPECT_EQ(pb::Field::TYPE_MESSAGE, t->fields(0).kind()); + EXPECT_EQ(pb::Field::CARDINALITY_OPTIONAL, t->fields(0).cardinality()); + EXPECT_EQ(1, t->fields(0).number()); + EXPECT_EQ("shelf", t->fields(0).name()); + EXPECT_EQ("type.googleapis.com/Shelf", t->fields(0).type_url()); + + t = GetType("type.googleapis.com/Shelf"); + + ASSERT_NE(nullptr, t); + EXPECT_EQ("Shelf", t->name()); + EXPECT_EQ(2, t->fields_size()); + EXPECT_EQ(pb::Field::TYPE_STRING, t->fields(0).kind()); + EXPECT_EQ(pb::Field::CARDINALITY_OPTIONAL, t->fields(0).cardinality()); + EXPECT_EQ(1, t->fields(0).number()); + EXPECT_EQ("name", t->fields(0).name()); + EXPECT_EQ(pb::Field::TYPE_STRING, t->fields(1).kind()); + EXPECT_EQ(pb::Field::CARDINALITY_OPTIONAL, t->fields(1).cardinality()); + EXPECT_EQ(2, t->fields(1).number()); + EXPECT_EQ("theme", t->fields(1).name()); +} + +TEST_F(ServiceConfigBasedTypeHelperTest, AllTypesTests) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + + ASSERT_NE(nullptr, GetType("type.googleapis.com/CreateShelfRequest")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/ListShelvesResponse")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/CreateBookRequest")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/ListBooksResponse")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/GetBookRequest")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/DeleteBookRequest")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/Shelf")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/Book")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/AuthorInfo")); + ASSERT_NE(nullptr, GetType("type.googleapis.com/Biography")); + + EXPECT_EQ("CreateShelfRequest", + GetType("type.googleapis.com/CreateShelfRequest")->name()); + EXPECT_EQ("ListShelvesResponse", + GetType("type.googleapis.com/ListShelvesResponse")->name()); + EXPECT_EQ("CreateBookRequest", + GetType("type.googleapis.com/CreateBookRequest")->name()); + EXPECT_EQ("ListBooksResponse", + GetType("type.googleapis.com/ListBooksResponse")->name()); + EXPECT_EQ("GetBookRequest", + GetType("type.googleapis.com/GetBookRequest")->name()); + EXPECT_EQ("DeleteBookRequest", + GetType("type.googleapis.com/DeleteBookRequest")->name()); + EXPECT_EQ("Shelf", GetType("type.googleapis.com/Shelf")->name()); + EXPECT_EQ("Book", GetType("type.googleapis.com/Book")->name()); + EXPECT_EQ("AuthorInfo", GetType("type.googleapis.com/AuthorInfo")->name()); + EXPECT_EQ("Biography", GetType("type.googleapis.com/Biography")->name()); + + EXPECT_EQ(nullptr, GetType("type.googleapis.com/DoesNotExist")); + EXPECT_EQ(nullptr, GetType("type.other.com/Shelf")); +} + +TEST_F(ServiceConfigBasedTypeHelperTest, FindFieldTests) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + + auto f = GetField("type.googleapis.com/CreateShelfRequest", "shelf"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("shelf", f->name()); + EXPECT_EQ(1, f->number()); + + f = GetField("type.googleapis.com/Shelf", "theme"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("theme", f->name()); + EXPECT_EQ(2, f->number()); + + f = GetField("type.googleapis.com/GetBookRequest", "shelf"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("shelf", f->name()); + EXPECT_EQ(1, f->number()); + + f = GetField("type.googleapis.com/GetBookRequest", "book"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("book", f->name()); + EXPECT_EQ(2, f->number()); +} + +TEST_F(ServiceConfigBasedTypeHelperTest, FindFieldCamelCaseTests) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + + auto f = GetField("type.googleapis.com/AuthorInfo", "firstName"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("first_name", f->name()); + EXPECT_EQ(1, f->number()); + + f = GetField("type.googleapis.com/AuthorInfo", "first_name"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("first_name", f->name()); + EXPECT_EQ(1, f->number()); + + f = GetField("type.googleapis.com/AuthorInfo", "lastName"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("last_name", f->name()); + EXPECT_EQ(2, f->number()); + + f = GetField("type.googleapis.com/AuthorInfo", "last_name"); + ASSERT_NE(nullptr, f); + EXPECT_EQ("last_name", f->name()); + EXPECT_EQ(2, f->number()); +} + +TEST_F(ServiceConfigBasedTypeHelperTest, ResolveFieldPathTests) { + ASSERT_TRUE(LoadService("bookstore_service.pb.txt")); + + std::vector field_path; + + // empty + EXPECT_TRUE(ResolveFieldPath("Shelf", "", &field_path)); + EXPECT_TRUE(field_path.empty()); + + // 1 level deep + EXPECT_TRUE(ResolveFieldPath("CreateShelfRequest", "shelf", &field_path)); + ASSERT_EQ(1, field_path.size()); + EXPECT_EQ("shelf", field_path[0]->name()); + + // 2 levels deep + EXPECT_TRUE( + ResolveFieldPath("CreateShelfRequest", "shelf.theme", &field_path)); + ASSERT_EQ(2, field_path.size()); + EXPECT_EQ("shelf", field_path[0]->name()); + EXPECT_EQ("theme", field_path[1]->name()); + + // 2 levels deep camel-case + EXPECT_TRUE( + ResolveFieldPath("CreateBookRequest", "book.authorInfo", &field_path)); + ASSERT_EQ(2, field_path.size()); + EXPECT_EQ("book", field_path[0]->name()); + EXPECT_EQ("author_info", field_path[1]->name()); + + // 2 levels deep non-camel-case + EXPECT_TRUE( + ResolveFieldPath("CreateBookRequest", "book.author_info", &field_path)); + ASSERT_EQ(2, field_path.size()); + EXPECT_EQ("book", field_path[0]->name()); + EXPECT_EQ("author_info", field_path[1]->name()); + + // 4 levels deep camel case + EXPECT_TRUE(ResolveFieldPath("CreateBookRequest", + "book.authorInfo.bio.yearBorn", &field_path)); + ASSERT_EQ(4, field_path.size()); + EXPECT_EQ("book", field_path[0]->name()); + EXPECT_EQ("author_info", field_path[1]->name()); + EXPECT_EQ("bio", field_path[2]->name()); + EXPECT_EQ("year_born", field_path[3]->name()); +} + +} // namespace + +} // namespace testing +} // namespace transcoding + +} // namespace api_manager +} // namespace google