diff --git a/contrib/endpoints/BUILD b/contrib/endpoints/BUILD new file mode 100644 index 00000000000..ac06d245289 --- /dev/null +++ b/contrib/endpoints/BUILD @@ -0,0 +1,26 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +exports_files(["LICENSE"]) + +config_setting( + name = "darwin", + values = { + "cpu": "darwin", + }, + visibility = ["//visibility:public"], +) diff --git a/contrib/endpoints/WORKSPACE b/contrib/endpoints/WORKSPACE new file mode 100644 index 00000000000..ed7b1b60a7b --- /dev/null +++ b/contrib/endpoints/WORKSPACE @@ -0,0 +1,186 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +# required by servicecontrol_client +git_repository( + name = "boringssl", + commit = "12c35d69008ae6b8486e435447445240509f7662", # 2016-10-24 + remote = "https://boringssl.googlesource.com/boringssl", +) + +bind( + name = "boringssl_crypto", + actual = "@boringssl//:crypto", +) + +# Required by gRPC. +bind( + name = "libssl", + actual = "@boringssl//:ssl", +) + +new_git_repository( + name = "zlib_git", + build_file = "third_party/BUILD.zlib", + commit = "50893291621658f355bc5b4d450a8d06a563053d", # v1.2.8 + remote = "https://github.com/madler/zlib.git", +) + +bind( + name = "zlib", + actual = "@zlib_git//:zlib" +) + +new_git_repository( + name = "nanopb_git", + build_file = "third_party/BUILD.nanopb", + commit = "f8ac463766281625ad710900479130c7fcb4d63b", + remote = "https://github.com/nanopb/nanopb.git", +) + +bind( + name = "nanopb", + actual = "@nanopb_git//:nanopb", +) + +git_repository( + name = "grpc_git", + commit = "d28417c856366df704200f544e72d31056931bce", + remote = "https://github.com/grpc/grpc.git", + init_submodules = True, +) + +bind( + name = "gpr", + actual = "@grpc_git//:gpr", +) + +bind( + name = "grpc", + actual = "@grpc_git//:grpc", +) + +bind( + name = "grpc_cpp_plugin", + actual = "@grpc_git//:grpc_cpp_plugin", +) + +bind( + name = "grpc++", + actual = "@grpc_git//:grpc++", +) + +bind( + name = "grpc_lib", + actual = "@grpc_git//:grpc++_reflection", +) + +git_repository( + name = "protobuf_git", + commit = "a428e42072765993ff674fda72863c9f1aa2d268", # v3.1.0 + remote = "https://github.com/google/protobuf.git", +) + +bind( + name = "protoc", + actual = "@protobuf_git//:protoc", +) + +bind( + name = "protobuf", + actual = "@protobuf_git//:protobuf", +) + +bind( + name = "cc_wkt_protos", + actual = "@protobuf_git//:cc_wkt_protos", +) + +bind( + name = "cc_wkt_protos_genproto", + actual = "@protobuf_git//:cc_wkt_protos_genproto", +) + +bind( + name = "protobuf_compiler", + actual = "@protobuf_git//:protoc_lib", +) + +bind( + name = "protobuf_clib", + actual = "@protobuf_git//:protobuf_lite", +) + +new_git_repository( + name = "googletest_git", + build_file = "third_party/BUILD.googletest", + commit = "d225acc90bc3a8c420a9bcd1f033033c1ccd7fe0", + remote = "https://github.com/google/googletest.git", +) + +bind( + name = "googletest", + actual = "@googletest_git//:googletest", +) + +bind( + name = "googletest_main", + actual = "@googletest_git//:googletest_main", +) + +bind( + name = "googletest_prod", + actual = "@googletest_git//:googletest_prod", +) + +new_git_repository( + name = "googleapis_git", + commit = "6c1d6d4067364a21f8ffefa3401b213d652bf121", + remote = "https://github.com/googleapis/googleapis.git", + build_file = "third_party/BUILD.googleapis", +) + +bind( + name = "servicecontrol", + actual = "@googleapis_git//:servicecontrol", +) + +bind( + name = "servicecontrol_genproto", + actual = "@googleapis_git//:servicecontrol_genproto", +) + +bind( + name = "service_config", + actual = "@googleapis_git//:service_config", +) + +bind( + name = "cloud_trace", + actual = "@googleapis_git//:cloud_trace", +) + +git_repository( + name = "servicecontrol_client_git", + commit = "d739d755365c6a13d0b4164506fd593f53932f5d", + remote = "https://github.com/cloudendpoints/service-control-client-cxx.git", +) + +bind( + name = "servicecontrol_client", + actual = "@servicecontrol_client_git//:service_control_client_lib", +) diff --git a/contrib/endpoints/include/BUILD b/contrib/endpoints/include/BUILD new file mode 100644 index 00000000000..6275f8aa11a --- /dev/null +++ b/contrib/endpoints/include/BUILD @@ -0,0 +1,72 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +exports_files(["version"]) + +cc_inc_library( + name = "api_manager", + hdrs = [ + ":headers", + ], + visibility = ["//visibility:public"], + deps = [ + "//src/api_manager", + ], +) + +cc_library( + name = "headers_only", + hdrs = [ + ":headers", + ], + visibility = [ + "//src/api_manager:__subpackages__", + ], +) + +filegroup( + name = "headers", + srcs = [ + "api_manager/api_manager.h", + "api_manager/compute_platform.h", + "api_manager/env_interface.h", + "api_manager/http_request.h", + "api_manager/method.h", + "api_manager/method_call_info.h", + "api_manager/periodic_timer.h", + "api_manager/protocol.h", + "api_manager/request.h", + "api_manager/request_handler_interface.h", + "api_manager/response.h", + "api_manager/service_control.h", + "api_manager/utils/status.h", + "api_manager/version.h", + ], +) + +genrule( + name = "api_manager_version_header", + srcs = [ + ":version", + ], + outs = [ + "api_manager/version.h", + ], + cmd = "$(location //tools:create_version) $(location :version) > $@", + tools = [ + "//tools:create_version", + ], +) diff --git a/contrib/endpoints/include/api_manager/api_manager.h b/contrib/endpoints/include/api_manager/api_manager.h new file mode 100644 index 00000000000..e535cc275c4 --- /dev/null +++ b/contrib/endpoints/include/api_manager/api_manager.h @@ -0,0 +1,127 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_API_MANAGER_H_ +#define API_MANAGER_API_MANAGER_H_ + +// An API Manager interface. + +#include +#include + +#include "google/api/service.pb.h" +#include "include/api_manager/env_interface.h" +#include "include/api_manager/request.h" +#include "include/api_manager/request_handler_interface.h" +#include "include/api_manager/service_control.h" + +namespace google { +namespace api_manager { + +// Data to summarize the API Manager statistics. +// Important note: please don't use std::string. These fields are directly +// copied into a shared memory. +struct ApiManagerStatistics { + service_control::Statistics service_control_statistics; +}; + +class ApiManager { + public: + virtual ~ApiManager() {} + + // Returns true if either auth is required or service control is configured. + virtual bool Enabled() const = 0; + + // Gets the service name. + virtual const std::string &service_name() const = 0; + + // Gets the service config + virtual const ::google::api::Service &service() const = 0; + + // Set the metadata server for GCP platforms. + virtual void SetMetadataServer(const std::string &server) = 0; + + // Sets the client auth secret and uses it to generate auth token. + // If it fails to generate an auth token, return failure. + virtual utils::Status SetClientAuthSecret(const std::string &secret) = 0; + + // Initializes the API Manager. It should be called: + // 1) Before first CreateRequestHandler(). + // 2) After certain provided environment is ready. Specifically for Nginx, + // It should be called inside InitProcess() of each worker process. + virtual utils::Status Init() = 0; + + // Closes the API Manager. After this, CreateRequestHandler() should not be + // called. + virtual utils::Status Close() = 0; + + // server_config has an option to disable logging to nginx error.log. + // This function checks server_config for that flag. + virtual bool get_logging_status_disabled() = 0; + + // Creates a RequestHandler to handle check and report for each request. + // Its usage: + // 1) Creates a RequestHandler object for each request, + // + // request_handler = api_manager->CreateRequestHandler(request); + // + // 2) Before forwarding the request to backend, calls Check(). + // + // request_handler->Check([](utils::Status status) { + // check status; + // }); + // + // 3) After the request is finished, calls Report(). + // + // request_handler->Report(response, [](){}); + // + virtual std::unique_ptr CreateRequestHandler( + std::unique_ptr request) = 0; + + // To get the api manager statistics. + virtual utils::Status GetStatistics( + ApiManagerStatistics *statistics) const = 0; + + protected: + ApiManager() {} + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ApiManager); +}; + +class ApiManagerFactory { + public: + ApiManagerFactory() {} + ~ApiManagerFactory() {} + + // Gets or creates an ApiManager instance. Service configurations with the + // same service names will resolve to the same live ApiManager instance. + // The environment is used iff the instance needs to be created; + // otherwise, it's deleted. This means that the returned ApiManager may + // use a different environment than the one provided. + std::shared_ptr GetOrCreateApiManager( + std::unique_ptr env, + const std::string &service_config, const std::string &server_config); + + private: + typedef std::map> ApiManagerMap; + ApiManagerMap api_manager_map_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ApiManagerFactory); +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_API_MANAGER_H_ diff --git a/contrib/endpoints/include/api_manager/compute_platform.h b/contrib/endpoints/include/api_manager/compute_platform.h new file mode 100644 index 00000000000..79144a81f22 --- /dev/null +++ b/contrib/endpoints/include/api_manager/compute_platform.h @@ -0,0 +1,44 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_COMPUTE_PLATFORM_H_ +#define API_MANAGER_COMPUTE_PLATFORM_H_ + +namespace google { +namespace api_manager { + +namespace compute_platform { + +enum ComputePlatform { UNKNOWN = 0, GAE_FLEX = 1, GCE = 2, GKE = 3 }; + +inline const char *ToString(ComputePlatform p) { + switch (p) { + case GAE_FLEX: + return "GAE Flex"; + case GCE: + return "GCE"; + case GKE: + return "GKE"; + case UNKNOWN: + default: + return "unknown"; + } +} + +} // namespace compute_platform + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_COMPUTE_PLATFORM_H_ diff --git a/contrib/endpoints/include/api_manager/env_interface.h b/contrib/endpoints/include/api_manager/env_interface.h new file mode 100644 index 00000000000..8b6f048d855 --- /dev/null +++ b/contrib/endpoints/include/api_manager/env_interface.h @@ -0,0 +1,66 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_ENV_INTERFACE_H_ +#define API_MANAGER_ENV_INTERFACE_H_ + +// An interface for the API Manager to access its environment. + +#include +#include +#include + +#include "include/api_manager/http_request.h" +#include "include/api_manager/periodic_timer.h" +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { + +class ApiManagerEnvInterface { + public: + virtual ~ApiManagerEnvInterface() {} + + enum LogLevel { DEBUG, INFO, WARNING, ERROR }; + + void LogDebug(const std::string &str) { LogDebug(str.c_str()); } + void LogInfo(const std::string &str) { LogInfo(str.c_str()); } + void LogWarning(const std::string &str) { LogWarning(str.c_str()); } + void LogError(const std::string &str) { LogError(str.c_str()); } + + void LogDebug(const char *message) { Log(DEBUG, message); } + void LogInfo(const char *message) { Log(INFO, message); } + void LogWarning(const char *message) { Log(WARNING, message); } + void LogError(const char *message) { Log(ERROR, message); } + + virtual void Log(LogLevel level, const char *message) = 0; + + // Simple periodic timer support. API Manager uses this method to get + // called at regular intervals of wall-clock time. + virtual std::unique_ptr StartPeriodicTimer( + std::chrono::milliseconds interval, + std::function continuation) = 0; + + // Environment support for issuing non-streaming HTTP 1.1 requests from + // the API manager. The environment takes ownership of the request, and is + // responsible for eventually invoking request->OnComplete() with the result + // of the request + // (possibly before returning). + virtual void RunHTTPRequest(std::unique_ptr request) = 0; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_ENV_INTERFACE_H_ diff --git a/contrib/endpoints/include/api_manager/http_request.h b/contrib/endpoints/include/api_manager/http_request.h new file mode 100644 index 00000000000..d33718501ca --- /dev/null +++ b/contrib/endpoints/include/api_manager/http_request.h @@ -0,0 +1,152 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_HTTP_REQUEST_H_ +#define API_MANAGER_HTTP_REQUEST_H_ + +#include +#include + +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { + +// Represents a non-streaming HTTP request issued by the API Manager and +// processed by the environment. +class HTTPRequest { + public: + // HTTPRequest constructor without headers in the callback function. + // Callback receives NGX_ERROR status code if the request fails to initiate or + // complete. + // Callback receives HTTP response code in status, if the request succeeds. + // The keys in headers passed to the callback are in lowercase. + HTTPRequest( + std::function&&, + std::string&&)> + callback) + : callback_(callback), + requires_response_headers_(false), + timeout_ms_(0), + max_retries_(0), + timeout_backoff_factor_(2.0) {} + + // A callback for the environment to invoke when the request is + // complete. This will be invoked by the environment exactly once, + // and then the environment will drop the std::unique_ptr + // referencing the HTTPRequest. + // Headers are supplied only if the request requires them; otherwise, it's an + // empty map. + void OnComplete(utils::Status status, + std::map&& headers, + std::string&& body) { + callback_(status, std::move(headers), std::move(body)); + } + + // Request properties. + const std::string& method() const { return method_; } + HTTPRequest& set_method(const std::string& value) { + method_ = value; + return *this; + } + HTTPRequest& set_method(std::string&& value) { + method_ = std::move(value); + return *this; + } + + const std::string& url() const { return url_; } + HTTPRequest& set_url(const std::string& value) { + url_ = value; + return *this; + } + HTTPRequest& set_url(std::string&& value) { + url_ = std::move(value); + return *this; + } + + const std::string& body() const { return body_; } + HTTPRequest& set_body(const std::string& value) { + body_ = value; + return *this; + } + HTTPRequest& set_body(std::string&& value) { + body_ = std::move(value); + return *this; + } + + HTTPRequest& set_auth_token(const std::string& value) { + if (!value.empty()) { + set_header("Authorization", "Bearer " + value); + } + return *this; + } + + int timeout_ms() const { return timeout_ms_; } + HTTPRequest& set_timeout_ms(int value) { + timeout_ms_ = value; + return *this; + } + + int max_retries() const { return max_retries_; } + HTTPRequest& set_max_retries(int value) { + max_retries_ = value; + return *this; + } + + double timeout_backoff_factor() const { return timeout_backoff_factor_; } + HTTPRequest& set_timeout_backoff_factor(double value) { + timeout_backoff_factor_ = value; + return *this; + } + + const std::map& request_headers() const { + return headers_; + } + HTTPRequest& set_header(const std::string& name, const std::string& value) { + headers_[name] = value; + return *this; + } + + bool requires_response_headers() const { return requires_response_headers_; } + HTTPRequest& set_requires_response_headers(bool value) { + requires_response_headers_ = value; + return *this; + } + + private: + std::function&&, + std::string&&)> + callback_; + std::string method_; + std::string url_; + std::string body_; + std::map headers_; + + // Indicates whether to extract headers from the response + bool requires_response_headers_; + + // Timeout, in milliseconds, for the request. + int timeout_ms_; + + // Maximum number of retries, 0 to skip + int max_retries_; + + // Exponential back-off for the retries + double timeout_backoff_factor_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_HTTP_REQUEST_H_ diff --git a/contrib/endpoints/include/api_manager/method.h b/contrib/endpoints/include/api_manager/method.h new file mode 100644 index 00000000000..6b7c01072b8 --- /dev/null +++ b/contrib/endpoints/include/api_manager/method.h @@ -0,0 +1,97 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_METHOD_H_ +#define API_MANAGER_METHOD_H_ + +#include +#include +#include + +namespace google { +namespace api_manager { + +// A minimal set of data attached to a method. +class MethodInfo { + public: + virtual ~MethodInfo() {} + + // Return the method name + virtual const std::string &name() const = 0; + + // Return the API name + virtual const std::string &api_name() const = 0; + + // Return the API version + virtual const std::string &api_version() const = 0; + + // Return the method selector + virtual const std::string &selector() const = 0; + + // Return if auth is enabled for this method. + virtual bool auth() const = 0; + + // Return if this method allows unregistered calls. + virtual bool allow_unregistered_calls() const = 0; + + // Check an issuer is allowed. + virtual bool isIssuerAllowed(const std::string &issuer) const = 0; + + // Check if an audience is allowed for an issuer. + virtual bool isAudienceAllowed( + const std::string &issuer, + const std::set &jwt_audiences) const = 0; + + // Get http header system parameters by name. + virtual const std::vector *http_header_parameters( + const std::string &name) const = 0; + + // Get url query system parameters by name. + virtual const std::vector *url_query_parameters( + const std::string &name) const = 0; + + // Get http header system parameters for api_key. + virtual const std::vector *api_key_http_headers() const = 0; + + // Get url query system parameters for api_key. + virtual const std::vector *api_key_url_query_parameters() + const = 0; + + // Get the backend address for this method. + virtual const std::string &backend_address() const = 0; + + // Get the RPC method full name. The full name has the following form: + // "//". + virtual const std::string &rpc_method_full_name() const = 0; + + // Get the request_type_url + virtual const std::string &request_type_url() const = 0; + + // Get whether request is streaming + virtual bool request_streaming() const = 0; + + // Get the response_type_url + virtual const std::string &response_type_url() const = 0; + + // Get whether response is streaming + virtual bool response_streaming() const = 0; + + // Get the names of url system parameters + virtual const std::set &system_query_parameter_names() const = 0; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_METHOD_H_ diff --git a/contrib/endpoints/include/api_manager/method_call_info.h b/contrib/endpoints/include/api_manager/method_call_info.h new file mode 100644 index 00000000000..48f965ab8bd --- /dev/null +++ b/contrib/endpoints/include/api_manager/method_call_info.h @@ -0,0 +1,53 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_METHOD_CALL_INFO_H_ +#define API_MANAGER_METHOD_CALL_INFO_H_ + +#include +#include + +#include "include/api_manager/method.h" + +namespace google { +namespace api_manager { + +// VariableBinding specifies a value for a single field in the request message. +// When transcoding HTTP/REST/JSON to gRPC/proto the request message is +// constructed using the HTTP body and the variable bindings (specified through +// request url). +struct VariableBinding { + // The location of the field in the protobuf message, where the value + // needs to be inserted, e.g. "shelf.theme" would mean the "theme" field + // of the nested "shelf" message of the request protobuf message. + std::vector field_path; + // The value to be inserted. + std::string value; +}; + +// Information that we store per each call of a method. It includes information +// about the method itself and variable bindings for the particular call. +struct MethodCallInfo { + // Method information + const MethodInfo* method_info; + // Variable bindings + std::vector variable_bindings; + // Body prefix (the field of the message where the HTTP body should go) + std::string body_field_path; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_METHOD_CALL_INFO_H_ diff --git a/contrib/endpoints/include/api_manager/periodic_timer.h b/contrib/endpoints/include/api_manager/periodic_timer.h new file mode 100644 index 00000000000..547cba03c96 --- /dev/null +++ b/contrib/endpoints/include/api_manager/periodic_timer.h @@ -0,0 +1,42 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_PERIODIC_TIMER_H_ +#define API_MANAGER_PERIODIC_TIMER_H_ + +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { + +// Represents a periodic timer created by API Manager's environment. +class PeriodicTimer { + public: + // Deletes the timer, stopping it first if needed. This method + // synchronizes with any outstanding invocation of the timer's + // callback function; this method must not be invoked from within + // the callback (doing so will cause undefined behavior). + virtual ~PeriodicTimer() {} + + // Stops the timer, preventing new invocations of the timer's + // callback function from being started. This method does not + // synchronize with outstanding invocations of the timer's callback + // function; it may be invoked from within the callback. + virtual void Stop() = 0; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_PERIODIC_TIMER_H_ diff --git a/contrib/endpoints/include/api_manager/protocol.h b/contrib/endpoints/include/api_manager/protocol.h new file mode 100644 index 00000000000..901934803d7 --- /dev/null +++ b/contrib/endpoints/include/api_manager/protocol.h @@ -0,0 +1,44 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_PROTOCOL_H_ +#define API_MANAGER_PROTOCOL_H_ + +namespace google { +namespace api_manager { + +namespace protocol { + +enum Protocol { UNKNOWN = 0, HTTP = 1, HTTPS = 2, GRPC = 3 }; + +inline const char *ToString(Protocol p) { + switch (p) { + case HTTP: + return "http"; + case HTTPS: + return "https"; + case GRPC: + return "grpc"; + case UNKNOWN: + default: + return "unknown"; + } +} + +} // namespace protocol + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_PROTOCOL_H_ diff --git a/contrib/endpoints/include/api_manager/request.h b/contrib/endpoints/include/api_manager/request.h new file mode 100644 index 00000000000..565b73d11f1 --- /dev/null +++ b/contrib/endpoints/include/api_manager/request.h @@ -0,0 +1,76 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_REQUEST_H_ +#define API_MANAGER_REQUEST_H_ + +#include +#include + +#include "include/api_manager/protocol.h" +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { + +// Request provides an interface for CallHandler::Check to use to +// query information about a request. +class Request { + public: + virtual ~Request() {} + + // Returns the HTTP method used for this call. + virtual std::string GetRequestHTTPMethod() = 0; + + // Returns the REST path or RPC path for this call. + virtual std::string GetRequestPath() = 0; + // Returns the query parameters + virtual std::string GetQueryParameters() = 0; + // Returns the request path before parsed. + virtual std::string GetUnparsedRequestPath() = 0; + + // Gets Client IP + // This will be used by service control Check() call. + virtual std::string GetClientIP() = 0; + + // Get GRPC stats. + virtual int64_t GetGrpcRequestBytes() = 0; + virtual int64_t GetGrpcResponseBytes() = 0; + virtual int64_t GetGrpcRequestMessageCounts() = 0; + virtual int64_t GetGrpcResponseMessageCounts() = 0; + + // Finds a HTTP query parameter with a name. Returns true if found. + virtual bool FindQuery(const std::string &name, std::string *query) = 0; + + // Finds a HTTP header with a name. Returns true if found. + // Don't support multiple headers with same name for now. In that case, + // the first header will be returned. + virtual bool FindHeader(const std::string &name, std::string *header) = 0; + + // Returns the protocol used for this call. + virtual ::google::api_manager::protocol::Protocol GetRequestProtocol() = 0; + + // Sets auth token to the request object. Caller of RequestHandler::Check + // need to use it compose error message if authentication fails. + virtual void SetAuthToken(const std::string &auth_token) = 0; + + // Adds a header to backend. If the header exists, overwrite its value + virtual utils::Status AddHeaderToBackend(const std::string &key, + const std::string &value) = 0; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_REQUEST_H_ diff --git a/contrib/endpoints/include/api_manager/request_handler_interface.h b/contrib/endpoints/include/api_manager/request_handler_interface.h new file mode 100644 index 00000000000..cae86561887 --- /dev/null +++ b/contrib/endpoints/include/api_manager/request_handler_interface.h @@ -0,0 +1,64 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_REQUEST_HANDLER_INTERFACE_H_ +#define API_MANAGER_REQUEST_HANDLER_INTERFACE_H_ + +#include "include/api_manager/method.h" +#include "include/api_manager/method_call_info.h" +#include "include/api_manager/response.h" +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { + +// Defines an interface for API Manager to handle a request. +// Callers call API Manager to create such interface, and call +// its Check() function to process all checking, and call +// its Report() function to send a report. +class RequestHandlerInterface { + public: + virtual ~RequestHandlerInterface(){}; + + // Makes all necessary checks for a request, + // All request data can be accessed from request object. + // continuation will be called once it is done or failed. + virtual void Check( + std::function continuation) = 0; + + // Sends a report. + virtual void Report(std::unique_ptr response, + std::function continuation) = 0; + + // Attempt to send intermediate report in streaming calls. + virtual void AttemptIntermediateReport() = 0; + + // Get the backend address. + virtual std::string GetBackendAddress() const = 0; + + // Get the full name of the RPC method in the "//" + // form. + virtual std::string GetRpcMethodFullName() const = 0; + + // Get the method info. + virtual const MethodInfo *method() const = 0; + + // Get the method info. + virtual const MethodCallInfo *method_call() const = 0; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_REQUEST_HANDLER_INTERFACE_H_ diff --git a/contrib/endpoints/include/api_manager/response.h b/contrib/endpoints/include/api_manager/response.h new file mode 100644 index 00000000000..1845f9a0041 --- /dev/null +++ b/contrib/endpoints/include/api_manager/response.h @@ -0,0 +1,51 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_RESPONSE_H_ +#define API_MANAGER_RESPONSE_H_ + +#include +#include + +#include "include/api_manager/protocol.h" +#include "include/api_manager/service_control.h" +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { + +// Response provides an interface for CallHandler::StartReport and +// CallHandler::CompleteReport to use to query information about a +// response. +class Response { + public: + virtual ~Response() {} + + // Returns the status associated with the response. + virtual utils::Status GetResponseStatus() = 0; + + // Returns the size of the initial request, in bytes. + virtual std::size_t GetRequestSize() = 0; + + // Returns the size of the response, in bytes. + virtual std::size_t GetResponseSize() = 0; + + // Gets the latency info. + virtual utils::Status GetLatencyInfo(service_control::LatencyInfo *info) = 0; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_RESPONSE_H_ diff --git a/contrib/endpoints/include/api_manager/service_control.h b/contrib/endpoints/include/api_manager/service_control.h new file mode 100644 index 00000000000..8a28f37f5c2 --- /dev/null +++ b/contrib/endpoints/include/api_manager/service_control.h @@ -0,0 +1,67 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_SERVICE_CONTROL_H_ +#define API_MANAGER_SERVICE_CONTROL_H_ + +namespace google { +namespace api_manager { +namespace service_control { + +// The statistics recorded by service control library. +// Important note: please don't use std::string. These fields are directly +// copied into a shared memory. +struct Statistics { + // Total number of Check() calls received. + uint64_t total_called_checks; + // Check sends to server from flushed cache items. + uint64_t send_checks_by_flush; + // Check sends to remote sever during Check() calls. + uint64_t send_checks_in_flight; + + // Total number of Report() calls received. + uint64_t total_called_reports; + // Report sends to server from flushed cache items. + uint64_t send_reports_by_flush; + // Report sends to remote sever during Report() calls. + uint64_t send_reports_in_flight; + + // The number of operations send, each input report has only 1 operation, but + // each report send to server may have multiple operations. The ratio of + // send_report_operations / total_called_reports will reflect report + // aggregation rate. send_report_operations may not reflect aggregation rate. + uint64_t send_report_operations; + + // Maximum report request size send to server. + uint64_t max_report_size; +}; + +// Per request latency statistics. +struct LatencyInfo { + // The request time in milliseconds. -1 if not available. + int64_t request_time_ms; + // The backend request time in milliseconds. -1 if not available. + int64_t backend_time_ms; + // The API Manager overhead time in milliseconds. -1 if not available. + int64_t overhead_time_ms; + + LatencyInfo() + : request_time_ms(-1), backend_time_ms(-1), overhead_time_ms(-1) {} +}; + +} // namespace service_control +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_SERVICE_CONTROL_H_ diff --git a/contrib/endpoints/include/api_manager/utils/status.h b/contrib/endpoints/include/api_manager/utils/status.h new file mode 100644 index 00000000000..a4c256f620e --- /dev/null +++ b/contrib/endpoints/include/api_manager/utils/status.h @@ -0,0 +1,124 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_UTILS_STATUS_H_ +#define API_MANAGER_UTILS_STATUS_H_ + +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/stubs/status.h" +#include "google/rpc/error_details.pb.h" +#include "google/rpc/status.pb.h" + +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace utils { + +// A Status object can be used to represent an error or an OK state. Error +// status messages have an error code, an error message, and an error cause. +// An OK status has a code of 0 or 200 and no message. +class Status final { + public: + enum ErrorCause { + // Internal proxy error (default) + INTERNAL = 0, + // External application error + APPLICATION = 1, + // Error in authentication + AUTH = 2, + // Error in service control check + SERVICE_CONTROL = 3 + }; + + // Constructs a status with an error code and message. If code == 0 + // message is ignored and a Status object identical to Status::OK + // is constructed. Error cause is optional and defaults to INTERNAL. + Status(int code, const std::string& message); + Status(int code, const std::string& message, ErrorCause error_cause); + ~Status() {} + + bool operator==(const Status& x) const; + bool operator!=(const Status& x) const { return !operator==(x); } + + // Get string representation of the error code + static std::string CodeToString(int code); + + // Get string representation of the error cause + static std::string ErrorCauseToString(ErrorCause error_cause); + + // Constructs a Status object from a protobuf Status. + static Status FromProto(const ::google::protobuf::util::Status& proto_status); + + // Pre-defined OK status. + static const Status& OK; + + // Pre-defined DONE status. + static const Status& DONE; + + // Returns true if this status is not an error + bool ok() const { return code_ == Code::OK || code_ == 200; } + + // Returns the error code held by this status. + int code() const { return code_; } + + // Returns the error message held by this status. + const std::string& message() const { return message_; } + + // Returns the error cause held by this status. + ErrorCause error_cause() const { return error_cause_; } + + // Returns the error code mapped to HTTP status codes. + int HttpCode() const; + + // Returns the error code mapped to protobuf canonical code. + Code CanonicalCode() const; + + // Returns a combination of the error code name and message. + std::string ToString() const; + + // Returns a representation of the error as a protobuf Status. + ::google::protobuf::util::Status ToProto() const; + + // Returns a representation of the error as a canonical status + ::google::rpc::Status ToCanonicalProto() const; + + // Returns a JSON representation of the error as a canonical status + std::string ToJson() const; + + private: + // Constructs the OK status. + Status(); + + // Error code. Zero means OK. Negative numbers are for control + // statuses (e.g. DECLINED). Positive numbers below 100 represent grpc + // status codes. Positive numbers 100 and greater represent HTTP status codes. + int code_; + + // The error message if this Status represents an error, otherwise an empty + // string if this is the OK status. + std::string message_; + + // Error cause indicating the origin of the error. + ErrorCause error_cause_; +}; + +} // namespace utils +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_UTILS_STATUS_H_ diff --git a/contrib/endpoints/include/version b/contrib/endpoints/include/version new file mode 100644 index 00000000000..9084fa2f716 --- /dev/null +++ b/contrib/endpoints/include/version @@ -0,0 +1 @@ +1.1.0 diff --git a/contrib/endpoints/src/api_manager/.gitignore b/contrib/endpoints/src/api_manager/.gitignore new file mode 100644 index 00000000000..63b25506f8f --- /dev/null +++ b/contrib/endpoints/src/api_manager/.gitignore @@ -0,0 +1 @@ +libesp.srcs.stamp diff --git a/contrib/endpoints/src/api_manager/BUILD b/contrib/endpoints/src/api_manager/BUILD new file mode 100644 index 00000000000..75a0428383c --- /dev/null +++ b/contrib/endpoints/src/api_manager/BUILD @@ -0,0 +1,264 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +package(default_visibility = [":__subpackages__"]) + +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") + +cc_proto_library( + name = "server_config_proto", + srcs = [ + "proto/server_config.proto", + ], + default_runtime = "//external:protobuf", + protoc = "//external:protoc", +) + +cc_proto_library( + name = "status_proto", + srcs = [ + "proto/api_manager_status.proto", + ], + include = ".", + default_runtime = "//external:protobuf", + protoc = "//external:protoc", +) + +cc_library( + name = "auth_headers", + hdrs = [ + "auth.h", + ], +) + +cc_library( + name = "impl_headers", + hdrs = [ + "api_manager_impl.h", + "config.h", + "gce_metadata.h", + "http_template.h", + "method_impl.h", + "path_matcher.h", + "path_matcher_node.h", + "request_handler.h", + ], + deps = [ + "//include:headers_only", + ], +) + +cc_library( + name = "api_manager", + srcs = [ + "api_manager_impl.cc", + "check_auth.cc", + "check_auth.h", + "check_service_control.cc", + "check_service_control.h", + "check_workflow.cc", + "check_workflow.h", + "config.cc", + "fetch_metadata.cc", + "fetch_metadata.h", + "gce_metadata.cc", + "http_template.cc", + "http_template.h", + "method_impl.cc", + "path_matcher.cc", + "path_matcher_node.cc", + "request_handler.cc", + ], + linkopts = select({ + "//:darwin": [], + "//conditions:default": [ + "-lm", + "-luuid", + ], + }), + visibility = [ + "//include:__subpackages__", + ], + deps = [ + ":auth_headers", + ":impl_headers", + ":server_config_proto", + "//external:cc_wkt_protos", + "//external:cloud_trace", + "//external:googletest_prod", + "//external:grpc++", + "//external:protobuf", + "//external:service_config", + "//external:servicecontrol", + "//external:servicecontrol_client", + "//src/api_manager/auth", + "//src/api_manager/cloud_trace", + "//src/api_manager/context", + "//src/api_manager/service_control", + "//src/api_manager/utils", + ], +) + +cc_test( + name = "config_test", + size = "small", + srcs = [ + "config_test.cc", + ], + linkstatic = 1, + deps = [ + ":api_manager", + ":mock_api_manager_environment", + "//external:googletest_main", + ], +) + +cc_test( + name = "gce_metadata_test", + size = "small", + srcs = [ + "gce_metadata_test.cc", + ], + linkstatic = 1, + deps = [ + ":api_manager", + "//external:googletest_main", + ], +) + +cc_test( + name = "api_manager_test", + size = "small", + srcs = [ + "api_manager_test.cc", + ], + linkstatic = 1, + deps = [ + ":api_manager", + ":mock_api_manager_environment", + "//external:googletest_main", + ], +) + +cc_test( + name = "http_template_test", + size = "small", + srcs = [ + "http_template_test.cc", + ], + linkstatic = 1, + deps = [ + ":api_manager", + "//external:googletest_main", + ], +) + +cc_test( + name = "method_test", + size = "small", + srcs = [ + "method_test.cc", + ], + linkstatic = 1, + deps = [ + ":api_manager", + "//external:googletest_main", + ], +) + +cc_test( + name = "path_matcher_test", + size = "small", + srcs = [ + "mock_method_info.h", + "path_matcher_test.cc", + ], + linkstatic = 1, + deps = [ + ":api_manager", + "//external:googletest_main", + ], +) + +cc_test( + name = "common_protos_test", + size = "small", + srcs = [ + "common_protos_test.cc", + ], + deps = [ + "//external:googletest_main", + "//external:service_config", + "//external:servicecontrol", + ], +) + +cc_test( + name = "server_config_proto_test", + size = "small", + srcs = [ + "server_config_proto_test.cc", + ], + data = [ + "proto/sample_server_config.pb.txt", + ], + deps = [ + ":server_config_proto", + "//external:googletest_main", + ], +) + +cc_inc_library( + name = "mock_api_manager_environment", + testonly = True, + hdrs = [ + "mock_api_manager_environment.h", + ], + deps = [ + # TODO: remove this dependency + ":api_manager", + ], +) + +cc_test( + name = "check_auth_test", + size = "small", + srcs = [ + "check_auth_test.cc", + "mock_request.h", + ], + linkstatic = 1, + deps = [ + ":api_manager", + ":mock_api_manager_environment", + "//external:googletest_main", + ], +) + +cc_test( + name = "fetch_metadata_test", + size = "small", + srcs = [ + "fetch_metadata_test.cc", + "mock_request.h", + ], + linkstatic = 1, + deps = [ + ":api_manager", + ":mock_api_manager_environment", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/api_manager_impl.cc b/contrib/endpoints/src/api_manager/api_manager_impl.cc new file mode 100644 index 00000000000..40e108d8caa --- /dev/null +++ b/contrib/endpoints/src/api_manager/api_manager_impl.cc @@ -0,0 +1,77 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/api_manager_impl.h" + +#include "src/api_manager/check_workflow.h" +#include "src/api_manager/request_handler.h" + +using ::google::api_manager::proto::ServerConfig; + +namespace google { +namespace api_manager { + +namespace { + +std::shared_ptr CreateApiManager( + std::unique_ptr env, + std::unique_ptr config) { + return std::shared_ptr( + new ApiManagerImpl(std::move(env), std::move(config))); +} + +} // namespace + +ApiManagerImpl::ApiManagerImpl(std::unique_ptr env, + std::unique_ptr config) + : service_context_( + new context::ServiceContext(std::move(env), std::move(config))) { + check_workflow_ = std::unique_ptr(new CheckWorkflow); + check_workflow_->RegisterAll(); +} + +std::unique_ptr ApiManagerImpl::CreateRequestHandler( + std::unique_ptr request_data) { + return std::unique_ptr(new RequestHandler( + check_workflow_, service_context_, std::move(request_data))); +} + +std::shared_ptr ApiManagerFactory::GetOrCreateApiManager( + std::unique_ptr env, + const std::string& service_config, const std::string& server_config) { + std::unique_ptr config = + Config::Create(env.get(), service_config, server_config); + if (config == nullptr) { + return nullptr; + } + + ApiManagerMap::iterator it; + std::tie(it, std::ignore) = api_manager_map_.emplace( + config->service_name(), std::weak_ptr()); + std::shared_ptr result = it->second.lock(); + + if (!result) { + // TODO: Handle the case where the caller gives us a different + // config with the same service name. + result = CreateApiManager(std::move(env), std::move(config)); + it->second = result; + } + + return result; +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/api_manager_impl.h b/contrib/endpoints/src/api_manager/api_manager_impl.h new file mode 100644 index 00000000000..76480f82f19 --- /dev/null +++ b/contrib/endpoints/src/api_manager/api_manager_impl.h @@ -0,0 +1,105 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_API_MANAGER_IMPL_H_ +#define API_MANAGER_API_MANAGER_IMPL_H_ + +#include "include/api_manager/api_manager.h" +#include "src/api_manager/context/service_context.h" +#include "src/api_manager/service_control/interface.h" + +namespace google { +namespace api_manager { + +class CheckWorkflow; + +// Implements ApiManager interface. +class ApiManagerImpl : public ApiManager { + public: + ApiManagerImpl(std::unique_ptr env, + std::unique_ptr config); + + virtual bool Enabled() const { return service_context_->Enabled(); } + + virtual const std::string &service_name() const { + return service_context_->service_name(); + } + + virtual const ::google::api::Service &service() const { + return service_context_->service(); + } + + virtual void SetMetadataServer(const std::string &server) { + service_context_->SetMetadataServer(server); + } + + virtual utils::Status SetClientAuthSecret(const std::string &secret) { + return service_context_->service_account_token()->SetClientAuthSecret( + secret); + } + + virtual utils::Status Init() { + if (service_context_->cloud_trace_aggregator()) { + service_context_->cloud_trace_aggregator()->Init(); + } + if (service_control()) { + return service_control()->Init(); + } else { + return utils::Status::OK; + } + } + + virtual utils::Status Close() { + if (service_context_->cloud_trace_aggregator()) { + service_context_->cloud_trace_aggregator()->SendAndClearTraces(); + } + if (service_control()) { + return service_control()->Close(); + } else { + return utils::Status::OK; + } + } + + virtual std::unique_ptr CreateRequestHandler( + std::unique_ptr request); + + virtual utils::Status GetStatistics(ApiManagerStatistics *statistics) const { + if (service_control()) { + return service_control()->GetStatistics( + &statistics->service_control_statistics); + } else { + return utils::Status::OK; + } + } + + virtual bool get_logging_status_disabled() { + return service_context_->DisableLogStatus(); + }; + + private: + service_control::Interface *service_control() const { + return service_context_->service_control(); + } + + // The check work flow. + std::shared_ptr check_workflow_; + + // Service context + std::shared_ptr service_context_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_API_MANAGER_IMPL_H_ diff --git a/contrib/endpoints/src/api_manager/api_manager_test.cc b/contrib/endpoints/src/api_manager/api_manager_test.cc new file mode 100644 index 00000000000..3a7d84fc63a --- /dev/null +++ b/contrib/endpoints/src/api_manager/api_manager_test.cc @@ -0,0 +1,149 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "include/api_manager/api_manager.h" +#include "gtest/gtest.h" +#include "src/api_manager/mock_api_manager_environment.h" + +#include + +namespace google { +namespace api_manager { + +namespace { + +class ApiManagerTest : public ::testing::Test { + protected: + std::shared_ptr MakeApiManager( + std::unique_ptr env, const char *service_config); + + private: + ApiManagerFactory factory_; +}; + +std::shared_ptr ApiManagerTest::MakeApiManager( + std::unique_ptr env, const char *service_config) { + return factory_.GetOrCreateApiManager(std::move(env), service_config, ""); +} + +TEST_F(ApiManagerTest, EmptyConfig) { + MockApiManagerEnvironment *raw_env = new MockApiManagerEnvironment(); + std::unique_ptr env(raw_env); + ASSERT_FALSE(MakeApiManager(std::move(env), "")); +} + +TEST_F(ApiManagerTest, EnvironmentLogging) { + MockApiManagerEnvironment env; + + ::testing::InSequence s; + EXPECT_CALL(env, Log(ApiManagerEnvInterface::LogLevel::DEBUG, "debug log")); + EXPECT_CALL(env, Log(ApiManagerEnvInterface::LogLevel::INFO, "info log")); + EXPECT_CALL(env, + Log(ApiManagerEnvInterface::LogLevel::WARNING, "warning log")); + EXPECT_CALL(env, Log(ApiManagerEnvInterface::LogLevel::ERROR, "error log")); + + env.LogDebug("debug log"); + env.LogInfo("info log"); + env.LogWarning("warning log"); + env.LogError("error log"); +} + +const char service_one[] = "name: \"service-one\"\n"; +const char service_two[] = "name: \"service-two\"\n"; + +TEST_F(ApiManagerTest, SameServiceNameYieldsSameApiManager) { + std::unique_ptr env_one( + new ::testing::NiceMock()); + std::unique_ptr env_two( + new ::testing::NiceMock()); + + std::shared_ptr esp1( + MakeApiManager(std::move(env_one), service_one)); + std::shared_ptr esp2( + MakeApiManager(std::move(env_two), service_one)); + + EXPECT_TRUE(esp1); + EXPECT_TRUE(esp2); + ASSERT_EQ(esp1.get(), esp2.get()); +} + +const char kServiceForStatistics[] = + "name: \"service-name\"\n" + "control: {\n" + " environment: \"http://127.0.0.1:8081\"\n" + "}\n"; + +TEST_F(ApiManagerTest, CorrectStatistics) { + std::unique_ptr env( + new ::testing::NiceMock()); + + std::shared_ptr api_manager( + MakeApiManager(std::move(env), kServiceForStatistics)); + EXPECT_TRUE(api_manager); + EXPECT_TRUE(api_manager->Enabled()); + api_manager->Init(); + ApiManagerStatistics statistics; + api_manager->GetStatistics(&statistics); + const service_control::Statistics &service_control_stat = + statistics.service_control_statistics; + EXPECT_EQ(0, service_control_stat.total_called_checks); + EXPECT_EQ(0, service_control_stat.send_checks_by_flush); + EXPECT_EQ(0, service_control_stat.send_checks_in_flight); + EXPECT_EQ(0, service_control_stat.total_called_reports); + EXPECT_EQ(0, service_control_stat.send_reports_by_flush); + EXPECT_EQ(0, service_control_stat.send_reports_in_flight); + EXPECT_EQ(0, service_control_stat.send_report_operations); +} + +TEST_F(ApiManagerTest, DifferentServiceNameYieldsDifferentApiManager) { + std::unique_ptr env_one( + new ::testing::NiceMock()); + std::unique_ptr env_two( + new ::testing::NiceMock()); + + std::shared_ptr esp1( + MakeApiManager(std::move(env_one), service_one)); + std::shared_ptr esp2( + MakeApiManager(std::move(env_two), service_two)); + + EXPECT_TRUE(esp1); + EXPECT_TRUE(esp2); + ASSERT_NE(esp1.get(), esp2.get()); +} + +TEST_F(ApiManagerTest, CreateAfterExpirationWorks) { + std::unique_ptr env_one( + new ::testing::NiceMock()); + std::unique_ptr env_two( + new ::testing::NiceMock()); + + std::shared_ptr esp1( + MakeApiManager(std::move(env_one), service_one)); + EXPECT_TRUE(esp1); + std::weak_ptr esp1_weak(esp1); + ASSERT_FALSE(esp1_weak.expired()); + esp1.reset(); + ASSERT_TRUE(esp1_weak.expired()); + + std::shared_ptr esp2( + MakeApiManager(std::move(env_two), service_one)); + ASSERT_TRUE(esp2); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth.h b/contrib/endpoints/src/api_manager/auth.h new file mode 100644 index 00000000000..aff735bc1aa --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth.h @@ -0,0 +1,60 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_H_ +#define API_MANAGER_AUTH_H_ + +#include +#include +#include + +namespace google { +namespace api_manager { + +// Holds authentication results, used as a bridge between host proxy +// and grpc auth lib. +struct UserInfo { + // Unique ID of authenticated user. + std::string id; + // Email address of authenticated user. + std::string email; + // Consumer ID that identifies client app, used for servicecontrol. + std::string consumer_id; + // Issuer of the incoming JWT. + // See https://tools.ietf.org/html/rfc7519. + std::string issuer; + // Audience of the incoming JWT. + // See https://tools.ietf.org/html/rfc7519. + std::set audiences; + // Authorized party of the incoming JWT. + // See http://openid.net/specs/openid-connect-core-1_0.html#IDToken + std::string authorized_party; + + // Returns audiences as a comma separated strings. + std::string AudiencesAsString() const { + std::ostringstream os; + for (auto it = audiences.begin(); it != audiences.end(); ++it) { + if (it != audiences.begin()) { + os << ","; + } + os << *it; + } + return os.str(); + } +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_AUTH_H_ diff --git a/contrib/endpoints/src/api_manager/auth/BUILD b/contrib/endpoints/src/api_manager/auth/BUILD new file mode 100644 index 00000000000..8c0a0d0e00c --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/BUILD @@ -0,0 +1,92 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +package(default_visibility = ["//src/api_manager:__subpackages__"]) + +cc_library( + name = "auth", + srcs = [ + "jwt_cache.cc", + ], + hdrs = [ + "certs.h", + "jwt_cache.h", + ], + linkopts = select({ + "//:darwin": [], + "//conditions:default": [ + "-lm", + "-luuid", + ], + }), + deps = [ + "//external:googletest_prod", + "//external:servicecontrol_client", + "//src/api_manager:auth_headers", + "//src/api_manager/auth/lib", + "//src/api_manager/utils", + ], +) + +cc_library( + name = "service_account_token", + srcs = [ + "service_account_token.cc", + ], + hdrs = [ + "service_account_token.h", + ], + linkopts = select({ + "//:darwin": [], + "//conditions:default": [ + "-lm", + "-luuid", + ], + }), + deps = [ + "//external:grpc++", + "//include:headers_only", + "//src/api_manager/auth/lib", + "//src/api_manager/utils", + ], +) + +cc_test( + name = "jwt_cache_test", + size = "small", + srcs = [ + "jwt_cache_test.cc", + ], + linkstatic = 1, + deps = [ + ":auth", + "//external:googletest_main", + ], +) + +cc_test( + name = "service_account_token_test", + size = "small", + srcs = [ + "service_account_token_test.cc", + ], + linkstatic = 1, + deps = [ + ":service_account_token", + "//external:googletest_main", + "//src/api_manager:mock_api_manager_environment", + ], +) diff --git a/contrib/endpoints/src/api_manager/auth/certs.h b/contrib/endpoints/src/api_manager/auth/certs.h new file mode 100644 index 00000000000..a1f2f791cd4 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/certs.h @@ -0,0 +1,52 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_CERTS_H_ +#define API_MANAGER_AUTH_CERTS_H_ + +#include +#include +#include + +namespace google { +namespace api_manager { +namespace auth { + +// A class to manage certs for token validation. +class Certs { + public: + void Update(const std::string& issuer, const std::string& cert, + std::chrono::system_clock::time_point expiration) { + issuer_cert_map_[issuer] = std::make_pair(cert, expiration); + } + + const std::pair* GetCert( + const std::string& iss) { + return issuer_cert_map_.find(iss) == issuer_cert_map_.end() + ? nullptr + : &(issuer_cert_map_[iss]); + } + + private: + // Map from issuer to a verification key and its absolute expiration time. + std::map > + issuer_cert_map_; +}; + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_AUTH_CERTS_H_ diff --git a/contrib/endpoints/src/api_manager/auth/jwt_cache.cc b/contrib/endpoints/src/api_manager/auth/jwt_cache.cc new file mode 100644 index 00000000000..718cecd3544 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/jwt_cache.cc @@ -0,0 +1,51 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/jwt_cache.h" + +using ::google::service_control_client::SimpleLRUCache; +using std::chrono::system_clock; + +namespace google { +namespace api_manager { +namespace auth { + +namespace { +// The maximum lifetime of a cache entry. Unit: seconds. +// TODO: This value should be configurable via server config. +const int kJwtCacheTimeout = 300; +// The number of entries in JWT cache. +// TODO: This value should be configurable via server config. +const int kJwtCacheSize = 100; +} // namespace + +JwtCache::JwtCache() : SimpleLRUCache(kJwtCacheSize) {} + +JwtCache::~JwtCache() { Clear(); } + +void JwtCache::Insert(const std::string& jwt, const UserInfo& user_info, + const system_clock::time_point& token_exp, + const system_clock::time_point& now) { + JwtValue* newval = new JwtValue(); + newval->user_info = user_info; + newval->exp = + std::min(token_exp, now + std::chrono::seconds(kJwtCacheTimeout)); + SimpleLRUCache::Insert(jwt, newval, 1); +} + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/jwt_cache.h b/contrib/endpoints/src/api_manager/auth/jwt_cache.h new file mode 100644 index 00000000000..6b06f58c28a --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/jwt_cache.h @@ -0,0 +1,56 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_JWT_CACHE_H_ +#define API_MANAGER_AUTH_JWT_CACHE_H_ + +#include +#include + +#include "src/api_manager/auth.h" +#include "utils/simple_lru_cache_inl.h" + +namespace google { +namespace api_manager { +namespace auth { + +// The value of a JwtCache entry. +struct JwtValue { + // User info extracted from the JWT. + UserInfo user_info; + + // Expiration time of the cache entry. This is the minimum of "exp" field in + // the JWT and [the time this cache entry is added + kJwtCacheTimeout]. + std::chrono::system_clock::time_point exp; +}; + +// A local cache that resides in ESP. The key of the cache is a JWT, +// and the value is of type JwtValue. +class JwtCache + : public google::service_control_client::SimpleLRUCache { + public: + JwtCache(); + ~JwtCache(); + + void Insert(const std::string& jwt, const UserInfo& user_info, + const std::chrono::system_clock::time_point& token_exp, + const std::chrono::system_clock::time_point& now); +}; + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_AUTH_JWT_CACHE_H_ diff --git a/contrib/endpoints/src/api_manager/auth/jwt_cache_test.cc b/contrib/endpoints/src/api_manager/auth/jwt_cache_test.cc new file mode 100644 index 00000000000..82cc4ed7d2d --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/jwt_cache_test.cc @@ -0,0 +1,102 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/jwt_cache.h" +#include +#include "gtest/gtest.h" + +using std::chrono::system_clock; + +namespace google { +namespace api_manager { +namespace auth { + +namespace { + +const char kId[] = "user1"; +const char kEmail[] = "user1@gmail.com"; +const char kConsumer[] = "consumer1"; +const char kIssuer[] = "iss1"; +const char kJwt[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1M" + "jNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20" + "iLCJzdWIiOiI2Mjg2NDU3NDE4ODEtbm9hYml1MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZ" + "GV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20iLCJhdWQiOiJodHRwOi8vbXlzZXJ2aWNlLmN" + "vbS9teWFwaSJ9.gq_4ucjddQDjYK5FJr_kXmMo2fgSEB6Js1zopcQLVpCKFDNb-TQ97go0wuk5" + "_vlSp_8I2ImrcdwYbAKqYCzcdyBXkAYoHCGgmY-v6MwZFUvrIaDzR_M3rmY8sQ8cdN3MN6ZRbB" + "6opHwDP1lUEx4bZn_ZBjJMPgqbIqGmhoT1UpfPF6P1eI7sXYru-4KVna0STOynLl3d7JYb7E-8" + "ifcjUJLhat8JR4zR8i4-zWjn6d6j_NI7ZvMROnao77D9YyhXv56zfsXRatKzzYtxPlQMz4AjP-" + "bUHfbHmhiIOOAeEKFuIVUAwM17j54M6VQ5jnAabY5O-ermLfwPiXvNt2L2SA=="; +const int kJwtCacheTimeout = 300; + +class TestJwtCache : public ::testing::Test { + public: + virtual void SetUp() { cache_.reset(new JwtCache()); } + + std::unique_ptr cache_; +}; + +// Test the Insert function in JwtCache class. +void InsertAndLookupImpl(JwtCache *cache, bool token_exp_earlier) { + ASSERT_EQ(nullptr, cache->Lookup(kJwt)); + + UserInfo user_info; + user_info.id = kId; + user_info.email = kEmail; + user_info.consumer_id = kConsumer; + user_info.issuer = kIssuer; + user_info.audiences.insert("aud1"); + user_info.audiences.insert("aud2"); + system_clock::time_point now = system_clock::now(); + + system_clock::time_point token_exp; + if (token_exp_earlier) { + token_exp = now + std::chrono::seconds(kJwtCacheTimeout - 1); + } else { + token_exp = now + std::chrono::seconds(kJwtCacheTimeout + 1); + } + cache->Insert(kJwt, user_info, token_exp, now); + JwtValue *val = cache->Lookup(kJwt); + ASSERT_NE(nullptr, val); + ASSERT_EQ(val->user_info.id, kId); + ASSERT_EQ(val->user_info.email, kEmail); + ASSERT_EQ(val->user_info.consumer_id, kConsumer); + ASSERT_EQ(val->user_info.issuer, kIssuer); + ASSERT_EQ(val->user_info.AudiencesAsString(), "aud1,aud2"); + if (token_exp_earlier) { + ASSERT_EQ(val->exp, token_exp); + } else { + ASSERT_EQ(val->exp, now + std::chrono::seconds(kJwtCacheTimeout)); + } + + cache->Release(kJwt, val); + cache->Remove(kJwt); + ASSERT_EQ(nullptr, cache->Lookup(kJwt)); +} + +TEST_F(TestJwtCache, InsertAndLookUp) { + // case 1: token expire sooner than 5 minutes. + InsertAndLookupImpl(cache_.get(), true); + + // case 2: token lifetime is 5 minutes. + InsertAndLookupImpl(cache_.get(), false); +} + +} // namespace + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/BUILD b/contrib/endpoints/src/api_manager/auth/lib/BUILD new file mode 100644 index 00000000000..50796c7699a --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/BUILD @@ -0,0 +1,107 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +cc_library( + name = "lib", + srcs = [ + "auth_jwt_validator.cc", + "auth_token.cc", + "base64.cc", + "grpc_internals.h", + "json.cc", + "json_util.cc", + ], + hdrs = [ + "auth_jwt_validator.h", + "auth_token.h", + "base64.h", + "json.h", + "json_util.h", + ], + visibility = ["//visibility:public"], + deps = [ + "//external:grpc", + "//external:protobuf", + "//include:headers_only", + "//src/api_manager:auth_headers", + "//src/api_manager/utils", + ], +) + +cc_test( + name = "auth_jwt_validator_test", + size = "small", + srcs = [ + "auth_jwt_validator_test.cc", + ], + linkstatic = 1, + deps = [ + ":lib", + "//external:googletest_main", + ], +) + +cc_test( + name = "auth_token_test", + size = "small", + srcs = [ + "auth_token_test.cc", + ], + linkstatic = 1, + deps = [ + ":lib", + "//external:googletest_main", + ], +) + +cc_test( + name = "json_test", + size = "small", + srcs = [ + "json_test.cc", + ], + linkstatic = 1, + deps = [ + ":lib", + "//external:googletest_main", + ], +) + +cc_test( + name = "base64_test", + size = "small", + srcs = [ + "base64_test.cc", + ], + linkstatic = 1, + deps = [ + ":lib", + "//external:googletest_main", + ], +) + +cc_test( + name = "json_util_test", + size = "small", + srcs = [ + "json_util_test.cc", + ], + linkstatic = 1, + deps = [ + ":lib", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc new file mode 100644 index 00000000000..a245454269c --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc @@ -0,0 +1,763 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/auth_jwt_validator.h" + +// Implementation of JWT token verification. + +// Support public keys in x509 format or JWK (Json Web Keys). +// -- Sample x509 keys +// { +// "8f3e950b309186540c314ecf348bb14f1784d79d": "-----BEGIN +// CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIIYJnxRhkHEz8wDQYJKoZIhvcNAQEFBQAwMTEvMC0GA1UE\nAxMmc2VjdXJldG9rZW4uc3lzdGVtLmdzZXJ2aWNlYWNjb3VudC5jb20wHhcNMTUw\nNjE2MDEwMzQxWhcNMTUwNjE3MTQwMzQxWjAxMS8wLQYDVQQDEyZzZWN1cmV0b2tl\nbi5zeXN0ZW0uZ3NlcnZpY2VhY2NvdW50LmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD\nggEPADCCAQoCggEBAMNKd/jkdD+ifIw806pawXZo656ycjL1KB/kUJbPopTzKKxZ\nR/eYJpd5BZIZPnWbXGvoY2kGne8jYJptQLLHr18u7TDVMpnh41jvLWYHXJv8Zd/W\n1HZk4t5mm4+JzZ2WUAx881aiEieO7/cMSIT3VC2I98fMFuEJ8jAWUDWY3KzHsXp0\nlj5lJknFCiESQ8s+UxFYHF/EgS8S2eJBvs2unq1a4NVan/GupA1OB5LrlFXm09Vt\na+dB4gulBrPh0/AslRd36uiXLRFnvAr+EF25WyZsUcq0ANCFx1Rd5z3Fv/5zC9hw\n3EeHEpc+NgovzPJ+IDHfqiU4BLTPgT70DYeLHUcCAwEAAaM4MDYwDAYDVR0TAQH/\nBAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwIwDQYJ\nKoZIhvcNAQEFBQADggEBAK2b2Mw5W0BtXS+onaKyyvC9O2Ysh7gTjjOGTVaTpYaB\nDg2vDgqFHM5xeYwMUf8O163rZz4R/DusZ5GsNav9BSsco9VsaIp5oIuy++tepnVN\niAdzK/bH/mo6w0s/+q46v3yhSE2Yd9WzKS9eSfQ6Yw/6y1rCgygTIVsdtKwN2u9L\nXj3RQGcHk7tgrICETqeMULZWMJQG2webNAu8bqkONgo+JP54QNVCzWgCznbCbmOR\nExlMusHMG60j8CxmMG/WLUhX+46m5HxVVx29AH8RhqpwmeFs17QXpGjOMW+ZL/Vf\nwmtv14KGfeX0z2A2iQAP5w6R1r6c+HWizj80mXHWI5U=\n-----END +// CERTIFICATE-----\n", +// "0f980915096a38d8de8d7398998c7fb9152e14fc": "-----BEGIN +// CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIIVhHsCCeHFBowDQYJKoZIhvcNAQEFBQAwMTEvMC0GA1UE\nAxMmc2VjdXJldG9rZW4uc3lzdGVtLmdzZXJ2aWNlYWNjb3VudC5jb20wHhcNMTUw\nNjE3MDA0ODQxWhcNMTUwNjE4MTM0ODQxWjAxMS8wLQYDVQQDEyZzZWN1cmV0b2tl\nbi5zeXN0ZW0uZ3NlcnZpY2VhY2NvdW50LmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD\nggEPADCCAQoCggEBAKbpeArSNTeYN977uD7GxgFhjghjsTFAq52IW04BdoXQmT9G\nP38s4q06UKGjaZbvEQmdxS+IX2BvswHxiOOgA210C4vRIBu6k1fAnt4JYBy1QHf8\n6C4K9cArp5Sx7/NJcTyu0cj/Ce1fi2iKcvuaQG7+e6VsERWjCFoUHbBohx9a92ch\nMVzQU3Bp8Ix6err6gsxxX8AcrgTN9Ux1Z2x6Ahd/x6Id2HkP8N4dGq72ksk1T9y6\n+Q8LmCzgILSyKvtVVW9G44neFDQvcvJyQljfM996b03yur4XRBs3dPS9AyJlGuN3\nagxBLwM2ieXyM73Za8khlR8PJMUcy4vA6zVHQeECAwEAAaM4MDYwDAYDVR0TAQH/\nBAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwIwDQYJ\nKoZIhvcNAQEFBQADggEBAJyWqYryl4K/DS0RCD66V4v8pZ5ja80Dr/dp8U7cH3Xi\nxHfCqtR5qmKZC48CHg4OPD9WSiVYFsDGLJn0KGU85AXghqcMSJN3ffABNMZBX5a6\nQGFjnvL4b2wZGgqXdaxstJ6pnaM6lQ5J6ZwFTMf0gaeo+jkwx9ENPRLudD5Mf91z\nLvLdp+gRWAlZ3Avo1YTG916kMnGRRwNJ7xgy1YSsEOUzzsNlQWca/XdGmj1I3BW/\nimaI/QRPePs3LlpPVtgu5jvOqyRpaNaYNQU7ki+jdEU4ZDOAvteqd5svXitfpB+a\nx5Bj4hUbZhw2U9AMMDmknhH4w3JKeKYGcdQrO/qVWFQ=\n-----END +// CERTIFICATE-----\n" +// } +// +// -- Sample JWK keys +// { +// "keys": [ +// { +// "kty": "RSA", +// "alg": "RS256", +// "use": "sig", +// "kid": "62a93512c9ee4c7f8067b5a216dade2763d32a47", +// "n": +// "0YWnm_eplO9BFtXszMRQNL5UtZ8HJdTH2jK7vjs4XdLkPW7YBkkm_2xNgcaVpkW0VT2l4mU3KftR-6s3Oa5Rnz5BrWEUkCTVVolR7VYksfqIB2I_x5yZHdOiomMTcm3DheUUCgbJRv5OKRnNqszA4xHn3tA3Ry8VO3X7BgKZYAUh9fyZTFLlkeAh0-bLK5zvqCmKW5QgDIXSxUTJxPjZCgfx1vmAfGqaJb-nvmrORXQ6L284c73DUL7mnt6wj3H6tVqPKA27j56N0TB1Hfx4ja6Slr8S4EB3F1luYhATa1PKUSH8mYDW11HolzZmTQpRoLV8ZoHbHEaTfqX_aYahIw", +// "e": "AQAB" +// }, +// { +// "kty": "RSA", +// "alg": "RS256", +// "use": "sig", +// "kid": "b3319a147514df7ee5e4bcdee51350cc890cc89e", +// "n": +// "qDi7Tx4DhNvPQsl1ofxxc2ePQFcs-L0mXYo6TGS64CY_2WmOtvYlcLNZjhuddZVV2X88m0MfwaSA16wE-RiKM9hqo5EY8BPXj57CMiYAyiHuQPp1yayjMgoE1P2jvp4eqF-BTillGJt5W5RuXti9uqfMtCQdagB8EC3MNRuU_KdeLgBy3lS3oo4LOYd-74kRBVZbk2wnmmb7IhP9OoLc1-7-9qU1uhpDxmE6JwBau0mDSwMnYDS4G_ML17dC-ZDtLd1i24STUw39KH0pcSdfFbL2NtEZdNeam1DDdk0iUtJSPZliUHJBI_pj8M-2Mn_oA8jBuI8YKwBqYkZCN1I95Q", +// "e": "AQAB" +// } +// ] +// } + +extern "C" { +#include +#include +} + +#include "grpc_internals.h" + +#include +#include +#include +#include +#include + +#include "src/api_manager/auth/lib/json_util.h" + +using std::string; +using std::chrono::system_clock; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace auth { +namespace { + +// JOSE header. see http://tools.ietf.org/html/rfc7515#section-4 +struct JoseHeader { + const char *alg; + const char *kid; +}; + +// An implementation of JwtValidator, hold ALL allocated memory data. +class JwtValidatorImpl : public JwtValidator { + public: + JwtValidatorImpl(const char *jwt, size_t jwt_len); + Status Parse(UserInfo *user_info); + Status VerifySignature(const char *pkey, size_t pkey_len); + system_clock::time_point &GetExpirationTime() { return exp_; } + ~JwtValidatorImpl(); + + private: + // Validates given JWT with pkey. + grpc_jwt_verifier_status Validate(const char *jwt, size_t jwt_len, + const char *pkey, size_t pkey_len, + const char *aud); + grpc_jwt_verifier_status ParseImpl(); + grpc_jwt_verifier_status VerifySignatureImpl(const char *pkey, + size_t pkey_len); + // Parses the audiences and removes the audiences from the json object. + void UpdateAudience(grpc_json *json); + + // Creates header_ from header_json_. + void CreateJoseHeader(); + // Checks required fields and fills User Info from claims_. + // And sets expiration time to exp_. + grpc_jwt_verifier_status FillUserInfoAndSetExp(UserInfo *user_info); + // Finds the public key and verifies JWT signature with it. + grpc_jwt_verifier_status FindAndVerifySignature(); + // Extracts the public key from x509 string (key) and sets it to pkey_. + // Returns true if successful. + bool ExtractPubkeyFromX509(const char *key); + // Extracts the public key from a jwk key (jkey) and sets it to pkey_. + // Returns true if successful. + bool ExtractPubkeyFromJwk(const grpc_json *jkey); + // Extracts the public key from jwk key set and verifies JWT signature with + // it. + grpc_jwt_verifier_status ExtractAndVerifyJwkKeys(const grpc_json *jwt_keys); + // Extracts the public key from pkey_json_ and verifies JWT signature with + // it. + grpc_jwt_verifier_status ExtractAndVerifyX509Keys(); + // Verifies signature with pkey_. + grpc_jwt_verifier_status VerifyPubkey(); + // Verifies RS signature. + grpc_jwt_verifier_status VerifyRsSignature(const char *pkey, size_t pkey_len); + // Verifies HS signature. + grpc_jwt_verifier_status VerifyHsSignature(const char *pkey, size_t pkey_len); + + // Not owned. + const char *jwt; + int jwt_len; + + JoseHeader *header_; + grpc_json *header_json_; + gpr_slice header_buffer_; + grpc_jwt_claims *claims_; + gpr_slice sig_buffer_; + gpr_slice signed_buffer_; + + std::set audiences_; + system_clock::time_point exp_; + + grpc_json *pkey_json_; + gpr_slice pkey_buffer_; + BIO *bio_; + X509 *x509_; + RSA *rsa_; + EVP_PKEY *pkey_; + EVP_MD_CTX *md_ctx_; +}; + +// Gets EVP_MD mapped from an alg (algorithm string). +const EVP_MD *EvpMdFromAlg(const char *alg); + +// Gets hash size from HS algorithm string. +size_t HashSizeFromAlg(const char *alg); + +// Parses str into grpc_json object. Does not own buffer. +grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, + gpr_slice *buffer); + +// Gets BIGNUM from b64 string, used for extracting pkey from jwk. +// Result owned by rsa_. +BIGNUM *BigNumFromBase64String(const char *b64); + +} // namespace + +std::unique_ptr JwtValidator::Create(const char *jwt, + size_t jwt_len) { + return std::unique_ptr(new JwtValidatorImpl(jwt, jwt_len)); +} + +namespace { +JwtValidatorImpl::JwtValidatorImpl(const char *jwt, size_t jwt_len) + : jwt(jwt), + jwt_len(jwt_len), + header_(nullptr), + header_json_(nullptr), + claims_(nullptr), + pkey_json_(nullptr), + bio_(nullptr), + x509_(nullptr), + rsa_(nullptr), + pkey_(nullptr), + md_ctx_(nullptr) { + header_buffer_ = gpr_empty_slice(); + signed_buffer_ = gpr_empty_slice(); + sig_buffer_ = gpr_empty_slice(); + pkey_buffer_ = gpr_empty_slice(); +} + +// Makes sure all data are cleaned up, both success and failure case. +JwtValidatorImpl::~JwtValidatorImpl() { + if (header_ != nullptr) { + gpr_free(header_); + } + if (header_json_ != nullptr) { + grpc_json_destroy(header_json_); + } + if (pkey_json_ != nullptr) { + grpc_json_destroy(pkey_json_); + } + if (claims_ != nullptr) { + grpc_jwt_claims_destroy(claims_); + } + if (!GPR_SLICE_IS_EMPTY(header_buffer_)) { + gpr_slice_unref(header_buffer_); + } + if (!GPR_SLICE_IS_EMPTY(signed_buffer_)) { + gpr_slice_unref(signed_buffer_); + } + if (!GPR_SLICE_IS_EMPTY(sig_buffer_)) { + gpr_slice_unref(sig_buffer_); + } + if (!GPR_SLICE_IS_EMPTY(pkey_buffer_)) { + gpr_slice_unref(pkey_buffer_); + } + if (bio_ != nullptr) { + BIO_free(bio_); + } + if (x509_ != nullptr) { + X509_free(x509_); + } + if (rsa_ != nullptr) { + RSA_free(rsa_); + } + if (pkey_ != nullptr) { + EVP_PKEY_free(pkey_); + } + if (md_ctx_ != nullptr) { + EVP_MD_CTX_destroy(md_ctx_); + } +} + +Status JwtValidatorImpl::Parse(UserInfo *user_info) { + grpc_jwt_verifier_status status = ParseImpl(); + if (status == GRPC_JWT_VERIFIER_OK) { + status = FillUserInfoAndSetExp(user_info); + if (status == GRPC_JWT_VERIFIER_OK) { + return Status::OK; + } + } + + return Status(Code::UNAUTHENTICATED, + grpc_jwt_verifier_status_to_string(status)); +} + +// Extracts and removes the audiences from the token. +// This is a workaround to deal with GRPC library not accepting +// multiple audiences. +void JwtValidatorImpl::UpdateAudience(grpc_json *json) { + grpc_json *cur; + for (cur = json->child; cur != nullptr; cur = cur->next) { + if (strcmp(cur->key, "aud") == 0) { + if (cur->type == GRPC_JSON_ARRAY) { + grpc_json *aud; + for (aud = cur->child; aud != nullptr; aud = aud->next) { + if (aud->type == GRPC_JSON_STRING && aud->value != nullptr) { + audiences_.insert(aud->value); + } + } + // Replaces the array of audiences with an empty string. + grpc_json *prev = cur->prev; + grpc_json *next = cur->next; + grpc_json_destroy(cur); + grpc_json *fake_audience = grpc_json_create(GRPC_JSON_STRING); + fake_audience->key = "aud"; + fake_audience->value = ""; + fake_audience->parent = json; + fake_audience->prev = prev; + fake_audience->next = next; + if (prev) { + prev->next = fake_audience; + } else { + json->child = fake_audience; + } + if (next) { + next->prev = fake_audience; + } + } else if (cur->type == GRPC_JSON_STRING && cur->value != nullptr) { + audiences_.insert(cur->value); + } + return; + } + } +} + +grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { + // ==================== + // Basic check. + // ==================== + if (jwt == nullptr || jwt_len <= 0) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + + // ==================== + // Creates Jose Header. + // ==================== + const char *cur = jwt; + const char *dot = strchr(cur, '.'); + if (dot == nullptr) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + header_json_ = DecodeBase64AndParseJson(cur, dot - cur, &header_buffer_); + CreateJoseHeader(); + if (header_ == nullptr) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + + // ============================= + // Creates Claims/Payload. + // ============================= + cur = dot + 1; + dot = strchr(cur, '.'); + if (dot == nullptr) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + + // claim_buffer is the only exception that requires deallocation for failure + // case, and it is owned by claims_ for successful case. + gpr_slice claims_buffer = gpr_empty_slice(); + grpc_json *claims_json = + DecodeBase64AndParseJson(cur, dot - cur, &claims_buffer); + if (claims_json == nullptr) { + if (!GPR_SLICE_IS_EMPTY(claims_buffer)) { + gpr_slice_unref(claims_buffer); + } + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + UpdateAudience(claims_json); + // Takes ownershp of claims_json and claims_buffer. + claims_ = grpc_jwt_claims_from_json(claims_json, claims_buffer); + if (claims_ == nullptr) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + // Check timestamp. + // Passing in its own audience to skip audience check. + // Audience check should be done by the caller. + grpc_jwt_verifier_status status = + grpc_jwt_claims_check(claims_, grpc_jwt_claims_audience(claims_)); + if (status != GRPC_JWT_VERIFIER_OK) { + return status; + } + + // ============================= + // Creates Buffer for signature check + // ============================= + size_t signed_jwt_len = (size_t)(dot - jwt); + signed_buffer_ = gpr_slice_from_copied_buffer(jwt, signed_jwt_len); + if (GPR_SLICE_IS_EMPTY(signed_buffer_)) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + cur = dot + 1; + sig_buffer_ = + grpc_base64_decode_with_len(cur, jwt_len - signed_jwt_len - 1, 1); + if (GPR_SLICE_IS_EMPTY(sig_buffer_)) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + + return GRPC_JWT_VERIFIER_OK; +} + +Status JwtValidatorImpl::VerifySignature(const char *pkey, size_t pkey_len) { + grpc_jwt_verifier_status status = VerifySignatureImpl(pkey, pkey_len); + if (status == GRPC_JWT_VERIFIER_OK) { + return Status::OK; + } else { + return Status(Code::UNAUTHENTICATED, + grpc_jwt_verifier_status_to_string(status)); + } +} + +grpc_jwt_verifier_status JwtValidatorImpl::VerifySignatureImpl( + const char *pkey, size_t pkey_len) { + if (jwt == nullptr || pkey == nullptr || jwt_len <= 0 || pkey_len <= 0) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + if (GPR_SLICE_IS_EMPTY(signed_buffer_) || GPR_SLICE_IS_EMPTY(sig_buffer_)) { + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + if (strncmp(header_->alg, "RS", 2) == 0) { // Asymmetric keys. + return VerifyRsSignature(pkey, pkey_len); + } else { // Symmetric key. + return VerifyHsSignature(pkey, pkey_len); + } +} + +void JwtValidatorImpl::CreateJoseHeader() { + if (header_json_ == nullptr) { + return; + } + const char *alg = GetStringValue(header_json_, "alg"); + if (alg == nullptr) { + gpr_log(GPR_ERROR, "Missing alg field."); + return; + } + if (EvpMdFromAlg(alg) == nullptr) { + gpr_log(GPR_ERROR, "Invalid alg field [%s].", alg); + return; + } + + header_ = reinterpret_cast(gpr_malloc(sizeof(JoseHeader))); + if (header_ == nullptr) { + gpr_log(GPR_ERROR, "Jose header creation failed"); + } + header_->alg = alg; + header_->kid = GetStringValue(header_json_, "kid"); +} + +grpc_jwt_verifier_status JwtValidatorImpl::FindAndVerifySignature() { + if (pkey_json_ == nullptr) { + gpr_log(GPR_ERROR, "The public keys are empty."); + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + if (header_ == nullptr) { + gpr_log(GPR_ERROR, "JWT header is empty."); + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + // JWK set https://tools.ietf.org/html/rfc7517#section-5. + const grpc_json *jwk_keys = GetProperty(pkey_json_, "keys"); + if (jwk_keys == nullptr) { + // Try x509 format. + return ExtractAndVerifyX509Keys(); + } else { + // JWK format. + return ExtractAndVerifyJwkKeys(jwk_keys); + } +} + +grpc_jwt_verifier_status JwtValidatorImpl::ExtractAndVerifyX509Keys() { + // Precondition (checked by caller): pkey_json_ and header_ are not nullptr. + if (header_->kid != nullptr) { + const char *value = GetStringValue(pkey_json_, header_->kid); + if (value == nullptr) { + gpr_log(GPR_ERROR, + "Cannot find matching key in key set for kid=%s and alg=%s", + header_->kid, header_->alg); + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + if (!ExtractPubkeyFromX509(value)) { + gpr_log(GPR_ERROR, "Failed to extract public key from X509 key (%s)", + header_->kid); + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + return VerifyPubkey(); + } + // If kid is not specified in the header, try all keys. If the JWT can be + // validated with any of the keys, the request is successful. + const grpc_json *cur; + for (cur = pkey_json_->child; cur != nullptr; cur = cur->next) { + if (cur->value == nullptr || !ExtractPubkeyFromX509(cur->value)) { + // Failed to extract public key from X509 key. + continue; + } + if (VerifyPubkey() == GRPC_JWT_VERIFIER_OK) { + return GRPC_JWT_VERIFIER_OK; + } + } + // header_->kid is nullptr. The JWT cannot be validated with any of the keys. + // Return error. + gpr_log(GPR_ERROR, + "The JWT cannot be validated with any of the public keys."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; +} + +bool JwtValidatorImpl::ExtractPubkeyFromX509(const char *key) { + if (bio_ != nullptr) { + BIO_free(bio_); + } + bio_ = BIO_new(BIO_s_mem()); + if (bio_ == nullptr) { + gpr_log(GPR_ERROR, "Unable to allocate a BIO object."); + return false; + } + if (BIO_write(bio_, key, strlen(key)) <= 0) { + gpr_log(GPR_ERROR, "BIO write error for key (%s).", key); + return false; + } + if (x509_ != nullptr) { + X509_free(x509_); + } + x509_ = PEM_read_bio_X509(bio_, nullptr, nullptr, nullptr); + if (x509_ == nullptr) { + gpr_log(GPR_ERROR, "Unable to parse x509 cert for key (%s).", key); + return false; + } + if (pkey_ != nullptr) { + EVP_PKEY_free(pkey_); + } + pkey_ = X509_get_pubkey(x509_); + if (pkey_ == nullptr) { + gpr_log(GPR_ERROR, "X509_get_pubkey failed"); + return false; + } + return true; +} + +grpc_jwt_verifier_status JwtValidatorImpl::ExtractAndVerifyJwkKeys( + const grpc_json *jwk_keys) { + // Precondition (checked by caller): jwk_keys and header_ are not nullptr. + if (jwk_keys->type != GRPC_JSON_ARRAY) { + gpr_log(GPR_ERROR, + "Unexpected value type of keys property in jwks key set."); + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + + const grpc_json *jkey = nullptr; + + // JWK format from https://tools.ietf.org/html/rfc7518#section-6. + for (jkey = jwk_keys->child; jkey != nullptr; jkey = jkey->next) { + if (jkey->type != GRPC_JSON_OBJECT) continue; + const char *alg = GetStringValue(jkey, "alg"); + if (alg == nullptr || strcmp(alg, header_->alg) != 0) { + continue; + } + const char *kid = GetStringValue(jkey, "kid"); + if (kid == nullptr || + (header_->kid != nullptr && strcmp(kid, header_->kid) != 0)) { + continue; + } + const char *kty = GetStringValue(jkey, "kty"); + if (kty == nullptr || strcmp(kty, "RSA") != 0) { + gpr_log(GPR_ERROR, "Missing or unsupported key type %s.", kty); + continue; + } + if (!ExtractPubkeyFromJwk(jkey)) { + // Failed to extract public key from this Jwk key. + continue; + } + if (header_->kid != nullptr) { + return VerifyPubkey(); + } else { + // If kid is not specified in the header, try all keys. If the JWT can be + // validated with any of the keys, the request is successful. + if (VerifyPubkey() == GRPC_JWT_VERIFIER_OK) { + return GRPC_JWT_VERIFIER_OK; + } + } + } + + if (header_->kid != nullptr) { + gpr_log(GPR_ERROR, + "Cannot find matching key in key set for kid=%s and alg=%s", + header_->kid, header_->alg); + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + // header_->kid is nullptr. The JWT cannot be validated with any of the keys. + // Return error. + gpr_log(GPR_ERROR, + "The JWT cannot be validated with any of the public keys."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; +} + +bool JwtValidatorImpl::ExtractPubkeyFromJwk(const grpc_json *jkey) { + if (rsa_ != nullptr) { + RSA_free(rsa_); + } + rsa_ = RSA_new(); + if (rsa_ == nullptr) { + gpr_log(GPR_ERROR, "Could not create rsa key."); + return false; + } + + const char *rsa_n = GetStringValue(jkey, "n"); + rsa_->n = rsa_n == nullptr ? nullptr : BigNumFromBase64String(rsa_n); + const char *rsa_e = GetStringValue(jkey, "e"); + rsa_->e = rsa_e == nullptr ? nullptr : BigNumFromBase64String(rsa_e); + + if (rsa_->e == nullptr || rsa_->n == nullptr) { + gpr_log(GPR_ERROR, "Missing RSA public key field."); + return false; + } + + if (pkey_ != nullptr) { + EVP_PKEY_free(pkey_); + } + pkey_ = EVP_PKEY_new(); + if (EVP_PKEY_set1_RSA(pkey_, rsa_) == 0) { + gpr_log(GPR_ERROR, "EVP_PKEY_ste1_RSA failed"); + return false; + } + return true; +} + +grpc_jwt_verifier_status JwtValidatorImpl::VerifyRsSignature(const char *pkey, + size_t pkey_len) { + pkey_buffer_ = gpr_slice_from_copied_buffer(pkey, pkey_len); + if (GPR_SLICE_IS_EMPTY(pkey_buffer_)) { + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + pkey_json_ = grpc_json_parse_string_with_len( + reinterpret_cast(GPR_SLICE_START_PTR(pkey_buffer_)), + GPR_SLICE_LENGTH(pkey_buffer_)); + if (pkey_json_ == nullptr) { + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + + return FindAndVerifySignature(); +} + +grpc_jwt_verifier_status JwtValidatorImpl::VerifyPubkey() { + if (pkey_ == nullptr) { + gpr_log(GPR_ERROR, "Cannot find public key."); + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + if (md_ctx_ != nullptr) { + EVP_MD_CTX_destroy(md_ctx_); + } + md_ctx_ = EVP_MD_CTX_create(); + if (md_ctx_ == nullptr) { + gpr_log(GPR_ERROR, "Could not create EVP_MD_CTX."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; + } + const EVP_MD *md = EvpMdFromAlg(header_->alg); + + GPR_ASSERT(md != nullptr); // Checked before. + + if (EVP_DigestVerifyInit(md_ctx_, nullptr, md, nullptr, pkey_) != 1) { + gpr_log(GPR_ERROR, "EVP_DigestVerifyInit failed."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; + } + if (EVP_DigestVerifyUpdate(md_ctx_, GPR_SLICE_START_PTR(signed_buffer_), + GPR_SLICE_LENGTH(signed_buffer_)) != 1) { + gpr_log(GPR_ERROR, "EVP_DigestVerifyUpdate failed."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; + } + if (EVP_DigestVerifyFinal(md_ctx_, GPR_SLICE_START_PTR(sig_buffer_), + GPR_SLICE_LENGTH(sig_buffer_)) != 1) { + gpr_log(GPR_ERROR, "JWT signature verification failed."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; + } + return GRPC_JWT_VERIFIER_OK; +} + +grpc_jwt_verifier_status JwtValidatorImpl::VerifyHsSignature(const char *pkey, + size_t pkey_len) { + const EVP_MD *md = EvpMdFromAlg(header_->alg); + GPR_ASSERT(md != nullptr); // Checked before. + + pkey_buffer_ = grpc_base64_decode_with_len(pkey, pkey_len, 1); + if (GPR_SLICE_IS_EMPTY(pkey_buffer_)) { + gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); + return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; + } + + unsigned char res[HashSizeFromAlg(header_->alg)]; + unsigned int res_len = 0; + HMAC(md, GPR_SLICE_START_PTR(pkey_buffer_), GPR_SLICE_LENGTH(pkey_buffer_), + GPR_SLICE_START_PTR(signed_buffer_), GPR_SLICE_LENGTH(signed_buffer_), + res, &res_len); + if (res_len == 0) { + gpr_log(GPR_ERROR, "Cannot compute HMAC from secret."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; + } + + if (res_len != GPR_SLICE_LENGTH(sig_buffer_) || + CRYPTO_memcmp(reinterpret_cast(GPR_SLICE_START_PTR(sig_buffer_)), + reinterpret_cast(res), res_len) != 0) { + gpr_log(GPR_ERROR, "JWT signature verification failed."); + return GRPC_JWT_VERIFIER_BAD_SIGNATURE; + } + return GRPC_JWT_VERIFIER_OK; +} + +grpc_jwt_verifier_status JwtValidatorImpl::FillUserInfoAndSetExp( + UserInfo *user_info) { + // Required fields. + const char *issuer = grpc_jwt_claims_issuer(claims_); + if (issuer == nullptr) { + gpr_log(GPR_ERROR, "Missing issuer field."); + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + if (audiences_.empty()) { + gpr_log(GPR_ERROR, "Missing audience field."); + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + const char *subject = grpc_jwt_claims_subject(claims_); + if (subject == nullptr) { + gpr_log(GPR_ERROR, "Missing subject field."); + return GRPC_JWT_VERIFIER_BAD_FORMAT; + } + user_info->issuer = issuer; + user_info->audiences = audiences_; + user_info->id = subject; + + // Optional field. + const grpc_json *grpc_json = grpc_jwt_claims_json(claims_); + const char *email = GetStringValue(grpc_json, "email"); + user_info->email = email == nullptr ? "" : email; + const char *authorized_party = GetStringValue(grpc_json, "azp"); + user_info->authorized_party = + authorized_party == nullptr ? "" : authorized_party; + + exp_ = system_clock::from_time_t(grpc_jwt_claims_expires_at(claims_).tv_sec); + + return GRPC_JWT_VERIFIER_OK; +} + +const EVP_MD *EvpMdFromAlg(const char *alg) { + if (strcmp(alg, "RS256") == 0 || strcmp(alg, "HS256") == 0) { + return EVP_sha256(); + } else if (strcmp(alg, "RS384") == 0 || strcmp(alg, "HS384") == 0) { + return EVP_sha384(); + } else if (strcmp(alg, "RS512") == 0 || strcmp(alg, "HS512") == 0) { + return EVP_sha512(); + } else { + return nullptr; + } +} + +// Gets hash byte size from HS algorithm string. +size_t HashSizeFromAlg(const char *alg) { + if (strcmp(alg, "HS256") == 0) { + return 32; + } else if (strcmp(alg, "HS384") == 0) { + return 48; + } else if (strcmp(alg, "HS512") == 0) { + return 64; + } else { + return 0; + } +} + +grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, + gpr_slice *buffer) { + grpc_json *json; + + *buffer = grpc_base64_decode_with_len(str, len, 1); + if (GPR_SLICE_IS_EMPTY(*buffer)) { + gpr_log(GPR_ERROR, "Invalid base64."); + return nullptr; + } + json = grpc_json_parse_string_with_len( + reinterpret_cast(GPR_SLICE_START_PTR(*buffer)), + GPR_SLICE_LENGTH(*buffer)); + if (json == nullptr) { + gpr_log(GPR_ERROR, "JSON parsing error."); + } + return json; +} + +BIGNUM *BigNumFromBase64String(const char *b64) { + BIGNUM *result = nullptr; + gpr_slice bin; + + if (b64 == nullptr) return nullptr; + bin = grpc_base64_decode(b64, 1); + if (GPR_SLICE_IS_EMPTY(bin)) { + gpr_log(GPR_ERROR, "Invalid base64 for big num."); + return nullptr; + } + result = BN_bin2bn(GPR_SLICE_START_PTR(bin), GPR_SLICE_LENGTH(bin), nullptr); + gpr_slice_unref(bin); + return result; +} + +} // namespace +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.h b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.h new file mode 100644 index 00000000000..5bc7d6df204 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.h @@ -0,0 +1,56 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_LIB_AUTH_JWT_VALIDATOR_H_ +#define API_MANAGER_AUTH_LIB_AUTH_JWT_VALIDATOR_H_ + +#include +#include +#include + +#include "include/api_manager/utils/status.h" +#include "src/api_manager/auth.h" + +using ::google::api_manager::utils::Status; + +namespace google { +namespace api_manager { +namespace auth { + +class JwtValidator { + public: + // Create JwtValidator with JWT. + static std::unique_ptr Create(const char *jwt, size_t jwt_len); + + // Parse JWT. + // Returns Status::OK when parsing is successful, and fills user_info. + // Otherwise, produces a status error message. + virtual Status Parse(UserInfo *user_info) = 0; + + // Verify signature. + // Returns Status::OK when signature verification is successful. + // Otherwise, produces a status error message. + virtual Status VerifySignature(const char *pkey, size_t pkey_len) = 0; + + // Returns the expiration time of the JWT. + virtual std::chrono::system_clock::time_point &GetExpirationTime() = 0; + + virtual ~JwtValidator() {} +}; + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif /* API_MANAGER_AUTH_LIB_AUTH_JWT_VALIDATOR_H_ */ diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator_test.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator_test.cc new file mode 100644 index 00000000000..9401a28ac83 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator_test.cc @@ -0,0 +1,498 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/auth_jwt_validator.h" + +#include "gtest/gtest.h" +#include "src/api_manager/auth/lib/auth_token.h" + +#include + +namespace google { +namespace api_manager { +namespace auth { + +namespace { + +// Testing service account. +const char kUserId[] = + "628645741881-noabiu23f5a8m8ovd8ucv698lj78vv0l@developer.gserviceaccount." + "com"; +const char kAudience[] = "http://myservice.com/myapi"; +const char kAuthorizedParty[] = "authorized@party.com"; + +// Generated via service account, and immediately revoked. +const char kWrongPrivateKey[] = + "{" + "\"private_key_id\": \"8f0ed82105b173c7267be7185c91b1d74cf52cfd\"," + "\"private_key\": \"-----BEGIN PRIVATE " + "KEY-----" + "\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDRHBpr7CcRVvWc\\n4lKS" + "A+" + "8Qf+uL6V2O2pctSwRUyN8f+rjgs3RJBt2kBg63zsaVUi0X95YjkgF7NzRl\\nRk4WEZSPty4V+" + "xCuxzouPfqHv1EwFDvC/pCmtVuYMHih2RSrmpKKISVTCdeNZ+Gk\\nXo2o4fN4FD+i/" + "FUx+" + "t5vmUIIjai8UxTHjsqzVEX16CE1aQA6XyqL7subbNyOaMk1\\nhTQsfF3ANm9KgxGZpMeYhaty" + "n" + "gi6IdX9vhW2k14wjEHYP1QtJjcesDLlXnN+" + "1cq3\\na44VFC1YCM0rvSHE5VzQU1LF7VgQs1CnT/" + "bpDLNvPsYTiGPdnbAnxaxBMGiGMg5m\\n4Ch14eRpAgMBAAECggEAPkulA2nC6cOCQE6cUquhW" + "M" + "UDIxdOq/Qq/" + "W9PxwJglmJX\\nGXnctrS46thzIgcT2gA1NuKnc8lXb6Gulk0vjhuGqpnjvOCiw67OgmAsdqxk" + "P" + "3KH\\nqzuzVDbLJrep+G13XvgZl9TwDaDs+k9sRU913E4T/" + "j3qB2As8UrPYWfC6FFrZ06+\\n7KrT6iZQi9eQtqCxOA/" + "qVHyVsRPBcPEAc3SDahBOyrub7U60O1UVGpZC3ZROJ3X/\\nK1vVWZVBrAnn/TIJ/" + "gklTZX7nIf9u+uZK0vyb3WXZdDQ+/JlkFL0/" + "IcV+ocsQYJs\\nCU3cJduTej0YEpCsnwz3vVRrUBl1GmWmfUdI2o/qUQKBgQDyYvRk/" + "etdfqazAcqI\\n1J1n4PJRpDhwArDVw5+" + "SYUkcb85WK8UiouK8ckmq8m0BYTIb7E2pzYgin5ggtvwY\\npwIh/" + "m0K3L7bzCuNnQHetYyQsuD+dqdzKw7aGilpvPARZlF7pC33WrJU5YP/jbNu\\n32iK/" + "iIpUj+Tizcqs1KwckWxrwKBgQDc2rGSYbm/" + "ebgdFn3l5WatycmgIE4koUuT\\nF2k9Uk+" + "DdsRstepxHSIEpm0gfwxsXaeg4AvDldOltFybIuoUw4RIBHlfR/xy4WEB\\nsZtHRWj/QXLS/" + "4RT/LIy/zkjzvdq82hQvcV14hBmVDmQqbJaEH2/2tv9ZK1QtTAP\\ndRA/" + "MDvJZwKBgBKQB31wgMTxPRz6ZyNhfQiGjqg39maFnjtQtvjD4JB/" + "84Jf6cIE\\nTW73JbMky7pOUkMXLr9xURqttD3VJatRpvUpgfpR+3/ju/" + "Ylbw46QyCVwmtadOp6\\nArIrTL6fTJdYiab5ZNfLp1qfFSPOG07DZ0M1wTH+" + "7YWEJN5tS0jeB35bAoGBAKXs\\ns94LB7dAJj/" + "MRxfySjsk0BM6UhsZByNiQlGsxko5b4dRAOqsfYNK2c/BQ78iea7W\\nxF/" + "T76edorl2+LBS184XdmxMM/" + "DHPM899TANiL3FGRRGnc9PmT3RG8e4VZAHgQaw\\nHGrdRX7rpjf2FiWuIBuEvSRZgBCTn6DtT" + "S" + "B8B17fAoGBALaWesLP4Kfwd27jeqhq\\nHma90f5qB8mndTexPE5sekETUsdXLwsouLX3U94wW" + "V" + "1cyd9sg56EScnrqUhyYRWf\\nxXyHgmWDIO5xJxUbduII/" + "oytxpKSo5CL6ZZZapYS4rasF800LUGZhfCqMO50xAcH\\nauL/" + "1ayCE32Uwdo5Ws7rU8PE\\n-----END PRIVATE KEY-----\\n\"," + "\"client_email\": " + "\"628645741881-ff881mb5f6n4onn1d4hl4e9vb87ecl1e@developer.gserviceaccount." + "com\"," + "\"client_id\": " + "\"628645741881-ff881mb5f6n4onn1d4hl4e9vb87ecl1e.apps.googleusercontent." + "com\"," + "\"type\": \"service_account\"" + "}"; + +const char kOkPrivateKey[] = + "{\"private_key_id\": " + "\"b3319a147514df7ee5e4bcdee51350cc890cc89e\",\"private_key\": " + "\"-----BEGIN PRIVATE " + "KEY-----" + "\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCoOLtPHgOE289C\\nyXWh" + "/HFzZ49AVyz4vSZdijpMZLrgJj/ZaY629iVws1mOG511lVXZfzybQx/" + "BpIDX\\nrAT5GIoz2GqjkRjwE9ePnsIyJgDKIe5A+nXJrKMyCgTU/" + "aO+" + "nh6oX4FOKWUYm3lb\\nlG5e2L26p8y0JB1qAHwQLcw1G5T8p14uAHLeVLeijgs5h37viREFVlu" + "TbCeaZvsi\\nE/" + "06gtzX7v72pTW6GkPGYTonAFq7SYNLAydgNLgb8wvXt0L5kO0t3WLbhJNTDf0o\\nfSlxJ18Vs" + "vY20Rl015qbUMN2TSJS0lI9mWJQckEj+mPwz7Yyf+" + "gDyMG4jxgrAGpi\\nRkI3Uj3lAgMBAAECggEAOuaaVyp4KvXYDVeC07QTeUgCdZHQkkuQemIi5" + "YrDkCZ0\\nZsi6CsAG/f4eVk6/" + "BGPEioItk2OeY+wYnOuDVkDMazjUpe7xH2ajLIt3DZ4W2q+" + "k\\nv6WyxmmnPqcZaAZjZiPxMh02pkqCNmqBxJolRxp23DtSxqR6lBoVVojinpnIwem6\\nxyU" + "l65u0mvlluMLCbKeGW/" + "K9bGxT+" + "qd3qWtYFLo5C3qQscXH4L0m96AjGgHUYW6M\\nFfs94ETNfHjqICbyvXOklabSVYenXVRL24TO" + "KIHWkywhi1wW+" + "Q6zHDADSdDVYw5l\\nDaXz7nMzJ2X7cuRP9zrPpxByCYUZeJDqej0Pi7h7ZQKBgQDdI7Yb3xFX" + "pbuPd1VS\\ntNMltMKzEp5uQ7FXyDNI6C8+" + "9TrjNMduTQ3REGqEcfdWA79FTJq95IM7RjXX9Aae\\np6cLekyH8MDH/" + "SI744vCedkD2bjpA6MNQrzNkaubzGJgzNiZhjIAqnDAD3ljHI61\\nNbADc32SQMejb6zlEh8h" + "ssSsXwKBgQDCvXhTIO/EuE/y5Kyb/" + "4RGMtVaQ2cpPCoB\\nGPASbEAHcsRk+4E7RtaoDQC1cBRy+" + "zmiHUA9iI9XZyqD2xwwM89fzqMj5Yhgukvo\\nXMxvMh8NrTneK9q3/" + "M3mV1AVg71FJQ2oBr8KOXSEbnF25V6/ara2+EpH2C2GDMAo\\npgEnZ0/" + "8OwKBgFB58IoQEdWdwLYjLW/" + "d0oGEWN6mRfXGuMFDYDaGGLuGrxmEWZdw\\nfzi4CquMdgBdeLwVdrLoeEGX+XxPmCEgzg/" + "FQBiwqtec7VpyIqhxg2J9V2elJS9s\\nPB1rh9I4/QxRP/" + "oO9h9753BdsUU6XUzg7t8ypl4VKRH3UCpFAANZdW1tAoGAK4ad\\ntjbOYHGxrOBflB5wOiByf" + "1JBZH4GBWjFf9iiFwgXzVpJcC5NHBKL7gG3EFwGba2M\\nBjTXlPmCDyaSDlQGLavJ2uQar0P0" + "Y2MabmANgMkO/hFfOXBPtQQe6jAfxayaeMvJ\\nN0fQOylUQvbRTodTf2HPeG9g/" + "W0sJem0qFH3FrECgYEAnwixjpd1Zm/diJuP0+Lb\\nYUzDP+Afy78IP3mXlbaQ/" + "RVd7fJzMx6HOc8s4rQo1m0Y84Ztot0vwm9+S54mxVSo\\n6tvh9q0D7VLDgf+" + "2NpnrDW7eMB3n0SrLJ83Mjc5rZ+wv7m033EPaWSr/TFtc/" + "MaF\\naOI20MEe3be96HHuWD3lTK0\u003d\\n-----END PRIVATE " + "KEY-----\\n\",\"client_email\": " + "\"628645741881-noabiu23f5a8m8ovd8ucv698lj78vv0l@developer.gserviceaccount." + "com\",\"client_id\": " + "\"628645741881-noabiu23f5a8m8ovd8ucv698lj78vv0l.apps.googleusercontent." + "com\",\"type\": \"service_account\"}"; + +const char kPublicKeyJwk[] = + "{\"keys\": [{\"kty\": \"RSA\",\"alg\": \"RS256\",\"use\": " + "\"sig\",\"kid\": \"62a93512c9ee4c7f8067b5a216dade2763d32a47\",\"n\": " + "\"0YWnm_eplO9BFtXszMRQNL5UtZ8HJdTH2jK7vjs4XdLkPW7YBkkm_" + "2xNgcaVpkW0VT2l4mU3KftR-6s3Oa5Rnz5BrWEUkCTVVolR7VYksfqIB2I_" + "x5yZHdOiomMTcm3DheUUCgbJRv5OKRnNqszA4xHn3tA3Ry8VO3X7BgKZYAUh9fyZTFLlkeAh0-" + "bLK5zvqCmKW5QgDIXSxUTJxPjZCgfx1vmAfGqaJb-" + "nvmrORXQ6L284c73DUL7mnt6wj3H6tVqPKA27j56N0TB1Hfx4ja6Slr8S4EB3F1luYhATa1PKU" + "SH8mYDW11HolzZmTQpRoLV8ZoHbHEaTfqX_aYahIw\",\"e\": \"AQAB\"},{\"kty\": " + "\"RSA\",\"alg\": \"RS256\",\"use\": \"sig\",\"kid\": " + "\"b3319a147514df7ee5e4bcdee51350cc890cc89e\",\"n\": " + "\"qDi7Tx4DhNvPQsl1ofxxc2ePQFcs-L0mXYo6TGS64CY_" + "2WmOtvYlcLNZjhuddZVV2X88m0MfwaSA16wE-" + "RiKM9hqo5EY8BPXj57CMiYAyiHuQPp1yayjMgoE1P2jvp4eqF-" + "BTillGJt5W5RuXti9uqfMtCQdagB8EC3MNRuU_KdeLgBy3lS3oo4LOYd-" + "74kRBVZbk2wnmmb7IhP9OoLc1-7-9qU1uhpDxmE6JwBau0mDSwMnYDS4G_ML17dC-" + "ZDtLd1i24STUw39KH0pcSdfFbL2NtEZdNeam1DDdk0iUtJSPZliUHJBI_pj8M-2Mn_" + "oA8jBuI8YKwBqYkZCN1I95Q\",\"e\": \"AQAB\"}]}"; + +const char kPublicKeyX509[] = + "{\"62a93512c9ee4c7f8067b5a216dade2763d32a47\": \"-----BEGIN " + "CERTIFICATE-----" + "\\nMIIDYDCCAkigAwIBAgIIEzRv3yOFGvcwDQYJKoZIhvcNAQEFBQAwUzFRME8GA1UE\\nAxNI" + "NjI4NjQ1NzQxODgxLW5vYWJpdTIzZjVhOG04b3ZkOHVjdjY5OGxqNzh2djBs\\nLmFwcHMuZ29" + "vZ2xldXNlcmNvbnRlbnQuY29tMB4XDTE1MDkxMTIzNDg0OVoXDTI1\\nMDkwODIzNDg0OVowUz" + "FRME8GA1UEAxNINjI4NjQ1NzQxODgxLW5vYWJpdTIzZjVh\\nOG04b3ZkOHVjdjY5OGxqNzh2d" + "jBsLmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29t\\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A" + "MIIBCgKCAQEA0YWnm/eplO9BFtXszMRQ\\nNL5UtZ8HJdTH2jK7vjs4XdLkPW7YBkkm/" + "2xNgcaVpkW0VT2l4mU3KftR+6s3Oa5R\\nnz5BrWEUkCTVVolR7VYksfqIB2I/" + "x5yZHdOiomMTcm3DheUUCgbJRv5OKRnNqszA\\n4xHn3tA3Ry8VO3X7BgKZYAUh9fyZTFLlkeA" + "h0+bLK5zvqCmKW5QgDIXSxUTJxPjZ\\nCgfx1vmAfGqaJb+" + "nvmrORXQ6L284c73DUL7mnt6wj3H6tVqPKA27j56N0TB1Hfx4\\nja6Slr8S4EB3F1luYhATa1" + "PKUSH8mYDW11HolzZmTQpRoLV8ZoHbHEaTfqX/" + "aYah\\nIwIDAQABozgwNjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/" + "wQEAwIHgDAWBgNVHSUB\\nAf8EDDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQUFAAOCAQEAP4gk" + "DCrPMI27/" + "QdN\\nwW0mUSFeDuM8VOIdxu6d8kTHZiGa2h6nTz5E+" + "twCdUuo6elGit3i5H93kFoaTpex\\nj/eDNoULdrzh+cxNAbYXd8XgDx788/" + "jm06qkwXd0I5s9KtzDo7xxuBCyGea2LlpM\\n2HOI4qFunjPjFX5EFdaT/Rh+qafepTKrF/" + "GQ7eGfWoFPbZ29Hs5y5zATJCDkstkY\\npnAya8O8I+" + "tfKjOkcra9nOhtck8BK94tm3bHPdL0OoqKynnoRCJzN5KPlSGqR/h9\\nSMBZzGtDOzA2sX/" + "8eyU6Rm4MV6/1/53+J6EIyarR5g3IK1dWmz/YT/YMCt6LhHTo\\n3yfXqQ==\\n-----END " + "CERTIFICATE-----\\n\",\"b3319a147514df7ee5e4bcdee51350cc890cc89e\": " + "\"-----BEGIN " + "CERTIFICATE-----" + "\\nMIIDYDCCAkigAwIBAgIICjE9gZxAlu8wDQYJKoZIhvcNAQEFBQAwUzFRME8GA1UE\\nAxNI" + "NjI4NjQ1NzQxODgxLW5vYWJpdTIzZjVhOG04b3ZkOHVjdjY5OGxqNzh2djBs\\nLmFwcHMuZ29" + "vZ2xldXNlcmNvbnRlbnQuY29tMB4XDTE1MDkxMzAwNTAyM1oXDTI1\\nMDkxMDAwNTAyM1owUz" + "FRME8GA1UEAxNINjI4NjQ1NzQxODgxLW5vYWJpdTIzZjVh\\nOG04b3ZkOHVjdjY5OGxqNzh2d" + "jBsLmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29t\\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A" + "MIIBCgKCAQEAqDi7Tx4DhNvPQsl1ofxx\\nc2ePQFcs+L0mXYo6TGS64CY/" + "2WmOtvYlcLNZjhuddZVV2X88m0MfwaSA16wE+" + "RiK\\nM9hqo5EY8BPXj57CMiYAyiHuQPp1yayjMgoE1P2jvp4eqF+" + "BTillGJt5W5RuXti9\\nuqfMtCQdagB8EC3MNRuU/" + "KdeLgBy3lS3oo4LOYd+74kRBVZbk2wnmmb7IhP9OoLc\\n1+7+" + "9qU1uhpDxmE6JwBau0mDSwMnYDS4G/" + "ML17dC+ZDtLd1i24STUw39KH0pcSdf\\nFbL2NtEZdNeam1DDdk0iUtJSPZliUHJBI/" + "pj8M+2Mn/" + "oA8jBuI8YKwBqYkZCN1I9\\n5QIDAQABozgwNjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/" + "wQEAwIHgDAWBgNVHSUB\\nAf8EDDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQUFAAOCAQEAHSPR" + "7fDAWyZ825IZ\\n86hEsQZCvmC0QbSzy62XisM/uHUO75BRFIAvC+zZAePCcNo/" + "nh6FtEM19wZpxLiK\\n0m2nqDMpRdw3Qt6BNhjJMozTxA2Xdipnfq+fGpa+" + "bMkVpnRZ53qAuwQpaKX6vagr\\nj83Bdx2b5WPQCg6xrQWsf79Vjj2U1hdw7+" + "klcF7tLef1p8qA/ezcNXmcZ4BpbpaO\\nN9M4/kQOA3Y2F3ISAaOJzCB25F259whjW+Uuqd/" + "L9Lb4gPPSUMSKy7Zy4Sn4il1U\\nFc94Mi9j13oeGvLOduNOStGu5XROIxDtCEjjn2y2SL2bPw" + "0qAlIzBeniiApkmYw/\\no6OLrg==\\n-----END CERTIFICATE-----\\n\"}"; + +// Token generated with the following header and payload and kOkPrivateKey. +// Header (kid is not specified): +// { +// "alg": "RS256", +// "typ": "JWT" +// } +// Payload: +// { +// "iss": "628645741881-" +// "noabiu23f5a8m8ovd8ucv698lj78vv0l@developer.gserviceaccount.com", +// "sub": "628645741881-" +// "noabiu23f5a8m8ovd8ucv698lj78vv0l@developer.gserviceaccount.com", +// "aud": "http://myservice.com/myapi" +// } +const char kTokenNoKid[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1M" + "jNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20" + "iLCJzdWIiOiI2Mjg2NDU3NDE4ODEtbm9hYml1MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZ" + "GV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20iLCJhdWQiOiJodHRwOi8vbXlzZXJ2aWNlLmN" + "vbS9teWFwaSJ9.gq_4ucjddQDjYK5FJr_kXmMo2fgSEB6Js1zopcQLVpCKFDNb-TQ97go0wuk5" + "_vlSp_8I2ImrcdwYbAKqYCzcdyBXkAYoHCGgmY-v6MwZFUvrIaDzR_M3rmY8sQ8cdN3MN6ZRbB" + "6opHwDP1lUEx4bZn_ZBjJMPgqbIqGmhoT1UpfPF6P1eI7sXYru-4KVna0STOynLl3d7JYb7E-8" + "ifcjUJLhat8JR4zR8i4-zWjn6d6j_NI7ZvMROnao77D9YyhXv56zfsXRatKzzYtxPlQMz4AjP-" + "bUHfbHmhiIOOAeEKFuIVUAwM17j54M6VQ5jnAabY5O-ermLfwPiXvNt2L2SA=="; + +// Token that has one dot. +const char kTokenOneDot[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1M"; + +// Token that does not have signature part. +const char kTokenNoSign[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1M" + "jNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20" + "iLCJzdWIiOiI2Mjg2NDU3NDE4ODEtbm9hYml1MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZ" + "GV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20iLCJhdWQiOiJodHRwOi8vbXlzZXJ2aWNlLmN" + "vbS9teWFwaSJ9."; + +// Token without "iss" in payload. +const char kTokenNoIss[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiI2Mjg2NDU3NDE4ODEtbm9hYml1" + "MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb" + "20iLCJhdWQiOiJodHRwOi8vbXlzZXJ2aWNlLmNvbS9teWFwaSIsImV4cCI6MjQ2MjMyNDAyMH" + "0=.m08o3yaV3Xhth0MdI8TviotIbew1TcBfepYMtAbwSzBsSFtTXUCpmyph4ld-ji7koSXipx" + "Sssq43qqSX_ywOBq4mfChmDZL-DLvJlGnIkF_ec2vn9FgOUFivPGcrPSkyOBcKGPj4JA764oO" + "Sg1VMN34sB9qc57qQHQBL3SYv1exyPOZksf9Y-UTLdbZJp0ACDeqEbvrOKhkylUrCtMtJZe0l" + "b8h4sSgZ5fiNev4eyX5M8WB6dCFmehAUabYop8_3XSy1Ufxo_g-Is9CXKStSOWxRlea7o_rNO" + "P1yrXjS-A7T-fCvqrEYYn4bZh7gYh4Q_Gu0SHlzwviYFzcfLpti7A=="; + +// Token without "sub" in payload. +const char kTokenNoSub[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1" + "MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb" + "20iLCJhdWQiOiJodHRwOi8vbXlzZXJ2aWNlLmNvbS9teWFwaSIsImV4cCI6MjQ2MjMyNDAyMH" + "0=.Rq5UHXlRxzNvklEacP9v2YfwoztSTdhN0A5ecGrH06mfqvsmqZ6OEHsTDUIFUE85BkSA71" + "vjDhkMQYxi2yn8TaJ9ELYFKggfIhuHX5BGF5PgdMZnIR3JDHDXQmOBLwGtBPtC0VUCbDnzG_P" + "j5Syd4yU0_4FbIl8B2iw6rlxER1JL38Hm7K4pFUPhEgTM1suM_bzMRPLOmInw4neJDhuZOaG0" + "7d__D1Z8Z6QCiXTEEMTjswT8cYxzeLp8EOdqKdiHIn6ynawJozjha7NS1GxA5-M2anQnzLf1y" + "h2Uc9Cg5Kx0PgYzjk5GkoWWl8zOCekgvzyLFNSttO3013WtUlw7dA=="; + +// Token without "aud" in payload. +const char kTokenNoAud[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1" + "MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb" + "20iLCJzdWIiOiI2Mjg2NDU3NDE4ODEtbm9hYml1MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MG" + "xAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20iLCJleHAiOjI0NjIzMjQwMjB9.IlOJAg" + "rpgO9q6A3D9uRqNQ_NwU83PuZn-YaB5LqGfLvAL6bd-JPuwN_RRlWRtj6SPowm7MGXpEw8PvR" + "vj1IV8RRhLzSfr_gA_6gAiYpRs-_wD2FGCl3wzhnBL387j9m8M04xUdt2-a1WfXBgPWpV-rZa" + "Q_jqi_pad26AXZtW-iit7fy2K7viH2PKxVQgFjjjtqTuLNoFJjH1fTd3tGZr0ynF4UcUjRWWg" + "QROJ9A3YNRSGM3ejfsXGCFUuREJ8G7X7dxukd19fnuX4u6iPziXqqFWtaJoaBYKkX4SQ254bl" + "CKvuNLFiokcJGxGyZ6STLUCB5cBvlQjzfD3ZSAUBKGtA=="; + +// Token with multiple audiences. +const char kTokenMultiAud[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1" + "MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb" + "20iLCJzdWIiOiI2Mjg2NDU3NDE4ODEtbm9hYml1MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MG" + "xAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20iLCJhdWQiOlsiaHR0cDovL215c2Vydml" + "jZS5jb20vbXlhcGkiLCJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20iXSwiZXhwIjoyNDYy" + "MzI0MDIwfQ==.Arx-W6LPwK74u9Wndg85TPGDGuJysyGe3ApqcA905kR0fzKYS8sH8lKdzbk6" + "xGYS1VMNIWpTrjM9Nf28ZB_r-1j_iYfjS4VREAbFlv4MGionYQDI7eDIpeh-CqeKFs3yPRJHV" + "nviZ5wRE-16qT3wvxdib3oIimWrnq6MVLL6WTXvz4OLTAJr74Oak88fF48KcCZKQ9Ffg1DF81" + "qGuAjFES_t6LmmSgi31nI5R6frO98k0DQgAv2m16qC2CidhzStwFIaFVYWBaOwkoB-RQaH8Zr" + "wgOlvF_7CgLuva1bhuYfDDc8jjgqU-vGKdctc87KK4tkc_OQiwe-RLH6o8sF5vw=="; + +// Token with the "azp" claim. +// Payload: +// { +// "iss": "628645741881-" +// "noabiu23f5a8m8ovd8ucv698lj78vv0l@developer.gserviceaccount.com", +// "sub": "628645741881-" +// "noabiu23f5a8m8ovd8ucv698lj78vv0l@developer.gserviceaccount.com", +// "aud": "http://myservice.com/myapi", +// "azp": "authorized@party.com" +// } +const char kTokenWithAzp[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI2Mjg2NDU3NDE4ODEtbm9hYml1" + "MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MGxAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb" + "20iLCJzdWIiOiI2Mjg2NDU3NDE4ODEtbm9hYml1MjNmNWE4bThvdmQ4dWN2Njk4bGo3OHZ2MG" + "xAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20iLCJhdWQiOiJodHRwOi8vbXlzZXJ2aWN" + "lLmNvbS9teWFwaSIsImF6cCI6ImF1dGhvcml6ZWRAcGFydHkuY29tIn0.I48qxD87l2k6UBw-" + "1NiP-JgBAyelOFSs3_dUlEeyvdwkeahcgyHBu7W8lOdESHci88uBof-6DjLgIJGQv0-9xGRuY" + "EfnBDtUkPjVDk_9NCipoi2X2HVrcLYY0AJFQnd57UL3bqDudY8-lx4QXxsczNMba6eyInibdw" + "B3VAsgcZZAqMGu91i-d12ayNodrKurCGY7tH_9bf7kxhtcB-YDAepLnaLZ4pOjTZe4Ap20G2p" + "Z_Wbu2Pc9Pq0kPHQo-e9gKCt403cI8MaVxDQkSolpjiVg29rul5m7k359q_XVexvsboHRVP2-" + "no5Y_Ge3KbA7XosymMYlal0J0iYHQuV_sw"; + +class JwtValidatorTest : public ::testing::Test { + public: + void SetUp() {} +}; + +// Test JwtValidator class for a given JWT (token) and public key (pkey). +void TestTokenWithPubkey(char *token, const char *pkey) { + ASSERT_TRUE(token != nullptr); + UserInfo user_info; + + std::unique_ptr validator = + JwtValidator::Create(token, strlen(token)); + Status status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(status.message(), ""); + ASSERT_EQ(kUserId, user_info.id); + ASSERT_TRUE(user_info.audiences.end() != user_info.audiences.find(kAudience)); + ASSERT_EQ(1U, user_info.audiences.size()); + ASSERT_EQ(kUserId, user_info.issuer); + + status = validator->VerifySignature(pkey, strlen(pkey)); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(status.message(), ""); + + // Wrong length. + validator = JwtValidator::Create(token + 1, strlen(token)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Wrong sig. + validator = JwtValidator::Create(token, strlen(token) - 1); + status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + status = validator->VerifySignature(pkey, strlen(pkey)); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_SIGNATURE") << status.message(); + + // Wrong key length. + validator = JwtValidator::Create(token, strlen(token)); + status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + status = validator->VerifySignature(pkey, strlen(pkey) - 1); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "KEY_RETRIEVAL_ERROR") << status.message(); + + // Empty key. + validator = JwtValidator::Create(token, strlen(token)); + status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + status = validator->VerifySignature("", 0); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); +} + +TEST_F(JwtValidatorTest, OkTokenX509) { + // TODO: update esp_get_auth_token to generate token with multiple + // audiences. + char *token = esp_get_auth_token(kOkPrivateKey, kAudience); + TestTokenWithPubkey(token, kPublicKeyX509); + + esp_grpc_free(token); +} + +TEST_F(JwtValidatorTest, OkTokenJwk) { + char *token = esp_get_auth_token(kOkPrivateKey, kAudience); + TestTokenWithPubkey(token, kPublicKeyJwk); + + esp_grpc_free(token); +} + +// This test uses a JWT with no "kid" specided in its header. +TEST_F(JwtValidatorTest, TokenNoKid) { + // Use public keys of JWK format to verify token signature. + TestTokenWithPubkey(const_cast(kTokenNoKid), kPublicKeyJwk); + + // Use public keys of X509 format to verify token signature. + TestTokenWithPubkey(const_cast(kTokenNoKid), kPublicKeyX509); +} + +TEST_F(JwtValidatorTest, ParseToken) { + // Multiple audiences. + UserInfo user_info; + std::unique_ptr validator = + JwtValidator::Create(kTokenMultiAud, strlen(kTokenMultiAud)); + Status status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(user_info.audiences.size(), 2U); + + // Token with only one dot. + const char *token = "a1234.b5678"; // should have 2 dots. + validator = JwtValidator::Create(token, strlen(token)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Token with three dots. + token = "a1234.b5678.c7890.d1234"; // should have 2 dots. + validator = JwtValidator::Create(token, strlen(token)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Token without dot. + token = "a1234"; // should have 2 dots. + validator = JwtValidator::Create(token, strlen(token)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Empty token. + token = ""; // should have 2 dots. + validator = JwtValidator::Create(token, strlen(token)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Token truncated in the second part. + validator = JwtValidator::Create(kTokenOneDot, strlen(kTokenOneDot)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Token without the last signature part. + validator = JwtValidator::Create(kTokenNoSign, strlen(kTokenNoSign)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Token without "iss" field in payload. + validator = JwtValidator::Create(kTokenNoIss, strlen(kTokenNoIss)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Token without "sub" field in payload. + validator = JwtValidator::Create(kTokenNoSub, strlen(kTokenNoSub)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); + + // Token without "aud" field in payload. + validator = JwtValidator::Create(kTokenNoAud, strlen(kTokenNoAud)); + status = validator->Parse(&user_info); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "BAD_FORMAT") << status.message(); +} + +TEST_F(JwtValidatorTest, WrongKey) { + char *token = esp_get_auth_token(kWrongPrivateKey, kAudience); + ASSERT_TRUE(token != nullptr); + UserInfo user_info; + + std::unique_ptr validator = + JwtValidator::Create(token, strlen(token)); + Status status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + + status = validator->VerifySignature(kPublicKeyX509, strlen(kPublicKeyX509)); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "KEY_RETRIEVAL_ERROR") << status.message(); + + status = validator->VerifySignature(kPublicKeyJwk, strlen(kPublicKeyJwk)); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.message(), "KEY_RETRIEVAL_ERROR") << status.message(); +} + +TEST_F(JwtValidatorTest, TokenWithAuthorizedParty) { + UserInfo user_info; + + // Token without "azp" claim. + std::unique_ptr validator = + JwtValidator::Create(kTokenNoKid, strlen(kTokenNoKid)); + Status status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(user_info.authorized_party.empty()); + + // Token with "azp" claim. + validator = JwtValidator::Create(kTokenWithAzp, strlen(kTokenWithAzp)); + status = validator->Parse(&user_info); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(user_info.authorized_party, kAuthorizedParty); +} + +} // namespace + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc new file mode 100644 index 00000000000..36d9c264aad --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc @@ -0,0 +1,231 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/auth_token.h" +#include "src/api_manager/auth/lib/json_util.h" + +#include +#include +#include +#include + +extern "C" { +#include "grpc/grpc.h" +#include "grpc/support/alloc.h" +#include "grpc/support/log.h" +#include "grpc/support/string_util.h" +#include "grpc/support/sync.h" +} + +#include "grpc_internals.h" + +using std::string; + +namespace google { +namespace api_manager { +namespace auth { + +namespace { + +#define GRPC_AUTH_JSON_TYPE_INVALID "invalid" + +const gpr_timespec TOKEN_LIFETIME = {3600, 0, GPR_TIMESPAN}; + +char *GenerateTokenSymmetricKey(const grpc_json *json, const char *audience); + +char *GenerateJoseHeader(const char *algorithm); +char *GenerateJwtClaim(const char *issuer, const char *subject, + const char *audience, gpr_timespec token_lifetime); +char *GenerateSignatueHs256(const char *data, const char *key); +string DotConcat(const string &str1, const string &str2); + +} // namespace + +// TODO: this function can return a string instead of char* that need to be +// freed. +char *esp_get_auth_token(const char *json_secret, const char *audience) { + char *scratchpad = gpr_strdup(json_secret); + grpc_json *json = grpc_json_parse_string(scratchpad); + char *token = nullptr; + + if (GetStringValue(json, "client_secret") != nullptr) { // Symmetric key. + token = GenerateTokenSymmetricKey(json, audience); + } else { + grpc_auth_json_key json_key = + grpc_auth_json_key_create_from_string(json_secret); + if (strcmp(json_key.type, GRPC_AUTH_JSON_TYPE_INVALID) != 0) { + token = grpc_jwt_encode_and_sign(&json_key, audience, TOKEN_LIFETIME, + nullptr); + grpc_auth_json_key_destruct(&json_key); + } + } + + if (json != nullptr) grpc_json_destroy(json); + gpr_free(scratchpad); + return token; +} + +void esp_grpc_free(char *token) { gpr_free(token); } + +// Parses a JSON service account auth token in the following format: +// { +// "access_token":" ... ", +// "expires_in":100, +// "token_type":"Bearer" +// } +// Returns true on success, false otherwise. On success, *token is set to the +// malloc'd auth token (value of 'access_token' JSON property) and *expires is +// set to the value of 'expires_in' property (token expiration in seconds. +bool esp_get_service_account_auth_token(char *input, size_t size, char **token, + int *expires) { + bool result = false; // fail by default + grpc_json *json = grpc_json_parse_string_with_len(input, size); + if (json) { + const char *access_token = GetStringValue(json, "access_token"); + const char *expires_in = GetNumberValue(json, "expires_in"); + + if (access_token && expires_in) { + *token = strdup(access_token); + if (*token) { + *expires = atoi(expires_in); + result = true; + } + } + grpc_json_destroy(json); + } + return result; +} + +namespace { + +char *GenerateTokenSymmetricKey(const grpc_json *json, const char *audience) { + const char *issuer = GetStringValue(json, "issuer"); + if (issuer == nullptr) { + // TODO: move to common logging. + gpr_log(GPR_ERROR, "Missing issuer"); + return nullptr; + } + const char *subject = GetStringValue(json, "subject"); + if (subject == nullptr) { + gpr_log(GPR_ERROR, "Missing subject"); + return nullptr; + } + + char *header = GenerateJoseHeader("HS256"); + if (header == nullptr) { + gpr_log(GPR_ERROR, "Unable to create JOSE header"); + return nullptr; + } + char *claims = GenerateJwtClaim(issuer, subject, audience, TOKEN_LIFETIME); + if (claims == nullptr) { + gpr_log(GPR_ERROR, "Unable to create JWT claims payload"); + return nullptr; + } + string signed_data = DotConcat(header, claims); + gpr_free(header); + gpr_free(claims); + + const char *secret = GetStringValue(json, "client_secret"); + char *signature = GenerateSignatueHs256(signed_data.c_str(), secret); + if (signature == nullptr) { + gpr_log(GPR_ERROR, "Unable to compute HS256 signature"); + return nullptr; + } + + string result = DotConcat(signed_data, signature); + gpr_free(signature); + // Match return type of esp_get_auth_token. + return gpr_strdup(result.c_str()); +} + +char *GenerateJoseHeader(const char *algorithm) { + grpc_json json_top; + memset(&json_top, 0, sizeof(json_top)); + json_top.type = GRPC_JSON_OBJECT; + + grpc_json json_algorithm, json_type; + + FillChild(&json_algorithm, nullptr, &json_top, "alg", algorithm, + GRPC_JSON_STRING); + FillChild(&json_type, &json_algorithm, &json_top, "typ", "JWT", + GRPC_JSON_STRING); + + char *json_str = grpc_json_dump_to_string(&json_top, 0); + char *result = grpc_base64_encode(json_str, strlen(json_str), 1, 0); + gpr_free(json_str); + return result; +} + +char *GenerateJwtClaim(const char *issuer, const char *subject, + const char *audience, gpr_timespec token_lifetime) { + grpc_json json_top; + memset(&json_top, 0, sizeof(json_top)); + json_top.type = GRPC_JSON_OBJECT; + + gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); + gpr_timespec expiration = gpr_time_add(now, token_lifetime); + char now_str[GPR_LTOA_MIN_BUFSIZE]; + char expiration_str[GPR_LTOA_MIN_BUFSIZE]; + gpr_ltoa(now.tv_sec, now_str); + gpr_ltoa(expiration.tv_sec, expiration_str); + + grpc_json json_issuer, json_subject, json_audience, json_now, json_expiration; + FillChild(&json_issuer, nullptr, &json_top, "iss", issuer, GRPC_JSON_STRING); + FillChild(&json_subject, &json_issuer, &json_top, "sub", subject, + GRPC_JSON_STRING); + FillChild(&json_audience, &json_subject, &json_top, "aud", audience, + GRPC_JSON_STRING); + FillChild(&json_now, &json_audience, &json_top, "iat", now_str, + GRPC_JSON_NUMBER); + FillChild(&json_expiration, &json_now, &json_top, "exp", expiration_str, + GRPC_JSON_NUMBER); + + char *json_str = grpc_json_dump_to_string(&json_top, 0); + char *result = grpc_base64_encode(json_str, strlen(json_str), 1, 0); + gpr_free(json_str); + return result; +} + +char *GenerateSignatueHs256(const char *data, const char *key) { + gpr_slice key_buffer = grpc_base64_decode(key, 1); + if (GPR_SLICE_IS_EMPTY(key_buffer)) { + gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); + return nullptr; + } + + unsigned char res[256 / 8]; + unsigned int res_len = 0; + HMAC(EVP_sha256(), GPR_SLICE_START_PTR(key_buffer), + GPR_SLICE_LENGTH(key_buffer), + reinterpret_cast(data), strlen(data), res, + &res_len); + gpr_slice_unref(key_buffer); + if (res_len == 0) { + gpr_log(GPR_ERROR, "Cannot compute HMAC from secret."); + return nullptr; + } + return grpc_base64_encode(res, res_len, 1, 0); +} + +string DotConcat(const string &str1, const string &str2) { + return str1 + "." + str2; +} + +} // namespace + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_token.h b/contrib/endpoints/src/api_manager/auth/lib/auth_token.h new file mode 100644 index 00000000000..f17358d96b8 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_token.h @@ -0,0 +1,47 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_LIB_AUTH_TOKEN_H_ +#define API_MANAGER_AUTH_LIB_AUTH_TOKEN_H_ + +#include + +namespace google { +namespace api_manager { +namespace auth { + +// Parse a json secret and generate auth_token +// Returned pointer need to be freed by esp_grpc_free +char *esp_get_auth_token(const char *json_secret, const char *audience); + +// Free a buffer allocated by gRPC library. +void esp_grpc_free(char *token); + +// Parses a JSON service account auth token in the following format: +// { +// "access_token":" ... ", +// "expires_in":100, +// "token_type":"Bearer" +// } +// Returns true on success, false otherwise. On success, *token is set to the +// malloc'd auth token (value of 'access_token' JSON property) and *expires is +// set to the value of 'expires_in' property (token expiration in seconds. +bool esp_get_service_account_auth_token(char *input, size_t size, char **token, + int *expires); + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_AUTH_LIB_AUTH_TOKEN_H_ diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_token_test.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_token_test.cc new file mode 100644 index 00000000000..a08d67133ec --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_token_test.cc @@ -0,0 +1,177 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/auth_token.h" +#include "gtest/gtest.h" + +#include + +namespace google { +namespace api_manager { +namespace auth { +namespace { + +class AuthTokenTest : public ::testing::Test { + protected: + char *good_token_; + char *empty_token_; + char *missing_access_token_; + char *missing_expires_in_token_; + char *bad_access_token_type_; + char *bad_expires_in_type_; + + void SetUp() { + good_token_ = strdup(good_token); + empty_token_ = strdup(empty_token); + missing_access_token_ = strdup(missing_access_token); + missing_expires_in_token_ = strdup(missing_expires_in_token); + bad_access_token_type_ = strdup(bad_access_token_type); + bad_expires_in_type_ = strdup(bad_expires_in_type); + } + + void TearDown() { + free(good_token_); + free(empty_token_); + free(missing_access_token_); + free(missing_expires_in_token_); + free(bad_access_token_type_); + free(bad_expires_in_type_); + } + + private: + static const char good_token[]; + static const char empty_token[]; + static const char missing_access_token[]; + static const char missing_expires_in_token[]; + static const char bad_access_token_type[]; + static const char bad_expires_in_type[]; +}; + +const char AuthTokenTest::good_token[] = + "{" + " \"access_token\":" + " \"ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_QtAS08i85nHq39HE3C2LTrCARA\"," + " \"expires_in\":3599," + " \"token_type\":\"Bearer\"" + "}"; + +const char AuthTokenTest::empty_token[] = "{ }"; + +const char AuthTokenTest::missing_access_token[] = + "{" + " \"expires_in\":3599," + " \"token_type\":\"Bearer\"" + "}"; + +const char AuthTokenTest::missing_expires_in_token[] = + "{" + " \"access_token\":" + " \"ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_QtAS08i85nHq39HE3C2LTrCARA\"," + " \"token_type\":\"Bearer\"" + "}"; + +const char AuthTokenTest::bad_access_token_type[] = + "{" + " \"access_token\":1234," + " \"expires_in\":3599," + " \"token_type\":\"Bearer\"" + "}"; + +const char AuthTokenTest::bad_expires_in_type[] = + "{" + " \"access_token\":" + " \"ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_QtAS08i85nHq39HE3C2LTrCARA\"," + " \"expires_in\":\"3599\"," + " \"token_type\":\"Bearer\"" + "}"; + +TEST_F(AuthTokenTest, ParseServiceAccountToken) { + char *token = nullptr; + int expires = 0; + + ASSERT_TRUE(esp_get_service_account_auth_token( + good_token_, strlen(good_token_), &token, &expires)); + ASSERT_STREQ("ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_QtAS08i85nHq39HE3C2LTrCARA", + token); + ASSERT_EQ(3599, expires); +} + +TEST_F(AuthTokenTest, ParseEmptyServiceAccountToken) { + char *token = nullptr; + int expires = 0; + + ASSERT_FALSE(esp_get_service_account_auth_token( + empty_token_, strlen(empty_token_), &token, &expires)); + ASSERT_EQ(nullptr, token); + ASSERT_EQ(0, expires); +} + +TEST_F(AuthTokenTest, ParseMissingAccessToken) { + char *token = nullptr; + int expires = 0; + + ASSERT_FALSE(esp_get_service_account_auth_token( + missing_access_token_, strlen(missing_access_token_), &token, &expires)); + ASSERT_EQ(nullptr, token); + ASSERT_EQ(0, expires); +} + +TEST_F(AuthTokenTest, ParseMissingExpiresIn) { + char *token = nullptr; + int expires = 0; + + ASSERT_FALSE(esp_get_service_account_auth_token( + missing_expires_in_token_, strlen(missing_expires_in_token_), &token, + &expires)); + ASSERT_EQ(nullptr, token); + ASSERT_EQ(0, expires); +} + +TEST_F(AuthTokenTest, ParseBadAccessTokenType) { + char *token = nullptr; + int expires = 0; + + ASSERT_FALSE(esp_get_service_account_auth_token( + bad_access_token_type_, strlen(bad_access_token_type_), &token, + &expires)); + ASSERT_EQ(nullptr, token); + ASSERT_EQ(0, expires); +} + +TEST_F(AuthTokenTest, ParseBadExpiresInType) { + char *token = nullptr; + int expires = 0; + + ASSERT_FALSE(esp_get_service_account_auth_token( + bad_expires_in_type_, strlen(bad_expires_in_type_), &token, &expires)); + ASSERT_EQ(nullptr, token); + ASSERT_EQ(0, expires); +} + +TEST_F(AuthTokenTest, ParseNullServiceAccountToken) { + char *token = nullptr; + int expires = 0; + + ASSERT_FALSE( + esp_get_service_account_auth_token(nullptr, 0, &token, &expires)); + ASSERT_EQ(nullptr, token); + ASSERT_EQ(0, expires); +} + +} // namespace +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/base64.cc b/contrib/endpoints/src/api_manager/auth/lib/base64.cc new file mode 100644 index 00000000000..1b24d6ab830 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/base64.cc @@ -0,0 +1,48 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/base64.h" + +#include +#include + +#include "src/api_manager/auth/lib/grpc_internals.h" + +namespace google { +namespace api_manager { +namespace auth { + +char *esp_base64_encode(const void *data, size_t data_size, bool url_safe, + bool multiline, bool padding) { + char *result = + grpc_base64_encode(data, data_size, url_safe ? 1 : 0, multiline ? 1 : 0); + if (result == nullptr) { + return result; + } + // grpc_base64_encode may have added padding. If not needed, remove them. + if (!padding) { + size_t len = strlen(result); + while (len > 0 && result[len - 1] == '=') { + len--; + } + result[len] = '\0'; + } + return result; +} + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/base64.h b/contrib/endpoints/src/api_manager/auth/lib/base64.h new file mode 100644 index 00000000000..54d6c756a8f --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/base64.h @@ -0,0 +1,33 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_LIB_BASE64_H_ +#define API_MANAGER_AUTH_LIB_BASE64_H_ + +#include + +namespace google { +namespace api_manager { +namespace auth { + +// Base64 encode a data. +// Returned buffer should be freed by esp_grpc_free. +char *esp_base64_encode(const void *data, size_t data_size, bool url_safe, + bool multiline, bool padding); + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif /* API_MANAGER_AUTH_LIB_BASE64_H_ */ diff --git a/contrib/endpoints/src/api_manager/auth/lib/base64_test.cc b/contrib/endpoints/src/api_manager/auth/lib/base64_test.cc new file mode 100644 index 00000000000..0a889e94e80 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/base64_test.cc @@ -0,0 +1,60 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/base64.h" +#include "gtest/gtest.h" +#include "src/api_manager/auth/lib/auth_token.h" + +namespace google { +namespace api_manager { +namespace auth { + +struct TestVector { + const char *data; + const char *encoded_padding; + const char *encoded_no_padding; +}; + +/* Test vectors from RFC 4648. */ +static const TestVector test_vectors[] = { + {"", "", ""}, + {"f", "Zg==", "Zg"}, + {"fo", "Zm8=", "Zm8"}, + {"foo", "Zm9v", "Zm9v"}, + {"foob", "Zm9vYg==", "Zm9vYg"}, + {"fooba", "Zm9vYmE=", "Zm9vYmE"}, + {"foobar", "Zm9vYmFy", "Zm9vYmFy"}, +}; + +TEST(EspBase64Test, EncodeTest) { + for (const auto &t : test_vectors) { + char *encoded_padding = + esp_base64_encode(t.data, strlen(t.data), true, false, true); + ASSERT_NE(nullptr, encoded_padding); + ASSERT_STREQ(t.encoded_padding, encoded_padding); + esp_grpc_free(encoded_padding); + + char *encoded_no_padding = + esp_base64_encode(t.data, strlen(t.data), true, false, false); + ASSERT_NE(nullptr, encoded_no_padding); + ASSERT_STREQ(t.encoded_no_padding, encoded_no_padding); + esp_grpc_free(encoded_no_padding); + } +} + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h new file mode 100644 index 00000000000..66074a12776 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h @@ -0,0 +1,200 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ +#define API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ + +// This header file contains definitions for all grpc internal +// dependencies. The code that depends on grpc internals should +// include this file instead of including the original headers +// in grpc. + +// This header file is for internal use only since it declares grpc +// internals that auth depends on. A public header file should not +// include any internal grpc header files. + +// TODO: Remove this dependency on gRPC internal implementation details, +// or work with gRPC team to support this functionality as a public API +// surface. + +extern "C" { + +#include +#include + +////////////////////////////////////////////////////// +// definitions from grpc/src/core/json/json_common.h +////////////////////////////////////////////////////// + +/* The various json types. */ +typedef enum { + GRPC_JSON_OBJECT, + GRPC_JSON_ARRAY, + GRPC_JSON_STRING, + GRPC_JSON_NUMBER, + GRPC_JSON_TRUE, + GRPC_JSON_FALSE, + GRPC_JSON_NULL, + GRPC_JSON_TOP_LEVEL +} grpc_json_type; + +////////////////////////////////////////////////////// +// definitions from grpc/src/core/json/json.h +////////////////////////////////////////////////////// + +/* A tree-like structure to hold json values. The key and value pointers + * are not owned by it. + */ +typedef struct grpc_json { + struct grpc_json *next; + struct grpc_json *prev; + struct grpc_json *child; + struct grpc_json *parent; + + grpc_json_type type; + const char *key; + const char *value; +} grpc_json; + +/* The next two functions are going to parse the input string, and + * modify it in the process, in order to use its space to store + * all of the keys and values for the returned object tree. + * + * They assume UTF-8 input stream, and will output UTF-8 encoded + * strings in the tree. The input stream's UTF-8 isn't validated, + * as in, what you input is what you get as an output. + * + * All the keys and values in the grpc_json objects will be strings + * pointing at your input buffer. + * + * Delete the allocated tree afterward using grpc_json_destroy(). + */ +grpc_json *grpc_json_parse_string_with_len(char *input, size_t size); +grpc_json *grpc_json_parse_string(char *input); + +/* Use these to create or delete a grpc_json object. + * Deletion is recursive. We will not attempt to free any of the strings + * in any of the objects of that tree. + */ +grpc_json *grpc_json_create(grpc_json_type type); +void grpc_json_destroy(grpc_json *json); + +/* This function will create a new string using gpr_realloc, and will + * deserialize the grpc_json tree into it. It'll be zero-terminated, + * but will be allocated in chunks of 256 bytes. + * + * The indent parameter controls the way the output is formatted. + * If indent is 0, then newlines will be suppressed as well, and the + * output will be condensed at its maximum. + */ +char *grpc_json_dump_to_string(grpc_json *json, int indent); + +////////////////////////////////////////////////////// +// definitions from grpc/src/core/security/base64 +////////////////////////////////////////////////////// + +/* Encodes data using base64. It is the caller's responsability to free + the returned char * using gpr_free. Returns nullptr on nullptr input. */ +char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, + int multiline); + +/* Decodes data according to the base64 specification. Returns an empty + slice in case of failure. */ +gpr_slice grpc_base64_decode(const char *b64, int url_safe); + +/* Same as above except that the length is provided by the caller. */ +gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, + int url_safe); + +////////////////////////////////////////////////////// +// definitions from grpc/src/core/auth/security/json_key.h +////////////////////////////////////////////////////// + +/* --- auth_json_key parsing. --- */ + +typedef struct { + const char *type; + char *private_key_id; + char *client_id; + char *client_email; + RSA *private_key; +} grpc_auth_json_key; + +/* Creates a json_key object from string. Returns an invalid object if a parsing + error has been encountered. */ +grpc_auth_json_key grpc_auth_json_key_create_from_string( + const char *json_string); + +/* Destructs the object. */ +void grpc_auth_json_key_destruct(grpc_auth_json_key *json_key); + +/* Caller is responsible for calling gpr_free on the returned value. May return + nullptr on invalid input. The scope parameter may be nullptr. */ +char *grpc_jwt_encode_and_sign(const grpc_auth_json_key *json_key, + const char *audience, + gpr_timespec token_lifetime, const char *scope); + +////////////////////////////////////////////////////// +// definitions from grpc/src/core/support/string.h +////////////////////////////////////////////////////// + +/* Minimum buffer size for calling ltoa */ +#define GPR_LTOA_MIN_BUFSIZE (3 * sizeof(long)) + +/* Convert a long to a string in base 10; returns the length of the + output string (or 0 on failure). + output must be at least GPR_LTOA_MIN_BUFSIZE bytes long. */ +int gpr_ltoa(long value, char *output); + +////////////////////////////////////////////////////// +// definitions from grpc/src/core/security/jwt_verifier.h +////////////////////////////////////////////////////// + +/* --- grpc_jwt_verifier_status. --- */ + +typedef enum { + GRPC_JWT_VERIFIER_OK = 0, + GRPC_JWT_VERIFIER_BAD_SIGNATURE, + GRPC_JWT_VERIFIER_BAD_FORMAT, + GRPC_JWT_VERIFIER_BAD_AUDIENCE, + GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR, + GRPC_JWT_VERIFIER_TIME_CONSTRAINT_FAILURE, + GRPC_JWT_VERIFIER_GENERIC_ERROR +} grpc_jwt_verifier_status; + +const char *grpc_jwt_verifier_status_to_string(grpc_jwt_verifier_status status); + +/* --- grpc_jwt_claims. --- */ + +typedef struct grpc_jwt_claims grpc_jwt_claims; + +void grpc_jwt_claims_destroy(grpc_jwt_claims *claims); + +/* Returns the whole JSON tree of the claims. */ +const grpc_json *grpc_jwt_claims_json(const grpc_jwt_claims *claims); + +/* Access to registered claims in https://tools.ietf.org/html/rfc7519#page-9 */ +const char *grpc_jwt_claims_subject(const grpc_jwt_claims *claims); +const char *grpc_jwt_claims_issuer(const grpc_jwt_claims *claims); +const char *grpc_jwt_claims_audience(const grpc_jwt_claims *claims); +gpr_timespec grpc_jwt_claims_expires_at(const grpc_jwt_claims *claims); + +/* --- TESTING ONLY exposed functions. --- */ + +grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer); +grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, + const char *audience); +} + +#endif // API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ diff --git a/contrib/endpoints/src/api_manager/auth/lib/json.cc b/contrib/endpoints/src/api_manager/auth/lib/json.cc new file mode 100644 index 00000000000..c3805a7f50b --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/json.cc @@ -0,0 +1,54 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/json.h" + +#include +#include + +#include "src/api_manager/auth/lib/json_util.h" + +namespace google { +namespace api_manager { +namespace auth { + +char *WriteUserInfoToJson(const UserInfo &user_info) { + grpc_json json_top; + memset(&json_top, 0, sizeof(json_top)); + json_top.type = GRPC_JSON_OBJECT; + + grpc_json json_issuer; + FillChild(&json_issuer, nullptr, &json_top, "issuer", + user_info.issuer.c_str(), GRPC_JSON_STRING); + + grpc_json json_id; + FillChild(&json_id, &json_issuer, &json_top, "id", user_info.id.c_str(), + GRPC_JSON_STRING); + + grpc_json json_email; + FillChild(&json_email, &json_id, &json_top, "email", user_info.email.c_str(), + GRPC_JSON_STRING); + + grpc_json json_consumer_id; + FillChild(&json_consumer_id, &json_email, &json_top, "consumer_id", + user_info.consumer_id.c_str(), GRPC_JSON_STRING); + + return grpc_json_dump_to_string(&json_top, 0); +} + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/json.h b/contrib/endpoints/src/api_manager/auth/lib/json.h new file mode 100644 index 00000000000..e7020289e27 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/json.h @@ -0,0 +1,32 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_LIB_JSON_H_ +#define API_MANAGER_AUTH_LIB_JSON_H_ + +#include "src/api_manager/auth.h" + +namespace google { +namespace api_manager { +namespace auth { + +// Write the UserInfo to Json string (NULL terminated). +// Returned buffer should be freed by esp_grpc_free. +char *WriteUserInfoToJson(const UserInfo &user_info); + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif /* API_MANAGER_AUTH_LIB_JSON_H_ */ diff --git a/contrib/endpoints/src/api_manager/auth/lib/json_test.cc b/contrib/endpoints/src/api_manager/auth/lib/json_test.cc new file mode 100644 index 00000000000..5f0e3ac15e8 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/json_test.cc @@ -0,0 +1,62 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/json.h" +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { +namespace auth { + +TEST(EspJsonTest, NormalDataTest) { + UserInfo user_info{"id", "email", "consumer_id", "iss", {"aud"}}; + static const char expected_json[] = + "{\"issuer\":\"iss\",\"id\":\"id\",\"email\":\"email\",\"consumer_id\":" + "\"consumer_id\"}"; + + ASSERT_STREQ(expected_json, WriteUserInfoToJson(user_info)); +} + +TEST(EspJsonTest, DoubleQuoteTest) { + UserInfo user_info{"id", "email \"with\" quote", "consumer_id", "iss", {}}; + static const char expected_json[] = + "{\"issuer\":\"iss\",\"id\":\"id\",\"email\":\"email \\\"with\\\" " + "quote\",\"consumer_id\":\"consumer_id\"}"; + + ASSERT_STREQ(expected_json, WriteUserInfoToJson(user_info)); +} + +TEST(EspJsonTest, SingleQuoteTest) { + UserInfo user_info{"id", "email 'with' quote", "consumer_id", "iss", {}}; + static const char expected_json[] = + "{\"issuer\":\"iss\",\"id\":\"id\",\"email\":\"email 'with' " + "quote\",\"consumer_id\":\"consumer_id\"}"; + + ASSERT_STREQ(expected_json, WriteUserInfoToJson(user_info)); +} + +TEST(EspJsonTest, SlashTest) { + UserInfo user_info{"id", "email \\with\\ quote", "consumer_id", "iss", {}}; + static const char expected_json[] = + "{\"issuer\":\"iss\",\"id\":\"id\",\"email\":\"email \\\\with\\\\ " + "quote\",\"consumer_id\":\"consumer_id\"}"; + + ASSERT_STREQ(expected_json, WriteUserInfoToJson(user_info)); +} + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/json_util.cc b/contrib/endpoints/src/api_manager/auth/lib/json_util.cc new file mode 100644 index 00000000000..5e7df14d7a0 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/json_util.cc @@ -0,0 +1,87 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/json_util.h" +#include +#include + +extern "C" { +#include "grpc/support/log.h" +} + +namespace google { +namespace api_manager { +namespace auth { + +namespace { + +bool isNullOrEmpty(const char *str) { return str == nullptr || *str == '\0'; } + +} // namespace + +const grpc_json *GetProperty(const grpc_json *json, const char *key) { + if (json == nullptr || key == nullptr) { + return nullptr; + } + const grpc_json *cur; + for (cur = json->child; cur != nullptr; cur = cur->next) { + if (strcmp(cur->key, key) == 0) return cur; + } + return nullptr; +} + +const char *GetPropertyValue(const grpc_json *json, const char *key, + grpc_json_type type) { + const grpc_json *cur = GetProperty(json, key); + if (cur != nullptr) { + if (cur->type != type) { + gpr_log(GPR_ERROR, "Unexpected type of a %s field [%s]: %d", key, + cur->value, type); + return nullptr; + } + return cur->value; + } + return nullptr; +} + +const char *GetStringValue(const grpc_json *json, const char *key) { + return GetPropertyValue(json, key, GRPC_JSON_STRING); +} + +const char *GetNumberValue(const grpc_json *json, const char *key) { + return GetPropertyValue(json, key, GRPC_JSON_NUMBER); +} + +void FillChild(grpc_json *child, grpc_json *brother, grpc_json *parent, + const char *key, const char *value, grpc_json_type type) { + if (isNullOrEmpty(key) || isNullOrEmpty(value)) { + return; + } + + memset(child, 0, sizeof(grpc_json)); + + if (brother) brother->next = child; + if (!parent->child) parent->child = child; + + child->parent = parent; + child->key = key; + child->value = value; + child->type = type; +} + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/lib/json_util.h b/contrib/endpoints/src/api_manager/auth/lib/json_util.h new file mode 100644 index 00000000000..af202c133ec --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/json_util.h @@ -0,0 +1,48 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_LIB_JSON_UTIL_H_ +#define API_MANAGER_AUTH_LIB_JSON_UTIL_H_ + +// This header file is for auth library internal use only +// since it directly includes a grpc header file. +// A public header file should not include any grpc header files. + +#include "src/api_manager/auth/lib/grpc_internals.h" + +namespace google { +namespace api_manager { +namespace auth { + +// Gets given JSON property by key name. +const grpc_json *GetProperty(const grpc_json *json, const char *key); + +// Gets string value by key or nullptr if no such key or property is not string +// type. +const char *GetStringValue(const grpc_json *json, const char *key); + +// Gets a value of a number property with a given key, or nullptr if no such key +// exists or the property is property is not number type. +const char *GetNumberValue(const grpc_json *json, const char *key); + +// Fill grpc_child with key, value and type, and setup links from/to +// brother/parents. +void FillChild(grpc_json *child, grpc_json *brother, grpc_json *parent, + const char *key, const char *value, grpc_json_type type); + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_AUTH_LIB_JSON_UTIL_H_ diff --git a/contrib/endpoints/src/api_manager/auth/lib/json_util_test.cc b/contrib/endpoints/src/api_manager/auth/lib/json_util_test.cc new file mode 100644 index 00000000000..21d3d751eec --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/lib/json_util_test.cc @@ -0,0 +1,113 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/lib/json_util.h" +#include "gtest/gtest.h" + +#include + +namespace google { +namespace api_manager { +namespace auth { +namespace { + +const char json_input[] = + "{" + " \"string\": \"string value\"," + " \"number\": 12345," + " \"null\": null," + " \"true\": true," + " \"false\": false," + " \"object\": { }," + " \"array\": [ ]," + "}"; + +TEST(JsonUtil, GetPropertyValue) { + char *json_copy = strdup(json_input); + grpc_json *json = + grpc_json_parse_string_with_len(json_copy, strlen(json_copy)); + + const char *string_value = GetStringValue(json, "string"); + ASSERT_STREQ("string value", string_value); + + const char *number_value = GetNumberValue(json, "number"); + ASSERT_STREQ("12345", number_value); + + grpc_json_destroy(json); + free(json_copy); +} + +TEST(JsonUtil, GetProperty) { + char *json_copy = strdup(json_input); + grpc_json *json = + grpc_json_parse_string_with_len(json_copy, strlen(json_copy)); + + const grpc_json *json_property; + + json_property = GetProperty(json, "string"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("string", json_property->key); + ASSERT_STREQ("string value", json_property->value); + ASSERT_EQ(GRPC_JSON_STRING, json_property->type); + + json_property = GetProperty(json, "number"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("number", json_property->key); + ASSERT_STREQ("12345", json_property->value); + ASSERT_EQ(GRPC_JSON_NUMBER, json_property->type); + + json_property = GetProperty(json, "null"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("null", json_property->key); + ASSERT_EQ(nullptr, json_property->value); + ASSERT_EQ(GRPC_JSON_NULL, json_property->type); + + json_property = GetProperty(json, "true"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("true", json_property->key); + ASSERT_EQ(nullptr, json_property->value); + ASSERT_EQ(GRPC_JSON_TRUE, json_property->type); + + json_property = GetProperty(json, "false"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("false", json_property->key); + ASSERT_EQ(nullptr, json_property->value); + ASSERT_EQ(GRPC_JSON_FALSE, json_property->type); + + json_property = GetProperty(json, "string"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("string", json_property->key); + ASSERT_STREQ("string value", json_property->value); + ASSERT_EQ(GRPC_JSON_STRING, json_property->type); + + json_property = GetProperty(json, "object"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("object", json_property->key); + ASSERT_EQ(GRPC_JSON_OBJECT, json_property->type); + + json_property = GetProperty(json, "array"); + ASSERT_NE(nullptr, json_property); + ASSERT_STREQ("array", json_property->key); + ASSERT_EQ(GRPC_JSON_ARRAY, json_property->type); + + grpc_json_destroy(json); + free(json_copy); +} + +} // namespace +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.cc b/contrib/endpoints/src/api_manager/auth/service_account_token.cc new file mode 100644 index 00000000000..a2986893538 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.cc @@ -0,0 +1,98 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/service_account_token.h" + +#include "google/protobuf/stubs/logging.h" +#include "src/api_manager/auth/lib/auth_token.h" + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace auth { + +namespace { + +// The expiration time in seconds for auth token calculated from client secret. +// This should not be changed since it has to match the hard-coded value in +// esp_get_auth_token() function. +// Token expired in 1 hour, reduce 100 seconds for grace buffer. +const int kClientSecretAuthTokenExpiration(3600 - 100); + +} // namespace + +Status ServiceAccountToken::SetClientAuthSecret(const std::string& secret) { + client_auth_secret_ = secret; + + for (unsigned int i = 0; i < JWT_TOKEN_TYPE_MAX; i++) { + if (!jwt_tokens_[i].audience().empty()) { + Status status = jwt_tokens_[i].GenerateJwtToken(client_auth_secret_); + if (!status.ok()) { + if (env_) { + env_->LogError("Failed to generate auth token."); + } + return status; + } + } + } + return Status::OK; +} + +void ServiceAccountToken::SetAudience(JWT_TOKEN_TYPE type, + const std::string& audience) { + GOOGLE_CHECK(type >= 0 && type < JWT_TOKEN_TYPE_MAX); + jwt_tokens_[type].set_audience(audience); +} + +const std::string& ServiceAccountToken::GetAuthToken(JWT_TOKEN_TYPE type) { + // Uses authentication secret if available. + if (!client_auth_secret_.empty()) { + GOOGLE_CHECK(type >= 0 && type < JWT_TOKEN_TYPE_MAX); + if (!jwt_tokens_[type].is_valid(0)) { + Status status = jwt_tokens_[type].GenerateJwtToken(client_auth_secret_); + if (!status.ok()) { + if (env_) { + env_->LogError("Failed to generate auth token."); + } + static std::string empty; + return empty; + } + } + return jwt_tokens_[type].token(); + } + return access_token_.token(); +} + +Status ServiceAccountToken::JwtTokenInfo::GenerateJwtToken( + const std::string& client_auth_secret) { + // Make sure audience is set. + GOOGLE_CHECK(!audience_.empty()); + char* token = + auth::esp_get_auth_token(client_auth_secret.c_str(), audience_.c_str()); + if (token == nullptr) { + return Status(Code::INVALID_ARGUMENT, + "Invalid client auth secret, the file may be corrupted."); + } + set_token(token, kClientSecretAuthTokenExpiration); + auth::esp_grpc_free(token); + return Status::OK; +} + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.h b/contrib/endpoints/src/api_manager/auth/service_account_token.h new file mode 100644 index 00000000000..b062d66bbd8 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.h @@ -0,0 +1,139 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_AUTH_SERVICE_ACCOUNT_TOKEN_H_ +#define API_MANAGER_AUTH_SERVICE_ACCOUNT_TOKEN_H_ + +#include + +#include "include/api_manager/env_interface.h" + +namespace google { +namespace api_manager { +namespace auth { + +// Stores service account tokens to access Google services, such as service +// control and cloud tracing. There are two kinds of auth token: +// 1) client auth secret is a client secret can be used to generate auth +// JWT token. But JWT token is audience specific. Need to generate auth +// JWT token for each service with its audience. +// 2) GCE service account token is fetched from GCP metadata server. +// This auth token can be used for any Google services. +class ServiceAccountToken { + public: + ServiceAccountToken(ApiManagerEnvInterface* env) : env_(env), state_(NONE) {} + + // Sets the client auth secret and it can be used to generate JWT token. + utils::Status SetClientAuthSecret(const std::string& secret); + + // Fetching state of the token from the metadata server + enum FetchState { NONE = 0, FETCHING, FETCHED, FAILED }; + + // Set fetching state + void set_state(FetchState state) { state_ = state; } + + // Get fetching state + FetchState state() const { return state_; } + + // Returns whether the client auth secret exists + bool has_client_secret() const { return !client_auth_secret_.empty(); } + + // Set access token value and expiration duration + void set_access_token(const std::string& token, time_t expiration) { + access_token_.set_token(token, expiration); + } + + // Returns true if access token is valid `duration` seconds from now. + // Use 0 for `duration` to check if the token is valid now. + bool is_access_token_valid(time_t duration) const { + return access_token_.is_valid(duration); + } + + // JWT token calcualted from client auth secret are audience dependent. + enum JWT_TOKEN_TYPE { + JWT_TOKEN_FOR_SERVICE_CONTROL = 0, + JWT_TOKEN_FOR_CLOUD_TRACING, + JWT_TOKEN_TYPE_MAX, + }; + // Set audience. Only calcualtes JWT token with specified audience. + void SetAudience(JWT_TOKEN_TYPE type, const std::string& audience); + + // Gets the auth token to access Google services. + // If client auth secret is specified, use it to calcualte JWT token. + // Otherwise, use the access token fetched from metadata server. + const std::string& GetAuthToken(JWT_TOKEN_TYPE type); + + private: + // Stores base token info. Used for both OAuth and JWT tokens. + class TokenInfo { + public: + // Token available and not expired in `duration` seconds + bool is_valid(time_t duration) const { + return !token_.empty() && expiration_time_ >= time(nullptr) + duration; + } + + // Set token and its expiration duration + void set_token(const std::string& token, time_t expiration) { + token_ = token; + expiration_time_ = time(nullptr) + expiration; + } + + // Get the token + const std::string& token() const { return token_; } + + // Get expiration time in seconds + time_t expiration_time() const { return expiration_time_; } + + private: + // The auth token. + std::string token_; + // The token expiration time. + time_t expiration_time_; + }; + + // Stores JWT token info + class JwtTokenInfo : public TokenInfo { + public: + void set_audience(const std::string audience) { audience_ = audience; } + const std::string& audience() const { return audience_; } + + // Generates auth JWT token from client auth secret. + utils::Status GenerateJwtToken(const std::string& client_auth_secret); + + private: + // The audiences. + std::string audience_; + }; + + // environment interface. + ApiManagerEnvInterface* env_; + + // The client auth secret which can be used to generate JWT auth token. + std::string client_auth_secret_; + + // JWT tokens calcualted from client auth secrect. + JwtTokenInfo jwt_tokens_[JWT_TOKEN_TYPE_MAX]; + + // GCE service account access token fetched from GCE metadata server. + TokenInfo access_token_; + + // Fetching state + FetchState state_; +}; + +} // namespace auth +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_AUTH_SERVICE_ACCOUNT_TOKEN_H_ diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token_test.cc b/contrib/endpoints/src/api_manager/auth/service_account_token_test.cc new file mode 100644 index 00000000000..260a9d2cc18 --- /dev/null +++ b/contrib/endpoints/src/api_manager/auth/service_account_token_test.cc @@ -0,0 +1,83 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/auth/service_account_token.h" + +#include "gtest/gtest.h" +#include "src/api_manager/mock_api_manager_environment.h" + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace auth { + +namespace { + +class ServiceAccountTokenTest : public ::testing::Test { + public: + void SetUp() { + env_.reset(new ::testing::NiceMock()); + sa_token_ = std::unique_ptr( + new ServiceAccountToken(env_.get())); + } + + std::unique_ptr env_; + std::unique_ptr sa_token_; +}; + +TEST_F(ServiceAccountTokenTest, TestAccessToken) { + // Needed. + ASSERT_FALSE(sa_token_->is_access_token_valid(0)); + // Adds a token, not needed now + sa_token_->set_access_token("Dummy Token", 1); + ASSERT_TRUE(sa_token_->is_access_token_valid(0)); + ASSERT_EQ("Dummy Token", + sa_token_->GetAuthToken( + ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL)); + + sleep(2); + // Token is expired, needed now. + ASSERT_FALSE(sa_token_->is_access_token_valid(0)); + // Returns expired token. + ASSERT_EQ("Dummy Token", + sa_token_->GetAuthToken( + ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL)); +} + +TEST_F(ServiceAccountTokenTest, TestClientAuthSecret) { + // Needed. + ASSERT_FALSE(sa_token_->is_access_token_valid(0)); + ASSERT_EQ(ServiceAccountToken::NONE, sa_token_->state()); + + sa_token_->SetAudience(ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL, + "audience"); + Status status = sa_token_->SetClientAuthSecret("Dummy secret"); + ASSERT_EQ(status.code(), Code::INVALID_ARGUMENT); + + // As long as there is a client auth secret. no need to get token. + ASSERT_TRUE(sa_token_->has_client_secret()); + // Returns empty string for an invalid secret. + ASSERT_EQ("", sa_token_->GetAuthToken( + ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL)); +} + +} // namespace + +} // namespace auth +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/check_auth.cc b/contrib/endpoints/src/api_manager/check_auth.cc new file mode 100644 index 00000000000..cac05f7a3ea --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_auth.cc @@ -0,0 +1,463 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +#include "src/api_manager/check_auth.h" + +#include +#include + +#include "include/api_manager/api_manager.h" +#include "include/api_manager/request.h" +#include "src/api_manager/auth.h" +#include "src/api_manager/auth/lib/auth_jwt_validator.h" +#include "src/api_manager/auth/lib/auth_token.h" +#include "src/api_manager/auth/lib/base64.h" +#include "src/api_manager/auth/lib/json.h" +#include "src/api_manager/auth/lib/json_util.h" +#include "src/api_manager/cloud_trace/cloud_trace.h" +#include "src/api_manager/utils/url_util.h" + +using ::google::api_manager::auth::Certs; +using ::google::api_manager::auth::JwtCache; +using ::google::api_manager::auth::JwtValue; +using ::google::api_manager::auth::GetStringValue; +using ::google::api_manager::auth::JwtValidator; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; +using std::chrono::system_clock; + +namespace google { +namespace api_manager { + +namespace { +const char kAccessTokenName[] = "access_token"; +const char kAuthHeader[] = "authorization"; +const char kBearer[] = "Bearer "; +// The lifetime of a public key cache entry. Unit: seconds. +const int kPubKeyCacheDuration = 300; + +// The header key to send endpoint api user info. +const char kEndpointApiUserInfo[] = "X-Endpoint-API-UserInfo"; + +// An AuthChecker object is created for every incoming request. It authenticates +// the request, extracts user info from the auth token and sets it to the +// request context. +class AuthChecker : public std::enable_shared_from_this { + public: + AuthChecker(std::shared_ptr context, + std::function continuation); + + // Check auth for a given request. This is the starting point to enter + // the auth state machine. + void Check(); + + private: + /*** Steps in auth state machine, ordered in execution sequence. ***/ + + // Not all the steps are executed for every request. + // For example, in case of a JWT cache hit, only four steps are executed: + // GetAuthToken() --> LookupJwtCache() --> CheckAudience() --> PassUserInfo() + // In the case of a JWT cache miss, but a key cache hit, the steps are: + // GetAuthToken() --> LookupJwtCache() --> ParseJwt() --> CheckAudience() --> + // InitKey() --> VerifySignature() --> PassUserInfo() + void GetAuthToken(); + + void LookupJwtCache(); + + void ParseJwt(); + + void CheckAudience(bool cache_hit); + + void InitKey(); + + void DiscoverJwksUri(const std::string &url); + + // Callback function for open ID discovery http fetch. + void PostFetchJwksUri(Status status, std::string &&body); + + void FetchPubKey(const std::string &url); + + // Callback function for public key http fetch. + void PostFetchPubKey(Status status, std::string &&body); + + void VerifySignature(); + + void PassUserInfoOnSuccess(); + + /*** Helper functions ***/ + + // Returns a shared pointer of this AuthChecker object. + std::shared_ptr GetPtr() { return shared_from_this(); } + + // Helper function to send a http GET request. + void HttpFetch(const std::string &url, + std::function continuation); + + // Authentication error + void Unauthenticated(const std::string &error); + + // Authorization error + void Unauthorized(const std::string &error); + + // Fetch error, takes upstream error + void FetchFailure(const std::string &error, Status status); + + /*** Member Variables. ***/ + + // Request context. + std::shared_ptr context_; + + // JWT validator. + std::unique_ptr validator_; + + // User info extracted from auth token. + UserInfo user_info_; + + // Pointer to access ESP running environment. + ApiManagerEnvInterface *env_; + + // auth token. + std::string auth_token_; + + // The final continuation function. + std::function on_done_; + + // Trace span for check auth. + std::shared_ptr trace_span_; +}; + +AuthChecker::AuthChecker(std::shared_ptr context, + std::function continuation) + : context_(context), + env_(context_->service_context()->env()), + on_done_(continuation) {} + +void AuthChecker::Check() { + if (!context_->service_context()->RequireAuth() || + context_->method() == nullptr || !context_->method()->auth()) { + env_->LogDebug("Auth not required."); + on_done_(Status::OK); + return; + } + + // CreateSpan returns nullptr if trace is disabled. + trace_span_.reset(CreateSpan(context_->cloud_trace(), "CheckAuth")); + + GetAuthToken(); + if (auth_token_.empty()) { + Unauthenticated("Missing or invalid credentials"); + return; + } + context_->request()->SetAuthToken(auth_token_); + + env_->LogDebug(std::string("auth token: ") + auth_token_); + + LookupJwtCache(); +} + +void AuthChecker::GetAuthToken() { + Request *r = context_->request(); + std::string auth_header; + if (!r->FindHeader(kAuthHeader, &auth_header)) { + // When authorization header is missing, check query parameter. + r->FindQuery(kAccessTokenName, &auth_token_); + return; + } + + static const size_t bearer_len = sizeof(kBearer) - 1; + if (auth_header.size() <= bearer_len || + auth_header.compare(0, bearer_len, kBearer) != 0) { + // Authorization header is not long enough, or authorization header does + // not begin with "Bearer ", set auth_token_ to empty string. + auth_token_ = std::string(); + return; + } + + auth_token_ = auth_header.substr(bearer_len); +} + +void AuthChecker::LookupJwtCache() { + bool remove = false; // whether or not need to remove an expired entry. + bool cache_hit = false; + JwtCache &jwt_cache = context_->service_context()->jwt_cache(); + { + JwtCache::ScopedLookup lookup(&jwt_cache, auth_token_); + if (lookup.Found()) { + JwtValue *val = lookup.value(); + if (system_clock::now() <= val->exp) { + // Cache hit and cache entry is not expired. + user_info_ = val->user_info; + cache_hit = true; + } else { + // Need to removes the expired cache entry. + remove = true; + } + } + } + if (remove) { + jwt_cache.Remove(auth_token_); + } + + if (cache_hit) { + CheckAudience(true); + } else { + ParseJwt(); + } +} + +void AuthChecker::ParseJwt() { + if (validator_ == nullptr) { + validator_ = JwtValidator::Create(auth_token_.c_str(), auth_token_.size()); + if (validator_ == nullptr) { + Unauthenticated("Internal error"); + return; + } + } + + Status status = validator_->Parse(&user_info_); + if (!status.ok()) { + Unauthenticated(status.message()); + return; + } + + CheckAudience(false); +} + +void AuthChecker::CheckAudience(bool cache_hit) { + std::string audience = user_info_.audiences.empty() + ? std::string() + : user_info_.AudiencesAsString(); + context_->set_auth_issuer(user_info_.issuer); + context_->set_auth_audience(audience); + context_->set_auth_authorized_party(user_info_.authorized_party); + + // Remove http/s header and trailing '/' for issuer. + std::string issuer = utils::GetUrlContent(user_info_.issuer); + if (!context_->method()->isIssuerAllowed(issuer)) { + Unauthenticated("Issuer not allowed"); + return; + } + + // The audience from the JWT must + // - Equals to service_name or + // - Explicitly allowed by the issuer in the method configuration. + // Otherwise the JWT is rejected. + const std::string &service_name = context_->service_context()->service_name(); + // Remove http/s header and trailing '/' for audiences. + std::set aud; + for (auto &it : user_info_.audiences) { + aud.insert(utils::GetUrlContent(it)); + } + if (aud.find(service_name) == aud.end() && + !context_->method()->isAudienceAllowed(issuer, aud)) { + Unauthorized("Audience not allowed"); + return; + } + if (cache_hit) { + PassUserInfoOnSuccess(); + } else { + InitKey(); + } +} + +void AuthChecker::InitKey() { + Certs &key_cache = context_->service_context()->certs(); + auto cert = key_cache.GetCert(user_info_.issuer); + + if (cert == nullptr || system_clock::now() > cert->second) { + // Key has not been fetched or has expired. + std::string url; + bool tryOpenId = + context_->service_context()->GetJwksUri(user_info_.issuer, &url); + if (url.empty()) { + Unauthenticated("Cannot determine the URI of the key"); + return; + } + + if (tryOpenId) { + DiscoverJwksUri(url); + } else { + // JwksUri is available. No need to try openID discovery. + FetchPubKey(url); + } + } else { + // Key is in the cache, next step is to verify signature. + VerifySignature(); + } +} + +void AuthChecker::DiscoverJwksUri(const std::string &url) { + auto pChecker = GetPtr(); + HttpFetch(url, [pChecker](Status status, std::string &&body) { + pChecker->PostFetchJwksUri(status, std::move(body)); + }); +} + +void AuthChecker::PostFetchJwksUri(Status status, std::string &&body) { + if (!status.ok()) { + context_->service_context()->SetJwksUri(user_info_.issuer, std::string(), + false); + FetchFailure("Unable to fetch URI of the key via OpenID discovery", status); + return; + } + + // Parse discovery doc and extract jwks_uri + grpc_json *discovery_json = grpc_json_parse_string_with_len( + const_cast(body.c_str()), body.size()); + const char *jwks_uri; + if (discovery_json != nullptr) { + jwks_uri = GetStringValue(discovery_json, "jwks_uri"); + grpc_json_destroy(discovery_json); + } else { + jwks_uri = nullptr; + } + + if (jwks_uri == nullptr) { + env_->LogError("OpenID discovery failed due to invalid doc format"); + context_->service_context()->SetJwksUri(user_info_.issuer, std::string(), + false); + Unauthenticated("Unable to parse URI of the key via OpenID discovery"); + return; + } + + // OpenID discovery completed. Set jwks_uri for the issuer in cache. + context_->service_context()->SetJwksUri(user_info_.issuer, jwks_uri, false); + + FetchPubKey(jwks_uri); +} + +void AuthChecker::FetchPubKey(const std::string &url) { + auto pChecker = GetPtr(); + HttpFetch(url, [pChecker](Status status, std::string &&body) { + pChecker->PostFetchPubKey(status, std::move(body)); + }); +} + +void AuthChecker::PostFetchPubKey(Status status, std::string &&body) { + if (!status.ok() || body.empty()) { + FetchFailure("Unable to fetch verification key", status); + return; + } + + Certs &key_cache = context_->service_context()->certs(); + key_cache.Update( + user_info_.issuer, std::move(body), + system_clock::now() + std::chrono::seconds(kPubKeyCacheDuration)); + VerifySignature(); +} + +void AuthChecker::VerifySignature() { + Certs &key_cache = context_->service_context()->certs(); + auto cert = key_cache.GetCert(user_info_.issuer); + if (cert == nullptr) { + Unauthenticated("Missing verification key"); + return; + } + + Status status = + validator_->VerifySignature(cert->first.c_str(), cert->first.size()); + if (!status.ok()) { + Unauthenticated(status.message()); + return; + } + + // Inserts the entry to JwtCache. + JwtCache &cache = context_->service_context()->jwt_cache(); + cache.Insert(auth_token_, user_info_, validator_->GetExpirationTime(), + system_clock::now()); + + PassUserInfoOnSuccess(); +} + +void AuthChecker::PassUserInfoOnSuccess() { + char *json_buf = auth::WriteUserInfoToJson(user_info_); + if (json_buf == nullptr) { + return; + } + char *base64_json_buf = auth::esp_base64_encode( + json_buf, strlen(json_buf), true, false, true /*padding*/); + context_->request()->AddHeaderToBackend(kEndpointApiUserInfo, + base64_json_buf); + auth::esp_grpc_free(json_buf); + auth::esp_grpc_free(base64_json_buf); + + TRACE(trace_span_) << "Authenticated."; + trace_span_.reset(); + on_done_(Status::OK); +} + +void AuthChecker::Unauthenticated(const std::string &error) { + TRACE(trace_span_) << "Authentication failed: " << error; + trace_span_.reset(); + on_done_(Status(Code::UNAUTHENTICATED, + std::string("JWT validation failed: ") + error, + Status::AUTH)); +} + +void AuthChecker::Unauthorized(const std::string &error) { + TRACE(trace_span_) << "Authorization failed: " << error; + trace_span_.reset(); + on_done_(Status(Code::PERMISSION_DENIED, + std::string("JWT validation failed: ") + error, + Status::AUTH)); +} + +void AuthChecker::FetchFailure(const std::string &error, Status status) { + // Append HTTP response code for the upstream statuses + trace_span_.reset(); + on_done_( + Status(Code::UNAUTHENTICATED, + std::string("JWT validation failed: ") + error + + (status.code() >= 300 + ? ". HTTP response code: " + std::to_string(status.code()) + : ""), + Status::AUTH)); +} + +void AuthChecker::HttpFetch( + const std::string &url, + std::function continuation) { + std::shared_ptr fetch_span( + CreateChildSpan(trace_span_.get(), "HttpFetch")); + env_->LogDebug(std::string("http fetch: ") + url); + TRACE(fetch_span) << "Http request URL: " << url; + + std::unique_ptr request( + new HTTPRequest([continuation, fetch_span]( + Status status, std::map &&, + std::string &&body) { + TRACE(fetch_span) << "Http response status: " << status.ToString(); + continuation(status, std::move(body)); + })); + if (!request) { + continuation(Status(Code::INTERNAL, "Out of memory"), ""); + return; + } + + request->set_method("GET").set_url(url); + env_->RunHTTPRequest(std::move(request)); +} + +} // namespace + +void CheckAuth(std::shared_ptr context, + std::function continuation) { + std::shared_ptr authChecker = + std::make_shared(context, continuation); + authChecker->Check(); +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/check_auth.h b/contrib/endpoints/src/api_manager/check_auth.h new file mode 100644 index 00000000000..3374352a981 --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_auth.h @@ -0,0 +1,32 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_CHECK_AUTH_H_ +#define API_MANAGER_CHECK_AUTH_H_ + +#include "include/api_manager/utils/status.h" +#include "src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// This function checks auth for a given request. +// It is called by CheckWorkflow class when processing a request. +void CheckAuth(std::shared_ptr context, + std::function continuation); + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_CHECK_AUTH_H_ diff --git a/contrib/endpoints/src/api_manager/check_auth_test.cc b/contrib/endpoints/src/api_manager/check_auth_test.cc new file mode 100644 index 00000000000..85f7f3b7dd6 --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_auth_test.cc @@ -0,0 +1,615 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/check_auth.h" + +#include "src/api_manager/check_workflow.h" +#include "src/api_manager/context/service_context.h" +#include "src/api_manager/mock_api_manager_environment.h" +#include "src/api_manager/mock_request.h" + +using ::testing::_; +using ::testing::AllOf; +using ::testing::DoAll; +using ::testing::Field; +using ::testing::Invoke; +using ::testing::Mock; +using ::testing::Return; + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +namespace { + +const char kAccessTokenName[] = "access_token"; +const char kAuthHeader[] = "authorization"; +const char kBearer[] = "Bearer "; + +const char kServiceConfig[] = + "name: \"endpoints-test.cloudendpointsapis.com\"\n" + "authentication {\n" + " providers: [\n" + " {\n" + " id: \"issuer1\"\n" + " issuer: \"https://issuer1.com\"\n" + " },\n" + " {\n" + " id: \"openid_fail\"\n" + " issuer: \"http://openid_fail\"\n" + " },\n" + " {\n" + " id: \"issuer2\"\n" + " issuer: \"https://issuer2.com\"\n" + " jwks_uri: \"https://issuer2.com/pubkey\"\n" + " }\n" + " ],\n" + " rules: {\n" + " selector: \"ListShelves\"\n" + " requirements: [\n" + " {\n" + " provider_id: \"issuer1\"\n" + " },\n" + " {\n" + " provider_id: \"openid_fail\"\n" + " },\n" + " {\n" + " provider_id: \"issuer2\"\n" + " }\n" + " ]\n" + " }\n" + "}\n" + "http {\n" + " rules {\n" + " selector: \"ListShelves\"\n" + " get: \"/ListShelves\"\n" + " }\n" + "}\n" + "control {\n" + " environment : \"http://127.0.0.1:8081\"\n" + "}\n"; + +// Auth token generated with the following header and payload. +//{ +// "alg": "RS256", +// "typ": "JWT", +// "kid": "b3319a147514df7ee5e4bcdee51350cc890cc89e" +//} +//{ +// "iss": "https://issuer1.com", +// "sub": "end-user-id", +// "aud": "endpoints-test.cloudendpointsapis.com", +// "iat": 1461779321, +// "exp": 2461782921 +//} +const char kToken[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "JlbmRwb2ludHMtdGVzdC5jbG91ZGVuZHBvaW50c2FwaXMuY29tIiwiaWF0IjoxNDYxNzc5MzIx" + "LCJleHAiOjI0NjE3ODI5MjF9.iiJj93x_KNSIh14nthz_N_" + "JsA6ZAZfQQxECrmhalrcw5jnlKKIwI_QFnv8y9EFyIHLfEt56-" + "GUXH7uLhed1sTJUNTd8sXyuFdSK1Cd3jAozvvZaYazhNIYC9ljh7hhm-" + "rHkirWTu1GxdRaJD3Az-B-VX3C_OWY9oHR2mhw0zxkEcMAgjf7GuGWr-AYtDmAMD_" + "fE8o7oXvD9eg506E5mcDa208m0N8Ysc3Ibdmfnux5B1pPB-5M-O2u2lVLjhne7SMCG9wv-" + "nnXCy9iAcTgCt6VmBtehZOBDQ0q8_08aWGoWBXntLwdXinVIs-zR-" + "4YUumdkEIIPF3IYE6ZAlloWG7Q"; + +// kToken2 is the same as kToken except that "sub" is "another-user-id". +const char kToken2[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ.eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViI" + "joiYW5vdGhlci11c2VyLWlkIiwiYXVkIjoiZW5kcG9pbnRzLXRlc3QuY2xvdWRlbmRwb2ludHN" + "hcGlzLmNvbSIsImlhdCI6MTQ2MTc3OTMyMSwiZXhwIjoyNDYxNzgyOTIxfQ.ASIXY1N3fmGuDY" + "8-Xg6lCxiXM51wtTiGRmcYMk6_q_91D3D9cswVMfNp7YLf6sA4KQFxoTFAiWRqxT-kPzF5o2O8" + "ga4CY0VZAsiXRm-YTe7O8T2kFrVJOkABQIyNZgln8Sm15bO1MSyClj8Ti2qRAYPeoDM57X0f-u" + "0nQsWv0X8BsLvlJfu1J-4n08l_eEUFZlYGwpWCKfwvklmWXYfXdcJMeQ_poVqOti8dvSnqVi_Z" + "0Yu1r5Xuq45q4WNqQ9PRk1HFsWB7uV25m8fU2VpbRLFka6F9MZu4dU9gZoGCGDTtauCHiMqBTv" + "6vON8GbcB7w8pEhS1hK6FOehe4qZKnSA"; + +// kTokenOpenIdFail is the same as kToken except that "iss" is +// "http://openid_fail". +const char kTokenOpenIdFail[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwOi8vb3BlbmlkX2ZhaWwiLCJzdWIiOiJlbmQtdXNlci1pZCIsImF1ZCI6Im" + "VuZHBvaW50cy10ZXN0LmNsb3VkZW5kcG9pbnRzYXBpcy5jb20iLCJpYXQiOjE0NjE3NzkzMjEs" + "ImV4cCI6MjQ2MTc4MjkyMX0.kauli_A55XB3AI-ZHrzlZs87VlMF_" + "iRyxAJs0BOCIZ0pYrsH4gKpheepafoJfYJcP5HYeC4JZuivKmfZn8hVjHZ-crXhxvnQf0AM-" + "nI4S80tuWewcvKQq3tpyoyjw0DAu4sI61ejCINvc2qEpiyp4jBcww1xxOFXbCOvSTbfJzISGSe" + "Kmqs5ryGHFyW-" + "rsGau030xa4ZnJo4qjzEaFqf9UwbWoEGhmJLHx6AWJUPnMtHN1YGZkCO7OXBk7gOOlVd5iNR-" + "OHDbpUEEYI2KM5N2MNdjN5QaAIwyvDnWTA3ivetbiiNP2sjt9Ar3fTkfO_" + "bjHTvoHiUKvPLTJfWeLVSzQ"; + +// kTokenOpenIdFail2 is the same as kTokenOpenIdFail except that "sub" is +// "another-user-id". +const char kTokenOpenIdFail2[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwOi8vb3BlbmlkX2ZhaWwiLCJzdWIiOiJhbm90aGVyLXVzZXItaWQiLCJhdW" + "QiOiJlbmRwb2ludHMtdGVzdC5jbG91ZGVuZHBvaW50c2FwaXMuY29tIiwiaWF0IjoxNDYxNzc5" + "MzIxLCJleHAiOjI0NjE3ODI5MjF9.DYsI1A0CDZUbIniGAQy0JXyE9KjhmbSMxNzIrm_" + "5ogvwAXgpx4vStYtGU6w_Lv1vAtQHONY6zK4qzNBi3h-" + "wSXQtaFxycRphopohA56vyT6dP0BMWJQuDvWBUyxYWApP49XuVkZCFy3BFbJ8ZniuhzkgrRhTN" + "-L8jUiopUd6mtdPPc7ZLoWcKXrtsTjlNSOH7r2VmJTCeWgVsDCBRceFdLyNy1tfz5ibaTgP-" + "oL7tUQ1VnRkRoedLm17LoPKtu4-dbGchIaUnYfP1__gbHW48_oj_" + "QSbutSQYsxtx2ESZ78JFd1kFX0Qs5YL7u2k93Za49S3SuWH9CV7bca0leIzEw"; + +// kTokenIssuer2 is the same as kToken except that "iss" is +// "https://issuer2.com". +const char kTokenIssuer2[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjIuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "JlbmRwb2ludHMtdGVzdC5jbG91ZGVuZHBvaW50c2FwaXMuY29tIiwiaWF0IjoxNDYxNzc5MzIx" + "LCJleHAiOjI0NjE3ODI5MjF9.iFPb5TSZXGBxzFKJ5FOZDr1sH_-5KEBRvJy5vtanbjX2H-" + "VU0aloWUeKCUGbtd9HdnHP4I7n-nivI1wltYK-E19aG3xswDZr3I5kg3JyNp-" + "T9a4EuQ7cue7ofxJi57l7vRDGQ-" + "9548QoenJP9vkJc4nb70xPF0CriwujaBr91jOaJmvc4W1ivXoIc1QgG9wHdRg8AgeIAaQhvnyj" + "F_Y9ut23lhAL8miYEs2ggwUSrImQzjed0t0205nz_" + "3rFZuei5DNEGbDoj6ja5jLvf11W2bCTkitEjYXUbomKm-RegWX6MjT-" + "zdhuVgY0vUaLFI1OFlKT7ArQQ5HZpw9lt4sKzg"; + +// kTokenExpired is the same as kToken except that "exp" is "1461782921". +const char kTokenExpired[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "JlbmRwb2ludHMtdGVzdC5jbG91ZGVuZHBvaW50c2FwaXMuY29tIiwiaWF0IjoxNDYxNzc5MzIx" + "LCJleHAiOjE0NjE3ODI5MjF9.Zlk_QD3FM5u-E7r2Vo4xrPfNUNtewfTmJPZUL_" + "MaLkWK3Do6ov0B8mj5nzf8fFyw3YTxhN7VjabDfwr2pTfVl0PA8Kx8XLz3R_" + "apAbP0UhKEBgYhv5GCLhiTTuq01QCDnhvcTVXRorWfe6ocC2GPTIHVA_M5U1v5nPopP-" + "Kp68fMQ3sro7mNIs-NtWQsjPEyk1cHu1wH0_pUjcVjex8SQb6vRxAEsF4sZq17cCRI-" + "6miwxYNzYyVCv-csNsxgTH8_kZbfXgk9WbJKE4k7XhIsU4K_cLmz8OysRw9IpsWQpmhHGHMXu-" + "-QBSDC3_5Lp5RU5oaEjHRd9AvXVsz1K1iaQ"; + +// kTokenBadAud is the same as kToken except that "aud" is "some-audience". +const char kTokenBadAud[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "Jzb21lLWF1ZGllbmNlIiwiaWF0IjoxNDYxNzc5MzIxLCJleHAiOjI0NjE3ODI5MjF9." + "KFNI2R5r9IdjvAWFEvrM6q7dHNkrbLfPZNK7u8XQmn4vbMChHVW0gYYbO-" + "eaLPl0xnGX9q40xiMkM3blZ4oRPTj5YF_GcG9S-2fTPXdbmOtbHbpfMG1W26n2ESH2UKpEAL-" + "wRWbC7ea2dsdHE2zEbQfbLAwRjX3m3JiJlLHFJkQcrxsj8PwEScNBRJLRrIA1c4EPwQRsiNR7w" + "VooZYt_8v2ClgdKx8I-iDa4zhOheIVgvOduKARW5p2yyptM__9Pr544ox-R1IO2YYh-" + "70mLN05YowDM268OOjkdR1wC5vtsXGns5ZmT-h1vdQXluMuz-S2ppR3EqTUip4rBMOSAEQ"; + +// kTokenHttpsAud is the same as kToken except that "aud" is +// "https://endpoints-test.cloudendpointsapis.com". +const char kTokenHttpsAud[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "JodHRwczovL2VuZHBvaW50cy10ZXN0LmNsb3VkZW5kcG9pbnRzYXBpcy5jb20iLCJpYXQiOjE0" + "NjE3NzkzMjEsImV4cCI6MjQ2MTc4MjkyMX0." + "YVL4imxp7jS0RdvQhz7zflaqQzX7Q7TxVGZF9iHy9cxZKnB0wxGgXSb7jl_" + "KZ2tVCXvQLvErQAxrADDHUOLpXGbgdImF1UJz0YPQGffiyYPvXch2207czH9erKRNdMSxDCHrc" + "976Rvb9VTO9JFCTTbRwcGgBWz4H-gO55oCbErJCchyXdjLiMPiww-otw8n4tKqcNZhp_" + "xNDxkRExbk0oQH04epoIvmgmh6snAF06bi652Ag6Z4E842017DIZdaoy3VySbBsDMpZU2YLMil" + "fLJYUe5b64D6YvAfVAGgB01s7UJJb1b9KXoash5_nvM6xG0syr9URGF-kbqbWVccclA"; + +// kTokenHttpsSlashAud is the same as kToken except that "aud" is +// ""https://endpoints-test.cloudendpointsapis.com/". +const char kTokenHttpsSlashAud[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "JodHRwczovL2VuZHBvaW50cy10ZXN0LmNsb3VkZW5kcG9pbnRzYXBpcy5jb20vIiwiaWF0Ijox" + "NDYxNzc5MzIxLCJleHAiOjI0NjE3ODI5MjF9.OPN_7kPW2XR2qDNOB-" + "RNL1YLBTqJagbK7O2aieEr8VRWpUDR-BcY4LgbrVTY9kRvV1pb_T_" + "lCxX6tLIkKqC4QOZ1NuOVFhL1yAAjCI8oZB30m43JE8I0m6aEqjzelYcMBJFKp2Wfk16Hj-" + "Ain797f0u1F1tYJiat67bCfaPFJwWBsHHvyPbJ7NnxFvRYN6F1b8ddT9qAbELxj4fsj1F9rE0O" + "dmcp0lLhUa2OYrpyyipD5hv0eZIj4Yxlt962Qb6ZhewJULKULueIsWFXq3QZ-FPHXO8-B-" + "ZBDv5_INzDpTaUK0htgVMMvcbqCcr2DdAlloaZXUnnEINy-d57SBsw1w"; + +// kTokenHttpAud is the same as kToken except that "aud" is +// "http://endpoints-test.cloudendpointsapis.com". +const char kTokenHttpAud[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "JodHRwczovL2VuZHBvaW50cy10ZXN0LmNsb3VkZW5kcG9pbnRzYXBpcy5jb20iLCJpYXQiOjE0" + "NjE3NzkzMjEsImV4cCI6MjQ2MTc4MjkyMX0." + "YVL4imxp7jS0RdvQhz7zflaqQzX7Q7TxVGZF9iHy9cxZKnB0wxGgXSb7jl_" + "KZ2tVCXvQLvErQAxrADDHUOLpXGbgdImF1UJz0YPQGffiyYPvXch2207czH9erKRNdMSxDCHrc" + "976Rvb9VTO9JFCTTbRwcGgBWz4H-gO55oCbErJCchyXdjLiMPiww-otw8n4tKqcNZhp_" + "xNDxkRExbk0oQH04epoIvmgmh6snAF06bi652Ag6Z4E842017DIZdaoy3VySbBsDMpZU2YLMil" + "fLJYUe5b64D6YvAfVAGgB01s7UJJb1b9KXoash5_nvM6xG0syr9URGF-kbqbWVccclA"; + +// kTokenHttpSlashAud is the same as kToken except that "aud" is +// ""http://endpoints-test.cloudendpointsapis.com/". +const char kTokenHttpSlashAud[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ." + "eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViIjoiZW5kLXVzZXItaWQiLCJhdWQiOi" + "JodHRwczovL2VuZHBvaW50cy10ZXN0LmNsb3VkZW5kcG9pbnRzYXBpcy5jb20vIiwiaWF0Ijox" + "NDYxNzc5MzIxLCJleHAiOjI0NjE3ODI5MjF9.OPN_7kPW2XR2qDNOB-" + "RNL1YLBTqJagbK7O2aieEr8VRWpUDR-BcY4LgbrVTY9kRvV1pb_T_" + "lCxX6tLIkKqC4QOZ1NuOVFhL1yAAjCI8oZB30m43JE8I0m6aEqjzelYcMBJFKp2Wfk16Hj-" + "Ain797f0u1F1tYJiat67bCfaPFJwWBsHHvyPbJ7NnxFvRYN6F1b8ddT9qAbELxj4fsj1F9rE0O" + "dmcp0lLhUa2OYrpyyipD5hv0eZIj4Yxlt962Qb6ZhewJULKULueIsWFXq3QZ-FPHXO8-B-" + "ZBDv5_INzDpTaUK0htgVMMvcbqCcr2DdAlloaZXUnnEINy-d57SBsw1w"; + +// kTokenWithNbf is the same as kToken but has an additional "nbf" claim set to +// 2461782921. +const char kTokenWithNbf[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImIzMzE5YTE0NzUxNGRmN2VlNWU0Ym" + "NkZWU1MTM1MGNjODkwY2M4OWUifQ.eyJpc3MiOiJodHRwczovL2lzc3VlcjEuY29tIiwic3ViI" + "joiZW5kLXVzZXItaWQiLCJhdWQiOiJlbmRwb2ludHMtdGVzdHMuY2xvdWRlbmRwb2ludHNhcGl" + "zLmNvbSIsImlhdCI6MTQ2MTc3OTMyMSwiZXhwIjoyNDYxNzgyOTIxLCJuYmYiOjI0NjE3ODI5M" + "jF9.eG8k_YeXPrmzkpu88PvvL_sP2FiG_VVsqG6zMxYleBKytGS1cUELVQzzlYNWeUh_w3q6EV" + "r_VeyrhbeUtQEsiDbeqWrz8fSaeUvgg0q1ndMo30YZxGx7gnFq5PKsDyyd_gi20J0P40y5ig5K" + "g4hXKlxpdJkUxwljmzkVvvy5N69EGvfIap474hHGKa1rpMZC2hfxAP0damJBShyGkr9qCnmBKn" + "5X-tA-XrqQjByzcdwu8D9jZSAdXsue285glUwPCpAYlRbrtrlhxHPS2pR2malTcR1PNXYEQD2G" + "x7n57Rg31-DuEoZAqWT5Tsr-cY8rn0cALm0AkFPyyC4OwI0O7w"; + +const char kOpenIdContent[] = "{\"jwks_uri\": \"https://issuer1.com/pubkey\"}"; + +const char kPubkey[] = + "{\"keys\": [{\"kty\": \"RSA\",\"alg\": \"RS256\",\"use\": " + "\"sig\",\"kid\": \"62a93512c9ee4c7f8067b5a216dade2763d32a47\",\"n\": " + "\"0YWnm_eplO9BFtXszMRQNL5UtZ8HJdTH2jK7vjs4XdLkPW7YBkkm_" + "2xNgcaVpkW0VT2l4mU3KftR-6s3Oa5Rnz5BrWEUkCTVVolR7VYksfqIB2I_" + "x5yZHdOiomMTcm3DheUUCgbJRv5OKRnNqszA4xHn3tA3Ry8VO3X7BgKZYAUh9fyZTFLlkeAh0-" + "bLK5zvqCmKW5QgDIXSxUTJxPjZCgfx1vmAfGqaJb-" + "nvmrORXQ6L284c73DUL7mnt6wj3H6tVqPKA27j56N0TB1Hfx4ja6Slr8S4EB3F1luYhATa1PKU" + "SH8mYDW11HolzZmTQpRoLV8ZoHbHEaTfqX_aYahIw\",\"e\": \"AQAB\"},{\"kty\": " + "\"RSA\",\"alg\": \"RS256\",\"use\": \"sig\",\"kid\": " + "\"b3319a147514df7ee5e4bcdee51350cc890cc89e\",\"n\": " + "\"qDi7Tx4DhNvPQsl1ofxxc2ePQFcs-L0mXYo6TGS64CY_" + "2WmOtvYlcLNZjhuddZVV2X88m0MfwaSA16wE-" + "RiKM9hqo5EY8BPXj57CMiYAyiHuQPp1yayjMgoE1P2jvp4eqF-" + "BTillGJt5W5RuXti9uqfMtCQdagB8EC3MNRuU_KdeLgBy3lS3oo4LOYd-" + "74kRBVZbk2wnmmb7IhP9OoLc1-7-9qU1uhpDxmE6JwBau0mDSwMnYDS4G_ML17dC-" + "ZDtLd1i24STUw39KH0pcSdfFbL2NtEZdNeam1DDdk0iUtJSPZliUHJBI_pj8M-2Mn_" + "oA8jBuI8YKwBqYkZCN1I95Q\",\"e\": \"AQAB\"}]}"; + +const char kIssuer1OpenIdUrl[] = + "https://issuer1.com/.well-known/openid-configuration"; + +const char kIssuer1PubkeyUrl[] = "https://issuer1.com/pubkey"; +const char kIssuer2PubkeyUrl[] = "https://issuer2.com/pubkey"; + +const char kOpenIdFailUrl[] = + "http://openid_fail/.well-known/openid-configuration"; + +// The header key to send endpoint api user info. +const char kEndpointApiUserInfo[] = "X-Endpoint-API-UserInfo"; + +// Base64 encoded string of +// { +// "issuer": "https://issuer1.com", +// "id": "end-user-id" +// } +const char kUserInfo_kSub_kIss[] = + "eyJpc3N1ZXIiOiJodHRwczovL2lzc3VlcjEuY29tIiwiaWQiOiJlbmQtdXNlci1pZCJ9"; + +// Base64 encoded string of +// { +// "issuer": "https://issuer1.com", +// "id": "another-user-id" +// } +const char kUserInfo_kSub2_kIss[] = + "eyJpc3N1ZXIiOiJodHRwczovL2lzc3VlcjEuY29tIiwiaWQiOiJhbm90aGVyLXVzZXItaWQif" + "Q=="; + +// Base64 encoded string of +// { +// "issuer": "https://issuer2.com", +// "id": "end-user-id" +// } +const char kUserInfo_kSub_kIss2[] = + "eyJpc3N1ZXIiOiJodHRwczovL2lzc3VlcjIuY29tIiwiaWQiOiJlbmQtdXNlci1pZCJ9"; + +class CheckAuthTest : public ::testing::Test { + public: + void SetUp() { + std::unique_ptr env( + new ::testing::NiceMock()); + // save the raw pointer of env before calling std::move(env). + raw_env_ = env.get(); + + std::unique_ptr config = + Config::Create(raw_env_, kServiceConfig, ""); + ASSERT_NE(config.get(), nullptr); + + service_context_ = std::make_shared( + std::move(env), std::move(config)); + ASSERT_NE(service_context_.get(), nullptr); + + std::unique_ptr request( + new ::testing::NiceMock()); + // save the raw pointer of request before calling std::move(request). + raw_request_ = request.get(); + + EXPECT_CALL(*raw_request_, GetRequestHTTPMethod()) + .WillOnce(Return(std::string("GET"))); + EXPECT_CALL(*raw_request_, GetRequestPath()) + .WillOnce(Return(std::string("/ListShelves"))); + EXPECT_CALL(*raw_request_, FindQuery(_, _)) + .WillOnce(Invoke([](const std::string &, std::string *apikey) { + *apikey = "apikey"; + return true; + })); + EXPECT_CALL(*raw_request_, FindHeader("referer", _)) + .WillOnce(Invoke([](const std::string &, std::string *http_referer) { + *http_referer = ""; + return true; + })); + EXPECT_CALL(*raw_request_, FindHeader("X-Cloud-Trace-Context", _)) + .WillOnce(Invoke([](const std::string &, std::string *trace_context) { + *trace_context = ""; + return true; + })); + context_ = std::make_shared(service_context_, + std::move(request)); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_request_)); + } + + void TestValidToken(const std::string &auth_token); + + MockApiManagerEnvironment *raw_env_; + std::shared_ptr service_context_; + MockRequest *raw_request_; + std::shared_ptr context_; +}; + +void CheckAuthTest::TestValidToken(const std::string &auth_token) { + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([auth_token](const std::string &, std::string *token) { + *token = std::string(kBearer) + auth_token; + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(auth_token)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)) + .Times(2) + .WillOnce(Invoke([](HTTPRequest *req) { + EXPECT_EQ(req->url(), kIssuer1OpenIdUrl); + std::string body(kOpenIdContent); + std::map empty; + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + })) + .WillOnce(Invoke([](HTTPRequest *req) { + EXPECT_EQ(req->url(), kIssuer1PubkeyUrl); + std::string body(kPubkey); + std::map empty; + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + })); + EXPECT_CALL(*raw_request_, + AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub_kIss)) + .WillOnce(Return(utils::Status::OK)); + + CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); }); +} + +// Positive test. +// Step 1: Check auth workflow that involves openID discovery and fetching +// public key. +// Step 2. Use the same auth token, which should be cached in JWT cache. +// Step 3. Use a different auth token signed by the same issuer. This time, +// token is not cached, but key is cached. +TEST_F(CheckAuthTest, TestOKAuth) { + // Step 1. Check auth requires open ID discovery and fetching public key. + TestValidToken(kToken); + + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_request_)); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_env_)); + + // Step 2. Check auth with the same auth token. This time the token is cached. + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kBearer) + std::string(kToken); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kToken)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0); + EXPECT_CALL(*raw_request_, + AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub_kIss)) + .WillOnce(Return(utils::Status::OK)); + + CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); }); + + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_request_)); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_env_)); + + // Step 3. Check auth with a different token signed by the same issuer. + // In this case, the token is not in the cache, but key is cached. + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kBearer) + std::string(kToken2); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kToken2)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0); + EXPECT_CALL(*raw_request_, + AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub2_kIss)) + .WillOnce(Return(utils::Status::OK)); + + CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); }); +} + +// Negative test: Test the case that openID discovery failed. +// Step 1. Try to fetch key URI via OpenID discovery but failed. +// Step 2. Use a different token signed by the same issuer, no HTTP request +// is sent this time because the failure result was cached. +TEST_F(CheckAuthTest, TestOpenIdFailed) { + // Step 1. Try to fetch key URI via OpenID discovery but failed. + // Use FindQuery to get auth token. + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(); + return false; + })); + EXPECT_CALL(*raw_request_, FindQuery(kAccessTokenName, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kTokenOpenIdFail); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kTokenOpenIdFail)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)) + .WillOnce(Invoke([](HTTPRequest *req) { + EXPECT_EQ(req->url(), kOpenIdFailUrl); + std::string body(""); + std::map empty; + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + })); + + CheckAuth(context_, [](Status status) { + ASSERT_EQ(status.code(), Code::UNAUTHENTICATED); + ASSERT_EQ(status.message(), + "JWT validation failed: Unable to parse " + "URI of the key via OpenID discovery"); + }); + + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_request_)); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_env_)); + + // Step 2. Use a different token signed by the same issuer, no HTTP request + // is sent this time because the failure result was cached. + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kBearer) + std::string(kTokenOpenIdFail2); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kTokenOpenIdFail2)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0); + + CheckAuth(context_, [](Status status) { + ASSERT_EQ(status.code(), Code::UNAUTHENTICATED); + ASSERT_EQ(status.message(), + "JWT validation failed: " + "Cannot determine the URI of the key"); + }); +} + +// jwks_uri is already specified in service config. Hence, no need to +// do openID discovery. +TEST_F(CheckAuthTest, TestNoOpenId) { + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kBearer) + std::string(kTokenIssuer2); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kTokenIssuer2)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)) + .WillOnce(Invoke([](HTTPRequest *req) { + EXPECT_EQ(req->url(), kIssuer2PubkeyUrl); + std::string body(kPubkey); + std::map empty; + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + })); + EXPECT_CALL(*raw_request_, + AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub_kIss2)) + .WillOnce(Return(utils::Status::OK)); + + CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); }); +} + +// Negative test: invalid token and expired token. +TEST_F(CheckAuthTest, TestInvalidToken) { + // Invalid token. + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = "bad_token"; + return true; + })); + + CheckAuth(context_, [](Status status) { + ASSERT_EQ(status.code(), Code::UNAUTHENTICATED); + ASSERT_EQ(status.message(), + "JWT validation failed: " + "Missing or invalid credentials"); + }); + + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_request_)); + + // Expired token. + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kBearer) + std::string(kTokenExpired); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kTokenExpired)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0); + + CheckAuth(context_, [](Status status) { + ASSERT_EQ(status.code(), Code::UNAUTHENTICATED); + ASSERT_EQ(status.message(), + "JWT validation failed: TIME_CONSTRAINT_FAILURE"); + }); + + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_request_)); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_env_)); + + // Token that is not ready to be used (i.e., current time is less than the + // "nbf" claim). + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kBearer) + std::string(kTokenWithNbf); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kTokenWithNbf)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0); + + CheckAuth(context_, [](Status status) { + ASSERT_EQ(status.code(), Code::UNAUTHENTICATED); + ASSERT_EQ(status.message(), + "JWT validation failed: TIME_CONSTRAINT_FAILURE"); + }); +} + +// Negative test: bad audience +TEST_F(CheckAuthTest, TestBadAudience) { + EXPECT_CALL(*raw_request_, FindHeader(kAuthHeader, _)) + .WillOnce(Invoke([](const std::string &, std::string *token) { + *token = std::string(kBearer) + std::string(kTokenBadAud); + return true; + })); + EXPECT_CALL(*raw_request_, SetAuthToken(kTokenBadAud)).Times(1); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0); + + CheckAuth(context_, [](Status status) { + ASSERT_EQ(status.code(), Code::PERMISSION_DENIED); + ASSERT_EQ(status.message(), "JWT validation failed: Audience not allowed"); + }); +} + +// Positive test: audience is service name with https prefix. +TEST_F(CheckAuthTest, TestHttpsAudience) { TestValidToken(kTokenHttpsAud); } + +// Positive test: audience is service name with https prefix and a trailing +// slash. +TEST_F(CheckAuthTest, TestHttpsSlashAudience) { + TestValidToken(kTokenHttpsSlashAud); +} + +// Positive test: audience is service name with http prefix. +TEST_F(CheckAuthTest, TestHttpAudience) { TestValidToken(kTokenHttpAud); } + +// Positive test: audience is service name with http prefix and a trailing +// slash. +TEST_F(CheckAuthTest, TestHttpSlashAudience) { + TestValidToken(kTokenHttpSlashAud); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/check_service_control.cc b/contrib/endpoints/src/api_manager/check_service_control.cc new file mode 100644 index 00000000000..5838f5f5c7b --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_service_control.cc @@ -0,0 +1,84 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// includes should be ordered. This seems like a bug in clang-format? +#include "src/api_manager/check_service_control.h" +#include "google/protobuf/stubs/status.h" +#include "src/api_manager/cloud_trace/cloud_trace.h" + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +void CheckServiceControl(std::shared_ptr context, + std::function continuation) { + std::shared_ptr trace_span( + CreateSpan(context->cloud_trace(), "CheckServiceControl")); + // If the method is not configured from the service config. + // or if not need to check service control, skip it. + if (!context->method()) { + if (context->request()->GetRequestHTTPMethod() == "OPTIONS") { + TRACE(trace_span) << "OPTIONS request is rejected"; + continuation(Status(Code::PERMISSION_DENIED, + "The service does not allow CORS traffic.", + Status::SERVICE_CONTROL)); + } else { + TRACE(trace_span) << "Method is not configured in the service config"; + continuation(Status(Code::NOT_FOUND, "Method does not exist.", + Status::SERVICE_CONTROL)); + } + return; + } else if (!context->service_context()->service_control()) { + TRACE(trace_span) << "Service control check is not needed"; + continuation(Status::OK); + return; + } + + if (context->api_key().empty()) { + if (context->method()->allow_unregistered_calls()) { + // Not need to call Check. + TRACE(trace_span) << "Service control check is not needed"; + continuation(Status::OK); + return; + } + + TRACE(trace_span) << "Failed at checking caller identity."; + continuation( + Status(Code::UNAUTHENTICATED, + "Method doesn't allow unregistered callers (callers without " + "established identity). Please use API Key or other form of " + "API consumer identity to call this API.", + Status::SERVICE_CONTROL)); + return; + } + + service_control::CheckRequestInfo info; + context->FillCheckRequestInfo(&info); + context->service_context()->service_control()->Check( + info, trace_span.get(), + [context, continuation, trace_span]( + Status status, const service_control::CheckResponseInfo &info) { + TRACE(trace_span) << "Check service control request returned with " + << "status " << status.ToString(); + // info is valid regardless status. + context->set_check_response_info(info); + continuation(status); + }); +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/check_service_control.h b/contrib/endpoints/src/api_manager/check_service_control.h new file mode 100644 index 00000000000..f6c0344e801 --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_service_control.h @@ -0,0 +1,31 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_CHECK_SERVICE_CONTROL_H_ +#define API_MANAGER_CHECK_SERVICE_CONTROL_H_ + +#include "include/api_manager/utils/status.h" +#include "src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// Call service control check. +void CheckServiceControl(std::shared_ptr, + std::function); + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_CHECK_SERVICE_CONTROL_H_ diff --git a/contrib/endpoints/src/api_manager/check_workflow.cc b/contrib/endpoints/src/api_manager/check_workflow.cc new file mode 100644 index 00000000000..2dc1b115985 --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_workflow.cc @@ -0,0 +1,63 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +#include "src/api_manager/check_workflow.h" +#include "src/api_manager/check_auth.h" +#include "src/api_manager/check_service_control.h" +#include "src/api_manager/fetch_metadata.h" + +using ::google::api_manager::utils::Status; + +namespace google { +namespace api_manager { + +void CheckWorkflow::RegisterAll() { + // Fetchs GCE metadata. + Register(FetchGceMetadata); + // Fetchs service account token. + Register(FetchServiceAccountToken); + // Authentication checks. + Register(CheckAuth); + // Checks service control. + Register(CheckServiceControl); +} + +void CheckWorkflow::Register(CheckHandler handler) { + handlers_.push_back(handler); +} + +void CheckWorkflow::Run(std::shared_ptr context) { + if (!handlers_.empty()) { + RunOneHandler(context, 0); + } else { + // Empty check handler list means: not need to check. + context->CompleteCheck(Status::OK); + } +} + +void CheckWorkflow::RunOneHandler( + std::shared_ptr context, size_t index) { + handlers_[index](context, [context, index, this](Status status) { + if (status.ok() && index + 1 < handlers_.size()) { + RunOneHandler(context, index + 1); + } else { + context->CompleteCheck(status); + } + }); +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/check_workflow.h b/contrib/endpoints/src/api_manager/check_workflow.h new file mode 100644 index 00000000000..f4820fe2825 --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_workflow.h @@ -0,0 +1,56 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_CHECK_WORKFLOW_H_ +#define API_MANAGER_CHECK_WORKFLOW_H_ + +#include "include/api_manager/utils/status.h" +#include "src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// The prototype for CheckHandler +typedef std::function, + std::function)> + CheckHandler; + +// A workflow to run all CheckHandlers +class CheckWorkflow { + public: + virtual ~CheckWorkflow() {} + + // Registers all known check handlers. + void RegisterAll(); + + // Runs the workflow to call each check handler sequentially. + void Run(std::shared_ptr context); + + private: + // Registers a check handler. The order is important. + // They will be executed in the order they are registered. + void Register(CheckHandler handler); + + // Runs one check handler with index. + void RunOneHandler(std::shared_ptr context, + size_t index); + + // A vector to store all check handlers. + std::vector handlers_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_CHECK_WORKFLOW_H_ diff --git a/contrib/endpoints/src/api_manager/cloud_trace/BUILD b/contrib/endpoints/src/api_manager/cloud_trace/BUILD new file mode 100644 index 00000000000..79d95f7a32d --- /dev/null +++ b/contrib/endpoints/src/api_manager/cloud_trace/BUILD @@ -0,0 +1,57 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +package(default_visibility = ["//src/api_manager:__subpackages__"]) + +cc_library( + name = "cloud_trace", + srcs = [ + "cloud_trace.cc", + "cloud_trace.h", + ], + deps = [ + "//src/api_manager/auth:service_account_token", + "//external:cloud_trace", + ], +) + +cc_test( + name = "cloud_trace_test", + size = "small", + srcs = [ + "cloud_trace_test.cc", + ], + linkstatic = 1, + deps = [ + ":cloud_trace", + "//external:googletest_main", + "//src/api_manager:mock_api_manager_environment", + "//external:cloud_trace", + ], +) + +cc_test( + name = "sampler_test", + size = "small", + srcs = [ + "sampler_test.cc", + ], + linkstatic = 1, + deps = [ + ":cloud_trace", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.cc b/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.cc new file mode 100644 index 00000000000..97ea66aba74 --- /dev/null +++ b/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.cc @@ -0,0 +1,418 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "cloud_trace.h" + +#include +#include +#include +#include +#include +#include +#include "google/protobuf/timestamp.pb.h" +#include "include/api_manager/utils/status.h" +#include "include/api_manager/version.h" +#include "src/api_manager/utils/marshalling.h" + +using google::api_manager::utils::Status; +using google::devtools::cloudtrace::v1::Trace; +using google::devtools::cloudtrace::v1::Traces; +using google::devtools::cloudtrace::v1::TraceSpan; +using google::devtools::cloudtrace::v1::TraceSpan_SpanKind; +using google::protobuf::Timestamp; + +namespace google { +namespace api_manager { +namespace cloud_trace { +namespace { + +const char kCloudTraceService[] = "/google.devtools.cloudtrace.v1.TraceService"; +// Cloud Trace agent label key +const char kCloudTraceAgentKey[] = "trace.cloud.google.com/agent"; +// Cloud Trace agent label value +const char kServiceAgent[] = "esp/" API_MANAGER_VERSION_STRING; +// Default trace options +const char kDefaultTraceOptions[] = "o=1"; + +// Generate a random unsigned 64-bit integer. +uint64_t RandomUInt64(); + +// Get a random string of 128 bit hex number +std::string RandomUInt128HexString(); + +// Get the timestamp for now. +void GetNow(Timestamp *ts); + +// Get a new Trace object stored in the trace parameter. The new object has +// the given trace id and contains a root span with default settings. +void GetNewTrace(std::string trace_id_str, const std::string &root_span_name, + Trace **trace); + +// Parse the trace context header. +// Assigns Trace object to the trace pointer if context is parsed correctly and +// trace is enabled. Otherwise the pointer is not modified. +// If trace is enabled, the option will be modified to the one passed in. +// +// Grammar of the context header: +// trace-id [“/” span-id] [ “;” “o” “=” trace_options ] +// +// trace-id := hex representation of a 128 bit value +// span-id := decimal representation of a 64 bit value +// trace-options := decimal representation of a 32 bit value +// +void GetTraceFromContextHeader(const std::string &trace_context, + const std::string &root_span_name, Trace **trace, + std::string *options); +} // namespace + +Sampler::Sampler(double qps) { + if (qps == 0.0) { + is_disabled_ = true; + } else { + duration_ = 1.0 / qps; + is_disabled_ = false; + } +} + +bool Sampler::On() { + if (is_disabled_) { + return false; + } + auto now = std::chrono::system_clock::now(); + std::chrono::duration diff = now - previous_; + if (diff.count() > duration_) { + previous_ = now; + return true; + } else { + return false; + } +}; + +void Sampler::Refresh() { + if (is_disabled_) { + return; + } + previous_ = std::chrono::system_clock::now(); +} + +Aggregator::Aggregator(auth::ServiceAccountToken *sa_token, + const std::string &cloud_trace_address, + int aggregate_time_millisec, int cache_max_size, + double minimum_qps, ApiManagerEnvInterface *env) + : sa_token_(sa_token), + cloud_trace_address_(cloud_trace_address), + aggregate_time_millisec_(aggregate_time_millisec), + cache_max_size_(cache_max_size), + traces_(new Traces), + env_(env), + sampler_(minimum_qps) { + sa_token_->SetAudience(auth::ServiceAccountToken::JWT_TOKEN_FOR_CLOUD_TRACING, + cloud_trace_address_ + kCloudTraceService); +} + +void Aggregator::Init() { + if (aggregate_time_millisec_ == 0) { + return; + } + timer_ = env_->StartPeriodicTimer( + std::chrono::milliseconds(aggregate_time_millisec_), + [this]() { SendAndClearTraces(); }); +} + +Aggregator::~Aggregator() { + if (timer_) { + timer_->Stop(); + } +} + +void Aggregator::SendAndClearTraces() { + if (traces_->traces_size() == 0 || project_id_.empty()) { + env_->LogDebug( + "Not sending request to CloudTrace: no traces or " + "project_id is empty."); + traces_->clear_traces(); + return; + } + + // Add project id into each trace object. + for (int i = 0; i < traces_->traces_size(); ++i) { + traces_->mutable_traces(i)->set_project_id(project_id_); + } + + std::unique_ptr http_request(new HTTPRequest( + [this](Status status, std::map &&, + std::string &&body) { + if (status.code() < 0) { + env_->LogError("Trace Request Failed." + status.ToString()); + } else { + env_->LogDebug("Trace Response: " + status.ToString() + "\n" + body); + } + })); + + std::string url = + cloud_trace_address_ + "/v1/projects/" + project_id_ + "/traces"; + + std::string request_body; + + ProtoToJson(*traces_, &request_body, utils::DEFAULT); + traces_->clear_traces(); + env_->LogDebug("Sending request to Cloud Trace."); + env_->LogDebug(request_body); + + http_request->set_url(url) + .set_method("PATCH") + .set_auth_token(sa_token_->GetAuthToken( + auth::ServiceAccountToken::JWT_TOKEN_FOR_CLOUD_TRACING)) + .set_header("Content-Type", "application/json") + .set_body(request_body); + + env_->RunHTTPRequest(std::move(http_request)); +} + +void Aggregator::AppendTrace(google::devtools::cloudtrace::v1::Trace *trace) { + traces_->mutable_traces()->AddAllocated(trace); + if (traces_->traces_size() > cache_max_size_) { + SendAndClearTraces(); + } +} + +CloudTrace::CloudTrace(Trace *trace, const std::string &options) + : trace_(trace), options_(options) { + // Root span must exist and must be the only span as of now. + root_span_ = trace_->mutable_spans(0); +} + +void CloudTrace::SetProjectId(const std::string &project_id) { + trace_->set_project_id(project_id); +} + +void CloudTrace::EndRootSpan() { GetNow(root_span_->mutable_end_time()); } + +CloudTraceSpan::CloudTraceSpan(CloudTrace *cloud_trace, + const std::string &span_name) + : cloud_trace_(cloud_trace) { + InitWithParentSpanId(span_name, cloud_trace_->root_span()->span_id()); +} + +CloudTraceSpan::CloudTraceSpan(CloudTraceSpan *parent, + const std::string &span_name) + : cloud_trace_(parent->cloud_trace_) { + InitWithParentSpanId(span_name, parent->trace_span_->span_id()); +} + +void CloudTraceSpan::InitWithParentSpanId(const std::string &span_name, + protobuf::uint64 parent_span_id) { + // TODO: this if is not needed, and probably the following two as well. + // Fully test and remove them. + if (!cloud_trace_) { + // Trace is disabled. + return; + } + trace_span_ = cloud_trace_->trace()->add_spans(); + trace_span_->set_kind(TraceSpan_SpanKind::TraceSpan_SpanKind_RPC_SERVER); + trace_span_->set_span_id(RandomUInt64()); + trace_span_->set_parent_span_id(parent_span_id); + trace_span_->set_name(span_name); + GetNow(trace_span_->mutable_start_time()); +} + +CloudTraceSpan::~CloudTraceSpan() { + if (!cloud_trace_) { + // Trace is disabled. + return; + } + GetNow(trace_span_->mutable_end_time()); + for (unsigned int i = 0; i < messages.size(); ++i) { + std::stringstream stream; + stream << std::setfill('0') << std::setw(3) << i; + std::string sequence = stream.str(); + trace_span_->mutable_labels()->insert({sequence, messages[i]}); + } +} + +void CloudTraceSpan::Write(const std::string &msg) { + if (!cloud_trace_) { + // Trace is disabled. + return; + } + messages.push_back(msg); +} + +CloudTrace *CreateCloudTrace(const std::string &trace_context, + const std::string &root_span_name, + Sampler *sampler) { + Trace *trace = nullptr; + std::string options; + GetTraceFromContextHeader(trace_context, root_span_name, &trace, &options); + if (trace) { + // When trace is triggered by the context header, refresh the previous + // timestamp in sampler. + if (sampler) { + sampler->Refresh(); + } + return new CloudTrace(trace, options); + } else if (sampler && sampler->On()) { + // Trace is turned on by sampler. + GetNewTrace(RandomUInt128HexString(), root_span_name, &trace); + return new CloudTrace(trace, kDefaultTraceOptions); + } else { + return nullptr; + } +} + +CloudTraceSpan *CreateSpan(CloudTrace *cloud_trace, const std::string &name) { + if (cloud_trace != nullptr) { + return new CloudTraceSpan(cloud_trace, name); + } else { + return nullptr; + } +} + +CloudTraceSpan *CreateChildSpan(CloudTraceSpan *parent, + const std::string &name) { + if (parent != nullptr) { + return new CloudTraceSpan(parent, name); + } else { + return nullptr; + } +} + +TraceStream::~TraceStream() { trace_span_->Write(info_.str()); } + +namespace { +// TODO: this method is duplicated with a similar method in +// api_manager/context/request_context.cc. Consider merge them by moving to a +// common position. +uint64_t RandomUInt64() { + static std::random_device random_device; + static std::mt19937 generator(random_device()); + static std::uniform_int_distribution distribution; + return distribution(generator); +} + +std::string RandomUInt128HexString() { + std::stringstream stream; + + stream << std::setfill('0') << std::setw(sizeof(uint64_t) * 2) << std::hex + << RandomUInt64(); + stream << std::setfill('0') << std::setw(sizeof(uint64_t) * 2) << std::hex + << RandomUInt64(); + + return stream.str(); +} + +void GetNow(Timestamp *ts) { + long long nanos = + std::chrono::duration_cast( + std::chrono::high_resolution_clock::now().time_since_epoch()) + .count(); + ts->set_seconds(nanos / 1000000000); + ts->set_nanos(nanos % 1000000000); +} + +void GetNewTrace(std::string trace_id_str, const std::string &root_span_name, + Trace **trace) { + *trace = new Trace; + (*trace)->set_trace_id(trace_id_str); + TraceSpan *root_span = (*trace)->add_spans(); + root_span->set_kind(TraceSpan_SpanKind::TraceSpan_SpanKind_RPC_SERVER); + root_span->set_span_id(RandomUInt64()); + root_span->set_name(root_span_name); + // Agent label is defined as "/". + root_span->mutable_labels()->insert({kCloudTraceAgentKey, kServiceAgent}); + GetNow(root_span->mutable_start_time()); +} + +void GetTraceFromContextHeader(const std::string &trace_context, + const std::string &root_span_name, Trace **trace, + std::string *options) { + std::stringstream header_stream(trace_context); + + std::string trace_and_span_id; + if (!getline(header_stream, trace_and_span_id, ';')) { + // When trace_context is empty; + return; + } + + bool trace_enabled = false; + std::string item; + while (getline(header_stream, item, ';')) { + if (item.substr(0, 2) == "o=") { + int value; + std::stringstream option_stream(item.substr(2)); + if ((option_stream >> value).fail() || !option_stream.eof()) { + return; + } + if (value < 0 || value > 0b11) { + // invalid option value. + return; + } + *options = trace_context.substr(trace_context.find_first_of(';') + 1); + // First bit indicates whether trace is enabled. + if (!(value & 1)) { + return; + } + // Trace is enabled, we can stop parsing the header. + trace_enabled = true; + break; + } + } + if (!trace_enabled) { + return; + } + + // Parse trace_id/span_id + std::string trace_id_str, span_id_str; + size_t slash_pos = trace_and_span_id.find_first_of('/'); + if (slash_pos == std::string::npos) { + trace_id_str = trace_and_span_id; + } else { + trace_id_str = trace_and_span_id.substr(0, slash_pos); + span_id_str = trace_and_span_id.substr(slash_pos + 1); + } + + // Trace id should be a 128-bit hex number (32 hex digits). + if (trace_id_str.size() != 32) { + return; + } + for (size_t i = 0; i < trace_id_str.size(); ++i) { + if (!isxdigit(trace_id_str[i])) { + return; + } + } + + uint64_t span_id = 0; + // Parse the span id, disable trace when span id is illegal. + if (!span_id_str.empty()) { + std::stringstream span_id_stream(span_id_str); + if ((span_id_stream >> span_id).fail() || !span_id_stream.eof()) { + return; + } + } + + // At this point, trace is enabled and trace id is successfully parsed. + GetNewTrace(trace_id_str, root_span_name, trace); + TraceSpan *root_span = (*trace)->mutable_spans(0); + // Set parent of root span to the given one if provided. + if (span_id != 0) { + root_span->set_parent_span_id(span_id); + } +} + +} // namespace +} // cloud_trace +} // api_manager +} // google diff --git a/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.h b/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.h new file mode 100644 index 00000000000..d5d0534e024 --- /dev/null +++ b/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.h @@ -0,0 +1,249 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// +#ifndef API_MANAGER_CLOUD_TRACE_CLOUD_TRACE_H_ +#define API_MANAGER_CLOUD_TRACE_CLOUD_TRACE_H_ + +#include +#include + +#include "google/devtools/cloudtrace/v1/trace.pb.h" +#include "google/protobuf/map.h" +#include "include/api_manager/env_interface.h" +#include "include/api_manager/periodic_timer.h" +#include "src/api_manager/auth/service_account_token.h" + +namespace google { +namespace api_manager { +namespace cloud_trace { + +// A helper class to determine if trace should be enabled for a request. +// A Sampler instance is put into the Aggregator class. +// Trace is triggered if the time interval between the request time and the +// previous trace enabled request is bigger than a threshold. +// The threshold is calculated from the qps. +class Sampler { + public: + Sampler(double qps); + + // Returns whether trace should be turned on for this request. + bool On(); + + // Refresh the previous timestamp to the current time. + void Refresh(); + + private: + bool is_disabled_; + std::chrono::time_point previous_; + double duration_; +}; + +// TODO: The Aggregator class is not thread safe. +// TODO: simplify class naming in this file. +// Stores cloud trace configurations shared within the job. There should be +// only one such instance. The instance is put in service_context. +class Aggregator final { + public: + Aggregator(auth::ServiceAccountToken *sa_token, + const std::string &cloud_trace_address, + int aggregate_time_millisec, int cache_max_size, + double minimum_qps, ApiManagerEnvInterface *env); + + ~Aggregator(); + + // Initializes the aggregator by setting up a periodic timer. At each timer + // invocation traces aggregated are sent to Cloud Trace API + void Init(); + + // Flush traces cached and clear the traces_ proto. + void SendAndClearTraces(); + + // Append a Trace to traces_, the appended trace may not be sent at the time + // of this function call. + void AppendTrace(google::devtools::cloudtrace::v1::Trace *trace); + + // Sets the producer project id + void SetProjectId(const std::string &project_id) { project_id_ = project_id; } + + // Get the sampler. + Sampler &sampler() { return sampler_; } + + private: + // ServiceAccountToken object to get auth tokens for Cloud Trace API. + auth::ServiceAccountToken *sa_token_; + + // Address for Cloud Trace API + std::string cloud_trace_address_; + + // The maximum time to hold a trace before sent to Cloud Trace. + int aggregate_time_millisec_; + + // The maximum number of traces that can be cached. + int cache_max_size_; + + // Traces protobuf to hold a list of Trace obejcts. + std::unique_ptr traces_; + + // The producer project id. + std::string project_id_; + + // ApiManager Env used to set up periodic timer. + ApiManagerEnvInterface *env_; + + // Timer to trigger flush trace. + std::unique_ptr timer_; + + // Sampler object to help determine if trace should be enabled for a request. + Sampler sampler_; +}; + +// Stores traces and metadata for one request. The instance of this class is +// put in request_context. +// When a CloudTrace instance is initialized, it creates a RootSpan called +// ESP_ROOT that will be a parent span of all other trace spans. Start time +// of this root span is recorded in constructor and end time is recorded when +// EndRootSpan is called. +class CloudTrace final { + public: + // Construct with give Trace proto object. This constructor must only be + // called with non-null pointer. + CloudTrace(google::devtools::cloudtrace::v1::Trace *trace, + const std::string &options); + + void SetProjectId(const std::string &project_id); + + void EndRootSpan(); + + google::devtools::cloudtrace::v1::TraceSpan *root_span() { + return root_span_; + } + + google::devtools::cloudtrace::v1::Trace *trace() { return trace_.get(); } + + // Releases ownership of all traces and returns it. + google::devtools::cloudtrace::v1::Trace *ReleaseTrace() { + return trace_.release(); + } + + const std::string &options() const { return options_; } + + private: + std::unique_ptr trace_; + google::devtools::cloudtrace::v1::TraceSpan *root_span_; + std::string options_; +}; + +// This class stores messages written to a single trace span. There can be +// multiple trace spans for one request. Typically an instance of this class is +// initialized at the beginning of a function that needs to be traced. +// +// When an instance of CloudTraceSpan is destructed, its messages are written +// to the corresponding CloudTrace. +// +// Start time and end time of the trace span is recorded in constructor and +// destructor. +// +// Example of initializing a trace span: +// std::shared_ptr trace_span(GetTraceSpan(cloud_trace, +// "MyFunc")); +// +class CloudTraceSpan { + public: + // Initializes a trace span whose parent is the api manager root. + CloudTraceSpan(CloudTrace *cloud_trace, const std::string &span_name); + + // Initializes a trace span using the given trace span as parent. + CloudTraceSpan(CloudTraceSpan *parent, const std::string &span_name); + + ~CloudTraceSpan(); + + const google::devtools::cloudtrace::v1::TraceSpan *trace_span() const { + return trace_span_; + } + + private: + friend class TraceStream; + void Write(const std::string &msg); + void InitWithParentSpanId(const std::string &span_name, + protobuf::uint64 parent_span_id); + CloudTrace *cloud_trace_; + google::devtools::cloudtrace::v1::TraceSpan *trace_span_; + std::vector messages; +}; + +// Parses the trace_context and determines if cloud trace should +// be produced for the request. If so, creates an initialized CloudTrace object. +// Otherwise return nullptr. +// For trace_context, pass the value of "X-Cloud-Trace-Context" HTTP header. +CloudTrace *CreateCloudTrace(const std::string &trace_context, + const std::string &root_span_name, + Sampler *sampler = nullptr); + +// Creates trace span if trace is enabled. +// Returns nullptr when cloud_trace is nullptr. +CloudTraceSpan *CreateSpan(CloudTrace *cloud_trace, const std::string &name); + +// Creates a child trace span with the given parent span. +// Returns nullptr if parent is nullptr. +CloudTraceSpan *CreateChildSpan(CloudTraceSpan *parent, + const std::string &name); + +// A helper class to create a stream-like write traces interface. +// +class TraceStream { + public: + TraceStream(std::shared_ptr trace_span) + : trace_span_(trace_span.get()){}; + + ~TraceStream(); + + template + TraceStream &operator<<(T const &value) { + info_ << value; + return *this; + } + + private: + CloudTraceSpan *trace_span_; + std::ostringstream info_; +}; + +// This class is used to explicitly ignore values in the conditional +// tracing macro TRACE_ENABLED, i.e. when trace is disabled this class becomes +// a no-op prefix. +class TraceMessageVoidify { + public: + TraceMessageVoidify() {} + // This has to be an operator with a precedence lower than << but + // higher than ?: + void operator&(const TraceStream &trace_out) {} +}; + +} // cloud_trace +} // api_manager +} // google + +// A helper macro to make tracing conditional. +#define TRACE_ENABLED(trace_span) \ + !(trace_span) ? (void)0 \ + : ::google::api_manager::cloud_trace::TraceMessageVoidify() & + +// The macro to write traces. Example usage: +// TRACE(trace_span) << "Some message: " << some_str; +#define TRACE(trace_span) \ + TRACE_ENABLED(trace_span) \ + ::google::api_manager::cloud_trace::TraceStream((trace_span)) + +#endif // API_MANAGER_CLOUD_TRACE_CLOUD_TRACE_H_ diff --git a/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace_test.cc b/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace_test.cc new file mode 100644 index 00000000000..f9b4deb1d73 --- /dev/null +++ b/contrib/endpoints/src/api_manager/cloud_trace/cloud_trace_test.cc @@ -0,0 +1,205 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/cloud_trace/cloud_trace.h" + +#include "google/devtools/cloudtrace/v1/trace.pb.h" +#include "gtest/gtest.h" +#include "src/api_manager/mock_api_manager_environment.h" + +using google::devtools::cloudtrace::v1::TraceSpan; + +namespace google { +namespace api_manager { +namespace cloud_trace { +namespace { + +class CloudTraceTest : public ::testing::Test { + public: + void SetUp() { + env_.reset(new ::testing::NiceMock()); + sa_token_ = std::unique_ptr( + new auth::ServiceAccountToken(env_.get())); + } + + std::unique_ptr env_; + std::unique_ptr sa_token_; +}; + +TEST_F(CloudTraceTest, TestCloudTrace) { + std::unique_ptr cloud_trace( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=1", "root-span")); + ASSERT_TRUE(cloud_trace); + + // After created, there should be a root span. + ASSERT_EQ(cloud_trace->trace()->spans_size(), 1); + ASSERT_EQ(cloud_trace->trace()->spans(0).name(), "root-span"); + // End time should be empty now. + ASSERT_EQ(cloud_trace->trace()->spans(0).end_time().DebugString(), ""); + + TraceSpan *trace_span = cloud_trace->trace()->add_spans(); + trace_span->set_name("Span1"); + + ASSERT_EQ(cloud_trace->trace()->spans_size(), 2); + ASSERT_EQ(cloud_trace->trace()->spans(1).name(), "Span1"); + + std::shared_ptr cloud_trace_span( + CreateSpan(cloud_trace.get(), "Span2")); + TRACE(cloud_trace_span) << "Message"; + cloud_trace_span.reset(); + + ASSERT_EQ(cloud_trace->trace()->spans_size(), 3); + ASSERT_EQ(cloud_trace->trace()->spans(2).name(), "Span2"); + ASSERT_EQ(cloud_trace->trace()->spans(2).labels().size(), 1); + ASSERT_EQ(cloud_trace->trace()->spans(2).labels().find("000")->second, + "Message"); + + cloud_trace->EndRootSpan(); + // After EndRootSpan, end time should not be empty. + ASSERT_NE(cloud_trace->trace()->spans(0).end_time().DebugString(), ""); +} + +TEST_F(CloudTraceTest, TestCloudTraceSpanDisabled) { + std::shared_ptr cloud_trace_span(CreateSpan(nullptr, "Span")); + // Ensure no core dump calling TRACE when cloud_trace_span is nullptr. + TRACE(cloud_trace_span) << "Message"; + ASSERT_FALSE(cloud_trace_span); +} + +TEST_F(CloudTraceTest, TestParseContextHeader) { + // Disabled if empty. + ASSERT_EQ(nullptr, CreateCloudTrace("", "")); + // Disabled for malformed prefix + ASSERT_EQ(nullptr, CreateCloudTrace("o=1", "")); + ASSERT_EQ(nullptr, CreateCloudTrace("o=", "")); + ASSERT_EQ(nullptr, CreateCloudTrace("o=1foo", "")); + ASSERT_EQ(nullptr, CreateCloudTrace("o=foo1", "")); + ASSERT_EQ(nullptr, CreateCloudTrace("o=113471230948140", "")); + ASSERT_EQ(nullptr, CreateCloudTrace("o=1;foo=bar", "")); + ASSERT_EQ(nullptr, CreateCloudTrace(";o=", "")); + ASSERT_EQ(nullptr, CreateCloudTrace(";o=1", "")); + ASSERT_EQ(nullptr, CreateCloudTrace(";o=1foo", "")); + ASSERT_EQ(nullptr, CreateCloudTrace(";o=foo1", "")); + ASSERT_EQ(nullptr, CreateCloudTrace(";o=113471230948140", "")); + ASSERT_EQ(nullptr, CreateCloudTrace(";o=1;foo=bar", "")); + // Disabled if trace id length < 32 + ASSERT_EQ(nullptr, CreateCloudTrace("123;o=1", "")); + // Disabled if trace id is not hex string ('q' in last position) + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962dq;o=1", "")); + // Disabled if no option invalid or not provided. + ASSERT_EQ(nullptr, CreateCloudTrace("e133eacd437d8a12068fd902af3962d8", "")); + ASSERT_EQ(nullptr, CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=4", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=-1", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=12345", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=1foo", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=foo1", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace( + "e133eacd437d8a12068fd902af3962d8;o=113471230948140", "")); + ASSERT_EQ(nullptr, CreateCloudTrace("e133eacd437d8a12068fd902af3962d8", "")); + // Disabled if option explicitly says so. Note: first bit of number "o" + // indicated whether trace is enabled. + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=0", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=2", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=0;o=1", "")); + // Disabled if span id is illegal + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8/xx;o=1", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8/1xx;o=1", "")); + ASSERT_EQ(nullptr, + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8/xx1;o=1", "")); + ASSERT_EQ( + nullptr, + CreateCloudTrace( + "e133eacd437d8a12068fd902af3962d8/18446744073709551616;o=1", "")); + + std::unique_ptr cloud_trace; + + // parent trace id should be 0(default) if span id is not provided. + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=1", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ(0, cloud_trace->root_span()->parent_span_id()); + ASSERT_EQ(1, cloud_trace->trace()->spans_size()); + ASSERT_EQ("o=1", cloud_trace->options()); + + // Should also be enabled for "o=3" + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=3", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ("o=3", cloud_trace->options()); + + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=1;", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ("o=1;", cloud_trace->options()); + + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=1;o=0", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ("o=1;o=0", cloud_trace->options()); + + // Verify capital hex digits should pass + cloud_trace.reset( + CreateCloudTrace("46F1ADB8573CC0F3C4156B5EA7E0E3DC;o=1", "")); + ASSERT_TRUE(cloud_trace); + + // Parent trace id should be set if span id is provided. + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8/12345;o=1", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ(12345, cloud_trace->root_span()->parent_span_id()); + // Parent trace id is max uint64 + cloud_trace.reset(CreateCloudTrace( + "e133eacd437d8a12068fd902af3962d8/18446744073709551615;o=1", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ(18446744073709551615U, cloud_trace->root_span()->parent_span_id()); + + // Should not crash if unrecognized option is provided. + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;foo=bar;o=1", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ("foo=bar;o=1", cloud_trace->options()); + + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;x;o=1", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ("x;o=1", cloud_trace->options()); + + cloud_trace.reset( + CreateCloudTrace("e133eacd437d8a12068fd902af3962d8;o=1;foo=bar", "")); + ASSERT_TRUE(cloud_trace); + ASSERT_EQ("o=1;foo=bar", cloud_trace->options()); +} + +} // namespace + +} // cloud_trace +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/cloud_trace/sampler_test.cc b/contrib/endpoints/src/api_manager/cloud_trace/sampler_test.cc new file mode 100644 index 00000000000..015fab15cb7 --- /dev/null +++ b/contrib/endpoints/src/api_manager/cloud_trace/sampler_test.cc @@ -0,0 +1,72 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/cloud_trace/cloud_trace.h" + +#include +#include + +#include "google/devtools/cloudtrace/v1/trace.pb.h" +#include "gtest/gtest.h" + +using google::devtools::cloudtrace::v1::TraceSpan; + +namespace google { +namespace api_manager { +namespace cloud_trace { +namespace { + +class SamplerTest : public ::testing::Test {}; + +TEST_F(SamplerTest, TestDefaultSetting) { + Sampler sampler(0.1); + + ASSERT_TRUE(sampler.On()); + ASSERT_FALSE(sampler.On()); +} + +TEST_F(SamplerTest, TestLargeQps) { + Sampler sampler(1.0); + + ASSERT_TRUE(sampler.On()); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + ASSERT_FALSE(sampler.On()); + std::this_thread::sleep_for(std::chrono::milliseconds(800)); + ASSERT_TRUE(sampler.On()); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + ASSERT_FALSE(sampler.On()); +} + +TEST_F(SamplerTest, TestRefresh) { + Sampler sampler(1.0); + ASSERT_TRUE(sampler.On()); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + ASSERT_FALSE(sampler.On()); + sampler.Refresh(); + std::this_thread::sleep_for(std::chrono::milliseconds(600)); + ASSERT_FALSE(sampler.On()); +} + +TEST_F(SamplerTest, TestDisabled) { + Sampler sampler(0.0); + ASSERT_FALSE(sampler.On()); +} + +} // namespace + +} // cloud_trace +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/common_protos_test.cc b/contrib/endpoints/src/api_manager/common_protos_test.cc new file mode 100644 index 00000000000..4ea1f5ae39e --- /dev/null +++ b/contrib/endpoints/src/api_manager/common_protos_test.cc @@ -0,0 +1,41 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "gtest/gtest.h" + +#include "google/api/service.pb.h" +#include "google/api/servicecontrol/v1/service_controller.pb.h" + +// Trivial tests that instantiate common protos +// to make sure the compile and link correctly. + +TEST(CommonProtos, ServiceConfig) { + ::google::api::Service service; + + service.set_name("bookstore"); + + ASSERT_EQ("bookstore", service.name()); +} + +TEST(CommonProtos, ServiceControl) { + ::google::api::servicecontrol::v1::CheckRequest cr; + + cr.set_service_name("bookstore"); + cr.mutable_operation()->set_operation_name("CreateShelf"); + + ASSERT_EQ("bookstore", cr.service_name()); + ASSERT_EQ("CreateShelf", cr.operation().operation_name()); +} diff --git a/contrib/endpoints/src/api_manager/config.cc b/contrib/endpoints/src/api_manager/config.cc new file mode 100644 index 00000000000..d6d7f8a56b1 --- /dev/null +++ b/contrib/endpoints/src/api_manager/config.cc @@ -0,0 +1,513 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/config.h" +#include "src/api_manager/utils/marshalling.h" +#include "src/api_manager/utils/stl_util.h" +#include "src/api_manager/utils/url_util.h" + +#include +#include +#include +#include + +#include "google/protobuf/io/tokenizer.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/text_format.h" + +using std::map; +using std::string; + +using ::google::api_manager::utils::Status; + +namespace google { +namespace api_manager { + +namespace { + +const char http_delete[] = "DELETE"; +const char http_get[] = "GET"; +// const char http_head[] = "HEAD"; +const char http_patch[] = "PATCH"; +const char http_post[] = "POST"; +const char http_put[] = "PUT"; +const char http_options[] = "OPTIONS"; +const string https_prefix = "https://"; +const string http_prefix = "http://"; +const string openid_config_path = ".well-known/openid-configuration"; + +class NoOpErrorCollector : public ::google::protobuf::io::ErrorCollector { + void AddError(int line, int column, const string &message) {} +}; + +bool ReadConfigFromString(const std::string &service_config, + ::google::protobuf::Message *service) { + // Try binary serialized proto first. Due to a bug in JSON parser, + // JSON parser may crash if presented with non-JSON data. + if (service->ParseFromString(service_config)) { + return true; + } + + // Try JSON. + Status status = utils::JsonToProto(service_config, service); + if (status.ok()) { + return true; + } + + // Try text format. + NoOpErrorCollector error_collector; + ::google::protobuf::TextFormat::Parser parser; + parser.RecordErrorsTo(&error_collector); + if (parser.ParseFromString(service_config, service)) { + return true; + } + return false; +} + +} // namespace + +Config::Config() {} + +MethodInfoImpl *Config::GetOrCreateMethodInfoImpl(const string &name, + const string &api_name, + const string &api_version) { + std::string selector; + std::string path; + + if (api_name.empty()) { + selector = name; + path = name; + } else { + // The corresponding selector is "{api.name}.{method.name}". + std::ostringstream selector_builder; + selector_builder << api_name << '.' << name; + selector = selector_builder.str(); + + // The corresponding RPC path is "/{api.name}/{method.name}". + std::ostringstream path_builder; + path_builder << '/' << api_name << '/' << name; + path = path_builder.str(); + } + + auto i = method_map_.find(selector); + if (i == std::end(method_map_)) { + auto info = + MethodInfoImplPtr(new MethodInfoImpl(name, api_name, api_version)); + info->set_selector(selector); + info->set_rpc_method_full_name(path); + i = method_map_.emplace(selector, std::move(info)).first; + } + return i->second.get(); +} + +bool Config::LoadHttpMethods(ApiManagerEnvInterface *env, + PathMatcherBuilder *pmb) { + std::set all_urls, urls_with_options; + // By default, allow_cors is false. This means that the default behavior + // of ESP is to reject all "OPTIONS" requests. If customers want to enable + // CORS, they need to set "allow_cors" to true in swagger config. + bool allow_cors = false; + for (const auto &endpoint : service_.endpoints()) { + if (endpoint.name() == service_.name() && endpoint.allow_cors()) { + allow_cors = true; + env->LogDebug("CORS is allowed."); + break; + } + } + + for (const auto &rule : service_.http().rules()) { + const string &selector = rule.selector(); + const string *url = nullptr; + const char *http_method = nullptr; + + switch (rule.pattern_case()) { + case ::google::api::HttpRule::kGet: + url = &rule.get(); + http_method = http_get; + break; + case ::google::api::HttpRule::kPut: + url = &rule.put(); + http_method = http_put; + break; + case ::google::api::HttpRule::kPost: + url = &rule.post(); + http_method = http_post; + break; + case ::google::api::HttpRule::kDelete: + url = &rule.delete_(); + http_method = http_delete; + break; + case ::google::api::HttpRule::kPatch: + url = &rule.patch(); + http_method = http_patch; + break; + case ::google::api::HttpRule::kCustom: + url = &rule.custom().path(); + http_method = rule.custom().kind().c_str(); + break; + default: + break; + } + + if (http_method == nullptr || url == nullptr || url->empty()) { + env->LogError("Invalid HTTP binding encountered."); + continue; + } + + MethodInfoImpl *mi = GetOrCreateMethodInfoImpl(selector, "", ""); + + if (!pmb->Register(service_.name(), http_method, *url, rule.body(), mi)) { + string error("Invalid HTTP template: "); + error += *url; + env->LogError(error.c_str()); + } else if (allow_cors) { + all_urls.insert(*url); + if (strcmp(http_method, http_options) == 0) { + urls_with_options.insert(*url); + } + } + } + + if (!allow_cors) { + return true; + } + + // Remove urls with options. + for (auto url : urls_with_options) { + all_urls.erase(url); + } + return AddOptionsMethodForAllUrls(env, pmb, all_urls); +} + +bool Config::AddOptionsMethodForAllUrls(ApiManagerEnvInterface *env, + PathMatcherBuilder *pmb, + const std::set &all_urls) { + // In order to support CORS. Http method OPTIONS needs to be added to + // the path_matcher for all urls except the ones already with options. + // For these OPTIONS methods, auth should be disabled and + // allow_unregistered_calls should be true. + + // All options have same selector as format: CORS.suffix. + // Appends suffix to make sure it is not used by any http rules. + string cors_selector_base = "CORS"; + string cors_selector = cors_selector_base; + int n = 0; + while (method_map_.find(cors_selector) != method_map_.end()) { + std::ostringstream suffix; + suffix << ++n; + cors_selector = cors_selector_base + "." + suffix.str(); + } + MethodInfoImpl *mi = GetOrCreateMethodInfoImpl(cors_selector, "", ""); + mi->set_auth(false); + mi->set_allow_unregistered_calls(true); + + for (auto url : all_urls) { + if (!pmb->Register(service_.name(), http_options, url, std::string(), mi)) { + env->LogError( + std::string("Failed to add http options template for url: " + url)); + } + } + + return true; +} + +bool Config::LoadRpcMethods(ApiManagerEnvInterface *env, + PathMatcherBuilder *pmb) { + for (const auto &api : service_.apis()) { + if (api.name().empty()) { + continue; + } + + for (const auto &method : api.methods()) { + // The name in the Api message is the package name followed by + // the API's simple name, and the name in the Method message is + // the simple name of the method. + MethodInfoImpl *mi = + GetOrCreateMethodInfoImpl(method.name(), api.name(), api.version()); + + // Initialize RPC method details + mi->set_request_type_url(method.request_type_url()); + mi->set_request_streaming(method.request_streaming()); + mi->set_response_type_url(method.response_type_url()); + mi->set_response_streaming(method.response_streaming()); + + if (!pmb->Register(service_.name(), http_post, mi->rpc_method_full_name(), + std::string(), mi)) { + string error("Invalid method: "); + error += mi->selector(); + env->LogError(error.c_str()); + } + } + } + return true; +} + +bool Config::LoadAuthentication(ApiManagerEnvInterface *env) { + // Parsing auth config. + const ::google::api::Authentication &auth = service_.authentication(); + map provider_id_issuer_map; + for (const auto &provider : auth.providers()) { + if (provider.id().empty()) { + env->LogError("Missing id field in AuthProvider."); + continue; + } + if (provider.issuer().empty()) { + string error = "Missing issuer field for provider: " + provider.id(); + env->LogError(error.c_str()); + continue; + } + if (!provider.jwks_uri().empty()) { + SetJwksUri(provider.issuer(), provider.jwks_uri(), false); + } else { + SetJwksUri(provider.issuer(), string(), true); + } + provider_id_issuer_map[provider.id()] = provider.issuer(); + } + + for (const auto &rule : auth.rules()) { + auto method = utils::FindOrNull(method_map_, rule.selector()); + if (method == nullptr) { + std::string error = "Not HTTP rule defined for: " + rule.selector(); + env->LogError(error.c_str()); + continue; + } + + // Allows disabling auth for a method when auth is enabled + // globally for the API. + (*method)->set_auth(rule.requirements_size() > 0); + for (const auto &requirement : rule.requirements()) { + const string &provider_id = requirement.provider_id(); + if (provider_id.empty()) { + std::string error = + "Missing provider_id field in requirements for: " + rule.selector(); + env->LogError(error.c_str()); + continue; + } + auto issuer = utils::FindOrNull(provider_id_issuer_map, provider_id); + if (issuer == nullptr) { + std::string error = "Undefined provider_id: " + provider_id; + env->LogError(error.c_str()); + } else { + (*method)->addAudiencesForIssuer(*issuer, requirement.audiences()); + } + } + } + return true; +} + +bool Config::LoadUsage(ApiManagerEnvInterface *env) { + for (const auto &rule : service_.usage().rules()) { + auto method = utils::FindOrNull(method_map_, rule.selector()); + if (method) { + (*method)->set_allow_unregistered_calls(rule.allow_unregistered_calls()); + } else { + std::string error = "Not HTTP rule defined for: " + rule.selector(); + env->LogError(error.c_str()); + } + } + return true; +} + +bool Config::LoadSystemParameters(ApiManagerEnvInterface *env) { + for (auto &rule : service_.system_parameters().rules()) { + auto method = utils::FindOrNull(method_map_, rule.selector()); + if (method) { + for (auto parameter : rule.parameters()) { + if (parameter.name().empty()) { + std::string error = "Missing parameter name for: " + rule.selector(); + env->LogError(error.c_str()); + } else { + if (!parameter.http_header().empty()) { + (*method)->add_http_header_parameter(parameter.name(), + parameter.http_header()); + } + if (!parameter.url_query_parameter().empty()) { + (*method)->add_url_query_parameter(parameter.name(), + parameter.url_query_parameter()); + } + } + } + (*method)->process_system_parameters(); + } else { + std::string error = "Not HTTP rule defined for: " + rule.selector(); + env->LogError(error.c_str()); + } + } + // For each method compile a set of system query parameter names. + // PathMatcher uses this set to ignore system query parameters when building + // variable bindings. + for (auto &m : method_map_) { + m.second->ProcessSystemQueryParameterNames(); + } + return true; +} + +bool Config::LoadBackends(ApiManagerEnvInterface *env) { + for (auto &rule : service_.backend().rules()) { + if (rule.address().empty()) { + continue; + } + auto method = utils::FindOrNull(method_map_, rule.selector()); + if (method) { + if (!(*method)->backend_address().empty()) { + std::string error = + "Duplicate a backend address for selector: " + rule.selector(); + env->LogError(error.c_str()); + continue; + } + (*method)->set_backend_address(rule.address()); + } else { + std::string error = + "No method matching backend selector: " + rule.selector(); + env->LogError(error.c_str()); + } + } + return true; +} + +bool Config::LoadService(ApiManagerEnvInterface *env, + const std::string &service_config) { + if (!service_config.empty()) { + if (!ReadConfigFromString(service_config, &service_)) { + env->LogError("Cannot load ESP configuration protocol buffer."); + return false; + } + + if (service_.name().empty()) { + env->LogError("Service name not specified in the API configuration."); + return false; + } + + string tf; + ::google::protobuf::TextFormat::PrintToString(service_, &tf); + env->LogDebug(tf.c_str()); + return true; + } + return false; +} + +void Config::LoadServerConfig(ApiManagerEnvInterface *env, + const std::string &server_config) { + if (!server_config.empty()) { + server_config_.reset(new proto::ServerConfig); + if (!ReadConfigFromString(server_config, server_config_.get())) { + env->LogError("Cannot load server configuration protocol buffer."); + server_config_.reset(); + } + } +} + +std::unique_ptr Config::Create(ApiManagerEnvInterface *env, + const std::string &service_config, + const std::string &server_config) { + std::unique_ptr config(new Config); + if (!config->LoadService(env, service_config)) { + return nullptr; + } + config->LoadServerConfig(env, server_config); + PathMatcherBuilder pmb(false /* strict_service_matching */); + // Load apis before http rules to store API versions + if (!config->LoadRpcMethods(env, &pmb)) { + return nullptr; + } + if (!config->LoadHttpMethods(env, &pmb)) { + return nullptr; + } + config->path_matcher_ = pmb.Build(); + if (!config->LoadAuthentication(env)) { + return nullptr; + } + if (!config->LoadUsage(env)) { + return nullptr; + } + if (!config->LoadSystemParameters(env)) { + return nullptr; + } + if (!config->LoadBackends(env)) { + return nullptr; + } + return config; +} + +const MethodInfo *Config::GetMethodInfo(const string &http_method, + const string &url) const { + return path_matcher_ == nullptr + ? nullptr + : path_matcher_->Lookup(service_.name(), http_method, url); +} + +MethodCallInfo Config::GetMethodCallInfo( + const std::string &http_method, const std::string &url, + const std::string &query_params) const { + MethodCallInfo call_info; + if (path_matcher_ == nullptr) { + call_info.method_info = nullptr; + } else { + call_info.method_info = path_matcher_->Lookup( + service_.name(), http_method, url, query_params, + &call_info.variable_bindings, &call_info.body_field_path); + } + return call_info; +} + +bool Config::GetJwksUri(const string &issuer, string *url) const { + std::string iss = utils::GetUrlContent(issuer); + auto it = issuer_jwks_uri_map_.find(iss); + if (it == issuer_jwks_uri_map_.end()) { + // Unknown issuer. + *url = string(); + return false; + } + if (!it->second.first.empty()) { + // jwksUri is not empty, return it. + *url = it->second.first; + return false; + } + // jwksUri is empty. + if (it->second.second) { + // openIdValid is true. We need to try open ID discovery to fetch jwksUri. + // Set url to discovery url. + if (issuer.compare(0, https_prefix.size(), https_prefix) != 0 && + issuer.compare(0, http_prefix.size(), http_prefix) != 0) { + // OpenID standard requires that "issuer" is a URL. However, + // not all providers strictly follow the standard. For example, + // Google ID token's issuer field is "accounts.google.com". + *url = https_prefix + issuer; + } else { + *url = issuer; + } + if ((*url).back() != '/') { + *url += '/'; + } + *url += openid_config_path; + return true; + } + // jwksUri is empty and openIdValid is false. This means that we have + // already tried openId discovery but failed to fetch jwkUri. + *url = string(); + return false; +} + +void Config::SetJwksUri(const string &issuer, const string &jwks_uri, + bool openid_valid) { + std::string iss = utils::GetUrlContent(issuer); + if (!iss.empty()) { + issuer_jwks_uri_map_[iss] = std::make_pair(jwks_uri, openid_valid); + } +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/config.h b/contrib/endpoints/src/api_manager/config.h new file mode 100644 index 00000000000..de8cab68c1e --- /dev/null +++ b/contrib/endpoints/src/api_manager/config.h @@ -0,0 +1,137 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_CONFIG_H_ +#define API_MANAGER_CONFIG_H_ + +#include +#include +#include +#include + +#include "google/api/service.pb.h" +#include "include/api_manager/env_interface.h" +#include "include/api_manager/method_call_info.h" +#include "src/api_manager/method_impl.h" +#include "src/api_manager/path_matcher.h" +#include "src/api_manager/proto/server_config.pb.h" + +namespace google { +namespace api_manager { + +class Config { + public: + // Creates a configuration object from service config + //- a serialized google::api::Service protocol buffer message + //(in either text or binary format). + // server_config is a buffer pointer to the server config string, it can be + // any of protobuf format json, binary or text. It can be nullptr if there is + // not server_config. + static std::unique_ptr Create(ApiManagerEnvInterface *env, + const std::string &service_config, + const std::string &server_config); + + // Returns server_config. nullptr if no server_config. + const proto::ServerConfig *server_config() const { + return server_config_.get(); + } + + // Looks-up the method config info using the given url and verb. + const MethodInfo *GetMethodInfo(const std::string &http_method, + const std::string &url) const; + + // Same as above but also returns the variable bindings extracted from the url + // according to the configured http rule (see + // https://github.com/googleapis/googleapis/blob/master/google/api/http.proto + // for more details). + MethodCallInfo GetMethodCallInfo(const std::string &http_method, + const std::string &url, + const std::string &query_params) const; + + const ::google::api::Service &service() const { return service_; } + + // TODO: Remove in favor of service(). + const std::string &service_name() const { return service_.name(); } + + // TODO: Remove in favor of service(). + bool HasAuth() const { return service_.has_authentication(); } + + // Returns true if the caller should try openId discovery to fetch jwksUri. + // url is set to the openId discovery link in this case. Returns false + // if openId discovery is not needed. This means either a valid jwksUri + // already exists, or a previous attempt to fetch jwksUri via openId + // discovery failed. + bool GetJwksUri(const std::string &issuer, std::string *tryOpenId) const; + + // Set jwskUri and openIdValid for a given issuer. + void SetJwksUri(const std::string &issuer, const std::string &jwks_uri, + bool openid_valid); + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Config); + + Config(); + + // Loads the service config into protobuf. + bool LoadService(ApiManagerEnvInterface *env, + const std::string &service_config); + + // Loads the server config into protobuf. + void LoadServerConfig(ApiManagerEnvInterface *env, + const std::string &server_config); + + // Create MethodInfo for HTTP methods, register them to PathMatcher. + bool LoadHttpMethods(ApiManagerEnvInterface *env, PathMatcherBuilder *pmb); + + // Add a special option method info for all URLs to support CORS. + bool AddOptionsMethodForAllUrls(ApiManagerEnvInterface *env, + PathMatcherBuilder *pmb, + const std::set &all_urls); + + // Create MethodInfo for RPC methods, register them to PathMatcher. + bool LoadRpcMethods(ApiManagerEnvInterface *env, PathMatcherBuilder *pmb); + + // Load Authentication info to MethodInfo. + bool LoadAuthentication(ApiManagerEnvInterface *env); + + // Load Usage info to MethodInfo. + bool LoadUsage(ApiManagerEnvInterface *env); + + // Load SystemParameters info to MethodInfo. + bool LoadSystemParameters(ApiManagerEnvInterface *env); + + // Gets the MethodInfoImpl creating it if necessary + MethodInfoImpl *GetOrCreateMethodInfoImpl(const std::string &name, + const std::string &api_name, + const std::string &api_version); + + // Load Backend info to MethodInfo. + bool LoadBackends(ApiManagerEnvInterface *env); + + ::google::api::Service service_; + std::unique_ptr server_config_; + PathMatcherPtr path_matcher_; + std::map method_map_; + // Maps issuer to {jwksUri, openIdValid} pair. + // jwksUri is populated either from service config, or by openId discovery. + // openIdValid means whether or not we need to try openId discovery to fetch + // jwksUri for the issuer. It is set to true if jwksUri is not provided in + // service config and we have not tried openId discovery to fetch jwksUri. + std::map> issuer_jwks_uri_map_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_CONFIG_H_ diff --git a/contrib/endpoints/src/api_manager/config_test.cc b/contrib/endpoints/src/api_manager/config_test.cc new file mode 100644 index 00000000000..a5a871e9dc9 --- /dev/null +++ b/contrib/endpoints/src/api_manager/config_test.cc @@ -0,0 +1,876 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/config.h" +#include "gtest/gtest.h" +#include "src/api_manager/mock_api_manager_environment.h" + +namespace google { +namespace api_manager { + +namespace { + +// Integration tests use text format or JSON config. Test the binary proto code +// here. +TEST(Config, CreateFromBinaryProto) { + ::google::api::Service service; + service.set_name("service-name"); + ::google::api::Http *http = service.mutable_http(); + ::google::api::HttpRule *rule = http->add_rules(); + + rule->set_get("/path/**"); + rule->set_selector("Paths.Get"); + + ::google::protobuf::Api *api = service.add_apis(); + api->set_name("Paths"); + api->add_methods()->set_name("Get"); + + std::string proto(service.SerializeAsString()); + ::testing::NiceMock env; + std::unique_ptr config = Config::Create(&env, proto, ""); + ASSERT_NE(nullptr, config.get()); + + const MethodInfo *method = config->GetMethodInfo("GET", "/path/a/b/c"); + ASSERT_NE(nullptr, method); + + std::string name = method->selector(); + ASSERT_EQ("Paths.Get", name); + ASSERT_FALSE(method->allow_unregistered_calls()); + + method = config->GetMethodInfo("POST", "/Paths/Get"); + ASSERT_NE(nullptr, method); + + ASSERT_EQ("Paths.Get", method->selector()); + ASSERT_FALSE(method->allow_unregistered_calls()); +} + +static const char kServerConfig[] = R"( +service_control_config { + check_aggregator_config { + cache_entries: 1000 + flush_interval_ms: 10 + response_expiration_ms: 20 + } + report_aggregator_config { + cache_entries: 1020 + flush_interval_ms: 15 + } +} +)"; + +const char kServiceNameConfig[] = "name: \"service-one\"\n"; + +TEST(Config, ServerConfigProto) { + ::testing::NiceMock env; + + std::unique_ptr config = + Config::Create(&env, kServiceNameConfig, kServerConfig); + + EXPECT_TRUE(config); + + auto server_config = config->server_config(); + EXPECT_NE(nullptr, server_config); + + ASSERT_EQ(1000, server_config->service_control_config() + .check_aggregator_config() + .cache_entries()); + ASSERT_EQ(15, server_config->service_control_config() + .report_aggregator_config() + .flush_interval_ms()); +} + +static const char kInvalidServerConfig[] = R"( +service_control_config { + type: 1 + config { + cache_entries: 1020 + flush_interval_ms: 15 + } +} +)"; + +TEST(Config, InvalidServerConfigProto) { + ::testing::NiceMock env; + + std::unique_ptr config = + Config::Create(&env, kServiceNameConfig, kInvalidServerConfig); + + EXPECT_TRUE(config); + + auto server_config = config->server_config(); + EXPECT_EQ(nullptr, server_config); +} + +const char invalid_config[] = "this is an invalid service config"; + +TEST(Config, InvalidConfig) { + ::testing::NiceMock env; + + std::unique_ptr config = Config::Create(&env, invalid_config, ""); + ASSERT_EQ(nullptr, config.get()); +} + +TEST(Config, NoServiceName) { + // Create a non-empty service config. + ::google::api::Service service; + service.mutable_control()->set_environment("servicecontrol.googleapis.com"); + + std::string service_binary; + ASSERT_TRUE(service.SerializeToString(&service_binary)); + + ::testing::NiceMock env; + std::unique_ptr config = Config::Create(&env, service_binary, ""); + ASSERT_EQ(nullptr, config.get()); +} + +static const char http_config_with_some_errors[] = + "name: \"config-test\"\n" + "http: {\n" + " rules: {\n" + " selector: \"Invalid.Get\"\n" + " get: \"/invalid/{open\"\n" // Invalid template. + " }\n" + " rules: {\n" + " selector: \"Invalid.Post\"" + " post: \"\"" // Invalid binding. + " }\n" + " rules: {\n" + " selector: \"Valid.Method\"\n" + " get: \"/valid/*\"" + " }\n" + "}\n" + "apis: {\n" + " methods: {\n" + " name: \"MethodInUnnamedApi\"\n" + " }\n" + "}\n" + "apis: {\n" + " name: \"test.api\"\n" + " methods: {\n" + " name: \"ValidMethod\"\n" + " }\n" + "}"; + +TEST(Config, HttpConfigLoading) { + ::testing::NiceMock env; + std::unique_ptr config = + Config::Create(&env, http_config_with_some_errors, ""); + ASSERT_NE(nullptr, config.get()); + + const MethodInfo *method = config->GetMethodInfo("GET", "/valid/foo"); + ASSERT_NE(nullptr, method); + + method = config->GetMethodInfo("POST", "/MethodInUnnamedApi"); + ASSERT_EQ(nullptr, method); + + method = config->GetMethodInfo("POST", "//MethodInUnnamedApi"); + ASSERT_EQ(nullptr, method); + + method = config->GetMethodInfo("POST", "/test.api/ValidMethod"); + ASSERT_NE(nullptr, method); +} + +static const char auth_config_with_some_errors[] = + "name: \"no-provider-test\"\n" + "authentication {\n" + " providers {\n" + " id: \"\"\n" // empty provider id + " issuer: \"issuer@gserviceaccount.com\"" + " }\n" + " providers {\n" + " id: \"provider-id\"\n" + " issuer: \"\"" // empty issuer + " }\n" + " rules {\n" + " selector: \"NoProvider.Method\"\n" + " requirements {\n" + " provider_id: \"test_auth\"\n" + " audiences: \"ok_audience\"\n" + " }\n" + " }\n" + "}\n" + "http {\n" + " rules {\n" + " selector: \"Good.Method\"\n" + " get: \"/method/*\"\n" + " }\n" + "}\n"; + +TEST(Config, MethodWithUnknownProvider) { + ::testing::NiceMock env; + + std::unique_ptr config = + Config::Create(&env, auth_config_with_some_errors, ""); + ASSERT_NE(nullptr, config.get()); + + const MethodInfo *method = config->GetMethodInfo("GET", "/method/good"); + ASSERT_NE(nullptr, method); + ASSERT_EQ("Good.Method", method->name()); +} + +static const char usage_config[] = + "name: \"usage-config\"\n" + "usage {\n" + " rules {\n" + " selector: \"Xyz.Method1\"\n" + " allow_unregistered_calls: true\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method2\"\n" + " allow_unregistered_calls: false\n" + " }\n" + "}\n" + "http {\n" + " rules {\n" + " selector: \"Xyz.Method1\"\n" + " get: \"/xyz/method1/*\"\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method2\"\n" + " get: \"/xyz/method2/*\"\n" + " }\n" + "}\n"; + +TEST(Config, TestLoadUsage) { + ::testing::NiceMock env; + + std::unique_ptr config = Config::Create(&env, usage_config, ""); + ASSERT_NE(nullptr, config.get()); + + const MethodInfo *method1 = config->GetMethodInfo("GET", "/xyz/method1/abc"); + ASSERT_EQ("Xyz.Method1", method1->name()); + ASSERT_TRUE(method1->allow_unregistered_calls()); + + const MethodInfo *method2 = config->GetMethodInfo("GET", "/xyz/method2/abc"); + ASSERT_EQ("Xyz.Method2", method2->name()); + ASSERT_FALSE(method2->allow_unregistered_calls()); +} + +static const char custom_method_config[] = + "name: \"custom-method-config\"\n" + "http {\n" + " rules {\n" + " selector: \"Xyz.Method1\"\n" + " get: \"/xyz/method1/*\"\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method2\"\n" + " get: \"/xyz/method2/*\"\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method3\"\n" + " custom: {\n" + " kind: \"OPTIONS\"\n" + " path: \"/xyz/method3/*\"\n" + " }\n" + " }\n" + "}\n"; + +TEST(Config, TestCustomMethod) { + ::testing::NiceMock env; + + std::unique_ptr config = + Config::Create(&env, custom_method_config, ""); + ASSERT_NE(nullptr, config.get()); + + const MethodInfo *method1 = config->GetMethodInfo("GET", "/xyz/method1/abc"); + ASSERT_EQ("Xyz.Method1", method1->name()); + + const MethodInfo *method2 = config->GetMethodInfo("GET", "/xyz/method2/abc"); + ASSERT_EQ("Xyz.Method2", method2->name()); + + const MethodInfo *method3 = + config->GetMethodInfo("OPTIONS", "/xyz/method3/abc"); + ASSERT_EQ("Xyz.Method3", method3->name()); +} + +static const char auth_config[] = + "name: \"auth-config-test\"\n" + "authentication {\n" + " providers {\n" + " id: \"provider-id1\"\n" + " issuer: \"issuer1@gserviceaccount.com\"\n" + " jwks_uri: \"https://www.googleapis.com/jwks_uri1\"\n" + " }\n" + " providers {\n" + " id: \"provider-id2\"\n" + " issuer: \"issuer2@gserviceaccount.com\"\n" + " jwks_uri: \"https://www.googleapis.com/jwks_uri2\"\n" + " }\n" + " providers {\n" + " id: \"esp-auth0\"\n" + " issuer: \"esp-jwk.auth0.com\"\n" + " }\n" + " providers {\n" + " id: \"google-x\"\n" + " issuer: \"accounts.google.com/\"\n" + " }\n" + " providers {\n" + " id: \"localhost\"\n" + " issuer: \"http://localhost\"\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method1\"\n" + " requirements {\n" + " provider_id: \"provider-id1\"\n" + " audiences: \"ok_audience1\"\n" + " }\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method2\"\n" + " requirements {\n" + " provider_id: \"provider-id2\"\n" + " audiences: \"ok_audience2\"\n" + " }\n" + " }\n" + "}\n" + "http {\n" + " rules {\n" + " selector: \"Xyz.Method1\"\n" + " get: \"/xyz/method1/*\"\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method2\"\n" + " get: \"/xyz/method2/*\"\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method3\"\n" + " get: \"/xyz/method3/*\"\n" + " }\n" + "}\n"; + +TEST(Config, TestLoadAuthentication) { + MockApiManagerEnvironmentWithLog env; + + std::unique_ptr config = Config::Create(&env, auth_config, ""); + ASSERT_NE(nullptr, config.get()); + + const MethodInfo *method1 = config->GetMethodInfo("GET", "/xyz/method1/abc"); + ASSERT_EQ("Xyz.Method1", method1->name()); + ASSERT_TRUE(method1->auth()); + ASSERT_TRUE(method1->isIssuerAllowed("issuer1@gserviceaccount.com")); + ASSERT_FALSE(method1->isIssuerAllowed("issuer2@gserviceaccount.com")); + ASSERT_TRUE(method1->isAudienceAllowed("issuer1@gserviceaccount.com", + {"ok_audience1"})); + ASSERT_FALSE(method1->isAudienceAllowed("issuer1@gserviceaccount.com", + {"ok_audience2"})); + + const MethodInfo *method2 = config->GetMethodInfo("GET", "/xyz/method2/abc"); + ASSERT_EQ("Xyz.Method2", method2->name()); + ASSERT_TRUE(method2->auth()); + ASSERT_FALSE(method2->isIssuerAllowed("issuer1@gserviceaccount.com")); + ASSERT_TRUE(method2->isIssuerAllowed("issuer2@gserviceaccount.com")); + ASSERT_FALSE(method2->isAudienceAllowed("issuer2@gserviceaccount.com", + {"ok_audience1"})); + ASSERT_TRUE(method2->isAudienceAllowed("issuer2@gserviceaccount.com", + {"ok_audience2"})); + + const MethodInfo *method3 = config->GetMethodInfo("GET", "/xyz/method3/abc"); + ASSERT_EQ("Xyz.Method3", method3->name()); + ASSERT_FALSE(method3->auth()); + + std::string url; + bool ret = config->GetJwksUri("issuer1@gserviceaccount.com", &url); + ASSERT_EQ("https://www.googleapis.com/jwks_uri1", url); + ASSERT_FALSE(ret); + ret = config->GetJwksUri("issuer2@gserviceaccount.com", &url); + ASSERT_EQ("https://www.googleapis.com/jwks_uri2", url); + ASSERT_FALSE(ret); + // A falure test: unknown issuer. + ret = config->GetJwksUri("issuer3@gserviceaccount.com", &url); + ASSERT_EQ("", url); + ASSERT_FALSE(ret); + // Returns openId discovery URL + ret = config->GetJwksUri("esp-jwk.auth0.com", &url); + ASSERT_EQ("https://esp-jwk.auth0.com/.well-known/openid-configuration", url); + ASSERT_TRUE(ret); + + ret = config->GetJwksUri("https://accounts.google.com/", &url); + ASSERT_EQ("https://accounts.google.com/.well-known/openid-configuration", + url); + ASSERT_TRUE(ret); + + ret = config->GetJwksUri("http://localhost", &url); + ASSERT_EQ("http://localhost/.well-known/openid-configuration", url); + ASSERT_TRUE(ret); + + ret = config->GetJwksUri("accounts.google.com/", &url); + ASSERT_EQ("https://accounts.google.com/.well-known/openid-configuration", + url); + ASSERT_TRUE(ret); +} + +static const char system_parameter_config[] = + "name: \"usage-config\"\n" + "system_parameters {\n" + " rules {\n" + " selector: \"Xyz.Method1\"\n" + " parameters {\n" + " name: \"name1\"\n" + " http_header: \"Header-Key1\"\n" + " url_query_parameter: \"paramer_key1\"\n" + " }\n" + " parameters {\n" + " name: \"name2\"\n" + " http_header: \"Header-Key2\"\n" + " url_query_parameter: \"paramer_key2\"\n" + " }\n" + " }\n" + "}\n" + "http {\n" + " rules {\n" + " selector: \"Xyz.Method1\"\n" + " get: \"/xyz/method1/*\"\n" + " }\n" + " rules {\n" + " selector: \"Xyz.Method2\"\n" + " get: \"/xyz/method2/*\"\n" + " }\n" + "}\n"; + +TEST(Config, TestLoadSystemParameters) { + MockApiManagerEnvironmentWithLog env; + + std::unique_ptr config = + Config::Create(&env, system_parameter_config, ""); + ASSERT_NE(nullptr, config.get()); + + const MethodInfo *method1 = config->GetMethodInfo("GET", "/xyz/method1/abc"); + ASSERT_EQ("Xyz.Method1", method1->name()); + ASSERT_EQ("Header-Key1", *method1->http_header_parameters("name1")->begin()); + ASSERT_EQ("Header-Key2", *method1->http_header_parameters("name2")->begin()); + ASSERT_EQ("paramer_key1", *method1->url_query_parameters("name1")->begin()); + ASSERT_EQ("paramer_key2", *method1->url_query_parameters("name2")->begin()); + + const MethodInfo *method2 = config->GetMethodInfo("GET", "/xyz/method2/abc"); + ASSERT_EQ("Xyz.Method2", method2->name()); + ASSERT_EQ(nullptr, method2->http_header_parameters("name1")); + ASSERT_EQ(nullptr, method2->http_header_parameters("name2")); + ASSERT_EQ(nullptr, method2->url_query_parameters("name1")); + ASSERT_EQ(nullptr, method2->url_query_parameters("name2")); +} + +static const char backends_config[] = + "name: \"backends-config\"\n" + "backend {\n" + " rules {\n" + " selector: \"test.api.MethodWithBackend\"\n" + " address: \"TestBackend:TestPort\"\n" + " }\n" + "}\n" + "apis {\n" + " name: \"test.api\"\n" + " methods {\n" + " name: \"MethodWithBackend\"\n" + " }\n" + " methods {\n" + " name: \"MethodWithoutBackend\"\n" + " }\n" + "}\n"; + +TEST(Config, LoadBackends) { + MockApiManagerEnvironmentWithLog env; + + std::unique_ptr config = Config::Create(&env, backends_config, ""); + ASSERT_TRUE(config); + + const MethodInfo *method_with_backend = + config->GetMethodInfo("POST", "/test.api/MethodWithBackend"); + ASSERT_NE(nullptr, method_with_backend); + EXPECT_EQ("TestBackend:TestPort", method_with_backend->backend_address()); + + const MethodInfo *method_without_backend = + config->GetMethodInfo("POST", "/test.api/MethodWithoutBackend"); + ASSERT_NE(nullptr, method_without_backend); + EXPECT_TRUE(method_without_backend->backend_address().empty()); +} + +TEST(Config, RpcMethodsWithHttpRules) { + MockApiManagerEnvironmentWithLog env; + + const char config_text[] = + R"( + name : "BookstoreApi" + apis { + name: "Bookstore" + methods { + name: "ListShelves" + request_type_url: "types.googleapis.com/google.protobuf.Empty" + response_type_url: "types.googleapis.com/Bookstore.ListShelvesResponse" + } + methods { + name: "CreateShelves" + request_streaming: true + request_type_url: "types.googleapis.com/Bookstore.Shelf" + response_streaming: true + response_type_url: "types.googleapis.com/Bookstore.Shelf" + } + } + http { + rules { + selector: "Bookstore.ListShelves" + get: "/shelves" + } + rules { + selector: "Bookstore.CreateShelves" + post: "/shelves" + } + } + )"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + ASSERT_TRUE(config); + + const MethodInfo *list_shelves = + config->GetMethodInfo("POST", "/Bookstore/ListShelves"); + + ASSERT_NE(nullptr, list_shelves); + EXPECT_EQ("/Bookstore/ListShelves", list_shelves->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/google.protobuf.Empty", + list_shelves->request_type_url()); + EXPECT_EQ(false, list_shelves->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.ListShelvesResponse", + list_shelves->response_type_url()); + EXPECT_EQ(false, list_shelves->response_streaming()); + + const MethodInfo *create_shelves = + config->GetMethodInfo("POST", "/Bookstore/CreateShelves"); + + ASSERT_NE(nullptr, create_shelves); + EXPECT_EQ("/Bookstore/CreateShelves", create_shelves->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/Bookstore.Shelf", + create_shelves->request_type_url()); + EXPECT_EQ(true, create_shelves->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.Shelf", + create_shelves->response_type_url()); + EXPECT_EQ(true, create_shelves->response_streaming()); + + // Matching through http rule path must yield the same method + EXPECT_EQ(list_shelves, config->GetMethodInfo("GET", "/shelves")); + EXPECT_EQ(create_shelves, config->GetMethodInfo("POST", "/shelves")); +} + +TEST(Config, RpcMethodsWithHttpRulesAndVariableBindings) { + MockApiManagerEnvironmentWithLog env; + + const char config_text[] = + R"( + name : "BookstoreApi" + apis { + name: "Bookstore" + methods { + name: "ListShelves" + request_type_url: "types.googleapis.com/google.protobuf.Empty" + response_type_url: "types.googleapis.com/Bookstore.ListShelvesResponse" + } + methods { + name: "ListBooks" + request_type_url: "types.googleapis.com/google.protobuf.Empty" + response_type_url: "types.googleapis.com/Bookstore.ListBooksResponse" + } + methods { + name: "CreateBook" + request_type_url: "types.googleapis.com/Bookstore.CreateBookRequest" + response_type_url: "types.googleapis.com/Bookstore.Book" + } + } + http { + rules { + selector: "Bookstore.ListShelves" + get: "/shelves" + } + rules { + selector: "Bookstore.ListBooks" + get: "/shelves/{shelf=*}/books" + } + rules { + selector: "Bookstore.CreateBook" + post: "/shelves/{shelf=*}/books" + body: "book" + } + rules { + selector: "Bookstore.CreateBook" + post: "/shelves/{shelf=*}/books/{book.id}/{book.author}" + body: "book.title" + } + } + system_parameters { + rules { + selector: "Bookstore.CreateBook" + parameters { + name: "system_paramter" + url_query_parameter: "sys" + } + parameters { + name: "system_paramter" + url_query_parameter: "system" + } + } + } + )"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + ASSERT_TRUE(config); + + MethodCallInfo list_shelves = + config->GetMethodCallInfo("GET", "/shelves", ""); + + ASSERT_NE(nullptr, list_shelves.method_info); + EXPECT_EQ("/Bookstore/ListShelves", + list_shelves.method_info->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/google.protobuf.Empty", + list_shelves.method_info->request_type_url()); + EXPECT_EQ(false, list_shelves.method_info->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.ListShelvesResponse", + list_shelves.method_info->response_type_url()); + EXPECT_EQ(false, list_shelves.method_info->response_streaming()); + EXPECT_EQ("", list_shelves.body_field_path); + EXPECT_EQ(0, list_shelves.variable_bindings.size()); + + MethodCallInfo list_books = + config->GetMethodCallInfo("GET", "/shelves/88/books", ""); + + ASSERT_NE(nullptr, list_books.method_info); + EXPECT_EQ("/Bookstore/ListBooks", + list_books.method_info->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/google.protobuf.Empty", + list_books.method_info->request_type_url()); + EXPECT_EQ(false, list_books.method_info->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.ListBooksResponse", + list_books.method_info->response_type_url()); + EXPECT_EQ(false, list_books.method_info->response_streaming()); + EXPECT_EQ("", list_books.body_field_path); + ASSERT_EQ(1, list_books.variable_bindings.size()); + EXPECT_EQ(std::vector(1, "shelf"), + list_books.variable_bindings[0].field_path); + EXPECT_EQ("88", list_books.variable_bindings[0].value); + + MethodCallInfo create_book = + config->GetMethodCallInfo("POST", "/shelves/99/books", ""); + + ASSERT_NE(nullptr, create_book.method_info); + EXPECT_EQ("/Bookstore/CreateBook", + create_book.method_info->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/Bookstore.CreateBookRequest", + create_book.method_info->request_type_url()); + EXPECT_EQ(false, create_book.method_info->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.Book", + create_book.method_info->response_type_url()); + EXPECT_EQ(false, create_book.method_info->response_streaming()); + EXPECT_EQ("book", create_book.body_field_path); + ASSERT_EQ(1, create_book.variable_bindings.size()); + EXPECT_EQ(std::vector(1, "shelf"), + create_book.variable_bindings[0].field_path); + EXPECT_EQ("99", create_book.variable_bindings[0].value); + + MethodCallInfo create_book_1 = + config->GetMethodCallInfo("POST", "/shelves/77/books/88/auth", ""); + + ASSERT_NE(nullptr, create_book_1.method_info); + EXPECT_EQ("/Bookstore/CreateBook", + create_book_1.method_info->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/Bookstore.CreateBookRequest", + create_book_1.method_info->request_type_url()); + EXPECT_EQ(false, create_book_1.method_info->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.Book", + create_book_1.method_info->response_type_url()); + EXPECT_EQ(false, create_book_1.method_info->response_streaming()); + EXPECT_EQ("book.title", create_book_1.body_field_path); + ASSERT_EQ(3, create_book_1.variable_bindings.size()); + + EXPECT_EQ(std::vector(1, "shelf"), + create_book_1.variable_bindings[0].field_path); + EXPECT_EQ("77", create_book_1.variable_bindings[0].value); + + EXPECT_EQ((std::vector{"book", "id"}), + create_book_1.variable_bindings[1].field_path); + EXPECT_EQ("88", create_book_1.variable_bindings[1].value); + + EXPECT_EQ((std::vector{"book", "author"}), + create_book_1.variable_bindings[2].field_path); + EXPECT_EQ("auth", create_book_1.variable_bindings[2].value); + + MethodCallInfo create_book_2 = config->GetMethodCallInfo( + "POST", "/shelves/55/books", "book.title=Readme"); + + ASSERT_NE(nullptr, create_book_2.method_info); + EXPECT_EQ("/Bookstore/CreateBook", + create_book_2.method_info->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/Bookstore.CreateBookRequest", + create_book_2.method_info->request_type_url()); + EXPECT_EQ(false, create_book_2.method_info->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.Book", + create_book_2.method_info->response_type_url()); + EXPECT_EQ(false, create_book_2.method_info->response_streaming()); + EXPECT_EQ("book", create_book_2.body_field_path); + ASSERT_EQ(2, create_book_2.variable_bindings.size()); + EXPECT_EQ(std::vector(1, "shelf"), + create_book_2.variable_bindings[0].field_path); + EXPECT_EQ("55", create_book_2.variable_bindings[0].value); + EXPECT_EQ((std::vector{"book", "title"}), + create_book_2.variable_bindings[1].field_path); + EXPECT_EQ("Readme", create_book_2.variable_bindings[1].value); + + MethodCallInfo create_book_3 = + config->GetMethodCallInfo("POST", "/shelves/321/books", + "book.id=123&key=0&api_key=1&sys=2&system=3"); + + ASSERT_NE(nullptr, create_book_3.method_info); + EXPECT_EQ("/Bookstore/CreateBook", + create_book_3.method_info->rpc_method_full_name()); + EXPECT_EQ("types.googleapis.com/Bookstore.CreateBookRequest", + create_book_3.method_info->request_type_url()); + EXPECT_EQ(false, create_book_3.method_info->request_streaming()); + EXPECT_EQ("types.googleapis.com/Bookstore.Book", + create_book_3.method_info->response_type_url()); + EXPECT_EQ(false, create_book_3.method_info->response_streaming()); + EXPECT_EQ("book", create_book_3.body_field_path); + ASSERT_EQ(2, create_book_3.variable_bindings.size()); + EXPECT_EQ(std::vector(1, "shelf"), + create_book_3.variable_bindings[0].field_path); + EXPECT_EQ("321", create_book_3.variable_bindings[0].value); + EXPECT_EQ((std::vector{"book", "id"}), + create_book_3.variable_bindings[1].field_path); + EXPECT_EQ("123", create_book_3.variable_bindings[1].value); +} + +TEST(Config, TestHttpOptions) { + MockApiManagerEnvironmentWithLog env; + + static const char config_text[] = R"( + name: "Service.Name" + endpoints { + name: "Service.Name" + allow_cors: true + } + http { + rules { + selector: "ListShelves" + get: "/shelves" + } + rules { + selector: "CorsShelves" + custom: { + kind: "OPTIONS" + path: "/shelves" + } + } + rules { + selector: "CreateShelf" + post: "/shelves" + } + rules { + selector: "GetShelf" + get: "/shelves/{shelf}" + } + rules { + selector: "DeleteShelf" + delete: "/shelves/{shelf}" + } + rules { + selector: "GetShelfBook" + get: "/shelves/{shelf}/books" + } + } +)"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + ASSERT_TRUE(config); + + // The one from service config. + auto method1 = config->GetMethodInfo("OPTIONS", "/shelves"); + ASSERT_NE(nullptr, method1); + ASSERT_EQ("CorsShelves", method1->name()); + ASSERT_FALSE(method1->auth()); + // For all service config specified method, default is NOT to allow + // unregistered calls. + ASSERT_FALSE(method1->allow_unregistered_calls()); + + // added by the code. + for (auto path : {"/shelves/{shelf}", "/shelves/{shelf}/books"}) { + auto method = config->GetMethodInfo("OPTIONS", path); + ASSERT_NE(nullptr, method); + ASSERT_EQ("CORS", method->name()); + ASSERT_FALSE(method->auth()); + // For all added OPTIONS methods, allow_unregistered_calls is true. + ASSERT_TRUE(method->allow_unregistered_calls()); + } + + // not registered path. + auto method2 = config->GetMethodInfo("OPTIONS", "/xyz"); + ASSERT_EQ(nullptr, method2); +} + +TEST(Config, TestHttpOptionsSelector) { + MockApiManagerEnvironmentWithLog env; + + static const char config_text[] = R"( + name: "Service.Name" + endpoints { + name: "Service.Name" + allow_cors: true + } + http { + rules { + selector: "CORS" + get: "/shelves" + } + rules { + selector: "CORS.1" + get: "/shelves/{shelf}" + } + } +)"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + ASSERT_TRUE(config); + + auto method1 = config->GetMethodInfo("OPTIONS", "/shelves"); + ASSERT_NE(nullptr, method1); + // selector for options should be appended with suffix. + ASSERT_EQ("CORS.2", method1->name()); + ASSERT_FALSE(method1->auth()); + ASSERT_TRUE(method1->allow_unregistered_calls()); +} + +TEST(Config, TestCorsDisabled) { + MockApiManagerEnvironmentWithLog env; + + static const char config_text[] = R"( + name: "Service.Name" + http { + rules { + selector: "CORS" + get: "/shelves" + } + rules { + selector: "CORS.1" + get: "/shelves/{shelf}" + } + } +)"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + ASSERT_TRUE(config); + + auto method1 = config->GetMethodInfo("OPTIONS", "/shelves"); + ASSERT_EQ(nullptr, method1); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/context/BUILD b/contrib/endpoints/src/api_manager/context/BUILD new file mode 100644 index 00000000000..da445c024c7 --- /dev/null +++ b/contrib/endpoints/src/api_manager/context/BUILD @@ -0,0 +1,53 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +package(default_visibility = ["//src/api_manager:__subpackages__"]) + +cc_library( + name = "context", + srcs = [ + "request_context.cc", + "service_context.cc", + ], + hdrs = [ + "request_context.h", + "service_context.h", + ], + linkopts = select({ + "//:darwin": [], + "//conditions:default": [ + "-lm", + "-luuid", + ], + }), + deps = [ + "//external:cc_wkt_protos", + "//external:cloud_trace", + "//external:googletest_prod", + "//external:grpc++", + "//external:protobuf", + "//external:service_config", + "//external:servicecontrol", + "//external:servicecontrol_client", + "//src/api_manager:impl_headers", + "//src/api_manager:server_config_proto", + "//src/api_manager/auth", + "//src/api_manager/auth:service_account_token", + "//src/api_manager/cloud_trace", + "//src/api_manager/service_control", + "//src/api_manager/utils", + ], +) diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc new file mode 100644 index 00000000000..28d37c99b86 --- /dev/null +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -0,0 +1,290 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// + +#include "src/api_manager/context/request_context.h" + +#include +#include + +using ::google::api_manager::utils::Status; + +namespace google { +namespace api_manager { +namespace context { + +namespace { + +// Cloud Trace Context Header +const char kCloudTraceContextHeader[] = "X-Cloud-Trace-Context"; + +// Log message prefix for a success method. +const char kMessage[] = "Method: "; +// Log message prefix for an ignored method. +const char kIgnoredMessage[] = + "Endpoints management skipped for an unrecognized HTTP call: "; +// Unknown HTTP verb. +const char kUnknownHttpVerb[] = ""; + +// Service control does not currently support logging with an empty +// operation name so we use this value until fix is available. +const char kUnrecognizedOperation[] = ""; + +// Maximum 36 byte string for UUID +const int kMaxUUIDBufSize = 40; + +// Default api key names +const char kDefaultApiKeyQueryName1[] = "key"; +const char kDefaultApiKeyQueryName2[] = "api_key"; +const char kDefaultApiKeyHeaderName[] = "x-api-key"; + +// Default location +const char kDefaultLocation[] = "us-central1"; + +// Genereates a UUID string +std::string GenerateUUID() { + char uuid_buf[kMaxUUIDBufSize]; + uuid_t uuid; + uuid_generate(uuid); + uuid_unparse(uuid, uuid_buf); + return uuid_buf; +} + +} // namespace + +using context::ServiceContext; + +RequestContext::RequestContext(std::shared_ptr service_context, + std::unique_ptr request) + : service_context_(service_context), + request_(std::move(request)), + is_first_report_(true) { + start_time_ = std::chrono::system_clock::now(); + last_report_time_ = std::chrono::steady_clock::now(); + operation_id_ = GenerateUUID(); + const std::string &method = request_->GetRequestHTTPMethod(); + const std::string &path = request_->GetRequestPath(); + std::string query_params = request_->GetQueryParameters(); + + // In addition to matching the method, service_context_->GetMethodCallInfo() + // will extract the variable bindings from the url. We need variable bindings + // only when we need to do transcoding. If this turns out to be a performance + // problem for non-transcoded calls, we have a couple of options: + // 1) Do not extract variable bindings here, and do the method matching again + // with extracting variable bindings when transcoding is needed. + // 2) Store all the pieces needed for extracting variable bindings (such as + // http template variables, url path parts) in MethodCallInfo and extract + // variables lazily when needed. + method_call_ = + service_context_->GetMethodCallInfo(method, path, query_params); + + if (method_call_.method_info) { + ExtractApiKey(); + } + request_->FindHeader("referer", &http_referer_); + + // Enable trace if tracing is not force disabled and the triggering header is + // set. + if (service_context_->cloud_trace_aggregator()) { + std::string trace_context_header; + request_->FindHeader(kCloudTraceContextHeader, &trace_context_header); + + std::string method_name = kUnrecognizedOperation; + if (method_call_.method_info) { + method_name = method_call_.method_info->selector(); + } + // qualify with the service name + method_name = service_context_->service_name() + "/" + method_name; + cloud_trace_.reset(cloud_trace::CreateCloudTrace( + trace_context_header, method_name, + &service_context_->cloud_trace_aggregator()->sampler())); + } +} + +void RequestContext::ExtractApiKey() { + bool api_key_defined = false; + auto url_queries = method()->api_key_url_query_parameters(); + if (url_queries) { + api_key_defined = true; + for (const auto &url_query : *url_queries) { + if (request_->FindQuery(url_query, &api_key_)) { + return; + } + } + } + + auto headers = method()->api_key_http_headers(); + if (headers) { + api_key_defined = true; + for (const auto &header : *headers) { + if (request_->FindHeader(header, &api_key_)) { + return; + } + } + } + + if (!api_key_defined) { + // If api_key is not specified for a method, + // check "key" first, if not, check "api_key" in query parameter. + if (!request_->FindQuery(kDefaultApiKeyQueryName1, &api_key_)) { + if (!request_->FindQuery(kDefaultApiKeyQueryName2, &api_key_)) { + request_->FindHeader(kDefaultApiKeyHeaderName, &api_key_); + } + } + } +} + +void RequestContext::CompleteCheck(Status status) { + // Makes sure set_check_continuation() is called. + // Only making sure CompleteCheck() is NOT called twice. + GOOGLE_CHECK(check_continuation_); + + auto temp_continuation = check_continuation_; + check_continuation_ = nullptr; + + temp_continuation(status); +} + +void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { + if (method()) { + info->operation_name = method()->selector(); + } else { + info->operation_name = kUnrecognizedOperation; + } + info->operation_id = operation_id_; + if (check_response_info_.is_api_key_valid) { + info->api_key = api_key_; + } + info->producer_project_id = service_context()->project_id(); + info->referer = http_referer_; + info->request_start_time = start_time_; +} + +void RequestContext::FillLocation(service_control::ReportRequestInfo *info) { + if (service_context()->gce_metadata()->has_valid_data() && + !service_context()->gce_metadata()->zone().empty()) { + info->location = service_context()->gce_metadata()->zone(); + } else { + info->location = kDefaultLocation; + } +} + +void RequestContext::FillComputePlatform( + service_control::ReportRequestInfo *info) { + compute_platform::ComputePlatform cp; + + GceMetadata *metadata = service_context()->gce_metadata(); + if (metadata == nullptr || !metadata->has_valid_data()) { + cp = compute_platform::UNKNOWN; + } else { + if (!metadata->gae_server_software().empty()) { + cp = compute_platform::GAE_FLEX; + } else if (!metadata->kube_env().empty()) { + cp = compute_platform::GKE; + } else { + cp = compute_platform::GCE; + } + } + + info->compute_platform = cp; +} + +void RequestContext::FillLogMessage(service_control::ReportRequestInfo *info) { + if (method()) { + info->api_method = method()->selector(); + info->api_name = method()->api_name(); + info->api_version = method()->api_version(); + info->log_message = std::string(kMessage) + method()->selector(); + } else { + std::string http_verb = info->method; + if (http_verb.empty()) { + http_verb = kUnknownHttpVerb; + } + info->log_message = std::string(kIgnoredMessage) + http_verb + " " + + request_->GetUnparsedRequestPath(); + } +} + +void RequestContext::FillCheckRequestInfo( + service_control::CheckRequestInfo *info) { + FillOperationInfo(info); + info->client_ip = request_->GetClientIP(); + info->allow_unregistered_calls = method()->allow_unregistered_calls(); +} + +void RequestContext::FillReportRequestInfo( + Response *response, service_control::ReportRequestInfo *info) { + FillOperationInfo(info); + FillLocation(info); + FillComputePlatform(info); + + info->url = request_->GetUnparsedRequestPath(); + info->method = request_->GetRequestHTTPMethod(); + + info->protocol = request_->GetRequestProtocol(); + info->check_response_info = check_response_info_; + + info->auth_issuer = auth_issuer_; + info->auth_audience = auth_audience_; + + if (!info->is_final_report) { + info->request_bytes = request_->GetGrpcRequestBytes(); + info->response_bytes = request_->GetGrpcResponseBytes(); + } else { + info->request_size = response->GetRequestSize(); + info->response_size = response->GetResponseSize(); + info->request_bytes = info->request_size; + info->response_bytes = info->response_size; + + info->streaming_request_message_counts = + request_->GetGrpcRequestMessageCounts(); + info->streaming_response_message_counts = + request_->GetGrpcResponseMessageCounts(); + + info->streaming_durations = + std::chrono::duration_cast( + std::chrono::system_clock::now() - start_time_) + .count(); + + info->status = response->GetResponseStatus(); + info->response_code = info->status.HttpCode(); + + // Must be after response_code and method are assigned. + FillLogMessage(info); + response->GetLatencyInfo(&info->latency); + } +} + +void RequestContext::StartBackendSpanAndSetTraceContext() { + backend_span_.reset(CreateSpan(cloud_trace_.get(), "Backend")); + + // Set trace context header to backend. The span id in the header will + // be the backend span's id. + std::ostringstream trace_context_stream; + trace_context_stream << cloud_trace()->trace()->trace_id() << "/" + << backend_span_->trace_span()->span_id() << ";" + << cloud_trace()->options(); + Status status = request()->AddHeaderToBackend(kCloudTraceContextHeader, + trace_context_stream.str()); + if (!status.ok()) { + service_context()->env()->LogError( + "Failed to set trace context header to backend."); + } +} + +} // namespace context +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/context/request_context.h b/contrib/endpoints/src/api_manager/context/request_context.h new file mode 100644 index 00000000000..2ef0b391a87 --- /dev/null +++ b/contrib/endpoints/src/api_manager/context/request_context.h @@ -0,0 +1,186 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_CONTEXT_REQUEST_CONTEXT_H_ +#define API_MANAGER_CONTEXT_REQUEST_CONTEXT_H_ + +#include +#include +#include + +#include "include/api_manager/method.h" +#include "include/api_manager/request.h" +#include "include/api_manager/response.h" +#include "src/api_manager/cloud_trace/cloud_trace.h" +#include "src/api_manager/context/service_context.h" +#include "src/api_manager/service_control/info.h" + +namespace google { +namespace api_manager { +namespace context { + +// Stores request related data to be used by CheckHandler. +class RequestContext { + public: + RequestContext(std::shared_ptr service_context, + std::unique_ptr request); + + // Get the ApiManagerImpl object. + context::ServiceContext *service_context() { return service_context_.get(); } + + // Get the request object. + Request *request() { return request_.get(); } + + // Get the method info. + const MethodInfo *method() const { return method_call_.method_info; } + + // Get the method info. + const MethodCallInfo *method_call() const { return &method_call_; } + + // Get the api key. + const std::string &api_key() const { return api_key_; } + + // set the final check continuation callback function. + void set_check_continuation( + std::function continuation) { + check_continuation_ = continuation; + } + + // set the is_api_key_valid field. + void set_check_response_info( + const service_control::CheckResponseInfo &check_response_info) { + check_response_info_ = check_response_info; + } + + // Fill CheckRequestInfo + void FillCheckRequestInfo(service_control::CheckRequestInfo *info); + + // Fill ReportRequestInfo + void FillReportRequestInfo(Response *response, + service_control::ReportRequestInfo *info); + + // Complete check. + void CompleteCheck(utils::Status status); + + // Sets auth issuer to request context. + void set_auth_issuer(const std::string &issuer) { auth_issuer_ = issuer; } + + // Sets auth audience to request context. + void set_auth_audience(const std::string &audience) { + auth_audience_ = audience; + } + + // Sets authorized party to request context. The authorized party is read + // from the "azp" claim in the auth token. + void set_auth_authorized_party(const std::string &authorized_party) { + auth_authorized_party_ = authorized_party; + } + + // Get CloudTrace object. + cloud_trace::CloudTrace *cloud_trace() { return cloud_trace_.get(); } + + // Marks the start of backend trace span, set the trace context header to + // backend. + void StartBackendSpanAndSetTraceContext(); + + // Marks the end of backend trace span. + void EndBackendSpan() { backend_span_.reset(); } + + // To indicate if the next report is the first_report or not. + bool is_first_report() const { return is_first_report_; } + void set_first_report(bool is_first_report) { + is_first_report_ = is_first_report; + } + + // Get the last intermediate report time point. + std::chrono::steady_clock::time_point last_report_time() const { + return last_report_time_; + } + // Set the last intermediate report time point. + void set_last_report_time(std::chrono::steady_clock::time_point tp) { + last_report_time_ = tp; + } + + private: + // Fill OperationInfo + void FillOperationInfo(service_control::OperationInfo *info); + + // Fill location info. + void FillLocation(service_control::ReportRequestInfo *info); + + // Fill compute platform information. + void FillComputePlatform(service_control::ReportRequestInfo *info); + + // Fill log message. + void FillLogMessage(service_control::ReportRequestInfo *info); + + // Extracts api-key + void ExtractApiKey(); + + // The ApiManagerImpl object. + std::shared_ptr service_context_; + + // request object to encapsulate request data. + std::unique_ptr request_; + + // The final check continuation + std::function check_continuation_; + + // The method info from service config. + MethodCallInfo method_call_; + + // Randomly generated UUID for each request, passed to service control + // Check and Report calls. + std::string operation_id_; + + // api key. + std::string api_key_; + + // Pass check response data to Report call. + service_control::CheckResponseInfo check_response_info_; + + // Needed by both Check() and Report, extract it once and store it here. + std::string http_referer_; + + // auth_issuer. It will be used in service control Report(). + std::string auth_issuer_; + + // auth_audience. It will be used in service control Report(). + std::string auth_audience_; + + // auth_authorized_party. It will be used in service control Check() and + // Report(). + std::string auth_authorized_party_; + + // Used by cloud tracing. + std::unique_ptr cloud_trace_; + + // Backend trace span. + std::shared_ptr backend_span_; + + // Start time of the request_context instantiation. + std::chrono::system_clock::time_point start_time_; + + // Flag to indicate the first report. + bool is_first_report_; + + // The time point of last intermediate report + std::chrono::steady_clock::time_point last_report_time_; +}; + +} // namespace context +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_CONTEXT_REQUEST_CONTEXT_H_ diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc new file mode 100644 index 00000000000..a0b7f1e8150 --- /dev/null +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -0,0 +1,137 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/context/service_context.h" + +#include "src/api_manager/service_control/aggregated.h" + +namespace google { +namespace api_manager { +namespace context { + +namespace { + +// Default Cloud Trace URL. Points to prod Cloud Trace. +const char kCloudTraceUrl[] = "https://cloudtrace.googleapis.com"; + +// Default maximum time to aggregate traces. +const int kDefaultAggregateTimeMillisec = 1000; + +// Default maximum amount of traces to aggregate. The amount should ensure +// the http request payload with the aggregated traces not reaching MB in size. +const int kDefaultTraceCacheMaxSize = 100; + +// Default trace sample rate, in QPS. +const double kDefaultTraceSampleQps = 0.1; + +// The time window to send intermediate report for Grpc streaming (second). +// Default to 10s. +const int kIntermediateReportInterval = 10; +} + +ServiceContext::ServiceContext(std::unique_ptr env, + std::unique_ptr config) + : env_(std::move(env)), + config_(std::move(config)), + service_account_token_(env_.get()), + service_control_(CreateInterface()), + cloud_trace_aggregator_(CreateCloudTraceAggregator()), + is_auth_force_disabled_(config_->server_config() && + config_->server_config() + ->api_authentication_config() + .force_disable()) { + intermediate_report_interval_ = kIntermediateReportInterval; + + // Check server_config override. + if (config_->server_config() && + config_->server_config()->has_service_control_config() && + config_->server_config() + ->service_control_config() + .intermediate_report_min_interval()) { + intermediate_report_interval_ = config_->server_config() + ->service_control_config() + .intermediate_report_min_interval(); + } +} + +MethodCallInfo ServiceContext::GetMethodCallInfo( + const std::string& http_method, const std::string& url, + const std::string& query_params) const { + if (config_ == nullptr) { + return MethodCallInfo(); + } + return config_->GetMethodCallInfo(http_method, url, query_params); +} + +const std::string& ServiceContext::project_id() const { + if (gce_metadata_.has_valid_data() && !gce_metadata_.project_id().empty()) { + return gce_metadata_.project_id(); + } else { + return config_->service().producer_project_id(); + } +} + +std::unique_ptr ServiceContext::CreateInterface() { + return std::unique_ptr( + service_control::Aggregated::Create(config_->service(), + config_->server_config(), env_.get(), + &service_account_token_)); +} + +std::unique_ptr +ServiceContext::CreateCloudTraceAggregator() { + // If force_disable is set in server config, completely disable tracing. + if (config_->server_config() && + config_->server_config()->cloud_tracing_config().force_disable()) { + env()->LogInfo( + "Cloud Trace is force disabled. There will be no trace written."); + return std::unique_ptr(); + } + + std::string url = kCloudTraceUrl; + int aggregate_time_millisec = kDefaultAggregateTimeMillisec; + int cache_max_size = kDefaultTraceCacheMaxSize; + double minimum_qps = kDefaultTraceSampleQps; + if (config_->server_config() && + config_->server_config()->has_cloud_tracing_config()) { + // If url_override is set in server config, use it to query Cloud Trace. + const auto& tracing_config = + config_->server_config()->cloud_tracing_config(); + if (!tracing_config.url_override().empty()) { + url = tracing_config.url_override(); + } + + // If aggregation config is set, take the values from it. + if (tracing_config.has_aggregation_config()) { + aggregate_time_millisec = + tracing_config.aggregation_config().time_millisec(); + cache_max_size = tracing_config.aggregation_config().cache_max_size(); + } + + // If sampling config is set, take the values from it. + if (tracing_config.has_samling_config()) { + minimum_qps = tracing_config.samling_config().minimum_qps(); + } + } + + return std::unique_ptr(new cloud_trace::Aggregator( + &service_account_token_, url, aggregate_time_millisec, cache_max_size, + minimum_qps, env_.get())); +} + +} // namespace context +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/context/service_context.h b/contrib/endpoints/src/api_manager/context/service_context.h new file mode 100644 index 00000000000..efb7f1d4e3b --- /dev/null +++ b/contrib/endpoints/src/api_manager/context/service_context.h @@ -0,0 +1,137 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_CONTEXT_SERVICE_CONTEXT_H_ +#define API_MANAGER_CONTEXT_SERVICE_CONTEXT_H_ + +#include "include/api_manager/method.h" +#include "src/api_manager/auth/certs.h" +#include "src/api_manager/auth/jwt_cache.h" +#include "src/api_manager/auth/service_account_token.h" +#include "src/api_manager/cloud_trace/cloud_trace.h" +#include "src/api_manager/config.h" +#include "src/api_manager/gce_metadata.h" +#include "src/api_manager/service_control/interface.h" + +namespace google { +namespace api_manager { + +namespace context { + +// Shared context across request for every service +// Each RequestContext will hold a refcount to this object. +class ServiceContext { + public: + ServiceContext(std::unique_ptr env, + std::unique_ptr config); + + bool Enabled() const { return RequireAuth() || service_control_; } + + const std::string &service_name() const { return config_->service_name(); } + + const ::google::api::Service &service() const { return config_->service(); } + + void SetMetadataServer(const std::string &server) { + metadata_server_ = server; + } + + auth::ServiceAccountToken *service_account_token() { + return &service_account_token_; + } + + ApiManagerEnvInterface *env() { return env_.get(); } + Config *config() { return config_.get(); } + + MethodCallInfo GetMethodCallInfo(const std::string &http_method, + const std::string &url, + const std::string &query_params) const; + + service_control::Interface *service_control() const { + return service_control_.get(); + } + + bool RequireAuth() const { + return !is_auth_force_disabled_ && config_->HasAuth(); + } + + auth::Certs &certs() { return certs_; } + auth::JwtCache &jwt_cache() { return jwt_cache_; } + + bool GetJwksUri(const std::string &issuer, std::string *url) { + return config_->GetJwksUri(issuer, url); + } + + void SetJwksUri(const std::string &issuer, const std::string &jwks_uri, + bool openid_valid) { + config_->SetJwksUri(issuer, jwks_uri, openid_valid); + } + + const std::string &metadata_server() const { return metadata_server_; } + GceMetadata *gce_metadata() { return &gce_metadata_; } + const std::string &project_id() const; + cloud_trace::Aggregator *cloud_trace_aggregator() const { + return cloud_trace_aggregator_.get(); + } + + bool DisableLogStatus() { + if (config_->server_config() && + config_->server_config()->has_experimental()) { + const auto &experimental = config_->server_config()->experimental(); + return experimental.disable_log_status(); + } + return false; + } + + int64_t intermediate_report_interval() const { + return intermediate_report_interval_; + } + + private: + std::unique_ptr CreateInterface(); + + std::unique_ptr CreateCloudTraceAggregator(); + + std::unique_ptr env_; + std::unique_ptr config_; + + auth::Certs certs_; + auth::JwtCache jwt_cache_; + + // service account tokens + auth::ServiceAccountToken service_account_token_; + + // The service control object. + std::unique_ptr service_control_; + + // The service control object. When trace is force disabled, this will be a + // nullptr. + std::unique_ptr cloud_trace_aggregator_; + + // meta data server. + std::string metadata_server_; + // GCE metadata + GceMetadata gce_metadata_; + + // Is auth force-disabled + bool is_auth_force_disabled_; + + // The time interval for grpc intermediate report. + int64_t intermediate_report_interval_; +}; + +} // namespace context +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_CONTEXT_SERVICE_CONTEXT_H_ diff --git a/contrib/endpoints/src/api_manager/fetch_metadata.cc b/contrib/endpoints/src/api_manager/fetch_metadata.cc new file mode 100644 index 00000000000..2f81c9f6236 --- /dev/null +++ b/contrib/endpoints/src/api_manager/fetch_metadata.cc @@ -0,0 +1,201 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +#include "src/api_manager/fetch_metadata.h" + +#include "include/api_manager/http_request.h" +#include "src/api_manager/auth/lib/auth_token.h" + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +namespace { + +// URL path for fetching compute metadata. +const char kComputeMetadata[] = "/computeMetadata/v1/?recursive=true"; +// URL path for fetching service-account +const char kMetadataServiceAccountToken[] = + "/computeMetadata/v1/instance/service-accounts/default/token"; +// Initial metadata fetch timeout (1s) +const int kMetadataFetchTimeout = 1000; +// Maximum number of retries to fetch metadata +const int kMetadataFetchRetries = 5; +// External status message for failure to fetch metadata +const char kFailedMetadataFetch[] = "Failed to fetch metadata"; +// External status message for failure to fetch service account token +const char kFailedTokenFetch[] = "Failed to fetch service account token"; +// External status message for token fetch in progress +const char kFetchingToken[] = "Fetching service account token"; +// External status message for token parse failure +const char kFailedTokenParse[] = "Failed to parse access token response"; +// Time window (in seconds) before expiration to initiate re-fetch +const char kTokenRefetchWindow = 60; + +// Issues a HTTP request to fetch the metadata. +void FetchMetadata( + context::RequestContext *context, const char *path, + std::function &&, + std::string &&)> + continuation) { + std::unique_ptr request(new HTTPRequest(continuation)); + request->set_method("GET") + .set_url(context->service_context()->metadata_server() + path) + .set_header("Metadata-Flavor", "Google") + .set_timeout_ms(kMetadataFetchTimeout) + .set_max_retries(kMetadataFetchRetries); + context->service_context()->env()->RunHTTPRequest(std::move(request)); +} +} // namespace + +void FetchGceMetadata(std::shared_ptr context, + std::function continuation) { + if (context->service_context()->metadata_server().empty()) { + // No need to fetching metadata, metadata server address is not set. + continuation(Status::OK); + return; + } + + auto env = context->service_context()->env(); + switch (context->service_context()->gce_metadata()->state()) { + case GceMetadata::FETCHED: + // Already have metadata. + env->LogDebug("Metadata already available. Fetch skipped."); + continuation(Status::OK); + return; + case GceMetadata::FAILED: + // Metadata fetch already failed. Permanent failure. + env->LogDebug("Metadata fetch previously failed. Skipping with error."); + continuation(Status(Code::INTERNAL, kFailedMetadataFetch)); + return; + case GceMetadata::FETCHING: + env->LogDebug("Another request fetching metadata. Duplicate fetch."); + continuation(Status(Code::UNAVAILABLE, kFailedMetadataFetch)); + return; + case GceMetadata::NONE: + default: + env->LogDebug("Fetching metadata."); + } + + context->service_context()->gce_metadata()->set_state(GceMetadata::FETCHING); + FetchMetadata( + context.get(), kComputeMetadata, + [context, continuation](Status status, std::map, + std::string &&body) { + // translate status to external status + if (status.ok()) { + status = + context->service_context()->gce_metadata()->ParseFromJson(&body); + } else { + status = Status(Code::INTERNAL, kFailedMetadataFetch); + } + + // update fetching state + context->service_context()->gce_metadata()->set_state( + status.ok() ? GceMetadata::FETCHED : GceMetadata::FAILED); + + continuation(status); + }); +} + +void FetchServiceAccountToken(std::shared_ptr context, + std::function continuation) { + const auto env = context->service_context()->env(); + const auto token = context->service_context()->service_account_token(); + + // If metadata server is not configured, skip it + // If client auth secret is available, skip fetching + if (context->service_context()->metadata_server().empty() || + token->has_client_secret()) { + continuation(Status::OK); + return; + } + + switch (token->state()) { + case auth::ServiceAccountToken::FETCHED: + // If token is going to last longer than the window, continue + if (token->is_access_token_valid(kTokenRefetchWindow)) { + continuation(Status::OK); + return; + } + + // If token is about to expire, initiate fetching a fresh token + // Expects token to last significantly longer than time lookahead + token->set_state(auth::ServiceAccountToken::NONE); + + // The first request within the token re-fetch window will carry on + // the token fetch, while subsequent requests in the window reuse + // the old token. + break; + case auth::ServiceAccountToken::FETCHING: + env->LogDebug("Service account token fetch in progress"); + // If token is still valid, continue + if (token->is_access_token_valid(0)) { + continuation(Status::OK); + } else { + continuation(Status(Code::UNAVAILABLE, kFetchingToken)); + } + return; + case auth::ServiceAccountToken::FAILED: + // permanent failure + continuation(Status(Code::INTERNAL, kFailedTokenFetch)); + return; + case auth::ServiceAccountToken::NONE: + default: + env->LogDebug("Need to fetch service account token"); + } + + token->set_state(auth::ServiceAccountToken::FETCHING); + FetchMetadata(context.get(), kMetadataServiceAccountToken, + [env, token, continuation]( + Status status, std::map &&, + std::string &&body) { + // fetch failed + if (!status.ok()) { + env->LogDebug("Failed to fetch service account token"); + token->set_state(auth::ServiceAccountToken::FAILED); + continuation(Status(Code::INTERNAL, kFailedTokenFetch)); + return; + } + + // process token from the body + char *auth_token = nullptr; + int expires = 0; + if (!auth::esp_get_service_account_auth_token( + const_cast(body.data()), body.length(), + &auth_token, &expires) || + token == nullptr) { + env->LogDebug("Failed to parse token response body"); + token->set_state(auth::ServiceAccountToken::FAILED); + continuation(Status(Code::INTERNAL, kFailedTokenParse)); + return; + } + + token->set_state(auth::ServiceAccountToken::FETCHED); + // Set expiration time a little bit earlier to avoid rejection + // of the actual service control requests. + // Even there is a prefetch window of 60 seconds, but prefetch + // window may not kick in if not on-going requests. + token->set_access_token(auth_token, expires - 50); + free(auth_token); + + continuation(Status::OK); + }); +} +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/fetch_metadata.h b/contrib/endpoints/src/api_manager/fetch_metadata.h new file mode 100644 index 00000000000..7f2e1b74614 --- /dev/null +++ b/contrib/endpoints/src/api_manager/fetch_metadata.h @@ -0,0 +1,35 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_FETCH_METADATA_H_ +#define API_MANAGER_FETCH_METADATA_H_ + +#include "include/api_manager/utils/status.h" +#include "src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// Fetchs GCE metadata from metadata server. +void FetchGceMetadata(std::shared_ptr, + std::function); + +// Fetchs service account token from metadata server. +void FetchServiceAccountToken(std::shared_ptr, + std::function); + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_FETCH_METADATA_H_ diff --git a/contrib/endpoints/src/api_manager/fetch_metadata_test.cc b/contrib/endpoints/src/api_manager/fetch_metadata_test.cc new file mode 100644 index 00000000000..aa397e013f4 --- /dev/null +++ b/contrib/endpoints/src/api_manager/fetch_metadata_test.cc @@ -0,0 +1,107 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/fetch_metadata.h" + +#include "src/api_manager/context/service_context.h" +#include "src/api_manager/mock_api_manager_environment.h" +#include "src/api_manager/mock_request.h" + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Mock; +using ::testing::Return; + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +namespace { + +const char kServiceConfig[] = R"( +{ + "name": "endpoints-test.cloudendpointsapis.com", + "control": { + "environment": "http://127.0.0.1:808" + } +})"; + +const char kEmptyBody[] = R"({})"; +const char kMetadataServer[] = "http://127.0.0.1:8090"; + +class FetchMetadataTest : public ::testing::Test { + public: + void SetUp() { + std::unique_ptr env( + new ::testing::NiceMock()); + // save the raw pointer of env before calling std::move(env). + raw_env_ = env.get(); + + std::unique_ptr config = + Config::Create(raw_env_, kServiceConfig, ""); + ASSERT_NE(config.get(), nullptr); + + service_context_ = std::make_shared( + std::move(env), std::move(config)); + ASSERT_NE(service_context_.get(), nullptr); + + service_context_->SetMetadataServer(kMetadataServer); + + std::unique_ptr request( + new ::testing::NiceMock()); + + context_ = std::make_shared(service_context_, + std::move(request)); + } + + MockApiManagerEnvironment *raw_env_; + std::shared_ptr service_context_; + std::shared_ptr context_; +}; + +TEST_F(FetchMetadataTest, FetchGceMetadataWithStatusOK) { + // FetchGceMetadata responses with headers and status OK. + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)) + .WillOnce(Invoke([](HTTPRequest *req) { + std::map empty; + std::string body(kEmptyBody); + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + })); + + FetchGceMetadata(context_, [](Status status) { ASSERT_TRUE(status.ok()); }); +} + +TEST_F(FetchMetadataTest, FetchGceMetadataWithStatusINTERNAL) { + // FetchGceMetadata responses with headers and status UNAVAILABLE. + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)) + .WillOnce(Invoke([](HTTPRequest *req) { + std::map empty; + std::string body(kEmptyBody); + req->OnComplete(Status(Code::UNKNOWN, ""), std::move(empty), + std::move(body)); + })); + + FetchGceMetadata(context_, [](Status status) { + ASSERT_EQ(Code::INTERNAL, status.code()); + }); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/gce_metadata.cc b/contrib/endpoints/src/api_manager/gce_metadata.cc new file mode 100644 index 00000000000..0ff0485421b --- /dev/null +++ b/contrib/endpoints/src/api_manager/gce_metadata.cc @@ -0,0 +1,71 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/gce_metadata.h" + +#include "google/protobuf/stubs/status.h" +#include "src/api_manager/auth/lib/json_util.h" + +using ::google::api_manager::auth::GetProperty; +using ::google::api_manager::auth::GetStringValue; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +namespace { + +// Assigns a const char * to std::string safely +inline std::string SafeAssign(const char *str) { return (str) ? str : ""; } + +} // namespace + +Status GceMetadata::ParseFromJson(std::string *json_str) { + // TODO: use protobuf to parse Json. + grpc_json *json = grpc_json_parse_string_with_len( + const_cast(json_str->data()), json_str->length()); + if (!json) { + return Status(Code::INVALID_ARGUMENT, + "Invalid JSON response from metadata server", + Status::INTERNAL); + } + + const grpc_json *project = GetProperty(json, "project"); + const grpc_json *instance = GetProperty(json, "instance"); + const grpc_json *attributes = GetProperty(instance, "attributes"); + + project_id_ = SafeAssign(GetStringValue(project, "projectId")); + zone_ = SafeAssign(GetStringValue(instance, "zone")); + gae_server_software_ = + SafeAssign(GetStringValue(attributes, "gae_server_software")); + kube_env_ = SafeAssign(GetStringValue(attributes, "kube-env")); + + grpc_json_destroy(json); + + // Only keep last portion of zone + if (!zone_.empty()) { + std::size_t last_slash = zone_.find_last_of("/"); + if (last_slash != std::string::npos) { + zone_ = zone_.substr(last_slash + 1); + } + } + + return Status::OK; +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/gce_metadata.h b/contrib/endpoints/src/api_manager/gce_metadata.h new file mode 100644 index 00000000000..f4d2f033932 --- /dev/null +++ b/contrib/endpoints/src/api_manager/gce_metadata.h @@ -0,0 +1,62 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_GCE_METADATA_H_ +#define API_MANAGER_GCE_METADATA_H_ + +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { + +// An environment information extracted from the Google Compute Engine metadata +// server. +class GceMetadata { + public: + enum FetchState { + NONE = 0, + // Fetching, + FETCHING, + // Fetch failed + FAILED, + // Data is go + FETCHED, + }; + GceMetadata() : state_(NONE) {} + + FetchState state() const { return state_; } + void set_state(FetchState state) { state_ = state; } + bool has_valid_data() const { return state_ == FETCHED; } + + utils::Status ParseFromJson(std::string* json); + + const std::string& project_id() const { return project_id_; } + const std::string& zone() const { return zone_; } + const std::string& gae_server_software() const { + return gae_server_software_; + } + const std::string& kube_env() const { return kube_env_; } + + private: + FetchState state_; + std::string project_id_; + std::string zone_; + std::string gae_server_software_; + std::string kube_env_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_GCE_METADATA_H_ diff --git a/contrib/endpoints/src/api_manager/gce_metadata_test.cc b/contrib/endpoints/src/api_manager/gce_metadata_test.cc new file mode 100644 index 00000000000..1a102990846 --- /dev/null +++ b/contrib/endpoints/src/api_manager/gce_metadata_test.cc @@ -0,0 +1,83 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/gce_metadata.h" +#include "gtest/gtest.h" + +using ::google::api_manager::utils::Status; + +namespace google { +namespace api_manager { + +namespace { + +const char metadata[] = R"( +{ + "instance": { + "attributes": { + "gae_server_software": "Google App Engine/1.9.38", + "kube-env": "Kubernetes environment" + }, + "zone": "projects/23479234856/zones/us-central1-f" + }, + "project": { + "numericProjectId": 23479234856, + "projectId": "esp-test-app" + } +})"; + +const char partial_metadata[] = R"( +{ + "instance": { + "zone": "projects/23479234856/zones/us-central1-f" + } +})"; + +TEST(Metadata, ExtractPropertyValue) { + GceMetadata env; + std::string meta_str(metadata); + Status status = env.ParseFromJson(&meta_str); + ASSERT_TRUE(status.ok()); + + ASSERT_EQ("us-central1-f", env.zone()); + ASSERT_EQ("Google App Engine/1.9.38", env.gae_server_software()); + ASSERT_EQ("Kubernetes environment", env.kube_env()); + ASSERT_EQ("esp-test-app", env.project_id()); +} + +TEST(Metadata, ExtractSomePropertyValues) { + GceMetadata env; + std::string meta_str(partial_metadata); + Status status = env.ParseFromJson(&meta_str); + ASSERT_TRUE(status.ok()); + ASSERT_EQ("us-central1-f", env.zone()); + ASSERT_TRUE(env.gae_server_software().empty()); + ASSERT_TRUE(env.kube_env().empty()); +} + +TEST(Metadata, ExtractErrors) { + std::string meta_str(metadata); + std::string half_str = meta_str.substr(0, meta_str.size() / 2); + // Half a Json to make it an invalid Json. + GceMetadata env; + Status status = env.ParseFromJson(&half_str); + ASSERT_FALSE(status.ok()); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/http_template.cc b/contrib/endpoints/src/api_manager/http_template.cc new file mode 100644 index 00000000000..24f49952ed3 --- /dev/null +++ b/contrib/endpoints/src/api_manager/http_template.cc @@ -0,0 +1,380 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include +#include +#include + +#include "src/api_manager/http_template.h" + +namespace google { +namespace api_manager { + +namespace { + +// TODO: implement an error sink. + +// HTTP Template Grammar: +// Questions: +// - what are the constraints on LITERAL and IDENT? +// - what is the character set for the grammar? +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +class Parser { + public: + Parser(const std::string &input) + : input_(input), tb_(0), te_(0), in_variable_(false) {} + + bool Parse() { + if (!ParseTemplate() || !ConsumedAllInput()) { + return false; + } + PostProcessVariables(); + return true; + } + + std::vector &segments() { return segments_; } + std::string &verb() { return verb_; } + std::vector &variables() { return variables_; } + + // only constant path segments are allowed after '**'. + bool ValidateParts() { + bool found_wild_card = false; + for (size_t i = 0; i < segments_.size(); i++) { + if (!found_wild_card) { + if (segments_[i] == HttpTemplate::kWildCardPathKey) { + found_wild_card = true; + } + } else if (segments_[i] == HttpTemplate::kSingleParameterKey || + segments_[i] == HttpTemplate::kWildCardPathPartKey || + segments_[i] == HttpTemplate::kWildCardPathKey) { + return false; + } + } + return true; + } + + private: + // Template = "/" Segments [ Verb ] ; + bool ParseTemplate() { + if (!Consume('/')) { + // Expected '/' + return false; + } + if (!ParseSegments()) { + return false; + } + + if (EnsureCurrent() && current_char() == ':') { + if (!ParseVerb()) { + return false; + } + } + return true; + } + + // Segments = Segment { "/" Segment } ; + bool ParseSegments() { + if (!ParseSegment()) { + return false; + } + + for (;;) { + if (!Consume('/')) break; + if (!ParseSegment()) { + return false; + } + } + + return true; + } + + // Segment = "*" | "**" | LITERAL | Variable ; + bool ParseSegment() { + if (!EnsureCurrent()) { + return false; + } + switch (current_char()) { + case '*': { + Consume('*'); + if (Consume('*')) { + // ** + segments_.push_back("**"); + if (in_variable_) { + return MarkVariableHasWildCardPath(); + } + return true; + } else { + segments_.push_back("*"); + return true; + } + } + + case '{': + return ParseVariable(); + default: + return ParseLiteralSegment(); + } + } + + // Variable = "{" FieldPath [ "=" Segments ] "}" ; + bool ParseVariable() { + if (!Consume('{')) { + return false; + } + if (!StartVariable()) { + return false; + } + if (!ParseFieldPath()) { + return false; + } + if (Consume('=')) { + if (!ParseSegments()) { + return false; + } + } else { + // {field_path} is equivalent to {field_path=*} + segments_.push_back("*"); + } + if (!EndVariable()) { + return false; + } + if (!Consume('}')) { + return false; + } + return true; + } + + bool ParseLiteralSegment() { + std::string ls; + if (!ParseLiteral(&ls)) { + return false; + } + segments_.push_back(ls); + return true; + } + + // FieldPath = IDENT { "." IDENT } ; + bool ParseFieldPath() { + if (!ParseIdentifier()) { + return false; + } + while (Consume('.')) { + if (!ParseIdentifier()) { + return false; + } + } + return true; + } + + // Verb = ":" LITERAL ; + bool ParseVerb() { + if (!Consume(':')) return false; + if (!ParseLiteral(&verb_)) return false; + return true; + } + + bool ParseIdentifier() { + std::string idf; + + // Initialize to false to handle empty literal. + bool result = false; + + while (NextChar()) { + char c; + switch (c = current_char()) { + case '.': + case '}': + case '=': + return result && AddFieldIdentifier(std::move(idf)); + default: + Consume(c); + idf.push_back(c); + break; + } + result = true; + } + return result && AddFieldIdentifier(std::move(idf)); + } + + bool ParseLiteral(std::string *lit) { + if (!EnsureCurrent()) { + return false; + } + + // Initialize to false in case we encounter an empty literal. + bool result = false; + + for (;;) { + char c; + switch (c = current_char()) { + case '/': + case ':': + case '}': + return result; + default: + Consume(c); + lit->push_back(c); + break; + } + + result = true; + + if (!NextChar()) { + break; + } + } + return result; + } + + bool Consume(char c) { + if (tb_ >= te_ && !NextChar()) { + return false; + } + if (current_char() != c) { + return false; + } + tb_++; + return true; + } + + bool ConsumedAllInput() { return tb_ >= input_.size(); } + + bool EnsureCurrent() { return tb_ < te_ || NextChar(); } + + bool NextChar() { + if (te_ < input_.size()) { + te_++; + return true; + } else { + return false; + } + } + + // Returns the character looked at. + char current_char() const { + return tb_ < te_ && te_ <= input_.size() ? input_[te_ - 1] : -1; + } + + HttpTemplate::Variable &CurrentVariable() { return variables_.back(); } + + bool StartVariable() { + if (!in_variable_) { + variables_.push_back(HttpTemplate::Variable{}); + CurrentVariable().start_segment = segments_.size(); + CurrentVariable().has_wildcard_path = false; + in_variable_ = true; + return true; + } else { + // nested variables are not allowed + return false; + } + } + + bool EndVariable() { + if (in_variable_ && !variables_.empty()) { + CurrentVariable().end_segment = segments_.size(); + in_variable_ = false; + return ValidateVariable(CurrentVariable()); + } else { + // something's wrong we're not in a variable + return false; + } + } + + bool AddFieldIdentifier(std::string id) { + if (in_variable_ && !variables_.empty()) { + CurrentVariable().field_path.emplace_back(std::move(id)); + return true; + } else { + // something's wrong we're not in a variable + return false; + } + } + + bool MarkVariableHasWildCardPath() { + if (in_variable_ && !variables_.empty()) { + CurrentVariable().has_wildcard_path = true; + return true; + } else { + // something's wrong we're not in a variable + return false; + } + } + + bool ValidateVariable(const HttpTemplate::Variable &var) { + return !var.field_path.empty() && (var.start_segment < var.end_segment) && + (var.end_segment <= static_cast(segments_.size())); + } + + void PostProcessVariables() { + for (auto &var : variables_) { + if (var.has_wildcard_path) { + // if the variable contains a '**', store the end_positon + // relative to the end, such that -1 corresponds to the end + // of the path. As we only support fixed path after '**', + // this will allow the matcher code to reconstruct the variable + // value based on the url segments. + var.end_segment = (var.end_segment - segments_.size() - 1); + + if (!verb_.empty()) { + // a custom verb will add an additional segment, so + // the end_postion needs a -1 + --var.end_segment; + } + } + } + } + + const std::string &input_; + + // Token delimiter indexes + size_t tb_; + size_t te_; + + // are we in nested Segments of a variable? + bool in_variable_; + + std::vector segments_; + std::string verb_; + std::vector variables_; +}; + +} // namespace + +const char HttpTemplate::kSingleParameterKey[] = "/."; + +const char HttpTemplate::kWildCardPathPartKey[] = "*"; + +const char HttpTemplate::kWildCardPathKey[] = "**"; + +HttpTemplate *HttpTemplate::Parse(const std::string &ht) { + Parser p(ht); + if (!p.Parse() || !p.ValidateParts()) { + return nullptr; + } + + return new HttpTemplate(std::move(p.segments()), std::move(p.verb()), + std::move(p.variables())); +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/http_template.h b/contrib/endpoints/src/api_manager/http_template.h new file mode 100644 index 00000000000..7b867bcbd20 --- /dev/null +++ b/contrib/endpoints/src/api_manager/http_template.h @@ -0,0 +1,70 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_HTTP_TEMPLATE_H_ +#define API_MANAGER_HTTP_TEMPLATE_H_ + +#include +#include + +namespace google { +namespace api_manager { + +class HttpTemplate { + public: + static HttpTemplate *Parse(const std::string &ht); + const std::vector &segments() const { return segments_; } + const std::string &verb() const { return verb_; } + + // The info about a variable binding {variable=subpath} in the template. + struct Variable { + // Specifies the range of segments [start_segment, end_segment) the + // variable binds to. Both start_segment and end_segment are 0 based. + // end_segment can also be negative, which means that the position is + // specified relative to the end such that -1 corresponds to the end + // of the path. + int start_segment; + int end_segment; + + // The path of the protobuf field the variable binds to. + std::vector field_path; + + // Do we have a ** in the variable template? + bool has_wildcard_path; + }; + + std::vector &Variables() { return variables_; } + + // '/.': match any single path segment. + static const char kSingleParameterKey[]; + // '*': Wildcard match for one path segment. + static const char kWildCardPathPartKey[]; + // '**': Wildcard match the remaining path. + static const char kWildCardPathKey[]; + + private: + HttpTemplate(std::vector &&segments, std::string &&verb, + std::vector &&variables) + : segments_(std::move(segments)), + verb_(std::move(verb)), + variables_(std::move(variables)) {} + const std::vector segments_; + std::string verb_; + std::vector variables_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_HTTP_TEMPLATE_H_ diff --git a/contrib/endpoints/src/api_manager/http_template_test.cc b/contrib/endpoints/src/api_manager/http_template_test.cc new file mode 100644 index 00000000000..49db614dac3 --- /dev/null +++ b/contrib/endpoints/src/api_manager/http_template_test.cc @@ -0,0 +1,514 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "http_template.h" +#include "gtest/gtest.h" + +#include +#include +#include + +namespace google { +namespace api_manager { + +typedef std::vector Segments; +typedef HttpTemplate::Variable Variable; +typedef std::vector Variables; +typedef std::vector FieldPath; + +bool operator==(const Variable &v1, const Variable &v2) { + return v1.field_path == v2.field_path && + v1.start_segment == v2.start_segment && + v1.end_segment == v2.end_segment && + v1.has_wildcard_path == v2.has_wildcard_path; +} + +std::string FieldPathToString(const FieldPath &fp) { + std::string s; + for (const auto &f : fp) { + if (!s.empty()) { + s += "."; + } + s += f; + } + return s; +} + +std::ostream &operator<<(std::ostream &os, const Variable &var) { + return os << "{ " << FieldPathToString(var.field_path) << ", [" + << var.start_segment << ", " << var.end_segment << "), " + << var.has_wildcard_path << "}"; +} + +std::ostream &operator<<(std::ostream &os, const Variables &vars) { + for (const auto &var : vars) { + os << var << std::endl; + } + return os; +} + +TEST(HttpTemplate, ParseTest1) { + HttpTemplate *ht = HttpTemplate::Parse("/shelves/{shelf}/books/{book}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"shelves", "*", "books", "*"}), ht->segments()); + ASSERT_EQ(Variables({ + Variable{1, 2, FieldPath{"shelf"}, false}, + Variable{3, 4, FieldPath{"book"}, false}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest2) { + HttpTemplate *ht = HttpTemplate::Parse("/shelves/**"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"shelves", "**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({}), ht->Variables()); +} + +TEST(HttpTemplate, ParseTest3) { + HttpTemplate *ht = HttpTemplate::Parse("/**"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables(), ht->Variables()); +} + +TEST(HttpTemplate, ParseTest4a) { + HttpTemplate *ht = HttpTemplate::Parse("/a:foo"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a"}), ht->segments()); + ASSERT_EQ("foo", ht->verb()); + ASSERT_EQ(Variables(), ht->Variables()); +} + +TEST(HttpTemplate, ParseTest4b) { + HttpTemplate *ht = HttpTemplate::Parse("/a/b/c:foo"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "b", "c"}), ht->segments()); + ASSERT_EQ("foo", ht->verb()); + ASSERT_EQ(Variables(), ht->Variables()); +} + +TEST(HttpTemplate, ParseTest5) { + HttpTemplate *ht = HttpTemplate::Parse("/*/**"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*", "**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables(), ht->Variables()); +} + +TEST(HttpTemplate, ParseTest6) { + HttpTemplate *ht = HttpTemplate::Parse("/*/a/**"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*", "a", "**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables(), ht->Variables()); +} + +TEST(HttpTemplate, ParseTest7) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{a.b.c}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, 2, FieldPath{"a", "b", "c"}, false}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest8) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{a.b.c=*}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, 2, FieldPath{"a", "b", "c"}, false}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest9) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=*}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, 2, FieldPath{"b"}, false}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest10) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=**}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, -1, FieldPath{"b"}, true}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest11) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=c/*}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "c", "*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, 3, FieldPath{"b"}, false}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest12) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=c/*/d}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "c", "*", "d"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, 4, FieldPath{"b"}, false}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest13) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=c/**}"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "c", "**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, -1, FieldPath{"b"}, true}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest14) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=c/**}/d/e"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "c", "**", "d", "e"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, -3, FieldPath{"b"}, true}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest15) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=c/**/d}/e"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "c", "**", "d", "e"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, -2, FieldPath{"b"}, true}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ParseTest16) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=c/**/d}/e:verb"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "c", "**", "d", "e"}), ht->segments()); + ASSERT_EQ("verb", ht->verb()); + ASSERT_EQ(Variables({ + Variable{1, -3, FieldPath{"b"}, true}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, CustomVerbTests) { + HttpTemplate *ht; + + ht = HttpTemplate::Parse("/*:verb"); + ASSERT_EQ(Segments({"*"}), ht->segments()); + ASSERT_EQ(Variables(), ht->Variables()); + + ht = HttpTemplate::Parse("/**:verb"); + ASSERT_EQ(Segments({"**"}), ht->segments()); + ASSERT_EQ(Variables(), ht->Variables()); + + ht = HttpTemplate::Parse("/{a}:verb"); + ASSERT_EQ(Segments({"*"}), ht->segments()); + ASSERT_EQ(Variables({ + Variable{0, 1, FieldPath{"a"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/a/b/*:verb"); + ASSERT_EQ(Segments({"a", "b", "*"}), ht->segments()); + ASSERT_EQ(Variables(), ht->Variables()); + + ht = HttpTemplate::Parse("/a/b/**:verb"); + ASSERT_EQ(Segments({"a", "b", "**"}), ht->segments()); + ASSERT_EQ(Variables(), ht->Variables()); + + ht = HttpTemplate::Parse("/a/b/{a}:verb"); + ASSERT_EQ(Segments({"a", "b", "*"}), ht->segments()); + ASSERT_EQ(Variables({ + Variable{2, 3, FieldPath{"a"}, false}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, MoreVariableTests) { + HttpTemplate *ht = HttpTemplate::Parse("/{x}"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 1, FieldPath{"x"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z}"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 1, FieldPath{"x", "y", "z"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x=*}"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 1, FieldPath{"x"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x=a/*}"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "*"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 2, FieldPath{"x"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=*/a/b}/c"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*", "a", "b", "c"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 3, FieldPath{"x", "y", "z"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x=**}"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -1, FieldPath{"x"}, true}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=**}"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"**"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -1, FieldPath{"x", "y", "z"}, true}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=a/**/b}"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "**", "b"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -1, FieldPath{"x", "y", "z"}, true}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=a/**/b}/c/d"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "**", "b", "c", "d"}), ht->segments()); + ASSERT_EQ("", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -3, FieldPath{"x", "y", "z"}, true}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, VariableAndCustomVerbTests) { + HttpTemplate *ht = HttpTemplate::Parse("/{x}:verb"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*"}), ht->segments()); + ASSERT_EQ("verb", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 1, FieldPath{"x"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z}:verb"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*"}), ht->segments()); + ASSERT_EQ("verb", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 1, FieldPath{"x", "y", "z"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=*/*}:verb"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"*", "*"}), ht->segments()); + ASSERT_EQ("verb", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, 2, FieldPath{"x", "y", "z"}, false}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x=**}:myverb"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"**"}), ht->segments()); + ASSERT_EQ("myverb", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -2, FieldPath{"x"}, true}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=**}:myverb"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"**"}), ht->segments()); + ASSERT_EQ("myverb", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -2, FieldPath{"x", "y", "z"}, true}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=a/**/b}:custom"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "**", "b"}), ht->segments()); + ASSERT_EQ("custom", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -2, FieldPath{"x", "y", "z"}, true}, + }), + ht->Variables()); + + ht = HttpTemplate::Parse("/{x.y.z=a/**/b}/c/d:custom"); + + ASSERT_NE(nullptr, ht); + ASSERT_EQ(Segments({"a", "**", "b", "c", "d"}), ht->segments()); + ASSERT_EQ("custom", ht->verb()); + ASSERT_EQ(Variables({ + Variable{0, -4, FieldPath{"x", "y", "z"}, true}, + }), + ht->Variables()); +} + +TEST(HttpTemplate, ErrorTests) { + ASSERT_EQ(nullptr, HttpTemplate::Parse("")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("//")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/{}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a//b")); + + ASSERT_EQ(nullptr, HttpTemplate::Parse(":verb")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/:verb")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/:verb")); + + ASSERT_EQ(nullptr, HttpTemplate::Parse(":")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/*:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/**:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/{var}:")); + + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b/:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b/*:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b/**:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b/{var}:")); + + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{var")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{var.")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{x=var:verb}")); + + ASSERT_EQ(nullptr, HttpTemplate::Parse("a")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("{x}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("{x=/a}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("{x=/a/b}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("a/b")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("a/b/{x}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("a/{x}/b")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("a/{x}/b:verb")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{var=/b}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/{var=a/{nested=b}}")); + + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a{x}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/{x}a")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a{x}b")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/{x}a{y}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b{x}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{x}b")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b{x}c")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{x}b{y}")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b{x}/s")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{x}b/s")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/b{x}c/s")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{x}b{y}/s")); +} + +TEST(HttpTemplate, ParseVerbTest2) { + HttpTemplate *ht = HttpTemplate::Parse("/a/*:verb"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(ht->segments(), Segments({"a", "*"})); + ASSERT_EQ("verb", ht->verb()); +} + +TEST(HttpTemplate, ParseVerbTest3) { + HttpTemplate *ht = HttpTemplate::Parse("/a/**:verb"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(ht->segments(), Segments({"a", "**"})); + ASSERT_EQ("verb", ht->verb()); +} + +TEST(HttpTemplate, ParseVerbTest4) { + HttpTemplate *ht = HttpTemplate::Parse("/a/{b=*}/**:verb"); + ASSERT_NE(nullptr, ht); + ASSERT_EQ(ht->segments(), Segments({"a", "*", "**"})); + ASSERT_EQ("verb", ht->verb()); +} + +TEST(HttpTemplate, ParseNonVerbTest) { + ASSERT_EQ(nullptr, HttpTemplate::Parse(":")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/*:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/**:")); + ASSERT_EQ(nullptr, HttpTemplate::Parse("/a/{b=*}/**:")); +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/method_impl.cc b/contrib/endpoints/src/api_manager/method_impl.cc new file mode 100644 index 00000000000..72ae939168b --- /dev/null +++ b/contrib/endpoints/src/api_manager/method_impl.cc @@ -0,0 +1,107 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/method_impl.h" +#include "src/api_manager/utils/url_util.h" + +#include + +using std::map; +using std::set; +using std::string; +using std::stringstream; + +namespace google { +namespace api_manager { + +// The name for api key in system parameter from service config. +const char api_key_parameter_name[] = "api_key"; + +MethodInfoImpl::MethodInfoImpl(const string &name, const string &api_name, + const string &api_version) + : name_(name), + api_name_(api_name), + api_version_(api_version), + auth_(false), + allow_unregistered_calls_(false), + api_key_http_headers_(nullptr), + api_key_url_query_parameters_(nullptr), + request_streaming_(false), + response_streaming_(false) {} + +void MethodInfoImpl::addAudiencesForIssuer(const string &issuer, + const string &audiences_list) { + if (issuer.empty()) { + return; + } + std::string iss = utils::GetUrlContent(issuer); + if (iss.empty()) { + return; + } + set &audiences = issuer_audiences_map_[iss]; + stringstream ss(audiences_list); + string audience; + // Audience list is comma-delimited. + while (getline(ss, audience, ',')) { + if (!audience.empty()) { // Only adds non-empty audience. + std::string aud = utils::GetUrlContent(audience); + if (!aud.empty()) { + audiences.insert(aud); + } + } + } +} + +bool MethodInfoImpl::isIssuerAllowed(const std::string &issuer) const { + return !issuer.empty() && + issuer_audiences_map_.find(issuer) != issuer_audiences_map_.end(); +} + +bool MethodInfoImpl::isAudienceAllowed( + const string &issuer, const std::set &jwt_audiences) const { + if (issuer.empty() || jwt_audiences.empty() || !isIssuerAllowed(issuer)) { + return false; + } + const set &audiences = issuer_audiences_map_.at(issuer); + for (const auto &it : jwt_audiences) { + if (audiences.find(it) != audiences.end()) { + return true; + } + } + return false; +} + +void MethodInfoImpl::process_system_parameters() { + api_key_http_headers_ = http_header_parameters(api_key_parameter_name); + api_key_url_query_parameters_ = url_query_parameters(api_key_parameter_name); +} + +void MethodInfoImpl::ProcessSystemQueryParameterNames() { + for (const auto ¶m : url_query_parameters_) { + for (const auto &name : param.second) { + system_query_parameter_names_.insert(name); + } + } + + if (!api_key_http_headers_ && !api_key_url_query_parameters_) { + // Adding the default api_key url query parameters + system_query_parameter_names_.insert("key"); + system_query_parameter_names_.insert("api_key"); + } +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/method_impl.h b/contrib/endpoints/src/api_manager/method_impl.h new file mode 100644 index 00000000000..c100c4ce166 --- /dev/null +++ b/contrib/endpoints/src/api_manager/method_impl.h @@ -0,0 +1,185 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_METHOD_IMPL_H_ +#define API_MANAGER_METHOD_IMPL_H_ + +#include +#include +#include + +#include "include/api_manager/method.h" +#include "src/api_manager/utils/stl_util.h" + +namespace google { +namespace api_manager { + +// An implementation of MethodInfo interface. +class MethodInfoImpl : public MethodInfo { + public: + MethodInfoImpl(const std::string &name, const std::string &api_name, + const std::string &api_version); + + const std::string &name() const { return name_; } + const std::string &api_name() const { return api_name_; } + const std::string &api_version() const { return api_version_; } + const std::string &selector() const { return selector_; } + bool auth() const { return auth_; } + bool allow_unregistered_calls() const { return allow_unregistered_calls_; } + + bool isIssuerAllowed(const std::string &issuer) const; + + bool isAudienceAllowed(const std::string &issuer, + const std::set &jwt_audiences) const; + + const std::vector *http_header_parameters( + const std::string &name) const { + return utils::FindOrNull(http_header_parameters_, name); + } + const std::vector *url_query_parameters( + const std::string &name) const { + return utils::FindOrNull(url_query_parameters_, name); + } + + const std::vector *api_key_http_headers() const { + return api_key_http_headers_; + } + + const std::vector *api_key_url_query_parameters() const { + return api_key_url_query_parameters_; + } + + const std::string &backend_address() const { return backend_address_; } + + const std::string &rpc_method_full_name() const { + return rpc_method_full_name_; + } + + const std::string &request_type_url() const { return request_type_url_; } + + bool request_streaming() const { return request_streaming_; } + + const std::string &response_type_url() const { return response_type_url_; } + + bool response_streaming() const { return response_streaming_; } + + // Adds allowed audiences (comma delimated, no space) for the issuer. + // audiences_list can be empty. + void addAudiencesForIssuer(const std::string &issuer, + const std::string &audiences_list); + void set_auth(bool v) { auth_ = v; } + void set_allow_unregistered_calls(bool v) { allow_unregistered_calls_ = v; } + + void add_http_header_parameter(const std::string &name, + const std::string &http_header) { + http_header_parameters_[name].push_back(http_header); + } + void add_url_query_parameter(const std::string &name, + const std::string &url_query_parameter) { + url_query_parameters_[name].push_back(url_query_parameter); + } + + // After add all system parameters, lookup some of them to cache + // their lookup results. + void process_system_parameters(); + + void set_selector(const std::string &selector) { selector_ = selector; } + + void set_backend_address(const std::string &address) { + backend_address_ = address; + } + + void set_rpc_method_full_name(std::string rpc_method_full_name) { + rpc_method_full_name_ = std::move(rpc_method_full_name); + } + + void set_request_type_url(std::string request_type_url) { + request_type_url_ = std::move(request_type_url); + } + + void set_request_streaming(bool request_streaming) { + request_streaming_ = request_streaming; + } + + void set_response_type_url(std::string response_type_url) { + response_type_url_ = std::move(response_type_url); + } + + void set_response_streaming(bool response_streaming) { + response_streaming_ = response_streaming; + } + + const std::set &system_query_parameter_names() const { + return system_query_parameter_names_; + } + + void ProcessSystemQueryParameterNames(); + + private: + // Method name + std::string name_; + // API name + std::string api_name_; + // API version + std::string api_version_; + // Whether auth is enabled. + bool auth_; + // Does the method allow unregistered callers (callers without client identity + // such as API Key)? + bool allow_unregistered_calls_; + // Issuers to allowed audiences map. + std::map > issuer_audiences_map_; + + // system parameter map of parameter name to http_header name. + std::map > http_header_parameters_; + + // system parameter map of parameter name to url query parameter name. + std::map > url_query_parameters_; + + // all the names of system query parameters + std::set system_query_parameter_names_; + + // http header for api_key. + const std::vector *api_key_http_headers_; + + const std::vector *api_key_url_query_parameters_; + + // The backend address for this method. + std::string backend_address_; + + // Method selector + std::string selector_; + + // The RPC method name + std::string rpc_method_full_name_; + + // The request type url + std::string request_type_url_; + + // Whether the request is streaming or not. + bool request_streaming_; + + // The response type url + std::string response_type_url_; + + // Whether the response is streaming or not. + bool response_streaming_; +}; + +typedef std::unique_ptr MethodInfoImplPtr; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_METHOD_IMPL_H_ diff --git a/contrib/endpoints/src/api_manager/method_test.cc b/contrib/endpoints/src/api_manager/method_test.cc new file mode 100644 index 00000000000..eef4ad3839f --- /dev/null +++ b/contrib/endpoints/src/api_manager/method_test.cc @@ -0,0 +1,125 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "gtest/gtest.h" +#include "src/api_manager/method_impl.h" + +using std::string; + +namespace google { +namespace api_manager { + +namespace { + +const char kMethodName[] = "method"; +const char kIssuer1[] = "iss1"; +const char kIssuer2[] = "iss2"; +const char kIssuer2https[] = "https://iss2"; +const char kIssuer3[] = "iss3"; +const char kIssuer3http[] = "http://iss3/"; +const char kIssuer4[] = "iss4"; + +TEST(MethodInfo, Create) { + MethodInfoImplPtr method_info(new MethodInfoImpl(kMethodName, "", "")); + ASSERT_FALSE(method_info->auth()); + ASSERT_FALSE(method_info->allow_unregistered_calls()); + ASSERT_EQ(kMethodName, method_info->name()); + + method_info->set_auth(true); + ASSERT_TRUE(method_info->auth()); + + method_info->set_allow_unregistered_calls(true); + ASSERT_TRUE(method_info->allow_unregistered_calls()); +} + +TEST(MethodInfo, IssueAndAudiences) { + MethodInfoImplPtr method_info(new MethodInfoImpl(kMethodName, "", "")); + method_info->addAudiencesForIssuer(kIssuer1, "aud1,aud2"); + method_info->addAudiencesForIssuer(kIssuer1, "aud3"); + method_info->addAudiencesForIssuer(kIssuer1, ","); + method_info->addAudiencesForIssuer(kIssuer1, ",aud4"); + method_info->addAudiencesForIssuer(kIssuer1, ""); + method_info->addAudiencesForIssuer(kIssuer1, ",aud5,,,"); + method_info->addAudiencesForIssuer(kIssuer2https, ",,,aud6"); + method_info->addAudiencesForIssuer(kIssuer3http, ""); + method_info->addAudiencesForIssuer(kIssuer3http, "https://aud7"); + method_info->addAudiencesForIssuer(kIssuer3http, "http://aud8"); + method_info->addAudiencesForIssuer(kIssuer3http, "https://aud9/"); + + ASSERT_TRUE(method_info->isIssuerAllowed(kIssuer1)); + ASSERT_TRUE(method_info->isIssuerAllowed(kIssuer2)); + ASSERT_TRUE(method_info->isIssuerAllowed(kIssuer3)); + ASSERT_FALSE(method_info->isIssuerAllowed(kIssuer4)); + + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud1"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud1", "audx"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud2"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud2", "audx"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud1", "aud2"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud3"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud4"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer1, {"aud5"})); + ASSERT_FALSE(method_info->isAudienceAllowed(kIssuer1, {"aud6"})); + + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer2, {"aud6"})); + ASSERT_FALSE(method_info->isAudienceAllowed(kIssuer2, {"aud1"})); + + ASSERT_FALSE(method_info->isAudienceAllowed(kIssuer3, {"aud1"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer3, {"aud7"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer3, {"aud8"})); + ASSERT_TRUE(method_info->isAudienceAllowed(kIssuer3, {"aud9"})); + + // some negative test cases + ASSERT_FALSE(method_info->isAudienceAllowed("", {"aud1"})); + ASSERT_FALSE(method_info->isAudienceAllowed(kIssuer1, {""})); + ASSERT_FALSE(method_info->isAudienceAllowed(kIssuer1, {})); +} + +TEST(MethodInfo, TestParameters) { + MethodInfoImplPtr method_info(new MethodInfoImpl(kMethodName, "", "")); + + ASSERT_EQ(nullptr, method_info->url_query_parameters("xyz")); + ASSERT_EQ(nullptr, method_info->http_header_parameters("xyz")); + + method_info->add_http_header_parameter("name1", "Http-Header1"); + method_info->add_http_header_parameter("name1", "Http-Header2"); + ASSERT_EQ(nullptr, method_info->http_header_parameters("name2")); + auto http_headers = method_info->http_header_parameters("name1"); + ASSERT_NE(nullptr, http_headers); + ASSERT_EQ(2ul, http_headers->size()); + ASSERT_EQ((*http_headers)[0], "Http-Header1"); + ASSERT_EQ((*http_headers)[1], "Http-Header2"); + + method_info->add_url_query_parameter("name1", "url_query1"); + method_info->add_url_query_parameter("name1", "url_query2"); + ASSERT_EQ(nullptr, method_info->url_query_parameters("name2")); + auto url_queries = method_info->url_query_parameters("name1"); + ASSERT_NE(nullptr, url_queries); + ASSERT_EQ(2ul, url_queries->size()); + ASSERT_EQ((*url_queries)[0], "url_query1"); + ASSERT_EQ((*url_queries)[1], "url_query2"); +} + +TEST(MethodInfo, PreservesBackendAddress) { + MethodInfoImplPtr method_info(new MethodInfoImpl(kMethodName, "", "")); + method_info->set_backend_address("backend"); + ASSERT_EQ(method_info->backend_address(), "backend"); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/mock_api_manager_environment.h b/contrib/endpoints/src/api_manager/mock_api_manager_environment.h new file mode 100644 index 00000000000..9a84cd068d4 --- /dev/null +++ b/contrib/endpoints/src/api_manager/mock_api_manager_environment.h @@ -0,0 +1,61 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_MOCK_ESP_ENVIRONMENT_H_ +#define API_MANAGER_MOCK_ESP_ENVIRONMENT_H_ + +#include "gmock/gmock.h" +#include "include/api_manager/api_manager.h" + +namespace google { +namespace api_manager { + +class MockApiManagerEnvironment : public ApiManagerEnvInterface { + public: + MOCK_METHOD2(Log, void(LogLevel, const char *)); + MOCK_METHOD1(MakeTag, void *(std::function)); + MOCK_METHOD2(StartPeriodicTimer, + std::unique_ptr(std::chrono::milliseconds, + std::function)); + MOCK_METHOD1(DoRunHTTPRequest, void(HTTPRequest *)); + virtual void RunHTTPRequest(std::unique_ptr req) { + DoRunHTTPRequest(req.get()); + } +}; + +// A useful mock class to log to stdout to debug Config loading failure. +// Replace +// ::testing::NiceMock env; +// With +// MockApiManagerEnvironmentWithLog env; +class MockApiManagerEnvironmentWithLog : public ApiManagerEnvInterface { + public: + void Log(LogLevel level, const char *message) { + std::cout << "LOG: " << message << std::endl; + } + void *MakeTag(std::function) { return nullptr; } + std::unique_ptr StartPeriodicTimer(std::chrono::milliseconds, + std::function) { + return std::unique_ptr(); + } + void RunHTTPRequest(std::unique_ptr request) { + std::map headers; + request->OnComplete(utils::Status::OK, std::move(headers), ""); + } +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_MOCK_ESP_ENVIRONMENT_H_ diff --git a/contrib/endpoints/src/api_manager/mock_method_info.h b/contrib/endpoints/src/api_manager/mock_method_info.h new file mode 100644 index 00000000000..472cf887521 --- /dev/null +++ b/contrib/endpoints/src/api_manager/mock_method_info.h @@ -0,0 +1,56 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_MOCK_METHOD_INFO_H_ +#define API_MANAGER_MOCK_METHOD_INFO_H_ + +#include "gmock/gmock.h" +#include "include/api_manager/method.h" + +namespace google { +namespace api_manager { + +class MockMethodInfo : public MethodInfo { + public: + virtual ~MockMethodInfo() {} + MOCK_CONST_METHOD0(name, const std::string&()); + MOCK_CONST_METHOD0(api_name, const std::string&()); + MOCK_CONST_METHOD0(api_version, const std::string&()); + MOCK_CONST_METHOD0(selector, const std::string&()); + MOCK_CONST_METHOD0(auth, bool()); + MOCK_CONST_METHOD0(allow_unregistered_calls, bool()); + MOCK_CONST_METHOD1(isIssuerAllowed, bool(const std::string&)); + MOCK_CONST_METHOD2(isAudienceAllowed, + bool(const std::string&, const std::set&)); + MOCK_CONST_METHOD1(http_header_parameters, + const std::vector*(const std::string&)); + MOCK_CONST_METHOD1(url_query_parameters, + const std::vector*(const std::string&)); + MOCK_CONST_METHOD0(api_key_http_headers, const std::vector*()); + MOCK_CONST_METHOD0(api_key_url_query_parameters, + const std::vector*()); + MOCK_CONST_METHOD0(backend_address, const std::string&()); + MOCK_CONST_METHOD0(rpc_method_full_name, const std::string&()); + MOCK_CONST_METHOD0(request_type_url, const std::string&()); + MOCK_CONST_METHOD0(request_streaming, bool()); + MOCK_CONST_METHOD0(response_type_url, const std::string&()); + MOCK_CONST_METHOD0(response_streaming, bool()); + MOCK_CONST_METHOD0(system_query_parameter_names, + const std::set&()); +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_MOCK_METHOD_INFO_H_ diff --git a/contrib/endpoints/src/api_manager/mock_request.h b/contrib/endpoints/src/api_manager/mock_request.h new file mode 100644 index 00000000000..2e8e7ae3ca7 --- /dev/null +++ b/contrib/endpoints/src/api_manager/mock_request.h @@ -0,0 +1,48 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_MOCK_REQUEST_H_ +#define API_MANAGER_MOCK_REQUEST_H_ + +#include "gmock/gmock.h" +#include "include/api_manager/request.h" + +namespace google { +namespace api_manager { + +class MockRequest : public Request { + public: + MOCK_METHOD2(FindQuery, bool(const std::string &, std::string *)); + MOCK_METHOD2(FindHeader, bool(const std::string &, std::string *)); + MOCK_METHOD2(AddHeaderToBackend, + utils::Status(const std::string &, const std::string &)); + MOCK_METHOD1(SetAuthToken, void(const std::string &)); + MOCK_METHOD0(GetRequestHTTPMethod, std::string()); + MOCK_METHOD0(GetRequestPath, std::string()); + MOCK_METHOD0(GetQueryParameters, std::string()); + MOCK_METHOD0(GetRequestProtocol, ::google::api_manager::protocol::Protocol()); + MOCK_METHOD0(GetUnparsedRequestPath, std::string()); + MOCK_METHOD0(GetInsecureCallerID, std::string()); + MOCK_METHOD0(GetClientIP, std::string()); + MOCK_METHOD0(GetRequestHeaders, std::multimap *()); + MOCK_METHOD0(GetGrpcRequestBytes, int64_t()); + MOCK_METHOD0(GetGrpcResponseBytes, int64_t()); + MOCK_METHOD0(GetGrpcRequestMessageCounts, int64_t()); + MOCK_METHOD0(GetGrpcResponseMessageCounts, int64_t()); +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_MOCK_REQUEST_H_ diff --git a/contrib/endpoints/src/api_manager/path_matcher.cc b/contrib/endpoints/src/api_manager/path_matcher.cc new file mode 100644 index 00000000000..6fa874f59ee --- /dev/null +++ b/contrib/endpoints/src/api_manager/path_matcher.cc @@ -0,0 +1,421 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/path_matcher.h" + +#include "include/api_manager/method.h" +#include "include/api_manager/method_call_info.h" +#include "src/api_manager/http_template.h" + +#include +#include +#include +#include + +using std::string; +using std::vector; + +namespace google { +namespace api_manager { + +namespace { + +const char kDefaultServiceName[] = "Default"; + +// Converts a request path into a format that can be used to perform a request +// lookup in the PathMatcher trie. This utility method sanitizes the request +// path and then splits the path into slash separated parts. Returns an empty +// vector if the sanitized path is "/". +// +// custom_verbs is a set of configured custom verbs that are used to match +// against any custom verbs in request path. If the request_path contains a +// custom verb not found in custom_verbs, it is treated as a part of the path. +// +// - Strips off query string: "/a?foo=bar" --> "/a" +// - Collapses extra slashes: "///" --> "/" +vector ExtractRequestParts(string req_path); + +// Looks up on a PathMatcherNode. +PathMatcherLookupResult LookupInPathMatcherNode(const PathMatcherNode& root, + const vector& parts, + const HttpMethod& http_method); + +PathMatcherNode::PathInfo TransformHttpTemplate(const HttpTemplate& ht); + +std::vector& split(const std::string& s, char delim, + std::vector& elems) { + std::stringstream ss(s); + std::string item; + while (std::getline(ss, item, delim)) { + elems.push_back(item); + } + return elems; +} + +inline bool IsReservedChar(char c) { + // Reserved characters according to RFC 6570 + switch (c) { + case '!': + case '#': + case '$': + case '&': + case '\'': + case '(': + case ')': + case '*': + case '+': + case ',': + case '/': + case ':': + case ';': + case '=': + case '?': + case '@': + case '[': + case ']': + return true; + default: + return false; + } +} + +// Check if an ASCII character is a hex digit. We can't use ctype's +// isxdigit() because it is affected by locale. This function is applied +// to the escaped characters in a url, not to natural-language +// strings, so locale should not be taken into account. +inline bool ascii_isxdigit(char c) { + return ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F') || + ('0' <= c && c <= '9'); +} + +inline int hex_digit_to_int(char c) { + /* Assume ASCII. */ + int x = static_cast(c); + if (x > '9') { + x += 9; + } + return x & 0xf; +} + +// This is a helper function for UrlUnescapeString. It takes a string and +// the index of where we are within that string. +// +// The function returns true if the next three characters are of the format: +// "%[0-9A-Fa-f]{2}". +// +// If the next three characters are an escaped character then this function will +// also return what character is escaped. +bool GetEscapedChar(const string& src, size_t i, bool unescape_reserved_chars, + char* out) { + if (i + 2 < src.size() && src[i] == '%') { + if (ascii_isxdigit(src[i + 1]) && ascii_isxdigit(src[i + 2])) { + char c = + (hex_digit_to_int(src[i + 1]) << 4) | hex_digit_to_int(src[i + 2]); + if (!unescape_reserved_chars && IsReservedChar(c)) { + return false; + } + *out = c; + return true; + } + } + return false; +} + +// Unescapes string 'part' and returns the unescaped string. Reserved characters +// (as specified in RFC 6570) are not escaped if unescape_reserved_chars is +// false. +std::string UrlUnescapeString(const std::string& part, + bool unescape_reserved_chars) { + std::string unescaped; + // Check whether we need to escape at all. + bool needs_unescaping = false; + char ch = '\0'; + for (size_t i = 0; i < part.size(); ++i) { + if (GetEscapedChar(part, i, unescape_reserved_chars, &ch)) { + needs_unescaping = true; + break; + } + } + if (!needs_unescaping) { + unescaped = part; + return unescaped; + } + + unescaped.resize(part.size()); + + char* begin = &(unescaped)[0]; + char* p = begin; + + for (size_t i = 0; i < part.size();) { + if (GetEscapedChar(part, i, unescape_reserved_chars, &ch)) { + *p++ = ch; + i += 3; + } else { + *p++ = part[i]; + i += 1; + } + } + + unescaped.resize(p - begin); + return unescaped; +} + +void ExtractBindingsFromPath(const std::vector& vars, + const std::vector& parts, + std::vector* bindings) { + for (const auto& var : vars) { + // Determine the subpath bound to the variable based on the + // [start_segment, end_segment) segment range of the variable. + // + // In case of matching "**" - end_segment is negative and is relative to + // the end such that end_segment = -1 will match all subsequent segments. + VariableBinding binding; + binding.field_path = var.field_path; + // Calculate the absolute index of the ending segment in case it's negative. + size_t end_segment = (var.end_segment >= 0) + ? var.end_segment + : parts.size() + var.end_segment + 1; + // It is multi-part match if we have more than one segment. We also make + // sure that a single URL segment match with ** is also considered a + // multi-part match by checking if it->second.end_segment is negative. + bool is_multipart = + (end_segment - var.start_segment) > 1 || var.end_segment < 0; + // Joins parts with "/" to form a path string. + for (size_t i = var.start_segment; i < end_segment; ++i) { + // For multipart matches only unescape non-reserved characters. + binding.value += UrlUnescapeString(parts[i], !is_multipart); + if (i < end_segment - 1) { + binding.value += "/"; + } + } + bindings->emplace_back(binding); + } +} + +void ExtractBindingsFromQueryParameters( + const std::string& query_params, const std::set& system_params, + std::vector* bindings) { + // The bindings in URL the query parameters have the following form: + // =value1&=value2&...&=valueN + // Query parameters may also contain system parameters such as `api_key`. + // We'll need to ignore these. Example: + // book.id=123&book.author=Neal%20Stephenson&api_key=AIzaSyAz7fhBkC35D2M + vector params; + split(query_params, '&', params); + for (const auto& param : params) { + size_t pos = param.find('='); + if (pos != 0 && pos != std::string::npos) { + auto name = param.substr(0, pos); + // Make sure the query parameter is not a system parameter (e.g. + // `api_key`) before adding the binding. + if (system_params.find(name) == std::end(system_params)) { + // The name of the parameter is a field path, which is a dot-delimited + // sequence of field names that identify the (potentially deep) field + // in the request, e.g. `book.author.name`. + VariableBinding binding; + split(name, '.', binding.field_path); + binding.value = UrlUnescapeString(param.substr(pos + 1), true); + bindings->emplace_back(std::move(binding)); + } + } + } +} + +} // namespace + +PathMatcher::PathMatcher(PathMatcherBuilder& builder) + : default_root_ptr_(builder.default_root_ptr_->Clone()), + strict_service_matching_(builder.strict_service_matching_), + custom_verbs_(builder.custom_verbs_), + methods_(std::move(builder.methods_)) { + for (auto key_value : builder.root_ptr_map_) { + utils::InsertIfNotPresent(&root_ptr_map_, key_value.first, + key_value.second->Clone()); + } +} + +PathMatcher::~PathMatcher() { utils::STLDeleteValues(&root_ptr_map_); } + +// Lookup is a wrapper method for the recursive node Lookup. First, the wrapper +// splits the request path into slash-separated path parts. Next, the method +// checks that the |http_method| is supported. If not, then it returns an empty +// WrapperGraph::SharedPtr. Next, this method invokes the node's Lookup on +// the extracted |parts|. Finally, it fills the mapping from variables to their +// values parsed from the path. +// TODO: cache results by adding get/put methods here (if profiling reveals +// benefit) +MethodInfo* PathMatcher::Lookup(const string& service_name, + const string& http_method, const string& url, + const string& query_params, + std::vector* variable_bindings, + std::string* body_field_path) const { + const vector parts = ExtractRequestParts(url); + + PathMatcherNode* root_ptr = utils::FindPtrOrNull(root_ptr_map_, service_name); + + // If service_name has not been registered to ESP and strict_service_matching_ + // is set to false, tries to lookup the method in all registered services. + if (root_ptr == nullptr && !strict_service_matching_) { + root_ptr = default_root_ptr_.get(); + } + if (root_ptr == nullptr) { + return nullptr; + } + + PathMatcherLookupResult lookup_result = + LookupInPathMatcherNode(*root_ptr, parts, http_method); + // In non strict match case, if not found from the map with the exact service, + // needs to try the default one again. + if (lookup_result.data == nullptr && !strict_service_matching_ && + root_ptr != default_root_ptr_.get()) { + root_ptr = default_root_ptr_.get(); + lookup_result = LookupInPathMatcherNode(*root_ptr, parts, http_method); + } + // Return nullptr if nothing is found or the result is marked for duplication. + if (lookup_result.data == nullptr || lookup_result.is_multiple) { + return nullptr; + } + MethodData* method_data = reinterpret_cast(lookup_result.data); + if (variable_bindings != nullptr) { + variable_bindings->clear(); + ExtractBindingsFromPath(method_data->variables, parts, variable_bindings); + ExtractBindingsFromQueryParameters( + query_params, method_data->method->system_query_parameter_names(), + variable_bindings); + } + if (body_field_path != nullptr) { + *body_field_path = method_data->body_field_path; + } + return method_data->method; +} + +MethodInfo* PathMatcher::Lookup(const string& service_name, + const string& http_method, + const string& path) const { + return Lookup(service_name, http_method, path, string(), nullptr, nullptr); +} + +// Initializes the builder with a root Path Segment +PathMatcherBuilder::PathMatcherBuilder(bool strict_service_matching) + : default_root_ptr_(new PathMatcherNode()), + strict_service_matching_(strict_service_matching) {} + +PathMatcherBuilder::~PathMatcherBuilder() { + utils::STLDeleteValues(&root_ptr_map_); +} + +PathMatcherPtr PathMatcherBuilder::Build() { + return PathMatcherPtr(new PathMatcher(*this)); +} + +void PathMatcherBuilder::InsertPathToNode(const PathMatcherNode::PathInfo& path, + void* method_data, + string service_name, + string http_method, + bool mark_duplicates, + PathMatcherNode* root_ptr) { + if (root_ptr->InsertPath(path, http_method, service_name, method_data, + mark_duplicates)) { + // VLOG(3) << "Registered WrapperGraph for " << + // http_template.as_string(); + } else { + // VLOG(3) << "Replaced WrapperGraph for " << http_template.as_string(); + } +} + +// This wrapper converts the |http_rule| into a HttpTemplate. Then, inserts the +// template into the trie. +bool PathMatcherBuilder::Register(string service_name, string http_method, + string http_template, string body_field_path, + MethodInfo* method) { + std::unique_ptr ht(HttpTemplate::Parse(http_template)); + if (nullptr == ht) { + return false; + } + PathMatcherNode::PathInfo path_info = TransformHttpTemplate(*ht); + if (path_info.path_info().size() == 0) { + return false; + } + // Create & initialize a MethodData struct. Then insert its pointer + // into the path matcher trie. + auto method_data = std::unique_ptr(new MethodData()); + method_data->method = method; + method_data->variables = std::move(ht->Variables()); + method_data->body_field_path = std::move(body_field_path); + // Don't mark batch methods as duplicates, since we insert them into each + // service, and their graphs are all the same. We'll just use the first one + // as the default. This allows batch requests on any service name to work. + InsertPathToNode(path_info, method_data.get(), kDefaultServiceName, + http_method, false, default_root_ptr_.get()); + PathMatcherNode* root_ptr = + utils::LookupOrInsertNew(&root_ptr_map_, service_name); + InsertPathToNode(path_info, method_data.get(), service_name, http_method, + true, root_ptr); + // Add the method_data to the methods_ vector for cleanup + methods_.emplace_back(std::move(method_data)); + return true; +} + +namespace { + +vector ExtractRequestParts(string path) { + // Remove query parameters. + path = path.substr(0, path.find_first_of('?')); + + // Replace last ':' with '/' to handle custom verb. + // But not for /foo:bar/const. + std::size_t last_colon_pos = path.find_last_of(':'); + std::size_t last_slash_pos = path.find_last_of('/'); + if (last_colon_pos != std::string::npos && last_colon_pos > last_slash_pos) { + path[last_colon_pos] = '/'; + } + + vector result; + if (path.size() > 0) { + split(path.substr(1), '/', result); + } + // Removes all trailing empty parts caused by extra "/". + while (!result.empty() && (*(--result.end())).empty()) { + result.pop_back(); + } + return result; +} + +PathMatcherLookupResult LookupInPathMatcherNode(const PathMatcherNode& root, + const vector& parts, + const HttpMethod& http_method) { + PathMatcherLookupResult result; + root.LookupPath(parts.begin(), parts.end(), http_method, &result); + return result; +} + +PathMatcherNode::PathInfo TransformHttpTemplate(const HttpTemplate& ht) { + PathMatcherNode::PathInfo::Builder builder; + + for (const string& part : ht.segments()) { + builder.AppendLiteralNode(part); + } + if (!ht.verb().empty()) { + builder.AppendLiteralNode(ht.verb()); + } + + return builder.Build(); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/path_matcher.h b/contrib/endpoints/src/api_manager/path_matcher.h new file mode 100644 index 00000000000..3ece80fb920 --- /dev/null +++ b/contrib/endpoints/src/api_manager/path_matcher.h @@ -0,0 +1,146 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_PATH_MATCHER_H_ +#define API_MANAGER_PATH_MATCHER_H_ + +#include +#include +#include +#include +#include + +#include "include/api_manager/method.h" +#include "include/api_manager/method_call_info.h" +#include "src/api_manager/http_template.h" +#include "src/api_manager/path_matcher_node.h" + +namespace google { +namespace api_manager { + +class PathMatcher; // required for typedef PathMatcherPtr +class PathMatcherBuilder; // required for PathMatcher constructor +class PathMatcherNode; + +typedef std::shared_ptr PathMatcherPtr; +typedef std::unordered_map + ServiceRootPathMatcherNodeMap; + +// The immutable, thread safe PathMatcher stores a mapping from a combination of +// a service (host) name and a HTTP path to your method (MethodInfo*). It is +// constructed with a PathMatcherBuilder and supports one operation: Lookup. +// Clients may use this method to locate your method (MethodInfo*) for a +// combination of service name and HTTP URL path. +// +// Usage example: +// 1) building the PathMatcher: +// PathMatcherBuilder builder(false); +// for each (service_name, http_method, url_path, associated method) +// builder.register(service_name, http_method, url_path, datat); +// PathMater matcher = builder.Build(); +// 2) lookup: +// MethodInfo * method = matcher.Lookup(service_name, http_method, +// url_path); +// if (method == nullptr) failed to find it. +// +class PathMatcher { + public: + // Creates a Path Matcher with a Builder by deep-copying the builder's root + // node. + explicit PathMatcher(PathMatcherBuilder& builder); + ~PathMatcher(); + + MethodInfo* Lookup(const std::string& service_name, + const std::string& http_method, const std::string& path, + const std::string& query_params, + std::vector* variable_bindings, + std::string* body_field_path) const; + + MethodInfo* Lookup(const std::string& service_name, + const std::string& http_method, + const std::string& path) const; + + private: + // A map between service names and their root path matcher nodes. + ServiceRootPathMatcherNodeMap root_ptr_map_; + // A root node shared by all services, i.e. paths of all services will be + // registered to this node. + std::unique_ptr default_root_ptr_; + // Whether requests with unregistered host name will be rejected. + bool strict_service_matching_; + // Holds the set of custom verbs found in configured templates. + std::set custom_verbs_; + // Data we store per each registered method + struct MethodData { + MethodInfo* method; + std::vector variables; + std::string body_field_path; + }; + // The info associated with each method. The path matcher nodes + // will hold pointers to MethodData objects in this vector. + std::vector> methods_; + + private: + friend class PathMatcherBuilder; +}; + +// This PathMatcherBuilder is used to register path-WrapperGraph pairs and +// instantiate an immutable, thread safe PathMatcher. +// +// The PathMatcherBuilder itself is NOT THREAD SAFE. +class PathMatcherBuilder { + public: + friend class PathMatcher; + PathMatcherBuilder(bool strict_service_matching); + ~PathMatcherBuilder(); + + // Registers a method. + // + // Registrations are one-to-one. If this function is called more than once, it + // replaces the existing method. Only the last registered method is stored. + // Return false if path is an invalid http template. + bool Register(std::string service_name, std::string http_method, + std::string path, std::string body_field_path, + MethodInfo* method); + + // Returns a shared_ptr to a thread safe PathMatcher that contains all + // registered path-WrapperGraph pairs. + PathMatcherPtr Build(); + + private: + // Inserts a path to a PathMatcherNode. + void InsertPathToNode(const PathMatcherNode::PathInfo& path, + void* method_data, std::string service_name, + std::string http_method, bool mark_duplicates, + PathMatcherNode* root_ptr); + // A map between service names and their root path matcher nodes. + ServiceRootPathMatcherNodeMap root_ptr_map_; + // A root node shared by all services, i.e. paths of all services will be + // registered to this node. + std::unique_ptr default_root_ptr_; + // Whether requests with unregistered host name will be rejected. + bool strict_service_matching_; + // The set of custom verbs configured. + // TODO: Perhaps this should not be at this level because there will + // be multiple templates in different services on a server. Consider moving + // this to PathMatcherNode. + std::set custom_verbs_; + typedef PathMatcher::MethodData MethodData; + std::vector> methods_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_PATH_MATCHER_H_ diff --git a/contrib/endpoints/src/api_manager/path_matcher_node.cc b/contrib/endpoints/src/api_manager/path_matcher_node.cc new file mode 100644 index 00000000000..2bfc9afb352 --- /dev/null +++ b/contrib/endpoints/src/api_manager/path_matcher_node.cc @@ -0,0 +1,218 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/path_matcher_node.h" +#include "src/api_manager/http_template.h" + +#include +#include + +using std::string; +using std::vector; + +namespace google { +namespace api_manager { + +const char HttpMethod_WILD_CARD[] = "*"; + +namespace { +// A convinent function to lookup a STL colllection with two keys. +// Lookup key1 first, if not found, lookup key2, or return nullptr. +template +const typename Collection::value_type::second_type* Find2KeysOrNull( + const Collection& collection, + const typename Collection::value_type::first_type& key1, + const typename Collection::value_type::first_type& key2) { + auto it = collection.find(key1); + if (it == collection.end()) { + it = collection.find(key2); + if (it == collection.end()) { + return nullptr; + } + } + return &it->second; +} +} // namespace + +PathMatcherNode::PathInfo::Builder& +PathMatcherNode::PathInfo::Builder::AppendLiteralNode(string name) { + if (name == HttpTemplate::kSingleParameterKey) { + // status_.Update(util::Status(util::error::INVALID_ARGUMENT, + // StrCat(name, " is a reserved node name."))); + } + path_.emplace_back(name); + return *this; +} + +PathMatcherNode::PathInfo::Builder& +PathMatcherNode::PathInfo::Builder::AppendSingleParameterNode() { + path_.emplace_back(HttpTemplate::kSingleParameterKey); + return *this; +} + +PathMatcherNode::PathInfo PathMatcherNode::PathInfo::Builder::Build() const { + return PathMatcherNode::PathInfo(*this); +} + +// TODO: When possible, replace the children_ map's plain pointers with +// smart pointers. (small_map and hash_map are not move-aware, so they cannot +// hold unique_ptrs) +PathMatcherNode::~PathMatcherNode() { utils::STLDeleteValues(&children_); } + +// Performs a deep copy of the parameter |node|. NB: WrapperGraph::SharedPtr is +// a shared_ptr, so wrapper_graph_map_ copies point to the same WrapperGraph. +// TODO: Consider returning by value. Since copying is not allowed and it +// would not be possible to heap-allocate a cloned node, would it still be +// possible to store children in an associative container? +PathMatcherNode* PathMatcherNode::Clone() const { + PathMatcherNode* clone = new PathMatcherNode(); + clone->result_map_ = result_map_; + // deep-copy literal children + for (const auto& entry : children_) { + clone->children_[entry.first] = entry.second->Clone(); + } + clone->wildcard_ = wildcard_; + return clone; +} + +// This recursive function performs an exhaustive DFS of the node's subtrie. +// The node attempts to find a match for the current part of the path among its +// children. Children are considered in sequence according to Google HTTP +// Template Spec matching precedence. If a match is found, the method recurses +// on the matching child with the next part in path. +// +// NB: If this path segment is of repeated-variable type and no matching child +// is found, the receiver recurses on itself with the next path part. +// +// Base Case: |current| is beyond the range of the path parts +// ========== +// The receiver node matched the final part in |path|. If a WrapperGraph exists +// for the given HTTP method, the method copies to the node's WrapperGraph to +// result and returns true. +void PathMatcherNode::LookupPath(const RequestPathParts::const_iterator current, + const RequestPathParts::const_iterator end, + HttpMethod http_method, + PathMatcherLookupResult* result) const { + // base case + if (current == end) { + if (!GetResultForHttpMethod(http_method, result)) { + // If we didn't find a wrapper graph at this node, check if we have one + // in a wildcard (**) child. If we do, use it. This will ensure we match + // the root with wildcard templates. + auto pair = children_.find(HttpTemplate::kWildCardPathKey); + if (pair != children_.end()) { + const PathMatcherNode* child = pair->second; + child->GetResultForHttpMethod(http_method, result); + } + } + return; + } + if (LookupPathFromChild(*current, current, end, http_method, result)) { + return; + } + // For wild card node, keeps searching for next path segment until either + // 1) reaching the end (/foo/** case), or 2) all remaining segments match + // one of child branches (/foo/**/bar/xyz case). + if (wildcard_) { + LookupPath(current + 1, end, http_method, result); + // Since only constant segments are allowed after wild card, no need to + // search another wild card nodes from children, so bail out here. + return; + } + + for (const string& child_key : + {HttpTemplate::kSingleParameterKey, HttpTemplate::kWildCardPathPartKey, + HttpTemplate::kWildCardPathKey}) { + if (LookupPathFromChild(child_key, current, end, http_method, result)) { + return; + } + } + return; +} + +bool PathMatcherNode::InsertPath(const PathInfo& node_path_info, + string http_method, string service_name, + void* method_data, bool mark_duplicates) { + return this->InsertTemplate(node_path_info.path_info().begin(), + node_path_info.path_info().end(), http_method, + service_name, method_data, mark_duplicates); +} + +// This method locates a matching child for the |current| path part, inserting a +// child if not present. Then, the method recurses on this matching child with +// the next template path part. +// +// Base Case: |current| is beyond the range of the path parts +// ========== +// This node matched the final part in the iterator of parts. This method +// updates the node's WrapperGraph for the specified HTTP method. +bool PathMatcherNode::InsertTemplate( + const vector::const_iterator current, + const vector::const_iterator end, HttpMethod http_method, + string service_name, void* method_data, bool mark_duplicates) { + if (current == end) { + PathMatcherLookupResult* const existing = utils::InsertOrReturnExisting( + &result_map_, http_method, PathMatcherLookupResult(method_data, false)); + if (existing != nullptr) { + existing->data = method_data; + if (mark_duplicates) { +#if 0 + LOG(WARNING) << "The HTTP path '" << http_method << ":" + << ConvertHttpRuleToString(*wrapper_graph->http_rule()) + << "' has already been registered to the host '" + << service_name << "'."; +#endif + existing->is_multiple = true; + } + return false; + } + return true; + } + PathMatcherNode* child = utils::LookupOrInsertNew(&children_, *current); + if (*current == HttpTemplate::kWildCardPathKey) { + child->set_wildcard(true); + } + return child->InsertTemplate(current + 1, end, http_method, service_name, + method_data, mark_duplicates); +} + +bool PathMatcherNode::LookupPathFromChild( + const string child_key, const RequestPathParts::const_iterator current, + const RequestPathParts::const_iterator end, HttpMethod http_method, + PathMatcherLookupResult* result) const { + auto pair = children_.find(child_key); + if (pair != children_.end()) { + pair->second->LookupPath(current + 1, end, http_method, result); + if (result != nullptr && result->data != nullptr) { + return true; + } + } + return false; +} + +bool PathMatcherNode::GetResultForHttpMethod( + HttpMethod key, PathMatcherLookupResult* result) const { + const PathMatcherLookupResult* found_p = + Find2KeysOrNull(result_map_, key, HttpMethod_WILD_CARD); + if (found_p != nullptr) { + *result = *found_p; + return true; + } + return false; +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/path_matcher_node.h b/contrib/endpoints/src/api_manager/path_matcher_node.h new file mode 100644 index 00000000000..90f961344fd --- /dev/null +++ b/contrib/endpoints/src/api_manager/path_matcher_node.h @@ -0,0 +1,202 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_PATH_MATCHER_NODE_H_ +#define API_MANAGER_PATH_MATCHER_NODE_H_ + +#include +#include +#include +#include +#include + +#include "src/api_manager/utils/stl_util.h" + +namespace google { +namespace api_manager { + +typedef std::string HttpMethod; + +struct PathMatcherLookupResult { + PathMatcherLookupResult() : data(nullptr), is_multiple(false) {} + + PathMatcherLookupResult(void* data, bool is_multiple) + : data(data), is_multiple(is_multiple) {} + + // The WrapperGraph that is registered to a method (or HTTP path). + void* data; + // Whether the method (or path) has been registered for more than once. + bool is_multiple; +}; + +// PathMatcherNodes represents a path part in a PathMatcher trie. Children nodes +// represent adjacent path parts. A node can have many literal children, one +// single-parameter child, and one repeated-parameter child. +// +// Thread Compatible. +class PathMatcherNode { + public: + // Provides information for inserting templates into the trie. Clients can + // instantiate PathInfo with the provided Builder. + class PathInfo { + public: + class Builder { + public: + friend class PathInfo; + Builder() : path_() {} + ~Builder() {} + + PathMatcherNode::PathInfo Build() const; + + // Appends a node that must match the string value of a request part. The + // strings "/." and "/.." are disallowed. + // + // Example: + // + // builder.AppendLiteralNode("a") + // .AppendLiteralNode("b") + // .AppendLiteralNode("c"); + // + // Matches the request path: a/b/c + Builder& AppendLiteralNode(std::string name); + + // Appends a node that ignores the string value and matches any single + // request part. + // + // Example: + // + // builder.AppendLiteralNode("a") + // .AppendSingleParameterNode() + // .AppendLiteralNode("c"); + // + // Matching request paths: a/foo/c, a/bar/c, a/1/c + Builder& AppendSingleParameterNode(); + + // TODO: Appends a node that ignores string values and matches any + // number of consecutive request parts. + // + // Example: + // + // builder.AppendLiteralNode("a") + // .AppendLiteralNode("b") + // .AppendRepeatedParameterNode(); + // + // Matching request paths: a/b/1/2/3/4/5, a/b/c + // Builder& AppendRepeatedParameterNode(); + + private: + std::vector path_; + }; // class Builder + + ~PathInfo() {} + + // Returns path information used to insert a new path into a PathMatcherNode + // trie. + const std::vector& path_info() const { return path_; } + + private: + explicit PathInfo(const Builder& builder) : path_(builder.path_) {} + std::vector path_; + }; // class PathInfo + + typedef std::vector RequestPathParts; + + // Creates a Root node with an empty WrapperGraph map. + PathMatcherNode() : result_map_(), children_(), wildcard_(false) {} + + ~PathMatcherNode(); + + // Creates a clone of this node and its subtrie + PathMatcherNode* Clone() const; + + // Searches subtrie by finding a matching child for the current path part. If + // a matching child exists, this function recurses on current + 1 with that + // child as the receiver. If a matching descendant is found for the last part + // in then this method copies the matching descendant's WrapperGraph, + // VariableBindingInfoMap to the result pointers. + void LookupPath(const RequestPathParts::const_iterator current, + const RequestPathParts::const_iterator end, + HttpMethod http_method, + PathMatcherLookupResult* result) const; + + // This method inserts a path of nodes into this subtrie. The WrapperGraph, + // VariableBindingInfoMap are inserted at the terminal descendant node. + // Returns true if the template didn't previously exist. Returns false + // otherwise and depends on if mark_duplicates is true, the template will be + // marked as having been registered for more than once and the lookup of the + // template will yield a special error reporting WrapperGraph. + bool InsertPath(const PathInfo& node_path_info, std::string http_method, + std::string service_name, void* method_data, + bool mark_duplicates); + + void set_wildcard(bool wildcard) { wildcard_ = wildcard; } + + private: + // This method inserts a path of nodes into this subtrie (described by the + // vector, starting from the |current| position in the iterator of path + // parts, and if necessary, creating intermediate nodes along the way. The + // WrapperGraph, VariableBindingInfoMap are inserted at the terminal + // descendant node (which corresponds to the string part in the iterator). + // Returns true if the template didn't previously exist. Returns false + // otherwise and depends on if mark_duplicates is true, the template will be + // marked as having been registered for more than once and the lookup of the + // template will yield a special error reporting WrapperGraph. + bool InsertTemplate(const std::vector::const_iterator current, + const std::vector::const_iterator end, + HttpMethod http_method, std::string service_name, + void* method_data, bool mark_duplicates); + + // Helper method for LookupPath. If the given child key exists, search + // continues on the child node pointed by the child key with the next part + // in the path. Returns true if found a match for the path eventually. + bool LookupPathFromChild(const std::string child_key, + const RequestPathParts::const_iterator current, + const RequestPathParts::const_iterator end, + HttpMethod http_method, + PathMatcherLookupResult* result) const; + + // If a WrapperGraph is found for the provided key, then this method returns + // true and copies the WrapperGraph to the provided result pointer. If no + // match is found, this method returns false and leaves the result unmodified. + // + // NB: If result == nullptr, method will return bool value without modifying + // result. + bool GetResultForHttpMethod(HttpMethod key, + PathMatcherLookupResult* result) const; + + std::map result_map_; + + // Lookup must be FAST + // + // n: the number of paths registered per client varies, but we can expect the + // size of |children_| to range from ~5 to ~100 entries. + // + // To ensure fast lookups when n grows large, it is prudent to consider an + // alternative to binary search on a sorted vector. + // + // hash_map provides O(1) expected lookups for all n, but has undesirable + // memory overhead when n is small. To address this overhead, we use a + // small_map hybrid that provides O(1) expected lookups with a small memory + // footprint. The small_map scales up to a hash_map when the number of literal + // children grows large. + std::unordered_map children_; + + // True if this node represents a wildcard path '**'. + bool wildcard_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_PATH_MATCHER_NODE_H_ diff --git a/contrib/endpoints/src/api_manager/path_matcher_test.cc b/contrib/endpoints/src/api_manager/path_matcher_test.cc new file mode 100644 index 00000000000..51104397dcd --- /dev/null +++ b/contrib/endpoints/src/api_manager/path_matcher_test.cc @@ -0,0 +1,737 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/path_matcher.h" + +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "include/api_manager/method.h" +#include "include/api_manager/method_call_info.h" +#include "src/api_manager/mock_method_info.h" +#include "src/api_manager/utils/stl_util.h" + +using ::testing::ReturnRef; + +namespace google { +namespace api_manager { + +typedef VariableBinding Binding; +typedef std::vector Bindings; +typedef std::vector FieldPath; + +bool operator==(const Binding& b1, const Binding& b2) { + return b1.field_path == b2.field_path && b1.value == b2.value; +} + +namespace { + +std::string FieldPathToString(const FieldPath& fp) { + std::string s; + for (const auto& f : fp) { + if (!s.empty()) { + s += "."; + } + s += f; + } + return s; +} + +} // namespace + +std::ostream& operator<<(std::ostream& os, const Binding& b) { + return os << "{ " << FieldPathToString(b.field_path) << "=" << b.value << "}"; +} + +std::ostream& operator<<(std::ostream& os, const Bindings& bindings) { + for (const auto& b : bindings) { + os << b << std::endl; + } + return os; +} + +namespace { + +class PathMatcherTest : public ::testing::Test { + protected: + PathMatcherTest() : builder_(false) {} + ~PathMatcherTest() { utils::STLDeleteElements(&stored_methods_); } + + MethodInfo* AddPathWithBodyFieldPath(std::string http_method, + std::string http_template, + std::string body_field_path) { + auto method = new ::testing::NiceMock(); + ON_CALL(*method, system_query_parameter_names()) + .WillByDefault(ReturnRef(empty_set_)); + if (!builder_.Register("", http_method, http_template, body_field_path, + method)) { + delete method; + return nullptr; + } + stored_methods_.push_back(method); + return method; + } + + MethodInfo* AddPathWithSystemParams( + std::string http_method, std::string http_template, + const std::set* system_params) { + auto method = new ::testing::NiceMock(); + ON_CALL(*method, system_query_parameter_names()) + .WillByDefault(ReturnRef(*system_params)); + if (!builder_.Register("", http_method, http_template, std::string(), + method)) { + delete method; + return nullptr; + } + stored_methods_.push_back(method); + return method; + } + + MethodInfo* AddPath(std::string http_method, std::string http_template) { + return AddPathWithBodyFieldPath(http_method, http_template, std::string()); + } + + MethodInfo* AddGetPath(std::string path) { return AddPath("GET", path); } + + void Build() { matcher_ = builder_.Build(); } + + MethodInfo* LookupWithBodyFieldPath(std::string method, std::string path, + Bindings* bindings, + std::string* body_field_path) { + return matcher_->Lookup("", method, path, "", bindings, body_field_path); + } + + MethodInfo* Lookup(std::string method, std::string path, Bindings* bindings) { + std::string body_field_path; + return matcher_->Lookup("", method, path, std::string(), bindings, + &body_field_path); + } + + MethodInfo* LookupWithParams(std::string method, std::string path, + std::string query_params, Bindings* bindings) { + std::string body_field_path; + return matcher_->Lookup("", method, path, query_params, bindings, + &body_field_path); + } + + MethodInfo* LookupNoBindings(std::string method, std::string path) { + Bindings bindings; + std::string body_field_path; + auto result = matcher_->Lookup("", method, path, std::string(), &bindings, + &body_field_path); + EXPECT_EQ(0, bindings.size()); + return result; + } + + private: + PathMatcherBuilder builder_; + PathMatcherPtr matcher_; + std::vector stored_methods_; + std::set empty_set_; +}; + +TEST_F(PathMatcherTest, WildCardMatchesRoot) { + MethodInfo* data = AddGetPath("/**"); + Build(); + + EXPECT_NE(nullptr, data); + + EXPECT_EQ(LookupNoBindings("GET", "/"), data); + EXPECT_EQ(LookupNoBindings("GET", "/a"), data); + EXPECT_EQ(LookupNoBindings("GET", "/a/"), data); +} + +TEST_F(PathMatcherTest, WildCardMatches) { + // '*' only matches one path segment, but '**' matches the remaining path. + MethodInfo* a__ = AddGetPath("/a/**"); + MethodInfo* b_ = AddGetPath("/b/*"); + MethodInfo* c_d__ = AddGetPath("/c/*/d/**"); + MethodInfo* c_de = AddGetPath("/c/*/d/e"); + MethodInfo* cfde = AddGetPath("/c/f/d/e"); + Build(); + + EXPECT_NE(nullptr, a__); + EXPECT_NE(nullptr, b_); + EXPECT_NE(nullptr, c_d__); + EXPECT_NE(nullptr, c_de); + EXPECT_NE(nullptr, cfde); + + EXPECT_EQ(LookupNoBindings("GET", "/a/b"), a__); + EXPECT_EQ(LookupNoBindings("GET", "/a/b/c"), a__); + EXPECT_EQ(LookupNoBindings("GET", "/b/c"), b_); + + EXPECT_EQ(LookupNoBindings("GET", "b/c/d"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/c/u/d/v"), c_d__); + EXPECT_EQ(LookupNoBindings("GET", "/c/v/d/w/x"), c_d__); + EXPECT_EQ(LookupNoBindings("GET", "/c/x/y/d/z"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/c//v/d/w/x"), nullptr); + + // Test that more specific match overrides wildcard "**"" match. + EXPECT_EQ(LookupNoBindings("GET", "/c/x/d/e"), c_de); + // Test that more specific match overrides wildcard "*"" match. + EXPECT_EQ(LookupNoBindings("GET", "/c/f/d/e"), cfde); +} + +TEST_F(PathMatcherTest, VariableBindings) { + MethodInfo* a_cde = AddGetPath("/a/{x}/c/d/e"); + MethodInfo* a_b_c = AddGetPath("/{x=a/*}/b/{y=*}/c"); + MethodInfo* ab_d__ = AddGetPath("/a/{x=b/*}/{y=d/**}"); + MethodInfo* alpha_beta__gamma = AddGetPath("/alpha/{x=*}/beta/{y=**}/gamma"); + MethodInfo* _a = AddGetPath("/{x=*}/a"); + MethodInfo* __ab = AddGetPath("/{x=**}/a/b"); + MethodInfo* ab_ = AddGetPath("/a/b/{x=*}"); + MethodInfo* abc__ = AddGetPath("/a/b/c/{x=**}"); + MethodInfo* _def__ = AddGetPath("/{x=*}/d/e/f/{y=**}"); + Build(); + + EXPECT_NE(nullptr, a_cde); + EXPECT_NE(nullptr, a_b_c); + EXPECT_NE(nullptr, ab_d__); + EXPECT_NE(nullptr, alpha_beta__gamma); + EXPECT_NE(nullptr, _a); + EXPECT_NE(nullptr, __ab); + EXPECT_NE(nullptr, ab_); + EXPECT_NE(nullptr, abc__); + EXPECT_NE(nullptr, _def__); + + Bindings bindings; + EXPECT_EQ(Lookup("GET", "/a/book/c/d/e", &bindings), a_cde); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "book"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/a/hello/b/world/c", &bindings), a_b_c); + EXPECT_EQ( + Bindings({ + Binding{FieldPath{"x"}, "a/hello"}, Binding{FieldPath{"y"}, "world"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/zoo/d/animal/tiger", &bindings), ab_d__); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "b/zoo"}, + Binding{FieldPath{"y"}, "d/animal/tiger"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/alpha/dog/beta/eat/bones/gamma", &bindings), + alpha_beta__gamma); + EXPECT_EQ( + Bindings({ + Binding{FieldPath{"x"}, "dog"}, Binding{FieldPath{"y"}, "eat/bones"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/foo/a", &bindings), _a); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "foo"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/foo/bar/a/b", &bindings), __ab); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "foo/bar"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/foo", &bindings), ab_); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "foo"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/c/foo/bar/baz", &bindings), abc__); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "foo/bar/baz"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/foo/d/e/f/bar/baz", &bindings), _def__); + EXPECT_EQ( + Bindings({ + Binding{FieldPath{"x"}, "foo"}, Binding{FieldPath{"y"}, "bar/baz"}, + }), + bindings); +} + +TEST_F(PathMatcherTest, PercentEscapesUnescapedForSingleSegment) { + MethodInfo* a_c = AddGetPath("/a/{x}/c"); + Build(); + + EXPECT_NE(nullptr, a_c); + + Bindings bindings; + EXPECT_EQ(Lookup("GET", "/a/p%20q%2Fr/c", &bindings), a_c); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "p q/r"}, + }), + bindings); +} + +namespace { + +char HexDigit(unsigned char digit, bool uppercase) { + if (digit < 10) { + return '0' + digit; + } else if (uppercase) { + return 'A' + digit - 10; + } else { + return 'a' + digit - 10; + } +} + +} // namespace { + +TEST_F(PathMatcherTest, PercentEscapesUnescapedForSingleSegmentAllAsciiChars) { + MethodInfo* a_c = AddGetPath("/{x}"); + Build(); + + EXPECT_NE(nullptr, a_c); + + for (int u = 0; u < 2; ++u) { + for (char c = 0; c < 0x7f; ++c) { + std::string path("/%"); + path += HexDigit((c & 0xf0) >> 4, 0 != u); + path += HexDigit(c & 0x0f, 0 != u); + + Bindings bindings; + EXPECT_EQ(Lookup("GET", path, &bindings), a_c); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, std::string(1, (char)c)}, + }), + bindings); + } + } +} + +TEST_F(PathMatcherTest, PercentEscapesNotUnescapedForMultiSegment1) { + MethodInfo* ap_q_c = AddGetPath("/a/{x=p/*/q/*}/c"); + Build(); + + EXPECT_NE(nullptr, ap_q_c); + + Bindings bindings; + EXPECT_EQ(Lookup("GET", "/a/p/foo%20foo/q/bar%2Fbar/c", &bindings), ap_q_c); + // space (%20) is escaped, but slash (%2F) isn't. + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "p/foo foo/q/bar%2Fbar"}}), + bindings); +} + +TEST_F(PathMatcherTest, PercentEscapesNotUnescapedForMultiSegment2) { + MethodInfo* a__c = AddGetPath("/a/{x=**}/c"); + Build(); + + EXPECT_NE(nullptr, a__c); + + Bindings bindings; + EXPECT_EQ(Lookup("GET", "/a/p/foo%20foo/q/bar%2Fbar/c", &bindings), a__c); + // space (%20) is escaped, but slash (%2F) isn't. + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "p/foo foo/q/bar%2Fbar"}}), + bindings); +} + +TEST_F(PathMatcherTest, OnlyUnreservedCharsAreUnescapedForMultiSegmentMatch) { + MethodInfo* a__c = AddGetPath("/a/{x=**}/c"); + Build(); + + EXPECT_NE(nullptr, a__c); + + Bindings bindings; + EXPECT_EQ( + Lookup("GET", + "/a/%21%23%24%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D/c", + &bindings), + a__c); + + // All %XX are reserved characters, they should be intact. + EXPECT_EQ(Bindings({Binding{ + FieldPath{"x"}, + "%21%23%24%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D"}}), + bindings); +} + +TEST_F(PathMatcherTest, VariableBindingsWithCustomVerb) { + MethodInfo* a_verb = AddGetPath("/a/{y=*}:verb"); + MethodInfo* ad__verb = AddGetPath("/a/{y=d/**}:verb"); + MethodInfo* _averb = AddGetPath("/{x=*}/a:verb"); + MethodInfo* __bverb = AddGetPath("/{x=**}/b:verb"); + MethodInfo* e_fverb = AddGetPath("/e/{x=*}/f:verb"); + MethodInfo* g__hverb = AddGetPath("/g/{x=**}/h:verb"); + Build(); + + EXPECT_NE(nullptr, a_verb); + EXPECT_NE(nullptr, ad__verb); + EXPECT_NE(nullptr, _averb); + EXPECT_NE(nullptr, __bverb); + EXPECT_NE(nullptr, e_fverb); + EXPECT_NE(nullptr, g__hverb); + + Bindings bindings; + EXPECT_EQ(Lookup("GET", "/a/world:verb", &bindings), a_verb); + EXPECT_EQ(Bindings({Binding{FieldPath{"y"}, "world"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/a/d/animal/tiger:verb", &bindings), ad__verb); + EXPECT_EQ(Bindings({Binding{FieldPath{"y"}, "d/animal/tiger"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/foo/a:verb", &bindings), _averb); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "foo"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/foo/bar/baz/b:verb", &bindings), __bverb); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "foo/bar/baz"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/e/foo/f:verb", &bindings), e_fverb); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "foo"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/g/foo/bar/h:verb", &bindings), g__hverb); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "foo/bar"}}), bindings); +} + +TEST_F(PathMatcherTest, ConstantSuffixesWithVariable) { + MethodInfo* ab__ = AddGetPath("/a/{x=b/**}"); + MethodInfo* ab__z = AddGetPath("/a/{x=b/**}/z"); + MethodInfo* ab__yz = AddGetPath("/a/{x=b/**}/y/z"); + MethodInfo* ab__verb = AddGetPath("/a/{x=b/**}:verb"); + MethodInfo* a__ = AddGetPath("/a/{x=**}"); + MethodInfo* c_d__e = AddGetPath("/c/{x=*}/{y=d/**}/e"); + MethodInfo* c_d__everb = AddGetPath("/c/{x=*}/{y=d/**}/e:verb"); + MethodInfo* f___g = AddGetPath("/f/{x=*}/{y=**}/g"); + MethodInfo* f___gverb = AddGetPath("/f/{x=*}/{y=**}/g:verb"); + MethodInfo* ab_yz__foo = AddGetPath("/a/{x=b/*/y/z/**}/foo"); + MethodInfo* ab___yzfoo = AddGetPath("/a/{x=b/*/**/y/z}/foo"); + Build(); + + EXPECT_NE(nullptr, ab__); + EXPECT_NE(nullptr, ab__z); + EXPECT_NE(nullptr, ab__yz); + EXPECT_NE(nullptr, ab__verb); + EXPECT_NE(nullptr, c_d__e); + EXPECT_NE(nullptr, c_d__everb); + EXPECT_NE(nullptr, f___g); + EXPECT_NE(nullptr, f___gverb); + EXPECT_NE(nullptr, ab_yz__foo); + EXPECT_NE(nullptr, ab___yzfoo); + + Bindings bindings; + + EXPECT_EQ(Lookup("GET", "/a/b/hello/world/c", &bindings), ab__); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "b/hello/world/c"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/world/c/z", &bindings), ab__z); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "b/world/c"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/world/c/y/z", &bindings), ab__yz); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "b/world/c"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/world/c:verb", &bindings), ab__verb); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "b/world/c"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/a/hello/b/world/c", &bindings), a__); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "hello/b/world/c"}}), bindings); + + EXPECT_EQ(Lookup("GET", "/c/hello/d/esp/world/e", &bindings), c_d__e); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "hello"}, + Binding{FieldPath{"y"}, "d/esp/world"}}), + bindings); + + EXPECT_EQ(Lookup("GET", "/c/hola/d/esp/mundo/e:verb", &bindings), c_d__everb); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "hola"}, + Binding{FieldPath{"y"}, "d/esp/mundo"}}), + bindings); + + EXPECT_EQ(Lookup("GET", "/f/foo/bar/baz/g", &bindings), f___g); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "foo"}, + Binding{FieldPath{"y"}, "bar/baz"}}), + bindings); + + EXPECT_EQ(Lookup("GET", "/f/foo/bar/baz/g:verb", &bindings), f___gverb); + EXPECT_EQ(Bindings({Binding{FieldPath{"x"}, "foo"}, + Binding{FieldPath{"y"}, "bar/baz"}}), + bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/foo/y/z/bar/baz/foo", &bindings), ab_yz__foo); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "b/foo/y/z/bar/baz"}, + }), + bindings); + + EXPECT_EQ(Lookup("GET", "/a/b/foo/bar/baz/y/z/foo", &bindings), ab___yzfoo); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "b/foo/bar/baz/y/z"}, + }), + bindings); +} + +TEST_F(PathMatcherTest, InvalidTemplates) { + EXPECT_EQ(nullptr, AddGetPath("/a{x=b/**}/{y=*}")); + EXPECT_EQ(nullptr, AddGetPath("/a{x=b/**}/bb/{y=*}")); + EXPECT_EQ(nullptr, AddGetPath("/a{x=b/**}/{y=**}")); + EXPECT_EQ(nullptr, AddGetPath("/a{x=b/**}/bb/{y=**}")); + + EXPECT_EQ(nullptr, AddGetPath("/a/**/*")); + EXPECT_EQ(nullptr, AddGetPath("/a/**/foo/*")); + EXPECT_EQ(nullptr, AddGetPath("/a/**/**")); + EXPECT_EQ(nullptr, AddGetPath("/a/**/foo/**")); +} + +TEST_F(PathMatcherTest, CustomVerbMatches) { + MethodInfo* some_const_verb = AddGetPath("/some/const:verb"); + MethodInfo* some__verb = AddGetPath("/some/*:verb"); + MethodInfo* some__foo_verb = AddGetPath("/some/*/foo:verb"); + MethodInfo* other__verb = AddGetPath("/other/**:verb"); + MethodInfo* other__const_verb = AddGetPath("/other/**/const:verb"); + Build(); + + EXPECT_NE(nullptr, some_const_verb); + EXPECT_NE(nullptr, some__verb); + EXPECT_NE(nullptr, some__foo_verb); + EXPECT_NE(nullptr, other__verb); + EXPECT_NE(nullptr, other__const_verb); + + EXPECT_EQ(LookupNoBindings("GET", "/some/const:verb"), some_const_verb); + EXPECT_EQ(LookupNoBindings("GET", "/some/other:verb"), some__verb); + EXPECT_EQ(LookupNoBindings("GET", "/some/other:verb/"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/some/bar/foo:verb"), some__foo_verb); + EXPECT_EQ(LookupNoBindings("GET", "/some/foo1/foo2/foo:verb"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/some/foo/bar:verb"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/other/bar/foo:verb"), other__verb); + EXPECT_EQ(LookupNoBindings("GET", "/other/bar/foo/const:verb"), + other__const_verb); +} + +TEST_F(PathMatcherTest, CustomVerbMatch2) { + MethodInfo* verb = AddGetPath("/*/*:verb"); + Build(); + EXPECT_EQ(LookupNoBindings("GET", "/some:verb/const:verb"), verb); +} + +TEST_F(PathMatcherTest, CustomVerbMatch3) { + EXPECT_NE(nullptr, AddGetPath("/foo/*")); + Build(); + + // This should match. But due to an implementation bug which + // blinkdly replacing last : with /, it will use /foo/other/verb + // to match /foo/* which will fail. + EXPECT_EQ(LookupNoBindings("GET", "/foo/other:verb"), nullptr); +} + +TEST_F(PathMatcherTest, CustomVerbMatch4) { + MethodInfo* a = AddGetPath("/foo/*/hello"); + Build(); + + EXPECT_NE(nullptr, a); + + // last slash is before last colon. + EXPECT_EQ(LookupNoBindings("GET", "/foo/other:verb/hello"), a); +} + +TEST_F(PathMatcherTest, RejectPartialMatches) { + MethodInfo* prefix_middle_suffix = AddGetPath("/prefix/middle/suffix"); + MethodInfo* prefix_middle = AddGetPath("/prefix/middle"); + MethodInfo* prefix = AddGetPath("/prefix"); + Build(); + + EXPECT_NE(nullptr, prefix_middle_suffix); + EXPECT_NE(nullptr, prefix_middle); + EXPECT_NE(nullptr, prefix); + + EXPECT_EQ(LookupNoBindings("GET", "/prefix/middle/suffix"), + prefix_middle_suffix); + EXPECT_EQ(LookupNoBindings("GET", "/prefix/middle"), prefix_middle); + EXPECT_EQ(LookupNoBindings("GET", "/prefix"), prefix); + + EXPECT_EQ(LookupNoBindings("GET", "/prefix/middle/suffix/other"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/prefix/middle/other"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/prefix/other"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/other"), nullptr); +} + +TEST_F(PathMatcherTest, LookupReturnsNullIfMatcherEmpty) { + Build(); + EXPECT_EQ(LookupNoBindings("GET", "a/b/blue/foo"), nullptr); +} + +TEST_F(PathMatcherTest, LookupSimplePaths) { + MethodInfo* pms = AddGetPath("/prefix/middle/suffix"); + MethodInfo* pmo = AddGetPath("/prefix/middle/othersuffix"); + MethodInfo* pos = AddGetPath("/prefix/othermiddle/suffix"); + MethodInfo* oms = AddGetPath("/otherprefix/middle/suffix"); + MethodInfo* os = AddGetPath("/otherprefix/suffix"); + Build(); + + EXPECT_NE(nullptr, pms); + EXPECT_NE(nullptr, pmo); + EXPECT_NE(nullptr, pos); + EXPECT_NE(nullptr, oms); + EXPECT_NE(nullptr, os); + + EXPECT_EQ(LookupNoBindings("GET", "/prefix/not/a/path"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/prefix/middle"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/prefix/not/othermiddle"), nullptr); + EXPECT_EQ(LookupNoBindings("GET", "/otherprefix/suffix/othermiddle"), + nullptr); + + EXPECT_EQ(LookupNoBindings("GET", "/prefix/middle/suffix"), pms); + EXPECT_EQ(LookupNoBindings("GET", "/prefix/middle/othersuffix"), pmo); + EXPECT_EQ(LookupNoBindings("GET", "/prefix/othermiddle/suffix"), pos); + EXPECT_EQ(LookupNoBindings("GET", "/otherprefix/middle/suffix"), oms); + EXPECT_EQ(LookupNoBindings("GET", "/otherprefix/suffix"), os); + EXPECT_EQ(LookupNoBindings("GET", "/otherprefix/suffix?foo=bar"), os); +} + +TEST_F(PathMatcherTest, ReplacevoidForPath) { + const std::string path = "/foo/bar"; + auto first_mock_proc = AddGetPath(path); + auto second_mock_proc = AddGetPath(path); + Build(); + + EXPECT_NE(nullptr, first_mock_proc); + EXPECT_NE(nullptr, second_mock_proc); + + EXPECT_NE(first_mock_proc, LookupNoBindings("GET", path)); + EXPECT_NE(second_mock_proc, LookupNoBindings("GET", path)); +} + +// If a path matches a complete branch of trie, but is longer than the branch +// (ie. the trie cannot match all the way to the end of the path), Lookup +// should return nullptr. +TEST_F(PathMatcherTest, LookupReturnsNullForOverspecifiedPath) { + EXPECT_NE(nullptr, AddGetPath("/a/b/c")); + EXPECT_NE(nullptr, AddGetPath("/a/b")); + Build(); + EXPECT_EQ(LookupNoBindings("GET", "/a/b/c/d"), nullptr); +} + +TEST_F(PathMatcherTest, ReturnNullvoidSharedPtrForUnderspecifiedPath) { + EXPECT_NE(nullptr, AddGetPath("/a/b/c/d")); + Build(); + EXPECT_EQ(LookupNoBindings("GET", "/a/b/c"), nullptr); +} + +TEST_F(PathMatcherTest, DifferentHttpMethod) { + auto ab = AddGetPath("/a/b"); + Build(); + EXPECT_NE(nullptr, ab); + EXPECT_EQ(LookupNoBindings("GET", "/a/b"), ab); + EXPECT_EQ(LookupNoBindings("POST", "/a/b"), nullptr); +} + +TEST_F(PathMatcherTest, BodyFieldPathTest) { + auto a = AddPathWithBodyFieldPath("GET", "/a", "b"); + auto cd = AddPathWithBodyFieldPath("GET", "/c/d", "e.f.g"); + Build(); + EXPECT_NE(nullptr, a); + EXPECT_NE(nullptr, cd); + std::string body_field_path; + EXPECT_EQ(LookupWithBodyFieldPath("GET", "/a", nullptr, &body_field_path), a); + EXPECT_EQ("b", body_field_path); + EXPECT_EQ(LookupWithBodyFieldPath("GET", "/c/d", nullptr, &body_field_path), + cd); + EXPECT_EQ("e.f.g", body_field_path); +} + +TEST_F(PathMatcherTest, VariableBindingsWithQueryParams) { + MethodInfo* a = AddGetPath("/a"); + MethodInfo* a_b = AddGetPath("/a/{x}/b"); + MethodInfo* a_b_c = AddGetPath("/a/{x}/b/{y}/c"); + Build(); + + EXPECT_NE(nullptr, a); + EXPECT_NE(nullptr, a_b); + EXPECT_NE(nullptr, a_b_c); + + Bindings bindings; + EXPECT_EQ(LookupWithParams("GET", "/a", "x=hello", &bindings), a); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "hello"}, + }), + bindings); + + EXPECT_EQ(LookupWithParams("GET", "/a/book/b", "y=shelf&z=author", &bindings), + a_b); + EXPECT_EQ( + Bindings({ + Binding{FieldPath{"x"}, "book"}, Binding{FieldPath{"y"}, "shelf"}, + Binding{FieldPath{"z"}, "author"}, + }), + bindings); + + EXPECT_EQ(LookupWithParams("GET", "/a/hello/b/endpoints/c", + "z=server&t=proxy", &bindings), + a_b_c); + EXPECT_EQ( + Bindings({ + Binding{FieldPath{"x"}, "hello"}, + Binding{FieldPath{"y"}, "endpoints"}, + Binding{FieldPath{"z"}, "server"}, Binding{FieldPath{"t"}, "proxy"}, + }), + bindings); +} + +TEST_F(PathMatcherTest, VariableBindingsWithQueryParamsEncoding) { + MethodInfo* a = AddGetPath("/a"); + Build(); + + EXPECT_NE(nullptr, a); + + Bindings bindings; + EXPECT_EQ(LookupWithParams("GET", "/a", "x=Hello%20world", &bindings), a); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "Hello world"}, + }), + bindings); + + EXPECT_EQ(LookupWithParams("GET", "/a", "x=%24%25%2F%20%0A", &bindings), a); + EXPECT_EQ(Bindings({ + Binding{FieldPath{"x"}, "$%/ \n"}, + }), + bindings); +} + +TEST_F(PathMatcherTest, VariableBindingsWithQueryParamsAndSystemParams) { + std::set system_params{"key", "api_key"}; + MethodInfo* a_b = AddPathWithSystemParams("GET", "/a/{x}/b", &system_params); + Build(); + + EXPECT_NE(nullptr, a_b); + + Bindings bindings; + EXPECT_EQ(LookupWithParams("GET", "/a/hello/b", "y=world&api_key=secret", + &bindings), + a_b); + EXPECT_EQ( + Bindings({ + Binding{FieldPath{"x"}, "hello"}, Binding{FieldPath{"y"}, "world"}, + }), + bindings); + EXPECT_EQ( + LookupWithParams("GET", "/a/hello/b", "key=secret&y=world", &bindings), + a_b); + EXPECT_EQ( + Bindings({ + Binding{FieldPath{"x"}, "hello"}, Binding{FieldPath{"y"}, "world"}, + }), + bindings); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/proto/api_manager_status.proto b/contrib/endpoints/src/api_manager/proto/api_manager_status.proto new file mode 100644 index 00000000000..fb9a43aa3eb --- /dev/null +++ b/contrib/endpoints/src/api_manager/proto/api_manager_status.proto @@ -0,0 +1,66 @@ +// Copyright (C) Extensible Service Proxy Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. +// +//////////////////////////////////////////////////////////////////////////////// +// +// This file contains proto representations of the JSON output of ESP's status. + +syntax = "proto3"; + +package google.api_manager.proto; + +// Proto representation of ::esp::common::ServiceControlStatistics +message ServiceControlStatistics { + // Total number of Check() calls received. + uint64 total_called_checks = 1; + // Check sends to server from flushed cache items. + uint64 send_checks_by_flush = 2; + // Check sends to remote sever during Check() calls. + uint64 send_checks_in_flight = 3; + + // Total number of Report() calls received. + uint64 total_called_reports = 4; + // Report sends to server from flushed cache items. + uint64 send_reports_by_flush = 5; + // Report sends to remote sever during Report() calls. + uint64 send_reports_in_flight = 6; + + // The number of operations send, each input report has only 1 operation, but + // each report send to server may have multiple operations. The ratio of + // send_report_operations / total_called_reports will reflect report + // aggregation rate. send_report_operations may not reflect aggregation rate. + uint64 send_report_operations = 7; + + // Maximum report size send to server. + uint64 max_report_size = 8; +} + +// Status for ESP instances +message EspStatus { + // Service name + string service_name = 1; + + // Statistics from service control client + ServiceControlStatistics service_control_statistics = 2; +} diff --git a/contrib/endpoints/src/api_manager/proto/sample_server_config.pb.txt b/contrib/endpoints/src/api_manager/proto/sample_server_config.pb.txt new file mode 100644 index 00000000000..b1b8f72cde5 --- /dev/null +++ b/contrib/endpoints/src/api_manager/proto/sample_server_config.pb.txt @@ -0,0 +1,37 @@ +service_control_config { + force_disable : false, + url_override : "http://chemist_override" + + check_aggregator_config { + cache_entries: 1000 + flush_interval_ms: 10 + response_expiration_ms: 20 + } + + report_aggregator_config { + cache_entries: 1020 + flush_interval_ms: 15 + } +} + +metadata_server_config { + enabled: true + url: "http://metadata_server" +} + +google_authentication_secret: "{" + " The client secret goes here. " + "}" + +cloud_tracing_config { + force_disable: false, + url_override: "http://cloud_tracing_override" +} + +api_authentication_config { + force_disable: true +} + +experimental { + disable_log_status: true +} diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto new file mode 100644 index 00000000000..8680199847b --- /dev/null +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -0,0 +1,155 @@ +// Copyright (C) Extensible Service Proxy Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. +// +//////////////////////////////////////////////////////////////////////////////// +// +syntax = "proto3"; + +package google.api_manager.proto; + +// Top-level server_config message. +message ServerConfig { + // Server config used by service control client. + ServiceControlConfig service_control_config = 1; + + // Metadata server config + MetadataServerConfig metadata_server_config = 2; + + // Google Authentication Secret + string google_authentication_secret = 3; + + // Server config used by service control client. + CloudTracingConfig cloud_tracing_config = 4; + + // Server config used for API authentication + ApiAuthenticationConfig api_authentication_config = 5; + + // Experimental flags + Experimental experimental = 999; +} + +// Server config for service control +message ServiceControlConfig { + // Allows to disable the service-control feature regardless of the + // service-control configuration in service config. + bool force_disable = 1; + + // The URL of the service control server to use (overrides that specified in + // service config). If the URL scheme is not provided, https:// is assumed. + string url_override = 2; + + // Check aggregator config + CheckAggregatorConfig check_aggregator_config = 3; + + // Report aggregator config + ReportAggregatorConfig report_aggregator_config = 4; + + // Timeout in milliseconds on service control check requests. + // If the value is <= 0, default timeout is 5000 milliseconds. + int32 check_timeout_ms = 5; + + // Timeout in milliseconds on service control report requests. + // If the value is <= 0, default timeout is 15000 milliseconds. + int32 report_timeout_ms = 6; + + // The intermediate reports for streaming calls should not be more frequent + // than this value (in seconds) + int32 intermediate_report_min_interval = 7; +} + +// Check aggregator config +message CheckAggregatorConfig { + // The maximum number of cache entries that can be kept in the aggregation + // cache. Cache is disabled when entries <= 0. + int32 cache_entries = 1; + + // The maximum milliseconds before aggregated check requests are flushed to + // the server. + int32 flush_interval_ms = 2; + + // The maximum milliseconds before a cached check response should be deleted. + int32 response_expiration_ms = 3; +} + +// Report aggregator config +message ReportAggregatorConfig { + // The maximum number of cache entries that can be kept in the aggregation + // cache. Cache is disabled when entries <= 0. + int32 cache_entries = 1; + + // The maximum milliseconds before aggregated report requests are flushed to + // the server. The cache entry is deleted after the flush. + int32 flush_interval_ms = 2; +} + +// Server config for Metadata Server +message MetadataServerConfig { + // Whether the metadata server is enabled or not. + bool enabled = 1; + // MetadataServer url, if not specified defaults to "http://169.254.169.254" + string url = 2; +} + +// Server config for cloud tracing +message CloudTracingConfig { + // Allows to disable the cloud-tracing feature regardless of the presence of + // the cloud tracing header. + bool force_disable = 1; + + // Override for the Cloud Tracing API url If the URL scheme is not provided, + // https:// is assumed. + string url_override = 2; + + // Config for trace aggregation. + CloudTracingAggregationConfig aggregation_config = 3; + + // Config for trace sampling. + CloudTracingSamplingConfig samling_config = 4; +} + +message CloudTracingAggregationConfig { + // The maximum time to hold a trace before sent to Cloud Trace API + int32 time_millisec = 1; + + // The maximum number of traces that can be cached. + int32 cache_max_size = 2; +} + +message CloudTracingSamplingConfig { + // ApiManager enables cloud trace with this minimum rate even all their + // incoming requests don't have cloud trace enabled. Default value is 0.1. + double minimum_qps = 1; +} + +// Server config for API Authentication +message ApiAuthenticationConfig { + // Allows to disable the API authentication regardless of the auth + // configuration in service config. + bool force_disable = 1; +} + +message Experimental { + // Disable timed printouts of ESP status to the error log. + bool disable_log_status = 1; +} diff --git a/contrib/endpoints/src/api_manager/request_handler.cc b/contrib/endpoints/src/api_manager/request_handler.cc new file mode 100644 index 00000000000..47acaac7c80 --- /dev/null +++ b/contrib/endpoints/src/api_manager/request_handler.cc @@ -0,0 +1,136 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/request_handler.h" + +#include "google/devtools/cloudtrace/v1/trace.pb.h" +#include "google/protobuf/stubs/logging.h" +#include "src/api_manager/auth/service_account_token.h" +#include "src/api_manager/check_workflow.h" +#include "src/api_manager/cloud_trace/cloud_trace.h" +#include "src/api_manager/utils/marshalling.h" + +using ::google::api_manager::utils::Status; +using google::devtools::cloudtrace::v1::Traces; + +namespace google { +namespace api_manager { + +void RequestHandler::Check(std::function continuation) { + auto interception = [continuation, this](Status status) { + if (status.ok() && context_->cloud_trace()) { + context_->StartBackendSpanAndSetTraceContext(); + } + continuation(status); + }; + + context_->set_check_continuation(interception); + + // Run the check flow. + check_workflow_->Run(context_); +} + +void RequestHandler::AttemptIntermediateReport() { + // For grpc streaming calls, we send intermediate reports to represent + // streaming stats. Specifically: + // 1) We send request_count in the first report to indicate the start of a + // stream. + // 2) We send request_bytes, response_bytes in intermediate reports, which + // triggered by timer. + // 3) In the final report, we send all metrics except request_count if it + // already sent. + // We only send intermediate streaming report if the time_interval > + // intermediate_report_interval(). + if (std::chrono::duration_cast( + std::chrono::steady_clock::now() - context_->last_report_time()) + .count() < + context_->service_context()->intermediate_report_interval()) { + return; + } + service_control::ReportRequestInfo info; + info.is_first_report = context_->is_first_report(); + info.is_final_report = false; + context_->FillReportRequestInfo(NULL, &info); + + // Calling service_control Report. + Status status = context_->service_context()->service_control()->Report(info); + if (!status.ok()) { + context_->service_context()->env()->LogError( + "Failed to send intermediate report to service control."); + } else { + context_->set_first_report(false); + } + context_->set_last_report_time(std::chrono::steady_clock::now()); +} + +// Sends a report. +void RequestHandler::Report(std::unique_ptr response, + std::function continuation) { + // Close backend trace span. + context_->EndBackendSpan(); + + if (context_->service_context()->service_control()) { + service_control::ReportRequestInfo info; + info.is_first_report = context_->is_first_report(); + info.is_final_report = true; + context_->FillReportRequestInfo(response.get(), &info); + // Calling service_control Report. + Status status = + context_->service_context()->service_control()->Report(info); + if (!status.ok()) { + context_->service_context()->env()->LogError( + "Failed to send report to service control."); + } + } + + if (context_->cloud_trace()) { + context_->cloud_trace()->EndRootSpan(); + // Always set the project_id to the latest one. + // + // this is how project_id is calculated: if gce metadata is fetched, use + // its project_id. Otherwise, use the project_id from service_config if it + // is configured. + // gce metadata fetching is started by the first request. While fetching is + // in progress, subsequent requests will fail. These failed requests may + // have wrong project_id until gce metadata is fetched successfully. + context_->service_context()->cloud_trace_aggregator()->SetProjectId( + context_->service_context()->project_id()); + context_->service_context()->cloud_trace_aggregator()->AppendTrace( + context_->cloud_trace()->ReleaseTrace()); + } + + continuation(); +} + +std::string RequestHandler::GetBackendAddress() const { + if (context_->method()) { + return context_->method()->backend_address(); + } else { + return std::string(); + } +} + +std::string RequestHandler::GetRpcMethodFullName() const { + if (context_ && context_->method() && + !context_->method()->rpc_method_full_name().empty()) { + return context_->method()->rpc_method_full_name(); + } else { + return std::string(); + } +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/request_handler.h b/contrib/endpoints/src/api_manager/request_handler.h new file mode 100644 index 00000000000..d2ecd19483f --- /dev/null +++ b/contrib/endpoints/src/api_manager/request_handler.h @@ -0,0 +1,68 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_REQUEST_HANDLER_H_ +#define API_MANAGER_REQUEST_HANDLER_H_ + +#include "check_workflow.h" +#include "include/api_manager/request_handler_interface.h" +#include "src/api_manager/api_manager_impl.h" +#include "src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// Implements RequestHandlerInterface. +class RequestHandler : public RequestHandlerInterface { + public: + RequestHandler(std::shared_ptr check_workflow, + std::shared_ptr service_context, + std::unique_ptr request_data) + : context_(new context::RequestContext(service_context, + std::move(request_data))), + check_workflow_(check_workflow) {} + + virtual ~RequestHandler(){}; + + virtual void Check(std::function continuation); + + virtual void Report(std::unique_ptr response, + std::function continuation); + + virtual void AttemptIntermediateReport(); + + virtual std::string GetBackendAddress() const; + + virtual std::string GetRpcMethodFullName() const; + + // Get the method info. + const MethodInfo *method() const { return context_->method(); } + + // Get the method info. + const MethodCallInfo *method_call() const { return context_->method_call(); } + + private: + // The context object needs to pass to the continuation function the check + // handler as a lambda capture so it can be passed to the next check handler. + // In order to control the life time of context object, a shared_ptr is used. + // This object holds a ref_count, the continuation will hold another one. + std::shared_ptr context_; + + std::shared_ptr check_workflow_; +}; + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_REQUEST_HANDLER_H_ diff --git a/contrib/endpoints/src/api_manager/server_config_proto_test.cc b/contrib/endpoints/src/api_manager/server_config_proto_test.cc new file mode 100644 index 00000000000..669add63c62 --- /dev/null +++ b/contrib/endpoints/src/api_manager/server_config_proto_test.cc @@ -0,0 +1,186 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "gtest/gtest.h" + +#include +#include +#include + +#include "google/protobuf/text_format.h" +#include "src/api_manager/proto/server_config.pb.h" + +using ::google::protobuf::TextFormat; + +using ::google::api_manager::proto::CheckAggregatorConfig; +using ::google::api_manager::proto::ReportAggregatorConfig; +using ::google::api_manager::proto::ServerConfig; +using ::google::api_manager::proto::ServiceControlConfig; + +// Trivial tests that instantiate server config protos +// to make sure each field is set correctly. +namespace google { +namespace api_manager { + +namespace { + +const char kServerConfig[] = R"( +service_control_config { + force_disable : false, + url_override : "http://chemist_override" + + check_aggregator_config { + cache_entries: 1000 + flush_interval_ms: 10 + response_expiration_ms: 20 + } + + report_aggregator_config { + cache_entries: 1020 + flush_interval_ms: 15 + } +} + +metadata_server_config { + enabled: true + url: "http://metadata_server" +} + +google_authentication_secret: "{" + "The client secret goes here." + "}" + +cloud_tracing_config { + force_disable: false, + url_override: "http://cloud_tracing_override" +} + +api_authentication_config { + force_disable: true +} + +experimental { + disable_log_status: false +} +)"; + +TEST(ServerConfigProto, ServerConfigFromString) { + ServerConfig server_config; + ASSERT_TRUE(TextFormat::ParseFromString(kServerConfig, &server_config)); + + // Check service_control_config + EXPECT_EQ(false, server_config.service_control_config().force_disable()); + EXPECT_EQ("http://chemist_override", + server_config.service_control_config().url_override()); + + EXPECT_EQ(1000, server_config.service_control_config() + .check_aggregator_config() + .cache_entries()); + EXPECT_EQ(10, server_config.service_control_config() + .check_aggregator_config() + .flush_interval_ms()); + EXPECT_EQ(20, server_config.service_control_config() + .check_aggregator_config() + .response_expiration_ms()); + EXPECT_EQ(1020, server_config.service_control_config() + .report_aggregator_config() + .cache_entries()); + EXPECT_EQ(15, server_config.service_control_config() + .report_aggregator_config() + .flush_interval_ms()); + + // Check metadata_server_config + EXPECT_EQ(true, server_config.metadata_server_config().enabled()); + EXPECT_EQ("http://metadata_server", + server_config.metadata_server_config().url()); + + // Check google_authentication_secret + EXPECT_EQ("{The client secret goes here.}", + server_config.google_authentication_secret()); + + // Check cloud_tracing_config + EXPECT_EQ(false, server_config.cloud_tracing_config().force_disable()); + EXPECT_EQ("http://cloud_tracing_override", + server_config.cloud_tracing_config().url_override()); + + // Check api_authentication_config + EXPECT_EQ(true, server_config.api_authentication_config().force_disable()); + + // Check disable_log_status + EXPECT_EQ(false, server_config.experimental().disable_log_status()); +} + +TEST(ServerConfigProto, ValidateSampleServerConfig) { + const char kSampleServerConfigPath[] = + "src/api_manager/proto/sample_server_config.pb.txt"; + + std::ifstream ifs(kSampleServerConfigPath); + ASSERT_TRUE(ifs) << "Failed to open the sample server config file." + << std::endl; + + std::ostringstream ss; + ss << ifs.rdbuf(); + + auto sample_server_config = ss.str(); + ASSERT_FALSE(sample_server_config.empty()) + << "Sample server config must not be empty" << std::endl; + + ServerConfig server_config; + EXPECT_TRUE(TextFormat::ParseFromString(ss.str(), &server_config)); + + // Check disable_log_status + EXPECT_EQ(true, server_config.experimental().disable_log_status()); +} + +TEST(ServerConfigProto, ServerConfigSetManually) { + ServerConfig server_config; + + ServiceControlConfig* service_control_config = + server_config.mutable_service_control_config(); + + CheckAggregatorConfig* check_config = + service_control_config->mutable_check_aggregator_config(); + check_config->set_cache_entries(2000); + check_config->set_flush_interval_ms(20); + check_config->set_response_expiration_ms(23); + + ReportAggregatorConfig* report_config = + service_control_config->mutable_report_aggregator_config(); + report_config->set_cache_entries(2500); + report_config->set_flush_interval_ms(25); + + ASSERT_EQ(2000, server_config.service_control_config() + .check_aggregator_config() + .cache_entries()); + ASSERT_EQ(20, server_config.service_control_config() + .check_aggregator_config() + .flush_interval_ms()); + ASSERT_EQ(23, server_config.service_control_config() + .check_aggregator_config() + .response_expiration_ms()); + + ASSERT_EQ(2500, server_config.service_control_config() + .report_aggregator_config() + .cache_entries()); + ASSERT_EQ(25, server_config.service_control_config() + .report_aggregator_config() + .flush_interval_ms()); +} + +} // namespace + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/BUILD b/contrib/endpoints/src/api_manager/service_control/BUILD new file mode 100644 index 00000000000..5dcfda50cc8 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/BUILD @@ -0,0 +1,123 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +package(default_visibility = ["//src/api_manager:__subpackages__"]) + +cc_library( + name = "service_control", + srcs = [ + "aggregated.cc", + "logs_metrics_loader.cc", + "logs_metrics_loader.h", + "proto.cc", + "url.cc", + "url.h", + ], + hdrs = [ + "aggregated.h", + "info.h", + "interface.h", + "proto.h", + ], + linkopts = select({ + "//:darwin": [], + "//conditions:default": [ + "-lm", + "-luuid", + ], + }), + deps = [ + "//external:cc_wkt_protos", + "//external:googletest_prod", + "//external:grpc++", + "//external:protobuf", + "//external:service_config", + "//external:servicecontrol", + "//external:servicecontrol_client", + "//src/api_manager:impl_headers", + "//src/api_manager:server_config_proto", + "//src/api_manager/cloud_trace", + "//src/api_manager/utils", + ], +) + +cc_test( + name = "logs_metrics_loader_test", + size = "small", + srcs = [ + "logs_metrics_loader_test.cc", + ], + linkstatic = 1, + deps = [ + ":service_control", + "//external:googletest_main", + ], +) + +cc_test( + name = "proto_test", + size = "small", + srcs = [ + "proto_test.cc", + ], + data = glob(["testdata/*.golden"]), + linkstatic = 1, + deps = [ + ":service_control", + "//external:googletest_main", + ], +) + +cc_test( + name = "url_test", + size = "small", + srcs = [ + "url_test.cc", + ], + linkstatic = 1, + deps = [ + ":service_control", + "//external:googletest_main", + "//src/api_manager:mock_api_manager_environment", + ], +) + +cc_test( + name = "aggregated_test", + size = "small", + srcs = [ + "aggregated_test.cc", + ], + linkstatic = 1, + deps = [ + ":service_control", + "//external:googletest_main", + "//src/api_manager:mock_api_manager_environment", + ], +) + +cc_test( + name = "check_response_test", + size = "small", + srcs = [ + "check_response_test.cc", + ], + linkstatic = 1, + deps = [ + ":service_control", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.cc b/contrib/endpoints/src/api_manager/service_control/aggregated.cc new file mode 100644 index 00000000000..ec8d8e31e2c --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.cc @@ -0,0 +1,454 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/aggregated.h" + +#include +#include +#include "src/api_manager/service_control/logs_metrics_loader.h" + +using ::google::api::servicecontrol::v1::CheckRequest; +using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::ReportRequest; +using ::google::api::servicecontrol::v1::ReportResponse; +using ::google::api_manager::proto::ServerConfig; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +using ::google::service_control_client::CheckAggregationOptions; +using ::google::service_control_client::ReportAggregationOptions; +using ::google::service_control_client::ServiceControlClient; +using ::google::service_control_client::ServiceControlClientOptions; +using ::google::service_control_client::TransportDoneFunc; + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +// Default config for check aggregator +const int kCheckAggregationEntries = 10000; +// Check doesn't support quota yet. It is safe to increase +// the cache life of check results. +// Cache life is 5 minutes. It will be refreshed every minute. +const int kCheckAggregationFlushIntervalMs = 60000; +const int kCheckAggregationExpirationMs = 300000; + +// Default config for report aggregator +const int kReportAggregationEntries = 10000; +const int kReportAggregationFlushIntervalMs = 1000; + +// The default connection timeout for check requests. +const int kCheckDefaultTimeoutInMs = 5000; +// The default connection timeout for report requests. +const int kReportDefaultTimeoutInMs = 15000; + +// The maximum protobuf pool size. All usages of pool alloc() and free() are +// within a function frame. If no con-current usage, pool size of 1 is enough. +// This number should correspond to maximum of concurrent calls. +const int kProtoPoolMaxSize = 100; + +// Defines protobuf content type. +const char application_proto[] = "application/x-protobuf"; + +// The service_control service name. used for as audience to generate JWT token. +const char servicecontrol_service[] = + "/google.api.servicecontrol.v1.ServiceController"; + +// Generates CheckAggregationOptions. +CheckAggregationOptions GetCheckAggregationOptions( + const ServerConfig* server_config) { + if (server_config && server_config->has_service_control_config() && + server_config->service_control_config().has_check_aggregator_config()) { + const auto& check_config = + server_config->service_control_config().check_aggregator_config(); + return CheckAggregationOptions(check_config.cache_entries(), + check_config.flush_interval_ms(), + check_config.response_expiration_ms()); + } + return CheckAggregationOptions(kCheckAggregationEntries, + kCheckAggregationFlushIntervalMs, + kCheckAggregationExpirationMs); +} + +// Generates ReportAggregationOptions. +ReportAggregationOptions GetReportAggregationOptions( + const ServerConfig* server_config) { + if (server_config && server_config->has_service_control_config() && + server_config->service_control_config().has_report_aggregator_config()) { + const auto& report_config = + server_config->service_control_config().report_aggregator_config(); + return ReportAggregationOptions(report_config.cache_entries(), + report_config.flush_interval_ms()); + } + return ReportAggregationOptions(kReportAggregationEntries, + kReportAggregationFlushIntervalMs); +} + +} // namespace + +template +std::unique_ptr Aggregated::ProtoPool::Alloc() { + std::lock_guard lock(mutex_); + if (!pool_.empty()) { + auto item = std::move(pool_.front()); + pool_.pop_front(); + item->Clear(); + return item; + } else { + return std::unique_ptr(new Type); + } +} + +template +void Aggregated::ProtoPool::Free(std::unique_ptr item) { + std::lock_guard lock(mutex_); + if (pool_.size() < kProtoPoolMaxSize) { + pool_.push_back(std::move(item)); + } +} + +Aggregated::Aggregated(const ::google::api::Service& service, + const ServerConfig* server_config, + ApiManagerEnvInterface* env, + auth::ServiceAccountToken* sa_token, + const std::set& logs, + const std::set& metrics, + const std::set& labels) + : service_(&service), + server_config_(server_config), + env_(env), + sa_token_(sa_token), + service_control_proto_(logs, metrics, labels, service.name(), + service.id()), + url_(service_, server_config), + mismatched_check_config_id(service.id()), + mismatched_report_config_id(service.id()), + max_report_size_(0) { + if (sa_token_) { + sa_token_->SetAudience( + auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL, + url_.service_control() + servicecontrol_service); + } +} + +Aggregated::Aggregated(const std::set& logs, + ApiManagerEnvInterface* env, + std::unique_ptr client) + : service_(nullptr), + server_config_(nullptr), + env_(env), + sa_token_(nullptr), + service_control_proto_(logs, "", ""), + url_(service_, server_config_), + client_(std::move(client)), + max_report_size_(0) {} + +Aggregated::~Aggregated() {} + +Status Aggregated::Init() { + // Init() can be called repeatedly. + if (client_) { + return Status::OK; + } + + // It is too early to create client_ at constructor. + // Client creation is calling env->StartPeriodicTimer. + // env->StartPeriodicTimer doens't work at constructor. + ServiceControlClientOptions options( + GetCheckAggregationOptions(server_config_), + GetReportAggregationOptions(server_config_)); + + std::stringstream ss; + ss << "Check_aggregation_options: " + << "num_entries: " << options.check_options.num_entries + << ", flush_interval_ms: " << options.check_options.flush_interval_ms + << ", expiration_ms: " << options.check_options.expiration_ms + << ", Report_aggregation_options: " + << "num_entries: " << options.report_options.num_entries + << ", flush_interval_ms: " << options.report_options.flush_interval_ms; + env_->LogInfo(ss.str().c_str()); + + options.check_transport = [this]( + const CheckRequest& request, CheckResponse* response, + TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; + options.report_transport = [this]( + const ReportRequest& request, ReportResponse* response, + TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; + + options.periodic_timer = [this](int interval_ms, + std::function callback) + -> std::unique_ptr<::google::service_control_client::PeriodicTimer> { + return std::unique_ptr<::google::service_control_client::PeriodicTimer>( + new ApiManagerPeriodicTimer(env_->StartPeriodicTimer( + std::chrono::milliseconds(interval_ms), callback))); + }; + client_ = ::google::service_control_client::CreateServiceControlClient( + service_->name(), service_->id(), options); + return Status::OK; +} + +Status Aggregated::Close() { + // Just destroy the client to flush all its cache. + client_.reset(); + return Status::OK; +} + +Status Aggregated::Report(const ReportRequestInfo& info) { + if (!client_) { + return Status(Code::INTERNAL, "Missing service control client"); + } + auto request = report_pool_.Alloc(); + Status status = service_control_proto_.FillReportRequest(info, request.get()); + if (!status.ok()) { + report_pool_.Free(std::move(request)); + return status; + } + ReportResponse* response = new ReportResponse; + client_->Report( + *request, response, + [this, response](const ::google::protobuf::util::Status& status) { + if (service_control_proto_.service_config_id() != + response->service_config_id()) { + if (mismatched_report_config_id != response->service_config_id()) { + env_->LogWarning( + "Received non-matching report response service config ID: '" + + response->service_config_id() + "', requested: '" + + service_control_proto_.service_config_id() + "'"); + mismatched_report_config_id = response->service_config_id(); + } + } + + if (!status.ok() && env_) { + env_->LogError(std::string("Service control report failed. " + + status.ToString())); + } + delete response; + }); + // There is no reference to request anymore at this point and it is safe to + // free request now. + report_pool_.Free(std::move(request)); + return Status::OK; +} + +void Aggregated::Check( + const CheckRequestInfo& info, cloud_trace::CloudTraceSpan* parent_span, + std::function on_done) { + std::shared_ptr trace_span( + CreateChildSpan(parent_span, "CheckServiceControlCache")); + CheckResponseInfo dummy_response_info; + if (!client_) { + on_done(Status(Code::INTERNAL, "Missing service control client"), + dummy_response_info); + return; + } + auto request = check_pool_.Alloc(); + Status status = service_control_proto_.FillCheckRequest(info, request.get()); + if (!status.ok()) { + on_done(status, dummy_response_info); + check_pool_.Free(std::move(request)); + return; + } + + CheckResponse* response = new CheckResponse; + bool allow_unregistered_calls = info.allow_unregistered_calls; + + auto check_on_done = [this, response, allow_unregistered_calls, on_done, + trace_span]( + const ::google::protobuf::util::Status& status) { + TRACE(trace_span) << "Check returned with status: " << status.ToString(); + CheckResponseInfo response_info; + + if (service_control_proto_.service_config_id() != + response->service_config_id()) { + if (mismatched_check_config_id != response->service_config_id()) { + env_->LogWarning( + "Received non-matching check response service config ID: '" + + response->service_config_id() + "', requested: '" + + service_control_proto_.service_config_id() + "'"); + mismatched_check_config_id = response->service_config_id(); + } + } + + if (status.ok()) { + Status status = Proto::ConvertCheckResponse( + *response, service_control_proto_.service_name(), &response_info); + // If allow_unregistered_calls is true, it is always OK to proceed. + if (allow_unregistered_calls) { + on_done(Status::OK, response_info); + } else { + on_done(status, response_info); + } + } else { + // If allow_unregistered_calls is true, it is always OK to proceed. + if (allow_unregistered_calls) { + on_done(Status::OK, response_info); + } else { + on_done(Status(status.error_code(), status.error_message(), + Status::SERVICE_CONTROL), + response_info); + } + } + delete response; + }; + + client_->Check( + *request, response, check_on_done, + [trace_span, this](const CheckRequest& request, CheckResponse* response, + TransportDoneFunc on_done) { + Call(request, response, on_done, trace_span.get()); + }); + // There is no reference to request anymore at this point and it is safe to + // free request now. + check_pool_.Free(std::move(request)); +} + +Status Aggregated::GetStatistics(Statistics* esp_stat) const { + if (!client_) { + return Status(Code::INTERNAL, "Missing service control client"); + } + + ::google::service_control_client::Statistics client_stat; + ::google::protobuf::util::Status status = + client_->GetStatistics(&client_stat); + + if (!status.ok()) { + return Status::FromProto(status); + } + esp_stat->total_called_checks = client_stat.total_called_checks; + esp_stat->send_checks_by_flush = client_stat.send_checks_by_flush; + esp_stat->send_checks_in_flight = client_stat.send_checks_in_flight; + esp_stat->total_called_reports = client_stat.total_called_reports; + esp_stat->send_reports_by_flush = client_stat.send_reports_by_flush; + esp_stat->send_reports_in_flight = client_stat.send_reports_in_flight; + esp_stat->send_report_operations = client_stat.send_report_operations; + esp_stat->max_report_size = max_report_size_; + + return Status::OK; +} + +template +void Aggregated::Call(const RequestType& request, ResponseType* response, + TransportDoneFunc on_done, + cloud_trace::CloudTraceSpan* parent_span) { + std::shared_ptr trace_span( + CreateChildSpan(parent_span, "Call ServiceControl server")); + std::unique_ptr http_request(new HTTPRequest([response, on_done, + trace_span, this]( + Status status, std::map&&, std::string&& body) { + TRACE(trace_span) << "HTTP response status: " << status.ToString(); + if (status.ok()) { + // Handle 200 response + if (!response->ParseFromString(body)) { + status = + Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); + } + } else { + const std::string& url = typeid(RequestType) == typeid(CheckRequest) + ? url_.check_url() + : url_.report_url(); + env_->LogError(std::string("Failed to call ") + url + ", Error: " + + status.ToString() + ", Response body: " + body); + + // Handle NGX error as opposed to pass-through error code + if (status.code() < 0) { + status = + Status(Code::UNAVAILABLE, "Failed to connect to service control"); + } else { + status = + Status(Code::UNAVAILABLE, + "Service control request failed with HTTP response code " + + std::to_string(status.code())); + } + } + on_done(status.ToProto()); + })); + + bool is_check = (typeid(RequestType) == typeid(CheckRequest)); + const std::string& url = is_check ? url_.check_url() : url_.report_url(); + TRACE(trace_span) << "Http request URL: " << url; + + std::string request_body; + request.SerializeToString(&request_body); + + if (!is_check && (request_body.size() > max_report_size_)) { + max_report_size_ = request_body.size(); + } + + http_request->set_url(url) + .set_method("POST") + .set_auth_token(GetAuthToken()) + .set_header("Content-Type", application_proto) + .set_body(request_body); + + // Set timeout on the request if it was so configured. + if (is_check) { + http_request->set_timeout_ms(kCheckDefaultTimeoutInMs); + } else { + http_request->set_timeout_ms(kReportDefaultTimeoutInMs); + } + if (server_config_ != nullptr && + server_config_->has_service_control_config()) { + const auto& config = server_config_->service_control_config(); + if (is_check) { + if (config.check_timeout_ms() > 0) { + http_request->set_timeout_ms(config.check_timeout_ms()); + } + } else { + if (config.report_timeout_ms() > 0) { + http_request->set_timeout_ms(config.report_timeout_ms()); + } + } + } + + env_->RunHTTPRequest(std::move(http_request)); +} + +const std::string& Aggregated::GetAuthToken() { + if (sa_token_) { + return sa_token_->GetAuthToken( + auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL); + } else { + static std::string empty; + return empty; + } +} + +Interface* Aggregated::Create(const ::google::api::Service& service, + const ServerConfig* server_config, + ApiManagerEnvInterface* env, + auth::ServiceAccountToken* sa_token) { + if (server_config && + server_config->service_control_config().force_disable()) { + env->LogError("Service control is disabled."); + return nullptr; + } + Url url(&service, server_config); + if (url.service_control().empty()) { + env->LogError( + "Service control address is not specified. Disabling API management."); + return nullptr; + } + std::set logs, metrics, labels; + Status s = LogsMetricsLoader::Load(service, &logs, &metrics, &labels); + return new Aggregated(service, server_config, env, sa_token, logs, metrics, + labels); +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.h b/contrib/endpoints/src/api_manager/service_control/aggregated.h new file mode 100644 index 00000000000..00c4a8c5f58 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.h @@ -0,0 +1,156 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_SERVICE_CONTROL_AGGREGATED_H_ +#define API_MANAGER_SERVICE_CONTROL_AGGREGATED_H_ + +#include "google/api/service.pb.h" +#include "google/api/servicecontrol/v1/service_controller.pb.h" +#include "include/api_manager/env_interface.h" +#include "include/service_control_client.h" +#include "src/api_manager/auth/service_account_token.h" +#include "src/api_manager/cloud_trace/cloud_trace.h" +#include "src/api_manager/proto/server_config.pb.h" +#include "src/api_manager/service_control/interface.h" +#include "src/api_manager/service_control/proto.h" +#include "src/api_manager/service_control/url.h" + +#include +#include + +namespace google { +namespace api_manager { +namespace service_control { + +// This implementation uses service-control-client-cxx module. +class Aggregated : public Interface { + public: + static Interface* Create(const ::google::api::Service& service, + const proto::ServerConfig* server_config, + ApiManagerEnvInterface* env, + auth::ServiceAccountToken* sa_token); + + virtual ~Aggregated(); + + virtual utils::Status Report(const ReportRequestInfo& info); + + virtual void Check( + const CheckRequestInfo& info, cloud_trace::CloudTraceSpan* parent_span, + std::function on_done); + + virtual utils::Status Init(); + virtual utils::Status Close(); + + virtual utils::Status GetStatistics(Statistics* stat) const; + + private: + // A timer object to wrap PeriodicTimer + class ApiManagerPeriodicTimer + : public ::google::service_control_client::PeriodicTimer { + public: + ApiManagerPeriodicTimer( + std::unique_ptr<::google::api_manager::PeriodicTimer> esp_timer) + : esp_timer_(std::move(esp_timer)) {} + + // Cancels the timer. + virtual void Stop() { + if (esp_timer_) esp_timer_->Stop(); + } + + private: + std::unique_ptr<::google::api_manager::PeriodicTimer> esp_timer_; + }; + + // The protobuf pool to reuse protobuf. Performance tests showed that reusing + // protobuf is faster than allocating a new protobuf. + template + class ProtoPool { + public: + // Allocates a protobuf. If there is one in the pool, uses it, otherwise + // creates a new one. + std::unique_ptr Alloc(); + // Frees a protobuf. If pool did not reach maximum size, stores it in the + // pool, otherwise frees it. + void Free(std::unique_ptr item); + + private: + // Protobuf pool to store used protobufs. + std::list> pool_; + // Mutex to protect the protobuf pool. + std::mutex mutex_; + }; + + friend class AggregatedTestWithMockedClient; + // Constructor for unit-test only. + Aggregated( + const std::set& logs, ApiManagerEnvInterface* env, + std::unique_ptr<::google::service_control_client::ServiceControlClient> + client); + // The constructor. + Aggregated(const ::google::api::Service& service, + const proto::ServerConfig* server_config, + ApiManagerEnvInterface* env, auth::ServiceAccountToken* sa_token, + const std::set& logs, + const std::set& metrics, + const std::set& labels); + + // Calls to service control server. + template + void Call(const RequestType& request, ResponseType* response, + ::google::service_control_client::TransportDoneFunc on_done, + cloud_trace::CloudTraceSpan* parent_span); + + // Gets the auth token to access service control server. + const std::string& GetAuthToken(); + + // the sevice config. + const ::google::api::Service* service_; + // the server config. + const proto::ServerConfig* server_config_; + + // The Api Manager environment interface. + ApiManagerEnvInterface* env_; + + // service account token. + auth::ServiceAccountToken* sa_token_; + + // The object to fill service control Check and Report protobuf. + Proto service_control_proto_; + + // Stores service control urls. + Url url_; + + // The service control client instance. + std::unique_ptr<::google::service_control_client::ServiceControlClient> + client_; + // The protobuf pool to reuse CheckRequest protobuf. + ProtoPool<::google::api::servicecontrol::v1::CheckRequest> check_pool_; + // The protobuf pool to reuse ReportRequest protobuf. + ProtoPool<::google::api::servicecontrol::v1::ReportRequest> report_pool_; + + // Mismatched config ID received for a check request + std::string mismatched_check_config_id; + + // Mismatched config ID received for a report request + std::string mismatched_report_config_id; + + // Maximum report size send to server. + uint64_t max_report_size_; +}; + +} // namespace service_control +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_SERVICE_CONTROL_AGGREGATED_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc b/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc new file mode 100644 index 00000000000..51139c9b18c --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc @@ -0,0 +1,213 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/aggregated.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "include/api_manager/utils/status.h" +#include "src/api_manager/mock_api_manager_environment.h" +#include "src/api_manager/service_control/proto.h" + +using ::google::api::servicecontrol::v1::CheckRequest; +using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::ReportRequest; +using ::google::api::servicecontrol::v1::ReportResponse; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; +using ::google::service_control_client::ServiceControlClient; +using ::google::service_control_client::TransportCheckFunc; +using ::google::service_control_client::TransportReportFunc; +using ::testing::Return; +using ::testing::Invoke; +using ::testing::_; + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { +void FillOperationInfo(OperationInfo* op) { + op->operation_id = "operation_id"; + op->operation_name = "operation_name"; + op->api_key = "api_key_x"; + op->producer_project_id = "project_id"; +} + +class MockServiceControClient : public ServiceControlClient { + public: + MOCK_METHOD3(Check, void(const CheckRequest&, CheckResponse*, DoneCallback)); + MOCK_METHOD2(Check, ::google::protobuf::util::Status(const CheckRequest&, + CheckResponse*)); + MOCK_METHOD4(Check, void(const CheckRequest&, CheckResponse*, DoneCallback, + TransportCheckFunc)); + MOCK_METHOD3(Report, + void(const ReportRequest&, ReportResponse*, DoneCallback)); + MOCK_METHOD2(Report, ::google::protobuf::util::Status(const ReportRequest&, + ReportResponse*)); + MOCK_METHOD4(Report, void(const ReportRequest&, ReportResponse*, DoneCallback, + TransportReportFunc)); + MOCK_CONST_METHOD1(GetStatistics, + ::google::protobuf::util::Status( + ::google::service_control_client::Statistics*)); +}; +} // namespace + +class AggregatedTestWithMockedClient : public ::testing::Test { + public: + void SetUp() { + env_.reset(new ::testing::NiceMock); + mock_client_ = new MockServiceControClient; + sc_lib_.reset( + new Aggregated({"local_test_log"}, env_.get(), + std::unique_ptr(mock_client_))); + ASSERT_TRUE((bool)(sc_lib_)); + } + + void Check(const CheckRequest& req, CheckResponse* res, + ServiceControlClient::DoneCallback on_done, + TransportCheckFunc transport) { + on_done(done_status_); + } + void Report(const ReportRequest& req, ReportResponse* res, + ServiceControlClient::DoneCallback on_done) { + on_done(done_status_); + } + + ::google::protobuf::util::Status done_status_; + std::unique_ptr env_; + MockServiceControClient* mock_client_; + std::unique_ptr sc_lib_; +}; + +TEST_F(AggregatedTestWithMockedClient, ReportTest) { + EXPECT_CALL(*mock_client_, Report(_, _, _)) + .WillOnce(Invoke(this, &AggregatedTestWithMockedClient::Report)); + ReportRequestInfo info; + FillOperationInfo(&info); + // mock the client to return OK + done_status_ = ::google::protobuf::util::Status::OK; + ASSERT_TRUE(sc_lib_->Report(info).ok()); +} + +TEST_F(AggregatedTestWithMockedClient, FailedReportTest) { + EXPECT_CALL(*mock_client_, Report(_, _, _)) + .WillOnce(Invoke(this, &AggregatedTestWithMockedClient::Report)); + ReportRequestInfo info; + FillOperationInfo(&info); + // mock the client to return failed status. + done_status_ = ::google::protobuf::util::Status( + Code::INTERNAL, "AggregatedTestWithMockedClient internal error"); + // Client layer failure is ignored. + ASSERT_TRUE(sc_lib_->Report(info).ok()); +} + +TEST_F(AggregatedTestWithMockedClient, FailedCheckRequiredFieldTest) { + CheckRequestInfo info; + FillOperationInfo(&info); + info.operation_name = nullptr; // Missing operation_name + sc_lib_->Check(info, nullptr, + [](Status status, const CheckResponseInfo& info) { + ASSERT_EQ(Code::INVALID_ARGUMENT, status.code()); + }); +} + +TEST_F(AggregatedTestWithMockedClient, CheckTest) { + EXPECT_CALL(*mock_client_, Check(_, _, _, _)) + .WillOnce(Invoke(this, &AggregatedTestWithMockedClient::Check)); + CheckRequestInfo info; + FillOperationInfo(&info); + // mock the client to return OK + done_status_ = ::google::protobuf::util::Status::OK; + sc_lib_->Check(info, nullptr, + [](Status status, const CheckResponseInfo& info) { + ASSERT_TRUE(status.ok()); + }); +} + +TEST_F(AggregatedTestWithMockedClient, FailedCheckTest) { + EXPECT_CALL(*mock_client_, Check(_, _, _, _)) + .WillOnce(Invoke(this, &AggregatedTestWithMockedClient::Check)); + CheckRequestInfo info; + FillOperationInfo(&info); + // mock the client to return OK + done_status_ = ::google::protobuf::util::Status( + Code::INTERNAL, "AggregatedTestWithMockedClient internal error"); + sc_lib_->Check(info, nullptr, + [](Status status, const CheckResponseInfo& info) { + ASSERT_EQ(status.code(), Code::INTERNAL); + }); +} + +class AggregatedTestWithRealClient : public ::testing::Test { + public: + void SetUp() { + service_.set_name("test_service"); + service_.mutable_control()->set_environment( + "servicecontrol.googleapis.com"); + env_.reset(new ::testing::NiceMock); + sc_lib_.reset(Aggregated::Create(service_, nullptr, env_.get(), nullptr)); + ASSERT_TRUE((bool)(sc_lib_)); + // This is the call actually creating the client. + sc_lib_->Init(); + } + + void DoRunHTTPRequest(HTTPRequest* request) { + std::map headers; + std::string body; + request->OnComplete(Status::OK, std::move(headers), std::move(body)); + } + + ::google::api::Service service_; + std::unique_ptr env_; + std::unique_ptr sc_lib_; +}; + +TEST_F(AggregatedTestWithRealClient, CheckOKTest) { + EXPECT_CALL(*env_, DoRunHTTPRequest(_)) + .WillOnce(Invoke(this, &AggregatedTestWithRealClient::DoRunHTTPRequest)); + + CheckRequestInfo info; + FillOperationInfo(&info); + sc_lib_->Check(info, nullptr, + [](Status status, const CheckResponseInfo& info) { + ASSERT_TRUE(status.ok()); + }); + + Statistics stat; + Status stat_status = sc_lib_->GetStatistics(&stat); + EXPECT_EQ(stat_status, Status::OK); + EXPECT_EQ(stat.total_called_checks, 1); + EXPECT_EQ(stat.send_checks_by_flush, 0); + EXPECT_EQ(stat.send_checks_in_flight, 1); + EXPECT_EQ(stat.send_report_operations, 0); +} + +TEST(AggregatedServiceControlTest, Create) { + // Verify that invalid service config yields nullptr. + ::google::api::Service + invalid_service; // only contains name, not service control address. + invalid_service.set_name("invalid-service"); + + std::unique_ptr env( + new ::testing::NiceMock); + std::unique_ptr sc_lib( + Aggregated::Create(invalid_service, nullptr, env.get(), nullptr)); + ASSERT_FALSE(sc_lib); +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/check_response_test.cc b/contrib/endpoints/src/api_manager/service_control/check_response_test.cc new file mode 100644 index 00000000000..390fdd07cc0 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/check_response_test.cc @@ -0,0 +1,142 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "gtest/gtest.h" +#include "include/api_manager/utils/status.h" +#include "src/api_manager/service_control/proto.h" + +namespace gasv1 = ::google::api::servicecontrol::v1; + +using ::google::api::servicecontrol::v1::CheckError; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +Status ConvertCheckErrorToStatus(gasv1::CheckError::Code code, + const char* error_detail, + const char* service_name) { + gasv1::CheckResponse response; + gasv1::CheckError* check_error = response.add_check_errors(); + CheckRequestInfo info; + check_error->set_code(code); + check_error->set_detail(error_detail); + return Proto::ConvertCheckResponse(response, service_name, nullptr); +} + +Status ConvertCheckErrorToStatus(gasv1::CheckError::Code code) { + gasv1::CheckResponse response; + std::string service_name; + response.add_check_errors()->set_code(code); + return Proto::ConvertCheckResponse(response, service_name, nullptr); +} + +} // namespace + +TEST(CheckResponseTest, AbortedWithInvalidArgumentWhenRespIsKeyInvalid) { + Status result = ConvertCheckErrorToStatus(CheckError::API_KEY_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(CheckResponseTest, AbortedWithInvalidArgumentWhenRespIsKeyExpired) { + Status result = ConvertCheckErrorToStatus(CheckError::API_KEY_EXPIRED); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithInvalidArgumentWhenRespIsBlockedWithNotFound) { + Status result = ConvertCheckErrorToStatus(CheckError::NOT_FOUND); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithInvalidArgumentWhenRespIsBlockedWithKeyNotFound) { + Status result = ConvertCheckErrorToStatus(CheckError::API_KEY_NOT_FOUND); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithServiceNotActivated) { + Status result = ConvertCheckErrorToStatus( + CheckError::SERVICE_NOT_ACTIVATED, "Service not activated.", "api_xxxx"); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); + EXPECT_EQ(result.message(), "API api_xxxx is not enabled for the project."); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithPermissionDenied) { + Status result = ConvertCheckErrorToStatus(CheckError::PERMISSION_DENIED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithIpAddressBlocked) { + Status result = ConvertCheckErrorToStatus(CheckError::IP_ADDRESS_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithRefererBlocked) { + Status result = ConvertCheckErrorToStatus(CheckError::REFERER_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithClientAppBlocked) { + Status result = ConvertCheckErrorToStatus(CheckError::CLIENT_APP_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenResponseIsBlockedWithProjectDeleted) { + Status result = ConvertCheckErrorToStatus(CheckError::PROJECT_DELETED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenResponseIsBlockedWithProjectInvalid) { + Status result = ConvertCheckErrorToStatus(CheckError::PROJECT_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(CheckResponseTest, + AbortedWithPermissionDeniedWhenResponseIsBlockedWithBillingDisabled) { + Status result = ConvertCheckErrorToStatus(CheckError::BILLING_DISABLED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(CheckResponseTest, FailOpenWhenResponseIsUnknownNamespaceLookup) { + EXPECT_TRUE( + ConvertCheckErrorToStatus(CheckError::NAMESPACE_LOOKUP_UNAVAILABLE).ok()); +} + +TEST(CheckResponseTest, FailOpenWhenResponseIsUnknownBillingStatus) { + EXPECT_TRUE( + ConvertCheckErrorToStatus(CheckError::BILLING_STATUS_UNAVAILABLE).ok()); +} + +TEST(CheckResponseTest, FailOpenWhenResponseIsUnknownServiceStatus) { + EXPECT_TRUE( + ConvertCheckErrorToStatus(CheckError::SERVICE_STATUS_UNAVAILABLE).ok()); +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/info.h b/contrib/endpoints/src/api_manager/service_control/info.h new file mode 100644 index 00000000000..58a2016dc5b --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/info.h @@ -0,0 +1,175 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_SERVICE_CONTROL_INFO_H_ +#define API_MANAGER_SERVICE_CONTROL_INFO_H_ + +#include "google/protobuf/stubs/stringpiece.h" + +#include +#include +#include +#include +#include + +#include "include/api_manager/compute_platform.h" +#include "include/api_manager/protocol.h" +#include "include/api_manager/service_control.h" +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { +namespace service_control { + +// Use the CheckRequestInfo and ReportRequestInfo to fill Service Control +// request protocol buffers. Use following two structures to pass +// in minimum info and call Fill functions to fill the protobuf. + +// Basic information about the API call (operation). +struct OperationInfo { + // Identity of the operation. It must be unique within the scope of the + // service. If the service calls Check() and Report() on the same operation, + // the two calls should carry the same operation id. + ::google::protobuf::StringPiece operation_id; + + // Fully qualified name of the operation. + ::google::protobuf::StringPiece operation_name; + + // The producer project id. + ::google::protobuf::StringPiece producer_project_id; + + // The API key. + ::google::protobuf::StringPiece api_key; + + // Uses Referer header, if the Referer header doesn't present, use the + // Origin header. If both of them not present, it's empty. + ::google::protobuf::StringPiece referer; + + // The start time of the call. Used to set operation.start_time for both Check + // and Report. + std::chrono::system_clock::time_point request_start_time; + + OperationInfo() {} +}; + +// Information to fill Check request protobuf. +struct CheckRequestInfo : public OperationInfo { + // The client IP address. + std::string client_ip; + // Whether the method allow unregistered calls. + bool allow_unregistered_calls; + + CheckRequestInfo() : allow_unregistered_calls(false) {} +}; + +// Stores the information substracted from the check response. +struct CheckResponseInfo { + // If the request have a valid api key. + bool is_api_key_valid; + // If service is activated. + bool service_is_activated; + + // By default api_key is valid and service is activated. + // They only set to false by the check response from server. + CheckResponseInfo() : is_api_key_valid(true), service_is_activated(true) {} +}; + +// Information to fill Report request protobuf. +struct ReportRequestInfo : public OperationInfo { + // The HTTP response code. + int response_code; + + // The response status. + utils::Status status; + + // Original request URL. + std::string url; + + // location of the service, such as us-central. + ::google::protobuf::StringPiece location; + // API name and version. + ::google::protobuf::StringPiece api_name; + ::google::protobuf::StringPiece api_version; + ::google::protobuf::StringPiece api_method; + + // The request size in bytes. -1 if not available. + int64_t request_size; + + // The response size in bytes. -1 if not available. + int64_t response_size; + + // per request latency. + LatencyInfo latency; + + // The message to log as INFO log. + std::string log_message; + + // Auth info: issuer and audience. + std::string auth_issuer; + std::string auth_audience; + + // Protocol used to issue the request. + protocol::Protocol protocol; + + // HTTP method. all-caps string such as "GET", "POST" etc. + std::string method; + + // A recognized compute platform (GAE, GCE, GKE). + compute_platform::ComputePlatform compute_platform; + + // If consumer data should be sent. + CheckResponseInfo check_response_info; + + // request message size till the current time point. + int64_t request_bytes; + + // response message size till the current time point. + int64_t response_bytes; + + // number of messages for a stream. + int64_t streaming_request_message_counts; + + // number of messages for a stream. + int64_t streaming_response_message_counts; + + // streaming duration (us) between first message and last message. + int64_t streaming_durations; + + // Flag to indicate the first report + bool is_first_report; + + // Flag to indicate the final report + bool is_final_report; + + ReportRequestInfo() + : response_code(200), + status(utils::Status::OK), + request_size(-1), + response_size(-1), + protocol(protocol::UNKNOWN), + compute_platform(compute_platform::UNKNOWN), + request_bytes(0), + response_bytes(0), + streaming_request_message_counts(0), + streaming_response_message_counts(0), + streaming_durations(0), + is_first_report(true), + is_final_report(true) {} +}; + +} // namespace service_control +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_SERVICE_CONTROL_INFO_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/interface.h b/contrib/endpoints/src/api_manager/service_control/interface.h new file mode 100644 index 00000000000..51d6c51e6af --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/interface.h @@ -0,0 +1,81 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_SERVICE_CONTROL_INTERFACE_H_ +#define API_MANAGER_SERVICE_CONTROL_INTERFACE_H_ + +#include "include/api_manager/service_control.h" +#include "include/api_manager/utils/status.h" +#include "src/api_manager/cloud_trace/cloud_trace.h" +#include "src/api_manager/service_control/info.h" + +namespace google { +namespace api_manager { +namespace service_control { + +// An interface to support calling Service Control services. +class Interface { + public: + virtual ~Interface() {} + + // Init() should be called before following calls + // Report + // Check + // GetStatistics + // But Init() should be called after key functions of ApiManagerEnvInterface + // are ready, such as Log, and StartPeriodicTimer. + // Specifically for Nginx, the object can be created at master process + // at config or postconfig. But Init() should be called at init_process + // of each worker. + virtual utils::Status Init() = 0; + + // Flushs out all items in the cache and close the object. + // After Close() is called, following methods should not be used. + // Report + // Check + // GetStatistics + // This method can be called repeatedly. + virtual utils::Status Close() = 0; + + // Sends a ServiceControl Report. + // Report calls are always aggregated and cached. + // Return utils::Status usually is OK unless some caching error. + // If status code is less than 20, as defined by + // google/protobuf/stubs/status.h, + // then the error is from processing response fields (e.g. INVALID_ARGUMENT). + // Reports may be sent to the service control server asynchronously if caching + // is enabled. HTTP request errors carry Nginx error code. + virtual utils::Status Report(const ReportRequestInfo& info) = 0; + + // Sends ServiceControl Check asynchronously. + // on_done() function will be called once it is completed. + // utils::Status in the on_done callback: + // If status.code is more than 100, it is the HTTP response status + // from the service control server. + // If status code is less than 20, within the ranges defined by + // google/protobuf/stubs/status.h, is from parsing error response + // body. + virtual void Check( + const CheckRequestInfo& info, cloud_trace::CloudTraceSpan* parent_span, + std::function on_done) = 0; + + // Get statistics of ServiceControl library. + virtual utils::Status GetStatistics(Statistics* stat) const = 0; +}; + +} // namespace service_control +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_SERVICE_CONTROL_INTERFACE_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader.cc b/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader.cc new file mode 100644 index 00000000000..69cfafd6715 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader.cc @@ -0,0 +1,228 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/logs_metrics_loader.h" + +#include + +#include "src/api_manager/service_control/proto.h" + +namespace google { +namespace api_manager { +namespace service_control { + +using ::google::api::LabelDescriptor; +using ::google::api::LogDescriptor; +using ::google::api::Logging; +using ::google::api::Logging_LoggingDestination; +using ::google::api::MetricDescriptor; +using ::google::api::MonitoredResourceDescriptor; +using ::google::api::Monitoring; +using ::google::api::Monitoring_MonitoringDestination; +using ::google::api::Service; +using ::google::api_manager::utils::Status; +using ::google::protobuf::RepeatedPtrField; +using ::google::protobuf::util::error::Code; + +Status LogsMetricsLoader::Load(const Service& service, + std::set* logs, + std::set* metrics, + std::set* labels) { + LogsMetricsLoader lml(Proto::IsLabelSupported, Proto::IsMetricSupported); + return lml.LoadLogsMetrics(service, logs, metrics, labels); +} + +Status LogsMetricsLoader::AddLabels( + const RepeatedPtrField& descriptors, + std::map* labels) { + for (int i = 0, size = descriptors.size(); i < size; i++) { + const LabelDescriptor& ld = descriptors.Get(i); + + // Check for pre-existing incompatible duplicates + auto existing = labels->find(ld.key()); + if (existing != labels->end()) { + if (existing->second.value_type() != ld.value_type()) { + return Status(Code::INVALID_ARGUMENT, + "Conflicting label in the configuration: " + ld.key()); + } + } + } + + // Only insert the labels into the output set once we validated them. + for (int i = 0, size = descriptors.size(); i < size; i++) { + const LabelDescriptor& ld = descriptors.Get(i); + if (label_supported_(ld)) { + labels->insert(std::map::value_type( + ld.key(), ld)); + } + } + + return Status::OK; +} + +Status LogsMetricsLoader::AddLogLabels( + const ::google::protobuf::RepeatedPtrField& descriptors, + const std::string& log_name, + std::map* labels) { + for (int i = 0, size = descriptors.size(); i < size; i++) { + const LogDescriptor& ld = descriptors.Get(i); + if (ld.name() == log_name) { + // Found the log. + return AddLabels(ld.labels(), labels); + } + } + return Status(Code::INVALID_ARGUMENT, "Log not found: " + log_name); +} + +Status LogsMetricsLoader::AddMonitoredResourceLabels( + const RepeatedPtrField& descriptors, + const std::string& monitored_resource_name, + std::map* labels) { + for (int i = 0, size = descriptors.size(); i < size; i++) { + const MonitoredResourceDescriptor& mr = descriptors.Get(i); + if (mr.type() == monitored_resource_name) { + // Found the monitored resource. + return AddLabels(mr.labels(), labels); + } + } + return Status(Code::INVALID_ARGUMENT, + "Monitored resource not found: " + monitored_resource_name); +} + +Status LogsMetricsLoader::AddLoggingDestinations( + const RepeatedPtrField& destinations, + const RepeatedPtrField& monitored_resources, + const RepeatedPtrField& log_descriptors, + std::set* logs, + std::map* labels) { + Status s = Status::OK; + for (int d = 0, dsize = destinations.size(); d < dsize; d++) { + const Logging_LoggingDestination& ld = destinations.Get(d); + + s = AddMonitoredResourceLabels(monitored_resources, ld.monitored_resource(), + labels); + if (!s.ok()) { + continue; // Skip bad monitored resource. + } + + // Store names of logs ESP should log into. + for (int l = 0, lsize = ld.logs_size(); l < lsize; l++) { + const std::string& log_name = ld.logs(l); + s = AddLogLabels(log_descriptors, log_name, labels); + if (!s.ok()) { + continue; // Skip bad log. + } + logs->insert(log_name); + } + } + return Status::OK; +} + +const MetricDescriptor* FindMetricDescriptor( + const RepeatedPtrField& metric_descriptors, + const std::string& metric_name) { + auto it = std::find_if(metric_descriptors.begin(), metric_descriptors.end(), + [&metric_name](const MetricDescriptor& md) { + return md.name() == metric_name; + }); + return it != metric_descriptors.end() ? &*it : nullptr; +} + +Status LogsMetricsLoader::AddMonitoringDestinations( + const RepeatedPtrField& destinations, + const RepeatedPtrField& monitored_resources, + const RepeatedPtrField& metric_descriptors, + std::map* metrics, + std::map* labels) { + Status s = Status::OK; + + for (int d = 0, dsize = destinations.size(); d < dsize; d++) { + const Monitoring_MonitoringDestination& md = destinations.Get(d); + s = AddMonitoredResourceLabels(monitored_resources, md.monitored_resource(), + labels); + if (!s.ok()) { + continue; // Skip bad monitored resource. + } + + for (int m = 0, msize = md.metrics_size(); m < msize; m++) { + const std::string& metric_name = md.metrics(m); + const MetricDescriptor* metric_descriptor = + FindMetricDescriptor(metric_descriptors, metric_name); + if (metric_descriptor == nullptr || + !metric_supported_(*metric_descriptor)) { + continue; // Skip unrecognized or unsupported metric. + } + + // Add metric specific labels. + s = AddLabels(metric_descriptor->labels(), labels); + if (!s.ok()) { + continue; // Skip bad metric. + } + + // Insert the metric to make sure we report it. + metrics->insert( + std::map::value_type( + metric_name, *metric_descriptor)); + } + } + + return Status::OK; +} + +Status LogsMetricsLoader::LoadLogsMetrics(const Service& service, + std::set* logs, + std::set* metrics, + std::set* labels) { + std::map labels_map; + std::map metrics_map; + + Status s = Status::OK; + + // ESP logs into all producer destination logs. + const Logging& logging = service.logging(); + s = AddLoggingDestinations(logging.producer_destinations(), + service.monitored_resources(), service.logs(), + logs, &labels_map); + if (!s.ok()) return s; + + // ESP reports producer and consumer metrics. + const Monitoring& monitoring = service.monitoring(); + s = AddMonitoringDestinations(monitoring.producer_destinations(), + service.monitored_resources(), + service.metrics(), &metrics_map, &labels_map); + if (!s.ok()) return s; + s = AddMonitoringDestinations(monitoring.consumer_destinations(), + service.monitored_resources(), + service.metrics(), &metrics_map, &labels_map); + if (!s.ok()) return s; + + std::set metrics_set; + for (auto it = metrics_map.begin(), end = metrics_map.end(); it != end; + it++) { + metrics->insert(it->first); + } + + std::set labels_set; + for (auto it = labels_map.begin(), end = labels_map.end(); it != end; it++) { + labels->insert(it->first); + } + + return Status::OK; +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader.h b/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader.h new file mode 100644 index 00000000000..d25b67b7201 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader.h @@ -0,0 +1,105 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_SERVICE_CONTROL_LOGS_METRICS_LOADER_H_ +#define API_MANAGER_SERVICE_CONTROL_LOGS_METRICS_LOADER_H_ + +#include + +#include "google/api/service.pb.h" +#include "gtest/gtest_prod.h" +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { +namespace service_control { + +class LogsMetricsLoader final { + public: + static utils::Status Load(const ::google::api::Service& service, + std::set* logs, + std::set* metrics, + std::set* labels); + + private: + LogsMetricsLoader(std::function + label_supported, + std::function + metric_supported) + : label_supported_(label_supported), + metric_supported_(metric_supported) {} + + utils::Status AddLabels( + const ::google::protobuf::RepeatedPtrField< + ::google::api::LabelDescriptor>& descriptors, + std::map* labels); + + utils::Status AddLogLabels( + const ::google::protobuf::RepeatedPtrField< ::google::api::LogDescriptor>& + descriptors, + const std::string& log_name, + std::map* labels); + + utils::Status AddMonitoredResourceLabels( + const ::google::protobuf::RepeatedPtrField< + ::google::api::MonitoredResourceDescriptor>& descriptors, + const std::string& monitored_resource_name, + std::map* labels); + + utils::Status AddLoggingDestinations( + const ::google::protobuf::RepeatedPtrField< + ::google::api::Logging_LoggingDestination>& destinations, + const ::google::protobuf::RepeatedPtrField< + ::google::api::MonitoredResourceDescriptor>& monitored_resources, + const ::google::protobuf::RepeatedPtrField< ::google::api::LogDescriptor>& + log_descriptors, + std::set* logs, + std::map* labels); + + utils::Status AddMonitoringDestinations( + const ::google::protobuf::RepeatedPtrField< + ::google::api::Monitoring_MonitoringDestination>& destinations, + const ::google::protobuf::RepeatedPtrField< + ::google::api::MonitoredResourceDescriptor>& monitored_resources, + const ::google::protobuf::RepeatedPtrField< + ::google::api::MetricDescriptor>& metric_descriptors, + std::map* metrics, + std::map* labels); + + utils::Status LoadLogsMetrics(const ::google::api::Service& service, + std::set* logs, + std::set* metrics, + std::set* labels); + + std::function label_supported_; + std::function metric_supported_; + + private: + FRIEND_TEST(LogsMetricsLoader, AddDuplicateLabels); + FRIEND_TEST(LogsMetricsLoader, AddConflictingLabels); + FRIEND_TEST(LogsMetricsLoader, AddUnsupportedLabels); + FRIEND_TEST(LogsMetricsLoader, AddRedundantLabels); + FRIEND_TEST(LogsMetricsLoader, AddNoLabels); + FRIEND_TEST(LogsMetricsLoader, AddLogLabels); + FRIEND_TEST(LogsMetricsLoader, AddMonitoredResourceLabels); + FRIEND_TEST(LogsMetricsLoader, AddLoggingDestinations); + FRIEND_TEST(LogsMetricsLoader, AddMonitoringDestinations); + FRIEND_TEST(LogsMetricsLoader, LoadLogsMetrics); +}; + +} // namespace service_control +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_SERVICE_CONTROL_LOGS_METRICS_LOADER_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader_test.cc b/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader_test.cc new file mode 100644 index 00000000000..bfaf63ef065 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/logs_metrics_loader_test.cc @@ -0,0 +1,558 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/logs_metrics_loader.h" + +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/text_format.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +using ::google::api::LabelDescriptor; +using ::google::api::LogDescriptor; +using ::google::api::MetricDescriptor; +using ::google::api::Service; +using ::google::api_manager::utils::Status; +using ::google::protobuf::RepeatedPtrField; + +using ::testing::_; +using ::testing::Eq; +using ::testing::Pair; +using ::testing::Property; +using ::testing::UnorderedElementsAre; + +template +Message Parse(const char *text) { + ::google::protobuf::TextFormat::Parser parser; + ::google::protobuf::io::ArrayInputStream input(text, strlen(text)); + Message msg; + bool success = parser.Parse(&input, &msg); + EXPECT_TRUE(success); + return msg; +} + +const char unsupported_prefix[] = "unsupported/"; + +bool StartsWith(const std::string &value, const char *prefix, size_t size) { + return value.compare(0, size, prefix) == 0; +} + +bool IsLabelSupported(const LabelDescriptor &ld) { + return !StartsWith(ld.key(), unsupported_prefix, + sizeof(unsupported_prefix) - 1); +} + +bool IsMetricSupported(const MetricDescriptor &md) { + return !StartsWith(md.name(), unsupported_prefix, + sizeof(unsupported_prefix) - 1); +} + +typedef std::map LabelMap; +typedef std::map MetricMap; + +} // namespace + +TEST(LogsMetricsLoaderTest, StartsWith) { + ASSERT_TRUE(StartsWith("abc", "ab", 2)); + ASSERT_FALSE( + StartsWith("abc", unsupported_prefix, sizeof(unsupported_prefix) - 1)); +} + +TEST(LogsMetricsLoaderTest, IsSupported) { + LabelDescriptor ld; + ld.set_key("a"); + ASSERT_TRUE(IsLabelSupported(ld)); + ld.set_key("unsupported"); + ASSERT_TRUE(IsLabelSupported(ld)); + ld.set_key("supported/foo"); + ASSERT_TRUE(IsLabelSupported(ld)); + ld.set_key("unsupported/"); + ASSERT_FALSE(IsLabelSupported(ld)); + ld.set_key("unsupported/foo"); + ASSERT_FALSE(IsLabelSupported(ld)); + + MetricDescriptor md; + md.set_name("a"); + ASSERT_TRUE(IsMetricSupported(md)); + md.set_name("unsupported"); + ASSERT_TRUE(IsMetricSupported(md)); + md.set_name("supported/foo"); + ASSERT_TRUE(IsMetricSupported(md)); + md.set_name("unsupported/"); + ASSERT_FALSE(IsMetricSupported(md)); + md.set_name("unsupported/foo"); + ASSERT_FALSE(IsMetricSupported(md)); +} + +// Configuration contains duplicate labels. We de-dupe them. +TEST(LogsMetricsLoader, AddDuplicateLabels) { + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + + { + RepeatedPtrField lds; + lds.Add()->set_key("duplicate"); + lds.Add()->set_key("duplicate"); + + LabelMap labels; + ASSERT_TRUE(lml.AddLabels(lds, &labels).ok()); + ASSERT_THAT(labels, UnorderedElementsAre(Pair("duplicate", _))); + } + { + // Unsupported duplicates are not added. + RepeatedPtrField lds; + lds.Add()->set_key("unsupported/duplicate"); + lds.Add()->set_key("unsupported/duplicate"); + + LabelMap empty; + ASSERT_TRUE(lml.AddLabels(lds, &empty).ok()); + ASSERT_TRUE(empty.empty()); + } +} + +// Set of labels so far collected contains label which is in conflict with label +// newly added. +TEST(LogsMetricsLoader, AddConflictingLabels) { + const char log_descriptor[] = // We parse LogDescriptor but all we need are + // its labels + "labels {" + " key: \"api-key\"" // lexicographically before 'conflicting' + "}" + "labels {" + " key: \"conflicting\"" + " value_type: BOOL" + "}" + "labels {" + " key: \"request-size\"" + " value_type: INT64" + "}"; + + LogDescriptor ld = Parse(log_descriptor); + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + + LabelDescriptor ld_string; + ld_string.set_key("conflicting"); + ld_string.set_value_type(LabelDescriptor::STRING); + LabelMap conflict({{"conflicting", ld_string}}); + ASSERT_FALSE(lml.AddLabels(ld.labels(), &conflict).ok()); + + // Assert that the label map was not modified. + ASSERT_THAT(conflict, + UnorderedElementsAre( + Pair("conflicting", Property(&LabelDescriptor::value_type, + Eq(LabelDescriptor::STRING))))); +} + +TEST(LogsMetricsLoader, AddUnsupportedLabels) { + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + RepeatedPtrField lds; + lds.Add()->set_key("unsupported/foo"); + lds.Add()->set_key("supported/baz"); + lds.Add()->set_key("unsupported/bar"); + + LabelMap labels; + ASSERT_TRUE(lml.AddLabels(lds, &labels).ok()); + ASSERT_THAT(labels, UnorderedElementsAre(Pair("supported/baz", _))); +} + +TEST(LogsMetricsLoader, AddRedundantLabels) { + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + const std::string name("redundant"); + RepeatedPtrField lds; + lds.Add()->set_key(name); + + LabelDescriptor redundant; + redundant.set_key(name); + LabelMap labels({{name, redundant}}); + + ASSERT_TRUE(lml.AddLabels(lds, &labels).ok()); + ASSERT_THAT(labels, UnorderedElementsAre(Pair(name, _))); +} + +TEST(LogsMetricsLoader, AddNoLabels) { + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + RepeatedPtrField lds; + + // Adding no labels must work. + LabelMap labels; + ASSERT_TRUE(lml.AddLabels(lds, &labels).ok()); + EXPECT_TRUE(labels.empty()); +} + +TEST(LogsMetricsLoader, AddLogLabels) { + const char service_config[] = // Only contains logs, that's all we need. + "logs {" + " name: \"endpoints\"" + " labels {" + " key: \"unsupported/endpoints\"" + " }" + " labels {" + " key: \"supported/endpoints\"" + " }" + "}" + "logs {" + " name: \"startpoints\"" + " labels {" + " key: \"supported/startpoints\"" + " }" + " labels {" + " key: \"unsupported/startpoints\"" + " }" + "}"; + + Service service = Parse(service_config); + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + + // Test that only supported labels from the named log are added. + { + LabelMap labels; + ASSERT_TRUE(lml.AddLogLabels(service.logs(), "endpoints", &labels).ok()); + ASSERT_THAT(labels, UnorderedElementsAre(Pair("supported/endpoints", _))); + } + { + LabelMap labels; + ASSERT_TRUE(lml.AddLogLabels(service.logs(), "startpoints", &labels).ok()); + ASSERT_THAT(labels, UnorderedElementsAre(Pair("supported/startpoints", _))); + } + + // Referencing log not present in the config --> error. + { + LabelMap labels; + ASSERT_FALSE(lml.AddLogLabels(service.logs(), "notfound", &labels).ok()); + ASSERT_TRUE(labels.empty()); + } +} + +TEST(LogsMetricsLoader, AddMonitoredResourceLabels) { + const char service_config[] = // Only contains monitored resources, that's + // all we need. + "monitored_resources {" + " type: \"endpoints.googleapis.com/endpoints\"" + " labels {" + " key: \"unsupported/endpoints\"" + " }" + " labels {" + " key: \"supported/endpoints\"" + " }" + "}" + "monitored_resources {" + " type: \"startpoints.googleapis.com/startpoints\"" + " labels {" + " key: \"supported/startpoints\"" + " }" + " labels {" + " key: \"unsupported/startpoints\"" + " }" + "}"; + + Service service = Parse(service_config); + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + + // Test that only supported labels from the specific monitored resource are + // added. + { + LabelMap labels; + ASSERT_TRUE(lml.AddMonitoredResourceLabels( + service.monitored_resources(), + "endpoints.googleapis.com/endpoints", &labels) + .ok()); + ASSERT_THAT(labels, UnorderedElementsAre(Pair("supported/endpoints", _))); + } + { + LabelMap labels; + ASSERT_TRUE(lml.AddMonitoredResourceLabels( + service.monitored_resources(), + "startpoints.googleapis.com/startpoints", &labels) + .ok()); + ASSERT_THAT(labels, UnorderedElementsAre(Pair("supported/startpoints", _))); + } + + // Referencing non-existent monitored resource --> error. + { + LabelMap labels; + ASSERT_FALSE(lml.AddMonitoredResourceLabels( + service.monitored_resources(), + "endpoints.googleapis.com/notfound", &labels) + .ok()); + ASSERT_TRUE(labels.empty()); + } +} + +TEST(LogsMetricsLoader, AddLoggingDestinations) { + const char service_config[] = + // A valid log with one supported label. + "logs {" + " name: \"endpoints-log\"" + " labels {" + " key: \"supported/endpoints-log-label\"" + " }" + " labels {" + " key: \"unsupported/endpoints-log-label\"" + " }" + "}" + + // A log otherwise unreferenced in the config. + "logs {" + " name: \"unreferenced-log\"" + " labels: {" + " key: \"supported/unreferenced-log-label\"" + " }" + "}" + + // Only one valid monitored resource. + "monitored_resources {" + " type: \"endpoints.googleapis.com/endpoints\"" + " labels {" + " key: \"unsupported/endpoints\"" + " }" + " labels {" + " key: \"supported/endpoints\"" + " }" + "}" + + // Logging section + "logging {" + // Invalid logging destination (bad resource, log). + " producer_destinations {" + " monitored_resource: \"bad-monitored-resource\"" + " logs: \"bad-monitored-resource-log\"" + " }" + + // Partly valid logging destination (good resource, some good logs). + " producer_destinations {" + " monitored_resource: \"endpoints.googleapis.com/endpoints\"" + " logs: \"bad-endpoints-log\"" + " logs: \"endpoints-log\"" + " }" + "}"; + + Service service = Parse(service_config); + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + + std::set logs; + LabelMap labels; + + Status s = lml.AddLoggingDestinations( + service.logging().producer_destinations(), service.monitored_resources(), + service.logs(), &logs, &labels); + ASSERT_TRUE(s.ok()); + + // Only one log was referenced, and valid. + ASSERT_THAT(logs, UnorderedElementsAre("endpoints-log")); + + // Only the supported labels from the monitored resource and the log + // are going to be reported. + ASSERT_THAT(labels, + UnorderedElementsAre(Pair("supported/endpoints", _), + Pair("supported/endpoints-log-label", _))); +} + +TEST(LogsMetricsLoader, AddMonitoringDestinations) { + const char service_config[] = + // A valid metric with one supported label. + "metrics {" + " name: \"supported/endpoints-metric\"" + " labels {" + " key: \"supported/endpoints-metric-label\"" + " }" + " labels {" + " key: \"unsupported/endpoints-metric-label\"" + " }" + "}" + + // An unsupported metric with a supported label. The supported label won't + // be reported because the metric isn't. + "metrics {" + " name: \"unsupported/unsupported-endpoints-metric\"" + " labels {" + " key: \"supported/unreferenced-metric-label\"" + " }" + "}" + + // Supported metric used by non-existent monitored resource. Thus, it + // won't be reported. + "metrics {" + " name: \"supported/non-existent-resource-metric\"" + " labels {" + " key: \"supported/non-existent-resource-metric-label\"" + " }" + "}" + + // Only one valid monitored resource. + "monitored_resources {" + " type: \"endpoints.googleapis.com/endpoints\"" + " labels {" + " key: \"unsupported/endpoints\"" + " }" + " labels {" + " key: \"supported/endpoints\"" + " }" + "}" + + // Monitoring section. Use consumer destinations only for the purpose of + // the test. + "monitoring {" + " consumer_destinations {" + " monitored_resource: \"endpoints.googleapis.com/endpoints\"" + " metrics: \"supported/endpoints-metric\"" + " metrics: \"unsupported/unsupported-endpoints-metric\"" + " metrics: \"supported/unknown-metric\"" + " }" + + // One valid metric but on unrecognized monitored resource. This should be + // skipped. + " consumer_destinations {" + " monitored_resource: \"endpoints.googleapis.com/non-existent\"" + " metrics: \"supported/endpoints-metric\"" + " metrics: \"unsupported/unsupported-endpoints-metric\"" + " metrics: \"supported/unknown-metric\"" + " metrics: \"supported/non-existent-resource-metric\"" + " }" + "}"; + + Service service = Parse(service_config); + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + + MetricMap metrics; + LabelMap labels; + Status s = lml.AddMonitoringDestinations( + service.monitoring().consumer_destinations(), + service.monitored_resources(), service.metrics(), &metrics, &labels); + ASSERT_TRUE(s.ok()); + + // Only the supported metrics from valid monitored resources have been added. + ASSERT_THAT(metrics, + UnorderedElementsAre(Pair("supported/endpoints-metric", _))); + + // Only the supported labels from the monitored resource and from the metric + // are recognized. + ASSERT_THAT(labels, UnorderedElementsAre( + Pair("supported/endpoints", _), + Pair("supported/endpoints-metric-label", _))); +} + +TEST(LogsMetricsLoader, LoadLogsMetrics) { + const char service_config[] = + // A valid log with one supported label. + "logs {" + " name: \"endpoints-log\"" + " labels {" + " key: \"supported/endpoints-log-label\"" + " }" + " labels {" + " key: \"unsupported/endpoints-log-label\"" + " }" + "}" + + // Shared metric. + "metrics {" + " name: \"supported/endpoints-metric\"" + " labels {" + " key: \"supported/endpoints-metric-label\"" + " }" + " labels {" + " key: \"unsupported/endpoints-metric-label\"" + " }" + "}" + // Consumer metric. + "metrics {" + " name: \"supported/endpoints-consumer-metric\"" + " labels {" + " key: \"supported/endpoints-metric-label\"" + " }" + " labels {" + " key: \"supported/endpoints-consumer-metric-label\"" + " }" + "}" + // Producer metric. + "metrics {" + " name: \"supported/endpoints-producer-metric\"" + " labels {" + " key: \"supported/endpoints-metric-label\"" + " }" + " labels {" + " key: \"supported/endpoints-producer-metric-label\"" + " }" + "}" + + // Only one valid monitored resource. + "monitored_resources {" + " type: \"endpoints.googleapis.com/endpoints\"" + " labels {" + " key: \"unsupported/endpoints\"" + " }" + " labels {" + " key: \"supported/endpoints\"" + " }" + "}" + + // Logging section + "logging {" + " producer_destinations {" + " monitored_resource: \"endpoints.googleapis.com/endpoints\"" + " logs: \"endpoints-log\"" + " }" + "}" + + // Monitoring section. + "monitoring {" + " consumer_destinations {" + " monitored_resource: \"endpoints.googleapis.com/endpoints\"" + " metrics: \"supported/endpoints-consumer-metric\"" + " metrics: \"supported/endpoints-metric\"" + " }" + + // One valid metric but on unrecognized monitored resource. This should be + // skipped. + " producer_destinations {" + " monitored_resource: \"endpoints.googleapis.com/endpoints\"" + " metrics: \"supported/endpoints-producer-metric\"" + " metrics: \"supported/endpoints-metric\"" + " }" + "}"; + + Service service = Parse(service_config); + LogsMetricsLoader lml(IsLabelSupported, IsMetricSupported); + + std::set logs, metrics, labels; + Status s = lml.LoadLogsMetrics(service, &logs, &metrics, &labels); + ASSERT_TRUE(s.ok()); + + ASSERT_THAT(logs, UnorderedElementsAre("endpoints-log")); + ASSERT_THAT(metrics, + UnorderedElementsAre("supported/endpoints-metric", + "supported/endpoints-consumer-metric", + "supported/endpoints-producer-metric")); + ASSERT_THAT( + labels, + UnorderedElementsAre( + "supported/endpoints", // from monitored resource + "supported/endpoints-log-label", // from log + "supported/endpoints-metric-label", // from both metrics + "supported/endpoints-consumer-metric-label", // from consumer metric + "supported/endpoints-producer-metric-label" // from producer metric + )); +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc new file mode 100644 index 00000000000..1c6fceb4458 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -0,0 +1,1096 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/proto.h" + +#include + +#include +#include + +#include "google/api/metric.pb.h" +#include "google/protobuf/timestamp.pb.h" +#include "include/api_manager/service_control.h" +#include "include/api_manager/version.h" +#include "src/api_manager/auth/lib/auth_token.h" +#include "src/api_manager/auth/lib/base64.h" +#include "utils/distribution_helper.h" + +using ::google::api::servicecontrol::v1::CheckError; +using ::google::api::servicecontrol::v1::CheckRequest; +using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::Distribution; +using ::google::api::servicecontrol::v1::LogEntry; +using ::google::api::servicecontrol::v1::MetricValue; +using ::google::api::servicecontrol::v1::MetricValueSet; +using ::google::api::servicecontrol::v1::Operation; +using ::google::api::servicecontrol::v1::ReportRequest; +using ::google::api_manager::utils::Status; +using ::google::protobuf::Map; +using ::google::protobuf::StringPiece; +using ::google::protobuf::Timestamp; +using ::google::protobuf::util::error::Code; +using ::google::service_control_client::DistributionHelper; + +namespace google { +namespace api_manager { +namespace service_control { + +struct SupportedMetric { + const char* name; + ::google::api::MetricDescriptor_MetricKind metric_kind; + ::google::api::MetricDescriptor_ValueType value_type; + + enum Mark { PRODUCER = 0, CONSUMER = 1 }; + enum Tag { START = 0, INTERMEDIATE = 1, FINAL = 2 }; + Tag tag; + Mark mark; + Status (*set)(const SupportedMetric& m, const ReportRequestInfo& info, + Operation* operation); +}; + +struct SupportedLabel { + const char* name; + ::google::api::LabelDescriptor_ValueType value_type; + + enum Kind { USER = 0, SYSTEM = 1 }; + Kind kind; + + Status (*set)(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels); +}; + +namespace { + +// Metric Helpers + +MetricValue* AddMetricValue(const char* metric_name, Operation* operation) { + MetricValueSet* metric_value_set = operation->add_metric_value_sets(); + metric_value_set->set_metric_name(metric_name); + return metric_value_set->add_metric_values(); +} + +void AddInt64Metric(const char* metric_name, int64_t value, + Operation* operation) { + MetricValue* metric_value = AddMetricValue(metric_name, operation); + metric_value->set_int64_value(value); +} + +// The parameters to initialize DistributionHelper +struct DistributionHelperOptions { + int buckets; + double growth; + double scale; +}; + +const DistributionHelperOptions time_distribution = {8, 10.0, 1e-6}; +const DistributionHelperOptions size_distribution = {8, 10.0, 1}; +const double kMsToSecs = 1e-3; + +Status AddDistributionMetric(const DistributionHelperOptions& options, + const char* metric_name, double value, + Operation* operation) { + MetricValue* metric_value = AddMetricValue(metric_name, operation); + Distribution distribution; + ::google::protobuf::util::Status proto_status = + DistributionHelper::InitExponential(options.buckets, options.growth, + options.scale, &distribution); + if (!proto_status.ok()) return Status::FromProto(proto_status); + proto_status = DistributionHelper::AddSample(value, &distribution); + if (!proto_status.ok()) return Status::FromProto(proto_status); + *metric_value->mutable_distribution_value() = distribution; + return Status::OK; +} + +// Metrics supported by ESP. + +Status set_int64_metric_to_constant_1(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + AddInt64Metric(m.name, 1l, operation); + return Status::OK; +} + +Status set_int64_metric_to_constant_1_if_http_error( + const SupportedMetric& m, const ReportRequestInfo& info, + Operation* operation) { + // Use status code >= 400 to determine request failed. + if (info.response_code >= 400) { + AddInt64Metric(m.name, 1l, operation); + } + return Status::OK; +} + +Status set_distribution_metric_to_request_size(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + if (info.request_size >= 0) { + return AddDistributionMetric(size_distribution, m.name, info.request_size, + operation); + } + return Status::OK; +} + +Status set_distribution_metric_to_response_size(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + if (info.response_size >= 0) { + return AddDistributionMetric(size_distribution, m.name, info.response_size, + operation); + } + return Status::OK; +} + +// TODO: Consider refactoring following 3 functions to avoid duplicate code +Status set_distribution_metric_to_request_time(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + if (info.latency.request_time_ms >= 0) { + double request_time_secs = info.latency.request_time_ms * kMsToSecs; + return AddDistributionMetric(time_distribution, m.name, request_time_secs, + operation); + } + return Status::OK; +} + +Status set_distribution_metric_to_backend_time(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + if (info.latency.backend_time_ms >= 0) { + double backend_time_secs = info.latency.backend_time_ms * kMsToSecs; + return AddDistributionMetric(time_distribution, m.name, backend_time_secs, + operation); + } + return Status::OK; +} + +Status set_distribution_metric_to_overhead_time(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + if (info.latency.overhead_time_ms >= 0) { + double overhead_time_secs = info.latency.overhead_time_ms * kMsToSecs; + return AddDistributionMetric(time_distribution, m.name, overhead_time_secs, + operation); + } + return Status::OK; +} + +Status set_int64_metric_to_request_bytes(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + if (info.request_bytes > 0) { + AddInt64Metric(m.name, info.request_bytes, operation); + } + return Status::OK; +} + +Status set_int64_metric_to_response_bytes(const SupportedMetric& m, + const ReportRequestInfo& info, + Operation* operation) { + if (info.response_bytes > 0) { + AddInt64Metric(m.name, info.response_bytes, operation); + } + return Status::OK; +} + +Status set_distribution_metric_to_streaming_request_message_counts( + const SupportedMetric& m, const ReportRequestInfo& info, + Operation* operation) { + if (info.streaming_request_message_counts > 0) { + AddDistributionMetric(size_distribution, m.name, + info.streaming_request_message_counts, operation); + } + return Status::OK; +} +Status set_distribution_metric_to_streaming_response_message_counts( + const SupportedMetric& m, const ReportRequestInfo& info, + Operation* operation) { + if (info.streaming_response_message_counts > 0) { + AddDistributionMetric(size_distribution, m.name, + info.streaming_response_message_counts, operation); + } + return Status::OK; +} + +Status set_distribution_metric_to_streaming_durations( + const SupportedMetric& m, const ReportRequestInfo& info, + Operation* operation) { + if (info.streaming_durations > 0) { + AddDistributionMetric(time_distribution, m.name, info.streaming_durations, + operation); + } + return Status::OK; +} +// Currently unsupported metrics: +// +const SupportedMetric supported_metrics[] = { + { + "serviceruntime.googleapis.com/api/consumer/request_count", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, SupportedMetric::START, + SupportedMetric::CONSUMER, set_int64_metric_to_constant_1, + }, + { + "serviceruntime.googleapis.com/api/producer/request_count", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, SupportedMetric::START, + SupportedMetric::PRODUCER, set_int64_metric_to_constant_1, + }, + { + "serviceruntime.googleapis.com/api/consumer/request_sizes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_request_size, + }, + { + "serviceruntime.googleapis.com/api/producer/request_sizes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_request_size, + }, + { + "serviceruntime.googleapis.com/api/consumer/response_sizes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_response_size, + }, + { + "serviceruntime.googleapis.com/api/producer/response_sizes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_response_size, + }, + { + "serviceruntime.googleapis.com/api/consumer/request_bytes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, + SupportedMetric::INTERMEDIATE, SupportedMetric::CONSUMER, + set_int64_metric_to_request_bytes, + }, + { + "serviceruntime.googleapis.com/api/consumer/response_bytes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, + SupportedMetric::INTERMEDIATE, SupportedMetric::CONSUMER, + set_int64_metric_to_response_bytes, + }, + { + "serviceruntime.googleapis.com/api/producer/request_bytes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, + SupportedMetric::INTERMEDIATE, SupportedMetric::PRODUCER, + set_int64_metric_to_request_bytes, + }, + { + "serviceruntime.googleapis.com/api/producer/response_bytes", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, + SupportedMetric::INTERMEDIATE, SupportedMetric::PRODUCER, + set_int64_metric_to_response_bytes, + }, + { + "serviceruntime.googleapis.com/api/consumer/error_count", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, SupportedMetric::FINAL, + SupportedMetric::CONSUMER, set_int64_metric_to_constant_1_if_http_error, + }, + { + "serviceruntime.googleapis.com/api/producer/error_count", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_INT64, SupportedMetric::FINAL, + SupportedMetric::PRODUCER, set_int64_metric_to_constant_1_if_http_error, + }, + { + "serviceruntime.googleapis.com/api/consumer/total_latencies", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_request_time, + }, + { + "serviceruntime.googleapis.com/api/producer/total_latencies", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_request_time, + }, + { + "serviceruntime.googleapis.com/api/consumer/backend_latencies", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_backend_time, + }, + { + "serviceruntime.googleapis.com/api/producer/backend_latencies", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_backend_time, + }, + { + "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_overhead_time, + }, + { + "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_overhead_time, + }, + { + "serviceruntime.googleapis.com/api/consumer/" + "streaming_request_message_counts", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_streaming_request_message_counts, + }, + { + "serviceruntime.googleapis.com/api/producer/" + "streaming_request_message_counts", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_streaming_request_message_counts, + }, + { + "serviceruntime.googleapis.com/api/consumer/" + "streaming_response_message_counts", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_streaming_response_message_counts, + }, + { + "serviceruntime.googleapis.com/api/producer/" + "streaming_response_message_counts", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_streaming_response_message_counts, + }, + { + "serviceruntime.googleapis.com/api/consumer/" + "streaming_durations", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::CONSUMER, + set_distribution_metric_to_streaming_durations, + }, + { + "serviceruntime.googleapis.com/api/producer/" + "streaming_durations", + ::google::api::MetricDescriptor_MetricKind_DELTA, + ::google::api::MetricDescriptor_ValueType_DISTRIBUTION, + SupportedMetric::FINAL, SupportedMetric::PRODUCER, + set_distribution_metric_to_streaming_durations, + }, + +}; +const int supported_metrics_count = + sizeof(supported_metrics) / sizeof(supported_metrics[0]); + +const char kServiceControlCallerIp[] = + "servicecontrol.googleapis.com/caller_ip"; +const char kServiceControlReferer[] = "servicecontrol.googleapis.com/referer"; +const char kServiceControlServiceAgent[] = + "servicecontrol.googleapis.com/service_agent"; +const char kServiceControlUserAgent[] = + "servicecontrol.googleapis.com/user_agent"; +const char kServiceControlPlatform[] = "servicecontrol.googleapis.com/platform"; + +// User agent label value +// The value for kUserAgent should be configured at service control server. +// Now it is configured as "ESP". +const char kUserAgent[] = "ESP"; + +// Service agent label value +const char kServiceAgent[] = "ESP/" API_MANAGER_VERSION_STRING; + +// /credential_id +Status set_credential_id(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + // The rule to set /credential_id is: + // 1) If api_key is available, set it as apiKey:API-KEY + // 2) If auth issuer and audience both are available, set it as: + // jwtAuth:issuer=base64(issuer)&audience=base64(audience) + if (!info.api_key.empty()) { + std::string credential_id("apikey:"); + credential_id += info.api_key.ToString(); + (*labels)[l.name] = credential_id; + } else if (!info.auth_issuer.empty()) { + // If auth is used, auth_issuer should NOT be empty since it is required. + char* base64_issuer = auth::esp_base64_encode( + info.auth_issuer.data(), info.auth_issuer.size(), true /* url_safe */, + false /* multiline */, false /* padding */); + if (base64_issuer == nullptr) { + return Status(Code::INTERNAL, "Out of memory"); + } + std::string credential_id("jwtauth:issuer="); + credential_id += base64_issuer; + auth::esp_grpc_free(base64_issuer); + + // auth audience is optional. + if (!info.auth_audience.empty()) { + char* base64_audience = auth::esp_base64_encode( + info.auth_audience.data(), info.auth_audience.size(), + true /* url_safe */, false /* multiline */, false /* padding */); + if (base64_audience == nullptr) { + return Status(Code::INTERNAL, "Out of memory"); + } + + credential_id += "&audience="; + credential_id += base64_audience; + auth::esp_grpc_free(base64_audience); + } + (*labels)[l.name] = credential_id; + } + return Status::OK; +} + +const char* error_types[10] = {"0xx", "1xx", "2xx", "3xx", "4xx", + "5xx", "6xx", "7xx", "8xx", "9xx"}; + +// /error_type +Status set_error_type(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + if (info.response_code >= 400) { + int code = (info.response_code / 100) % 10; + if (error_types[code]) { + (*labels)[l.name] = error_types[code]; + } + } + return Status::OK; +} + +// /protocol +Status set_protocol(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + (*labels)[l.name] = protocol::ToString(info.protocol); + return Status::OK; +} + +// /referer +Status set_referer(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + if (!info.referer.empty()) { + (*labels)[l.name] = info.referer; + } + return Status::OK; +} + +// /response_code +Status set_response_code(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + if (!info.is_final_report) return Status::OK; + char response_code_buf[20]; + snprintf(response_code_buf, sizeof(response_code_buf), "%d", + info.response_code); + (*labels)[l.name] = response_code_buf; + return Status::OK; +} + +// /response_code_class +Status set_response_code_class(const SupportedLabel& l, + const ReportRequestInfo& info, + Map* labels) { + if (!info.is_final_report) return Status::OK; + (*labels)[l.name] = error_types[(info.response_code / 100) % 10]; + return Status::OK; +} + +// /status_code +Status set_status_code(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + if (!info.is_final_report) return Status::OK; + char status_code_buf[20]; + snprintf(status_code_buf, sizeof(status_code_buf), "%d", + info.status.CanonicalCode()); + (*labels)[l.name] = status_code_buf; + return Status::OK; +} + +// cloud.googleapis.com/location +Status set_location(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + if (!info.location.empty()) { + (*labels)[l.name] = info.location; + } + return Status::OK; +} + +// serviceruntime.googleapis.com/api_method +Status set_api_method(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + if (!info.api_method.empty()) { + (*labels)[l.name] = info.api_method; + } + return Status::OK; +} + +// serviceruntime.googleapis.com/api_version +Status set_api_version(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + if (!info.api_version.empty()) { + (*labels)[l.name] = info.api_version; + } + return Status::OK; +} + +// servicecontrol.googleapis.com/platform +Status set_platform(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + (*labels)[l.name] = compute_platform::ToString(info.compute_platform); + return Status::OK; +} + +// servicecontrol.googleapis.com/service_agent +Status set_service_agent(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + (*labels)[l.name] = kServiceAgent; + return Status::OK; +} + +// serviceruntime.googleapis.com/user_agent +Status set_user_agent(const SupportedLabel& l, const ReportRequestInfo& info, + Map* labels) { + (*labels)[l.name] = kUserAgent; + return Status::OK; +} + +const SupportedLabel supported_labels[] = { + { + "/credential_id", ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::USER, set_credential_id, + }, + { + "/end_user", ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::USER, nullptr, + }, + { + "/end_user_country", ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::USER, nullptr, + }, + { + "/error_type", ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::USER, set_error_type, + }, + { + "/protocol", ::google::api::LabelDescriptor::STRING, + SupportedLabel::USER, set_protocol, + }, + { + "/referer", ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::USER, set_referer, + }, + { + "/response_code", ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::USER, set_response_code, + }, + { + "/response_code_class", ::google::api::LabelDescriptor::STRING, + SupportedLabel::USER, set_response_code_class, + }, + { + "/status_code", ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::USER, set_status_code, + }, + { + "appengine.googleapis.com/clone_id", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + nullptr, + }, + { + "appengine.googleapis.com/module_id", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + nullptr, + }, + { + "appengine.googleapis.com/replica_index", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + nullptr, + }, + { + "appengine.googleapis.com/version_id", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + nullptr, + }, + { + "cloud.googleapis.com/location", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + set_location, + }, + { + "cloud.googleapis.com/project", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + nullptr, + }, + { + "cloud.googleapis.com/region", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + nullptr, + }, + { + "cloud.googleapis.com/resource_id", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + nullptr, + }, + { + "cloud.googleapis.com/resource_type", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + nullptr, + }, + { + "cloud.googleapis.com/service", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + nullptr, + }, + { + "cloud.googleapis.com/zone", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + nullptr, + }, + { + "cloud.googleapis.com/uid", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + nullptr, + }, + { + "serviceruntime.googleapis.com/api_method", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + set_api_method, + }, + { + "serviceruntime.googleapis.com/api_version", + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::USER, + set_api_version, + }, + { + kServiceControlCallerIp, + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + nullptr, + }, + { + kServiceControlReferer, ::google::api::LabelDescriptor_ValueType_STRING, + SupportedLabel::SYSTEM, nullptr, + }, + { + kServiceControlServiceAgent, + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + set_service_agent, + }, + { + kServiceControlUserAgent, + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + set_user_agent, + }, + { + kServiceControlPlatform, + ::google::api::LabelDescriptor_ValueType_STRING, SupportedLabel::SYSTEM, + set_platform, + }, +}; + +const int supported_labels_count = + sizeof(supported_labels) / sizeof(supported_labels[0]); + +// Supported intrinsic labels: +// "servicecontrol.googleapis.com/operation_name": Operation.operation_name +// "servicecontrol.googleapis.com/consumer_id": Operation.consumer_id + +// Unsupported service control labels: +// "servicecontrol.googleapis.com/android_package_name" +// "servicecontrol.googleapis.com/android_cert_fingerprint" +// "servicecontrol.googleapis.com/ios_bundle_id" +// "servicecontrol.googleapis.com/credential_project_number" + +// Define Service Control constant strings +const char kConsumerIdApiKey[] = "api_key:"; + +// Following names for for Log struct_playload field names: +const char kLogFieldNameTimestamp[] = "timestamp"; +const char kLogFieldNameApiName[] = "api_name"; +const char kLogFieldNameApiVersion[] = "api_version"; +const char kLogFieldNameApiMethod[] = "api_method"; +const char kLogFieldNameApiKey[] = "api_key"; +const char kLogFieldNameProducerProjectId[] = "producer_project_id"; +const char kLogFieldNameReferer[] = "referer"; +const char kLogFieldNameLocation[] = "location"; +const char kLogFieldNameRequestSize[] = "request_size"; +const char kLogFieldNameResponseSize[] = "response_size"; +const char kLogFieldNameHttpMethod[] = "http_method"; +const char kLogFieldNameHttpResponseCode[] = "http_response_code"; +const char kLogFieldNameLogMessage[] = "log_message"; +const char kLogFieldNameRequestLatency[] = "request_latency_in_ms"; +const char kLogFieldNameUrl[] = "url"; +const char kLogFieldNameErrorCause[] = "error_cause"; + +// Convert timestamp from time_point to Timestamp +Timestamp CreateTimestamp(std::chrono::system_clock::time_point tp) { + Timestamp time_stamp; + long long nanos = std::chrono::duration_cast( + tp.time_since_epoch()) + .count(); + + time_stamp.set_seconds(nanos / 1000000000); + time_stamp.set_nanos(nanos % 1000000000); + return time_stamp; +} + +Timestamp GetCurrentTimestamp() { + return CreateTimestamp(std::chrono::system_clock::now()); +} + +Status VerifyRequiredCheckFields(const OperationInfo& info) { + if (info.operation_id.empty()) { + return Status(Code::INVALID_ARGUMENT, "operation_id is required.", + Status::SERVICE_CONTROL); + } + if (info.operation_name.empty()) { + return Status(Code::INVALID_ARGUMENT, "operation_name is required.", + Status::SERVICE_CONTROL); + } + return Status::OK; +} + +Status VerifyRequiredReportFields(const OperationInfo& info) { + return Status::OK; +} + +void SetOperationCommonFields(const OperationInfo& info, + const Timestamp& current_time, Operation* op) { + if (!info.operation_id.empty()) { + op->set_operation_id(info.operation_id); + } + if (!info.operation_name.empty()) { + op->set_operation_name(info.operation_name); + } + if (!info.api_key.empty()) { + op->set_consumer_id(std::string(kConsumerIdApiKey) + + std::string(info.api_key)); + } + *op->mutable_start_time() = CreateTimestamp(info.request_start_time); + *op->mutable_end_time() = current_time; +} + +void FillLogEntry(const ReportRequestInfo& info, const std::string& name, + const Timestamp& current_time, LogEntry* log_entry) { + log_entry->set_name(name); + *log_entry->mutable_timestamp() = current_time; + auto severity = (info.response_code >= 400) ? google::logging::type::ERROR + : google::logging::type::INFO; + log_entry->set_severity(severity); + + auto* fields = log_entry->mutable_struct_payload()->mutable_fields(); + (*fields)[kLogFieldNameTimestamp].set_number_value( + (double)current_time.seconds() + + (double)current_time.nanos() / (double)1000000000.0); + if (!info.producer_project_id.empty()) { + (*fields)[kLogFieldNameProducerProjectId].set_string_value( + info.producer_project_id); + } + if (!info.api_key.empty()) { + (*fields)[kLogFieldNameApiKey].set_string_value(info.api_key); + } + if (!info.referer.empty()) { + (*fields)[kLogFieldNameReferer].set_string_value(info.referer); + } + if (!info.api_name.empty()) { + (*fields)[kLogFieldNameApiName].set_string_value(info.api_name); + } + if (!info.api_version.empty()) { + (*fields)[kLogFieldNameApiVersion].set_string_value(info.api_version); + } + if (!info.url.empty()) { + (*fields)[kLogFieldNameUrl].set_string_value(info.url); + } + if (!info.api_method.empty()) { + (*fields)[kLogFieldNameApiMethod].set_string_value(info.api_method); + } + if (!info.location.empty()) { + (*fields)[kLogFieldNameLocation].set_string_value(info.location); + } + if (!info.log_message.empty()) { + (*fields)[kLogFieldNameLogMessage].set_string_value(info.log_message); + } + + (*fields)[kLogFieldNameHttpResponseCode].set_number_value(info.response_code); + + if (info.request_size >= 0) { + (*fields)[kLogFieldNameRequestSize].set_number_value(info.request_size); + } + if (info.response_size >= 0) { + (*fields)[kLogFieldNameResponseSize].set_number_value(info.response_size); + } + if (info.latency.request_time_ms >= 0) { + (*fields)[kLogFieldNameRequestLatency].set_number_value( + info.latency.request_time_ms); + } + if (!info.method.empty()) { + (*fields)[kLogFieldNameHttpMethod].set_string_value(info.method); + } + if (info.response_code >= 400) { + (*fields)[kLogFieldNameErrorCause].set_string_value( + Status::ErrorCauseToString(info.status.error_cause())); + } +} + +template +std::vector FilterPointers( + const Element* first, const Element* last, + std::function pred) { + std::vector filtered; + while (first < last) { + if (pred(first)) { + filtered.push_back(first); + } + first++; + } + return filtered; +} + +} // namespace + +Proto::Proto(const std::set& logs, const std::string& service_name, + const std::string& service_config_id) + : logs_(logs.begin(), logs.end()), + metrics_(FilterPointers( + supported_metrics, supported_metrics + supported_metrics_count, + [](const struct SupportedMetric* m) { return m->set != nullptr; })), + labels_(FilterPointers( + supported_labels, supported_labels + supported_labels_count, + [](const struct SupportedLabel* l) { return l->set != nullptr; })), + service_name_(service_name), + service_config_id_(service_config_id) {} + +Proto::Proto(const std::set& logs, + const std::set& metrics, + const std::set& labels, + const std::string& service_name, + const std::string& service_config_id) + : logs_(logs.begin(), logs.end()), + metrics_(FilterPointers( + supported_metrics, supported_metrics + supported_metrics_count, + [&metrics](const struct SupportedMetric* m) { + return m->set && metrics.find(m->name) != metrics.end(); + })), + labels_(FilterPointers( + supported_labels, supported_labels + supported_labels_count, + [&labels](const struct SupportedLabel* l) { + return l->set && (l->kind == SupportedLabel::SYSTEM || + labels.find(l->name) != labels.end()); + })), + service_name_(service_name), + service_config_id_(service_config_id) {} + +Status Proto::FillCheckRequest(const CheckRequestInfo& info, + CheckRequest* request) { + Status status = VerifyRequiredCheckFields(info); + if (!status.ok()) { + return status; + } + request->set_service_name(service_name_); + request->set_service_config_id(service_config_id_); + + Timestamp current_time = GetCurrentTimestamp(); + Operation* op = request->mutable_operation(); + SetOperationCommonFields(info, current_time, op); + + auto* labels = op->mutable_labels(); + if (!info.client_ip.empty()) { + (*labels)[kServiceControlCallerIp] = info.client_ip; + } + if (!info.referer.empty()) { + (*labels)[kServiceControlReferer] = info.referer; + } + (*labels)[kServiceControlUserAgent] = kUserAgent; + (*labels)[kServiceControlServiceAgent] = kServiceAgent; + return Status::OK; +} + +Status Proto::FillReportRequest(const ReportRequestInfo& info, + ReportRequest* request) { + Status status = VerifyRequiredReportFields(info); + if (!status.ok()) { + return status; + } + request->set_service_name(service_name_); + request->set_service_config_id(service_config_id_); + + Timestamp current_time = GetCurrentTimestamp(); + Operation* op = request->add_operations(); + SetOperationCommonFields(info, current_time, op); + + // Only populate metrics if we can associate them with a method/operation. + if (!info.operation_id.empty() && !info.operation_name.empty()) { + Map* labels = op->mutable_labels(); + // Set all labels. + for (auto it = labels_.begin(), end = labels_.end(); it != end; it++) { + const SupportedLabel* l = *it; + if (l->set) { + status = (l->set)(*l, info, labels); + if (!status.ok()) return status; + } + } + + // Not to send consumer metrics for following cases: + // 1) api_key is not provided, or + // 2) the service is not activated for the consumer project, + bool send_consumer_metric = true; + if (info.api_key.empty() || + !info.check_response_info.service_is_activated) { + send_consumer_metric = false; + } + // Populate all metrics. + for (auto it = metrics_.begin(), end = metrics_.end(); it != end; it++) { + const SupportedMetric* m = *it; + if (send_consumer_metric || m->mark != SupportedMetric::CONSUMER) { + if (m->set) { + if ((info.is_first_report && m->tag == SupportedMetric::START) || + (info.is_final_report && + (m->tag == SupportedMetric::FINAL || + m->tag == SupportedMetric::INTERMEDIATE)) || + (!info.is_final_report && + m->tag == SupportedMetric::INTERMEDIATE)) { + status = (m->set)(*m, info, op); + if (!status.ok()) return status; + } + } + } + } + } + + // Fill log entries. + if (info.is_final_report) { + for (auto it = logs_.begin(), end = logs_.end(); it != end; it++) { + FillLogEntry(info, *it, current_time, op->add_log_entries()); + } + } + + return Status::OK; +} + +Status Proto::ConvertCheckResponse(const CheckResponse& check_response, + const std::string& service_name, + CheckResponseInfo* check_response_info) { + if (check_response.check_errors().size() == 0) { + return Status::OK; + } + + // TODO: aggregate status responses for all errors (including error.detail) + // TODO: report a detailed status to the producer project, but hide it from + // consumer + // TODO: unless they are the same entity + const CheckError& error = check_response.check_errors(0); + switch (error.code()) { + case CheckError::NOT_FOUND: // The consumer's project id is not found. + return Status(Code::INVALID_ARGUMENT, + "Client project not found. Please pass a valid project.", + Status::SERVICE_CONTROL); + case CheckError::API_KEY_NOT_FOUND: + if (check_response_info) check_response_info->is_api_key_valid = false; + return Status(Code::INVALID_ARGUMENT, + "API key not found. Please pass a valid API key.", + Status::SERVICE_CONTROL); + case CheckError::API_KEY_EXPIRED: + if (check_response_info) check_response_info->is_api_key_valid = false; + return Status(Code::INVALID_ARGUMENT, + "API key expired. Please renew the API key.", + Status::SERVICE_CONTROL); + case CheckError::API_KEY_INVALID: + if (check_response_info) check_response_info->is_api_key_valid = false; + return Status(Code::INVALID_ARGUMENT, + "API key not valid. Please pass a valid API key.", + Status::SERVICE_CONTROL); + case CheckError::SERVICE_NOT_ACTIVATED: + if (check_response_info) + check_response_info->service_is_activated = false; + return Status(Code::PERMISSION_DENIED, + std::string("API ") + service_name + + " is not enabled for the project.", + Status::SERVICE_CONTROL); + case CheckError::PERMISSION_DENIED: + return Status(Code::PERMISSION_DENIED, "Permission denied.", + Status::SERVICE_CONTROL); + case CheckError::IP_ADDRESS_BLOCKED: + return Status(Code::PERMISSION_DENIED, "IP address blocked.", + Status::SERVICE_CONTROL); + case CheckError::REFERER_BLOCKED: + return Status(Code::PERMISSION_DENIED, "Referer blocked.", + Status::SERVICE_CONTROL); + case CheckError::CLIENT_APP_BLOCKED: + return Status(Code::PERMISSION_DENIED, "Client application blocked.", + Status::SERVICE_CONTROL); + case CheckError::PROJECT_DELETED: + return Status(Code::PERMISSION_DENIED, "Project has been deleted.", + Status::SERVICE_CONTROL); + case CheckError::PROJECT_INVALID: + return Status(Code::INVALID_ARGUMENT, + "Client project not valid. Please pass a valid project.", + Status::SERVICE_CONTROL); + case CheckError::BILLING_DISABLED: + return Status(Code::PERMISSION_DENIED, + std::string("API ") + service_name + + " has billing disabled. Please enable it.", + Status::SERVICE_CONTROL); + case CheckError::NAMESPACE_LOOKUP_UNAVAILABLE: + case CheckError::SERVICE_STATUS_UNAVAILABLE: + case CheckError::BILLING_STATUS_UNAVAILABLE: + // Fail open for internal server errors per recommendation + return Status::OK; + default: + return Status( + Code::INTERNAL, + std::string("Request blocked due to unsupported error code: ") + + std::to_string(error.code()), + Status::SERVICE_CONTROL); + } + return Status::OK; +} + +bool Proto::IsMetricSupported(const ::google::api::MetricDescriptor& metric) { + for (int i = 0; i < supported_metrics_count; i++) { + const SupportedMetric& m = supported_metrics[i]; + if (metric.name() == m.name && metric.metric_kind() == m.metric_kind && + metric.value_type() == m.value_type) { + return true; + } + } + return false; +} + +bool Proto::IsLabelSupported(const ::google::api::LabelDescriptor& label) { + for (int i = 0; i < supported_labels_count; i++) { + const SupportedLabel& l = supported_labels[i]; + if (label.key() == l.name && label.value_type() == l.value_type) { + return true; + } + } + return false; +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/proto.h b/contrib/endpoints/src/api_manager/service_control/proto.h new file mode 100644 index 00000000000..321de44b861 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/proto.h @@ -0,0 +1,84 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_SERVICE_CONTROL_PROTO_H_ +#define API_MANAGER_SERVICE_CONTROL_PROTO_H_ + +#include "google/api/label.pb.h" +#include "google/api/metric.pb.h" +#include "google/api/servicecontrol/v1/service_controller.pb.h" +#include "include/api_manager/utils/status.h" +#include "src/api_manager/service_control/info.h" + +namespace google { +namespace api_manager { +namespace service_control { + +class Proto final { + public: + // Initializes Proto with all supported metrics and labels. + Proto(const std::set& logs, const std::string& service_name, + const std::string& service_config_id); + + // Initializes Proto with specified (and supported) metrics and + // labels. + Proto(const std::set& logs, const std::set& metrics, + const std::set& labels, const std::string& service_name, + const std::string& service_config_id); + + // Fills the CheckRequest protobuf from info. + // There are some logic inside the Fill functions beside just filling + // the fields, such as if both consumer_projecd_id and api_key present, + // one has to set to operation.producer_project_id and the other has to + // set to label. + // FillCheckRequest function should copy the strings pointed by info. + // These buffers may be freed after the FillCheckRequest call. + utils::Status FillCheckRequest( + const CheckRequestInfo& info, + ::google::api::servicecontrol::v1::CheckRequest* request); + + // Fills the CheckRequest protobuf from info. + // FillReportRequest function should copy the strings pointed by info. + // These buffers may be freed after the FillReportRequest call. + utils::Status FillReportRequest( + const ReportRequestInfo& info, + ::google::api::servicecontrol::v1::ReportRequest* request); + + // Converts the response status information in the CheckResponse protocol + // buffer into utils::Status and returns and returns 'check_response_info' + // subtracted from this CheckResponse. + // project_id is used when generating error message for project_id related + // failures. + static utils::Status ConvertCheckResponse( + const ::google::api::servicecontrol::v1::CheckResponse& response, + const std::string& service_name, CheckResponseInfo* check_response_info); + + static bool IsMetricSupported(const ::google::api::MetricDescriptor& metric); + static bool IsLabelSupported(const ::google::api::LabelDescriptor& label); + const std::string& service_name() const { return service_name_; } + const std::string& service_config_id() const { return service_config_id_; } + + private: + const std::vector logs_; + const std::vector metrics_; + const std::vector labels_; + const std::string service_name_; + const std::string service_config_id_; +}; + +} // namespace service_control +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_SERVICE_CONTROL_PROTO_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/proto_test.cc b/contrib/endpoints/src/api_manager/service_control/proto_test.cc new file mode 100644 index 00000000000..0670f072ff4 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/proto_test.cc @@ -0,0 +1,333 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/proto.h" +#include "gtest/gtest.h" + +#include +#include +#include + +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/text_format.h" + +#include "include/api_manager/version.h" + +namespace gasv1 = ::google::api::servicecontrol::v1; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; +using ::google::protobuf::Timestamp; + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +const char kTestdata[] = "src/api_manager/service_control/testdata/"; + +std::string ReadTestBaseline(const char* input_file_name) { + std::string file_name = std::string(kTestdata) + input_file_name; + + std::string contents; + std::ifstream input_file; + input_file.open(file_name, std::ifstream::in | std::ifstream::binary); + EXPECT_TRUE(input_file.is_open()) << file_name; + input_file.seekg(0, std::ios::end); + contents.reserve(input_file.tellg()); + input_file.seekg(0, std::ios::beg); + contents.assign((std::istreambuf_iterator(input_file)), + (std::istreambuf_iterator())); + + // Replace instances of {{service_agent_version}} with the expected service + // agent version. + std::string placeholder = "{{service_agent_version}}"; + std::string value = API_MANAGER_VERSION_STRING; + size_t current = 0; + while ((current = contents.find(placeholder, current)) != std::string::npos) { + contents.replace(current, placeholder.length(), value); + current += value.length(); + } + return contents; +} + +void FillOperationInfo(OperationInfo* op) { + op->operation_id = "operation_id"; + op->operation_name = "operation_name"; + op->api_key = "api_key_x"; + op->producer_project_id = "project_id"; +} + +void FillCheckRequestInfo(CheckRequestInfo* request) { + request->client_ip = "1.2.3.4"; + request->referer = "referer"; +} + +void FillReportRequestInfo(ReportRequestInfo* request) { + request->referer = "referer"; + request->response_code = 200; + request->location = "us-central"; + request->api_name = "api-name"; + request->api_version = "api-version"; + request->api_method = "api-method"; + request->request_size = 100; + request->response_size = 1024 * 1024; + request->log_message = "test-method is called"; + request->latency.request_time_ms = 123; + request->latency.backend_time_ms = 101; + request->latency.overhead_time_ms = 22; + request->protocol = protocol::HTTP; + request->compute_platform = compute_platform::GKE; + request->auth_issuer = "auth-issuer"; + request->auth_audience = "auth-audience"; + + request->request_bytes = 100; + request->response_bytes = 1024 * 1024; +} + +void SetFixTimeStamps(gasv1::Operation* op) { + Timestamp fix_time; + fix_time.set_seconds(100000); + fix_time.set_nanos(100000); + *op->mutable_start_time() = fix_time; + *op->mutable_end_time() = fix_time; + if (op->log_entries().size() > 0) { + *op->mutable_log_entries(0)->mutable_timestamp() = fix_time; + op->mutable_log_entries(0) + ->mutable_struct_payload() + ->mutable_fields() + ->erase("timestamp"); + } +} + +std::string CheckRequestToString(gasv1::CheckRequest* request) { + gasv1::Operation* op = request->mutable_operation(); + SetFixTimeStamps(op); + + std::string text; + google::protobuf::TextFormat::PrintToString(*request, &text); + return text; +} + +std::string ReportRequestToString(gasv1::ReportRequest* request) { + gasv1::Operation* op = request->mutable_operations(0); + SetFixTimeStamps(op); + + std::string text; + google::protobuf::TextFormat::PrintToString(*request, &text); + return text; +} + +class ProtoTest : public ::testing::Test { + protected: + ProtoTest() : scp_({"local_test_log"}, "test_service", "2016-09-19r0") {} + + Proto scp_; +}; + +TEST(Proto, TestProtobufStruct) { + // Verify if ::google::protobuf::Struct works. + // If the main binary code is compiled with CXXFLAGS=-std=c++11, + // and protobuf library is not, ::google::protobuf::Struct will crash. + ::google::protobuf::Struct st; + auto* fields = st.mutable_fields(); + (*fields)["test"].set_string_value("value"); + ASSERT_FALSE(fields->empty()); +} + +TEST_F(ProtoTest, FillGoodCheckRequestTest) { + CheckRequestInfo info; + FillOperationInfo(&info); + FillCheckRequestInfo(&info); + + gasv1::CheckRequest request; + ASSERT_TRUE(scp_.FillCheckRequest(info, &request).ok()); + + std::string text = CheckRequestToString(&request); + std::string expected_text = ReadTestBaseline("check_request.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillNoApiKeyCheckRequestTest) { + CheckRequestInfo info; + info.operation_id = "operation_id"; + info.operation_name = "operation_name"; + info.producer_project_id = "project_id"; + + gasv1::CheckRequest request; + ASSERT_TRUE(scp_.FillCheckRequest(info, &request).ok()); + + std::string text = CheckRequestToString(&request); + std::string expected_text = + ReadTestBaseline("check_request_no_api_key.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, CheckRequestMissingOperationNameTest) { + CheckRequestInfo info; + info.operation_id = "operation_id"; + + gasv1::CheckRequest request; + ASSERT_EQ(Code::INVALID_ARGUMENT, + scp_.FillCheckRequest(info, &request).code()); +} + +TEST_F(ProtoTest, CheckRequestMissingOperationIdTest) { + CheckRequestInfo info; + info.operation_name = "operation_name"; + + gasv1::CheckRequest request; + ASSERT_EQ(Code::INVALID_ARGUMENT, + scp_.FillCheckRequest(info, &request).code()); +} + +TEST_F(ProtoTest, FillGoodReportRequestTest) { + ReportRequestInfo info; + FillOperationInfo(&info); + FillReportRequestInfo(&info); + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + std::string text = ReportRequestToString(&request); + std::string expected_text = ReadTestBaseline("report_request.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillStartReportRequestTest) { + ReportRequestInfo info; + info.is_first_report = true; + info.is_final_report = false; + FillOperationInfo(&info); + FillReportRequestInfo(&info); + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + std::string text = ReportRequestToString(&request); + std::string expected_text = ReadTestBaseline("first_report_request.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillIntermediateReportRequestTest) { + ReportRequestInfo info; + info.is_first_report = false; + info.is_final_report = false; + FillOperationInfo(&info); + FillReportRequestInfo(&info); + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + std::string text = ReportRequestToString(&request); + std::string expected_text = + ReadTestBaseline("intermediate_report_request.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillFinalReportRequestTest) { + ReportRequestInfo info; + info.is_first_report = false; + info.is_final_report = true; + FillOperationInfo(&info); + FillReportRequestInfo(&info); + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + std::string text = ReportRequestToString(&request); + std::string expected_text = ReadTestBaseline("final_report_request.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillReportRequestFailedTest) { + ReportRequestInfo info; + FillOperationInfo(&info); + // Remove api_key to test not api_key case for + // producer_project_id and credential_id. + info.api_key = ""; + FillReportRequestInfo(&info); + + // Use 401 as a failed response code. + info.response_code = 401; + + // Use the corresponding status for that response code. + info.status = Status(info.response_code, "", Status::APPLICATION); + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + std::string text = ReportRequestToString(&request); + std::string expected_text = ReadTestBaseline("report_request_failed.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillReportRequestEmptyOptionalTest) { + ReportRequestInfo info; + FillOperationInfo(&info); + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + std::string text = ReportRequestToString(&request); + std::string expected_text = + ReadTestBaseline("report_request_empty_optional.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, CredentailIdApiKeyTest) { + ReportRequestInfo info; + FillOperationInfo(&info); + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + ASSERT_EQ(request.operations(0).labels().at("/credential_id"), + "apikey:api_key_x"); +} + +TEST_F(ProtoTest, CredentailIdIssuerOnlyTest) { + ReportRequestInfo info; + FillOperationInfo(&info); + info.api_key = ""; + info.auth_issuer = "auth-issuer"; + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + ASSERT_EQ(request.operations(0).labels().at("/credential_id"), + "jwtauth:issuer=YXV0aC1pc3N1ZXI"); +} + +TEST_F(ProtoTest, CredentailIdIssuerAudienceTest) { + ReportRequestInfo info; + FillOperationInfo(&info); + info.api_key = ""; + info.auth_issuer = "auth-issuer"; + info.auth_audience = "auth-audience"; + + gasv1::ReportRequest request; + ASSERT_TRUE(scp_.FillReportRequest(info, &request).ok()); + + ASSERT_EQ(request.operations(0).labels().at("/credential_id"), + "jwtauth:issuer=YXV0aC1pc3N1ZXI&audience=YXV0aC1hdWRpZW5jZQ"); +} + +} // namespace + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/check_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/check_request.golden new file mode 100644 index 00000000000..620006ddeac --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/check_request.golden @@ -0,0 +1,31 @@ +service_name: "test_service" +operation { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/check_request_no_api_key.golden b/contrib/endpoints/src/api_manager/service_control/testdata/check_request_no_api_key.golden new file mode 100644 index 00000000000..2d7d861e53e --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/check_request_no_api_key.golden @@ -0,0 +1,22 @@ +service_name: "test_service" +operation { + operation_id: "operation_id" + operation_name: "operation_name" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden new file mode 100644 index 00000000000..b9eff9f28eb --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden @@ -0,0 +1,429 @@ +service_name: "test_service" +operations { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "/credential_id" + value: "apikey:api_key_x" + } + labels { + key: "/protocol" + value: "http" + } + labels { + key: "/referer" + value: "referer" + } + labels { + key: "/response_code" + value: "200" + } + labels { + key: "/response_code_class" + value: "2xx" + } + labels { + key: "/status_code" + value: "0" + } + labels { + key: "cloud.googleapis.com/location" + value: "us-central" + } + labels { + key: "servicecontrol.googleapis.com/platform" + value: "GKE" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + labels { + key: "serviceruntime.googleapis.com/api_method" + value: "api-method" + } + labels { + key: "serviceruntime.googleapis.com/api_version" + value: "api-version" + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_sizes" + metric_values { + distribution_value { + count: 1 + mean: 100 + minimum: 100 + maximum: 100 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_sizes" + metric_values { + distribution_value { + count: 1 + mean: 100 + minimum: 100 + maximum: 100 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/response_sizes" + metric_values { + distribution_value { + count: 1 + mean: 1048576 + minimum: 1048576 + maximum: 1048576 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_sizes" + metric_values { + distribution_value { + count: 1 + mean: 1048576 + minimum: 1048576 + maximum: 1048576 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/response_bytes" + metric_values { + int64_value: 1048576 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_bytes" + metric_values { + int64_value: 1048576 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/total_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.123 + minimum: 0.123 + maximum: 0.123 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/total_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.123 + minimum: 0.123 + maximum: 0.123 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/backend_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.101 + minimum: 0.101 + maximum: 0.101 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/backend_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.101 + minimum: 0.101 + maximum: 0.101 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.022 + minimum: 0.022 + maximum: 0.022 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_overhead_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.022 + minimum: 0.022 + maximum: 0.022 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + log_entries { + struct_payload { + fields { + key: "api_key" + value { + string_value: "api_key_x" + } + } + fields { + key: "api_method" + value { + string_value: "api-method" + } + } + fields { + key: "api_name" + value { + string_value: "api-name" + } + } + fields { + key: "api_version" + value { + string_value: "api-version" + } + } + fields { + key: "http_response_code" + value { + number_value: 200 + } + } + fields { + key: "location" + value { + string_value: "us-central" + } + } + fields { + key: "log_message" + value { + string_value: "test-method is called" + } + } + fields { + key: "producer_project_id" + value { + string_value: "project_id" + } + } + fields { + key: "referer" + value { + string_value: "referer" + } + } + fields { + key: "request_latency_in_ms" + value { + number_value: 123 + } + } + fields { + key: "request_size" + value { + number_value: 100 + } + } + fields { + key: "response_size" + value { + number_value: 1048576 + } + } + } + name: "local_test_log" + timestamp { + seconds: 100000 + nanos: 100000 + } + severity: INFO + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/first_report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/first_report_request.golden new file mode 100644 index 00000000000..ac196d48452 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/first_report_request.golden @@ -0,0 +1,87 @@ +service_name: "test_service" +operations { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "/credential_id" + value: "apikey:api_key_x" + } + labels { + key: "/protocol" + value: "http" + } + labels { + key: "/referer" + value: "referer" + } + labels { + key: "cloud.googleapis.com/location" + value: "us-central" + } + labels { + key: "servicecontrol.googleapis.com/platform" + value: "GKE" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + labels { + key: "serviceruntime.googleapis.com/api_method" + value: "api-method" + } + labels { + key: "serviceruntime.googleapis.com/api_version" + value: "api-version" + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_count" + metric_values { + int64_value: 1 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_count" + metric_values { + int64_value: 1 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/response_bytes" + metric_values { + int64_value: 1048576 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_bytes" + metric_values { + int64_value: 1048576 + } + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/intermediate_report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/intermediate_report_request.golden new file mode 100644 index 00000000000..ddc3dc24031 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/intermediate_report_request.golden @@ -0,0 +1,75 @@ +service_name: "test_service" +operations { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "/credential_id" + value: "apikey:api_key_x" + } + labels { + key: "/protocol" + value: "http" + } + labels { + key: "/referer" + value: "referer" + } + labels { + key: "cloud.googleapis.com/location" + value: "us-central" + } + labels { + key: "servicecontrol.googleapis.com/platform" + value: "GKE" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + labels { + key: "serviceruntime.googleapis.com/api_method" + value: "api-method" + } + labels { + key: "serviceruntime.googleapis.com/api_version" + value: "api-version" + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/response_bytes" + metric_values { + int64_value: 1048576 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_bytes" + metric_values { + int64_value: 1048576 + } + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden new file mode 100644 index 00000000000..5d8a266ef88 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden @@ -0,0 +1,441 @@ +service_name: "test_service" +operations { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "/credential_id" + value: "apikey:api_key_x" + } + labels { + key: "/protocol" + value: "http" + } + labels { + key: "/referer" + value: "referer" + } + labels { + key: "/response_code" + value: "200" + } + labels { + key: "/response_code_class" + value: "2xx" + } + labels { + key: "/status_code" + value: "0" + } + labels { + key: "cloud.googleapis.com/location" + value: "us-central" + } + labels { + key: "servicecontrol.googleapis.com/platform" + value: "GKE" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + labels { + key: "serviceruntime.googleapis.com/api_method" + value: "api-method" + } + labels { + key: "serviceruntime.googleapis.com/api_version" + value: "api-version" + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_count" + metric_values { + int64_value: 1 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_count" + metric_values { + int64_value: 1 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_sizes" + metric_values { + distribution_value { + count: 1 + mean: 100 + minimum: 100 + maximum: 100 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_sizes" + metric_values { + distribution_value { + count: 1 + mean: 100 + minimum: 100 + maximum: 100 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/response_sizes" + metric_values { + distribution_value { + count: 1 + mean: 1048576 + minimum: 1048576 + maximum: 1048576 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_sizes" + metric_values { + distribution_value { + count: 1 + mean: 1048576 + minimum: 1048576 + maximum: 1048576 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/response_bytes" + metric_values { + int64_value: 1048576 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_bytes" + metric_values { + int64_value: 1048576 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/total_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.123 + minimum: 0.123 + maximum: 0.123 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/total_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.123 + minimum: 0.123 + maximum: 0.123 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/backend_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.101 + minimum: 0.101 + maximum: 0.101 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/backend_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.101 + minimum: 0.101 + maximum: 0.101 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.022 + minimum: 0.022 + maximum: 0.022 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_overhead_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.022 + minimum: 0.022 + maximum: 0.022 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + log_entries { + struct_payload { + fields { + key: "api_key" + value { + string_value: "api_key_x" + } + } + fields { + key: "api_method" + value { + string_value: "api-method" + } + } + fields { + key: "api_name" + value { + string_value: "api-name" + } + } + fields { + key: "api_version" + value { + string_value: "api-version" + } + } + fields { + key: "http_response_code" + value { + number_value: 200 + } + } + fields { + key: "location" + value { + string_value: "us-central" + } + } + fields { + key: "log_message" + value { + string_value: "test-method is called" + } + } + fields { + key: "producer_project_id" + value { + string_value: "project_id" + } + } + fields { + key: "referer" + value { + string_value: "referer" + } + } + fields { + key: "request_latency_in_ms" + value { + number_value: 123 + } + } + fields { + key: "request_size" + value { + number_value: 100 + } + } + fields { + key: "response_size" + value { + number_value: 1048576 + } + } + } + name: "local_test_log" + timestamp { + seconds: 100000 + nanos: 100000 + } + severity: INFO + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_empty_optional.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_empty_optional.golden new file mode 100644 index 00000000000..075536478d4 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_empty_optional.golden @@ -0,0 +1,87 @@ +service_name: "test_service" +operations { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "/credential_id" + value: "apikey:api_key_x" + } + labels { + key: "/protocol" + value: "unknown" + } + labels { + key: "/response_code" + value: "200" + } + labels { + key: "/response_code_class" + value: "2xx" + } + labels { + key: "/status_code" + value: "0" + } + labels { + key: "servicecontrol.googleapis.com/platform" + value: "unknown" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/consumer/request_count" + metric_values { + int64_value: 1 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_count" + metric_values { + int64_value: 1 + } + } + log_entries { + struct_payload { + fields { + key: "api_key" + value { + string_value: "api_key_x" + } + } + fields { + key: "http_response_code" + value { + number_value: 200 + } + } + fields { + key: "producer_project_id" + value { + string_value: "project_id" + } + } + } + name: "local_test_log" + timestamp { + seconds: 100000 + nanos: 100000 + } + severity: INFO + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden new file mode 100644 index 00000000000..a3ffd58c30d --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden @@ -0,0 +1,302 @@ +service_name: "test_service" +operations { + operation_id: "operation_id" + operation_name: "operation_name" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "/credential_id" + value: "jwtauth:issuer=YXV0aC1pc3N1ZXI&audience=YXV0aC1hdWRpZW5jZQ" + } + labels { + key: "/error_type" + value: "4xx" + } + labels { + key: "/protocol" + value: "http" + } + labels { + key: "/referer" + value: "referer" + } + labels { + key: "/response_code" + value: "401" + } + labels { + key: "/response_code_class" + value: "4xx" + } + labels { + key: "/status_code" + value: "16" + } + labels { + key: "cloud.googleapis.com/location" + value: "us-central" + } + labels { + key: "servicecontrol.googleapis.com/platform" + value: "GKE" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + labels { + key: "serviceruntime.googleapis.com/api_method" + value: "api-method" + } + labels { + key: "serviceruntime.googleapis.com/api_version" + value: "api-version" + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_count" + metric_values { + int64_value: 1 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_sizes" + metric_values { + distribution_value { + count: 1 + mean: 100 + minimum: 100 + maximum: 100 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_sizes" + metric_values { + distribution_value { + count: 1 + mean: 1048576 + minimum: 1048576 + maximum: 1048576 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_bytes" + metric_values { + int64_value: 100 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/response_bytes" + metric_values { + int64_value: 1048576 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/error_count" + metric_values { + int64_value: 1 + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/total_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.123 + minimum: 0.123 + maximum: 0.123 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/backend_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.101 + minimum: 0.101 + maximum: 0.101 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + metric_value_sets { + metric_name: "serviceruntime.googleapis.com/api/producer/request_overhead_latencies" + metric_values { + distribution_value { + count: 1 + mean: 0.022 + minimum: 0.022 + maximum: 0.022 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 1 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + exponential_buckets { + num_finite_buckets: 8 + growth_factor: 10 + scale: 1e-06 + } + } + } + } + log_entries { + struct_payload { + fields { + key: "api_method" + value { + string_value: "api-method" + } + } + fields { + key: "api_name" + value { + string_value: "api-name" + } + } + fields { + key: "api_version" + value { + string_value: "api-version" + } + } + fields { + key: "error_cause" + value { + string_value: "application" + } + } + fields { + key: "http_response_code" + value { + number_value: 401 + } + } + fields { + key: "location" + value { + string_value: "us-central" + } + } + fields { + key: "log_message" + value { + string_value: "test-method is called" + } + } + fields { + key: "producer_project_id" + value { + string_value: "project_id" + } + } + fields { + key: "referer" + value { + string_value: "referer" + } + } + fields { + key: "request_latency_in_ms" + value { + number_value: 123 + } + } + fields { + key: "request_size" + value { + number_value: 100 + } + } + fields { + key: "response_size" + value { + number_value: 1048576 + } + } + } + name: "local_test_log" + timestamp { + seconds: 100000 + nanos: 100000 + } + severity: ERROR + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/url.cc b/contrib/endpoints/src/api_manager/service_control/url.cc new file mode 100644 index 00000000000..cf02c06c6e9 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/url.cc @@ -0,0 +1,74 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/url.h" + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +// Service Control check and report URL paths: +// /v1/services/{service}:check +// /v1/services/{service}:report +const char v1_services_path[] = "/v1/services/"; +const char check_verb[] = ":check"; +const char report_verb[] = ":report"; +const char http[] = "http://"; +const char https[] = "https://"; + +// Finds service control server URL. Supports server config override +const std::string& GetServiceControlAddress( + const ::google::api::Service* service, + const proto::ServerConfig* server_config) { + // Return the value from the server config override if present. + if (server_config && server_config->has_service_control_config()) { + const ::google::api_manager::proto::ServiceControlConfig& scc = + server_config->service_control_config(); + if (!scc.url_override().empty()) { + return scc.url_override(); + } + } + + // Otherwise, use the default from service config. + return service->control().environment(); +} + +} // namespace + +Url::Url(const ::google::api::Service* service, + const proto::ServerConfig* server_config) { + // Precompute check and report URLs + if (service) { + service_control_ = GetServiceControlAddress(service, server_config); + } + + if (!service_control_.empty()) { + if (service_control_.compare(0, sizeof(http) - 1, http) != 0 && + service_control_.compare(0, sizeof(https) - 1, https) != 0) { + service_control_ = https + service_control_; // https is default + } + + std::string path = service_control_ + v1_services_path + service->name(); + check_url_ = path + check_verb; + report_url_ = path + report_verb; + } +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/url.h b/contrib/endpoints/src/api_manager/service_control/url.h new file mode 100644 index 00000000000..065f6ceaf5e --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/url.h @@ -0,0 +1,47 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_SERVICE_CONTROL_URL_H_ +#define API_MANAGER_SERVICE_CONTROL_URL_H_ + +#include "google/api/service.pb.h" +#include "src/api_manager/proto/server_config.pb.h" + +namespace google { +namespace api_manager { +namespace service_control { + +// This class implements service control url related logic. +class Url { + public: + Url(const ::google::api::Service* service, + const ::google::api_manager::proto::ServerConfig* server_config); + + // Pre-computed url for service control. + const std::string& service_control() const { return service_control_; } + const std::string& check_url() const { return check_url_; } + const std::string& report_url() const { return report_url_; } + + private: + // Pre-computed url for service control methods. + std::string service_control_; + std::string check_url_; + std::string report_url_; +}; + +} // namespace service_control +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_SERVICE_CONTROL_URL_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/url_test.cc b/contrib/endpoints/src/api_manager/service_control/url_test.cc new file mode 100644 index 00000000000..933f084db5b --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/url_test.cc @@ -0,0 +1,78 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/service_control/url.h" + +#include "gtest/gtest.h" +#include "src/api_manager/config.h" +#include "src/api_manager/mock_api_manager_environment.h" + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +const char prepend_https_config[] = R"( +name: "https-config" +control: { + environment: "servicecontrol.googleapis.com" +} +)"; + +static char server_config[] = R"( +service_control_config { + url_override: "servicecontrol-testing.googleapis.com" +} +)"; + +TEST(UrlTest, PrependHttps) { + std::unique_ptr env( + new ::testing::NiceMock()); + std::unique_ptr config( + Config::Create(env.get(), prepend_https_config, "")); + ASSERT_TRUE(config); + Url url(&config->service(), config->server_config()); + // https:// got prepended by default. + ASSERT_EQ("https://servicecontrol.googleapis.com", url.service_control()); + ASSERT_EQ( + "https://servicecontrol.googleapis.com/v1/services/https-config:check", + url.check_url()); + ASSERT_EQ( + "https://servicecontrol.googleapis.com/v1/services/https-config:report", + url.report_url()); +} + +TEST(UrlTest, ServerControlOverride) { + std::unique_ptr env( + new ::testing::NiceMock()); + std::unique_ptr config( + Config::Create(env.get(), prepend_https_config, server_config)); + ASSERT_TRUE(config); + ASSERT_TRUE(config->server_config()); + Url url(&config->service(), config->server_config()); + ASSERT_EQ("https://servicecontrol-testing.googleapis.com", + url.service_control()); +} + +} // namespace + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/utils/BUILD b/contrib/endpoints/src/api_manager/utils/BUILD new file mode 100644 index 00000000000..59078c9951e --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/BUILD @@ -0,0 +1,87 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +package(default_visibility = [ + "//src/api_manager:__subpackages__", +]) + +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") + +cc_library( + name = "utils", + srcs = [ + "marshalling.cc", + "status.cc", + "url_util.cc", + ], + hdrs = [ + "marshalling.h", + "stl_util.h", + "url_util.h", + ], + linkopts = select({ + "//:darwin": [], + "//conditions:default": [ + "-lm", + "-luuid", + ], + }), + deps = [ + "//external:cc_wkt_protos", + "//external:protobuf", + "//external:servicecontrol", # for google/rpc/status.proto + "//include:headers_only", + ], +) + +cc_test( + name = "marshalling_test", + size = "small", + srcs = [ + "marshalling_test.cc", + ], + linkstatic = 1, + deps = [ + ":utils", + "//external:googletest_main", + ], +) + +cc_test( + name = "status_test", + size = "small", + srcs = [ + "status_test.cc", + ], + linkstatic = 1, + deps = [ + ":utils", + "//external:googletest_main", + ], +) + +cc_test( + name = "url_util_test", + size = "small", + srcs = [ + "url_util_test.cc", + ], + linkstatic = 1, + deps = [ + ":utils", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/utils/marshalling.cc b/contrib/endpoints/src/api_manager/utils/marshalling.cc new file mode 100644 index 00000000000..f2d52b6f4e7 --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/marshalling.cc @@ -0,0 +1,131 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/utils/marshalling.h" + +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/util/json_util.h" +#include "google/protobuf/util/type_resolver.h" +#include "google/protobuf/util/type_resolver_util.h" + +using ::google::protobuf::Message; +using ::google::protobuf::util::TypeResolver; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace utils { + +namespace { +const char kTypeUrlPrefix[] = "type.googleapis.com"; + +// Creation function used by static lazy init. +TypeResolver* CreateTypeResolver() { + return ::google::protobuf::util::NewTypeResolverForDescriptorPool( + kTypeUrlPrefix, ::google::protobuf::DescriptorPool::generated_pool()); +} + +// Returns the singleton type resolver, creating it on first call. +TypeResolver* GetTypeResolver() { + static TypeResolver* resolver = CreateTypeResolver(); + return resolver; +} +} // namespace + +std::string GetTypeUrl(const Message& message) { + return std::string(kTypeUrlPrefix) + "/" + + message.GetDescriptor()->full_name(); +} + +Status ProtoToJson(const Message& message, std::string* result, int options) { + ::google::protobuf::util::JsonPrintOptions json_options; + if (options & JsonOptions::PRETTY_PRINT) { + json_options.add_whitespace = true; + } + if (options & JsonOptions::OUTPUT_DEFAULTS) { + json_options.always_print_primitive_fields = true; + } + // TODO: Skip going to bytes and use ProtoObjectSource directly. + ::google::protobuf::util::Status status = + ::google::protobuf::util::BinaryToJsonString( + GetTypeResolver(), GetTypeUrl(message), message.SerializeAsString(), + result, json_options); + return Status::FromProto(status); +} + +Status ProtoToJson(const Message& message, + ::google::protobuf::io::ZeroCopyOutputStream* json, + int options) { + ::google::protobuf::util::JsonPrintOptions json_options; + if (options & JsonOptions::PRETTY_PRINT) { + json_options.add_whitespace = true; + } + if (options & JsonOptions::OUTPUT_DEFAULTS) { + json_options.always_print_primitive_fields = true; + } + // TODO: Skip going to bytes and use ProtoObjectSource directly. + std::string binary = message.SerializeAsString(); + ::google::protobuf::io::ArrayInputStream binary_stream(binary.data(), + binary.size()); + ::google::protobuf::util::Status status = + ::google::protobuf::util::BinaryToJsonStream( + GetTypeResolver(), GetTypeUrl(message), &binary_stream, json, + json_options); + return Status::FromProto(status); +} + +Status JsonToProto(const std::string& json, Message* message) { + ::google::protobuf::util::JsonParseOptions options; + options.ignore_unknown_fields = true; + std::string binary; + ::google::protobuf::util::Status status = + ::google::protobuf::util::JsonToBinaryString( + GetTypeResolver(), GetTypeUrl(*message), json, &binary, options); + if (!status.ok()) { + return Status::FromProto(status); + } + if (message->ParseFromString(binary)) { + return Status::OK; + } + return Status( + Code::INTERNAL, + "Unable to parse bytes generated from JsonToBinaryString as proto."); +} + +Status JsonToProto(::google::protobuf::io::ZeroCopyInputStream* json, + ::google::protobuf::Message* message) { + ::google::protobuf::util::JsonParseOptions options; + options.ignore_unknown_fields = true; + std::string binary; + ::google::protobuf::io::StringOutputStream output(&binary); + ::google::protobuf::util::Status status = + ::google::protobuf::util::JsonToBinaryStream( + GetTypeResolver(), GetTypeUrl(*message), json, &output, options); + + if (!status.ok()) { + return Status::FromProto(status); + } + if (message->ParseFromString(binary)) { + return Status::OK; + } + return Status( + Code::INTERNAL, + "Unable to parse bytes generated from JsonToBinaryString as proto."); +} + +} // namespace utils +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/utils/marshalling.h b/contrib/endpoints/src/api_manager/utils/marshalling.h new file mode 100644 index 00000000000..b517673aadd --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/marshalling.h @@ -0,0 +1,68 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_UTILS_MARSHALLING_H_ +#define API_MANAGER_UTILS_MARSHALLING_H_ + +#include + +#include "google/protobuf/message.h" +#include "include/api_manager/utils/status.h" + +namespace google { +namespace api_manager { +namespace utils { + +// Options for JSON output. These should be OR'd together so must be 2^n. +enum JsonOptions { + // Use the default behavior (useful when no options are needed). + DEFAULT = 0, + + // Enables pretty printing of the output. + PRETTY_PRINT = 1, + + // Prints default values for primitive fields. + OUTPUT_DEFAULTS = 2, +}; + +// Returns the type URL for a protobuf Message. This is useful when embedding +// a message inside an Any, for example. +std::string GetTypeUrl(const ::google::protobuf::Message& message); + +// Converts a protobuf into a JSON string. The options field is a OR'd set of +// the available JsonOptions. +// TODO: Support generating to an output buffer. +Status ProtoToJson(const ::google::protobuf::Message& message, + std::string* result, int options); + +// Converts a protobuf into a JSON string and writes it into the output stream. +// The options parameter is an OR'd set of the available JsonOptions. +Status ProtoToJson(const ::google::protobuf::Message& message, + ::google::protobuf::io::ZeroCopyOutputStream* json, + int options); + +// Converts a json string into a protobuf message. +// TODO: Support parsing directly from an input buffer. +Status JsonToProto(const std::string& json, + ::google::protobuf::Message* message); + +// Converts a json input stream into a protobuf message. +Status JsonToProto(::google::protobuf::io::ZeroCopyInputStream* json, + ::google::protobuf::Message* message); + +} // namespace utils +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_UTILS_MARSHALLING_H_ diff --git a/contrib/endpoints/src/api_manager/utils/marshalling_test.cc b/contrib/endpoints/src/api_manager/utils/marshalling_test.cc new file mode 100644 index 00000000000..48b567d95fe --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/marshalling_test.cc @@ -0,0 +1,145 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/utils/marshalling.h" + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/struct.pb.h" +#include "google/rpc/error_details.pb.h" +#include "gtest/gtest.h" +#include "include/api_manager/utils/status.h" + +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace utils { + +TEST(Marshalling, ProtoToJsonDefaultOptions) { + ::google::rpc::DebugInfo debug; + debug.set_detail("Something went wrong."); + debug.add_stack_entries("first"); + debug.add_stack_entries("second"); + + std::string result; + EXPECT_EQ(Status::OK, ProtoToJson(debug, &result, JsonOptions::DEFAULT)); + EXPECT_EQ( + "{\"stackEntries\":[\"first\",\"second\"],\"detail\":\"Something went " + "wrong.\"}", + result); +} + +TEST(Marshalling, ProtoToJsonPrettyPrintOption) { + ::google::rpc::DebugInfo debug; + debug.set_detail("Something went wrong."); + debug.add_stack_entries("first"); + debug.add_stack_entries("second"); + + std::string result; + EXPECT_EQ(Status::OK, ProtoToJson(debug, &result, JsonOptions::PRETTY_PRINT)); + EXPECT_EQ( + "{\n \"stackEntries\": [\n \"first\",\n \"second\"\n ],\n \"detail\": " + "\"Something went wrong.\"\n}\n", + result); +} + +TEST(Marshalling, ProtoToJsonStream) { + ::google::rpc::DebugInfo debug; + debug.set_detail("Something went wrong."); + debug.add_stack_entries("first"); + debug.add_stack_entries("second"); + + std::string result; + ::google::protobuf::io::StringOutputStream json(&result); + EXPECT_EQ(Status::OK, + ProtoToJson(debug, &json, JsonOptions::PRETTY_PRINT | + JsonOptions::OUTPUT_DEFAULTS)); + EXPECT_EQ( + "{\n \"stackEntries\": [\n \"first\",\n \"second\"\n ],\n \"detail\": " + "\"Something went wrong.\"\n}\n", + result); +} + +TEST(Marshalling, JsonToProtoParsing) { + std::string json = + "{\"stackEntries\":[\"first\",\"second\"],\"detail\":\"Something went " + "wrong.\"}"; + + ::google::rpc::DebugInfo debug; + EXPECT_EQ(Status::OK, JsonToProto(json, &debug)); + EXPECT_EQ("Something went wrong.", debug.detail()); + EXPECT_EQ(2, debug.stack_entries_size()); + EXPECT_EQ("first", debug.stack_entries(0)); + EXPECT_EQ("second", debug.stack_entries(1)); +} + +TEST(Marshalling, JsonToProtoParsingWithExtraFields) { + std::string json = + "{\"stackEntries\":[\"first\",\"second\"],\"detail\":\"Something went " + "wrong.\", \"unknownEntries\": 0}"; + + ::google::rpc::DebugInfo debug; + EXPECT_EQ(Status::OK, JsonToProto(json, &debug)); + EXPECT_EQ("Something went wrong.", debug.detail()); + EXPECT_EQ(2, debug.stack_entries_size()); + EXPECT_EQ("first", debug.stack_entries(0)); + EXPECT_EQ("second", debug.stack_entries(1)); +} + +TEST(Marshalling, JsonToProtoParsingJwtPayload) { + std::string json = + R"({ + "iss": "23028304136-191r4v40tn4g96jf1saccr3vf1ke8aer@developer.gserviceaccount.com", + "sub": "23028304136-191r4v40tn4g96jf1saccr3vf1ke8aer@developer.gserviceaccount.com", + "aud": "bookstore-esp-echo.cloudendpointsapis.com", + "iat": 1462581603, + "exp": 1462585203 + })"; + + ::google::protobuf::Struct st; + + EXPECT_EQ(Status::OK, JsonToProto(json, &st)); + EXPECT_EQ(5, st.fields_size()); + EXPECT_EQ(1462581603, static_cast(st.fields().at("iat").number_value())); + EXPECT_EQ("bookstore-esp-echo.cloudendpointsapis.com", + st.fields().at("aud").string_value()); +} + +TEST(Marshalling, JsonToProtoParsingJwtPayloadWithMultipleAudiences) { + std::string json = + R"({ + "iss": "23028304136-191r4v40tn4g96jf1saccr3vf1ke8aer@developer.gserviceaccount.com", + "sub": "23028304136-191r4v40tn4g96jf1saccr3vf1ke8aer@developer.gserviceaccount.com", + "aud": ["bookstore-esp-echo.cloudendpointsapis.com", "bookstore2"], + "iat": 1462581603, + "exp": 1462585203 + })"; + + ::google::protobuf::Struct st; + + EXPECT_EQ(Status::OK, JsonToProto(json, &st)); + EXPECT_EQ(5, st.fields_size()); + EXPECT_EQ(1462581603, static_cast(st.fields().at("iat").number_value())); + EXPECT_TRUE(st.fields().at("aud").has_list_value()); + EXPECT_EQ(2, st.fields().at("aud").list_value().values_size()); + EXPECT_EQ("bookstore-esp-echo.cloudendpointsapis.com", + st.fields().at("aud").list_value().values(0).string_value()); +} + +} // namespace utils +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/utils/status.cc b/contrib/endpoints/src/api_manager/utils/status.cc new file mode 100644 index 00000000000..7b93d1bcaf7 --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/status.cc @@ -0,0 +1,442 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "include/api_manager/utils/status.h" + +#include + +#include "src/api_manager/utils/marshalling.h" + +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace utils { + +Status::Status(int code, const std::string& message, ErrorCause error_cause) + : code_(code == 200 ? Code::OK : code), + message_(message), + error_cause_(error_cause) {} + +Status::Status(int code, const std::string& message) + : Status(code, message, Status::INTERNAL) {} + +Status::Status() : Status(Code::OK, "", Status::INTERNAL) {} + +bool Status::operator==(const Status& x) const { + if (code_ != x.code_ || message_ != x.message_ || + error_cause_ != x.error_cause_) { + return false; + } + return true; +} + +/* static */ std::string Status::CodeToString(int code) { + // NGX error codes are negative. These are generally control codes. + if (code < 0) { + switch (code) { + case -1: + return "ERROR"; + case -2: + return "AGAIN"; + case -3: + return "BUSY"; + case -4: + return "DONE"; + case -5: + return "DECLINED"; + case -6: + return "ABORT"; + default: { + std::ostringstream ngx_out; + ngx_out << "UNKNOWN(" << code << ")"; + return ngx_out.str(); + } + } + } + + // Codes from 0 to 99 are in the protobuf canonical space, so we map those to + // their enum names. + if (code < 100) { + switch (code) { + case Code::OK: + return "OK"; + case Code::CANCELLED: + return "CANCELLED"; + case Code::UNKNOWN: + return "UNKNOWN"; + case Code::INVALID_ARGUMENT: + return "INVALID_ARGUMENT"; + case Code::DEADLINE_EXCEEDED: + return "DEADLINE_EXCEEDED"; + case Code::NOT_FOUND: + return "NOT_FOUND"; + case Code::ALREADY_EXISTS: + return "ALREADY_EXISTS"; + case Code::PERMISSION_DENIED: + return "PERMISSION_DENIED"; + case Code::UNAUTHENTICATED: + return "UNAUTHENTICATED"; + case Code::RESOURCE_EXHAUSTED: + return "RESOURCE_EXHAUSTED"; + case Code::FAILED_PRECONDITION: + return "FAILED_PRECONDITION"; + case Code::ABORTED: + return "ABORTED"; + case Code::OUT_OF_RANGE: + return "OUT_OF_RANGE"; + case Code::UNIMPLEMENTED: + return "UNIMPLEMENTED"; + case Code::INTERNAL: + return "INTERNAL"; + case Code::UNAVAILABLE: + return "UNAVAILABLE"; + case Code::DATA_LOSS: + return "DATA_LOSS"; + default: { + std::ostringstream canonical_out; + canonical_out << "UNKNOWN(" << code << ")"; + return canonical_out.str(); + } + } + } + + // Codes >= 100 are in the HTTP space, so we map to the HTTP meaning. We don't + // cover all of the code extensions, so any leftovers result in a generic + // error message with the code attached. + switch (code) { + case 100: + return "CONTINUE"; + case 200: + return "OK"; + case 201: + return "CREATED"; + case 202: + return "ACCEPTED"; + case 204: + return "NO_CONTENT"; + case 301: + return "MOVED_PERMANENTLY"; + case 302: + return "FOUND"; + case 303: + return "SEE_OTHER"; + case 304: + return "NOT_MODIFIED"; + case 305: + return "USE_PROXY"; + case 307: + return "TEMPORARY_REDIRECT"; + case 308: + return "PERMANENT_REDIRECT"; + case 400: + return "BAD_REQUEST"; + case 401: + return "UNAUTHORIZED"; + case 402: + return "PAYMENT_REQUIRED"; + case 403: + return "FORBIDDEN"; + case 404: + return "NOT_FOUND"; + case 405: + return "METHOD_NOT_ALLOWED"; + case 406: + return "NOT_ACCEPTABLE"; + case 408: + return "REQUEST_TIMEOUT"; + case 409: + return "CONFLICT"; + case 410: + return "GONE"; + case 411: + return "LENGTH_REQUIRED"; + case 412: + return "PRECONDITION_FAILED"; + case 413: + return "PAYLOAD_TOO_LARGE"; + case 414: + return "REQUEST_URI_TOO_LONG"; + case 415: + return "UNSUPPORTED_MEDIA_TYPE"; + case 416: + return "RANGE_NOT_SATISFIABLE"; + case 417: + return "EXPECTATION_FAILED"; + case 418: + return "IM_A_TEAPOT"; + case 419: + return "AUTHENTICATION_TIMEOUT"; + case 428: + return "PRECONDITION_REQUIRED"; + case 429: + return "TOO_MANY_REQUESTS"; + case 431: + return "REQUEST_HEADERS_TOO_LARGE"; + case 444: + return "NO_RESPONSE"; + case 499: + return "CLIENT_CLOSED_REQUEST"; + case 500: + return "INTERNAL_SERVER_ERROR"; + case 501: + return "NOT_IMPLEMENTED"; + case 502: + return "BAD_GATEWAY"; + case 503: + return "SERVICE_UNAVAILABLE"; + case 504: + return "GATEWAY_TIMEOUT"; + default: { + std::ostringstream http_out; + if (code >= 200 && code < 300) + http_out << "OK("; + else if (code >= 400 && code < 500) + http_out << "INVALID_REQUEST("; + else if (code >= 500 && code < 600) + http_out << "SERVER_ERROR("; + else + http_out << "UNKNOWN("; + http_out << code << ")"; + return http_out.str(); + } + } +} + +/* static */ std::string Status::ErrorCauseToString(ErrorCause error_cause) { + switch (error_cause) { + default: + case INTERNAL: + return "internal"; + case APPLICATION: + return "application"; + case AUTH: + return "auth"; + case SERVICE_CONTROL: + return "service_control"; + } +} + +/* static */ Status Status::FromProto( + const ::google::protobuf::util::Status& proto_status) { + if (proto_status.ok()) { + return OK; + } + return Status(proto_status.error_code(), + proto_status.error_message().ToString()); +} + +::google::protobuf::util::Status Status::ToProto() const { + ::google::protobuf::util::Status result(CanonicalCode(), message_); + return result; +} + +/* static */ const Status& Status::OK = Status(); +/* static */ const Status& Status::DONE = Status(-4, ""); + +// Note: We return 400 instead of 412 for failed precondition as the meaning of +// 412 is more specific than FAILED_PRECONDITION. If a 412 is desired create an +// error using 412 and it will be mapped in the other direction to the +// FAILED_PRECONDITION canonical code as desired. +int Status::HttpCode() const { + // If the code is already an HTTP error code, just return it intact. + if (code_ >= 100) { + return code_; + } + + // Map NGINX status codes to HTTP status codes. + if (code_ < 0) { + switch (code_) { + case -1: + // ERROR -> INTERNAL ERROR + return 500; + case -2: + // AGAIN -> CONTINUE + return 100; + case -3: + // BUSY -> TOO MANY REQUESTS + return 429; + case -4: + // DONE -> ACCEPTED + return 202; + case -5: + // DECLINED -> NOT FOUND + return 404; + case -6: + // ABORT -> BAD REQUEST + return 400; + default: + // UNKNOWN -> INTERNAL_SERVER_ERROR + return 500; + } + } + + // Map Canonical codes to HTTP status codes. This is based on the mapping + // defined by the protobuf http error space. + switch (code_) { + case Code::OK: + return 200; + case Code::CANCELLED: + return 499; + case Code::UNKNOWN: + return 500; + case Code::INVALID_ARGUMENT: + return 400; + case Code::DEADLINE_EXCEEDED: + return 504; + case Code::NOT_FOUND: + return 404; + case Code::ALREADY_EXISTS: + return 409; + case Code::PERMISSION_DENIED: + return 403; + case Code::RESOURCE_EXHAUSTED: + return 429; + case Code::FAILED_PRECONDITION: + return 400; + case Code::ABORTED: + return 409; + case Code::OUT_OF_RANGE: + return 400; + case Code::UNIMPLEMENTED: + return 501; + case Code::INTERNAL: + return 500; + case Code::UNAVAILABLE: + return 503; + case Code::DATA_LOSS: + return 500; + case Code::UNAUTHENTICATED: + return 401; + default: + return 500; + } +} + +Code Status::CanonicalCode() const { + // Map NGNX status codes to canonical codes. + if (code_ < 0) { + switch (code_) { + case -1: + // ERROR -> INTERNAL + return Code::INTERNAL; + case -2: + // AGAIN -> CANCELLED + return Code::CANCELLED; + case -3: + // BUSY -> PERMISSION DENIED + return Code::PERMISSION_DENIED; + case -4: + // DONE -> ALREADY EXISTS + return Code::ALREADY_EXISTS; + case -5: + // DECLINED -> NOT FOUND + return Code::NOT_FOUND; + case -6: + // ABORT -> ABORTED + return Code::ABORTED; + default: + // UNKNOWN -> UNKNOWN; + return Code::UNKNOWN; + } + } + + // The space from 0 to 99 is for canonical codes, so we leave it as is. + if (code_ < 100) { + return (Code)code_; + } + + // Map HTTP error codes to canonical codes. This is based on the mapping + // defined by the protobuf HTTP error space. + switch (code_) { + case 400: + return Code::INVALID_ARGUMENT; + case 403: + return Code::PERMISSION_DENIED; + case 404: + return Code::NOT_FOUND; + case 409: + return Code::ABORTED; + case 416: + return Code::OUT_OF_RANGE; + case 429: + return Code::RESOURCE_EXHAUSTED; + case 499: + return Code::CANCELLED; + case 504: + return Code::DEADLINE_EXCEEDED; + case 501: + return Code::UNIMPLEMENTED; + case 503: + return Code::UNAVAILABLE; + case 401: + return Code::UNAUTHENTICATED; + default: { + if (code_ >= 200 && code_ < 300) return Code::OK; + if (code_ >= 400 && code_ < 500) return Code::FAILED_PRECONDITION; + if (code_ >= 500 && code_ < 600) return Code::INTERNAL; + return Code::UNKNOWN; + } + } +} + +::google::rpc::Status Status::ToCanonicalProto() const { + ::google::rpc::Status status; + status.set_code(CanonicalCode()); + status.set_message(message_); + + ::google::rpc::DebugInfo info; + info.set_detail(Status::ErrorCauseToString(error_cause_)); + status.add_details()->PackFrom(info); + + return status; +} + +std::string Status::ToJson() const { + ::google::rpc::Status proto = ToCanonicalProto(); + std::string result; + int options = JsonOptions::PRETTY_PRINT | JsonOptions::OUTPUT_DEFAULTS; + Status status = ProtoToJson(proto, &result, options); + if (!status.ok()) { + // If translation failed, try outputting the json translation failure itself + // as a JSON error. This should only happen if one of the error details had + // an unresolvable type url. + proto = status.ToCanonicalProto(); + status = ProtoToJson(proto, &result, options); + if (!status.ok()) { + // This should never happen but just in case we do a non-json response. + result = "Unable to generate error response: "; + result.append(status.message_); + } + } + return result; +} + +std::string Status::ToString() const { + if (code_ == Code::OK) { + return "OK"; + } else { + if (message_.empty()) { + return Status::CodeToString(code_); + } else { + return Status::CodeToString(code_) + ": " + message_; + } + } +} + +} // namespace utils +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/utils/status_test.cc b/contrib/endpoints/src/api_manager/utils/status_test.cc new file mode 100644 index 00000000000..d9e1e36f349 --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/status_test.cc @@ -0,0 +1,251 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "include/api_manager/utils/status.h" + +#include "google/protobuf/any.pb.h" +#include "google/rpc/error_details.pb.h" +#include "google/rpc/status.pb.h" +#include "gtest/gtest.h" +#include "src/api_manager/utils/marshalling.h" + +namespace google { +namespace api_manager { +namespace utils { + +namespace { + +using ::google::protobuf::util::error::Code; + +TEST(Status, StatusOkReturnsTrueWhenAppropriate) { + EXPECT_TRUE(Status::OK.ok()); + EXPECT_TRUE(Status(0, "OK").ok()); + EXPECT_TRUE(Status(200, "OK").ok()); + + EXPECT_FALSE(Status(-1, "Internal Error").ok()); + EXPECT_FALSE(Status(400, "Invalid Parameter").ok()); + EXPECT_FALSE(Status(5, "Not Found").ok()); +} + +TEST(Status, ToHttpCodeMapping) { + EXPECT_EQ(200, Status::OK.HttpCode()); + EXPECT_EQ(404, Status(5, "Not Found").HttpCode()); + EXPECT_EQ(200, Status(200, "OK").HttpCode()); + + // Nginx codes + EXPECT_EQ(500, Status(-1, "").HttpCode()); + EXPECT_EQ(100, Status(-2, "").HttpCode()); + EXPECT_EQ(429, Status(-3, "").HttpCode()); + EXPECT_EQ(202, Status(-4, "").HttpCode()); + EXPECT_EQ(404, Status(-5, "").HttpCode()); + EXPECT_EQ(400, Status(-6, "").HttpCode()); + EXPECT_EQ(500, Status(-7, "").HttpCode()); + + // Canonical codes. + EXPECT_EQ(200, Status(Code::OK, "").HttpCode()); + EXPECT_EQ(499, Status(Code::CANCELLED, "").HttpCode()); + EXPECT_EQ(500, Status(Code::UNKNOWN, "").HttpCode()); + EXPECT_EQ(400, Status(Code::INVALID_ARGUMENT, "").HttpCode()); + EXPECT_EQ(504, Status(Code::DEADLINE_EXCEEDED, "").HttpCode()); + EXPECT_EQ(404, Status(Code::NOT_FOUND, "").HttpCode()); + EXPECT_EQ(409, Status(Code::ALREADY_EXISTS, "").HttpCode()); + EXPECT_EQ(403, Status(Code::PERMISSION_DENIED, "").HttpCode()); + EXPECT_EQ(401, Status(Code::UNAUTHENTICATED, "").HttpCode()); + EXPECT_EQ(429, Status(Code::RESOURCE_EXHAUSTED, "").HttpCode()); + EXPECT_EQ(400, Status(Code::FAILED_PRECONDITION, "").HttpCode()); + EXPECT_EQ(409, Status(Code::ABORTED, "").HttpCode()); + EXPECT_EQ(400, Status(Code::OUT_OF_RANGE, "").HttpCode()); + EXPECT_EQ(501, Status(Code::UNIMPLEMENTED, "").HttpCode()); + EXPECT_EQ(500, Status(Code::INTERNAL, "").HttpCode()); + EXPECT_EQ(503, Status(Code::UNAVAILABLE, "").HttpCode()); + EXPECT_EQ(500, Status(Code::DATA_LOSS, "").HttpCode()); + EXPECT_EQ(500, Status(70, "").HttpCode()); +} + +TEST(Status, ToCanonicalMapping) { + EXPECT_EQ(0, Status::OK.CanonicalCode()); + EXPECT_EQ(5, Status(404, "Not Found").CanonicalCode()); + EXPECT_EQ(4, Status(4, "Deadline Exceeded").CanonicalCode()); + + EXPECT_EQ(Code::OK, Status(200, "").CanonicalCode()); + EXPECT_EQ(Code::CANCELLED, Status(-2, "").CanonicalCode()); + EXPECT_EQ(Code::UNKNOWN, Status(700, "").CanonicalCode()); + EXPECT_EQ(Code::INVALID_ARGUMENT, Status(400, "").CanonicalCode()); + EXPECT_EQ(Code::DEADLINE_EXCEEDED, Status(504, "").CanonicalCode()); + EXPECT_EQ(Code::NOT_FOUND, Status(-5, "").CanonicalCode()); + EXPECT_EQ(Code::NOT_FOUND, Status(404, "").CanonicalCode()); + EXPECT_EQ(Code::ALREADY_EXISTS, Status(-4, "").CanonicalCode()); + EXPECT_EQ(Code::PERMISSION_DENIED, Status(-3, "").CanonicalCode()); + EXPECT_EQ(Code::PERMISSION_DENIED, Status(403, "").CanonicalCode()); + EXPECT_EQ(Code::UNAUTHENTICATED, Status(401, "").CanonicalCode()); + EXPECT_EQ(Code::RESOURCE_EXHAUSTED, Status(429, "").CanonicalCode()); + EXPECT_EQ(Code::FAILED_PRECONDITION, Status(450, "").CanonicalCode()); + EXPECT_EQ(Code::ABORTED, Status(409, "").CanonicalCode()); + EXPECT_EQ(Code::OUT_OF_RANGE, Status(416, "").CanonicalCode()); + EXPECT_EQ(Code::UNIMPLEMENTED, Status(501, "").CanonicalCode()); + EXPECT_EQ(Code::INTERNAL, Status(500, "").CanonicalCode()); + EXPECT_EQ(Code::UNAVAILABLE, Status(503, "").CanonicalCode()); +} + +TEST(Status, ToProtoIncludesCodeAndMessage) { + Status status(400, "Invalid Parameter"); + ::google::protobuf::util::Status proto = status.ToProto(); + EXPECT_EQ(Code::INVALID_ARGUMENT, proto.error_code()); + EXPECT_EQ("Invalid Parameter", proto.error_message()); +} + +TEST(Status, ToJsonIncludesCodeAndMessage) { + EXPECT_EQ( + "{\n" + " \"code\": 5,\n" + " \"message\": \"Unknown Element\",\n" + " \"details\": [\n" + " {\n" + " \"@type\": \"type.googleapis.com/google.rpc.DebugInfo\",\n" + " \"stackEntries\": [],\n" + " \"detail\": \"auth\"\n" + " }\n" + " ]\n" + "}\n", + Status(5, "Unknown Element", Status::AUTH).ToJson()); +} + +TEST(Status, ToJsonIncludesDetails) { + Status status(400, "Invalid Parameter"); + + EXPECT_EQ( + "{\n" + " \"code\": 3,\n" + " \"message\": \"Invalid Parameter\",\n" + " \"details\": [\n" + " {\n" + " \"@type\": \"type.googleapis.com/google.rpc.DebugInfo\",\n" + " \"stackEntries\": [],\n" + " \"detail\": \"internal\"\n" + " }\n" + " ]\n" + "}\n", + status.ToJson()); +} + +TEST(Status, ToStringPrintsNgxCodes) { + EXPECT_EQ("ERROR", Status(-1, "").ToString()); + EXPECT_EQ("AGAIN", Status(-2, "").ToString()); + EXPECT_EQ("BUSY", Status(-3, "").ToString()); + EXPECT_EQ("DONE", Status(-4, "").ToString()); + EXPECT_EQ("DECLINED", Status(-5, "").ToString()); + EXPECT_EQ("ABORT", Status(-6, "").ToString()); + EXPECT_EQ("UNKNOWN(-17)", Status(-17, "").ToString()); + EXPECT_EQ("AGAIN: Try Again!", Status(-2, "Try Again!").ToString()); + EXPECT_EQ("ABORT: Uh oh", Status(-6, "Uh oh").ToString()); +} + +TEST(Status, ToStringPrintsCanonicalCodes) { + EXPECT_EQ("OK", Status(Code::OK, "").ToString()); + EXPECT_EQ("CANCELLED", Status(Code::CANCELLED, "").ToString()); + EXPECT_EQ("UNKNOWN", Status(Code::UNKNOWN, "").ToString()); + EXPECT_EQ("INVALID_ARGUMENT", Status(Code::INVALID_ARGUMENT, "").ToString()); + EXPECT_EQ("DEADLINE_EXCEEDED", + Status(Code::DEADLINE_EXCEEDED, "").ToString()); + EXPECT_EQ("NOT_FOUND", Status(Code::NOT_FOUND, "").ToString()); + EXPECT_EQ("ALREADY_EXISTS", Status(Code::ALREADY_EXISTS, "").ToString()); + EXPECT_EQ("PERMISSION_DENIED", + Status(Code::PERMISSION_DENIED, "").ToString()); + EXPECT_EQ("UNAUTHENTICATED", Status(Code::UNAUTHENTICATED, "").ToString()); + EXPECT_EQ("RESOURCE_EXHAUSTED", + Status(Code::RESOURCE_EXHAUSTED, "").ToString()); + EXPECT_EQ("FAILED_PRECONDITION", + Status(Code::FAILED_PRECONDITION, "").ToString()); + EXPECT_EQ("ABORTED", Status(Code::ABORTED, "").ToString()); + EXPECT_EQ("OUT_OF_RANGE", Status(Code::OUT_OF_RANGE, "").ToString()); + EXPECT_EQ("UNIMPLEMENTED", Status(Code::UNIMPLEMENTED, "").ToString()); + EXPECT_EQ("INTERNAL", Status(Code::INTERNAL, "").ToString()); + EXPECT_EQ("UNAVAILABLE", Status(Code::UNAVAILABLE, "").ToString()); + EXPECT_EQ("DATA_LOSS", Status(Code::DATA_LOSS, "").ToString()); + + EXPECT_EQ("UNKNOWN(50): Unknown error code.", + Status(50, "Unknown error code.").ToString()); + + EXPECT_EQ("INVALID_ARGUMENT: Unknown parameter 'foo'", + Status(3, "Unknown parameter 'foo'").ToString()); + EXPECT_EQ("UNIMPLEMENTED: Support forthcoming", + Status(12, "Support forthcoming").ToString()); +} + +TEST(Status, ToStringPrintsHttpCodes) { + EXPECT_EQ("INTERNAL_SERVER_ERROR: Bad configuration", + Status(500, "Bad configuration").ToString()); + EXPECT_EQ("IM_A_TEAPOT: Short and stout", + Status(418, "Short and stout").ToString()); + + EXPECT_EQ("CONTINUE", Status(100, "").ToString()); + EXPECT_EQ("OK", Status(200, "").ToString()); + EXPECT_EQ("CREATED", Status(201, "").ToString()); + EXPECT_EQ("ACCEPTED", Status(202, "").ToString()); + EXPECT_EQ("NO_CONTENT", Status(204, "").ToString()); + EXPECT_EQ("MOVED_PERMANENTLY", Status(301, "").ToString()); + EXPECT_EQ("FOUND", Status(302, "").ToString()); + EXPECT_EQ("SEE_OTHER", Status(303, "").ToString()); + EXPECT_EQ("NOT_MODIFIED", Status(304, "").ToString()); + EXPECT_EQ("USE_PROXY", Status(305, "").ToString()); + EXPECT_EQ("TEMPORARY_REDIRECT", Status(307, "").ToString()); + EXPECT_EQ("PERMANENT_REDIRECT", Status(308, "").ToString()); + EXPECT_EQ("BAD_REQUEST", Status(400, "").ToString()); + EXPECT_EQ("UNAUTHORIZED", Status(401, "").ToString()); + EXPECT_EQ("PAYMENT_REQUIRED", Status(402, "").ToString()); + EXPECT_EQ("FORBIDDEN", Status(403, "").ToString()); + EXPECT_EQ("NOT_FOUND", Status(404, "").ToString()); + EXPECT_EQ("METHOD_NOT_ALLOWED", Status(405, "").ToString()); + EXPECT_EQ("NOT_ACCEPTABLE", Status(406, "").ToString()); + EXPECT_EQ("REQUEST_TIMEOUT", Status(408, "").ToString()); + EXPECT_EQ("CONFLICT", Status(409, "").ToString()); + EXPECT_EQ("GONE", Status(410, "").ToString()); + EXPECT_EQ("LENGTH_REQUIRED", Status(411, "").ToString()); + EXPECT_EQ("PRECONDITION_FAILED", Status(412, "").ToString()); + EXPECT_EQ("PAYLOAD_TOO_LARGE", Status(413, "").ToString()); + EXPECT_EQ("REQUEST_URI_TOO_LONG", Status(414, "").ToString()); + EXPECT_EQ("UNSUPPORTED_MEDIA_TYPE", Status(415, "").ToString()); + EXPECT_EQ("RANGE_NOT_SATISFIABLE", Status(416, "").ToString()); + EXPECT_EQ("EXPECTATION_FAILED", Status(417, "").ToString()); + EXPECT_EQ("IM_A_TEAPOT", Status(418, "").ToString()); + EXPECT_EQ("AUTHENTICATION_TIMEOUT", Status(419, "").ToString()); + EXPECT_EQ("PRECONDITION_REQUIRED", Status(428, "").ToString()); + EXPECT_EQ("TOO_MANY_REQUESTS", Status(429, "").ToString()); + EXPECT_EQ("REQUEST_HEADERS_TOO_LARGE", Status(431, "").ToString()); + EXPECT_EQ("NO_RESPONSE", Status(444, "").ToString()); + EXPECT_EQ("CLIENT_CLOSED_REQUEST", Status(499, "").ToString()); + EXPECT_EQ("INTERNAL_SERVER_ERROR", Status(500, "").ToString()); + EXPECT_EQ("NOT_IMPLEMENTED", Status(501, "").ToString()); + EXPECT_EQ("BAD_GATEWAY", Status(502, "").ToString()); + EXPECT_EQ("SERVICE_UNAVAILABLE", Status(503, "").ToString()); + EXPECT_EQ("GATEWAY_TIMEOUT", Status(504, "").ToString()); +} + +TEST(Status, ToStringPrintsIntegerCodeForUnknownHttpCodes) { + EXPECT_EQ("SERVER_ERROR(529): Something Screwy", + Status(529, "Something Screwy").ToString()); + EXPECT_EQ("INVALID_REQUEST(472): I don't even", + Status(472, "I don't even").ToString()); + EXPECT_EQ("OK(237): OK I guess", Status(237, "OK I guess").ToString()); + EXPECT_EQ("UNKNOWN(987): Nine, Eight, Seven", + Status(987, "Nine, Eight, Seven").ToString()); +} + +} // namespace + +} // namespace utils +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/utils/stl_util.h b/contrib/endpoints/src/api_manager/utils/stl_util.h new file mode 100644 index 00000000000..19e960a11e0 --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/stl_util.h @@ -0,0 +1,323 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_UTILS_STL_UTIL_H_ +#define API_MANAGER_UTILS_STL_UTIL_H_ + +namespace util { +namespace gtl { +namespace map_util_internal { + +// Local implementation of RemoveConst to avoid including base/type_traits.h. +template +struct RemoveConst { + typedef T type; +}; +template +struct RemoveConst : RemoveConst {}; + +// PointeeType

::type is the pointee of smart or raw pointer P. +template +struct PointeeTypeImpl { + typedef typename P::element_type type; +}; +template +struct PointeeTypeImpl { + typedef T type; +}; +template +struct PointeeType : PointeeTypeImpl

{}; +template +struct PointeeType : PointeeType

{}; +template +struct PointeeType : PointeeType

{}; + +} // namespace map_util_internal +} // namespace gtl +} // namespace util + +namespace google { +namespace api_manager { +namespace utils { + +// Calls delete (non-array version) on pointers in the range [begin, end). +// +// Note: If you're calling this on an entire container, you probably want to +// call STLDeleteElements(&container) instead (which also clears the container), +// or use an ElementDeleter. +template +void STLDeleteContainerPointers(ForwardIterator begin, ForwardIterator end) { + while (begin != end) { + ForwardIterator temp = begin; + ++begin; + delete *temp; + } +} + +// Calls delete (non-array version) on BOTH items (pointers) in each pair in the +// range [begin, end). +template +void STLDeleteContainerPairPointers(ForwardIterator begin, + ForwardIterator end) { + while (begin != end) { + ForwardIterator temp = begin; + ++begin; + delete temp->first; + delete temp->second; + } +} + +// Calls delete (non-array version) on the FIRST item (pointer) in each pair in +// the range [begin, end). +template +void STLDeleteContainerPairFirstPointers(ForwardIterator begin, + ForwardIterator end) { + while (begin != end) { + ForwardIterator temp = begin; + ++begin; + delete temp->first; + } +} + +// Calls delete (non-array version) on the SECOND item (pointer) in each pair in +// the range [begin, end). +// +// Note: If you're calling this on an entire container, you probably want to +// call STLDeleteValues(&container) instead, or use ValueDeleter. +template +void STLDeleteContainerPairSecondPointers(ForwardIterator begin, + ForwardIterator end) { + while (begin != end) { + ForwardIterator temp = begin; + ++begin; + delete temp->second; + } +} + +// Deletes all the elements in an STL container and clears the container. This +// function is suitable for use with a vector, set, hash_set, or any other STL +// container which defines sensible begin(), end(), and clear() methods. +// +// If container is nullptr, this function is a no-op. +// +// As an alternative to calling STLDeleteElements() directly, consider +// ElementDeleter (defined below), which ensures that your container's elements +// are deleted when the ElementDeleter goes out of scope. +template +void STLDeleteElements(T* container) { + if (!container) return; + STLDeleteContainerPointers(container->begin(), container->end()); + container->clear(); +} + +// Given an STL container consisting of (key, value) pairs, STLDeleteValues +// deletes all the "value" components and clears the container. Does nothing in +// the case it's given a nullptr. +template +void STLDeleteValues(T* v) { + if (!v) return; + STLDeleteContainerPairSecondPointers(v->begin(), v->end()); + v->clear(); +} + +// Returns a reference to the pointer associated with key. If not found, +// a pointee is constructed and added to the map. In that case, the new +// pointee is value-initialized (aka "default-constructed"). +// Useful for containers of the form Map, where Ptr is pointer-like. +template +typename Collection::value_type::second_type& LookupOrInsertNew( + Collection* const collection, + const typename Collection::value_type::first_type& key) { + typedef typename Collection::value_type::second_type Mapped; + typedef + typename util::gtl::map_util_internal::PointeeType::type Element; + std::pair ret = + collection->insert(typename Collection::value_type(key, Mapped())); + if (ret.second) { + ret.first->second = Mapped(new Element()); + } + return ret.first->second; +} + +// A variant that accepts and forwards a pointee constructor argument. +template +typename Collection::value_type::second_type& LookupOrInsertNew( + Collection* const collection, + const typename Collection::value_type::first_type& key, const Arg& arg) { + typedef typename Collection::value_type::second_type Mapped; + typedef + typename util::gtl::map_util_internal::PointeeType::type Element; + std::pair ret = + collection->insert(typename Collection::value_type(key, Mapped())); + if (ret.second) { + ret.first->second = Mapped(new Element(arg)); + } + return ret.first->second; +} + +// Inserts the given key and value into the given collection if and only if the +// given key did NOT already exist in the collection. If the key previously +// existed in the collection, the value is not changed. Returns true if the +// key-value pair was inserted; returns false if the key was already present. +template +bool InsertIfNotPresent(Collection* const collection, + const typename Collection::value_type& vt) { + return collection->insert(vt).second; +} + +// Same as above except the key and value are passed separately. +template +bool InsertIfNotPresent( + Collection* const collection, + const typename Collection::value_type::first_type& key, + const typename Collection::value_type::second_type& value) { + return InsertIfNotPresent(collection, + typename Collection::value_type(key, value)); +} + +// Inserts the given key-value pair into the collection. Returns true if and +// only if the key from the given pair didn't previously exist. Otherwise, the +// value in the map is replaced with the value from the given pair. +template +bool InsertOrUpdate(Collection* const collection, + const typename Collection::value_type& vt) { + std::pair ret = collection->insert(vt); + if (!ret.second) { + // update + ret.first->second = vt.second; + return false; + } + return true; +} + +// Same as above, except that the key and value are passed separately. +template +bool InsertOrUpdate(Collection* const collection, + const typename Collection::value_type::first_type& key, + const typename Collection::value_type::second_type& value) { + return InsertOrUpdate(collection, + typename Collection::value_type(key, value)); +} + +// Tries to insert the given key-value pair into the collection. Returns nullptr +// if the insert succeeds. Otherwise, returns a pointer to the existing value. +// +// This complements UpdateReturnCopy in that it allows to update only after +// verifying the old value and still insert quickly without having to look up +// twice. Unlike UpdateReturnCopy this also does not come with the issue of an +// undefined previous* in case new data was inserted. +template +typename Collection::value_type::second_type* InsertOrReturnExisting( + Collection* const collection, const typename Collection::value_type& vt) { + std::pair ret = collection->insert(vt); + if (ret.second) { + return nullptr; // Inserted, no existing previous value. + } else { + return &ret.first->second; // Return address of already existing value. + } +} + +// Same as above, except for explicit key and data. +template +typename Collection::value_type::second_type* InsertOrReturnExisting( + Collection* const collection, + const typename Collection::value_type::first_type& key, + const typename Collection::value_type::second_type& data) { + return InsertOrReturnExisting(collection, + typename Collection::value_type(key, data)); +} + +// Returns a const reference to the value associated with the given key if it +// exists, otherwise returns a const reference to the provided default value. +// +// WARNING: If a temporary object is passed as the default "value," +// this function will return a reference to that temporary object, +// which will be destroyed at the end of the statement. A common +// example: if you have a map with string values, and you pass a char* +// as the default "value," either use the returned value immediately +// or store it in a string (not string&). +template +const typename Collection::value_type::second_type& FindWithDefault( + const Collection& collection, + const typename Collection::value_type::first_type& key, + const typename Collection::value_type::second_type& value) { + typename Collection::const_iterator it = collection.find(key); + if (it == collection.end()) { + return value; + } + return it->second; +} + +// Returns a pointer to the const value associated with the given key if it +// exists, or nullptr otherwise. +template +const typename Collection::value_type::second_type* FindOrNull( + const Collection& collection, + const typename Collection::value_type::first_type& key) { + typename Collection::const_iterator it = collection.find(key); + if (it == collection.end()) { + return 0; + } + return &it->second; +} + +// Same as above but returns a pointer to the non-const value. +template +typename Collection::value_type::second_type* FindOrNull( + Collection& collection, // NOLINT + const typename Collection::value_type::first_type& key) { + typename Collection::iterator it = collection.find(key); + if (it == collection.end()) { + return 0; + } + return &it->second; +} + +// Returns the pointer value associated with the given key. If none is found, +// nullptr is returned. The function is designed to be used with a map of keys +// to pointers. +// +// This function does not distinguish between a missing key and a key mapped +// to a nullptr value. +template +typename Collection::value_type::second_type FindPtrOrNull( + const Collection& collection, + const typename Collection::value_type::first_type& key) { + typename Collection::const_iterator it = collection.find(key); + if (it == collection.end()) { + return typename Collection::value_type::second_type(); + } + return it->second; +} + +// Same as above, except takes non-const reference to collection. +// +// This function is needed for containers that propagate constness to the +// pointee, such as boost::ptr_map. +template +typename Collection::value_type::second_type FindPtrOrNull( + Collection& collection, // NOLINT + const typename Collection::value_type::first_type& key) { + typename Collection::iterator it = collection.find(key); + if (it == collection.end()) { + return typename Collection::value_type::second_type(); + } + return it->second; +} + +} // namespace utils +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_UTILS_STL_UTIL_H_ diff --git a/contrib/endpoints/src/api_manager/utils/url_util.cc b/contrib/endpoints/src/api_manager/utils/url_util.cc new file mode 100644 index 00000000000..60eedc2ef4b --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/url_util.cc @@ -0,0 +1,42 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +#include "src/api_manager/utils/url_util.h" + +namespace google { +namespace api_manager { +namespace utils { + +std::string GetUrlContent(const std::string &url) { + static const std::string https_prefix = "https://"; + static const std::string http_prefix = "http://"; + std::string result; + if (url.compare(0, https_prefix.size(), https_prefix) == 0) { + result = url.substr(https_prefix.size()); + } else if (url.compare(0, http_prefix.size(), http_prefix) == 0) { + result = url.substr(http_prefix.size()); + } else { + result = url; + } + if (result.back() == '/') { + result.pop_back(); + } + return result; +} + +} // namespace utils +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/utils/url_util.h b/contrib/endpoints/src/api_manager/utils/url_util.h new file mode 100644 index 00000000000..2c002b37faf --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/url_util.h @@ -0,0 +1,32 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_UTILS_URL_UTIL_H_ +#define API_MANAGER_UTILS_URL_UTIL_H_ + +#include + +namespace google { +namespace api_manager { +namespace utils { + +// Strips off https or http prefix and trailing '/' from a URL, and returns the +// processed string. +std::string GetUrlContent(const std::string &url); + +} // namespace utils +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_UTILS_URL_UTIL_H_ diff --git a/contrib/endpoints/src/api_manager/utils/url_util_test.cc b/contrib/endpoints/src/api_manager/utils/url_util_test.cc new file mode 100644 index 00000000000..07e3693d139 --- /dev/null +++ b/contrib/endpoints/src/api_manager/utils/url_util_test.cc @@ -0,0 +1,35 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "src/api_manager/utils/url_util.h" + +#include "gtest/gtest.h" + +namespace google { +namespace api_manager { +namespace utils { + +TEST(UrlUtil, UrlContentTest) { + EXPECT_EQ("a.com", GetUrlContent("https://a.com/")); + EXPECT_EQ("xyz", GetUrlContent("http://xyz")); + EXPECT_EQ("xyz", GetUrlContent("https://xyz")); + EXPECT_EQ("a.com.", GetUrlContent("http://a.com./")); + EXPECT_EQ("xyz", GetUrlContent("xyz/")); +} + +} // namespace utils +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/third_party/BUILD.googleapis b/contrib/endpoints/third_party/BUILD.googleapis new file mode 100644 index 00000000000..e82368a3e84 --- /dev/null +++ b/contrib/endpoints/third_party/BUILD.googleapis @@ -0,0 +1,93 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +licenses(["notice"]) + +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") + +exports_files(glob(["google/**"])) + +cc_proto_library( + name = "servicecontrol", + srcs = [ + "google/api/servicecontrol/v1/check_error.proto", + "google/api/servicecontrol/v1/distribution.proto", + "google/api/servicecontrol/v1/log_entry.proto", + "google/api/servicecontrol/v1/metric_value.proto", + "google/api/servicecontrol/v1/operation.proto", + "google/api/servicecontrol/v1/service_controller.proto", + "google/logging/type/http_request.proto", + "google/logging/type/log_severity.proto", + "google/rpc/error_details.proto", + "google/rpc/status.proto", + "google/type/money.proto", + ], + include = ".", + visibility = ["//visibility:public"], + deps = [ + ":service_config", + ], + protoc = "//external:protoc", + default_runtime = "//external:protobuf", +) + +cc_proto_library( + name = "service_config", + srcs = [ + "google/api/annotations.proto", + "google/api/auth.proto", + "google/api/backend.proto", + "google/api/billing.proto", + "google/api/consumer.proto", + "google/api/context.proto", + "google/api/control.proto", + "google/api/documentation.proto", + "google/api/endpoint.proto", + "google/api/http.proto", + "google/api/label.proto", + "google/api/log.proto", + "google/api/logging.proto", + "google/api/metric.proto", + "google/api/monitored_resource.proto", + "google/api/monitoring.proto", + "google/api/service.proto", + "google/api/system_parameter.proto", + "google/api/usage.proto", + ], + include = ".", + visibility = ["//visibility:public"], + deps = [ + "//external:cc_wkt_protos", + ], + protoc = "//external:protoc", + default_runtime = "//external:protobuf", +) + +cc_proto_library( + name = "cloud_trace", + srcs = [ + "google/devtools/cloudtrace/v1/trace.proto", + ], + include = ".", + default_runtime = "//external:protobuf", + protoc = "//external:protoc", + visibility = ["//visibility:public"], + deps = [ + ":service_config", + "//external:cc_wkt_protos", + ], +) diff --git a/contrib/endpoints/third_party/BUILD.googletest b/contrib/endpoints/third_party/BUILD.googletest new file mode 100644 index 00000000000..dbf7e64add4 --- /dev/null +++ b/contrib/endpoints/third_party/BUILD.googletest @@ -0,0 +1,56 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +cc_library( + name = "googletest", + srcs = [ + "googletest/src/gtest-all.cc", + "googlemock/src/gmock-all.cc", + ], + hdrs = glob([ + "googletest/include/**/*.h", + "googlemock/include/**/*.h", + "googletest/src/*.cc", + "googletest/src/*.h", + "googlemock/src/*.cc", + ]), + includes = [ + "googlemock", + "googletest", + "googletest/include", + "googlemock/include", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "googletest_main", + srcs = ["googlemock/src/gmock_main.cc"], + visibility = ["//visibility:public"], + deps = [":googletest"], +) + +cc_library( + name = "googletest_prod", + hdrs = [ + "googletest/include/gtest/gtest_prod.h", + ], + includes = [ + "googletest/include", + ], + visibility = ["//visibility:public"], +) diff --git a/contrib/endpoints/third_party/BUILD.nanopb b/contrib/endpoints/third_party/BUILD.nanopb new file mode 100644 index 00000000000..263c1c28aea --- /dev/null +++ b/contrib/endpoints/third_party/BUILD.nanopb @@ -0,0 +1,54 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +licenses(["notice"]) + +exports_files(["LICENSE.txt"]) + +cc_library( + name = "nanopb", + srcs = [ + "pb.h", + "pb_common.c", + "pb_common.h", + "pb_decode.c", + "pb_decode.h", + "pb_encode.c", + "pb_encode.h", + ], + hdrs = [":includes"], + visibility = [ + "//visibility:public", + ], +) + +genrule( + name = "includes", + srcs = [ + "pb.h", + "pb_common.h", + "pb_decode.h", + "pb_encode.h", + ], + outs = [ + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h", + ], + cmd = "mkdir -p $(@D)/third_party/nanopb && cp $(SRCS) $(@D)/third_party/nanopb", +) diff --git a/contrib/endpoints/third_party/BUILD.zlib b/contrib/endpoints/third_party/BUILD.zlib new file mode 100644 index 00000000000..a288e8c4488 --- /dev/null +++ b/contrib/endpoints/third_party/BUILD.zlib @@ -0,0 +1,57 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +licenses(["notice"]) + +exports_files(["README"]) + +cc_library( + name = "zlib", + srcs = [ + "adler32.c", + "crc32.c", + "crc32.h", + "deflate.c", + "deflate.h", + "infback.c", + "inffast.c", + "inffast.h", + "inffixed.h", + "inflate.c", + "inflate.h", + "inftrees.c", + "inftrees.h", + "trees.c", + "trees.h", + "zconf.h", + "zutil.c", + "zutil.h", + ], + hdrs = [ + "zlib.h", + ], + copts = [ + "-Wno-shift-negative-value", + "-Wno-unknown-warning-option", + ], + defines = [ + "Z_SOLO", + ], + visibility = [ + "//visibility:public", + ], +) diff --git a/contrib/endpoints/tools/BUILD b/contrib/endpoints/tools/BUILD new file mode 100644 index 00000000000..31fcea7f862 --- /dev/null +++ b/contrib/endpoints/tools/BUILD @@ -0,0 +1,23 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +sh_binary( + name = "create_version", + srcs = [ + "create_version.sh", + ], + visibility = ["//visibility:public"], +) diff --git a/contrib/endpoints/tools/create_version.sh b/contrib/endpoints/tools/create_version.sh new file mode 100755 index 00000000000..6b46de300e8 --- /dev/null +++ b/contrib/endpoints/tools/create_version.sh @@ -0,0 +1,50 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +read VERSION < "${1:-/dev/stdin}" +MAJOR="${VERSION%%\.*}" +REST="${VERSION#*\.}" +MINOR="${REST%%\.*}" +REVISION="${VERSION##*\.}" + +cat <