From 78b931b935d69422a0c6a0c9424ac50c054b0677 Mon Sep 17 00:00:00 2001 From: enricoschiattarella Date: Tue, 10 Jan 2017 13:26:08 -0800 Subject: [PATCH 01/49] Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect (#38) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect * Fixed style. --- contrib/tools/server/Makefile | 6 ++ contrib/tools/server/README.md | 25 +++++ contrib/tools/server/server.c | 169 +++++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 contrib/tools/server/Makefile create mode 100644 contrib/tools/server/README.md create mode 100644 contrib/tools/server/server.c diff --git a/contrib/tools/server/Makefile b/contrib/tools/server/Makefile new file mode 100644 index 00000000000..d7c7d758686 --- /dev/null +++ b/contrib/tools/server/Makefile @@ -0,0 +1,6 @@ +server: + $(CC) -g -o server server.c + +clean: + rm server + diff --git a/contrib/tools/server/README.md b/contrib/tools/server/README.md new file mode 100644 index 00000000000..f962c2c640b --- /dev/null +++ b/contrib/tools/server/README.md @@ -0,0 +1,25 @@ +# Sample TCP Server + +This is a simple TCP server that listens on the specified port (3490 by default) and replies to incoming connections by providing info about src/destination ip/port. + +It demonstrates the use of the getsockopt() system call with the ORIGINAL_DST option to retrieve the original destination ip/port after an iptables redirect. + +So, for example, if you have the server listening on port 3490 on the local machine and an iptables rule like: + +``` +iptables -t nat -I OUTPUT 1 -p tcp --dport 4000:5000 -j REDIRECT --to-port 3490 +``` +your will see: + +``` +$ telnet localhost 3490 +FROM 127.0.0.1:44978, TO 127.0.0.1:3490, ORIG DEST 127.0.0.1:3490 + +$ telnet localhost 4100 +FROM 127.0.0.1:35476, TO 127.0.0.1:3490, ORIG DEST 127.0.0.1:4100 + +$ telnet 1.1.1.1 5000 +FROM 100.100.100.100:60275, TO 127.0.0.1:3490, ORIG DEST 1.1.1.1:5000 +``` + + diff --git a/contrib/tools/server/server.c b/contrib/tools/server/server.c new file mode 100644 index 00000000000..77a2d97e004 --- /dev/null +++ b/contrib/tools/server/server.c @@ -0,0 +1,169 @@ +/* +** server.c + * Demo server to check that we can extract all parameters from an incoming +*request + * and respond to it +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFAULT_PORT \ + "3490" // the port users will be connecting to, if not specified on command + // line + +#define QUEUE_SIZE 10 // pending connections queue size + +void sigchld_handler(int s) { + int saved_errno = errno; + + while (waitpid(-1, NULL, WNOHANG) > 0) + ; + + errno = saved_errno; +} + +int main(int argc, char *argv[]) { + struct sigaction sa; + const int yes = 1; + char *port = DEFAULT_PORT; + int rv; + + if (argc > 1) { + port = argv[1]; + } + + struct addrinfo hints; + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; // use my IP + + struct addrinfo *server_info = NULL; + if ((rv = getaddrinfo(NULL, port, &hints, &server_info)) != 0) { + fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv)); + return 1; + } + + struct addrinfo *p; + int sockfd; // listen on sock_fd + for (p = server_info; p != NULL; p = p->ai_next) { + if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1) { + perror("server: socket"); + continue; + } + + if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) { + perror("setsockopt"); + exit(1); + } + + if (bind(sockfd, p->ai_addr, p->ai_addrlen) == -1) { + close(sockfd); + perror("server: bind"); + continue; + } + + break; + } + + freeaddrinfo(server_info); + + if (p == NULL) { + fprintf(stderr, "server: failed to bind\n"); + exit(1); + } + if (listen(sockfd, QUEUE_SIZE) == -1) { + perror("listen"); + exit(1); + } + + sa.sa_handler = sigchld_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + if (sigaction(SIGCHLD, &sa, NULL) == -1) { + perror("sigaction"); + exit(1); + } + + char bind_addr_str[INET6_ADDRSTRLEN] = {0}; + struct sockaddr_in *bind_sock_addr = (struct sockaddr_in *)p->ai_addr; + inet_ntop(p->ai_family, &bind_sock_addr->sin_addr, bind_addr_str, + p->ai_addrlen); + printf("server %s: waiting for connections on port %s:%u...\n", port, + bind_addr_str, ntohs(bind_sock_addr->sin_port)); + + while (1) { + socklen_t addr_len = sizeof(struct sockaddr_storage); + int addr_str_len = INET6_ADDRSTRLEN; + + struct sockaddr_storage their_addr = { + 0}; // connector's address information + int accepted_fd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_len); + if (accepted_fd == -1) { + perror("accept"); + continue; + } + + struct sockaddr_in *src_sock_addr = (struct sockaddr_in *)&their_addr; + char their_addr_str[INET6_ADDRSTRLEN] = {0}; + inet_ntop(their_addr.ss_family, &src_sock_addr->sin_addr, their_addr_str, + addr_str_len); + printf("server %s: got connection FROM %s:%u\n", port, their_addr_str, + ntohs(src_sock_addr->sin_port)); + + struct sockaddr_storage my_addr = {0}; // my address information + struct sockaddr_in *dst_sock_addr = (struct sockaddr_in *)&my_addr; + char my_addr_str[INET6_ADDRSTRLEN] = {0}; + getsockname(accepted_fd, (struct sockaddr *)dst_sock_addr, &addr_len); + inet_ntop(my_addr.ss_family, &dst_sock_addr->sin_addr, my_addr_str, + addr_str_len); + printf("server %s: got connection TO %s:%u\n", port, my_addr_str, + ntohs(dst_sock_addr->sin_port)); + + struct sockaddr_storage orig_addr = {0}; // orig address information + struct sockaddr_in *orig_sock_addr = (struct sockaddr_in *)&orig_addr; + char orig_addr_str[INET6_ADDRSTRLEN] = {0}; + int status = getsockopt(accepted_fd, SOL_IP, SO_ORIGINAL_DST, + orig_sock_addr, &addr_len); + + if (status == 0) { + inet_ntop(orig_addr.ss_family, &orig_sock_addr->sin_addr, orig_addr_str, + addr_str_len); + printf("server %s: ORIG DEST %s:%u\n", port, orig_addr_str, + ntohs(orig_sock_addr->sin_port)); + } else { + printf("Could not get orig destination from accepted socket.\n"); + } + + if (!fork()) { // this is the child process + + close(sockfd); + char msg[256] = {0}; + snprintf(msg, 256, "FROM %s:%u, TO %s:%u, ORIG DEST %s:%u\n", + their_addr_str, ntohs(src_sock_addr->sin_port), my_addr_str, + ntohs(dst_sock_addr->sin_port), orig_addr_str, + ntohs(orig_sock_addr->sin_port)); + + if (send(accepted_fd, msg, strlen(msg), 0) == -1) { + perror("send"); + } + close(accepted_fd); + exit(0); + } + close(accepted_fd); + } + + return 0; +} From 8b3a4429965ed1e792704e795e5d22782eca505b Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 12 Jan 2017 11:23:47 -0800 Subject: [PATCH 02/49] Rebase Envoy (#41) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 56fabce07ac..4460b4c1c9d 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "6b1336a786ebe56c45a1a349ddf706e0526c1ec1", # 2017-01-03 + commit = "38990918a8672a0332b1ca2e048db90a5166d678", build_file_content = BUILD, ) From c3abd2aeb2b43e2c68b71b48449a5fa0de0f2fb8 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 12 Jan 2017 14:52:56 -0800 Subject: [PATCH 03/49] Update prototype to use iptables (#42) --- src/envoy/prototype/README.md | 13 +++++++++++++ src/envoy/prototype/envoy-esp.conf | 11 +++++++++-- test/backend/echo/echo.go | 21 +++++++-------------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md index 2ddbe98459c..00e73a169d6 100644 --- a/src/envoy/prototype/README.md +++ b/src/envoy/prototype/README.md @@ -36,6 +36,19 @@ This Proxy will use Envoy and talk to Mixer server. go run echo.go ``` +* Modify your iptables: + +``` + sudo iptables -t nat -A OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 +``` + +Once you are done, you should remove this rule: + +``` + sudo iptables -t nat -D OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 +``` + + * Start Envoy proxy, run ``` diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index dff3bd67b1a..11690bed565 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -1,7 +1,14 @@ { "listeners": [ + { + "port": 9092, + "bind_to_port": true, + "use_original_dst": true, + "filters": [] + }, { "port": 9090, + "bind_to_port": false, "filters": [ { "type": "read", @@ -26,7 +33,7 @@ }, "access_log": [ { - "path": "/tmp/access.envoy" + "path": "/dev/stdout" } ], "filters": [ @@ -50,7 +57,7 @@ } ], "admin": { - "access_log_path": "/tmp/access.envoy", + "access_log_path": "/dev/stdout", "port": 9001 }, "cluster_manager": { diff --git a/test/backend/echo/echo.go b/test/backend/echo/echo.go index 51aaa07e353..ef13f515ff1 100644 --- a/test/backend/echo/echo.go +++ b/test/backend/echo/echo.go @@ -23,19 +23,22 @@ import ( "io/ioutil" "net/http" "strconv" - "sync" - "time" ) var ( port = flag.Int("port", 8080, "default http port") - mu sync.Mutex requests = 0 data = 0 ) func handler(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%v %v %v %v\n", r.Method, r.URL, r.Proto, r.RemoteAddr) + for name, headers := range r.Header { + for _, h := range headers { + fmt.Printf("%v: %v\n", name, h) + } + } body, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -51,24 +54,14 @@ func handler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(body) - mu.Lock() requests++ data += len(body) - defer mu.Unlock() + fmt.Printf("Requests Requests: %v Data: %v\n", requests, data) } func main() { flag.Parse() - go func() { - for { - mu.Lock() - fmt.Printf("Requests Requests: %v Data: %v\n", requests, data) - mu.Unlock() - time.Sleep(time.Second) - } - }() - fmt.Printf("Listening on port %v\n", *port) http.HandleFunc("/", handler) From add363a037389ebf985defa8898465b3269e380f Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 12 Jan 2017 20:18:53 -0800 Subject: [PATCH 04/49] Rebase to fixed Envoy (#43) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 4460b4c1c9d..dd36a23d456 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "38990918a8672a0332b1ca2e048db90a5166d678", + commit = "c36bb76e304ab4c5584a07303e2953a170888319", build_file_content = BUILD, ) From ee3cdeeb33b1de0b87c80809802de5246cbb3e25 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Fri, 13 Jan 2017 08:49:25 -0800 Subject: [PATCH 05/49] Handle HEAD request. (#34) * Handle HEAD request. * Try with GET if HEAD fails. * Address comments. * Format file. --- .../src/api_manager/context/service_context.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index b9f753f0625..d8fc9dfc409 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -41,6 +41,9 @@ const double kDefaultTraceSampleQps = 0.1; // The time window to send intermediate report for Grpc streaming (second). // Default to 10s. const int kIntermediateReportInterval = 10; + +const char kHTTPHeadMethod[] = "HEAD"; +const char kHTTPGetMethod[] = "GET"; } ServiceContext::ServiceContext(std::unique_ptr env, @@ -74,7 +77,15 @@ MethodCallInfo ServiceContext::GetMethodCallInfo( if (config_ == nullptr) { return MethodCallInfo(); } - return config_->GetMethodCallInfo(http_method, url, query_params); + MethodCallInfo method_call_info = + config_->GetMethodCallInfo(http_method, url, query_params); + // HEAD should be treated as GET unless it is specified from service_config. + if (method_call_info.method_info == nullptr && + http_method == kHTTPHeadMethod) { + method_call_info = + config_->GetMethodCallInfo(kHTTPGetMethod, url, query_params); + } + return method_call_info; } const std::string& ServiceContext::project_id() const { From efc32a2a83434e7169f00c56c08c4b7361febf3b Mon Sep 17 00:00:00 2001 From: Kuat Date: Fri, 20 Jan 2017 16:39:05 -0800 Subject: [PATCH 06/49] Expose bazel target (#48) --- src/envoy/prototype/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/envoy/prototype/BUILD b/src/envoy/prototype/BUILD index e5cbaeba10c..5c8ab20e27a 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/prototype/BUILD @@ -15,6 +15,8 @@ ################################################################################ # +package(default_visibility = [":__subpackages__"]) + cc_binary( name = "envoy_esp", srcs = [ From 92541b79f45fb3ada59e36d3abd87bd7724b27f9 Mon Sep 17 00:00:00 2001 From: Kuat Date: Fri, 20 Jan 2017 16:51:15 -0800 Subject: [PATCH 07/49] Try again (#49) --- src/envoy/prototype/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/prototype/BUILD b/src/envoy/prototype/BUILD index 5c8ab20e27a..258f2046cd7 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/prototype/BUILD @@ -15,7 +15,7 @@ ################################################################################ # -package(default_visibility = [":__subpackages__"]) +package(default_visibility = ["//visibility:public"]) cc_binary( name = "envoy_esp", From cea882799327dbea20ce849be36088a349a35f18 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 26 Jan 2017 22:02:18 -0800 Subject: [PATCH 08/49] Integrate with mixer client. (#55) * Integrate with mixer client. * Restore repositories.bzl back. --- WORKSPACE | 22 +- contrib/endpoints/repositories.bzl | 52 +--- contrib/endpoints/src/api_manager/mixer/BUILD | 6 +- .../endpoints/src/api_manager/mixer/mixer.cc | 255 ++++++------------ .../endpoints/src/api_manager/mixer/mixer.h | 6 +- src/envoy/prototype/api_manager_filter.cc | 9 +- src/envoy/prototype/envoy-esp.conf | 12 - src/envoy/prototype/server_config.pb.txt | 4 +- 8 files changed, 129 insertions(+), 237 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index bbafd205918..95dcdde5cdc 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,16 +31,34 @@ googletest_repositories() load( "//contrib/endpoints:repositories.bzl", "grpc_repositories", - "mixerapi_repositories", + "mixer_client_repositories", "servicecontrol_client_repositories", ) grpc_repositories() -mixerapi_repositories() +mixer_client_repositories() servicecontrol_client_repositories() +# Workaround for Bazel > 0.4.0 since it needs newer protobuf.bzl from: +# https://github.com/google/protobuf/pull/2246 +# Do not use this git_repository for anything else than protobuf.bzl +new_git_repository( + name = "protobuf_bzl", + # Injecting an empty BUILD file to prevent using any build target + build_file_content = "", + commit = "05090726144b6e632c50f47720ff51049bfcbef6", + remote = "https://github.com/google/protobuf.git", +) + +load( + "@mixerclient_git//:repositories.bzl", + "mixerapi_repositories", +) + +mixerapi_repositories(protobuf_repo="@protobuf_bzl//") + load( "//src/envoy:repositories.bzl", "envoy_repositories", diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index f3df61722a2..7b77aa91c5a 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -334,51 +334,15 @@ def servicecontrol_client_repositories(bind=True): actual = "@servicecontrol_client_git//:service_control_client_lib", ) -def mixerapi_repositories(protobuf_repo="@protobuf_git//", bind=True): - BUILD = """ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -licenses(["notice"]) - -load("{}:protobuf.bzl", "cc_proto_library") - -cc_proto_library( - name = "mixer_api_cc_proto", - srcs = glob( - ["mixer/api/v1/*.proto"], - ), - default_runtime = "//external:protobuf", - protoc = "//external:protoc", - visibility = ["//visibility:public"], - deps = [ - "//external:cc_wkt_protos", - "//external:servicecontrol", - ], -) -""".format(protobuf_repo) - - native.new_git_repository( - name = "mixerapi_git", - commit = "fc5a396185edc72d06d1937f30a8148a37d4fc1b", - remote = "https://github.com/istio/api.git", - build_file_content = BUILD, +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = "1569430f1e27b31e23c029c6bec0d8d5062d9e55", + remote = "https://github.com/istio/mixerclient.git", ) + if bind: native.bind( - name = "mixer_api_cc_proto", - actual = "@mixerapi_git//:mixer_api_cc_proto", + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", ) diff --git a/contrib/endpoints/src/api_manager/mixer/BUILD b/contrib/endpoints/src/api_manager/mixer/BUILD index 8082c61e201..affe09c7ddd 100644 --- a/contrib/endpoints/src/api_manager/mixer/BUILD +++ b/contrib/endpoints/src/api_manager/mixer/BUILD @@ -32,11 +32,11 @@ cc_library( ], }), deps = [ - "//external:grpc++", - "//external:mixer_api_cc_proto", - "//external:protobuf", "//contrib/endpoints/src/api_manager:impl_headers", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", + "//external:grpc++", + "//external:mixer_client_lib", + "//external:protobuf", ], ) diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc index 0c0bb5d793e..47f6c4a8a2f 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ b/contrib/endpoints/src/api_manager/mixer/mixer.cc @@ -14,169 +14,103 @@ */ #include "contrib/endpoints/src/api_manager/mixer/mixer.h" -#include -#include "mixer/api/v1/service.pb.h" - using ::google::api_manager::utils::Status; -using ::google::protobuf::util::error::Code; -using ::google::protobuf::Map; +using ::istio::mixer_client::Attributes; namespace google { namespace api_manager { namespace mixer { namespace { -const char kMixerServiceName[] = "istio.mixer.v1.Mixer"; - -enum AttributeIndex { - ATTR_SERVICE_NAME = 0, - ATTR_PEER_ID, - ATTR_OPERATION_NAME, - ATTR_API_KEY, - ATTR_RESPONSE_CODE, - ATTR_URL, - ATTR_LOCATION, - ATTR_API_NAME, - ATTR_API_VERSION, - ATTR_API_METHOD, - ATTR_REQUEST_SIZE, - ATTR_RESPONSE_SIZE, - ATTR_LOG_MESSAGE, -}; - -struct AttributeDict { - int index; - std::string name; -} kAttributeNames[] = { - { - ATTR_SERVICE_NAME, "serviceName", - }, - { - ATTR_PEER_ID, "peerId", - }, - { - ATTR_OPERATION_NAME, "operationName", - }, - { - ATTR_API_KEY, "apiKey", - }, - { - ATTR_RESPONSE_CODE, "responseCode", - }, - { - ATTR_URL, "URL", - }, - { - ATTR_LOCATION, "location", - }, - { - ATTR_API_NAME, "apiName", - }, - { - ATTR_API_VERSION, "apiVersion", - }, - { - ATTR_API_METHOD, "apiMethod", - }, - { - ATTR_REQUEST_SIZE, "requestSize", - }, - { - ATTR_RESPONSE_SIZE, "responesSize", - }, - { - ATTR_LOG_MESSAGE, "logMessage", - }, -}; - -void SetAttributeDict(Map* dict) { - for (auto attr : kAttributeNames) { - (*dict)[attr.index] = attr.name; - } +const std::string kAttrNameServiceName = "serviceName"; +const std::string kAttrNamePeerId = "peerId"; +const std::string kAttrNameOperationName = "operationName"; +const std::string kAttrNameApiKey = "apiKey"; +const std::string kAttrNameResponseCode = "responseCode"; +const std::string kAttrNameURL = "url"; +const std::string kAttrNameLocation = "location"; +const std::string kAttrNameApiName = "apiName"; +const std::string kAttrNameApiVersion = "apiVersion"; +const std::string kAttrNameApiMethod = "apiMethod"; +const std::string kAttrNameRequestSize = "requestSize"; +const std::string kAttrNameResponseSize = "responseSize"; +const std::string kAttrNameLogMessage = "logMessage"; + +Attributes::Value StringValue(const std::string& str) { + Attributes::Value v; + v.type = Attributes::Value::STRING; + v.str_v = str; + return v; } -void CovertToPb(const service_control::CheckRequestInfo& info, - const std::string& service_name, - ::istio::mixer::v1::Attributes* attr) { - SetAttributeDict(attr->mutable_dictionary()); - - auto* str_attrs = attr->mutable_string_attributes(); - (*str_attrs)[ATTR_SERVICE_NAME] = service_name; - (*str_attrs)[ATTR_PEER_ID] = "Proxy"; - (*str_attrs)[ATTR_OPERATION_NAME] = info.operation_name; - (*str_attrs)[ATTR_API_KEY] = info.api_key; +Attributes::Value Int64Value(int64_t value) { + Attributes::Value v; + v.type = Attributes::Value::INT64; + v.value.int64_v = value; + return v; } -void CovertToPb(const service_control::ReportRequestInfo& info, - const std::string& service_name, - ::istio::mixer::v1::Attributes* attr) { - SetAttributeDict(attr->mutable_dictionary()); +void FillCheckAttributes(const service_control::CheckRequestInfo& info, + const std::string& service_name, + ::istio::mixer_client::Attributes* attr) { + attr->attributes[kAttrNameServiceName] = StringValue(service_name); + attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); + attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); + attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); +} - auto* str_attrs = attr->mutable_string_attributes(); - (*str_attrs)[ATTR_SERVICE_NAME] = service_name; - (*str_attrs)[ATTR_PEER_ID] = "Proxy"; - (*str_attrs)[ATTR_OPERATION_NAME] = info.operation_name; - (*str_attrs)[ATTR_API_KEY] = info.api_key; +void FillReportAttributes(const service_control::ReportRequestInfo& info, + const std::string& service_name, + ::istio::mixer_client::Attributes* attr) { + attr->attributes[kAttrNameServiceName] = StringValue(service_name); + attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); + attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); + attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - (*str_attrs)[ATTR_URL] = info.url; - (*str_attrs)[ATTR_LOCATION] = info.location; + attr->attributes[kAttrNameURL] = StringValue(info.url); + attr->attributes[kAttrNameLocation] = StringValue(info.location); - (*str_attrs)[ATTR_API_NAME] = info.api_name; - (*str_attrs)[ATTR_API_VERSION] = info.api_version; - (*str_attrs)[ATTR_API_METHOD] = info.api_method; + attr->attributes[kAttrNameApiName] = StringValue(info.api_name); + attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); + attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - (*str_attrs)[ATTR_LOG_MESSAGE] = info.log_message; + attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - auto* int_attrs = attr->mutable_int64_attributes(); - (*int_attrs)[ATTR_RESPONSE_CODE] = info.response_code; - (*int_attrs)[ATTR_REQUEST_SIZE] = info.request_size; - (*int_attrs)[ATTR_RESPONSE_SIZE] = info.response_size; + attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); + attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); + attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); } } // namespace Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) - : env_(env), request_index_(0), config_(config) {} + : env_(env), config_(config) {} Mixer::~Mixer() {} -Status Mixer::Init() { return Status::OK; } +Status Mixer::Init() { + ::istio::mixer_client::MixerClientOptions options; + options.mixer_server = + config_->server_config()->mixer_options().mixer_server(); + mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + return Status::OK; +} Status Mixer::Close() { return Status::OK; } Status Mixer::Report(const service_control::ReportRequestInfo& info) { - std::unique_ptr grpc_request(new GRPCRequest([this]( - Status status, std::string&& body) { - if (status.ok()) { - // Handle 200 response - ::istio::mixer::v1::ReportResponse response; - if (!response.ParseFromString(body)) { - status = - Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); - env_->LogError(std::string("Failed parse report response: ") + body); - } - env_->LogInfo(std::string("Report response: ") + response.DebugString()); - } else { - env_->LogError(std::string("Failed to call Mixer::report, Error: ") + - status.ToString()); - } - })); - - ::istio::mixer::v1::ReportRequest request; - request.set_request_index(++request_index_); - CovertToPb(info, config_->service_name(), request.mutable_attribute_update()); - env_->LogInfo(std::string("Send Report: ") + request.DebugString()); - - std::string request_body; - request.SerializeToString(&request_body); - - grpc_request - ->set_server(config_->server_config()->mixer_options().mixer_server()) - .set_service(kMixerServiceName) - .set_method("Report") - .set_body(request_body); - - env_->RunGRPCRequest(std::move(grpc_request)); + ::istio::mixer_client::Attributes attributes; + FillReportAttributes(info, config_->service_name(), &attributes); + env_->LogInfo("Send Report: "); + mixer_client_->Report( + attributes, [this](const ::google::protobuf::util::Status& status) { + if (status.ok()) { + env_->LogInfo("Report response: OK"); + } else { + env_->LogError(std::string("Failed to call Mixer::report, Error: ") + + status.ToString()); + } + }); return Status::OK; } @@ -185,40 +119,23 @@ void Mixer::Check( cloud_trace::CloudTraceSpan* parent_span, std::function on_done) { - std::unique_ptr grpc_request(new GRPCRequest([this, on_done]( - Status status, std::string&& body) { - if (status.ok()) { - // Handle 200 response - ::istio::mixer::v1::CheckResponse response; - if (!response.ParseFromString(body)) { - status = - Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); - env_->LogError(std::string("Failed parse check response: ") + body); - } - env_->LogInfo(std::string("Check response: ") + response.DebugString()); - } else { - env_->LogError(std::string("Failed to call Mixer::check, Error: ") + - status.ToString()); - } - service_control::CheckResponseInfo info; - on_done(status, info); - })); - - ::istio::mixer::v1::CheckRequest request; - request.set_request_index(++request_index_); - CovertToPb(info, config_->service_name(), request.mutable_attribute_update()); - env_->LogInfo(std::string("Send Check: ") + request.DebugString()); - - std::string request_body; - request.SerializeToString(&request_body); - - grpc_request - ->set_server(config_->server_config()->mixer_options().mixer_server()) - .set_service(kMixerServiceName) - .set_method("Check") - .set_body(request_body); - - env_->RunGRPCRequest(std::move(grpc_request)); + ::istio::mixer_client::Attributes attributes; + FillCheckAttributes(info, config_->service_name(), &attributes); + env_->LogInfo("Send Check: "); + mixer_client_->Check( + attributes, + [this, on_done](const ::google::protobuf::util::Status& status) { + if (status.ok()) { + env_->LogInfo("Check response: OK"); + } else { + env_->LogError(std::string("Failed to call Mixer::check, Error: ") + + status.ToString()); + } + service_control::CheckResponseInfo info; + on_done(Status(status.error_code(), status.error_message(), + Status::SERVICE_CONTROL), + info); + }); } Status Mixer::GetStatistics(service_control::Statistics* esp_stat) const { diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h index 0318a5f06f1..4cf2d1f54a7 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ b/contrib/endpoints/src/api_manager/mixer/mixer.h @@ -18,6 +18,7 @@ #include "contrib/endpoints/include/api_manager/env_interface.h" #include "contrib/endpoints/src/api_manager/config.h" #include "contrib/endpoints/src/api_manager/service_control/interface.h" +#include "include/client.h" namespace google { namespace api_manager { @@ -51,9 +52,10 @@ class Mixer : public service_control::Interface { // The Api Manager environment interface. ApiManagerEnvInterface* env_; - int64_t request_index_; - + // The config. const Config* config_; + // The mixer client + std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; }; } // namespace mixer diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 2fca48de23c..4ccd9e6d120 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -214,13 +214,16 @@ class Instance : public Http::StreamFilter, status.ToJson()); if (!status.ok() && state_ != Responded) { state_ = Responded; - Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), - status.ToJson()); + decoder_callbacks_->dispatcher().post([this, status]() { + Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), + status.ToJson()); + }); return; } state_ = Complete; if (!initiating_call_) { - decoder_callbacks_->continueDecoding(); + decoder_callbacks_->dispatcher().post( + [this]() { decoder_callbacks_->continueDecoding(); }); } } diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index 11690bed565..d3ec54f405d 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -73,18 +73,6 @@ } ] }, - { - "name": "mixer_server", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "features": "http2", - "hosts": [ - { - "url": "tcp://localhost:9091" - } - ] - }, { "name": "api_manager", "connect_timeout_ms": 5000, diff --git a/src/envoy/prototype/server_config.pb.txt b/src/envoy/prototype/server_config.pb.txt index 9703886cc9e..19cb6cbb2c0 100644 --- a/src/envoy/prototype/server_config.pb.txt +++ b/src/envoy/prototype/server_config.pb.txt @@ -2,5 +2,5 @@ cloud_tracing_config { force_disable: true } mixer_options { - mixer_server: "mixer_server" -} \ No newline at end of file + mixer_server: "localhost:9091" +} From 6d7f0eebec4d00856987f9e8f55e4b9b4ddd8455 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 26 Jan 2017 23:38:51 -0800 Subject: [PATCH 09/49] Add originIp and originHost attributes. (#56) --- .../endpoints/include/api_manager/request.h | 2 + .../api_manager/context/request_context.cc | 3 +- .../endpoints/src/api_manager/mixer/mixer.cc | 107 ++++++++++++------ .../endpoints/src/api_manager/mixer/mixer.h | 10 ++ .../src/api_manager/service_control/info.h | 8 +- src/envoy/prototype/api_manager_filter.cc | 1 + 6 files changed, 95 insertions(+), 36 deletions(-) diff --git a/contrib/endpoints/include/api_manager/request.h b/contrib/endpoints/include/api_manager/request.h index ad604b5a41b..8d96ff356d6 100644 --- a/contrib/endpoints/include/api_manager/request.h +++ b/contrib/endpoints/include/api_manager/request.h @@ -43,6 +43,8 @@ class Request { // Gets Client IP // This will be used by service control Check() call. virtual std::string GetClientIP() = 0; + // Gets Client Host. + virtual std::string GetClientHost() { return ""; } // Get GRPC stats. virtual int64_t GetGrpcRequestBytes() = 0; diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index cbf4e925e37..59cc7c3d53f 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -171,6 +171,8 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->producer_project_id = service_context()->project_id(); info->referer = http_referer_; info->request_start_time = start_time_; + info->client_ip = request_->GetClientIP(); + info->client_host = request_->GetClientHost(); } void RequestContext::FillLocation(service_control::ReportRequestInfo *info) { @@ -221,7 +223,6 @@ void RequestContext::FillLogMessage(service_control::ReportRequestInfo *info) { void RequestContext::FillCheckRequestInfo( service_control::CheckRequestInfo *info) { FillOperationInfo(info); - info->client_ip = request_->GetClientIP(); info->allow_unregistered_calls = method()->allow_unregistered_calls(); } diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc index 47f6c4a8a2f..e9ea9a30ea5 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ b/contrib/endpoints/src/api_manager/mixer/mixer.cc @@ -22,8 +22,11 @@ namespace api_manager { namespace mixer { namespace { +const std::string kProxyPeerID = "Istio/Proxy"; + const std::string kAttrNameServiceName = "serviceName"; const std::string kAttrNamePeerId = "peerId"; +const std::string kAttrNameOperationId = "operationId"; const std::string kAttrNameOperationName = "operationName"; const std::string kAttrNameApiKey = "apiKey"; const std::string kAttrNameResponseCode = "responseCode"; @@ -35,6 +38,9 @@ const std::string kAttrNameApiMethod = "apiMethod"; const std::string kAttrNameRequestSize = "requestSize"; const std::string kAttrNameResponseSize = "responseSize"; const std::string kAttrNameLogMessage = "logMessage"; +const std::string kAttrNameResponseTime = "responseTime"; +const std::string kAttrNameOriginIp = "originIp"; +const std::string kAttrNameOriginHost = "originHost"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -50,37 +56,6 @@ Attributes::Value Int64Value(int64_t value) { return v; } -void FillCheckAttributes(const service_control::CheckRequestInfo& info, - const std::string& service_name, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(service_name); - attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); -} - -void FillReportAttributes(const service_control::ReportRequestInfo& info, - const std::string& service_name, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(service_name); - attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - - attr->attributes[kAttrNameURL] = StringValue(info.url); - attr->attributes[kAttrNameLocation] = StringValue(info.location); - - attr->attributes[kAttrNameApiName] = StringValue(info.api_name); - attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); - attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - - attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - - attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); - attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); - attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); -} - } // namespace Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) @@ -98,9 +73,75 @@ Status Mixer::Init() { Status Mixer::Close() { return Status::OK; } +void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, + ::istio::mixer_client::Attributes* attr) { + attr->attributes[kAttrNameServiceName] = StringValue(config_->service_name()); + attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); + + if (!info.operation_id.empty()) { + attr->attributes[kAttrNameOperationId] = StringValue(info.operation_id); + } + if (!info.operation_name.empty()) { + attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); + } + if (!info.api_key.empty()) { + attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); + } + if (!info.client_ip.empty()) { + attr->attributes[kAttrNameOriginIp] = StringValue(info.client_ip); + } + if (!info.client_host.empty()) { + attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); + } +} + +void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, + ::istio::mixer_client::Attributes* attr) { + FillCommonAttributes(info, attr); +} + +void Mixer::FillReportAttributes(const service_control::ReportRequestInfo& info, + ::istio::mixer_client::Attributes* attr) { + FillCommonAttributes(info, attr); + + if (!info.url.empty()) { + attr->attributes[kAttrNameURL] = StringValue(info.url); + } + if (!info.location.empty()) { + attr->attributes[kAttrNameLocation] = StringValue(info.location); + } + + if (!info.api_name.empty()) { + attr->attributes[kAttrNameApiName] = StringValue(info.api_name); + } + if (!info.api_version.empty()) { + attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); + } + if (!info.api_method.empty()) { + attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); + } + + if (!info.log_message.empty()) { + attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); + } + + attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); + if (info.request_size >= 0) { + attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); + } + if (info.response_size >= 0) { + attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); + } + + if (info.latency.request_time_ms >= 0) { + attr->attributes[kAttrNameResponseTime] = + Int64Value(info.latency.request_time_ms); + } +} + Status Mixer::Report(const service_control::ReportRequestInfo& info) { ::istio::mixer_client::Attributes attributes; - FillReportAttributes(info, config_->service_name(), &attributes); + FillReportAttributes(info, &attributes); env_->LogInfo("Send Report: "); mixer_client_->Report( attributes, [this](const ::google::protobuf::util::Status& status) { @@ -120,7 +161,7 @@ void Mixer::Check( std::function on_done) { ::istio::mixer_client::Attributes attributes; - FillCheckAttributes(info, config_->service_name(), &attributes); + FillCheckAttributes(info, &attributes); env_->LogInfo("Send Check: "); mixer_client_->Check( attributes, diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h index 4cf2d1f54a7..b020d2636f9 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ b/contrib/endpoints/src/api_manager/mixer/mixer.h @@ -50,6 +50,16 @@ class Mixer : public service_control::Interface { // The constructor. Mixer(ApiManagerEnvInterface* env, const Config* config); + // Fill common attributes for both check and report. + void FillCommonAttributes(const service_control::OperationInfo& info, + ::istio::mixer_client::Attributes* attr); + // Fill attributes for check. + void FillCheckAttributes(const service_control::CheckRequestInfo& info, + ::istio::mixer_client::Attributes* attr); + // Fill attributes for report. + void FillReportAttributes(const service_control::ReportRequestInfo& info, + ::istio::mixer_client::Attributes* attr); + // The Api Manager environment interface. ApiManagerEnvInterface* env_; // The config. diff --git a/contrib/endpoints/src/api_manager/service_control/info.h b/contrib/endpoints/src/api_manager/service_control/info.h index 15364ad58c6..f203057cc9e 100644 --- a/contrib/endpoints/src/api_manager/service_control/info.h +++ b/contrib/endpoints/src/api_manager/service_control/info.h @@ -60,13 +60,17 @@ struct OperationInfo { // and Report. std::chrono::system_clock::time_point request_start_time; + // The client IP address. + std::string client_ip; + + // The client host name. + std::string client_host; + OperationInfo() {} }; // Information to fill Check request protobuf. struct CheckRequestInfo : public OperationInfo { - // The client IP address. - std::string client_ip; // Whether the method allow unregistered calls. bool allow_unregistered_calls; diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 4ccd9e6d120..869d8e6c500 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -81,6 +81,7 @@ class Request : public google::api_manager::Request { return header_map_.Path()->value().c_str(); } virtual std::string GetClientIP() override { return ""; } + virtual std::string GetClientHost() override { return ""; } virtual bool FindQuery(const std::string& name, std::string* query) override { if (!query_parsed_) { auto header = header_map_.Path(); From 4cd5c9f1d7c8367239021c35b9d72229a82353de Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 27 Jan 2017 17:54:34 -0800 Subject: [PATCH 10/49] Add uuid-dev dependency in README.md (#45) --- src/envoy/prototype/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md index 00e73a169d6..dc73f63f6c9 100644 --- a/src/envoy/prototype/README.md +++ b/src/envoy/prototype/README.md @@ -1,6 +1,11 @@ This Proxy will use Envoy and talk to Mixer server. +## Install dependencies + +``` + apt-get install uuid-dev +``` ## Build Mixer server From bcdb75d2703050cb3d7b9b11b242a2304068cf90 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Sat, 28 Jan 2017 14:15:11 -0800 Subject: [PATCH 11/49] Extract originIp and OriginHost. (#57) * Extract originIp and OriginHost. * Make header x-forwarded-host const. --- src/envoy/prototype/api_manager_filter.cc | 56 +++++++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 869d8e6c500..d616ddfdf0f 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -13,6 +13,12 @@ namespace Http { namespace ApiManager { +namespace { + +// Define lower case string for X-Forwarded-Host. +const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); + +} // namespace std::string ReadFile(const std::string& file_name) { std::ifstream t(file_name); @@ -65,12 +71,18 @@ typedef std::shared_ptr ConfigPtr; class Request : public google::api_manager::Request { private: HeaderMap& header_map_; + std::string downstream_address_; + std::string virtual_host_; bool query_parsed_; std::map query_params_; public: - Request(HeaderMap& header_map) - : header_map_(header_map), query_parsed_(false) {} + Request(HeaderMap& header_map, const std::string& downstream_address, + const std::string& virtual_host) + : header_map_(header_map), + downstream_address_(downstream_address), + virtual_host_(virtual_host), + query_parsed_(false) {} virtual std::string GetRequestHTTPMethod() override { return header_map_.Method()->value().c_str(); } @@ -80,8 +92,31 @@ class Request : public google::api_manager::Request { virtual std::string GetUnparsedRequestPath() override { return header_map_.Path()->value().c_str(); } - virtual std::string GetClientIP() override { return ""; } - virtual std::string GetClientHost() override { return ""; } + + virtual std::string GetClientIP() override { + if (!header_map_.ForwardedFor()) { + return downstream_address_; + } + std::vector xff_address_list = + StringUtil::split(header_map_.ForwardedFor()->value().c_str(), ','); + if (xff_address_list.empty()) { + return downstream_address_; + } + return xff_address_list.front(); + } + + virtual std::string GetClientHost() override { + const HeaderEntry* entry = header_map_.get(kHeaderNameXFH); + if (entry == nullptr) { + return virtual_host_; + } + auto xff_list = StringUtil::split(entry->value().c_str(), ','); + if (xff_list.empty()) { + return virtual_host_; + } + return xff_list.back(); + } + virtual bool FindQuery(const std::string& name, std::string* query) override { if (!query_parsed_) { auto header = header_map_.Path(); @@ -159,6 +194,15 @@ class Instance : public Http::StreamFilter, bool initiating_call_; + std::string getRouteVirtualHost(HeaderMap& headers) const { + const Router::Route* route = + decoder_callbacks_->routeTable().route(headers); + if (route && route->routeEntry()) { + return route->routeEntry()->virtualHost().name(); + } + return ""; + } + public: Instance(ConfigPtr config) : api_manager_(config->api_manager()), @@ -170,7 +214,9 @@ class Instance : public Http::StreamFilter, FilterHeadersStatus decodeHeaders(HeaderMap& headers, bool end_stream) override { log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr request(new Request(headers)); + std::unique_ptr request( + new Request(headers, decoder_callbacks_->downstreamAddress(), + getRouteVirtualHost(headers))); request_handler_ = api_manager_->CreateRequestHandler(std::move(request)); state_ = Calling; initiating_call_ = true; From 2c9c1af3d6ac5ab4d295444c30acd42599edfcb3 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Mon, 30 Jan 2017 15:03:42 -0800 Subject: [PATCH 12/49] Update buckets for UI. (#58) * Update buckets for UI. * Only update time_distribution. --- .../src/api_manager/service_control/proto.cc | 2 +- .../testdata/final_report_request.golden | 150 ++++++++++++++++-- .../testdata/report_request.golden | 150 ++++++++++++++++-- .../testdata/report_request_failed.golden | 75 ++++++++- 4 files changed, 346 insertions(+), 31 deletions(-) diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 17b921db076..3e7eba64d21 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -96,7 +96,7 @@ struct DistributionHelperOptions { double scale; }; -const DistributionHelperOptions time_distribution = {8, 10.0, 1e-6}; +const DistributionHelperOptions time_distribution = {29, 2.0, 1e-6}; const DistributionHelperOptions size_distribution = {8, 10.0, 1}; const double kMsToSecs = 1e-3; diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden index b9eff9f28eb..d60b039987d 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden @@ -201,13 +201,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -227,13 +248,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -253,13 +295,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -279,13 +342,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -304,14 +388,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -330,14 +435,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden index 5d8a266ef88..faf96383418 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden @@ -213,13 +213,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -239,13 +260,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -265,13 +307,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -291,13 +354,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -316,14 +400,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -342,14 +447,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden index a3ffd58c30d..710772cd215 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden @@ -152,13 +152,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -178,13 +199,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -203,14 +245,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } From 27962094adf801685a8547284fb60ca7c75ad1f8 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 31 Jan 2017 16:12:56 -0800 Subject: [PATCH 13/49] Add targetService attribute. (#59) --- contrib/endpoints/repositories.bzl | 2 +- contrib/endpoints/src/api_manager/mixer/mixer.cc | 11 +++++++++++ contrib/endpoints/src/api_manager/mixer/mixer.h | 2 ++ src/envoy/prototype/api_manager_env.cc | 2 +- src/envoy/prototype/api_manager_filter.cc | 3 +-- src/envoy/repositories.bzl | 2 +- 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index 7b77aa91c5a..b1f14aae8b5 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -337,7 +337,7 @@ def servicecontrol_client_repositories(bind=True): def mixer_client_repositories(bind=True): native.git_repository( name = "mixerclient_git", - commit = "1569430f1e27b31e23c029c6bec0d8d5062d9e55", + commit = "80e450a5126960e8e6337c3631cf2ef984038eab", remote = "https://github.com/istio/mixerclient.git", ) diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc index e9ea9a30ea5..53837edaf67 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ b/contrib/endpoints/src/api_manager/mixer/mixer.cc @@ -23,6 +23,7 @@ namespace mixer { namespace { const std::string kProxyPeerID = "Istio/Proxy"; +const std::string kEnvNameTargetService = "TARGET_SERVICE"; const std::string kAttrNameServiceName = "serviceName"; const std::string kAttrNamePeerId = "peerId"; @@ -41,6 +42,7 @@ const std::string kAttrNameLogMessage = "logMessage"; const std::string kAttrNameResponseTime = "responseTime"; const std::string kAttrNameOriginIp = "originIp"; const std::string kAttrNameOriginHost = "originHost"; +const std::string kAttrNameTargetService = "targetService"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -68,6 +70,10 @@ Status Mixer::Init() { options.mixer_server = config_->server_config()->mixer_options().mixer_server(); mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + auto target_service = getenv(kEnvNameTargetService.c_str()); + if (target_service) { + target_service_ = target_service; + } return Status::OK; } @@ -93,6 +99,9 @@ void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, if (!info.client_host.empty()) { attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); } + if (!target_service_.empty()) { + attr->attributes[kAttrNameTargetService] = StringValue(target_service_); + } } void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, @@ -143,6 +152,7 @@ Status Mixer::Report(const service_control::ReportRequestInfo& info) { ::istio::mixer_client::Attributes attributes; FillReportAttributes(info, &attributes); env_->LogInfo("Send Report: "); + env_->LogInfo(attributes.DebugString()); mixer_client_->Report( attributes, [this](const ::google::protobuf::util::Status& status) { if (status.ok()) { @@ -163,6 +173,7 @@ void Mixer::Check( ::istio::mixer_client::Attributes attributes; FillCheckAttributes(info, &attributes); env_->LogInfo("Send Check: "); + env_->LogInfo(attributes.DebugString()); mixer_client_->Check( attributes, [this, on_done](const ::google::protobuf::util::Status& status) { diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h index b020d2636f9..81276e1227d 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ b/contrib/endpoints/src/api_manager/mixer/mixer.h @@ -66,6 +66,8 @@ class Mixer : public service_control::Interface { const Config* config_; // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // Target service + std::string target_service_; }; } // namespace mixer diff --git a/src/envoy/prototype/api_manager_env.cc b/src/envoy/prototype/api_manager_env.cc index 6b1619a6faa..f8536bcc10a 100644 --- a/src/envoy/prototype/api_manager_env.cc +++ b/src/envoy/prototype/api_manager_env.cc @@ -103,7 +103,7 @@ class HTTPRequest : public Http::Message { virtual void body(Buffer::InstancePtr &&body) override {} virtual HeaderMap *trailers() override { return nullptr; } virtual void trailers(HeaderMapPtr &&trailers) override {} - virtual std::string bodyAsString() override { return ""; } + virtual std::string bodyAsString() const override { return ""; } }; class HTTPRequestCallbacks : public AsyncClient::Callbacks { diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index d616ddfdf0f..87a3beeb48a 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -195,8 +195,7 @@ class Instance : public Http::StreamFilter, bool initiating_call_; std::string getRouteVirtualHost(HeaderMap& headers) const { - const Router::Route* route = - decoder_callbacks_->routeTable().route(headers); + const Router::Route* route = decoder_callbacks_->route(); if (route && route->routeEntry()) { return route->routeEntry()->virtualHost().name(); } diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index dd36a23d456..e19efc6dce1 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "c36bb76e304ab4c5584a07303e2953a170888319", + commit = "39f42378fa41c10996d4c3ffba534951de30ceb8", build_file_content = BUILD, ) From 8218f92089a6ac36f851f3cacbf761e443e5f8a1 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 1 Feb 2017 12:08:31 -0800 Subject: [PATCH 14/49] Use envoy new access_log handler for sending Report. (#60) * use access_log handler. * Not to use Loggable base class. --- src/envoy/prototype/api_manager_filter.cc | 61 ++++++++++++++--------- src/envoy/prototype/envoy-esp.conf | 3 +- src/envoy/repositories.bzl | 2 +- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 87a3beeb48a..3983a4d8b41 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -163,24 +163,30 @@ class Request : public google::api_manager::Request { }; class Response : public google::api_manager::Response { + const AccessLog::RequestInfo& request_info_; + + public: + Response(const AccessLog::RequestInfo& request_info) + : request_info_(request_info) {} + google::api_manager::utils::Status GetResponseStatus() { return google::api_manager::utils::Status::OK; } - std::size_t GetRequestSize() { return 0; } + std::size_t GetRequestSize() { return request_info_.bytesReceived(); } - std::size_t GetResponseSize() { return 0; } + std::size_t GetResponseSize() { return request_info_.bytesSent(); } google::api_manager::utils::Status GetLatencyInfo( google::api_manager::service_control::LatencyInfo* info) { + info->request_time_ms = request_info_.duration().count(); return google::api_manager::utils::Status::OK; } }; const Http::HeaderMapImpl BadRequest{{Http::Headers::get().Status, "400"}}; -class Instance : public Http::StreamFilter, - public Logger::Loggable { +class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { private: std::shared_ptr api_manager_; std::unique_ptr @@ -207,12 +213,12 @@ class Instance : public Http::StreamFilter, : api_manager_(config->api_manager()), state_(NotStarted), initiating_call_(false) { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); } FilterHeadersStatus decodeHeaders(HeaderMap& headers, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); std::unique_ptr request( new Request(headers, decoder_callbacks_->downstreamAddress(), getRouteVirtualHost(headers))); @@ -227,13 +233,13 @@ class Instance : public Http::StreamFilter, if (state_ == Complete) { return FilterHeadersStatus::Continue; } - log().debug("Called ApiManager::Instance : {} Stop", __func__); + Log().debug("Called ApiManager::Instance : {} Stop", __func__); return FilterHeadersStatus::StopIteration; } FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override { - log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, + Log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, data.length(), end_stream); if (state_ == Calling) { return FilterDataStatus::StopIterationAndBuffer; @@ -242,7 +248,7 @@ class Instance : public Http::StreamFilter, } FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); if (state_ == Calling) { return FilterTrailersStatus::StopIteration; } @@ -250,13 +256,13 @@ class Instance : public Http::StreamFilter, } void setDecoderFilterCallbacks( StreamDecoderFilterCallbacks& callbacks) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); decoder_callbacks_ = &callbacks; decoder_callbacks_->addResetStreamCallback( [this]() { state_ = Responded; }); } void completeCheck(const google::api_manager::utils::Status& status) { - log().debug("Called ApiManager::Instance : check complete {}", + Log().debug("Called ApiManager::Instance : check complete {}", status.ToJson()); if (!status.ok() && state_ != Responded) { state_ = Responded; @@ -275,30 +281,37 @@ class Instance : public Http::StreamFilter, virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterHeadersStatus::Continue; } virtual FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterDataStatus::Continue; } virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterTrailersStatus::Continue; } virtual void setEncoderFilterCallbacks( StreamEncoderFilterCallbacks& callbacks) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); encoder_callbacks_ = &callbacks; } - // note: cannot extend ~ActiveStream for access log, placing it here - ~Instance() { - log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr response(new Response()); - request_handler_->Report(std::move(response), - [this]() { log().debug("Report returns"); }); + virtual void log(const HeaderMap* request_headers, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info) override { + Log().debug("Called ApiManager::Instance : {}", __func__); + std::unique_ptr response( + new Response(request_info)); + request_handler_->Report(std::move(response), []() {}); + } + + spdlog::logger& Log() { + static spdlog::logger& instance = + Logger::Registry::getLog(Logger::Id::http); + return instance; } }; } @@ -320,8 +333,10 @@ class ApiManagerConfig : public HttpFilterConfigFactory { new Http::ApiManager::Config(config, server)); return [api_manager_config]( Http::FilterChainFactoryCallbacks& callbacks) -> void { - auto instance = new Http::ApiManager::Instance(api_manager_config); - callbacks.addStreamFilter(Http::StreamFilterPtr{instance}); + std::shared_ptr instance( + new Http::ApiManager::Instance(api_manager_config)); + callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); }; } }; diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index d3ec54f405d..27ea9f1b397 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -85,6 +85,5 @@ ] } ] - }, - "tracing_enabled": "true" + } } diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index e19efc6dce1..d14053308e1 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "39f42378fa41c10996d4c3ffba534951de30ceb8", + commit = "0bac7508c6803ec315c2228672728281b99149bd", build_file_content = BUILD, ) From 1d2d6a84989321ed20c3504704ce6c249188c166 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 1 Feb 2017 17:57:55 -0800 Subject: [PATCH 15/49] Update to the latest envoy with #396. (#61) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index d14053308e1..7cc4cb2db48 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "0bac7508c6803ec315c2228672728281b99149bd", + commit = "02c6fc97b4c21d25ab596a25208fbe283e927f6a", build_file_content = BUILD, ) From 53fd02610571f8c05af2e50fea911f89f935e678 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 2 Feb 2017 11:52:57 -0800 Subject: [PATCH 16/49] Fix tclap dependency fetching error (#62) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 7cc4cb2db48..94f371ebc7d 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -221,7 +221,7 @@ cc_library( native.new_http_archive( name = "tclap_tar", - url = "https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download", + url = "https://storage.googleapis.com/istio-build-deps/tclap-1.2.1.tar.gz", type = "tar.gz", strip_prefix = "tclap-1.2.1", build_file_content = BUILD, From 33f54ed2b98c9e57a74e78bbb34756764c206f7a Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Mon, 6 Feb 2017 10:08:35 -0800 Subject: [PATCH 17/49] Integrate mixer client directly with envoy. (#66) * Integrate mixer client directly with envoy. * Send response header in Report. * rename filter name from esp to mixer. * add README. --- src/envoy/mixer/BUILD | 41 +++++ src/envoy/mixer/README.md | 49 ++++++ src/envoy/mixer/envoy-mixer.conf | 71 ++++++++ src/envoy/mixer/http_control.cc | 184 +++++++++++++++++++++ src/envoy/mixer/http_control.h | 63 ++++++++ src/envoy/mixer/http_filter.cc | 267 +++++++++++++++++++++++++++++++ 6 files changed, 675 insertions(+) create mode 100644 src/envoy/mixer/BUILD create mode 100644 src/envoy/mixer/README.md create mode 100644 src/envoy/mixer/envoy-mixer.conf create mode 100644 src/envoy/mixer/http_control.cc create mode 100644 src/envoy/mixer/http_control.h create mode 100644 src/envoy/mixer/http_filter.cc diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD new file mode 100644 index 00000000000..428c87101b9 --- /dev/null +++ b/src/envoy/mixer/BUILD @@ -0,0 +1,41 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "filter_lib", + srcs = [ + "http_control.cc", + "http_control.h", + "http_filter.cc", + ], + deps = [ + "//external:mixer_client_lib", + "@envoy_git//:envoy-common", + ], + alwayslink = 1, +) + +cc_binary( + name = "envoy_esp", + linkstatic = 1, + deps = [ + ":filter_lib", + "@envoy_git//:envoy-main", + ], +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md new file mode 100644 index 00000000000..94276e37826 --- /dev/null +++ b/src/envoy/mixer/README.md @@ -0,0 +1,49 @@ + +This Proxy will use Envoy and talk to Mixer server. + +## Build Mixer server + +* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: + +``` + cd $(ISTIO)/mixer + bazel build ...:all +``` + +## Build Envoy proxy + +* Build target envoy_esp: + +``` + bazel build //src/envoy/mixer:envoy_esp +``` + +## How to run it + +* Start mixer server. In mixer folder run: + +``` + bazel-bin/cmd/server/mixs server +``` + + The server will run at port 9091 + +* Start backend Echo server. + +``` + cd test/backend/echo + go run echo.go +``` + +* Start Envoy proxy, run + +``` + bazel-bin/src/envoy/mixer/envoy_esp -c src/envoy/prototype/envoy-mixer.conf +``` + +* Then issue HTTP request to proxy. + +``` + curl http://localhost:9090/echo -d "hello world" +``` + diff --git a/src/envoy/mixer/envoy-mixer.conf b/src/envoy/mixer/envoy-mixer.conf new file mode 100644 index 00000000000..3b494c96b59 --- /dev/null +++ b/src/envoy/mixer/envoy-mixer.conf @@ -0,0 +1,71 @@ +{ + "listeners": [ + { + "port": 9090, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "localhost:9091" + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + } + ], + "admin": { + "access_log_path": "/dev/stdout", + "port": 9001 + }, + "cluster_manager": { + "clusters": [ + { + "name": "service1", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:8080" + } + ] + } + ] + } +} diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc new file mode 100644 index 00000000000..39e96ecbd3c --- /dev/null +++ b/src/envoy/mixer/http_control.cc @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/http_control.h" +#include "common/common/utility.h" +#include "common/http/utility.h" + +using ::google::protobuf::util::Status; +using ::istio::mixer_client::Attributes; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +const std::string kProxyPeerID = "Istio/Proxy"; +const std::string kEnvNameTargetService = "TARGET_SERVICE"; + +// Define lower case string for X-Forwarded-Host. +const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); + +const std::string kRequestHeaderPrefix = "requestHeader:"; +const std::string kRequestParamPrefix = "requestParameter:"; +const std::string kResponseHeaderPrefix = "responseHeader:"; + +// Define attribute names +const std::string kAttrNamePeerId = "peerId"; +const std::string kAttrNameURL = "url"; +const std::string kAttrNameHttpMethod = "httpMethod"; +const std::string kAttrNameRequestSize = "requestSize"; +const std::string kAttrNameResponseSize = "responseSize"; +const std::string kAttrNameLogMessage = "logMessage"; +const std::string kAttrNameResponseTime = "responseTime"; +const std::string kAttrNameOriginIp = "originIp"; +const std::string kAttrNameOriginHost = "originHost"; +const std::string kAttrNameTargetService = "targetService"; + +Attributes::Value StringValue(const std::string& str) { + Attributes::Value v; + v.type = Attributes::Value::STRING; + v.str_v = str; + return v; +} + +Attributes::Value Int64Value(int64_t value) { + Attributes::Value v; + v.type = Attributes::Value::INT64; + v.value.int64_v = value; + return v; +} + +void SetStringAttribute(const std::string& name, const std::string& value, + Attributes* attr) { + if (!value.empty()) { + attr->attributes[name] = StringValue(value); + } +} + +std::string GetFirstForwardedFor(const HeaderMap& header_map) { + if (!header_map.ForwardedFor()) { + return ""; + } + std::vector xff_address_list = + StringUtil::split(header_map.ForwardedFor()->value().c_str(), ','); + if (xff_address_list.empty()) { + return ""; + } + return xff_address_list.front(); +} + +std::string GetLastForwardedHost(const HeaderMap& header_map) { + const HeaderEntry* entry = header_map.get(kHeaderNameXFH); + if (entry == nullptr) { + return ""; + } + auto xff_list = StringUtil::split(entry->value().c_str(), ','); + if (xff_list.empty()) { + return ""; + } + return xff_list.back(); +} + +void FillRequestHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + // Pass in all headers + header_map.iterate( + [](const HeaderEntry& header, void* context) { + auto attr = static_cast(context); + attr->attributes[kRequestHeaderPrefix + header.key().c_str()] = + StringValue(header.value().c_str()); + }, + attr); + + // Pass in all Query parameters. + auto path = header_map.Path(); + if (path != nullptr) { + for (const auto& it : Utility::parseQueryString(path->value().c_str())) { + attr->attributes[kRequestParamPrefix + it.first] = StringValue(it.second); + } + } + + SetStringAttribute(kAttrNameOriginIp, GetFirstForwardedFor(header_map), attr); + SetStringAttribute(kAttrNameOriginHost, GetLastForwardedHost(header_map), + attr); +} + +void FillResponseHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + header_map.iterate( + [](const HeaderEntry& header, void* context) { + auto attr = static_cast(context); + attr->attributes[kResponseHeaderPrefix + header.key().c_str()] = + StringValue(header.value().c_str()); + }, + attr); +} + +void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, + Attributes* attr) { + if (info.bytesReceived() >= 0) { + attr->attributes[kAttrNameRequestSize] = Int64Value(info.bytesReceived()); + } + if (info.bytesSent() >= 0) { + attr->attributes[kAttrNameResponseSize] = Int64Value(info.bytesSent()); + } + if (info.duration().count() >= 0) { + attr->attributes[kAttrNameResponseTime] = + Int64Value(info.duration().count()); + } +} + +} // namespace + +HttpControl::HttpControl(const std::string& mixer_server) { + ::istio::mixer_client::MixerClientOptions options; + options.mixer_server = mixer_server; + mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + + auto target_service = getenv(kEnvNameTargetService.c_str()); + if (target_service) { + target_service_ = target_service; + } +} + +void HttpControl::FillCheckAttributes(const HeaderMap& header_map, + Attributes* attr) { + FillRequestHeaderAttributes(header_map, attr); + + SetStringAttribute(kAttrNameTargetService, target_service_, attr); + attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); +} + +void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, + DoneFunc on_done) { + FillCheckAttributes(headers, &request_data->attributes); + log().debug("Send Check: {}", request_data->attributes.DebugString()); + mixer_client_->Check(request_data->attributes, on_done); +} + +void HttpControl::Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, + DoneFunc on_done) { + // Use all Check attributes for Report. + // Add additional Report attributes. + FillResponseHeaderAttributes(*response_headers, &request_data->attributes); + FillRequestInfoAttributes(request_info, &request_data->attributes); + log().debug("Send Report: {}", request_data->attributes.DebugString()); + mixer_client_->Report(request_data->attributes, on_done); +} + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h new file mode 100644 index 00000000000..71c7e5620d6 --- /dev/null +++ b/src/envoy/mixer/http_control.h @@ -0,0 +1,63 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "envoy/http/access_log.h" +#include "include/client.h" + +namespace Http { +namespace Mixer { + +// Store data from Check to report +struct HttpRequestData { + ::istio::mixer_client::Attributes attributes; +}; +typedef std::shared_ptr HttpRequestDataPtr; + +// The mixer client class to control HTTP requests. +// It has Check() to validate if a request can be processed. +// At the end of request, call Report(). +class HttpControl final : public Logger::Loggable { + public: + // The constructor. + HttpControl(const std::string& mixer_server); + + // Make mixer check call. + void Check(HttpRequestDataPtr request_data, HeaderMap& headers, + ::istio::mixer_client::DoneFunc on_done); + + // Make mixer report call. + void Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, + ::istio::mixer_client::DoneFunc on_done); + + private: + void FillCheckAttributes(const HeaderMap& header_map, + ::istio::mixer_client::Attributes* attr); + + // The mixer client + std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // Target service + std::string target_service_; +}; + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc new file mode 100644 index 00000000000..ec209b1ff98 --- /dev/null +++ b/src/envoy/mixer/http_filter.cc @@ -0,0 +1,267 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/http_control.h" + +using ::google::protobuf::util::Status; +using StatusCode = ::google::protobuf::util::error::Code; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +// Define lower case string for X-Forwarded-Host. +const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); + +// Convert Status::code to HTTP code +int HttpCode(int code) { + // Map Canonical codes to HTTP status codes. This is based on the mapping + // defined by the protobuf http error space. + switch (code) { + case StatusCode::OK: + return 200; + case StatusCode::CANCELLED: + return 499; + case StatusCode::UNKNOWN: + return 500; + case StatusCode::INVALID_ARGUMENT: + return 400; + case StatusCode::DEADLINE_EXCEEDED: + return 504; + case StatusCode::NOT_FOUND: + return 404; + case StatusCode::ALREADY_EXISTS: + return 409; + case StatusCode::PERMISSION_DENIED: + return 403; + case StatusCode::RESOURCE_EXHAUSTED: + return 429; + case StatusCode::FAILED_PRECONDITION: + return 400; + case StatusCode::ABORTED: + return 409; + case StatusCode::OUT_OF_RANGE: + return 400; + case StatusCode::UNIMPLEMENTED: + return 501; + case StatusCode::INTERNAL: + return 500; + case StatusCode::UNAVAILABLE: + return 503; + case StatusCode::DATA_LOSS: + return 500; + case StatusCode::UNAUTHENTICATED: + return 401; + default: + return 500; + } +} + +} // namespace + +class Config : public Logger::Loggable { + private: + std::shared_ptr http_control_; + Upstream::ClusterManager& cm_; + + public: + Config(const Json::Object& config, Server::Instance& server) + : cm_(server.clusterManager()) { + std::string mixer_server; + if (config.hasObject("mixer_server")) { + mixer_server = config.getString("mixer_server"); + } else { + log().error( + "mixer_server is required but not specified in the config: {}", + __func__); + } + + http_control_ = std::make_shared(mixer_server); + log().debug("Called Mixer::Config contructor with mixer_server: ", + mixer_server); + } + + std::shared_ptr& http_control() { return http_control_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { + private: + std::shared_ptr http_control_; + std::shared_ptr request_data_; + + enum State { NotStarted, Calling, Complete, Responded }; + State state_; + + StreamDecoderFilterCallbacks* decoder_callbacks_; + StreamEncoderFilterCallbacks* encoder_callbacks_; + + bool initiating_call_; + + public: + Instance(ConfigPtr config) + : http_control_(config->http_control()), + state_(NotStarted), + initiating_call_(false) { + Log().debug("Called Mixer::Instance : {}", __func__); + } + + // Jump thread; on_done will be called at the dispatcher thread. + DoneFunc wrapper(DoneFunc on_done) { + auto& dispatcher = decoder_callbacks_->dispatcher(); + return [&dispatcher, on_done](const Status& status) { + dispatcher.post([status, on_done]() { on_done(status); }); + }; + } + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + state_ = Calling; + initiating_call_ = true; + request_data_ = std::make_shared(); + http_control_->Check( + request_data_, headers, + wrapper([this](const Status& status) { completeCheck(status); })); + initiating_call_ = false; + + if (state_ == Complete) { + return FilterHeadersStatus::Continue; + } + Log().debug("Called Mixer::Instance : {} Stop", __func__); + return FilterHeadersStatus::StopIteration; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {} ({}, {})", __func__, data.length(), + end_stream); + if (state_ == Calling) { + return FilterDataStatus::StopIterationAndBuffer; + } + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + if (state_ == Calling) { + return FilterTrailersStatus::StopIteration; + } + return FilterTrailersStatus::Continue; + } + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + decoder_callbacks_ = &callbacks; + decoder_callbacks_->addResetStreamCallback( + [this]() { state_ = Responded; }); + } + void completeCheck(const Status& status) { + Log().debug("Called Mixer::Instance : check complete {}", + status.ToString()); + if (!status.ok() && state_ != Responded) { + state_ = Responded; + Utility::sendLocalReply(*decoder_callbacks_, + Code(HttpCode(status.error_code())), + status.ToString()); + return; + } + state_ = Complete; + if (!initiating_call_) { + decoder_callbacks_->continueDecoding(); + } + } + + virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterHeadersStatus::Continue; + } + virtual FilterDataStatus encodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterDataStatus::Continue; + } + virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterTrailersStatus::Continue; + } + virtual void setEncoderFilterCallbacks( + StreamEncoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + encoder_callbacks_ = &callbacks; + } + + virtual void log(const HeaderMap* request_headers, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info) override { + Log().debug("Called Mixer::Instance : {}", __func__); + // Make sure not to use any class members at the callback. + // The class may be gone when it is called. + // Log() is a static function so it is OK. + http_control_->Report(request_data_, response_headers, request_info, + [](const Status& status) { + Log().debug("Report returns status: {}", + status.ToString()); + }); + } + + static spdlog::logger& Log() { + static spdlog::logger& instance = + Logger::Registry::getLog(Logger::Id::http); + return instance; + } +}; + +} // namespace Mixer +} // namespace Http + +namespace Server { +namespace Configuration { + +class MixerConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Both || name != "mixer") { + return nullptr; + } + + Http::Mixer::ConfigPtr mixer_config( + new Http::Mixer::Config(config, server)); + return + [mixer_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::Mixer::Instance(mixer_config)); + callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server From eb8bfccc391e528fba316c4e081515eb18583e5c Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Mon, 6 Feb 2017 12:18:42 -0800 Subject: [PATCH 18/49] Add release binary script. (#68) --- script/release-binary | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 script/release-binary diff --git a/script/release-binary b/script/release-binary new file mode 100755 index 00000000000..9bd7b51bf36 --- /dev/null +++ b/script/release-binary @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +# The bucket name to store proxy binary +BUCKET_NAME="istio-build/proxy" + +# The proxy binary name. +BINARY_FORMAT='proxy-alpha-%H' +BINARY_NAME="$(git show -q HEAD --pretty=format:"${BINARY_FORMAT}")" + +# Build the binary +bazel build --config=release //src/envoy/mixer:envoy_esp + +SRC="bazel-bin/src/envoy/mixer/envoy_esp" +DST="gs://${BUCKET_NAME}/${BINARY_NAME}" + +# Copy it to the bucket. +echo "Copying ${SRC} to ${DST}" +gsutil cp ${SRC} ${DST} + + From 89f0fbfe5c2105793583cdd4ac2286c87961b98f Mon Sep 17 00:00:00 2001 From: Kuat Date: Mon, 6 Feb 2017 15:47:29 -0800 Subject: [PATCH 19/49] Push tar.gz to GCS (#69) * Push tar.gz to GCS * Rename envoy_esp --- script/release-binary | 10 ++++++---- src/envoy/mixer/BUILD | 12 ++++++++++-- src/envoy/mixer/README.md | 6 +++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/script/release-binary b/script/release-binary index 9bd7b51bf36..af90b1aa0da 100755 --- a/script/release-binary +++ b/script/release-binary @@ -16,18 +16,20 @@ # ################################################################################ # +set -ex + +# Make sure to this script on x86_64 Ubuntu Xenial # The bucket name to store proxy binary BUCKET_NAME="istio-build/proxy" # The proxy binary name. -BINARY_FORMAT='proxy-alpha-%H' +BINARY_FORMAT='envoy-alpha-%H.tar.gz' BINARY_NAME="$(git show -q HEAD --pretty=format:"${BINARY_FORMAT}")" # Build the binary -bazel build --config=release //src/envoy/mixer:envoy_esp - -SRC="bazel-bin/src/envoy/mixer/envoy_esp" +bazel build --config=release //src/envoy/mixer:envoy_tar +SRC="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" DST="gs://${BUCKET_NAME}/${BINARY_NAME}" # Copy it to the bucket. diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 428c87101b9..45c96e25f96 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -15,7 +15,7 @@ ################################################################################ # -package(default_visibility = ["//visibility:public"]) +load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") cc_library( name = "filter_lib", @@ -32,10 +32,18 @@ cc_library( ) cc_binary( - name = "envoy_esp", + name = "envoy", linkstatic = 1, deps = [ ":filter_lib", "@envoy_git//:envoy-main", ], ) + +pkg_tar( + name = "envoy_tar", + extension = "tar.gz", + files = [":envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 94276e37826..9094d748442 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -12,10 +12,10 @@ This Proxy will use Envoy and talk to Mixer server. ## Build Envoy proxy -* Build target envoy_esp: +* Build target envoy: ``` - bazel build //src/envoy/mixer:envoy_esp + bazel build //src/envoy/mixer:envoy ``` ## How to run it @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/envoy_esp -c src/envoy/prototype/envoy-mixer.conf + bazel-bin/src/envoy/mixer/envoy -c src/envoy/prototype/envoy-mixer.conf ``` * Then issue HTTP request to proxy. From 76355366e4adc6d4002d69d5aafa57e1606338ee Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 8 Feb 2017 09:34:56 -0800 Subject: [PATCH 20/49] Remove mixer_client from api_manager. (#72) --- WORKSPACE | 10 +- contrib/endpoints/repositories.bzl | 12 - .../endpoints/src/api_manager/context/BUILD | 1 - .../api_manager/context/service_context.cc | 15 +- contrib/endpoints/src/api_manager/mixer/BUILD | 42 - .../endpoints/src/api_manager/mixer/mixer.cc | 204 ----- .../endpoints/src/api_manager/mixer/mixer.h | 77 -- .../src/api_manager/proto/server_config.proto | 9 - .../api_manager/server_config_proto_test.cc | 6 - .../BUILD => mixer/repositories.bzl} | 29 +- src/envoy/prototype/README.md | 73 -- src/envoy/prototype/api_manager_env.cc | 219 ------ src/envoy/prototype/api_manager_env.h | 33 - src/envoy/prototype/api_manager_filter.cc | 346 -------- .../prototype/dummy_api_manager_cluster.py | 91 --- src/envoy/prototype/envoy-esp.conf | 89 --- .../prototype/generic_service_config.json | 736 ------------------ src/envoy/prototype/server_config.pb.txt | 6 - 18 files changed, 22 insertions(+), 1976 deletions(-) delete mode 100644 contrib/endpoints/src/api_manager/mixer/BUILD delete mode 100644 contrib/endpoints/src/api_manager/mixer/mixer.cc delete mode 100644 contrib/endpoints/src/api_manager/mixer/mixer.h rename src/envoy/{prototype/BUILD => mixer/repositories.bzl} (57%) delete mode 100644 src/envoy/prototype/README.md delete mode 100644 src/envoy/prototype/api_manager_env.cc delete mode 100644 src/envoy/prototype/api_manager_env.h delete mode 100644 src/envoy/prototype/api_manager_filter.cc delete mode 100644 src/envoy/prototype/dummy_api_manager_cluster.py delete mode 100644 src/envoy/prototype/envoy-esp.conf delete mode 100644 src/envoy/prototype/generic_service_config.json delete mode 100644 src/envoy/prototype/server_config.pb.txt diff --git a/WORKSPACE b/WORKSPACE index 95dcdde5cdc..e152300d73d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,16 +31,20 @@ googletest_repositories() load( "//contrib/endpoints:repositories.bzl", "grpc_repositories", - "mixer_client_repositories", "servicecontrol_client_repositories", ) grpc_repositories() -mixer_client_repositories() - servicecontrol_client_repositories() +load( + "//src/envoy/mixer:repositories.bzl", + "mixer_client_repositories", +) + +mixer_client_repositories() + # Workaround for Bazel > 0.4.0 since it needs newer protobuf.bzl from: # https://github.com/google/protobuf/pull/2246 # Do not use this git_repository for anything else than protobuf.bzl diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index b1f14aae8b5..e3363cc86d2 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -334,15 +334,3 @@ def servicecontrol_client_repositories(bind=True): actual = "@servicecontrol_client_git//:service_control_client_lib", ) -def mixer_client_repositories(bind=True): - native.git_repository( - name = "mixerclient_git", - commit = "80e450a5126960e8e6337c3631cf2ef984038eab", - remote = "https://github.com/istio/mixerclient.git", - ) - - if bind: - native.bind( - name = "mixer_client_lib", - actual = "@mixerclient_git//:mixer_client_lib", - ) diff --git a/contrib/endpoints/src/api_manager/context/BUILD b/contrib/endpoints/src/api_manager/context/BUILD index e158c13fc92..59365102318 100644 --- a/contrib/endpoints/src/api_manager/context/BUILD +++ b/contrib/endpoints/src/api_manager/context/BUILD @@ -39,7 +39,6 @@ cc_library( "//contrib/endpoints/src/api_manager/auth", "//contrib/endpoints/src/api_manager/auth:service_account_token", "//contrib/endpoints/src/api_manager/cloud_trace", - "//contrib/endpoints/src/api_manager/mixer", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", "//external:cc_wkt_protos", diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index d8fc9dfc409..e9598a05814 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -15,7 +15,6 @@ //////////////////////////////////////////////////////////////////////////////// // #include "contrib/endpoints/src/api_manager/context/service_context.h" -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" #include "contrib/endpoints/src/api_manager/service_control/aggregated.h" @@ -97,16 +96,10 @@ const std::string& ServiceContext::project_id() const { } std::unique_ptr ServiceContext::CreateInterface() { - if (config_->server_config() && - config_->server_config()->has_mixer_options()) { - return std::unique_ptr( - mixer::Mixer::Create(env_.get(), config_.get())); - } else { - return std::unique_ptr( - service_control::Aggregated::Create( - config_->service(), config_->server_config(), env_.get(), - &service_account_token_)); - } + return std::unique_ptr( + service_control::Aggregated::Create(config_->service(), + config_->server_config(), env_.get(), + &service_account_token_)); } std::unique_ptr diff --git a/contrib/endpoints/src/api_manager/mixer/BUILD b/contrib/endpoints/src/api_manager/mixer/BUILD deleted file mode 100644 index affe09c7ddd..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -package(default_visibility = ["//contrib/endpoints/src/api_manager:__subpackages__"]) - -cc_library( - name = "mixer", - srcs = [ - "mixer.cc", - ], - hdrs = [ - "mixer.h", - ], - linkopts = select({ - "//:darwin": [], - "//conditions:default": [ - "-lm", - "-luuid", - ], - }), - deps = [ - "//contrib/endpoints/src/api_manager:impl_headers", - "//contrib/endpoints/src/api_manager/service_control", - "//contrib/endpoints/src/api_manager/utils", - "//external:grpc++", - "//external:mixer_client_lib", - "//external:protobuf", - ], -) diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc deleted file mode 100644 index 53837edaf67..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" - -using ::google::api_manager::utils::Status; -using ::istio::mixer_client::Attributes; - -namespace google { -namespace api_manager { -namespace mixer { -namespace { - -const std::string kProxyPeerID = "Istio/Proxy"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; - -const std::string kAttrNameServiceName = "serviceName"; -const std::string kAttrNamePeerId = "peerId"; -const std::string kAttrNameOperationId = "operationId"; -const std::string kAttrNameOperationName = "operationName"; -const std::string kAttrNameApiKey = "apiKey"; -const std::string kAttrNameResponseCode = "responseCode"; -const std::string kAttrNameURL = "url"; -const std::string kAttrNameLocation = "location"; -const std::string kAttrNameApiName = "apiName"; -const std::string kAttrNameApiVersion = "apiVersion"; -const std::string kAttrNameApiMethod = "apiMethod"; -const std::string kAttrNameRequestSize = "requestSize"; -const std::string kAttrNameResponseSize = "responseSize"; -const std::string kAttrNameLogMessage = "logMessage"; -const std::string kAttrNameResponseTime = "responseTime"; -const std::string kAttrNameOriginIp = "originIp"; -const std::string kAttrNameOriginHost = "originHost"; -const std::string kAttrNameTargetService = "targetService"; - -Attributes::Value StringValue(const std::string& str) { - Attributes::Value v; - v.type = Attributes::Value::STRING; - v.str_v = str; - return v; -} - -Attributes::Value Int64Value(int64_t value) { - Attributes::Value v; - v.type = Attributes::Value::INT64; - v.value.int64_v = value; - return v; -} - -} // namespace - -Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) - : env_(env), config_(config) {} - -Mixer::~Mixer() {} - -Status Mixer::Init() { - ::istio::mixer_client::MixerClientOptions options; - options.mixer_server = - config_->server_config()->mixer_options().mixer_server(); - mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); - auto target_service = getenv(kEnvNameTargetService.c_str()); - if (target_service) { - target_service_ = target_service; - } - return Status::OK; -} - -Status Mixer::Close() { return Status::OK; } - -void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(config_->service_name()); - attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); - - if (!info.operation_id.empty()) { - attr->attributes[kAttrNameOperationId] = StringValue(info.operation_id); - } - if (!info.operation_name.empty()) { - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - } - if (!info.api_key.empty()) { - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - } - if (!info.client_ip.empty()) { - attr->attributes[kAttrNameOriginIp] = StringValue(info.client_ip); - } - if (!info.client_host.empty()) { - attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); - } - if (!target_service_.empty()) { - attr->attributes[kAttrNameTargetService] = StringValue(target_service_); - } -} - -void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); -} - -void Mixer::FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); - - if (!info.url.empty()) { - attr->attributes[kAttrNameURL] = StringValue(info.url); - } - if (!info.location.empty()) { - attr->attributes[kAttrNameLocation] = StringValue(info.location); - } - - if (!info.api_name.empty()) { - attr->attributes[kAttrNameApiName] = StringValue(info.api_name); - } - if (!info.api_version.empty()) { - attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); - } - if (!info.api_method.empty()) { - attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - } - - if (!info.log_message.empty()) { - attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - } - - attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); - if (info.request_size >= 0) { - attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); - } - if (info.response_size >= 0) { - attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); - } - - if (info.latency.request_time_ms >= 0) { - attr->attributes[kAttrNameResponseTime] = - Int64Value(info.latency.request_time_ms); - } -} - -Status Mixer::Report(const service_control::ReportRequestInfo& info) { - ::istio::mixer_client::Attributes attributes; - FillReportAttributes(info, &attributes); - env_->LogInfo("Send Report: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Report( - attributes, [this](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Report response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::report, Error: ") + - status.ToString()); - } - }); - return Status::OK; -} - -void Mixer::Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done) { - ::istio::mixer_client::Attributes attributes; - FillCheckAttributes(info, &attributes); - env_->LogInfo("Send Check: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Check( - attributes, - [this, on_done](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Check response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::check, Error: ") + - status.ToString()); - } - service_control::CheckResponseInfo info; - on_done(Status(status.error_code(), status.error_message(), - Status::SERVICE_CONTROL), - info); - }); -} - -Status Mixer::GetStatistics(service_control::Statistics* esp_stat) const { - return Status::OK; -} - -service_control::Interface* Mixer::Create(ApiManagerEnvInterface* env, - const Config* config) { - return new Mixer(env, config); -} - -} // namespace mixer -} // namespace api_manager -} // namespace google diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h deleted file mode 100644 index 81276e1227d..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef API_MANAGER_MIXER_MIXER_H_ -#define API_MANAGER_MIXER_MIXER_H_ - -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "contrib/endpoints/src/api_manager/config.h" -#include "contrib/endpoints/src/api_manager/service_control/interface.h" -#include "include/client.h" - -namespace google { -namespace api_manager { -namespace mixer { - -// This implementation uses service-control-client-cxx module. -class Mixer : public service_control::Interface { - public: - static service_control::Interface* Create(ApiManagerEnvInterface* env, - const Config* config); - - virtual ~Mixer(); - - virtual utils::Status Report(const service_control::ReportRequestInfo& info); - - virtual void Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done); - - virtual utils::Status Init(); - virtual utils::Status Close(); - - virtual utils::Status GetStatistics(service_control::Statistics* stat) const; - - private: - // The constructor. - Mixer(ApiManagerEnvInterface* env, const Config* config); - - // Fill common attributes for both check and report. - void FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for check. - void FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for report. - void FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - - // The Api Manager environment interface. - ApiManagerEnvInterface* env_; - // The config. - const Config* config_; - // The mixer client - std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // Target service - std::string target_service_; -}; - -} // namespace mixer -} // namespace api_manager -} // namespace google - -#endif // API_MANAGER_MIXER_MIXER_H_ diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 343414aceae..1fe04af6d07 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -35,10 +35,6 @@ message ServerConfig { // Server config used for API authentication ApiAuthenticationConfig api_authentication_config = 5; - // Mixer option flag. If not present, default to use service_control. When - // Envoy/esp talks to Mixer, has to specify this field. - MixerOptions mixer_options = 6; - // Experimental flags Experimental experimental = 999; } @@ -143,11 +139,6 @@ message ApiAuthenticationConfig { bool force_disable = 1; } -message MixerOptions { - // For envoy, it is the cluster name for mixer server. - string mixer_server = 1; -} - message Experimental { // Disable timed printouts of ESP status to the error log. bool disable_log_status = 1; diff --git a/contrib/endpoints/src/api_manager/server_config_proto_test.cc b/contrib/endpoints/src/api_manager/server_config_proto_test.cc index b39309727f1..80c62e05471 100644 --- a/contrib/endpoints/src/api_manager/server_config_proto_test.cc +++ b/contrib/endpoints/src/api_manager/server_config_proto_test.cc @@ -76,9 +76,6 @@ experimental { disable_log_status: false } -mixer_options { - mixer_server: "mixer_server" -} )"; TEST(ServerConfigProto, ServerConfigFromString) { @@ -125,9 +122,6 @@ TEST(ServerConfigProto, ServerConfigFromString) { // Check disable_log_status EXPECT_EQ(false, server_config.experimental().disable_log_status()); - - // Check mixer options - EXPECT_EQ("mixer_server", server_config.mixer_options().mixer_server()); } TEST(ServerConfigProto, ValidateSampleServerConfig) { diff --git a/src/envoy/prototype/BUILD b/src/envoy/mixer/repositories.bzl similarity index 57% rename from src/envoy/prototype/BUILD rename to src/envoy/mixer/repositories.bzl index 258f2046cd7..d9888fb0daf 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/mixer/repositories.bzl @@ -15,22 +15,15 @@ ################################################################################ # -package(default_visibility = ["//visibility:public"]) +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = "80e450a5126960e8e6337c3631cf2ef984038eab", + remote = "https://github.com/istio/mixerclient.git", + ) -cc_binary( - name = "envoy_esp", - srcs = [ - "api_manager_filter.cc", - "api_manager_env.cc", - "api_manager_env.h", - ], - deps = [ - "//external:protobuf", - "//contrib/endpoints/include:api_manager", - "//contrib/endpoints/src/grpc/transcoding:transcoding", - "//external:servicecontrol", - "@envoy_git//:envoy-common", - "@envoy_git//:envoy-main" - ], - linkstatic=1, -) + if bind: + native.bind( + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", + ) diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md deleted file mode 100644 index dc73f63f6c9..00000000000 --- a/src/envoy/prototype/README.md +++ /dev/null @@ -1,73 +0,0 @@ - -This Proxy will use Envoy and talk to Mixer server. - -## Install dependencies - -``` - apt-get install uuid-dev -``` - -## Build Mixer server - -* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: - -``` - cd $(ISTIO)/mixer - bazel build ...:all -``` - -## Build Envoy proxy - -* Build target envoy_esp: - -``` - bazel build //src/envoy/prototype:envoy_esp -``` - -## How to run it - -* Start mixer server. In mixer folder run: - -``` - bazel-bin/cmd/server/mixs server -``` - - The server will run at port 9091 - -* Start backend Echo server. - -``` - cd test/backend/echo - go run echo.go -``` - -* Modify your iptables: - -``` - sudo iptables -t nat -A OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - -Once you are done, you should remove this rule: - -``` - sudo iptables -t nat -D OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - - -* Start Envoy proxy, run - -``` - bazel-bin/src/envoy/prototype/envoy_esp -c src/envoy/prototype/envoy-esp.conf -``` - -* Then issue HTTP request to proxy. - -``` - curl http://localhost:9090/echo?key=API-KEY -d "hello world" -``` - -## How to add attributes or facts - -Now only some of attributes are passed to mixer. If you want to add more attributes, you can -modify this -[file](https://github.com/istio/proxy/blob/master/contrib/endpoints/src/api_manager/mixer/mixer.cc). diff --git a/src/envoy/prototype/api_manager_env.cc b/src/envoy/prototype/api_manager_env.cc deleted file mode 100644 index f8536bcc10a..00000000000 --- a/src/envoy/prototype/api_manager_env.cc +++ /dev/null @@ -1,219 +0,0 @@ -#include "api_manager_env.h" - -#include "common/http/headers.h" -#include "common/http/message_impl.h" -#include "envoy/event/timer.h" -#include "google/protobuf/stubs/status.h" -#include "source/common/grpc/common.h" - -using ::google::api_manager::utils::Status; -using ::google::protobuf::util::error::Code; - -namespace Http { -namespace ApiManager { - -void Http::ApiManager::Env::Log(LogLevel level, const char *message) { - switch (level) { - case LogLevel::DEBUG: - log().debug("{}", message); - break; - case LogLevel::INFO: - log().info("{}", message); - break; - case LogLevel::WARNING: - log().warn("{}", message); - break; - case LogLevel::ERROR: - log().error("{}", message); - break; - } -} - -class PeriodicTimer : public google::api_manager::PeriodicTimer, - public Logger::Loggable { - private: - Server::Instance &server_; - Event::TimerPtr timer_; - - public: - PeriodicTimer(Server::Instance &server) : server_(server) {} - ~PeriodicTimer() { Stop(); } - void Stop() { - if (timer_) { - timer_->disableTimer(); - timer_ = nullptr; - } - } - void Schedule(std::chrono::milliseconds interval, - std::function continuation) { - Stop(); - timer_ = server_.dispatcher().createTimer([this, continuation, interval]() { - continuation(); - Schedule(interval, continuation); - }); - timer_->enableTimer(interval); - } -}; - -std::unique_ptr Env::StartPeriodicTimer( - std::chrono::milliseconds interval, std::function continuation) { - log().debug("start periodic timer"); - auto single = new PeriodicTimer(server); - single->Schedule(interval, continuation); - std::unique_ptr timer(single); - return timer; -} - -static const LowerCaseString kApiManagerUrl("x-api-manager-url"); -static const LowerCaseString kGrpcTEKey("te"); -static const std::string kGrpcTEValue("trailers"); - -class HTTPRequest : public Http::Message { - private: - HeaderMapImpl header_map_; - Buffer::OwnedImpl body_; - - public: - HTTPRequest(google::api_manager::HTTPRequest *request) - : body_(request->body()) { - header_map_.addStaticKey(Headers::get().Method, request->method()); - - size_t path_pos = request->url().find('/', 8); - if (path_pos == std::string::npos) { - header_map_.addStaticKey(Headers::get().Path, "/"); - } else { - header_map_.addStaticKey(Headers::get().Path, - request->url().substr(path_pos)); - } - - header_map_.addStaticKey(Headers::get().Scheme, "http"); - header_map_.addStaticKey(Headers::get().Host, "localhost"); - header_map_.addStaticKey(Headers::get().ContentLength, body_.length()); - header_map_.addStaticKey(kApiManagerUrl, request->url()); - for (const auto header : request->request_headers()) { - LowerCaseString lower_key(header.first); - HeaderString key, value; - key.setCopy(lower_key.get().data(), lower_key.get().size()); - value.setCopy(header.second.data(), header.second.size()); - header_map_.addViaMove(std::move(key), std::move(value)); - } - } - virtual HeaderMap &headers() override { return header_map_; } - virtual Buffer::Instance *body() override { return &body_; } - virtual void body(Buffer::InstancePtr &&body) override {} - virtual HeaderMap *trailers() override { return nullptr; } - virtual void trailers(HeaderMapPtr &&trailers) override {} - virtual std::string bodyAsString() const override { return ""; } -}; - -class HTTPRequestCallbacks : public AsyncClient::Callbacks { - private: - std::unique_ptr request_; - std::unique_ptr sent_request_; - - public: - HTTPRequestCallbacks( - std::unique_ptr &&request) - : request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - std::map headers; - response->headers().iterate( - [&](const HeaderEntry &header, void *) -> void { - // TODO: fix it - // headers.emplace(header.key().c_str(), header.value().c_str()); - }, - nullptr); - request_->OnComplete(status, std::move(headers), response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to HTTP server."); - std::map headers; - request_->OnComplete(status, std::move(headers), ""); - delete this; - } -}; - -namespace { -// Copy the code here from envoy/grpc/common.cc -Buffer::InstancePtr SerializeGrpcBody(const std::string &body_str) { - // http://www.grpc.io/docs/guides/wire.html - Buffer::InstancePtr body(new Buffer::OwnedImpl()); - uint8_t compressed = 0; - body->add(&compressed, sizeof(compressed)); - uint32_t size = htonl(body_str.size()); - body->add(&size, sizeof(size)); - body->add(body_str); - return body; -} -Http::MessagePtr PrepareGrpcHeaders(const std::string &upstream_cluster, - const std::string &service_full_name, - const std::string &method_name) { - Http::MessagePtr message(new Http::RequestMessageImpl()); - message->headers().insertMethod().value( - Http::Headers::get().MethodValues.Post); - message->headers().insertPath().value( - fmt::format("/{}/{}", service_full_name, method_name)); - message->headers().insertHost().value(upstream_cluster); - message->headers().insertContentType().value(Grpc::Common::GRPC_CONTENT_TYPE); - message->headers().addStatic(kGrpcTEKey, kGrpcTEValue); - return message; -} -} // annoymous namespace - -class GrpcRequestCallbacks : public AsyncClient::Callbacks { - private: - Env *env_; - std::unique_ptr request_; - - public: - GrpcRequestCallbacks( - Env *env, std::unique_ptr &&request) - : env_(env), request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - Grpc::Common::validateResponse(*response); - env_->LogInfo("pass validate"); - // remove 5 bytes of grpc header - response->body()->drain(5); - request_->OnComplete(status, response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to gRPC server."); - request_->OnComplete(status, ""); - delete this; - } -}; - -void Env::RunHTTPRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster("api_manager"); - - MessagePtr message{new HTTPRequest(request.get())}; - HTTPRequestCallbacks *callbacks = - new HTTPRequestCallbacks(std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} - -void Env::RunGRPCRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster(request->server()); - - Http::MessagePtr message = - PrepareGrpcHeaders("localhost", request->service(), request->method()); - message->body(SerializeGrpcBody(request->body())); - auto callbacks = new GrpcRequestCallbacks(this, std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} -} -} diff --git a/src/envoy/prototype/api_manager_env.h b/src/envoy/prototype/api_manager_env.h deleted file mode 100644 index 0ae8136f5be..00000000000 --- a/src/envoy/prototype/api_manager_env.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "precompiled/precompiled.h" - -#include "common/common/logger.h" -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "envoy/upstream/cluster_manager.h" -#include "server/server.h" - -namespace Http { -namespace ApiManager { - -class Env : public google::api_manager::ApiManagerEnvInterface, - public Logger::Loggable { - private: - Server::Instance& server; - Upstream::ClusterManager& cm_; - - public: - Env(Server::Instance& server) - : server(server), cm_(server.clusterManager()){}; - - virtual void Log(LogLevel level, const char* message) override; - virtual std::unique_ptr - StartPeriodicTimer(std::chrono::milliseconds interval, - std::function continuation) override; - virtual void RunHTTPRequest( - std::unique_ptr request) override; - virtual void RunGRPCRequest( - std::unique_ptr request) override; -}; -} -} diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc deleted file mode 100644 index 3983a4d8b41..00000000000 --- a/src/envoy/prototype/api_manager_filter.cc +++ /dev/null @@ -1,346 +0,0 @@ -#include "precompiled/precompiled.h" - -#include "api_manager_env.h" -#include "common/common/logger.h" -#include "common/grpc/common.h" -#include "common/http/filter/ratelimit.h" -#include "common/http/headers.h" -#include "common/http/utility.h" -#include "contrib/endpoints/include/api_manager/api_manager.h" -#include "contrib/endpoints/src/grpc/transcoding/transcoder.h" -#include "envoy/server/instance.h" -#include "server/config/network/http_connection_manager.h" - -namespace Http { -namespace ApiManager { -namespace { - -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); - -} // namespace - -std::string ReadFile(const std::string& file_name) { - std::ifstream t(file_name); - std::string content((std::istreambuf_iterator(t)), - std::istreambuf_iterator()); - return content; -} - -class Config : public Logger::Loggable { - private: - google::api_manager::ApiManagerFactory api_manager_factory_; - std::shared_ptr api_manager_; - Upstream::ClusterManager& cm_; - - public: - Config(const Json::Object& config, Server::Instance& server) - : cm_(server.clusterManager()) { - std::string service_config_content; - if (config.hasObject("service_config")) { - const std::string service_config = config.getString("service_config"); - service_config_content = ReadFile(service_config); - } else { - log().error( - "Service_config is required but not specified in the config: {}", - __func__); - } - - std::string server_config_content; - if (config.hasObject("server_config")) { - const std::string server_config = config.getString("server_config"); - server_config_content = ReadFile(server_config); - } - std::unique_ptr env( - new Env(server)); - - api_manager_ = api_manager_factory_.GetOrCreateApiManager( - std::move(env), service_config_content, server_config_content); - - api_manager_->Init(); - log().debug("Called ApiManager::Config constructor: {}", __func__); - } - - std::shared_ptr& api_manager() { - return api_manager_; - } -}; - -typedef std::shared_ptr ConfigPtr; - -class Request : public google::api_manager::Request { - private: - HeaderMap& header_map_; - std::string downstream_address_; - std::string virtual_host_; - bool query_parsed_; - std::map query_params_; - - public: - Request(HeaderMap& header_map, const std::string& downstream_address, - const std::string& virtual_host) - : header_map_(header_map), - downstream_address_(downstream_address), - virtual_host_(virtual_host), - query_parsed_(false) {} - virtual std::string GetRequestHTTPMethod() override { - return header_map_.Method()->value().c_str(); - } - virtual std::string GetRequestPath() override { - return header_map_.Path()->value().c_str(); - } - virtual std::string GetUnparsedRequestPath() override { - return header_map_.Path()->value().c_str(); - } - - virtual std::string GetClientIP() override { - if (!header_map_.ForwardedFor()) { - return downstream_address_; - } - std::vector xff_address_list = - StringUtil::split(header_map_.ForwardedFor()->value().c_str(), ','); - if (xff_address_list.empty()) { - return downstream_address_; - } - return xff_address_list.front(); - } - - virtual std::string GetClientHost() override { - const HeaderEntry* entry = header_map_.get(kHeaderNameXFH); - if (entry == nullptr) { - return virtual_host_; - } - auto xff_list = StringUtil::split(entry->value().c_str(), ','); - if (xff_list.empty()) { - return virtual_host_; - } - return xff_list.back(); - } - - virtual bool FindQuery(const std::string& name, std::string* query) override { - if (!query_parsed_) { - auto header = header_map_.Path(); - if (header != nullptr) { - std::string path = header->value().c_str(); - Utility::parseQueryString(path).swap(query_params_); - } - query_parsed_ = true; - } - auto entry = query_params_.find(name); - if (entry == query_params_.end()) { - return false; - } - *query = entry->second; - return true; - } - - virtual bool FindHeader(const std::string& name, - std::string* header) override { - LowerCaseString key(name); - const HeaderEntry* entry = header_map_.get(key); - if (entry == nullptr) { - return false; - } - *header = entry->value().c_str(); - return true; - } - - virtual google::api_manager::protocol::Protocol GetRequestProtocol() - override { - return google::api_manager::protocol::Protocol::HTTP; - } - virtual google::api_manager::utils::Status AddHeaderToBackend( - const std::string& key, const std::string& value) override { - return google::api_manager::utils::Status::OK; - } - virtual void SetAuthToken(const std::string& auth_token) override {} - - virtual int64_t GetGrpcRequestBytes() { return 0; } - virtual int64_t GetGrpcResponseBytes() { return 0; } - virtual int64_t GetGrpcRequestMessageCounts() { return 0; } - virtual int64_t GetGrpcResponseMessageCounts() { return 0; } - virtual std::string GetQueryParameters() { return ""; } -}; - -class Response : public google::api_manager::Response { - const AccessLog::RequestInfo& request_info_; - - public: - Response(const AccessLog::RequestInfo& request_info) - : request_info_(request_info) {} - - google::api_manager::utils::Status GetResponseStatus() { - return google::api_manager::utils::Status::OK; - } - - std::size_t GetRequestSize() { return request_info_.bytesReceived(); } - - std::size_t GetResponseSize() { return request_info_.bytesSent(); } - - google::api_manager::utils::Status GetLatencyInfo( - google::api_manager::service_control::LatencyInfo* info) { - info->request_time_ms = request_info_.duration().count(); - return google::api_manager::utils::Status::OK; - } -}; - -const Http::HeaderMapImpl BadRequest{{Http::Headers::get().Status, "400"}}; - -class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { - private: - std::shared_ptr api_manager_; - std::unique_ptr - request_handler_; - - enum State { NotStarted, Calling, Complete, Responded }; - State state_; - - StreamDecoderFilterCallbacks* decoder_callbacks_; - StreamEncoderFilterCallbacks* encoder_callbacks_; - - bool initiating_call_; - - std::string getRouteVirtualHost(HeaderMap& headers) const { - const Router::Route* route = decoder_callbacks_->route(); - if (route && route->routeEntry()) { - return route->routeEntry()->virtualHost().name(); - } - return ""; - } - - public: - Instance(ConfigPtr config) - : api_manager_(config->api_manager()), - state_(NotStarted), - initiating_call_(false) { - Log().debug("Called ApiManager::Instance : {}", __func__); - } - - FilterHeadersStatus decodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr request( - new Request(headers, decoder_callbacks_->downstreamAddress(), - getRouteVirtualHost(headers))); - request_handler_ = api_manager_->CreateRequestHandler(std::move(request)); - state_ = Calling; - initiating_call_ = true; - request_handler_->Check([this](google::api_manager::utils::Status status) { - completeCheck(status); - }); - initiating_call_ = false; - - if (state_ == Complete) { - return FilterHeadersStatus::Continue; - } - Log().debug("Called ApiManager::Instance : {} Stop", __func__); - return FilterHeadersStatus::StopIteration; - } - - FilterDataStatus decodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, - data.length(), end_stream); - if (state_ == Calling) { - return FilterDataStatus::StopIterationAndBuffer; - } - return FilterDataStatus::Continue; - } - - FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - if (state_ == Calling) { - return FilterTrailersStatus::StopIteration; - } - return FilterTrailersStatus::Continue; - } - void setDecoderFilterCallbacks( - StreamDecoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - decoder_callbacks_ = &callbacks; - decoder_callbacks_->addResetStreamCallback( - [this]() { state_ = Responded; }); - } - void completeCheck(const google::api_manager::utils::Status& status) { - Log().debug("Called ApiManager::Instance : check complete {}", - status.ToJson()); - if (!status.ok() && state_ != Responded) { - state_ = Responded; - decoder_callbacks_->dispatcher().post([this, status]() { - Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), - status.ToJson()); - }); - return; - } - state_ = Complete; - if (!initiating_call_) { - decoder_callbacks_->dispatcher().post( - [this]() { decoder_callbacks_->continueDecoding(); }); - } - } - - virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterHeadersStatus::Continue; - } - virtual FilterDataStatus encodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterDataStatus::Continue; - } - virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterTrailersStatus::Continue; - } - virtual void setEncoderFilterCallbacks( - StreamEncoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - encoder_callbacks_ = &callbacks; - } - - virtual void log(const HeaderMap* request_headers, - const HeaderMap* response_headers, - const AccessLog::RequestInfo& request_info) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr response( - new Response(request_info)); - request_handler_->Report(std::move(response), []() {}); - } - - spdlog::logger& Log() { - static spdlog::logger& instance = - Logger::Registry::getLog(Logger::Id::http); - return instance; - } -}; -} -} - -namespace Server { -namespace Configuration { - -class ApiManagerConfig : public HttpFilterConfigFactory { - public: - HttpFilterFactoryCb tryCreateFilterFactory( - HttpFilterType type, const std::string& name, const Json::Object& config, - const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Both || name != "esp") { - return nullptr; - } - - Http::ApiManager::ConfigPtr api_manager_config( - new Http::ApiManager::Config(config, server)); - return [api_manager_config]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::ApiManager::Instance(api_manager_config)); - callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); - callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); - }; - } -}; - -static RegisterHttpFilterConfigFactory register_; -} -} diff --git a/src/envoy/prototype/dummy_api_manager_cluster.py b/src/envoy/prototype/dummy_api_manager_cluster.py deleted file mode 100644 index 3f83476aae8..00000000000 --- a/src/envoy/prototype/dummy_api_manager_cluster.py +++ /dev/null @@ -1,91 +0,0 @@ -from BaseHTTPServer import BaseHTTPRequestHandler -from BaseHTTPServer import HTTPServer -from subprocess import Popen, PIPE, STDOUT - -PUBLIC_KEY = '''{ - "keys": [ - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "686f442dead739e241342519849bd0d140707a86", - "n": "yVGRzBnVmCHC9uMH_mCna7FiSKOjMEsDdgf8hds41KYfUHH9cp4P41iQEBnqPYvQiMYaZDuatkMDh25ukmkSyfLwJIVQpnHgMDwoZ1cx83Z0PwC8wAAJYqKyItKNfFN4UJ6DZIOVU-Iqgxi8VOtGwMNx2LiD1NoFVfXyz52UJ_QLiUGzErVwTGv4UD6NtaWKFkctTnEG-9rZvDF8QknnzxAomVa2OcV8OHeszx6N8UE1tm9Kq4xj2Uu8D3dDrfu2jr45Pi6RHIZOTAnu8G7wTDNaiGbENrbHSk6WAjLZBOcWZj9SDlDlwH2xFoKdpNKRmLKpPQblHem_y1KRYwvJTw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "d57d66bfbff089769e3545589572d3b53955a8cb", - "n": "zX99YCqicbn_j-5YTJ-2FsgONUg7cmqJiwvHXrUopRvn2Tukrd0B5Sg-Rq1hdZTYgIym7lSMw_zLxIQCH54sUfydX5MMWr6FOxVUbYl-E0Oko85Yer9dFv61rN0USj_A12QRMmjCZkcqH_6MWWuA1QWaejyStopjpLEYnUD3bP6oS604eZWkkOp8Nu-Vg4NqqX7ZClIcQqe03xv3sFHiPuhB-qaifhpIPpKCiYSKxAY4_GxwCJ_ml_uJ5k1tIrDykAlie6aWxv8hogOXrQmNRCO2Qcumwb7d9cci1UxsEYOtpZxhTiZqWsbrBxwfvLqU_rvsCT8vOjrBPOkTtYl22w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "516ca9a83b130d4b605684fe7df83b43f33fba43", - "n": "rSLxCvtRLoJUt2xPTSjDgN8_7kuBLsZz0JD4HRiwvId_IYY-T5MsUirBx01AtI3KRlmeQ-1QIvET4cNN99QL8U6e7Hh75VWJvQhh0_TFdKbv_75b81-rzx-EvIoVe0czeY3QeRp3DDORRk9o7xgri6U3VfT2zcQIDAPY14h3wLHE5DQuH9EpbxjS_wNrQxRYZH_mWgmWU7h001WHWRDKdEIgtzQkinKOTig90Wy4pt_vgMGcv0mh_wYvIcmB62Qj1sHUUOa5NJWUaZLC4Stj7CH0dLiBrVqg3JFqvO60Oo3wrcdmNQl0RaRHFjBYLameUPSP6M2BevfCRx8-Ix26rw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "33a681b36b8983913b05f8d5eab165c60694b6fc", - "n": "pRbWhLXIOvF1Naxh5n2evZqpJ3kPEFL4b-jHRHOhMnzqsi_aeQtIwYLVM8zdYYfgCxRq8umG1YwMenBKKPzRWr4MkFAB64O3UPgdyg-3je-fgCMziqS_3KH7AXHekxG_ZHpwbkgilRMtJNiDnSZWGad8XAfW3VFct4RqRAaf7h-6Za0IM7R3u4VYkUfosNqKtoDJDQrng9Nbv9ryUk8u1WikKF0M1r-wrZoDA7QFRAFkbdfYyvHL2daUflNDIXmFkUeHGGApMlJQ3wJk7Tln4txGMSUdJoD3JWEyKa2W1WshtqBHnQb8VlL77H-ch9zbn5pGZCoJ0MsgJr0vjoea8w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "986f2c30b9843defdc8f8debe138d640d1d67d9d", - "n": "qfE-u8I60zP_Vg3UsP_X9Qv1idkrnsXKkcQcsGWUdqs4jM2VixGBJDMJWbP55aJA93Sl49_dzAv_lr5-ctc1L04ke6nHX19EEBDtFTjGKyEdI-X1C-6HXCQmpI1XvUA8DzTOIZd5KEXJgBA9tpn6qkMAHzXyMHOfg8nhW36She4QggFjfF_RDKOA2-jRXDIjinOmIaLhVq4hsC9hCshhsfreLPw3HH8UhRONbkoFU_8ZBgAQLdVB1TQwp-ZDiVyh9od9a-RmfIiUl27AK5LDrtpVRCtXj6bv9OMx4QWVX8G-NfQexDwAp1pICaE5qYyJbiIK25E07vPTJ1ZtMpodFQ", - "e": "AQAB" - } - ] -} -''' - -def print_body(binary, expected): - p = Popen(['bazel-bin/src/tools/service_control_json_gen', - '--text', - '--'+expected, - '--stdin'], stdin=PIPE) - p.communicate(input=binary) - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - self.handle_request("GET") - - def do_POST(self): - self.handle_request("POST") - - - - def handle_request(self, method): - print(method, self.path, self.headers.items()) - url = self.headers.get('x-api-manager-url', '') - if url == 'https://www.googleapis.com/service_accounts/v1/jwk/loadtest@esp-test-client.iam.gserviceaccount.com': - self.send_response(200) - self.send_header('Content-Length', str(len(PUBLIC_KEY))) - self.end_headers() - self.wfile.write(PUBLIC_KEY) - else: - #content_len = self.headers.getheader('content-length', 0) - #post_body = self.rfile.read(int(content_len)) - if url.endswith(":report"): - print_body(post_body, "report_request") - elif url.endswith(":check"): - print_body(post_body, "check_request") - - self.send_response(200) - self.send_header('Content-Length', "0") - self.end_headers() - - -if __name__ == '__main__': - server = HTTPServer(('localhost', 8081), Handler) - print 'Starting server, use to stop' - server.serve_forever() diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf deleted file mode 100644 index 27ea9f1b397..00000000000 --- a/src/envoy/prototype/envoy-esp.conf +++ /dev/null @@ -1,89 +0,0 @@ -{ - "listeners": [ - { - "port": 9092, - "bind_to_port": true, - "use_original_dst": true, - "filters": [] - }, - { - "port": 9090, - "bind_to_port": false, - "filters": [ - { - "type": "read", - "name": "http_connection_manager", - "config": { - "codec_type": "auto", - "stat_prefix": "ingress_http", - "route_config": { - "virtual_hosts": [ - { - "name": "backend", - "domains": ["*"], - "routes": [ - { - "timeout_ms": 0, - "prefix": "/", - "cluster": "service1" - } - ] - } - ] - }, - "access_log": [ - { - "path": "/dev/stdout" - } - ], - "filters": [ - { - "type": "both", - "name": "esp", - "config": { - "service_config": "src/envoy/prototype/generic_service_config.json", - "server_config": "src/envoy/prototype/server_config.pb.txt" - } - }, - { - "type": "decoder", - "name": "router", - "config": {} - } - ] - } - } - ] - } - ], - "admin": { - "access_log_path": "/dev/stdout", - "port": 9001 - }, - "cluster_manager": { - "clusters": [ - { - "name": "service1", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8080" - } - ] - }, - { - "name": "api_manager", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8081" - } - ] - } - ] - } -} diff --git a/src/envoy/prototype/generic_service_config.json b/src/envoy/prototype/generic_service_config.json deleted file mode 100644 index ac926e5e4c2..00000000000 --- a/src/envoy/prototype/generic_service_config.json +++ /dev/null @@ -1,736 +0,0 @@ -{ - "apis": [ - { - "methods": [ - { - "name": "Get", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Delete", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Patch", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Post", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Put", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "generic_qiwzhang_app_1_appspot_com_1_0_0", - "sourceContext": { - "fileName": "sss.yaml" - }, - "version": "1.0.0" - } - ], - "authentication": {}, - "backend": { - "rules": [ - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "configVersion": 3, - "context": {}, - "control": { - "environment": "servicecontrol.googleapis.com" - }, - "customError": {}, - "documentation": {}, - "enums": [ - { - "enumvalue": [ - { - "name": "NULL_VALUE" - } - ], - "name": "google.protobuf.NullValue", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "http": { - "rules": [ - { - "get": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "delete": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "patch": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "post": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "put": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "id": "2017-01-06r0", - "logging": { - "producerDestinations": [ - { - "logs": [ - "endpoints_log" - ], - "monitoredResource": "api" - } - ] - }, - "logs": [ - { - "name": "endpoints_log" - } - ], - "metrics": [ - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/error_count", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user_country" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/referer" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/consumer_id" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_billable_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_bytes", - "valueType": "INT64" - } - ], - "monitoredResources": [ - { - "labels": [ - { - "key": "cloud.googleapis.com/location" - }, - { - "key": "cloud.googleapis.com/uid" - }, - { - "key": "serviceruntime.googleapis.com/api_version" - }, - { - "key": "serviceruntime.googleapis.com/api_method" - }, - { - "key": "serviceruntime.googleapis.com/consumer_project" - }, - { - "key": "cloud.googleapis.com/project" - }, - { - "key": "cloud.googleapis.com/service" - } - ], - "type": "api" - } - ], - "monitoring": { - "consumerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/consumer/request_count", - "serviceruntime.googleapis.com/api/consumer/error_count", - "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "serviceruntime.googleapis.com/api/consumer/total_latencies", - "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "serviceruntime.googleapis.com/api/consumer/request_sizes", - "serviceruntime.googleapis.com/api/consumer/response_sizes", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "serviceruntime.googleapis.com/api/consumer/request_bytes", - "serviceruntime.googleapis.com/api/consumer/response_bytes" - ], - "monitoredResource": "api" - } - ], - "producerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/producer/request_count", - "serviceruntime.googleapis.com/api/producer/error_count", - "serviceruntime.googleapis.com/api/producer/total_latencies", - "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/request_sizes", - "serviceruntime.googleapis.com/api/producer/response_sizes", - "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_durations", - "serviceruntime.googleapis.com/api/producer/request_bytes", - "serviceruntime.googleapis.com/api/producer/response_bytes", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes" - ], - "monitoredResource": "api" - } - ] - }, - "name": "generic.qiwzhang-app-1.appspot.com", - "producerProjectId": "qiwzhang-app-1", - "systemParameters": {}, - "title": "Basic HTTP service configuration", - "types": [ - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "values", - "kind": "TYPE_MESSAGE", - "name": "values", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.ListValue", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "fields", - "kind": "TYPE_MESSAGE", - "name": "fields", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Struct.FieldsEntry" - } - ], - "name": "google.protobuf.Struct", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "key", - "kind": "TYPE_STRING", - "name": "key", - "number": 1 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "value", - "kind": "TYPE_MESSAGE", - "name": "value", - "number": 2, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.Struct.FieldsEntry", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "name": "google.protobuf.Empty", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "nullValue", - "kind": "TYPE_ENUM", - "name": "null_value", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.NullValue" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "numberValue", - "kind": "TYPE_DOUBLE", - "name": "number_value", - "number": 2 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "stringValue", - "kind": "TYPE_STRING", - "name": "string_value", - "number": 3 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "boolValue", - "kind": "TYPE_BOOL", - "name": "bool_value", - "number": 4 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "structValue", - "kind": "TYPE_MESSAGE", - "name": "struct_value", - "number": 5, - "typeUrl": "type.googleapis.com/google.protobuf.Struct" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "listValue", - "kind": "TYPE_MESSAGE", - "name": "list_value", - "number": 6, - "typeUrl": "type.googleapis.com/google.protobuf.ListValue" - } - ], - "name": "google.protobuf.Value", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "usage": { - "rules": [ - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "visibility": {} -} diff --git a/src/envoy/prototype/server_config.pb.txt b/src/envoy/prototype/server_config.pb.txt deleted file mode 100644 index 19cb6cbb2c0..00000000000 --- a/src/envoy/prototype/server_config.pb.txt +++ /dev/null @@ -1,6 +0,0 @@ -cloud_tracing_config { - force_disable: true -} -mixer_options { - mixer_server: "localhost:9091" -} From a6a33fdd6f2cd6b563018ed26f6f9f82bb0f06c8 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Wed, 8 Feb 2017 12:42:11 -0800 Subject: [PATCH 21/49] Update mixer client SHA. (#74) --- src/envoy/mixer/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index d9888fb0daf..d5dfe4ac19c 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -18,7 +18,7 @@ def mixer_client_repositories(bind=True): native.git_repository( name = "mixerclient_git", - commit = "80e450a5126960e8e6337c3631cf2ef984038eab", + commit = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac", remote = "https://github.com/istio/mixerclient.git", ) From d29a195a7478cd5bf56706aed50c874d813cb5e6 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Wed, 8 Feb 2017 12:49:30 -0800 Subject: [PATCH 22/49] Update readme. (#73) --- src/envoy/mixer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 9094d748442..43304785198 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/envoy -c src/envoy/prototype/envoy-mixer.conf + bazel-bin/src/envoy/mixer/envoy -c src/envoy/mixer/envoy-mixer.conf ``` * Then issue HTTP request to proxy. From fdac61b43eaf7963a9f79073a631d790f67535c3 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 16:58:32 -0800 Subject: [PATCH 23/49] Adds Jenkinsfile and updates release-binary to create a SHA. (#71) * Adds Jenkinsfile and update release-binary * Update Jenkinsfile and gitignore * Fixes typo and use normal build Node * Uses default bazel config * Using batch mode * Update bazel memory settings * Do not use Jenkins bazel env * Set .bazelrc for postsubmit --- .bazelrc.jenkins | 8 ++++++ .gitignore | 2 ++ Jenkinsfile | 61 +++++++++++++++++++++++++++++++++++++++++++ script/release-binary | 15 ++++++----- 4 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 .bazelrc.jenkins create mode 100644 Jenkinsfile diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins new file mode 100644 index 00000000000..664f7b13380 --- /dev/null +++ b/.bazelrc.jenkins @@ -0,0 +1,8 @@ +# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. +startup --host_jvm_args=-Xmx8192m +startup --host_jvm_args=-Xms8192m +startup --batch + +# This is so we understand failures better +build --verbose_failures + diff --git a/.gitignore b/.gitignore index a6ef824c1f8..469486c8f36 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /bazel-* +.idea/* +*.iml diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..db6abcaa5f4 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,61 @@ +#!groovy + +@Library('testutils') + +import org.istio.testutils.Utilities +import org.istio.testutils.GitUtilities +import org.istio.testutils.Bazel + +// Utilities shared amongst modules +def gitUtils = new GitUtilities() +def utils = new Utilities() +def bazel = new Bazel() + +node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') +} + +mainFlow(utils) { + if (utils.runStage('PRESUBMIT')) { + def success = true + utils.updatePullRequest('run') + try { + presubmit(gitUtils, bazel) + } catch (Exception e) { + success = false + throw e + } finally { + utils.updatePullRequest('verify', success) + } + } + if (utils.runStage('POSTSUBMIT')) { + buildNode(gitUtils) { + bazel.updateBazelRc() + sh 'script/release-binary' + } + } +} + +def presubmit(gitUtils, bazel) { + buildNode(gitUtils) { + stage('Code Check') { + sh('script/check-style') + } + bazel.updateBazelRc() + stage('Bazel Fetch') { + bazel.fetch('-k //...') + } + stage('Bazel Build') { + bazel.build('//...') + } + stage('Bazel Tests') { + bazel.test('//...') + } + stage('Push Test Binary') { + sh 'script/release-binary' + } + } +} diff --git a/script/release-binary b/script/release-binary index af90b1aa0da..bf83a7d3115 100755 --- a/script/release-binary +++ b/script/release-binary @@ -24,16 +24,19 @@ set -ex BUCKET_NAME="istio-build/proxy" # The proxy binary name. -BINARY_FORMAT='envoy-alpha-%H.tar.gz' -BINARY_NAME="$(git show -q HEAD --pretty=format:"${BINARY_FORMAT}")" +SHA="$(git rev-parse --verify HEAD)" +BINARY_NAME="envoy-alpha-${SHA}.tar.gz" +SHA256_NAME="envoy-alpha-${SHA}.sha256" # Build the binary bazel build --config=release //src/envoy/mixer:envoy_tar -SRC="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" -DST="gs://${BUCKET_NAME}/${BINARY_NAME}" +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +ln "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" +DST="gs://${BUCKET_NAME}/" # Copy it to the bucket. -echo "Copying ${SRC} to ${DST}" -gsutil cp ${SRC} ${DST} +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}" From 6e372fc510de08da504c2dd21060c09255c1aee7 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 8 Feb 2017 19:18:26 -0800 Subject: [PATCH 24/49] Update grpc and protobuf (#70) * protobuf v3.2.0 * grpc v1.1.1 * Align auth lib with grpc 1.1.1 --- contrib/endpoints/repositories.bzl | 5 +- .../auth/lib/auth_jwt_validator.cc | 47 +++-- .../src/api_manager/auth/lib/auth_token.cc | 3 +- .../src/api_manager/auth/lib/grpc_internals.h | 176 +----------------- repositories.bzl | 4 +- 5 files changed, 40 insertions(+), 195 deletions(-) diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index e3363cc86d2..bae14e92d61 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -163,7 +163,7 @@ def grpc_repositories(bind=True): native.git_repository( name = "grpc_git", - commit = "d28417c856366df704200f544e72d31056931bce", + commit = "bb3edafea245a9780cc4c10f0b58da21e8193f38", # v1.1.1 remote = "https://github.com/grpc/grpc.git", ) @@ -190,7 +190,7 @@ def grpc_repositories(bind=True): native.bind( name = "grpc_lib", - actual = "@grpc_git//:grpc++_reflection", + actual = "@grpc_git//:grpc++_codegen_proto", ) def googleapis_repositories(protobuf_repo="@protobuf_git//", bind=True): @@ -333,4 +333,3 @@ def servicecontrol_client_repositories(bind=True): name = "servicecontrol_client", actual = "@servicecontrol_client_git//:service_control_client_lib", ) - diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc index 8da6ef2166c..1c84bdb837d 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc @@ -150,6 +150,8 @@ class JwtValidatorImpl : public JwtValidator { RSA *rsa_; EVP_PKEY *pkey_; EVP_MD_CTX *md_ctx_; + + grpc_exec_ctx exec_ctx_; }; // Gets EVP_MD mapped from an alg (algorithm string). @@ -159,12 +161,12 @@ const EVP_MD *EvpMdFromAlg(const char *alg); size_t HashSizeFromAlg(const char *alg); // Parses str into grpc_json object. Does not own buffer. -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer); +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer); // Gets BIGNUM from b64 string, used for extracting pkey from jwk. // Result owned by rsa_. -BIGNUM *BigNumFromBase64String(const char *b64); +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64); } // namespace @@ -185,7 +187,8 @@ JwtValidatorImpl::JwtValidatorImpl(const char *jwt, size_t jwt_len) x509_(nullptr), rsa_(nullptr), pkey_(nullptr), - md_ctx_(nullptr) { + md_ctx_(nullptr), + exec_ctx_(GRPC_EXEC_CTX_INIT) { header_buffer_ = gpr_empty_slice(); signed_buffer_ = gpr_empty_slice(); sig_buffer_ = gpr_empty_slice(); @@ -204,7 +207,7 @@ JwtValidatorImpl::~JwtValidatorImpl() { grpc_json_destroy(pkey_json_); } if (claims_ != nullptr) { - grpc_jwt_claims_destroy(claims_); + grpc_jwt_claims_destroy(&exec_ctx_, claims_); } if (!GPR_SLICE_IS_EMPTY(header_buffer_)) { gpr_slice_unref(header_buffer_); @@ -304,7 +307,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { if (dot == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } - header_json_ = DecodeBase64AndParseJson(cur, dot - cur, &header_buffer_); + header_json_ = + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &header_buffer_); CreateJoseHeader(); if (header_ == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; @@ -323,7 +327,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { // case, and it is owned by claims_ for successful case. gpr_slice claims_buffer = gpr_empty_slice(); grpc_json *claims_json = - DecodeBase64AndParseJson(cur, dot - cur, &claims_buffer); + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &claims_buffer); if (claims_json == nullptr) { if (!GPR_SLICE_IS_EMPTY(claims_buffer)) { gpr_slice_unref(claims_buffer); @@ -332,10 +336,13 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { } UpdateAudience(claims_json); // Takes ownershp of claims_json and claims_buffer. - claims_ = grpc_jwt_claims_from_json(claims_json, claims_buffer); - if (claims_ == nullptr) { + claims_ = grpc_jwt_claims_from_json(&exec_ctx_, claims_json, claims_buffer); + + // issuer is mandatory. grpc_jwt_claims_issuer checks if claims_ is nullptr. + if (grpc_jwt_claims_issuer(claims_) == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } + // Check timestamp. // Passing in its own audience to skip audience check. // Audience check should be done by the caller. @@ -354,8 +361,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { return GRPC_JWT_VERIFIER_BAD_FORMAT; } cur = dot + 1; - sig_buffer_ = - grpc_base64_decode_with_len(cur, jwt_len - signed_jwt_len - 1, 1); + sig_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, cur, + jwt_len - signed_jwt_len - 1, 1); if (GPR_SLICE_IS_EMPTY(sig_buffer_)) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } @@ -576,9 +583,11 @@ bool JwtValidatorImpl::ExtractPubkeyFromJwk(const grpc_json *jkey) { } const char *rsa_n = GetStringValue(jkey, "n"); - rsa_->n = rsa_n == nullptr ? nullptr : BigNumFromBase64String(rsa_n); + rsa_->n = + rsa_n == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_n); const char *rsa_e = GetStringValue(jkey, "e"); - rsa_->e = rsa_e == nullptr ? nullptr : BigNumFromBase64String(rsa_e); + rsa_->e = + rsa_e == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_e); if (rsa_->e == nullptr || rsa_->n == nullptr) { gpr_log(GPR_ERROR, "Missing RSA public key field."); @@ -651,7 +660,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::VerifyHsSignature(const char *pkey, const EVP_MD *md = EvpMdFromAlg(header_->alg); GPR_ASSERT(md != nullptr); // Checked before. - pkey_buffer_ = grpc_base64_decode_with_len(pkey, pkey_len, 1); + pkey_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, pkey, pkey_len, 1); if (GPR_SLICE_IS_EMPTY(pkey_buffer_)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; @@ -735,11 +744,11 @@ size_t HashSizeFromAlg(const char *alg) { } } -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer) { +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer) { grpc_json *json; - *buffer = grpc_base64_decode_with_len(str, len, 1); + *buffer = grpc_base64_decode_with_len(exec_ctx, str, len, 1); if (GPR_SLICE_IS_EMPTY(*buffer)) { gpr_log(GPR_ERROR, "Invalid base64."); return nullptr; @@ -753,12 +762,12 @@ grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, return json; } -BIGNUM *BigNumFromBase64String(const char *b64) { +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64) { BIGNUM *result = nullptr; gpr_slice bin; if (b64 == nullptr) return nullptr; - bin = grpc_base64_decode(b64, 1); + bin = grpc_base64_decode(exec_ctx, b64, 1); if (GPR_SLICE_IS_EMPTY(bin)) { gpr_log(GPR_ERROR, "Invalid base64 for big num."); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc index 72b53790158..85f06ca0417 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc @@ -200,7 +200,8 @@ char *GenerateJwtClaim(const char *issuer, const char *subject, } char *GenerateSignatueHs256(const char *data, const char *key) { - gpr_slice key_buffer = grpc_base64_decode(key, 1); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_slice key_buffer = grpc_base64_decode(&exec_ctx, key, 1); if (GPR_SLICE_IS_EMPTY(key_buffer)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h index 66074a12776..cdf58840654 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h +++ b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h @@ -15,11 +15,6 @@ #ifndef API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ #define API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ -// This header file contains definitions for all grpc internal -// dependencies. The code that depends on grpc internals should -// include this file instead of including the original headers -// in grpc. - // This header file is for internal use only since it declares grpc // internals that auth depends on. A public header file should not // include any internal grpc header files. @@ -30,171 +25,12 @@ extern "C" { -#include -#include - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json_common.h -////////////////////////////////////////////////////// - -/* The various json types. */ -typedef enum { - GRPC_JSON_OBJECT, - GRPC_JSON_ARRAY, - GRPC_JSON_STRING, - GRPC_JSON_NUMBER, - GRPC_JSON_TRUE, - GRPC_JSON_FALSE, - GRPC_JSON_NULL, - GRPC_JSON_TOP_LEVEL -} grpc_json_type; - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json.h -////////////////////////////////////////////////////// - -/* A tree-like structure to hold json values. The key and value pointers - * are not owned by it. - */ -typedef struct grpc_json { - struct grpc_json *next; - struct grpc_json *prev; - struct grpc_json *child; - struct grpc_json *parent; - - grpc_json_type type; - const char *key; - const char *value; -} grpc_json; - -/* The next two functions are going to parse the input string, and - * modify it in the process, in order to use its space to store - * all of the keys and values for the returned object tree. - * - * They assume UTF-8 input stream, and will output UTF-8 encoded - * strings in the tree. The input stream's UTF-8 isn't validated, - * as in, what you input is what you get as an output. - * - * All the keys and values in the grpc_json objects will be strings - * pointing at your input buffer. - * - * Delete the allocated tree afterward using grpc_json_destroy(). - */ -grpc_json *grpc_json_parse_string_with_len(char *input, size_t size); -grpc_json *grpc_json_parse_string(char *input); - -/* Use these to create or delete a grpc_json object. - * Deletion is recursive. We will not attempt to free any of the strings - * in any of the objects of that tree. - */ -grpc_json *grpc_json_create(grpc_json_type type); -void grpc_json_destroy(grpc_json *json); - -/* This function will create a new string using gpr_realloc, and will - * deserialize the grpc_json tree into it. It'll be zero-terminated, - * but will be allocated in chunks of 256 bytes. - * - * The indent parameter controls the way the output is formatted. - * If indent is 0, then newlines will be suppressed as well, and the - * output will be condensed at its maximum. - */ -char *grpc_json_dump_to_string(grpc_json *json, int indent); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/base64 -////////////////////////////////////////////////////// - -/* Encodes data using base64. It is the caller's responsability to free - the returned char * using gpr_free. Returns nullptr on nullptr input. */ -char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, - int multiline); - -/* Decodes data according to the base64 specification. Returns an empty - slice in case of failure. */ -gpr_slice grpc_base64_decode(const char *b64, int url_safe); - -/* Same as above except that the length is provided by the caller. */ -gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, - int url_safe); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/auth/security/json_key.h -////////////////////////////////////////////////////// - -/* --- auth_json_key parsing. --- */ - -typedef struct { - const char *type; - char *private_key_id; - char *client_id; - char *client_email; - RSA *private_key; -} grpc_auth_json_key; - -/* Creates a json_key object from string. Returns an invalid object if a parsing - error has been encountered. */ -grpc_auth_json_key grpc_auth_json_key_create_from_string( - const char *json_string); - -/* Destructs the object. */ -void grpc_auth_json_key_destruct(grpc_auth_json_key *json_key); - -/* Caller is responsible for calling gpr_free on the returned value. May return - nullptr on invalid input. The scope parameter may be nullptr. */ -char *grpc_jwt_encode_and_sign(const grpc_auth_json_key *json_key, - const char *audience, - gpr_timespec token_lifetime, const char *scope); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/support/string.h -////////////////////////////////////////////////////// - -/* Minimum buffer size for calling ltoa */ -#define GPR_LTOA_MIN_BUFSIZE (3 * sizeof(long)) - -/* Convert a long to a string in base 10; returns the length of the - output string (or 0 on failure). - output must be at least GPR_LTOA_MIN_BUFSIZE bytes long. */ -int gpr_ltoa(long value, char *output); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/jwt_verifier.h -////////////////////////////////////////////////////// - -/* --- grpc_jwt_verifier_status. --- */ - -typedef enum { - GRPC_JWT_VERIFIER_OK = 0, - GRPC_JWT_VERIFIER_BAD_SIGNATURE, - GRPC_JWT_VERIFIER_BAD_FORMAT, - GRPC_JWT_VERIFIER_BAD_AUDIENCE, - GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR, - GRPC_JWT_VERIFIER_TIME_CONSTRAINT_FAILURE, - GRPC_JWT_VERIFIER_GENERIC_ERROR -} grpc_jwt_verifier_status; - -const char *grpc_jwt_verifier_status_to_string(grpc_jwt_verifier_status status); - -/* --- grpc_jwt_claims. --- */ - -typedef struct grpc_jwt_claims grpc_jwt_claims; - -void grpc_jwt_claims_destroy(grpc_jwt_claims *claims); - -/* Returns the whole JSON tree of the claims. */ -const grpc_json *grpc_jwt_claims_json(const grpc_jwt_claims *claims); - -/* Access to registered claims in https://tools.ietf.org/html/rfc7519#page-9 */ -const char *grpc_jwt_claims_subject(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_issuer(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_audience(const grpc_jwt_claims *claims); -gpr_timespec grpc_jwt_claims_expires_at(const grpc_jwt_claims *claims); - -/* --- TESTING ONLY exposed functions. --- */ - -grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer); -grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, - const char *audience); +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_common.h" +#include "src/core/lib/security/credentials/jwt/json_token.h" +#include "src/core/lib/security/credentials/jwt/jwt_verifier.h" +#include "src/core/lib/security/util/b64.h" +#include "src/core/lib/support/string.h" } #endif // API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ diff --git a/repositories.bzl b/repositories.bzl index 8764aa24d32..31fe15ffb63 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -36,7 +36,7 @@ def boringssl_repositories(bind=True): def protobuf_repositories(bind=True): native.git_repository( name = "protobuf_git", - commit = "a428e42072765993ff674fda72863c9f1aa2d268", # v3.1.0 + commit = "593e917c176b5bc5aafa57bf9f6030d749d91cd5", # v3.2.0 remote = "https://github.com/google/protobuf.git", ) @@ -68,7 +68,7 @@ def protobuf_repositories(bind=True): native.bind( name = "protobuf_clib", - actual = "@protobuf_git//:protobuf_lite", + actual = "@protobuf_git//:protoc_lib", ) From 28105ca86ab5487990eb7e13dd8a2ced1fb708be Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 9 Feb 2017 13:10:47 -0800 Subject: [PATCH 25/49] Add sourceService. (#78) --- src/envoy/mixer/http_control.cc | 11 ++++++++++- src/envoy/mixer/http_control.h | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 39e96ecbd3c..b1b5ba816e5 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -25,7 +25,6 @@ namespace Mixer { namespace { const std::string kProxyPeerID = "Istio/Proxy"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; // Define lower case string for X-Forwarded-Host. const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); @@ -44,6 +43,11 @@ const std::string kAttrNameLogMessage = "logMessage"; const std::string kAttrNameResponseTime = "responseTime"; const std::string kAttrNameOriginIp = "originIp"; const std::string kAttrNameOriginHost = "originHost"; + +const std::string kEnvNameSourceService = "SOURCE_SERVICE"; +const std::string kEnvNameTargetService = "TARGET_SERVICE"; + +const std::string kAttrNameSourceService = "sourceService"; const std::string kAttrNameTargetService = "targetService"; Attributes::Value StringValue(const std::string& str) { @@ -147,6 +151,10 @@ HttpControl::HttpControl(const std::string& mixer_server) { options.mixer_server = mixer_server; mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + auto source_service = getenv(kEnvNameSourceService.c_str()); + if (source_service) { + source_service_ = source_service; + } auto target_service = getenv(kEnvNameTargetService.c_str()); if (target_service) { target_service_ = target_service; @@ -157,6 +165,7 @@ void HttpControl::FillCheckAttributes(const HeaderMap& header_map, Attributes* attr) { FillRequestHeaderAttributes(header_map, attr); + SetStringAttribute(kAttrNameSourceService, source_service_, attr); SetStringAttribute(kAttrNameTargetService, target_service_, attr); attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); } diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index 71c7e5620d6..312016e419c 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -55,6 +55,8 @@ class HttpControl final : public Logger::Loggable { // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // Source service + std::string source_service_; // Target service std::string target_service_; }; From 21f1cb3a9333e2cda7658e09aace7fdb96c6b917 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 9 Feb 2017 15:22:10 -0800 Subject: [PATCH 26/49] Add script to build docker image. (#77) * Add script to build docker image. * Add start_envoy for docker image. --- WORKSPACE | 16 +++++ script/release-docker | 46 +++++++++++++ src/envoy/mixer/BUILD | 40 +++++++++++ src/envoy/mixer/README.md | 2 +- .../{envoy-mixer.conf => envoy.conf.template} | 6 +- src/envoy/mixer/start_envoy | 67 +++++++++++++++++++ 6 files changed, 173 insertions(+), 4 deletions(-) create mode 100755 script/release-docker rename src/envoy/mixer/{envoy-mixer.conf => envoy.conf.template} (92%) create mode 100755 src/envoy/mixer/start_envoy diff --git a/WORKSPACE b/WORKSPACE index e152300d73d..3b255e3fa7a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -69,3 +69,19 @@ load( ) envoy_repositories() + +new_http_archive( + name = "docker_ubuntu", + build_file_content = """ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +docker_build( + name = "xenial", + tars = ["xenial/ubuntu-xenial-core-cloudimg-amd64-root.tar.gz"], + visibility = ["//visibility:public"], +) +""", + sha256 = "de31e6fcb843068965de5945c11a6f86399be5e4208c7299fb7311634fb41943", + strip_prefix = "docker-brew-ubuntu-core-e406914e5f648003dfe8329b512c30c9ad0d2f9c", + type = "zip", + url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", +) diff --git a/script/release-docker b/script/release-docker new file mode 100755 index 00000000000..d21caffee55 --- /dev/null +++ b/script/release-docker @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +set -ex + +# This docker image can be used as: +# +# docker run IMAGE -a backend_address -p PORT -m MIXER_SERVER +# + +PROJECT=istio-testing +IMAGE_PREFIX="gcr.io/${PROJECT}/proxy" + +DATE_PART=$(date +"%Y%m%d") +SHA_PART=$(git show -q HEAD --pretty=format:%h) +DOCKER_TAG="${DATE_PART}${SHA_PART}" + +IMAGE_NAME="${IMAGE_PREFIX}:${DOCKER_TAG}" + +gcloud docker --authorize-only + +bazel run --config=release //src/envoy/mixer:proxy "${IMAGE_NAME}" + +gcloud docker -- push "${IMAGE_NAME}" + +IMAGE_LATEST="${IMAGE_PREFIX}:latest" + +docker tag -f "${IMAGE_NAME}" "${IMAGE_LATEST}" + +gcloud docker -- push "${IMAGE_LATEST}" + diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 45c96e25f96..917e627e073 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -16,6 +16,7 @@ # load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") cc_library( name = "filter_lib", @@ -47,3 +48,42 @@ pkg_tar( mode = "0755", package_dir = "/usr/local/bin/", ) + +pkg_tar( + name = "start_envoy_tar", + extension = "tar.gz", + files = ["start_envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) + +pkg_tar( + name = "config_tar", + extension = "tar.gz", + files = ["envoy.conf.template"], + mode = "0666", + package_dir = "/etc/opt/proxy", + tags = ["manual"], +) + +docker_build( + name = "proxy", + base = "@docker_ubuntu//:xenial", + entrypoint = [ + "/usr/local/bin/start_envoy", + "-e", + "/usr/local/bin/envoy", + "-c", + "/etc/opt/proxy/envoy.conf", + "-t", + "/etc/opt/proxy/envoy.conf.template", + ], + ports = ["9090"], + repository = "istio", + tags = ["manual"], + tars = [ + ":config_tar", + ":envoy_tar", + ":start_envoy_tar", + ], +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 43304785198..21f7403f77c 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/envoy -c src/envoy/mixer/envoy-mixer.conf + bazel-bin/src/envoy/mixer/start_envoy ``` * Then issue HTTP request to proxy. diff --git a/src/envoy/mixer/envoy-mixer.conf b/src/envoy/mixer/envoy.conf.template similarity index 92% rename from src/envoy/mixer/envoy-mixer.conf rename to src/envoy/mixer/envoy.conf.template index 3b494c96b59..f1a00d1fb3e 100644 --- a/src/envoy/mixer/envoy-mixer.conf +++ b/src/envoy/mixer/envoy.conf.template @@ -1,7 +1,7 @@ { "listeners": [ { - "port": 9090, + "port": ${PORT}, "bind_to_port": true, "filters": [ { @@ -35,7 +35,7 @@ "type": "both", "name": "mixer", "config": { - "mixer_server": "localhost:9091" + "mixer_server": "${MIXER_SERVER}" } }, { @@ -62,7 +62,7 @@ "lb_type": "round_robin", "hosts": [ { - "url": "tcp://localhost:8080" + "url": "tcp://${BACKEND}" } ] } diff --git a/src/envoy/mixer/start_envoy b/src/envoy/mixer/start_envoy new file mode 100755 index 00000000000..0a24b959ae2 --- /dev/null +++ b/src/envoy/mixer/start_envoy @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +function usage() { + [[ -n "${1}" ]] && echo "${1}" + + cat < "${CONFIG}" + +"${ENVOY}" -c "${CONFIG}" "${DEBUG}" + From 673b902a4da5b03fdad670af363ca265864505e9 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 10 Feb 2017 11:11:57 -0800 Subject: [PATCH 27/49] Use official attribute names (#80) * Use official attribute names * fix format --- src/envoy/mixer/http_control.cc | 37 +++++++++++---------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index b1b5ba816e5..6ae266ef088 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -24,31 +24,25 @@ namespace Http { namespace Mixer { namespace { -const std::string kProxyPeerID = "Istio/Proxy"; - // Define lower case string for X-Forwarded-Host. const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); -const std::string kRequestHeaderPrefix = "requestHeader:"; -const std::string kRequestParamPrefix = "requestParameter:"; -const std::string kResponseHeaderPrefix = "responseHeader:"; +const std::string kRequestHeaderPrefix = "request.headers."; +const std::string kResponseHeaderPrefix = "response.headers."; // Define attribute names -const std::string kAttrNamePeerId = "peerId"; -const std::string kAttrNameURL = "url"; -const std::string kAttrNameHttpMethod = "httpMethod"; -const std::string kAttrNameRequestSize = "requestSize"; -const std::string kAttrNameResponseSize = "responseSize"; -const std::string kAttrNameLogMessage = "logMessage"; -const std::string kAttrNameResponseTime = "responseTime"; -const std::string kAttrNameOriginIp = "originIp"; -const std::string kAttrNameOriginHost = "originHost"; +const std::string kAttrNameRequestPath = "request.path"; +const std::string kAttrNameRequestSize = "request.size"; +const std::string kAttrNameResponseSize = "response.size"; +const std::string kAttrNameResponseTime = "response.time"; +const std::string kAttrNameOriginIp = "origin.ip"; +const std::string kAttrNameOriginHost = "origin.host"; const std::string kEnvNameSourceService = "SOURCE_SERVICE"; const std::string kEnvNameTargetService = "TARGET_SERVICE"; -const std::string kAttrNameSourceService = "sourceService"; -const std::string kAttrNameTargetService = "targetService"; +const std::string kAttrNameSourceService = "source.service"; +const std::string kAttrNameTargetService = "target.service"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -106,14 +100,8 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, }, attr); - // Pass in all Query parameters. - auto path = header_map.Path(); - if (path != nullptr) { - for (const auto& it : Utility::parseQueryString(path->value().c_str())) { - attr->attributes[kRequestParamPrefix + it.first] = StringValue(it.second); - } - } - + SetStringAttribute(kAttrNameRequestPath, header_map.Path()->value().c_str(), + attr); SetStringAttribute(kAttrNameOriginIp, GetFirstForwardedFor(header_map), attr); SetStringAttribute(kAttrNameOriginHost, GetLastForwardedHost(header_map), attr); @@ -167,7 +155,6 @@ void HttpControl::FillCheckAttributes(const HeaderMap& header_map, SetStringAttribute(kAttrNameSourceService, source_service_, attr); SetStringAttribute(kAttrNameTargetService, target_service_, attr); - attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); } void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, From 366f5ae62eaf24888240c8b998cc43f3a0c975ca Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Fri, 10 Feb 2017 11:53:45 -0800 Subject: [PATCH 28/49] Creates a KEY for mixer client dep. Updates release-binary (#79) * Updated mixer repo to use a key for commit * release-binary skip build if file exists. --- script/release-binary | 15 ++++++++++----- src/envoy/mixer/repositories.bzl | 4 +++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/script/release-binary b/script/release-binary index bf83a7d3115..61d446533b8 100755 --- a/script/release-binary +++ b/script/release-binary @@ -19,24 +19,29 @@ set -ex # Make sure to this script on x86_64 Ubuntu Xenial +UBUNTU_RELEASE="$(lsb_release -c -s)" +[[ "${UBUNTU_RELEASE}" == 'xenial' ]] || { echo 'must run on Ubuntu Xenial'; exit 1; } # The bucket name to store proxy binary -BUCKET_NAME="istio-build/proxy" +DST="gs://istio-build/proxy" # The proxy binary name. SHA="$(git rev-parse --verify HEAD)" BINARY_NAME="envoy-alpha-${SHA}.tar.gz" SHA256_NAME="envoy-alpha-${SHA}.sha256" +# If binary already exists skip. +gsutil stat "${DST}/${BINARY_NAME}" \ + && { echo 'Binary already exists'; exit 0; } \ + || echo 'Building a new binary.' + # Build the binary bazel build --config=release //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" ln "${BAZEL_TARGET}" "${BINARY_NAME}" sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" -DST="gs://${BUCKET_NAME}/" # Copy it to the bucket. -echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}" -gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}" - +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index d5dfe4ac19c..1ec26f2a47c 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,10 +15,12 @@ ################################################################################ # +MIXER_CLIENT = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac" + def mixer_client_repositories(bind=True): native.git_repository( name = "mixerclient_git", - commit = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac", + commit = MIXER_CLIENT, remote = "https://github.com/istio/mixerclient.git", ) From a39d55542630f92885911bd9e86b468444baff39 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 10 Feb 2017 15:36:17 -0800 Subject: [PATCH 29/49] Update src/envoy/mixer/README. (#82) --- src/envoy/mixer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 21f7403f77c..20400c64045 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/start_envoy + src/envoy/mixer/start_envoy ``` * Then issue HTTP request to proxy. From 88cfb2474f4cf6c032ead911c17dc9debd779841 Mon Sep 17 00:00:00 2001 From: wattli Date: Fri, 10 Feb 2017 15:49:13 -0800 Subject: [PATCH 30/49] Fix src/envoy/mixer/README.md (#85) --- src/envoy/mixer/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 20400c64045..5bf2198b30d 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -24,6 +24,8 @@ This Proxy will use Envoy and talk to Mixer server. ``` bazel-bin/cmd/server/mixs server + --globalConfigFile testdata/globalconfig.yml + --serviceConfigFile testdata/serviceconfig.yml --logtostderr ``` The server will run at port 9091 From d5a878a10b899332639a46d83ce1e56356575baa Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Mon, 13 Feb 2017 16:27:14 -0800 Subject: [PATCH 31/49] Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. --- src/envoy/mixer/http_control.cc | 24 ++++++------------------ src/envoy/mixer/http_control.h | 9 ++++----- src/envoy/mixer/http_filter.cc | 10 +++++++++- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 6ae266ef088..870d70956fb 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -38,12 +38,6 @@ const std::string kAttrNameResponseTime = "response.time"; const std::string kAttrNameOriginIp = "origin.ip"; const std::string kAttrNameOriginHost = "origin.host"; -const std::string kEnvNameSourceService = "SOURCE_SERVICE"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; - -const std::string kAttrNameSourceService = "source.service"; -const std::string kAttrNameTargetService = "target.service"; - Attributes::Value StringValue(const std::string& str) { Attributes::Value v; v.type = Attributes::Value::STRING; @@ -134,27 +128,21 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, } // namespace -HttpControl::HttpControl(const std::string& mixer_server) { +HttpControl::HttpControl(const std::string& mixer_server, + std::map&& attributes) + : config_attributes_(std::move(attributes)) { ::istio::mixer_client::MixerClientOptions options; options.mixer_server = mixer_server; mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); - - auto source_service = getenv(kEnvNameSourceService.c_str()); - if (source_service) { - source_service_ = source_service; - } - auto target_service = getenv(kEnvNameTargetService.c_str()); - if (target_service) { - target_service_ = target_service; - } } void HttpControl::FillCheckAttributes(const HeaderMap& header_map, Attributes* attr) { FillRequestHeaderAttributes(header_map, attr); - SetStringAttribute(kAttrNameSourceService, source_service_, attr); - SetStringAttribute(kAttrNameTargetService, target_service_, attr); + for (const auto& attribute : config_attributes_) { + SetStringAttribute(attribute.first, attribute.second, attr); + } } void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index 312016e419c..bf52e49a81a 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -37,7 +37,8 @@ typedef std::shared_ptr HttpRequestDataPtr; class HttpControl final : public Logger::Loggable { public: // The constructor. - HttpControl(const std::string& mixer_server); + HttpControl(const std::string& mixer_server, + std::map&& attributes); // Make mixer check call. void Check(HttpRequestDataPtr request_data, HeaderMap& headers, @@ -55,10 +56,8 @@ class HttpControl final : public Logger::Loggable { // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // Source service - std::string source_service_; - // Target service - std::string target_service_; + // The attributes read from the config file. + std::map config_attributes_; }; } // namespace Mixer diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index ec209b1ff98..a37ac032ee1 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -96,7 +96,15 @@ class Config : public Logger::Loggable { __func__); } - http_control_ = std::make_shared(mixer_server); + std::map attributes; + if (config.hasObject("attributes")) { + for (const std::string& attr : config.getStringArray("attributes")) { + attributes[attr] = config.getString(attr); + } + } + + http_control_ = + std::make_shared(mixer_server, std::move(attributes)); log().debug("Called Mixer::Config contructor with mixer_server: ", mixer_server); } From 25adae9db6ab1b499815ba7a8fd87d4309cdff69 Mon Sep 17 00:00:00 2001 From: GregHanson Date: Tue, 14 Feb 2017 15:51:40 -0600 Subject: [PATCH 32/49] updated SHA to point to newer envoy with RDS API feature (#94) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 94f371ebc7d..ec140710592 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "02c6fc97b4c21d25ab596a25208fbe283e927f6a", + commit = "fa1d9680d809668fef2ec9386769c79486029f04", build_file_content = BUILD, ) From 1cdc2ac135ba642d79fb584104232b2964fd5c6e Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 14 Feb 2017 16:35:15 -0800 Subject: [PATCH 33/49] Disable travis on stable branches (#96) --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index d8559616e6d..88be42c285b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ sudo: required dist: xenial +branches: + except: + - stable + lang: go go: From 2f69cc985fc28e8131eba36ee24c21cfa535384e Mon Sep 17 00:00:00 2001 From: Kuat Date: Wed, 15 Feb 2017 14:36:06 -0800 Subject: [PATCH 34/49] Publish debug binaries (no release yet) (#98) --- script/release-binary | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/release-binary b/script/release-binary index 61d446533b8..13c4aea2bc7 100755 --- a/script/release-binary +++ b/script/release-binary @@ -36,7 +36,7 @@ gsutil stat "${DST}/${BINARY_NAME}" \ || echo 'Building a new binary.' # Build the binary -bazel build --config=release //src/envoy/mixer:envoy_tar +bazel build -c dbg //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" ln "${BAZEL_TARGET}" "${BINARY_NAME}" sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" @@ -44,4 +44,3 @@ sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" # Copy it to the bucket. echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" - From 84881332f5e3f31e64b06e46d61964b5ed390bac Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Thu, 16 Feb 2017 13:38:01 -0800 Subject: [PATCH 35/49] Copies the binary instead of linking for release (#102) --- script/release-binary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release-binary b/script/release-binary index 13c4aea2bc7..98bbc67a173 100755 --- a/script/release-binary +++ b/script/release-binary @@ -38,7 +38,7 @@ gsutil stat "${DST}/${BINARY_NAME}" \ # Build the binary bazel build -c dbg //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" -ln "${BAZEL_TARGET}" "${BINARY_NAME}" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" # Copy it to the bucket. From 0804a07680b2bc6a6ab93ffccead554b9680b6b5 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 21 Feb 2017 12:46:14 -0800 Subject: [PATCH 36/49] Not to use api_key if its service is not actived. (#109) --- .../src/api_manager/context/request_context.cc | 3 ++- .../src/api_manager/service_control/proto.cc | 15 +++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 59cc7c3d53f..75bcb177f9d 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -165,7 +165,8 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->operation_name = kUnrecognizedOperation; } info->operation_id = operation_id_; - if (check_response_info_.is_api_key_valid) { + if (check_response_info_.is_api_key_valid && + check_response_info_.service_is_activated) { info->api_key = api_key_; } info->producer_project_id = service_context()->project_id(); diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 3e7eba64d21..694f299d22c 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -956,14 +956,13 @@ Status Proto::FillReportRequest(const ReportRequestInfo& info, } } - // Not to send consumer metrics for following cases: - // 1) api_key is not provided, or - // 2) the service is not activated for the consumer project, - bool send_consumer_metric = true; - if (info.api_key.empty() || - !info.check_response_info.service_is_activated) { - send_consumer_metric = false; - } + // Not to send consumer metrics if api_key is empty. + // api_key is empty in one of following cases: + // 1) api_key is not provided, + // 2) api_key is invalid determined by the server from the Check call. + // 3) the service is not activated for the consumer project. + bool send_consumer_metric = !info.api_key.empty(); + // Populate all metrics. for (auto it = metrics_.begin(), end = metrics_.end(); it != end; it++) { const SupportedMetric* m = *it; From 7444da1c3c22abaab03e9fbe67ba3e77a1cea590 Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 21 Feb 2017 17:36:10 -0800 Subject: [PATCH 37/49] Update envoy and add c-ares (#107) * Update envoy and add c-ares depedencies * Update release script with debug and normal binary * remove debug ls * formatting --- script/release-binary | 14 ++++- src/envoy/repositories.bzl | 125 ++++++++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 2 deletions(-) diff --git a/script/release-binary b/script/release-binary index 98bbc67a173..0ff4dd63f91 100755 --- a/script/release-binary +++ b/script/release-binary @@ -35,7 +35,19 @@ gsutil stat "${DST}/${BINARY_NAME}" \ && { echo 'Binary already exists'; exit 0; } \ || echo 'Building a new binary.' -# Build the binary +# Build the release binary +bazel build --config=release //src/envoy/mixer:envoy_tar +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" + +# Copy it to the bucket. +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" + +# Build the debug binary +BINARY_NAME="envoy-debug-${SHA}.tar.gz" +SHA256_NAME="envoy-debug-${SHA}.sha256" bazel build -c dbg //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index ec140710592..150f445015e 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -442,6 +442,127 @@ cc_library( actual = "@nghttp2_tar//:nghttp2", ) +def ares_repositories(bind=True): + BUILD = """ +cc_library( + name = "ares", + srcs = [ + "ares__close_sockets.c", + "ares__get_hostent.c", + "ares__read_line.c", + "ares__timeval.c", + "ares_cancel.c", + "ares_create_query.c", + "ares_data.c", + "ares_destroy.c", + "ares_expand_name.c", + "ares_expand_string.c", + "ares_fds.c", + "ares_free_hostent.c", + "ares_free_string.c", + "ares_getenv.c", + "ares_gethostbyaddr.c", + "ares_gethostbyname.c", + "ares_getnameinfo.c", + "ares_getopt.c", + "ares_getsock.c", + "ares_init.c", + "ares_library_init.c", + "ares_llist.c", + "ares_mkquery.c", + "ares_nowarn.c", + "ares_options.c", + "ares_parse_a_reply.c", + "ares_parse_aaaa_reply.c", + "ares_parse_mx_reply.c", + "ares_parse_naptr_reply.c", + "ares_parse_ns_reply.c", + "ares_parse_ptr_reply.c", + "ares_parse_soa_reply.c", + "ares_parse_srv_reply.c", + "ares_parse_txt_reply.c", + "ares_platform.c", + "ares_process.c", + "ares_query.c", + "ares_search.c", + "ares_send.c", + "ares_strcasecmp.c", + "ares_strdup.c", + "ares_strerror.c", + "ares_timeout.c", + "ares_version.c", + "ares_writev.c", + "bitncmp.c", + "inet_net_pton.c", + "inet_ntop.c", + "windows_port.c", + ], + hdrs = [ + "ares_config.h", + "ares.h", + "ares_build.h", + "ares_data.h", + "ares_dns.h", + "ares_getenv.h", + "ares_getopt.h", + "ares_inet_net_pton.h", + "ares_iphlpapi.h", + "ares_ipv6.h", + "ares_library_init.h", + "ares_llist.h", + "ares_nowarn.h", + "ares_platform.h", + "ares_private.h", + "ares_rules.h", + "ares_setup.h", + "ares_strcasecmp.h", + "ares_strdup.h", + "ares_version.h", + "ares_writev.h", + "bitncmp.h", + "nameser.h", + "setup_once.h", + ], + copts = [ + "-DHAVE_CONFIG_H", + ], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "config", + srcs = glob(["**/*"]), + outs = ["ares_config.h"], + cmd = "pushd external/cares_git ; ./buildconf ; ./configure ; cp ares_config.h ../../$@", + visibility = ["//visibility:public"], +) + +genrule( + name = "ares_build", + srcs = [ + "ares_build.h.dist", + ], + outs = [ + "ares_build.h", + ], + cmd = "cp $(SRCS) $@", + visibility = ["//visibility:public"], +) +""" + + native.new_git_repository( + name = "cares_git", + remote = "https://github.com/c-ares/c-ares.git", + commit = "7691f773af79bf75a62d1863fd0f13ebf9dc51b1", # v1.12.0 + build_file_content = BUILD, + ) + + if bind: + native.bind( + name = "ares", + actual = "@cares_git//:ares", + ) def envoy_repositories(bind=True): libevent_repositories(bind) @@ -451,6 +572,7 @@ def envoy_repositories(bind=True): http_parser_repositories(bind) rapidjson_repositories(bind) nghttp2_repositories(bind) + ares_repositories(bind) BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") @@ -541,6 +663,7 @@ cc_library( alwayslink=1, deps = [ ":envoy-ratelimit-pb", + "//external:ares", "//external:libssl", "//external:nghttp2", "//external:spdlog", @@ -629,6 +752,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "fa1d9680d809668fef2ec9386769c79486029f04", + commit = "abef64f738d269f6c7f8265233d3ea377eef9596", # 2/16/2017 build_file_content = BUILD, ) From d09325482e840f9ea73b96791535e88b04387a55 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Tue, 21 Feb 2017 18:29:40 -0800 Subject: [PATCH 38/49] Send StatusCode Attributes to Mixer. (#110) --- src/envoy/mixer/http_control.cc | 15 ++++++++++++--- src/envoy/mixer/http_control.h | 2 +- src/envoy/mixer/http_filter.cc | 12 +++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 870d70956fb..bbff6b94992 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -37,6 +37,7 @@ const std::string kAttrNameResponseSize = "response.size"; const std::string kAttrNameResponseTime = "response.time"; const std::string kAttrNameOriginIp = "origin.ip"; const std::string kAttrNameOriginHost = "origin.host"; +const std::string kResponseStatusCode = "response.status.code"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -113,7 +114,7 @@ void FillResponseHeaderAttributes(const HeaderMap& header_map, } void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, - Attributes* attr) { + int check_status_code, Attributes* attr) { if (info.bytesReceived() >= 0) { attr->attributes[kAttrNameRequestSize] = Int64Value(info.bytesReceived()); } @@ -124,6 +125,13 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, attr->attributes[kAttrNameResponseTime] = Int64Value(info.duration().count()); } + if (info.responseCode().valid()) { + attr->attributes[kResponseStatusCode] = + StringValue(std::to_string(info.responseCode().value())); + } else { + attr->attributes[kResponseStatusCode] = + StringValue(std::to_string(check_status_code)); + } } } // namespace @@ -155,11 +163,12 @@ void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, void HttpControl::Report(HttpRequestDataPtr request_data, const HeaderMap* response_headers, const AccessLog::RequestInfo& request_info, - DoneFunc on_done) { + int check_status, DoneFunc on_done) { // Use all Check attributes for Report. // Add additional Report attributes. FillResponseHeaderAttributes(*response_headers, &request_data->attributes); - FillRequestInfoAttributes(request_info, &request_data->attributes); + FillRequestInfoAttributes(request_info, check_status, + &request_data->attributes); log().debug("Send Report: {}", request_data->attributes.DebugString()); mixer_client_->Report(request_data->attributes, on_done); } diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index bf52e49a81a..65863d950fa 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -47,7 +47,7 @@ class HttpControl final : public Logger::Loggable { // Make mixer report call. void Report(HttpRequestDataPtr request_data, const HeaderMap* response_headers, - const AccessLog::RequestInfo& request_info, + const AccessLog::RequestInfo& request_info, int check_status_code, ::istio::mixer_client::DoneFunc on_done); private: diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index a37ac032ee1..d0796b506dd 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -105,7 +105,7 @@ class Config : public Logger::Loggable { http_control_ = std::make_shared(mixer_server, std::move(attributes)); - log().debug("Called Mixer::Config contructor with mixer_server: ", + log().debug("Called Mixer::Config constructor with mixer_server: ", mixer_server); } @@ -126,12 +126,14 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { StreamEncoderFilterCallbacks* encoder_callbacks_; bool initiating_call_; + int check_status_code_; public: Instance(ConfigPtr config) : http_control_(config->http_control()), state_(NotStarted), - initiating_call_(false) { + initiating_call_(false), + check_status_code_(HttpCode(StatusCode::UNKNOWN)) { Log().debug("Called Mixer::Instance : {}", __func__); } @@ -190,8 +192,8 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { status.ToString()); if (!status.ok() && state_ != Responded) { state_ = Responded; - Utility::sendLocalReply(*decoder_callbacks_, - Code(HttpCode(status.error_code())), + check_status_code_ = HttpCode(status.error_code()); + Utility::sendLocalReply(*decoder_callbacks_, Code(check_status_code_), status.ToString()); return; } @@ -229,7 +231,7 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { // The class may be gone when it is called. // Log() is a static function so it is OK. http_control_->Report(request_data_, response_headers, request_info, - [](const Status& status) { + check_status_code_, [](const Status& status) { Log().debug("Report returns status: {}", status.ToString()); }); From ef1aecf0f18cfbdb44a0fa247eba510b4785595c Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 22 Feb 2017 10:04:36 -0800 Subject: [PATCH 39/49] Add send_attribute filter. (#115) * Add send_attribute filter. * Fix format * rename variable serialized_attributes_ * Address the comments. --- src/envoy/mixer/BUILD | 13 +++ src/envoy/mixer/README.md | 48 +++++++++ src/envoy/mixer/envoy.conf.template | 68 +++++++++++- src/envoy/mixer/forward_attribute_filter.cc | 114 ++++++++++++++++++++ src/envoy/mixer/http_control.cc | 30 ++++-- src/envoy/mixer/http_control.h | 4 +- src/envoy/mixer/http_filter.cc | 22 ++-- src/envoy/mixer/string_map.proto | 23 ++++ src/envoy/mixer/utils.cc | 50 +++++++++ src/envoy/mixer/utils.h | 39 +++++++ 10 files changed, 391 insertions(+), 20 deletions(-) create mode 100644 src/envoy/mixer/forward_attribute_filter.cc create mode 100644 src/envoy/mixer/string_map.proto create mode 100644 src/envoy/mixer/utils.cc create mode 100644 src/envoy/mixer/utils.h diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 917e627e073..d04b5105029 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -17,15 +17,28 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") + +cc_proto_library( + name = "string_map_proto", + srcs = ["string_map.proto"], + default_runtime = "//external:protobuf", + protoc = "//external:protoc", + visibility = ["//visibility:public"], +) cc_library( name = "filter_lib", srcs = [ + "forward_attribute_filter.cc", "http_control.cc", "http_control.h", "http_filter.cc", + "utils.cc", + "utils.h", ], deps = [ + ":string_map_proto", "//external:mixer_client_lib", "@envoy_git//:envoy-common", ], diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 5bf2198b30d..d7d39f30e03 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -49,3 +49,51 @@ This Proxy will use Envoy and talk to Mixer server. curl http://localhost:9090/echo -d "hello world" ``` +## How to configurate HTTP filters + +This module has two HTTP filters: +1. mixer filter: intercept all HTTP requests, call the mixer. +2. forward_attribute filter: Forward attributes to the upstream istio/proxy. + +### *mixer* filter: + +This filter will intercept all HTTP requests and call Mixer. Here is its config: + +``` + "filters": [ + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "${MIXER_SERVER}", + "attributes" : { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* mixer_server is required +* attributes: these attributes will be send to the mixer + +### *forward_attribute* HTTP filter: + +This filer will forward attributes to the upstream istio/proxy. + +``` + "filters": [ + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* attributes: these attributes will be forwarded to the upstream istio/proxy. + + + diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template index f1a00d1fb3e..c5377632388 100644 --- a/src/envoy/mixer/envoy.conf.template +++ b/src/envoy/mixer/envoy.conf.template @@ -35,7 +35,62 @@ "type": "both", "name": "mixer", "config": { - "mixer_server": "${MIXER_SERVER}" + "mixer_server": "${MIXER_SERVER}", + "attributes": { + "target.uid": "POD222", + "target.namespace": "XYZ222" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + }, + { + "port": 7070, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service2" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "source.uid": "POD11", + "source.namespace": "XYZ11" + } } }, { @@ -65,6 +120,17 @@ "url": "tcp://${BACKEND}" } ] + }, + { + "name": "service2", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:9090" + } + ] } ] } diff --git a/src/envoy/mixer/forward_attribute_filter.cc b/src/envoy/mixer/forward_attribute_filter.cc new file mode 100644 index 00000000000..08d58916f89 --- /dev/null +++ b/src/envoy/mixer/forward_attribute_filter.cc @@ -0,0 +1,114 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/base64.h" +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/utils.h" + +namespace Http { +namespace ForwardAttribute { +namespace { + +// The Json object name to specify attributes which will be forwarded +// to the upstream istio proxy. +const std::string kJsonNameAttributes("attributes"); + +} // namespace + +class Config : public Logger::Loggable { + private: + std::string attributes_; + + public: + Config(const Json::Object& config) { + Utils::StringMap attributes = + Utils::ExtractStringMap(config, kJsonNameAttributes); + if (!attributes.empty()) { + std::string serialized_str = Utils::SerializeStringMap(attributes); + attributes_ = + Base64::encode(serialized_str.c_str(), serialized_str.size()); + } + } + + const std::string& attributes() const { return attributes_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class ForwardAttributeFilter : public Http::StreamDecoderFilter { + private: + ConfigPtr config_; + + public: + ForwardAttributeFilter(ConfigPtr config) : config_(config) {} + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + if (!config_->attributes().empty()) { + headers.addStatic(Utils::kIstioAttributeHeader, config_->attributes()); + } + return FilterHeadersStatus::Continue; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + return FilterTrailersStatus::Continue; + } + + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override {} +}; + +} // namespace ForwardAttribute +} // namespace Http + +namespace Server { +namespace Configuration { + +class ForwardAttributeConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Decoder || name != "forward_attribute") { + return nullptr; + } + + Http::ForwardAttribute::ConfigPtr add_header_config( + new Http::ForwardAttribute::Config(config)); + return [add_header_config]( + Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::ForwardAttribute::ForwardAttributeFilter( + add_header_config)); + callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index bbff6b94992..7e1f7deffcd 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -1,4 +1,5 @@ -/* +/* Copyright 2017 Istio Authors. All Rights Reserved. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,9 +14,14 @@ */ #include "src/envoy/mixer/http_control.h" + +#include "common/common/base64.h" #include "common/common/utility.h" #include "common/http/utility.h" +#include "src/envoy/mixer/string_map.pb.h" +#include "src/envoy/mixer/utils.h" + using ::google::protobuf::util::Status; using ::istio::mixer_client::Attributes; using ::istio::mixer_client::DoneFunc; @@ -77,7 +83,8 @@ std::string GetLastForwardedHost(const HeaderMap& header_map) { if (entry == nullptr) { return ""; } - auto xff_list = StringUtil::split(entry->value().c_str(), ','); + std::vector xff_list = + StringUtil::split(entry->value().c_str(), ','); if (xff_list.empty()) { return ""; } @@ -89,7 +96,7 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, // Pass in all headers header_map.iterate( [](const HeaderEntry& header, void* context) { - auto attr = static_cast(context); + Attributes* attr = static_cast(context); attr->attributes[kRequestHeaderPrefix + header.key().c_str()] = StringValue(header.value().c_str()); }, @@ -106,7 +113,7 @@ void FillResponseHeaderAttributes(const HeaderMap& header_map, Attributes* attr) { header_map.iterate( [](const HeaderEntry& header, void* context) { - auto attr = static_cast(context); + Attributes* attr = static_cast(context); attr->attributes[kResponseHeaderPrefix + header.key().c_str()] = StringValue(header.value().c_str()); }, @@ -144,8 +151,19 @@ HttpControl::HttpControl(const std::string& mixer_server, mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); } -void HttpControl::FillCheckAttributes(const HeaderMap& header_map, - Attributes* attr) { +void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { + // Extract attributes from x-istio-attributes header + const HeaderEntry* entry = header_map.get(Utils::kIstioAttributeHeader); + if (entry) { + ::istio::proxy::mixer::StringMap pb; + std::string str(entry->value().c_str(), entry->value().size()); + pb.ParseFromString(Base64::decode(str)); + for (const auto& it : pb.map()) { + SetStringAttribute(it.first, it.second, attr); + } + header_map.remove(Utils::kIstioAttributeHeader); + } + FillRequestHeaderAttributes(header_map, attr); for (const auto& attribute : config_attributes_) { diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index 65863d950fa..c5938cf369d 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -1,4 +1,4 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. +/* Copyright 2017 Istio Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,7 @@ class HttpControl final : public Logger::Loggable { ::istio::mixer_client::DoneFunc on_done); private: - void FillCheckAttributes(const HeaderMap& header_map, + void FillCheckAttributes(HeaderMap& header_map, ::istio::mixer_client::Attributes* attr); // The mixer client diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index d0796b506dd..a990b0fd7e3 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -1,4 +1,4 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. +/* Copyright 2017 Istio Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include "envoy/server/instance.h" #include "server/config/network/http_connection_manager.h" #include "src/envoy/mixer/http_control.h" +#include "src/envoy/mixer/utils.h" using ::google::protobuf::util::Status; using StatusCode = ::google::protobuf::util::error::Code; @@ -30,8 +31,11 @@ namespace Http { namespace Mixer { namespace { -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); +// The Json object name for mixer-server. +const std::string kJsonNameMixerServer("mixer_server"); + +// The Json object name for static attributes. +const std::string kJsonNameMixerAttributes("attributes"); // Convert Status::code to HTTP code int HttpCode(int code) { @@ -88,20 +92,16 @@ class Config : public Logger::Loggable { Config(const Json::Object& config, Server::Instance& server) : cm_(server.clusterManager()) { std::string mixer_server; - if (config.hasObject("mixer_server")) { - mixer_server = config.getString("mixer_server"); + if (config.hasObject(kJsonNameMixerServer)) { + mixer_server = config.getString(kJsonNameMixerServer); } else { log().error( "mixer_server is required but not specified in the config: {}", __func__); } - std::map attributes; - if (config.hasObject("attributes")) { - for (const std::string& attr : config.getStringArray("attributes")) { - attributes[attr] = config.getString(attr); - } - } + std::map attributes = + Utils::ExtractStringMap(config, kJsonNameMixerAttributes); http_control_ = std::make_shared(mixer_server, std::move(attributes)); diff --git a/src/envoy/mixer/string_map.proto b/src/envoy/mixer/string_map.proto new file mode 100644 index 00000000000..bd9140c37d7 --- /dev/null +++ b/src/envoy/mixer/string_map.proto @@ -0,0 +1,23 @@ +// Copyright 2017 Istio Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.proxy.mixer; + +// A message with a map of string to string. It is used to serialize +// a string map. +message StringMap { + map map = 1; +} diff --git a/src/envoy/mixer/utils.cc b/src/envoy/mixer/utils.cc new file mode 100644 index 00000000000..6dc3a78356d --- /dev/null +++ b/src/envoy/mixer/utils.cc @@ -0,0 +1,50 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/utils.h" +#include "src/envoy/mixer/string_map.pb.h" + +namespace Http { +namespace Utils { + +const LowerCaseString kIstioAttributeHeader("x-istio-attributes"); + +StringMap ExtractStringMap(const Json::Object& json, const std::string& name) { + StringMap map; + if (json.hasObject(name)) { + Json::ObjectPtr json_obj = json.getObject(name); + Json::Object* raw_obj = json_obj.get(); + json_obj->iterate( + [&map, raw_obj](const std::string& key, const Json::Object&) -> bool { + map[key] = raw_obj->getString(key); + return true; + }); + } + return map; +} + +std::string SerializeStringMap(const StringMap& string_map) { + ::istio::proxy::mixer::StringMap pb; + ::google::protobuf::Map* map_pb = pb.mutable_map(); + for (const auto& it : string_map) { + (*map_pb)[it.first] = it.second; + } + std::string str; + pb.SerializeToString(&str); + return str; +} + +} // namespace Utils +} // namespace Http diff --git a/src/envoy/mixer/utils.h b/src/envoy/mixer/utils.h new file mode 100644 index 00000000000..9273a9a2d8b --- /dev/null +++ b/src/envoy/mixer/utils.h @@ -0,0 +1,39 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/http/headers.h" +#include "envoy/json/json_object.h" + +namespace Http { +namespace Utils { + +// The internal header to pass istio attributes. +extern const LowerCaseString kIstioAttributeHeader; + +// The string map. +typedef std::map StringMap; + +// Extracts name/value attributes from a json object. +StringMap ExtractStringMap(const Json::Object& json, const std::string& name); + +// Serialize a string map to string. +std::string SerializeStringMap(const StringMap& map); + +} // namespace Utils +} // namespace Http From 20e19d217be714ce1440b6dcdeab21cf1c14c2de Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Wed, 22 Feb 2017 12:50:08 -0800 Subject: [PATCH 40/49] Fail request if api_key is not valid. (#116) * Fail request if api_key is not valid. * Format code. * Update comments. * Address comment. --- .../src/api_manager/service_control/aggregated.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.cc b/contrib/endpoints/src/api_manager/service_control/aggregated.cc index d2754d69dbd..e59a69dc8cc 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.cc @@ -287,8 +287,14 @@ void Aggregated::Check( if (status.ok()) { Status status = Proto::ConvertCheckResponse( *response, service_control_proto_.service_name(), &response_info); - // If allow_unregistered_calls is true, it is always OK to proceed. - if (allow_unregistered_calls) { + // If server replied with either invalid api_key or not activated service, + // the request is rejected even allow_unregistered_calls is true. Most + // likely, users provide a wrong api key. By failing the request, the + // users will be notified with the error and have chance to correct it. + // Otherwise, the Report call will fail. It is very hard to notice and + // debug the Report failure. + if (allow_unregistered_calls && response_info.is_api_key_valid && + response_info.service_is_activated) { on_done(Status::OK, response_info); } else { on_done(status, response_info); From 7fe10df1ef85fd5b3417c58173387bd077360027 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 23 Feb 2017 16:04:30 -0800 Subject: [PATCH 41/49] Rename response.http.code (#125) --- src/envoy/mixer/http_control.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 7e1f7deffcd..84e31de5528 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -43,7 +43,7 @@ const std::string kAttrNameResponseSize = "response.size"; const std::string kAttrNameResponseTime = "response.time"; const std::string kAttrNameOriginIp = "origin.ip"; const std::string kAttrNameOriginHost = "origin.host"; -const std::string kResponseStatusCode = "response.status.code"; +const std::string kResponseHttpCode = "response.http.code"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -133,11 +133,10 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, Int64Value(info.duration().count()); } if (info.responseCode().valid()) { - attr->attributes[kResponseStatusCode] = - StringValue(std::to_string(info.responseCode().value())); + attr->attributes[kResponseHttpCode] = + Int64Value(info.responseCode().value()); } else { - attr->attributes[kResponseStatusCode] = - StringValue(std::to_string(check_status_code)); + attr->attributes[kResponseHttpCode] = Int64Value(check_status_code); } } From fdfb73f4ea245dc158d8cce5f4fd66f551099465 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 24 Feb 2017 17:21:27 -0800 Subject: [PATCH 42/49] Send headers as string map. (#129) * Send headers as string map. * Remove origin.ip and origin.host. * Fix format --- src/envoy/mixer/http_control.cc | 120 +++++++++++++++---------------- src/envoy/mixer/repositories.bzl | 2 +- 2 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 84e31de5528..677a8a9a3f3 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -30,19 +30,17 @@ namespace Http { namespace Mixer { namespace { -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); - -const std::string kRequestHeaderPrefix = "request.headers."; -const std::string kResponseHeaderPrefix = "response.headers."; - // Define attribute names -const std::string kAttrNameRequestPath = "request.path"; -const std::string kAttrNameRequestSize = "request.size"; -const std::string kAttrNameResponseSize = "response.size"; -const std::string kAttrNameResponseTime = "response.time"; -const std::string kAttrNameOriginIp = "origin.ip"; -const std::string kAttrNameOriginHost = "origin.host"; +const std::string kRequestPath = "request.path"; +const std::string kRequestHost = "request.host"; +const std::string kRequestSize = "request.size"; +const std::string kRequestTime = "request.time"; +const std::string kRequestHeaders = "request.headers"; + +const std::string kResponseHeaders = "response.headers"; +const std::string kResponseSize = "response.size"; +const std::string kResponseTime = "response.time"; +const std::string kResponseLatency = "response.latency"; const std::string kResponseHttpCode = "response.http.code"; Attributes::Value StringValue(const std::string& str) { @@ -52,6 +50,14 @@ Attributes::Value StringValue(const std::string& str) { return v; } +Attributes::Value StringMapValue( + std::map&& string_map) { + Attributes::Value v; + v.type = Attributes::Value::STRING_MAP; + v.string_map_v.swap(string_map); + return v; +} + Attributes::Value Int64Value(int64_t value) { Attributes::Value v; v.type = Attributes::Value::INT64; @@ -59,79 +65,70 @@ Attributes::Value Int64Value(int64_t value) { return v; } -void SetStringAttribute(const std::string& name, const std::string& value, - Attributes* attr) { - if (!value.empty()) { - attr->attributes[name] = StringValue(value); - } +Attributes::Value TimeValue( + std::chrono::time_point value) { + Attributes::Value v; + v.type = Attributes::Value::TIME; + v.time_v = value; + return v; } -std::string GetFirstForwardedFor(const HeaderMap& header_map) { - if (!header_map.ForwardedFor()) { - return ""; - } - std::vector xff_address_list = - StringUtil::split(header_map.ForwardedFor()->value().c_str(), ','); - if (xff_address_list.empty()) { - return ""; - } - return xff_address_list.front(); +Attributes::Value DurationValue(std::chrono::nanoseconds value) { + Attributes::Value v; + v.type = Attributes::Value::DURATION; + v.duration_nanos_v = value; + return v; } -std::string GetLastForwardedHost(const HeaderMap& header_map) { - const HeaderEntry* entry = header_map.get(kHeaderNameXFH); - if (entry == nullptr) { - return ""; - } - std::vector xff_list = - StringUtil::split(entry->value().c_str(), ','); - if (xff_list.empty()) { - return ""; +void SetStringAttribute(const std::string& name, const std::string& value, + Attributes* attr) { + if (!value.empty()) { + attr->attributes[name] = StringValue(value); } - return xff_list.back(); } -void FillRequestHeaderAttributes(const HeaderMap& header_map, - Attributes* attr) { - // Pass in all headers +std::map ExtractHeaders(const HeaderMap& header_map) { + std::map headers; header_map.iterate( [](const HeaderEntry& header, void* context) { - Attributes* attr = static_cast(context); - attr->attributes[kRequestHeaderPrefix + header.key().c_str()] = - StringValue(header.value().c_str()); + std::map* header_map = + static_cast*>(context); + (*header_map)[header.key().c_str()] = header.value().c_str(); }, - attr); + &headers); + return headers; +} - SetStringAttribute(kAttrNameRequestPath, header_map.Path()->value().c_str(), - attr); - SetStringAttribute(kAttrNameOriginIp, GetFirstForwardedFor(header_map), attr); - SetStringAttribute(kAttrNameOriginHost, GetLastForwardedHost(header_map), - attr); +void FillRequestHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + SetStringAttribute(kRequestPath, header_map.Path()->value().c_str(), attr); + SetStringAttribute(kRequestHost, header_map.Host()->value().c_str(), attr); + attr->attributes[kRequestTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kRequestHeaders] = + StringMapValue(ExtractHeaders(header_map)); } void FillResponseHeaderAttributes(const HeaderMap& header_map, Attributes* attr) { - header_map.iterate( - [](const HeaderEntry& header, void* context) { - Attributes* attr = static_cast(context); - attr->attributes[kResponseHeaderPrefix + header.key().c_str()] = - StringValue(header.value().c_str()); - }, - attr); + attr->attributes[kResponseHeaders] = + StringMapValue(ExtractHeaders(header_map)); + attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); } void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, int check_status_code, Attributes* attr) { if (info.bytesReceived() >= 0) { - attr->attributes[kAttrNameRequestSize] = Int64Value(info.bytesReceived()); + attr->attributes[kRequestSize] = Int64Value(info.bytesReceived()); } if (info.bytesSent() >= 0) { - attr->attributes[kAttrNameResponseSize] = Int64Value(info.bytesSent()); + attr->attributes[kResponseSize] = Int64Value(info.bytesSent()); } - if (info.duration().count() >= 0) { - attr->attributes[kAttrNameResponseTime] = - Int64Value(info.duration().count()); + + if (info.duration().count() > 0) { + attr->attributes[kResponseLatency] = DurationValue( + std::chrono::duration_cast(info.duration())); } + if (info.responseCode().valid()) { attr->attributes[kResponseHttpCode] = Int64Value(info.responseCode().value()); @@ -184,6 +181,7 @@ void HttpControl::Report(HttpRequestDataPtr request_data, // Use all Check attributes for Report. // Add additional Report attributes. FillResponseHeaderAttributes(*response_headers, &request_data->attributes); + FillRequestInfoAttributes(request_info, check_status, &request_data->attributes); log().debug("Send Report: {}", request_data->attributes.DebugString()); diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 1ec26f2a47c..76a4aa9a113 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac" +MIXER_CLIENT = "d0c57ab0d74887feb11f8644fa98538208f14dbc" def mixer_client_repositories(bind=True): native.git_repository( From 3d402e6defd367ea3e3047a86451110951a4ed22 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Fri, 24 Feb 2017 17:50:23 -0800 Subject: [PATCH 43/49] unify bazel's docker build targets with other istio repos (#127) --- WORKSPACE | 8 ++++++++ docker/BUILD | 11 +++++++++++ src/envoy/mixer/BUILD | 12 ++++++++---- src/envoy/mixer/proxy_docker.bzl | 9 +++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 docker/BUILD create mode 100644 src/envoy/mixer/proxy_docker.bzl diff --git a/WORKSPACE b/WORKSPACE index 3b255e3fa7a..69546d28831 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -85,3 +85,11 @@ docker_build( type = "zip", url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", ) + +# See github.com/istio/manager/blob/master/docker/debug/build-and-publish-debug-image.sh +# for instructions on how to re-build and publish this base image layer. +http_file( + name = "ubuntu_xenial_debug", + url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug.tar.gz", + sha256 = "02f0ea4b04012562dec4b75ee4337ac77a0003418d02a91bde1b4b4d162a41e4", +) diff --git a/docker/BUILD b/docker/BUILD new file mode 100644 index 00000000000..6e017ee6064 --- /dev/null +++ b/docker/BUILD @@ -0,0 +1,11 @@ +# Add aliases for envoy specific targets so that the docker build +# targets are consistent for all istio repos, e.g. //docker: +alias( + name = "proxy", + actual = "//src/envoy/mixer:proxy", +) + +alias( + name = "proxy_debug", + actual = "//src/envoy/mixer:proxy_debug", +) diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index d04b5105029..132bc364a1d 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -15,8 +15,9 @@ ################################################################################ # + load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") -load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +load("//src/envoy/mixer:proxy_docker.bzl", "proxy_docker_build") load("@protobuf_git//:protobuf.bzl", "cc_proto_library") cc_proto_library( @@ -79,9 +80,11 @@ pkg_tar( tags = ["manual"], ) -docker_build( - name = "proxy", - base = "@docker_ubuntu//:xenial", +proxy_docker_build( + images = [ + {"name": "proxy", "base": "@docker_ubuntu//:xenial"}, + {"name": "proxy_debug", "base": "@ubuntu_xenial_debug//file"}, + ], entrypoint = [ "/usr/local/bin/start_envoy", "-e", @@ -99,4 +102,5 @@ docker_build( ":envoy_tar", ":start_envoy_tar", ], + visibility = ["//visibility:public"], ) diff --git a/src/envoy/mixer/proxy_docker.bzl b/src/envoy/mixer/proxy_docker.bzl new file mode 100644 index 00000000000..8522729628f --- /dev/null +++ b/src/envoy/mixer/proxy_docker.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") + +def proxy_docker_build(images, **kwargs): + for image in images: + docker_build( + name = image['name'], + base = image['base'], + **kwargs + ) From d71d5f3d2b4fa47d18e060b81a5b9fb3f2032f33 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Tue, 28 Feb 2017 14:21:18 -0800 Subject: [PATCH 44/49] update base debug docker image reference (#133) --- WORKSPACE | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 69546d28831..ddfa05d578b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -86,10 +86,13 @@ docker_build( url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", ) + +DEBUG_BASE_IMAGE_SHA="3f57ae2aceef79e4000fb07ec850bbf4bce811e6f81dc8cfd970e16cdf33e622" + # See github.com/istio/manager/blob/master/docker/debug/build-and-publish-debug-image.sh # for instructions on how to re-build and publish this base image layer. http_file( name = "ubuntu_xenial_debug", - url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug.tar.gz", - sha256 = "02f0ea4b04012562dec4b75ee4337ac77a0003418d02a91bde1b4b4d162a41e4", + url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug-" + DEBUG_BASE_IMAGE_SHA + ".tar.gz", + sha256 = DEBUG_BASE_IMAGE_SHA, ) From f5281adb8e1a84e93815e06c67864a55e2f0d8e5 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 28 Feb 2017 15:12:53 -0800 Subject: [PATCH 45/49] Update postsubmit to create docker images (#132) --- Jenkinsfile | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index db6abcaa5f4..a8ded63359b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,30 +11,20 @@ def gitUtils = new GitUtilities() def utils = new Utilities() def bazel = new Bazel() -node { - gitUtils.initialize() - // Proxy does build work correctly with Hazelcast. - // Must use .bazelrc.jenkins - bazel.setVars('', '') -} - mainFlow(utils) { - if (utils.runStage('PRESUBMIT')) { - def success = true - utils.updatePullRequest('run') - try { + pullRequest(utils) { + node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') + } + + if (utils.runStage('PRESUBMIT')) { presubmit(gitUtils, bazel) - } catch (Exception e) { - success = false - throw e - } finally { - utils.updatePullRequest('verify', success) } - } - if (utils.runStage('POSTSUBMIT')) { - buildNode(gitUtils) { - bazel.updateBazelRc() - sh 'script/release-binary' + if (utils.runStage('POSTSUBMIT')) { + postsubmit(gitUtils, bazel, utils) } } } @@ -59,3 +49,17 @@ def presubmit(gitUtils, bazel) { } } } + +def postsubmit(gitUtils, bazel, utils) { + buildNode(gitUtils) { + bazel.updateBazelRc() + stage('Push Binary') { + sh 'script/release-binary' + } + stage('Docker Push') { + def images = 'proxy,proxy_debug' + def tags = "${gitUtils.GIT_SHORT_SHA},\$(date +%Y-%m-%d-%H.%M.%S),latest" + utils.publishDockerImages(images, tags) + } + } +} \ No newline at end of file From 0c7c549dd722e01fb7690f9d588ae645df9dde6a Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 28 Feb 2017 16:54:00 -0800 Subject: [PATCH 46/49] Adding config release for bazel build (#135) --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a8ded63359b..9c59a146e50 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -59,7 +59,7 @@ def postsubmit(gitUtils, bazel, utils) { stage('Docker Push') { def images = 'proxy,proxy_debug' def tags = "${gitUtils.GIT_SHORT_SHA},\$(date +%Y-%m-%d-%H.%M.%S),latest" - utils.publishDockerImages(images, tags) + utils.publishDockerImages(images, tags, 'release') } } } \ No newline at end of file From fc9497a8d99820c9084dec8a23b5b723949bfbdf Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 28 Feb 2017 17:11:01 -0800 Subject: [PATCH 47/49] Fix mixer client crash. (#136) --- src/envoy/mixer/http_control.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 677a8a9a3f3..6e64659c2bb 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -108,10 +108,12 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, StringMapValue(ExtractHeaders(header_map)); } -void FillResponseHeaderAttributes(const HeaderMap& header_map, +void FillResponseHeaderAttributes(const HeaderMap* header_map, Attributes* attr) { - attr->attributes[kResponseHeaders] = - StringMapValue(ExtractHeaders(header_map)); + if (header_map) { + attr->attributes[kResponseHeaders] = + StringMapValue(ExtractHeaders(*header_map)); + } attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); } @@ -180,7 +182,7 @@ void HttpControl::Report(HttpRequestDataPtr request_data, int check_status, DoneFunc on_done) { // Use all Check attributes for Report. // Add additional Report attributes. - FillResponseHeaderAttributes(*response_headers, &request_data->attributes); + FillResponseHeaderAttributes(response_headers, &request_data->attributes); FillRequestInfoAttributes(request_info, check_status, &request_data->attributes); From 83e1d58c8e841c47fb9c4b14352249f9e96f927e Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 1 Mar 2017 16:30:27 -0800 Subject: [PATCH 48/49] Get mixerclient with response parsing. (#138) --- src/envoy/mixer/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 76a4aa9a113..2f665d71c4b 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "d0c57ab0d74887feb11f8644fa98538208f14dbc" +MIXER_CLIENT = "5b5745f29ac5a8babe79ada573defaa83f3bb9e7" def mixer_client_repositories(bind=True): native.git_repository( From a118aea2493fabbd2bfadb4dcf5f873f81980402 Mon Sep 17 00:00:00 2001 From: wattli Date: Wed, 1 Mar 2017 19:00:35 -0800 Subject: [PATCH 49/49] Update nghttp2 to sync with envoy (#140) * Fix src/envoy/mixer/README.md * Update nghttp2 to sync with envoy * update * fix typo --- src/envoy/repositories.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 150f445015e..ca7e9468ded 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -431,8 +431,8 @@ cc_library( native.new_http_archive( name = "nghttp2_tar", - url = "https://github.com/nghttp2/nghttp2/releases/download/v1.14.1/nghttp2-1.14.1.tar.gz", - strip_prefix = "nghttp2-1.14.1", + url = "https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz", + strip_prefix = "nghttp2-1.20.0", build_file_content = BUILD, ) @@ -752,6 +752,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "abef64f738d269f6c7f8265233d3ea377eef9596", # 2/16/2017 + commit = "70e5d651b55d356770529e5bee9c6b2707d9cf21", # 3/1/2017 build_file_content = BUILD, )