diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins new file mode 100644 index 00000000000..664f7b13380 --- /dev/null +++ b/.bazelrc.jenkins @@ -0,0 +1,8 @@ +# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. +startup --host_jvm_args=-Xmx8192m +startup --host_jvm_args=-Xms8192m +startup --batch + +# This is so we understand failures better +build --verbose_failures + diff --git a/.gitignore b/.gitignore index a6ef824c1f8..469486c8f36 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /bazel-* +.idea/* +*.iml diff --git a/.travis.yml b/.travis.yml index d8559616e6d..88be42c285b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ sudo: required dist: xenial +branches: + except: + - stable + lang: go go: diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..9c59a146e50 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,65 @@ +#!groovy + +@Library('testutils') + +import org.istio.testutils.Utilities +import org.istio.testutils.GitUtilities +import org.istio.testutils.Bazel + +// Utilities shared amongst modules +def gitUtils = new GitUtilities() +def utils = new Utilities() +def bazel = new Bazel() + +mainFlow(utils) { + pullRequest(utils) { + node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') + } + + if (utils.runStage('PRESUBMIT')) { + presubmit(gitUtils, bazel) + } + if (utils.runStage('POSTSUBMIT')) { + postsubmit(gitUtils, bazel, utils) + } + } +} + +def presubmit(gitUtils, bazel) { + buildNode(gitUtils) { + stage('Code Check') { + sh('script/check-style') + } + bazel.updateBazelRc() + stage('Bazel Fetch') { + bazel.fetch('-k //...') + } + stage('Bazel Build') { + bazel.build('//...') + } + stage('Bazel Tests') { + bazel.test('//...') + } + stage('Push Test Binary') { + sh 'script/release-binary' + } + } +} + +def postsubmit(gitUtils, bazel, utils) { + buildNode(gitUtils) { + bazel.updateBazelRc() + stage('Push Binary') { + sh 'script/release-binary' + } + stage('Docker Push') { + def images = 'proxy,proxy_debug' + def tags = "${gitUtils.GIT_SHORT_SHA},\$(date +%Y-%m-%d-%H.%M.%S),latest" + utils.publishDockerImages(images, tags, 'release') + } + } +} \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index 95dcdde5cdc..ddfa05d578b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,16 +31,20 @@ googletest_repositories() load( "//contrib/endpoints:repositories.bzl", "grpc_repositories", - "mixer_client_repositories", "servicecontrol_client_repositories", ) grpc_repositories() -mixer_client_repositories() - servicecontrol_client_repositories() +load( + "//src/envoy/mixer:repositories.bzl", + "mixer_client_repositories", +) + +mixer_client_repositories() + # Workaround for Bazel > 0.4.0 since it needs newer protobuf.bzl from: # https://github.com/google/protobuf/pull/2246 # Do not use this git_repository for anything else than protobuf.bzl @@ -65,3 +69,30 @@ load( ) envoy_repositories() + +new_http_archive( + name = "docker_ubuntu", + build_file_content = """ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +docker_build( + name = "xenial", + tars = ["xenial/ubuntu-xenial-core-cloudimg-amd64-root.tar.gz"], + visibility = ["//visibility:public"], +) +""", + sha256 = "de31e6fcb843068965de5945c11a6f86399be5e4208c7299fb7311634fb41943", + strip_prefix = "docker-brew-ubuntu-core-e406914e5f648003dfe8329b512c30c9ad0d2f9c", + type = "zip", + url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", +) + + +DEBUG_BASE_IMAGE_SHA="3f57ae2aceef79e4000fb07ec850bbf4bce811e6f81dc8cfd970e16cdf33e622" + +# See github.com/istio/manager/blob/master/docker/debug/build-and-publish-debug-image.sh +# for instructions on how to re-build and publish this base image layer. +http_file( + name = "ubuntu_xenial_debug", + url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug-" + DEBUG_BASE_IMAGE_SHA + ".tar.gz", + sha256 = DEBUG_BASE_IMAGE_SHA, +) diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index bed9315a267..c77a3f72f58 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -163,7 +163,7 @@ def grpc_repositories(bind=True): native.git_repository( name = "grpc_git", - commit = "d28417c856366df704200f544e72d31056931bce", + commit = "bb3edafea245a9780cc4c10f0b58da21e8193f38", # v1.1.1 remote = "https://github.com/grpc/grpc.git", ) @@ -190,7 +190,7 @@ def grpc_repositories(bind=True): native.bind( name = "grpc_lib", - actual = "@grpc_git//:grpc++_reflection", + actual = "@grpc_git//:grpc++_codegen_proto", ) def googleapis_repositories(protobuf_repo="@protobuf_git//", bind=True): @@ -335,16 +335,3 @@ def servicecontrol_client_repositories(bind=True): name = "servicecontrol_client", actual = "@servicecontrol_client_git//:service_control_client_lib", ) - -def mixer_client_repositories(bind=True): - native.git_repository( - name = "mixerclient_git", - commit = "80e450a5126960e8e6337c3631cf2ef984038eab", - remote = "https://github.com/istio/mixerclient.git", - ) - - if bind: - native.bind( - name = "mixer_client_lib", - actual = "@mixerclient_git//:mixer_client_lib", - ) diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc index 2df9a843b71..962d3980bea 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc @@ -150,6 +150,8 @@ class JwtValidatorImpl : public JwtValidator { RSA *rsa_; EVP_PKEY *pkey_; EVP_MD_CTX *md_ctx_; + + grpc_exec_ctx exec_ctx_; }; // Gets EVP_MD mapped from an alg (algorithm string). @@ -159,12 +161,12 @@ const EVP_MD *EvpMdFromAlg(const char *alg); size_t HashSizeFromAlg(const char *alg); // Parses str into grpc_json object. Does not own buffer. -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer); +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer); // Gets BIGNUM from b64 string, used for extracting pkey from jwk. // Result owned by rsa_. -BIGNUM *BigNumFromBase64String(const char *b64); +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64); } // namespace @@ -185,7 +187,8 @@ JwtValidatorImpl::JwtValidatorImpl(const char *jwt, size_t jwt_len) x509_(nullptr), rsa_(nullptr), pkey_(nullptr), - md_ctx_(nullptr) { + md_ctx_(nullptr), + exec_ctx_(GRPC_EXEC_CTX_INIT) { header_buffer_ = gpr_empty_slice(); signed_buffer_ = gpr_empty_slice(); sig_buffer_ = gpr_empty_slice(); @@ -204,7 +207,7 @@ JwtValidatorImpl::~JwtValidatorImpl() { grpc_json_destroy(pkey_json_); } if (claims_ != nullptr) { - grpc_jwt_claims_destroy(claims_); + grpc_jwt_claims_destroy(&exec_ctx_, claims_); } if (!GPR_SLICE_IS_EMPTY(header_buffer_)) { gpr_slice_unref(header_buffer_); @@ -304,7 +307,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { if (dot == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } - header_json_ = DecodeBase64AndParseJson(cur, dot - cur, &header_buffer_); + header_json_ = + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &header_buffer_); CreateJoseHeader(); if (header_ == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; @@ -323,7 +327,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { // case, and it is owned by claims_ for successful case. gpr_slice claims_buffer = gpr_empty_slice(); grpc_json *claims_json = - DecodeBase64AndParseJson(cur, dot - cur, &claims_buffer); + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &claims_buffer); if (claims_json == nullptr) { if (!GPR_SLICE_IS_EMPTY(claims_buffer)) { gpr_slice_unref(claims_buffer); @@ -332,10 +336,13 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { } UpdateAudience(claims_json); // Takes ownershp of claims_json and claims_buffer. - claims_ = grpc_jwt_claims_from_json(claims_json, claims_buffer); - if (claims_ == nullptr) { + claims_ = grpc_jwt_claims_from_json(&exec_ctx_, claims_json, claims_buffer); + + // issuer is mandatory. grpc_jwt_claims_issuer checks if claims_ is nullptr. + if (grpc_jwt_claims_issuer(claims_) == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } + // Check timestamp. // Passing in its own audience to skip audience check. // Audience check should be done by the caller. @@ -354,8 +361,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { return GRPC_JWT_VERIFIER_BAD_FORMAT; } cur = dot + 1; - sig_buffer_ = - grpc_base64_decode_with_len(cur, jwt_len - signed_jwt_len - 1, 1); + sig_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, cur, + jwt_len - signed_jwt_len - 1, 1); if (GPR_SLICE_IS_EMPTY(sig_buffer_)) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } @@ -576,9 +583,11 @@ bool JwtValidatorImpl::ExtractPubkeyFromJwk(const grpc_json *jkey) { } const char *rsa_n = GetStringValue(jkey, "n"); - rsa_->n = rsa_n == nullptr ? nullptr : BigNumFromBase64String(rsa_n); + rsa_->n = + rsa_n == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_n); const char *rsa_e = GetStringValue(jkey, "e"); - rsa_->e = rsa_e == nullptr ? nullptr : BigNumFromBase64String(rsa_e); + rsa_->e = + rsa_e == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_e); if (rsa_->e == nullptr || rsa_->n == nullptr) { gpr_log(GPR_ERROR, "Missing RSA public key field."); @@ -651,7 +660,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::VerifyHsSignature(const char *pkey, const EVP_MD *md = EvpMdFromAlg(header_->alg); GPR_ASSERT(md != nullptr); // Checked before. - pkey_buffer_ = grpc_base64_decode_with_len(pkey, pkey_len, 1); + pkey_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, pkey, pkey_len, 1); if (GPR_SLICE_IS_EMPTY(pkey_buffer_)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; @@ -742,11 +751,11 @@ size_t HashSizeFromAlg(const char *alg) { } } -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer) { +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer) { grpc_json *json; - *buffer = grpc_base64_decode_with_len(str, len, 1); + *buffer = grpc_base64_decode_with_len(exec_ctx, str, len, 1); if (GPR_SLICE_IS_EMPTY(*buffer)) { gpr_log(GPR_ERROR, "Invalid base64."); return nullptr; @@ -760,12 +769,12 @@ grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, return json; } -BIGNUM *BigNumFromBase64String(const char *b64) { +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64) { BIGNUM *result = nullptr; gpr_slice bin; if (b64 == nullptr) return nullptr; - bin = grpc_base64_decode(b64, 1); + bin = grpc_base64_decode(exec_ctx, b64, 1); if (GPR_SLICE_IS_EMPTY(bin)) { gpr_log(GPR_ERROR, "Invalid base64 for big num."); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc index 72b53790158..85f06ca0417 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc @@ -200,7 +200,8 @@ char *GenerateJwtClaim(const char *issuer, const char *subject, } char *GenerateSignatueHs256(const char *data, const char *key) { - gpr_slice key_buffer = grpc_base64_decode(key, 1); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_slice key_buffer = grpc_base64_decode(&exec_ctx, key, 1); if (GPR_SLICE_IS_EMPTY(key_buffer)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h index 66074a12776..cdf58840654 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h +++ b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h @@ -15,11 +15,6 @@ #ifndef API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ #define API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ -// This header file contains definitions for all grpc internal -// dependencies. The code that depends on grpc internals should -// include this file instead of including the original headers -// in grpc. - // This header file is for internal use only since it declares grpc // internals that auth depends on. A public header file should not // include any internal grpc header files. @@ -30,171 +25,12 @@ extern "C" { -#include -#include - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json_common.h -////////////////////////////////////////////////////// - -/* The various json types. */ -typedef enum { - GRPC_JSON_OBJECT, - GRPC_JSON_ARRAY, - GRPC_JSON_STRING, - GRPC_JSON_NUMBER, - GRPC_JSON_TRUE, - GRPC_JSON_FALSE, - GRPC_JSON_NULL, - GRPC_JSON_TOP_LEVEL -} grpc_json_type; - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json.h -////////////////////////////////////////////////////// - -/* A tree-like structure to hold json values. The key and value pointers - * are not owned by it. - */ -typedef struct grpc_json { - struct grpc_json *next; - struct grpc_json *prev; - struct grpc_json *child; - struct grpc_json *parent; - - grpc_json_type type; - const char *key; - const char *value; -} grpc_json; - -/* The next two functions are going to parse the input string, and - * modify it in the process, in order to use its space to store - * all of the keys and values for the returned object tree. - * - * They assume UTF-8 input stream, and will output UTF-8 encoded - * strings in the tree. The input stream's UTF-8 isn't validated, - * as in, what you input is what you get as an output. - * - * All the keys and values in the grpc_json objects will be strings - * pointing at your input buffer. - * - * Delete the allocated tree afterward using grpc_json_destroy(). - */ -grpc_json *grpc_json_parse_string_with_len(char *input, size_t size); -grpc_json *grpc_json_parse_string(char *input); - -/* Use these to create or delete a grpc_json object. - * Deletion is recursive. We will not attempt to free any of the strings - * in any of the objects of that tree. - */ -grpc_json *grpc_json_create(grpc_json_type type); -void grpc_json_destroy(grpc_json *json); - -/* This function will create a new string using gpr_realloc, and will - * deserialize the grpc_json tree into it. It'll be zero-terminated, - * but will be allocated in chunks of 256 bytes. - * - * The indent parameter controls the way the output is formatted. - * If indent is 0, then newlines will be suppressed as well, and the - * output will be condensed at its maximum. - */ -char *grpc_json_dump_to_string(grpc_json *json, int indent); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/base64 -////////////////////////////////////////////////////// - -/* Encodes data using base64. It is the caller's responsability to free - the returned char * using gpr_free. Returns nullptr on nullptr input. */ -char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, - int multiline); - -/* Decodes data according to the base64 specification. Returns an empty - slice in case of failure. */ -gpr_slice grpc_base64_decode(const char *b64, int url_safe); - -/* Same as above except that the length is provided by the caller. */ -gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, - int url_safe); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/auth/security/json_key.h -////////////////////////////////////////////////////// - -/* --- auth_json_key parsing. --- */ - -typedef struct { - const char *type; - char *private_key_id; - char *client_id; - char *client_email; - RSA *private_key; -} grpc_auth_json_key; - -/* Creates a json_key object from string. Returns an invalid object if a parsing - error has been encountered. */ -grpc_auth_json_key grpc_auth_json_key_create_from_string( - const char *json_string); - -/* Destructs the object. */ -void grpc_auth_json_key_destruct(grpc_auth_json_key *json_key); - -/* Caller is responsible for calling gpr_free on the returned value. May return - nullptr on invalid input. The scope parameter may be nullptr. */ -char *grpc_jwt_encode_and_sign(const grpc_auth_json_key *json_key, - const char *audience, - gpr_timespec token_lifetime, const char *scope); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/support/string.h -////////////////////////////////////////////////////// - -/* Minimum buffer size for calling ltoa */ -#define GPR_LTOA_MIN_BUFSIZE (3 * sizeof(long)) - -/* Convert a long to a string in base 10; returns the length of the - output string (or 0 on failure). - output must be at least GPR_LTOA_MIN_BUFSIZE bytes long. */ -int gpr_ltoa(long value, char *output); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/jwt_verifier.h -////////////////////////////////////////////////////// - -/* --- grpc_jwt_verifier_status. --- */ - -typedef enum { - GRPC_JWT_VERIFIER_OK = 0, - GRPC_JWT_VERIFIER_BAD_SIGNATURE, - GRPC_JWT_VERIFIER_BAD_FORMAT, - GRPC_JWT_VERIFIER_BAD_AUDIENCE, - GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR, - GRPC_JWT_VERIFIER_TIME_CONSTRAINT_FAILURE, - GRPC_JWT_VERIFIER_GENERIC_ERROR -} grpc_jwt_verifier_status; - -const char *grpc_jwt_verifier_status_to_string(grpc_jwt_verifier_status status); - -/* --- grpc_jwt_claims. --- */ - -typedef struct grpc_jwt_claims grpc_jwt_claims; - -void grpc_jwt_claims_destroy(grpc_jwt_claims *claims); - -/* Returns the whole JSON tree of the claims. */ -const grpc_json *grpc_jwt_claims_json(const grpc_jwt_claims *claims); - -/* Access to registered claims in https://tools.ietf.org/html/rfc7519#page-9 */ -const char *grpc_jwt_claims_subject(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_issuer(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_audience(const grpc_jwt_claims *claims); -gpr_timespec grpc_jwt_claims_expires_at(const grpc_jwt_claims *claims); - -/* --- TESTING ONLY exposed functions. --- */ - -grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer); -grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, - const char *audience); +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_common.h" +#include "src/core/lib/security/credentials/jwt/json_token.h" +#include "src/core/lib/security/credentials/jwt/jwt_verifier.h" +#include "src/core/lib/security/util/b64.h" +#include "src/core/lib/support/string.h" } #endif // API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ diff --git a/contrib/endpoints/src/api_manager/context/BUILD b/contrib/endpoints/src/api_manager/context/BUILD index e158c13fc92..59365102318 100644 --- a/contrib/endpoints/src/api_manager/context/BUILD +++ b/contrib/endpoints/src/api_manager/context/BUILD @@ -39,7 +39,6 @@ cc_library( "//contrib/endpoints/src/api_manager/auth", "//contrib/endpoints/src/api_manager/auth:service_account_token", "//contrib/endpoints/src/api_manager/cloud_trace", - "//contrib/endpoints/src/api_manager/mixer", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", "//external:cc_wkt_protos", diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 59cc7c3d53f..75bcb177f9d 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -165,7 +165,8 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->operation_name = kUnrecognizedOperation; } info->operation_id = operation_id_; - if (check_response_info_.is_api_key_valid) { + if (check_response_info_.is_api_key_valid && + check_response_info_.service_is_activated) { info->api_key = api_key_; } info->producer_project_id = service_context()->project_id(); diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index 0e99c247676..28ce9271f80 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -15,7 +15,6 @@ //////////////////////////////////////////////////////////////////////////////// // #include "contrib/endpoints/src/api_manager/context/service_context.h" -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" #include "contrib/endpoints/src/api_manager/service_control/aggregated.h" @@ -105,16 +104,10 @@ const std::string& ServiceContext::project_id() const { } std::unique_ptr ServiceContext::CreateInterface() { - if (config_->server_config() && - config_->server_config()->has_mixer_options()) { - return std::unique_ptr( - mixer::Mixer::Create(env_.get(), config_.get())); - } else { - return std::unique_ptr( - service_control::Aggregated::Create( - config_->service(), config_->server_config(), env_.get(), - &service_account_token_)); - } + return std::unique_ptr( + service_control::Aggregated::Create(config_->service(), + config_->server_config(), env_.get(), + &service_account_token_)); } std::unique_ptr diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc deleted file mode 100644 index 53837edaf67..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" - -using ::google::api_manager::utils::Status; -using ::istio::mixer_client::Attributes; - -namespace google { -namespace api_manager { -namespace mixer { -namespace { - -const std::string kProxyPeerID = "Istio/Proxy"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; - -const std::string kAttrNameServiceName = "serviceName"; -const std::string kAttrNamePeerId = "peerId"; -const std::string kAttrNameOperationId = "operationId"; -const std::string kAttrNameOperationName = "operationName"; -const std::string kAttrNameApiKey = "apiKey"; -const std::string kAttrNameResponseCode = "responseCode"; -const std::string kAttrNameURL = "url"; -const std::string kAttrNameLocation = "location"; -const std::string kAttrNameApiName = "apiName"; -const std::string kAttrNameApiVersion = "apiVersion"; -const std::string kAttrNameApiMethod = "apiMethod"; -const std::string kAttrNameRequestSize = "requestSize"; -const std::string kAttrNameResponseSize = "responseSize"; -const std::string kAttrNameLogMessage = "logMessage"; -const std::string kAttrNameResponseTime = "responseTime"; -const std::string kAttrNameOriginIp = "originIp"; -const std::string kAttrNameOriginHost = "originHost"; -const std::string kAttrNameTargetService = "targetService"; - -Attributes::Value StringValue(const std::string& str) { - Attributes::Value v; - v.type = Attributes::Value::STRING; - v.str_v = str; - return v; -} - -Attributes::Value Int64Value(int64_t value) { - Attributes::Value v; - v.type = Attributes::Value::INT64; - v.value.int64_v = value; - return v; -} - -} // namespace - -Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) - : env_(env), config_(config) {} - -Mixer::~Mixer() {} - -Status Mixer::Init() { - ::istio::mixer_client::MixerClientOptions options; - options.mixer_server = - config_->server_config()->mixer_options().mixer_server(); - mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); - auto target_service = getenv(kEnvNameTargetService.c_str()); - if (target_service) { - target_service_ = target_service; - } - return Status::OK; -} - -Status Mixer::Close() { return Status::OK; } - -void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(config_->service_name()); - attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); - - if (!info.operation_id.empty()) { - attr->attributes[kAttrNameOperationId] = StringValue(info.operation_id); - } - if (!info.operation_name.empty()) { - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - } - if (!info.api_key.empty()) { - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - } - if (!info.client_ip.empty()) { - attr->attributes[kAttrNameOriginIp] = StringValue(info.client_ip); - } - if (!info.client_host.empty()) { - attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); - } - if (!target_service_.empty()) { - attr->attributes[kAttrNameTargetService] = StringValue(target_service_); - } -} - -void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); -} - -void Mixer::FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); - - if (!info.url.empty()) { - attr->attributes[kAttrNameURL] = StringValue(info.url); - } - if (!info.location.empty()) { - attr->attributes[kAttrNameLocation] = StringValue(info.location); - } - - if (!info.api_name.empty()) { - attr->attributes[kAttrNameApiName] = StringValue(info.api_name); - } - if (!info.api_version.empty()) { - attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); - } - if (!info.api_method.empty()) { - attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - } - - if (!info.log_message.empty()) { - attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - } - - attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); - if (info.request_size >= 0) { - attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); - } - if (info.response_size >= 0) { - attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); - } - - if (info.latency.request_time_ms >= 0) { - attr->attributes[kAttrNameResponseTime] = - Int64Value(info.latency.request_time_ms); - } -} - -Status Mixer::Report(const service_control::ReportRequestInfo& info) { - ::istio::mixer_client::Attributes attributes; - FillReportAttributes(info, &attributes); - env_->LogInfo("Send Report: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Report( - attributes, [this](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Report response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::report, Error: ") + - status.ToString()); - } - }); - return Status::OK; -} - -void Mixer::Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done) { - ::istio::mixer_client::Attributes attributes; - FillCheckAttributes(info, &attributes); - env_->LogInfo("Send Check: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Check( - attributes, - [this, on_done](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Check response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::check, Error: ") + - status.ToString()); - } - service_control::CheckResponseInfo info; - on_done(Status(status.error_code(), status.error_message(), - Status::SERVICE_CONTROL), - info); - }); -} - -Status Mixer::GetStatistics(service_control::Statistics* esp_stat) const { - return Status::OK; -} - -service_control::Interface* Mixer::Create(ApiManagerEnvInterface* env, - const Config* config) { - return new Mixer(env, config); -} - -} // namespace mixer -} // namespace api_manager -} // namespace google diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h deleted file mode 100644 index 81276e1227d..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef API_MANAGER_MIXER_MIXER_H_ -#define API_MANAGER_MIXER_MIXER_H_ - -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "contrib/endpoints/src/api_manager/config.h" -#include "contrib/endpoints/src/api_manager/service_control/interface.h" -#include "include/client.h" - -namespace google { -namespace api_manager { -namespace mixer { - -// This implementation uses service-control-client-cxx module. -class Mixer : public service_control::Interface { - public: - static service_control::Interface* Create(ApiManagerEnvInterface* env, - const Config* config); - - virtual ~Mixer(); - - virtual utils::Status Report(const service_control::ReportRequestInfo& info); - - virtual void Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done); - - virtual utils::Status Init(); - virtual utils::Status Close(); - - virtual utils::Status GetStatistics(service_control::Statistics* stat) const; - - private: - // The constructor. - Mixer(ApiManagerEnvInterface* env, const Config* config); - - // Fill common attributes for both check and report. - void FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for check. - void FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for report. - void FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - - // The Api Manager environment interface. - ApiManagerEnvInterface* env_; - // The config. - const Config* config_; - // The mixer client - std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // Target service - std::string target_service_; -}; - -} // namespace mixer -} // namespace api_manager -} // namespace google - -#endif // API_MANAGER_MIXER_MIXER_H_ diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 9e325c3f1a0..cbee58f8da8 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -35,10 +35,6 @@ message ServerConfig { // Server config used for API authentication ApiAuthenticationConfig api_authentication_config = 5; - // Mixer option flag. If not present, default to use service_control. When - // Envoy/esp talks to Mixer, has to specify this field. - MixerOptions mixer_options = 6; - // Server config used for API authorization via Firebase Rules. ApiCheckSecurityRulesConfig api_check_security_rules_config = 7; @@ -152,11 +148,6 @@ message ApiCheckSecurityRulesConfig { string firebase_server = 1; } -message MixerOptions { - // For envoy, it is the cluster name for mixer server. - string mixer_server = 1; -} - message Experimental { // Disable timed printouts of ESP status to the error log. bool disable_log_status = 1; diff --git a/contrib/endpoints/src/api_manager/server_config_proto_test.cc b/contrib/endpoints/src/api_manager/server_config_proto_test.cc index b39309727f1..80c62e05471 100644 --- a/contrib/endpoints/src/api_manager/server_config_proto_test.cc +++ b/contrib/endpoints/src/api_manager/server_config_proto_test.cc @@ -76,9 +76,6 @@ experimental { disable_log_status: false } -mixer_options { - mixer_server: "mixer_server" -} )"; TEST(ServerConfigProto, ServerConfigFromString) { @@ -125,9 +122,6 @@ TEST(ServerConfigProto, ServerConfigFromString) { // Check disable_log_status EXPECT_EQ(false, server_config.experimental().disable_log_status()); - - // Check mixer options - EXPECT_EQ("mixer_server", server_config.mixer_options().mixer_server()); } TEST(ServerConfigProto, ValidateSampleServerConfig) { diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.cc b/contrib/endpoints/src/api_manager/service_control/aggregated.cc index d2754d69dbd..e59a69dc8cc 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.cc @@ -287,8 +287,14 @@ void Aggregated::Check( if (status.ok()) { Status status = Proto::ConvertCheckResponse( *response, service_control_proto_.service_name(), &response_info); - // If allow_unregistered_calls is true, it is always OK to proceed. - if (allow_unregistered_calls) { + // If server replied with either invalid api_key or not activated service, + // the request is rejected even allow_unregistered_calls is true. Most + // likely, users provide a wrong api key. By failing the request, the + // users will be notified with the error and have chance to correct it. + // Otherwise, the Report call will fail. It is very hard to notice and + // debug the Report failure. + if (allow_unregistered_calls && response_info.is_api_key_valid && + response_info.service_is_activated) { on_done(Status::OK, response_info); } else { on_done(status, response_info); diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 3e7eba64d21..694f299d22c 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -956,14 +956,13 @@ Status Proto::FillReportRequest(const ReportRequestInfo& info, } } - // Not to send consumer metrics for following cases: - // 1) api_key is not provided, or - // 2) the service is not activated for the consumer project, - bool send_consumer_metric = true; - if (info.api_key.empty() || - !info.check_response_info.service_is_activated) { - send_consumer_metric = false; - } + // Not to send consumer metrics if api_key is empty. + // api_key is empty in one of following cases: + // 1) api_key is not provided, + // 2) api_key is invalid determined by the server from the Check call. + // 3) the service is not activated for the consumer project. + bool send_consumer_metric = !info.api_key.empty(); + // Populate all metrics. for (auto it = metrics_.begin(), end = metrics_.end(); it != end; it++) { const SupportedMetric* m = *it; diff --git a/docker/BUILD b/docker/BUILD new file mode 100644 index 00000000000..6e017ee6064 --- /dev/null +++ b/docker/BUILD @@ -0,0 +1,11 @@ +# Add aliases for envoy specific targets so that the docker build +# targets are consistent for all istio repos, e.g. //docker: +alias( + name = "proxy", + actual = "//src/envoy/mixer:proxy", +) + +alias( + name = "proxy_debug", + actual = "//src/envoy/mixer:proxy_debug", +) diff --git a/repositories.bzl b/repositories.bzl index 8764aa24d32..31fe15ffb63 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -36,7 +36,7 @@ def boringssl_repositories(bind=True): def protobuf_repositories(bind=True): native.git_repository( name = "protobuf_git", - commit = "a428e42072765993ff674fda72863c9f1aa2d268", # v3.1.0 + commit = "593e917c176b5bc5aafa57bf9f6030d749d91cd5", # v3.2.0 remote = "https://github.com/google/protobuf.git", ) @@ -68,7 +68,7 @@ def protobuf_repositories(bind=True): native.bind( name = "protobuf_clib", - actual = "@protobuf_git//:protobuf_lite", + actual = "@protobuf_git//:protoc_lib", ) diff --git a/script/release-binary b/script/release-binary new file mode 100755 index 00000000000..0ff4dd63f91 --- /dev/null +++ b/script/release-binary @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +set -ex + +# Make sure to this script on x86_64 Ubuntu Xenial +UBUNTU_RELEASE="$(lsb_release -c -s)" +[[ "${UBUNTU_RELEASE}" == 'xenial' ]] || { echo 'must run on Ubuntu Xenial'; exit 1; } + +# The bucket name to store proxy binary +DST="gs://istio-build/proxy" + +# The proxy binary name. +SHA="$(git rev-parse --verify HEAD)" +BINARY_NAME="envoy-alpha-${SHA}.tar.gz" +SHA256_NAME="envoy-alpha-${SHA}.sha256" + +# If binary already exists skip. +gsutil stat "${DST}/${BINARY_NAME}" \ + && { echo 'Binary already exists'; exit 0; } \ + || echo 'Building a new binary.' + +# Build the release binary +bazel build --config=release //src/envoy/mixer:envoy_tar +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" + +# Copy it to the bucket. +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" + +# Build the debug binary +BINARY_NAME="envoy-debug-${SHA}.tar.gz" +SHA256_NAME="envoy-debug-${SHA}.sha256" +bazel build -c dbg //src/envoy/mixer:envoy_tar +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" + +# Copy it to the bucket. +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" diff --git a/contrib/endpoints/src/api_manager/mixer/BUILD b/script/release-docker old mode 100644 new mode 100755 similarity index 51% rename from contrib/endpoints/src/api_manager/mixer/BUILD rename to script/release-docker index affe09c7ddd..d21caffee55 --- a/contrib/endpoints/src/api_manager/mixer/BUILD +++ b/script/release-docker @@ -1,3 +1,5 @@ +#!/bin/bash +# # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,29 +16,31 @@ # ################################################################################ # -package(default_visibility = ["//contrib/endpoints/src/api_manager:__subpackages__"]) - -cc_library( - name = "mixer", - srcs = [ - "mixer.cc", - ], - hdrs = [ - "mixer.h", - ], - linkopts = select({ - "//:darwin": [], - "//conditions:default": [ - "-lm", - "-luuid", - ], - }), - deps = [ - "//contrib/endpoints/src/api_manager:impl_headers", - "//contrib/endpoints/src/api_manager/service_control", - "//contrib/endpoints/src/api_manager/utils", - "//external:grpc++", - "//external:mixer_client_lib", - "//external:protobuf", - ], -) +set -ex + +# This docker image can be used as: +# +# docker run IMAGE -a backend_address -p PORT -m MIXER_SERVER +# + +PROJECT=istio-testing +IMAGE_PREFIX="gcr.io/${PROJECT}/proxy" + +DATE_PART=$(date +"%Y%m%d") +SHA_PART=$(git show -q HEAD --pretty=format:%h) +DOCKER_TAG="${DATE_PART}${SHA_PART}" + +IMAGE_NAME="${IMAGE_PREFIX}:${DOCKER_TAG}" + +gcloud docker --authorize-only + +bazel run --config=release //src/envoy/mixer:proxy "${IMAGE_NAME}" + +gcloud docker -- push "${IMAGE_NAME}" + +IMAGE_LATEST="${IMAGE_PREFIX}:latest" + +docker tag -f "${IMAGE_NAME}" "${IMAGE_LATEST}" + +gcloud docker -- push "${IMAGE_LATEST}" + diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD new file mode 100644 index 00000000000..132bc364a1d --- /dev/null +++ b/src/envoy/mixer/BUILD @@ -0,0 +1,106 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + + +load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("//src/envoy/mixer:proxy_docker.bzl", "proxy_docker_build") +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") + +cc_proto_library( + name = "string_map_proto", + srcs = ["string_map.proto"], + default_runtime = "//external:protobuf", + protoc = "//external:protoc", + visibility = ["//visibility:public"], +) + +cc_library( + name = "filter_lib", + srcs = [ + "forward_attribute_filter.cc", + "http_control.cc", + "http_control.h", + "http_filter.cc", + "utils.cc", + "utils.h", + ], + deps = [ + ":string_map_proto", + "//external:mixer_client_lib", + "@envoy_git//:envoy-common", + ], + alwayslink = 1, +) + +cc_binary( + name = "envoy", + linkstatic = 1, + deps = [ + ":filter_lib", + "@envoy_git//:envoy-main", + ], +) + +pkg_tar( + name = "envoy_tar", + extension = "tar.gz", + files = [":envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) + +pkg_tar( + name = "start_envoy_tar", + extension = "tar.gz", + files = ["start_envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) + +pkg_tar( + name = "config_tar", + extension = "tar.gz", + files = ["envoy.conf.template"], + mode = "0666", + package_dir = "/etc/opt/proxy", + tags = ["manual"], +) + +proxy_docker_build( + images = [ + {"name": "proxy", "base": "@docker_ubuntu//:xenial"}, + {"name": "proxy_debug", "base": "@ubuntu_xenial_debug//file"}, + ], + entrypoint = [ + "/usr/local/bin/start_envoy", + "-e", + "/usr/local/bin/envoy", + "-c", + "/etc/opt/proxy/envoy.conf", + "-t", + "/etc/opt/proxy/envoy.conf.template", + ], + ports = ["9090"], + repository = "istio", + tags = ["manual"], + tars = [ + ":config_tar", + ":envoy_tar", + ":start_envoy_tar", + ], + visibility = ["//visibility:public"], +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md new file mode 100644 index 00000000000..d7d39f30e03 --- /dev/null +++ b/src/envoy/mixer/README.md @@ -0,0 +1,99 @@ + +This Proxy will use Envoy and talk to Mixer server. + +## Build Mixer server + +* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: + +``` + cd $(ISTIO)/mixer + bazel build ...:all +``` + +## Build Envoy proxy + +* Build target envoy: + +``` + bazel build //src/envoy/mixer:envoy +``` + +## How to run it + +* Start mixer server. In mixer folder run: + +``` + bazel-bin/cmd/server/mixs server + --globalConfigFile testdata/globalconfig.yml + --serviceConfigFile testdata/serviceconfig.yml --logtostderr +``` + + The server will run at port 9091 + +* Start backend Echo server. + +``` + cd test/backend/echo + go run echo.go +``` + +* Start Envoy proxy, run + +``` + src/envoy/mixer/start_envoy +``` + +* Then issue HTTP request to proxy. + +``` + curl http://localhost:9090/echo -d "hello world" +``` + +## How to configurate HTTP filters + +This module has two HTTP filters: +1. mixer filter: intercept all HTTP requests, call the mixer. +2. forward_attribute filter: Forward attributes to the upstream istio/proxy. + +### *mixer* filter: + +This filter will intercept all HTTP requests and call Mixer. Here is its config: + +``` + "filters": [ + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "${MIXER_SERVER}", + "attributes" : { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* mixer_server is required +* attributes: these attributes will be send to the mixer + +### *forward_attribute* HTTP filter: + +This filer will forward attributes to the upstream istio/proxy. + +``` + "filters": [ + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* attributes: these attributes will be forwarded to the upstream istio/proxy. + + + diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template new file mode 100644 index 00000000000..c5377632388 --- /dev/null +++ b/src/envoy/mixer/envoy.conf.template @@ -0,0 +1,137 @@ +{ + "listeners": [ + { + "port": ${PORT}, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "${MIXER_SERVER}", + "attributes": { + "target.uid": "POD222", + "target.namespace": "XYZ222" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + }, + { + "port": 7070, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service2" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "source.uid": "POD11", + "source.namespace": "XYZ11" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + } + ], + "admin": { + "access_log_path": "/dev/stdout", + "port": 9001 + }, + "cluster_manager": { + "clusters": [ + { + "name": "service1", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://${BACKEND}" + } + ] + }, + { + "name": "service2", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:9090" + } + ] + } + ] + } +} diff --git a/src/envoy/mixer/forward_attribute_filter.cc b/src/envoy/mixer/forward_attribute_filter.cc new file mode 100644 index 00000000000..08d58916f89 --- /dev/null +++ b/src/envoy/mixer/forward_attribute_filter.cc @@ -0,0 +1,114 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/base64.h" +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/utils.h" + +namespace Http { +namespace ForwardAttribute { +namespace { + +// The Json object name to specify attributes which will be forwarded +// to the upstream istio proxy. +const std::string kJsonNameAttributes("attributes"); + +} // namespace + +class Config : public Logger::Loggable { + private: + std::string attributes_; + + public: + Config(const Json::Object& config) { + Utils::StringMap attributes = + Utils::ExtractStringMap(config, kJsonNameAttributes); + if (!attributes.empty()) { + std::string serialized_str = Utils::SerializeStringMap(attributes); + attributes_ = + Base64::encode(serialized_str.c_str(), serialized_str.size()); + } + } + + const std::string& attributes() const { return attributes_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class ForwardAttributeFilter : public Http::StreamDecoderFilter { + private: + ConfigPtr config_; + + public: + ForwardAttributeFilter(ConfigPtr config) : config_(config) {} + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + if (!config_->attributes().empty()) { + headers.addStatic(Utils::kIstioAttributeHeader, config_->attributes()); + } + return FilterHeadersStatus::Continue; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + return FilterTrailersStatus::Continue; + } + + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override {} +}; + +} // namespace ForwardAttribute +} // namespace Http + +namespace Server { +namespace Configuration { + +class ForwardAttributeConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Decoder || name != "forward_attribute") { + return nullptr; + } + + Http::ForwardAttribute::ConfigPtr add_header_config( + new Http::ForwardAttribute::Config(config)); + return [add_header_config]( + Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::ForwardAttribute::ForwardAttributeFilter( + add_header_config)); + callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc new file mode 100644 index 00000000000..6e64659c2bb --- /dev/null +++ b/src/envoy/mixer/http_control.cc @@ -0,0 +1,194 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/http_control.h" + +#include "common/common/base64.h" +#include "common/common/utility.h" +#include "common/http/utility.h" + +#include "src/envoy/mixer/string_map.pb.h" +#include "src/envoy/mixer/utils.h" + +using ::google::protobuf::util::Status; +using ::istio::mixer_client::Attributes; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +// Define attribute names +const std::string kRequestPath = "request.path"; +const std::string kRequestHost = "request.host"; +const std::string kRequestSize = "request.size"; +const std::string kRequestTime = "request.time"; +const std::string kRequestHeaders = "request.headers"; + +const std::string kResponseHeaders = "response.headers"; +const std::string kResponseSize = "response.size"; +const std::string kResponseTime = "response.time"; +const std::string kResponseLatency = "response.latency"; +const std::string kResponseHttpCode = "response.http.code"; + +Attributes::Value StringValue(const std::string& str) { + Attributes::Value v; + v.type = Attributes::Value::STRING; + v.str_v = str; + return v; +} + +Attributes::Value StringMapValue( + std::map&& string_map) { + Attributes::Value v; + v.type = Attributes::Value::STRING_MAP; + v.string_map_v.swap(string_map); + return v; +} + +Attributes::Value Int64Value(int64_t value) { + Attributes::Value v; + v.type = Attributes::Value::INT64; + v.value.int64_v = value; + return v; +} + +Attributes::Value TimeValue( + std::chrono::time_point value) { + Attributes::Value v; + v.type = Attributes::Value::TIME; + v.time_v = value; + return v; +} + +Attributes::Value DurationValue(std::chrono::nanoseconds value) { + Attributes::Value v; + v.type = Attributes::Value::DURATION; + v.duration_nanos_v = value; + return v; +} + +void SetStringAttribute(const std::string& name, const std::string& value, + Attributes* attr) { + if (!value.empty()) { + attr->attributes[name] = StringValue(value); + } +} + +std::map ExtractHeaders(const HeaderMap& header_map) { + std::map headers; + header_map.iterate( + [](const HeaderEntry& header, void* context) { + std::map* header_map = + static_cast*>(context); + (*header_map)[header.key().c_str()] = header.value().c_str(); + }, + &headers); + return headers; +} + +void FillRequestHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + SetStringAttribute(kRequestPath, header_map.Path()->value().c_str(), attr); + SetStringAttribute(kRequestHost, header_map.Host()->value().c_str(), attr); + attr->attributes[kRequestTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kRequestHeaders] = + StringMapValue(ExtractHeaders(header_map)); +} + +void FillResponseHeaderAttributes(const HeaderMap* header_map, + Attributes* attr) { + if (header_map) { + attr->attributes[kResponseHeaders] = + StringMapValue(ExtractHeaders(*header_map)); + } + attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); +} + +void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, + int check_status_code, Attributes* attr) { + if (info.bytesReceived() >= 0) { + attr->attributes[kRequestSize] = Int64Value(info.bytesReceived()); + } + if (info.bytesSent() >= 0) { + attr->attributes[kResponseSize] = Int64Value(info.bytesSent()); + } + + if (info.duration().count() > 0) { + attr->attributes[kResponseLatency] = DurationValue( + std::chrono::duration_cast(info.duration())); + } + + if (info.responseCode().valid()) { + attr->attributes[kResponseHttpCode] = + Int64Value(info.responseCode().value()); + } else { + attr->attributes[kResponseHttpCode] = Int64Value(check_status_code); + } +} + +} // namespace + +HttpControl::HttpControl(const std::string& mixer_server, + std::map&& attributes) + : config_attributes_(std::move(attributes)) { + ::istio::mixer_client::MixerClientOptions options; + options.mixer_server = mixer_server; + mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); +} + +void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { + // Extract attributes from x-istio-attributes header + const HeaderEntry* entry = header_map.get(Utils::kIstioAttributeHeader); + if (entry) { + ::istio::proxy::mixer::StringMap pb; + std::string str(entry->value().c_str(), entry->value().size()); + pb.ParseFromString(Base64::decode(str)); + for (const auto& it : pb.map()) { + SetStringAttribute(it.first, it.second, attr); + } + header_map.remove(Utils::kIstioAttributeHeader); + } + + FillRequestHeaderAttributes(header_map, attr); + + for (const auto& attribute : config_attributes_) { + SetStringAttribute(attribute.first, attribute.second, attr); + } +} + +void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, + DoneFunc on_done) { + FillCheckAttributes(headers, &request_data->attributes); + log().debug("Send Check: {}", request_data->attributes.DebugString()); + mixer_client_->Check(request_data->attributes, on_done); +} + +void HttpControl::Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, + int check_status, DoneFunc on_done) { + // Use all Check attributes for Report. + // Add additional Report attributes. + FillResponseHeaderAttributes(response_headers, &request_data->attributes); + + FillRequestInfoAttributes(request_info, check_status, + &request_data->attributes); + log().debug("Send Report: {}", request_data->attributes.DebugString()); + mixer_client_->Report(request_data->attributes, on_done); +} + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h new file mode 100644 index 00000000000..c5938cf369d --- /dev/null +++ b/src/envoy/mixer/http_control.h @@ -0,0 +1,64 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "envoy/http/access_log.h" +#include "include/client.h" + +namespace Http { +namespace Mixer { + +// Store data from Check to report +struct HttpRequestData { + ::istio::mixer_client::Attributes attributes; +}; +typedef std::shared_ptr HttpRequestDataPtr; + +// The mixer client class to control HTTP requests. +// It has Check() to validate if a request can be processed. +// At the end of request, call Report(). +class HttpControl final : public Logger::Loggable { + public: + // The constructor. + HttpControl(const std::string& mixer_server, + std::map&& attributes); + + // Make mixer check call. + void Check(HttpRequestDataPtr request_data, HeaderMap& headers, + ::istio::mixer_client::DoneFunc on_done); + + // Make mixer report call. + void Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, int check_status_code, + ::istio::mixer_client::DoneFunc on_done); + + private: + void FillCheckAttributes(HeaderMap& header_map, + ::istio::mixer_client::Attributes* attr); + + // The mixer client + std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // The attributes read from the config file. + std::map config_attributes_; +}; + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc new file mode 100644 index 00000000000..a990b0fd7e3 --- /dev/null +++ b/src/envoy/mixer/http_filter.cc @@ -0,0 +1,277 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/http_control.h" +#include "src/envoy/mixer/utils.h" + +using ::google::protobuf::util::Status; +using StatusCode = ::google::protobuf::util::error::Code; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +// The Json object name for mixer-server. +const std::string kJsonNameMixerServer("mixer_server"); + +// The Json object name for static attributes. +const std::string kJsonNameMixerAttributes("attributes"); + +// Convert Status::code to HTTP code +int HttpCode(int code) { + // Map Canonical codes to HTTP status codes. This is based on the mapping + // defined by the protobuf http error space. + switch (code) { + case StatusCode::OK: + return 200; + case StatusCode::CANCELLED: + return 499; + case StatusCode::UNKNOWN: + return 500; + case StatusCode::INVALID_ARGUMENT: + return 400; + case StatusCode::DEADLINE_EXCEEDED: + return 504; + case StatusCode::NOT_FOUND: + return 404; + case StatusCode::ALREADY_EXISTS: + return 409; + case StatusCode::PERMISSION_DENIED: + return 403; + case StatusCode::RESOURCE_EXHAUSTED: + return 429; + case StatusCode::FAILED_PRECONDITION: + return 400; + case StatusCode::ABORTED: + return 409; + case StatusCode::OUT_OF_RANGE: + return 400; + case StatusCode::UNIMPLEMENTED: + return 501; + case StatusCode::INTERNAL: + return 500; + case StatusCode::UNAVAILABLE: + return 503; + case StatusCode::DATA_LOSS: + return 500; + case StatusCode::UNAUTHENTICATED: + return 401; + default: + return 500; + } +} + +} // namespace + +class Config : public Logger::Loggable { + private: + std::shared_ptr http_control_; + Upstream::ClusterManager& cm_; + + public: + Config(const Json::Object& config, Server::Instance& server) + : cm_(server.clusterManager()) { + std::string mixer_server; + if (config.hasObject(kJsonNameMixerServer)) { + mixer_server = config.getString(kJsonNameMixerServer); + } else { + log().error( + "mixer_server is required but not specified in the config: {}", + __func__); + } + + std::map attributes = + Utils::ExtractStringMap(config, kJsonNameMixerAttributes); + + http_control_ = + std::make_shared(mixer_server, std::move(attributes)); + log().debug("Called Mixer::Config constructor with mixer_server: ", + mixer_server); + } + + std::shared_ptr& http_control() { return http_control_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { + private: + std::shared_ptr http_control_; + std::shared_ptr request_data_; + + enum State { NotStarted, Calling, Complete, Responded }; + State state_; + + StreamDecoderFilterCallbacks* decoder_callbacks_; + StreamEncoderFilterCallbacks* encoder_callbacks_; + + bool initiating_call_; + int check_status_code_; + + public: + Instance(ConfigPtr config) + : http_control_(config->http_control()), + state_(NotStarted), + initiating_call_(false), + check_status_code_(HttpCode(StatusCode::UNKNOWN)) { + Log().debug("Called Mixer::Instance : {}", __func__); + } + + // Jump thread; on_done will be called at the dispatcher thread. + DoneFunc wrapper(DoneFunc on_done) { + auto& dispatcher = decoder_callbacks_->dispatcher(); + return [&dispatcher, on_done](const Status& status) { + dispatcher.post([status, on_done]() { on_done(status); }); + }; + } + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + state_ = Calling; + initiating_call_ = true; + request_data_ = std::make_shared(); + http_control_->Check( + request_data_, headers, + wrapper([this](const Status& status) { completeCheck(status); })); + initiating_call_ = false; + + if (state_ == Complete) { + return FilterHeadersStatus::Continue; + } + Log().debug("Called Mixer::Instance : {} Stop", __func__); + return FilterHeadersStatus::StopIteration; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {} ({}, {})", __func__, data.length(), + end_stream); + if (state_ == Calling) { + return FilterDataStatus::StopIterationAndBuffer; + } + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + if (state_ == Calling) { + return FilterTrailersStatus::StopIteration; + } + return FilterTrailersStatus::Continue; + } + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + decoder_callbacks_ = &callbacks; + decoder_callbacks_->addResetStreamCallback( + [this]() { state_ = Responded; }); + } + void completeCheck(const Status& status) { + Log().debug("Called Mixer::Instance : check complete {}", + status.ToString()); + if (!status.ok() && state_ != Responded) { + state_ = Responded; + check_status_code_ = HttpCode(status.error_code()); + Utility::sendLocalReply(*decoder_callbacks_, Code(check_status_code_), + status.ToString()); + return; + } + state_ = Complete; + if (!initiating_call_) { + decoder_callbacks_->continueDecoding(); + } + } + + virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterHeadersStatus::Continue; + } + virtual FilterDataStatus encodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterDataStatus::Continue; + } + virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterTrailersStatus::Continue; + } + virtual void setEncoderFilterCallbacks( + StreamEncoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + encoder_callbacks_ = &callbacks; + } + + virtual void log(const HeaderMap* request_headers, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info) override { + Log().debug("Called Mixer::Instance : {}", __func__); + // Make sure not to use any class members at the callback. + // The class may be gone when it is called. + // Log() is a static function so it is OK. + http_control_->Report(request_data_, response_headers, request_info, + check_status_code_, [](const Status& status) { + Log().debug("Report returns status: {}", + status.ToString()); + }); + } + + static spdlog::logger& Log() { + static spdlog::logger& instance = + Logger::Registry::getLog(Logger::Id::http); + return instance; + } +}; + +} // namespace Mixer +} // namespace Http + +namespace Server { +namespace Configuration { + +class MixerConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Both || name != "mixer") { + return nullptr; + } + + Http::Mixer::ConfigPtr mixer_config( + new Http::Mixer::Config(config, server)); + return + [mixer_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::Mixer::Instance(mixer_config)); + callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server diff --git a/src/envoy/mixer/proxy_docker.bzl b/src/envoy/mixer/proxy_docker.bzl new file mode 100644 index 00000000000..8522729628f --- /dev/null +++ b/src/envoy/mixer/proxy_docker.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") + +def proxy_docker_build(images, **kwargs): + for image in images: + docker_build( + name = image['name'], + base = image['base'], + **kwargs + ) diff --git a/src/envoy/prototype/BUILD b/src/envoy/mixer/repositories.bzl similarity index 57% rename from src/envoy/prototype/BUILD rename to src/envoy/mixer/repositories.bzl index 258f2046cd7..2f665d71c4b 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/mixer/repositories.bzl @@ -15,22 +15,17 @@ ################################################################################ # -package(default_visibility = ["//visibility:public"]) +MIXER_CLIENT = "5b5745f29ac5a8babe79ada573defaa83f3bb9e7" -cc_binary( - name = "envoy_esp", - srcs = [ - "api_manager_filter.cc", - "api_manager_env.cc", - "api_manager_env.h", - ], - deps = [ - "//external:protobuf", - "//contrib/endpoints/include:api_manager", - "//contrib/endpoints/src/grpc/transcoding:transcoding", - "//external:servicecontrol", - "@envoy_git//:envoy-common", - "@envoy_git//:envoy-main" - ], - linkstatic=1, -) +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = MIXER_CLIENT, + remote = "https://github.com/istio/mixerclient.git", + ) + + if bind: + native.bind( + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", + ) diff --git a/src/envoy/mixer/start_envoy b/src/envoy/mixer/start_envoy new file mode 100755 index 00000000000..0a24b959ae2 --- /dev/null +++ b/src/envoy/mixer/start_envoy @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +function usage() { + [[ -n "${1}" ]] && echo "${1}" + + cat < "${CONFIG}" + +"${ENVOY}" -c "${CONFIG}" "${DEBUG}" + diff --git a/src/envoy/mixer/string_map.proto b/src/envoy/mixer/string_map.proto new file mode 100644 index 00000000000..bd9140c37d7 --- /dev/null +++ b/src/envoy/mixer/string_map.proto @@ -0,0 +1,23 @@ +// Copyright 2017 Istio Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.proxy.mixer; + +// A message with a map of string to string. It is used to serialize +// a string map. +message StringMap { + map map = 1; +} diff --git a/src/envoy/mixer/utils.cc b/src/envoy/mixer/utils.cc new file mode 100644 index 00000000000..6dc3a78356d --- /dev/null +++ b/src/envoy/mixer/utils.cc @@ -0,0 +1,50 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/utils.h" +#include "src/envoy/mixer/string_map.pb.h" + +namespace Http { +namespace Utils { + +const LowerCaseString kIstioAttributeHeader("x-istio-attributes"); + +StringMap ExtractStringMap(const Json::Object& json, const std::string& name) { + StringMap map; + if (json.hasObject(name)) { + Json::ObjectPtr json_obj = json.getObject(name); + Json::Object* raw_obj = json_obj.get(); + json_obj->iterate( + [&map, raw_obj](const std::string& key, const Json::Object&) -> bool { + map[key] = raw_obj->getString(key); + return true; + }); + } + return map; +} + +std::string SerializeStringMap(const StringMap& string_map) { + ::istio::proxy::mixer::StringMap pb; + ::google::protobuf::Map* map_pb = pb.mutable_map(); + for (const auto& it : string_map) { + (*map_pb)[it.first] = it.second; + } + std::string str; + pb.SerializeToString(&str); + return str; +} + +} // namespace Utils +} // namespace Http diff --git a/src/envoy/mixer/utils.h b/src/envoy/mixer/utils.h new file mode 100644 index 00000000000..9273a9a2d8b --- /dev/null +++ b/src/envoy/mixer/utils.h @@ -0,0 +1,39 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/http/headers.h" +#include "envoy/json/json_object.h" + +namespace Http { +namespace Utils { + +// The internal header to pass istio attributes. +extern const LowerCaseString kIstioAttributeHeader; + +// The string map. +typedef std::map StringMap; + +// Extracts name/value attributes from a json object. +StringMap ExtractStringMap(const Json::Object& json, const std::string& name); + +// Serialize a string map to string. +std::string SerializeStringMap(const StringMap& map); + +} // namespace Utils +} // namespace Http diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md deleted file mode 100644 index dc73f63f6c9..00000000000 --- a/src/envoy/prototype/README.md +++ /dev/null @@ -1,73 +0,0 @@ - -This Proxy will use Envoy and talk to Mixer server. - -## Install dependencies - -``` - apt-get install uuid-dev -``` - -## Build Mixer server - -* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: - -``` - cd $(ISTIO)/mixer - bazel build ...:all -``` - -## Build Envoy proxy - -* Build target envoy_esp: - -``` - bazel build //src/envoy/prototype:envoy_esp -``` - -## How to run it - -* Start mixer server. In mixer folder run: - -``` - bazel-bin/cmd/server/mixs server -``` - - The server will run at port 9091 - -* Start backend Echo server. - -``` - cd test/backend/echo - go run echo.go -``` - -* Modify your iptables: - -``` - sudo iptables -t nat -A OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - -Once you are done, you should remove this rule: - -``` - sudo iptables -t nat -D OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - - -* Start Envoy proxy, run - -``` - bazel-bin/src/envoy/prototype/envoy_esp -c src/envoy/prototype/envoy-esp.conf -``` - -* Then issue HTTP request to proxy. - -``` - curl http://localhost:9090/echo?key=API-KEY -d "hello world" -``` - -## How to add attributes or facts - -Now only some of attributes are passed to mixer. If you want to add more attributes, you can -modify this -[file](https://github.com/istio/proxy/blob/master/contrib/endpoints/src/api_manager/mixer/mixer.cc). diff --git a/src/envoy/prototype/api_manager_env.cc b/src/envoy/prototype/api_manager_env.cc deleted file mode 100644 index f8536bcc10a..00000000000 --- a/src/envoy/prototype/api_manager_env.cc +++ /dev/null @@ -1,219 +0,0 @@ -#include "api_manager_env.h" - -#include "common/http/headers.h" -#include "common/http/message_impl.h" -#include "envoy/event/timer.h" -#include "google/protobuf/stubs/status.h" -#include "source/common/grpc/common.h" - -using ::google::api_manager::utils::Status; -using ::google::protobuf::util::error::Code; - -namespace Http { -namespace ApiManager { - -void Http::ApiManager::Env::Log(LogLevel level, const char *message) { - switch (level) { - case LogLevel::DEBUG: - log().debug("{}", message); - break; - case LogLevel::INFO: - log().info("{}", message); - break; - case LogLevel::WARNING: - log().warn("{}", message); - break; - case LogLevel::ERROR: - log().error("{}", message); - break; - } -} - -class PeriodicTimer : public google::api_manager::PeriodicTimer, - public Logger::Loggable { - private: - Server::Instance &server_; - Event::TimerPtr timer_; - - public: - PeriodicTimer(Server::Instance &server) : server_(server) {} - ~PeriodicTimer() { Stop(); } - void Stop() { - if (timer_) { - timer_->disableTimer(); - timer_ = nullptr; - } - } - void Schedule(std::chrono::milliseconds interval, - std::function continuation) { - Stop(); - timer_ = server_.dispatcher().createTimer([this, continuation, interval]() { - continuation(); - Schedule(interval, continuation); - }); - timer_->enableTimer(interval); - } -}; - -std::unique_ptr Env::StartPeriodicTimer( - std::chrono::milliseconds interval, std::function continuation) { - log().debug("start periodic timer"); - auto single = new PeriodicTimer(server); - single->Schedule(interval, continuation); - std::unique_ptr timer(single); - return timer; -} - -static const LowerCaseString kApiManagerUrl("x-api-manager-url"); -static const LowerCaseString kGrpcTEKey("te"); -static const std::string kGrpcTEValue("trailers"); - -class HTTPRequest : public Http::Message { - private: - HeaderMapImpl header_map_; - Buffer::OwnedImpl body_; - - public: - HTTPRequest(google::api_manager::HTTPRequest *request) - : body_(request->body()) { - header_map_.addStaticKey(Headers::get().Method, request->method()); - - size_t path_pos = request->url().find('/', 8); - if (path_pos == std::string::npos) { - header_map_.addStaticKey(Headers::get().Path, "/"); - } else { - header_map_.addStaticKey(Headers::get().Path, - request->url().substr(path_pos)); - } - - header_map_.addStaticKey(Headers::get().Scheme, "http"); - header_map_.addStaticKey(Headers::get().Host, "localhost"); - header_map_.addStaticKey(Headers::get().ContentLength, body_.length()); - header_map_.addStaticKey(kApiManagerUrl, request->url()); - for (const auto header : request->request_headers()) { - LowerCaseString lower_key(header.first); - HeaderString key, value; - key.setCopy(lower_key.get().data(), lower_key.get().size()); - value.setCopy(header.second.data(), header.second.size()); - header_map_.addViaMove(std::move(key), std::move(value)); - } - } - virtual HeaderMap &headers() override { return header_map_; } - virtual Buffer::Instance *body() override { return &body_; } - virtual void body(Buffer::InstancePtr &&body) override {} - virtual HeaderMap *trailers() override { return nullptr; } - virtual void trailers(HeaderMapPtr &&trailers) override {} - virtual std::string bodyAsString() const override { return ""; } -}; - -class HTTPRequestCallbacks : public AsyncClient::Callbacks { - private: - std::unique_ptr request_; - std::unique_ptr sent_request_; - - public: - HTTPRequestCallbacks( - std::unique_ptr &&request) - : request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - std::map headers; - response->headers().iterate( - [&](const HeaderEntry &header, void *) -> void { - // TODO: fix it - // headers.emplace(header.key().c_str(), header.value().c_str()); - }, - nullptr); - request_->OnComplete(status, std::move(headers), response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to HTTP server."); - std::map headers; - request_->OnComplete(status, std::move(headers), ""); - delete this; - } -}; - -namespace { -// Copy the code here from envoy/grpc/common.cc -Buffer::InstancePtr SerializeGrpcBody(const std::string &body_str) { - // http://www.grpc.io/docs/guides/wire.html - Buffer::InstancePtr body(new Buffer::OwnedImpl()); - uint8_t compressed = 0; - body->add(&compressed, sizeof(compressed)); - uint32_t size = htonl(body_str.size()); - body->add(&size, sizeof(size)); - body->add(body_str); - return body; -} -Http::MessagePtr PrepareGrpcHeaders(const std::string &upstream_cluster, - const std::string &service_full_name, - const std::string &method_name) { - Http::MessagePtr message(new Http::RequestMessageImpl()); - message->headers().insertMethod().value( - Http::Headers::get().MethodValues.Post); - message->headers().insertPath().value( - fmt::format("/{}/{}", service_full_name, method_name)); - message->headers().insertHost().value(upstream_cluster); - message->headers().insertContentType().value(Grpc::Common::GRPC_CONTENT_TYPE); - message->headers().addStatic(kGrpcTEKey, kGrpcTEValue); - return message; -} -} // annoymous namespace - -class GrpcRequestCallbacks : public AsyncClient::Callbacks { - private: - Env *env_; - std::unique_ptr request_; - - public: - GrpcRequestCallbacks( - Env *env, std::unique_ptr &&request) - : env_(env), request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - Grpc::Common::validateResponse(*response); - env_->LogInfo("pass validate"); - // remove 5 bytes of grpc header - response->body()->drain(5); - request_->OnComplete(status, response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to gRPC server."); - request_->OnComplete(status, ""); - delete this; - } -}; - -void Env::RunHTTPRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster("api_manager"); - - MessagePtr message{new HTTPRequest(request.get())}; - HTTPRequestCallbacks *callbacks = - new HTTPRequestCallbacks(std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} - -void Env::RunGRPCRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster(request->server()); - - Http::MessagePtr message = - PrepareGrpcHeaders("localhost", request->service(), request->method()); - message->body(SerializeGrpcBody(request->body())); - auto callbacks = new GrpcRequestCallbacks(this, std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} -} -} diff --git a/src/envoy/prototype/api_manager_env.h b/src/envoy/prototype/api_manager_env.h deleted file mode 100644 index 0ae8136f5be..00000000000 --- a/src/envoy/prototype/api_manager_env.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "precompiled/precompiled.h" - -#include "common/common/logger.h" -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "envoy/upstream/cluster_manager.h" -#include "server/server.h" - -namespace Http { -namespace ApiManager { - -class Env : public google::api_manager::ApiManagerEnvInterface, - public Logger::Loggable { - private: - Server::Instance& server; - Upstream::ClusterManager& cm_; - - public: - Env(Server::Instance& server) - : server(server), cm_(server.clusterManager()){}; - - virtual void Log(LogLevel level, const char* message) override; - virtual std::unique_ptr - StartPeriodicTimer(std::chrono::milliseconds interval, - std::function continuation) override; - virtual void RunHTTPRequest( - std::unique_ptr request) override; - virtual void RunGRPCRequest( - std::unique_ptr request) override; -}; -} -} diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc deleted file mode 100644 index 3983a4d8b41..00000000000 --- a/src/envoy/prototype/api_manager_filter.cc +++ /dev/null @@ -1,346 +0,0 @@ -#include "precompiled/precompiled.h" - -#include "api_manager_env.h" -#include "common/common/logger.h" -#include "common/grpc/common.h" -#include "common/http/filter/ratelimit.h" -#include "common/http/headers.h" -#include "common/http/utility.h" -#include "contrib/endpoints/include/api_manager/api_manager.h" -#include "contrib/endpoints/src/grpc/transcoding/transcoder.h" -#include "envoy/server/instance.h" -#include "server/config/network/http_connection_manager.h" - -namespace Http { -namespace ApiManager { -namespace { - -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); - -} // namespace - -std::string ReadFile(const std::string& file_name) { - std::ifstream t(file_name); - std::string content((std::istreambuf_iterator(t)), - std::istreambuf_iterator()); - return content; -} - -class Config : public Logger::Loggable { - private: - google::api_manager::ApiManagerFactory api_manager_factory_; - std::shared_ptr api_manager_; - Upstream::ClusterManager& cm_; - - public: - Config(const Json::Object& config, Server::Instance& server) - : cm_(server.clusterManager()) { - std::string service_config_content; - if (config.hasObject("service_config")) { - const std::string service_config = config.getString("service_config"); - service_config_content = ReadFile(service_config); - } else { - log().error( - "Service_config is required but not specified in the config: {}", - __func__); - } - - std::string server_config_content; - if (config.hasObject("server_config")) { - const std::string server_config = config.getString("server_config"); - server_config_content = ReadFile(server_config); - } - std::unique_ptr env( - new Env(server)); - - api_manager_ = api_manager_factory_.GetOrCreateApiManager( - std::move(env), service_config_content, server_config_content); - - api_manager_->Init(); - log().debug("Called ApiManager::Config constructor: {}", __func__); - } - - std::shared_ptr& api_manager() { - return api_manager_; - } -}; - -typedef std::shared_ptr ConfigPtr; - -class Request : public google::api_manager::Request { - private: - HeaderMap& header_map_; - std::string downstream_address_; - std::string virtual_host_; - bool query_parsed_; - std::map query_params_; - - public: - Request(HeaderMap& header_map, const std::string& downstream_address, - const std::string& virtual_host) - : header_map_(header_map), - downstream_address_(downstream_address), - virtual_host_(virtual_host), - query_parsed_(false) {} - virtual std::string GetRequestHTTPMethod() override { - return header_map_.Method()->value().c_str(); - } - virtual std::string GetRequestPath() override { - return header_map_.Path()->value().c_str(); - } - virtual std::string GetUnparsedRequestPath() override { - return header_map_.Path()->value().c_str(); - } - - virtual std::string GetClientIP() override { - if (!header_map_.ForwardedFor()) { - return downstream_address_; - } - std::vector xff_address_list = - StringUtil::split(header_map_.ForwardedFor()->value().c_str(), ','); - if (xff_address_list.empty()) { - return downstream_address_; - } - return xff_address_list.front(); - } - - virtual std::string GetClientHost() override { - const HeaderEntry* entry = header_map_.get(kHeaderNameXFH); - if (entry == nullptr) { - return virtual_host_; - } - auto xff_list = StringUtil::split(entry->value().c_str(), ','); - if (xff_list.empty()) { - return virtual_host_; - } - return xff_list.back(); - } - - virtual bool FindQuery(const std::string& name, std::string* query) override { - if (!query_parsed_) { - auto header = header_map_.Path(); - if (header != nullptr) { - std::string path = header->value().c_str(); - Utility::parseQueryString(path).swap(query_params_); - } - query_parsed_ = true; - } - auto entry = query_params_.find(name); - if (entry == query_params_.end()) { - return false; - } - *query = entry->second; - return true; - } - - virtual bool FindHeader(const std::string& name, - std::string* header) override { - LowerCaseString key(name); - const HeaderEntry* entry = header_map_.get(key); - if (entry == nullptr) { - return false; - } - *header = entry->value().c_str(); - return true; - } - - virtual google::api_manager::protocol::Protocol GetRequestProtocol() - override { - return google::api_manager::protocol::Protocol::HTTP; - } - virtual google::api_manager::utils::Status AddHeaderToBackend( - const std::string& key, const std::string& value) override { - return google::api_manager::utils::Status::OK; - } - virtual void SetAuthToken(const std::string& auth_token) override {} - - virtual int64_t GetGrpcRequestBytes() { return 0; } - virtual int64_t GetGrpcResponseBytes() { return 0; } - virtual int64_t GetGrpcRequestMessageCounts() { return 0; } - virtual int64_t GetGrpcResponseMessageCounts() { return 0; } - virtual std::string GetQueryParameters() { return ""; } -}; - -class Response : public google::api_manager::Response { - const AccessLog::RequestInfo& request_info_; - - public: - Response(const AccessLog::RequestInfo& request_info) - : request_info_(request_info) {} - - google::api_manager::utils::Status GetResponseStatus() { - return google::api_manager::utils::Status::OK; - } - - std::size_t GetRequestSize() { return request_info_.bytesReceived(); } - - std::size_t GetResponseSize() { return request_info_.bytesSent(); } - - google::api_manager::utils::Status GetLatencyInfo( - google::api_manager::service_control::LatencyInfo* info) { - info->request_time_ms = request_info_.duration().count(); - return google::api_manager::utils::Status::OK; - } -}; - -const Http::HeaderMapImpl BadRequest{{Http::Headers::get().Status, "400"}}; - -class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { - private: - std::shared_ptr api_manager_; - std::unique_ptr - request_handler_; - - enum State { NotStarted, Calling, Complete, Responded }; - State state_; - - StreamDecoderFilterCallbacks* decoder_callbacks_; - StreamEncoderFilterCallbacks* encoder_callbacks_; - - bool initiating_call_; - - std::string getRouteVirtualHost(HeaderMap& headers) const { - const Router::Route* route = decoder_callbacks_->route(); - if (route && route->routeEntry()) { - return route->routeEntry()->virtualHost().name(); - } - return ""; - } - - public: - Instance(ConfigPtr config) - : api_manager_(config->api_manager()), - state_(NotStarted), - initiating_call_(false) { - Log().debug("Called ApiManager::Instance : {}", __func__); - } - - FilterHeadersStatus decodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr request( - new Request(headers, decoder_callbacks_->downstreamAddress(), - getRouteVirtualHost(headers))); - request_handler_ = api_manager_->CreateRequestHandler(std::move(request)); - state_ = Calling; - initiating_call_ = true; - request_handler_->Check([this](google::api_manager::utils::Status status) { - completeCheck(status); - }); - initiating_call_ = false; - - if (state_ == Complete) { - return FilterHeadersStatus::Continue; - } - Log().debug("Called ApiManager::Instance : {} Stop", __func__); - return FilterHeadersStatus::StopIteration; - } - - FilterDataStatus decodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, - data.length(), end_stream); - if (state_ == Calling) { - return FilterDataStatus::StopIterationAndBuffer; - } - return FilterDataStatus::Continue; - } - - FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - if (state_ == Calling) { - return FilterTrailersStatus::StopIteration; - } - return FilterTrailersStatus::Continue; - } - void setDecoderFilterCallbacks( - StreamDecoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - decoder_callbacks_ = &callbacks; - decoder_callbacks_->addResetStreamCallback( - [this]() { state_ = Responded; }); - } - void completeCheck(const google::api_manager::utils::Status& status) { - Log().debug("Called ApiManager::Instance : check complete {}", - status.ToJson()); - if (!status.ok() && state_ != Responded) { - state_ = Responded; - decoder_callbacks_->dispatcher().post([this, status]() { - Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), - status.ToJson()); - }); - return; - } - state_ = Complete; - if (!initiating_call_) { - decoder_callbacks_->dispatcher().post( - [this]() { decoder_callbacks_->continueDecoding(); }); - } - } - - virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterHeadersStatus::Continue; - } - virtual FilterDataStatus encodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterDataStatus::Continue; - } - virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterTrailersStatus::Continue; - } - virtual void setEncoderFilterCallbacks( - StreamEncoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - encoder_callbacks_ = &callbacks; - } - - virtual void log(const HeaderMap* request_headers, - const HeaderMap* response_headers, - const AccessLog::RequestInfo& request_info) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr response( - new Response(request_info)); - request_handler_->Report(std::move(response), []() {}); - } - - spdlog::logger& Log() { - static spdlog::logger& instance = - Logger::Registry::getLog(Logger::Id::http); - return instance; - } -}; -} -} - -namespace Server { -namespace Configuration { - -class ApiManagerConfig : public HttpFilterConfigFactory { - public: - HttpFilterFactoryCb tryCreateFilterFactory( - HttpFilterType type, const std::string& name, const Json::Object& config, - const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Both || name != "esp") { - return nullptr; - } - - Http::ApiManager::ConfigPtr api_manager_config( - new Http::ApiManager::Config(config, server)); - return [api_manager_config]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::ApiManager::Instance(api_manager_config)); - callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); - callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); - }; - } -}; - -static RegisterHttpFilterConfigFactory register_; -} -} diff --git a/src/envoy/prototype/dummy_api_manager_cluster.py b/src/envoy/prototype/dummy_api_manager_cluster.py deleted file mode 100644 index 3f83476aae8..00000000000 --- a/src/envoy/prototype/dummy_api_manager_cluster.py +++ /dev/null @@ -1,91 +0,0 @@ -from BaseHTTPServer import BaseHTTPRequestHandler -from BaseHTTPServer import HTTPServer -from subprocess import Popen, PIPE, STDOUT - -PUBLIC_KEY = '''{ - "keys": [ - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "686f442dead739e241342519849bd0d140707a86", - "n": "yVGRzBnVmCHC9uMH_mCna7FiSKOjMEsDdgf8hds41KYfUHH9cp4P41iQEBnqPYvQiMYaZDuatkMDh25ukmkSyfLwJIVQpnHgMDwoZ1cx83Z0PwC8wAAJYqKyItKNfFN4UJ6DZIOVU-Iqgxi8VOtGwMNx2LiD1NoFVfXyz52UJ_QLiUGzErVwTGv4UD6NtaWKFkctTnEG-9rZvDF8QknnzxAomVa2OcV8OHeszx6N8UE1tm9Kq4xj2Uu8D3dDrfu2jr45Pi6RHIZOTAnu8G7wTDNaiGbENrbHSk6WAjLZBOcWZj9SDlDlwH2xFoKdpNKRmLKpPQblHem_y1KRYwvJTw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "d57d66bfbff089769e3545589572d3b53955a8cb", - "n": "zX99YCqicbn_j-5YTJ-2FsgONUg7cmqJiwvHXrUopRvn2Tukrd0B5Sg-Rq1hdZTYgIym7lSMw_zLxIQCH54sUfydX5MMWr6FOxVUbYl-E0Oko85Yer9dFv61rN0USj_A12QRMmjCZkcqH_6MWWuA1QWaejyStopjpLEYnUD3bP6oS604eZWkkOp8Nu-Vg4NqqX7ZClIcQqe03xv3sFHiPuhB-qaifhpIPpKCiYSKxAY4_GxwCJ_ml_uJ5k1tIrDykAlie6aWxv8hogOXrQmNRCO2Qcumwb7d9cci1UxsEYOtpZxhTiZqWsbrBxwfvLqU_rvsCT8vOjrBPOkTtYl22w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "516ca9a83b130d4b605684fe7df83b43f33fba43", - "n": "rSLxCvtRLoJUt2xPTSjDgN8_7kuBLsZz0JD4HRiwvId_IYY-T5MsUirBx01AtI3KRlmeQ-1QIvET4cNN99QL8U6e7Hh75VWJvQhh0_TFdKbv_75b81-rzx-EvIoVe0czeY3QeRp3DDORRk9o7xgri6U3VfT2zcQIDAPY14h3wLHE5DQuH9EpbxjS_wNrQxRYZH_mWgmWU7h001WHWRDKdEIgtzQkinKOTig90Wy4pt_vgMGcv0mh_wYvIcmB62Qj1sHUUOa5NJWUaZLC4Stj7CH0dLiBrVqg3JFqvO60Oo3wrcdmNQl0RaRHFjBYLameUPSP6M2BevfCRx8-Ix26rw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "33a681b36b8983913b05f8d5eab165c60694b6fc", - "n": "pRbWhLXIOvF1Naxh5n2evZqpJ3kPEFL4b-jHRHOhMnzqsi_aeQtIwYLVM8zdYYfgCxRq8umG1YwMenBKKPzRWr4MkFAB64O3UPgdyg-3je-fgCMziqS_3KH7AXHekxG_ZHpwbkgilRMtJNiDnSZWGad8XAfW3VFct4RqRAaf7h-6Za0IM7R3u4VYkUfosNqKtoDJDQrng9Nbv9ryUk8u1WikKF0M1r-wrZoDA7QFRAFkbdfYyvHL2daUflNDIXmFkUeHGGApMlJQ3wJk7Tln4txGMSUdJoD3JWEyKa2W1WshtqBHnQb8VlL77H-ch9zbn5pGZCoJ0MsgJr0vjoea8w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "986f2c30b9843defdc8f8debe138d640d1d67d9d", - "n": "qfE-u8I60zP_Vg3UsP_X9Qv1idkrnsXKkcQcsGWUdqs4jM2VixGBJDMJWbP55aJA93Sl49_dzAv_lr5-ctc1L04ke6nHX19EEBDtFTjGKyEdI-X1C-6HXCQmpI1XvUA8DzTOIZd5KEXJgBA9tpn6qkMAHzXyMHOfg8nhW36She4QggFjfF_RDKOA2-jRXDIjinOmIaLhVq4hsC9hCshhsfreLPw3HH8UhRONbkoFU_8ZBgAQLdVB1TQwp-ZDiVyh9od9a-RmfIiUl27AK5LDrtpVRCtXj6bv9OMx4QWVX8G-NfQexDwAp1pICaE5qYyJbiIK25E07vPTJ1ZtMpodFQ", - "e": "AQAB" - } - ] -} -''' - -def print_body(binary, expected): - p = Popen(['bazel-bin/src/tools/service_control_json_gen', - '--text', - '--'+expected, - '--stdin'], stdin=PIPE) - p.communicate(input=binary) - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - self.handle_request("GET") - - def do_POST(self): - self.handle_request("POST") - - - - def handle_request(self, method): - print(method, self.path, self.headers.items()) - url = self.headers.get('x-api-manager-url', '') - if url == 'https://www.googleapis.com/service_accounts/v1/jwk/loadtest@esp-test-client.iam.gserviceaccount.com': - self.send_response(200) - self.send_header('Content-Length', str(len(PUBLIC_KEY))) - self.end_headers() - self.wfile.write(PUBLIC_KEY) - else: - #content_len = self.headers.getheader('content-length', 0) - #post_body = self.rfile.read(int(content_len)) - if url.endswith(":report"): - print_body(post_body, "report_request") - elif url.endswith(":check"): - print_body(post_body, "check_request") - - self.send_response(200) - self.send_header('Content-Length', "0") - self.end_headers() - - -if __name__ == '__main__': - server = HTTPServer(('localhost', 8081), Handler) - print 'Starting server, use to stop' - server.serve_forever() diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf deleted file mode 100644 index 27ea9f1b397..00000000000 --- a/src/envoy/prototype/envoy-esp.conf +++ /dev/null @@ -1,89 +0,0 @@ -{ - "listeners": [ - { - "port": 9092, - "bind_to_port": true, - "use_original_dst": true, - "filters": [] - }, - { - "port": 9090, - "bind_to_port": false, - "filters": [ - { - "type": "read", - "name": "http_connection_manager", - "config": { - "codec_type": "auto", - "stat_prefix": "ingress_http", - "route_config": { - "virtual_hosts": [ - { - "name": "backend", - "domains": ["*"], - "routes": [ - { - "timeout_ms": 0, - "prefix": "/", - "cluster": "service1" - } - ] - } - ] - }, - "access_log": [ - { - "path": "/dev/stdout" - } - ], - "filters": [ - { - "type": "both", - "name": "esp", - "config": { - "service_config": "src/envoy/prototype/generic_service_config.json", - "server_config": "src/envoy/prototype/server_config.pb.txt" - } - }, - { - "type": "decoder", - "name": "router", - "config": {} - } - ] - } - } - ] - } - ], - "admin": { - "access_log_path": "/dev/stdout", - "port": 9001 - }, - "cluster_manager": { - "clusters": [ - { - "name": "service1", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8080" - } - ] - }, - { - "name": "api_manager", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8081" - } - ] - } - ] - } -} diff --git a/src/envoy/prototype/generic_service_config.json b/src/envoy/prototype/generic_service_config.json deleted file mode 100644 index ac926e5e4c2..00000000000 --- a/src/envoy/prototype/generic_service_config.json +++ /dev/null @@ -1,736 +0,0 @@ -{ - "apis": [ - { - "methods": [ - { - "name": "Get", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Delete", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Patch", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Post", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Put", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "generic_qiwzhang_app_1_appspot_com_1_0_0", - "sourceContext": { - "fileName": "sss.yaml" - }, - "version": "1.0.0" - } - ], - "authentication": {}, - "backend": { - "rules": [ - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "configVersion": 3, - "context": {}, - "control": { - "environment": "servicecontrol.googleapis.com" - }, - "customError": {}, - "documentation": {}, - "enums": [ - { - "enumvalue": [ - { - "name": "NULL_VALUE" - } - ], - "name": "google.protobuf.NullValue", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "http": { - "rules": [ - { - "get": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "delete": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "patch": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "post": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "put": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "id": "2017-01-06r0", - "logging": { - "producerDestinations": [ - { - "logs": [ - "endpoints_log" - ], - "monitoredResource": "api" - } - ] - }, - "logs": [ - { - "name": "endpoints_log" - } - ], - "metrics": [ - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/error_count", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user_country" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/referer" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/consumer_id" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_billable_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_bytes", - "valueType": "INT64" - } - ], - "monitoredResources": [ - { - "labels": [ - { - "key": "cloud.googleapis.com/location" - }, - { - "key": "cloud.googleapis.com/uid" - }, - { - "key": "serviceruntime.googleapis.com/api_version" - }, - { - "key": "serviceruntime.googleapis.com/api_method" - }, - { - "key": "serviceruntime.googleapis.com/consumer_project" - }, - { - "key": "cloud.googleapis.com/project" - }, - { - "key": "cloud.googleapis.com/service" - } - ], - "type": "api" - } - ], - "monitoring": { - "consumerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/consumer/request_count", - "serviceruntime.googleapis.com/api/consumer/error_count", - "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "serviceruntime.googleapis.com/api/consumer/total_latencies", - "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "serviceruntime.googleapis.com/api/consumer/request_sizes", - "serviceruntime.googleapis.com/api/consumer/response_sizes", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "serviceruntime.googleapis.com/api/consumer/request_bytes", - "serviceruntime.googleapis.com/api/consumer/response_bytes" - ], - "monitoredResource": "api" - } - ], - "producerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/producer/request_count", - "serviceruntime.googleapis.com/api/producer/error_count", - "serviceruntime.googleapis.com/api/producer/total_latencies", - "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/request_sizes", - "serviceruntime.googleapis.com/api/producer/response_sizes", - "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_durations", - "serviceruntime.googleapis.com/api/producer/request_bytes", - "serviceruntime.googleapis.com/api/producer/response_bytes", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes" - ], - "monitoredResource": "api" - } - ] - }, - "name": "generic.qiwzhang-app-1.appspot.com", - "producerProjectId": "qiwzhang-app-1", - "systemParameters": {}, - "title": "Basic HTTP service configuration", - "types": [ - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "values", - "kind": "TYPE_MESSAGE", - "name": "values", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.ListValue", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "fields", - "kind": "TYPE_MESSAGE", - "name": "fields", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Struct.FieldsEntry" - } - ], - "name": "google.protobuf.Struct", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "key", - "kind": "TYPE_STRING", - "name": "key", - "number": 1 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "value", - "kind": "TYPE_MESSAGE", - "name": "value", - "number": 2, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.Struct.FieldsEntry", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "name": "google.protobuf.Empty", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "nullValue", - "kind": "TYPE_ENUM", - "name": "null_value", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.NullValue" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "numberValue", - "kind": "TYPE_DOUBLE", - "name": "number_value", - "number": 2 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "stringValue", - "kind": "TYPE_STRING", - "name": "string_value", - "number": 3 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "boolValue", - "kind": "TYPE_BOOL", - "name": "bool_value", - "number": 4 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "structValue", - "kind": "TYPE_MESSAGE", - "name": "struct_value", - "number": 5, - "typeUrl": "type.googleapis.com/google.protobuf.Struct" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "listValue", - "kind": "TYPE_MESSAGE", - "name": "list_value", - "number": 6, - "typeUrl": "type.googleapis.com/google.protobuf.ListValue" - } - ], - "name": "google.protobuf.Value", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "usage": { - "rules": [ - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "visibility": {} -} diff --git a/src/envoy/prototype/server_config.pb.txt b/src/envoy/prototype/server_config.pb.txt deleted file mode 100644 index 19cb6cbb2c0..00000000000 --- a/src/envoy/prototype/server_config.pb.txt +++ /dev/null @@ -1,6 +0,0 @@ -cloud_tracing_config { - force_disable: true -} -mixer_options { - mixer_server: "localhost:9091" -} diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 94f371ebc7d..ca7e9468ded 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -431,8 +431,8 @@ cc_library( native.new_http_archive( name = "nghttp2_tar", - url = "https://github.com/nghttp2/nghttp2/releases/download/v1.14.1/nghttp2-1.14.1.tar.gz", - strip_prefix = "nghttp2-1.14.1", + url = "https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz", + strip_prefix = "nghttp2-1.20.0", build_file_content = BUILD, ) @@ -442,6 +442,127 @@ cc_library( actual = "@nghttp2_tar//:nghttp2", ) +def ares_repositories(bind=True): + BUILD = """ +cc_library( + name = "ares", + srcs = [ + "ares__close_sockets.c", + "ares__get_hostent.c", + "ares__read_line.c", + "ares__timeval.c", + "ares_cancel.c", + "ares_create_query.c", + "ares_data.c", + "ares_destroy.c", + "ares_expand_name.c", + "ares_expand_string.c", + "ares_fds.c", + "ares_free_hostent.c", + "ares_free_string.c", + "ares_getenv.c", + "ares_gethostbyaddr.c", + "ares_gethostbyname.c", + "ares_getnameinfo.c", + "ares_getopt.c", + "ares_getsock.c", + "ares_init.c", + "ares_library_init.c", + "ares_llist.c", + "ares_mkquery.c", + "ares_nowarn.c", + "ares_options.c", + "ares_parse_a_reply.c", + "ares_parse_aaaa_reply.c", + "ares_parse_mx_reply.c", + "ares_parse_naptr_reply.c", + "ares_parse_ns_reply.c", + "ares_parse_ptr_reply.c", + "ares_parse_soa_reply.c", + "ares_parse_srv_reply.c", + "ares_parse_txt_reply.c", + "ares_platform.c", + "ares_process.c", + "ares_query.c", + "ares_search.c", + "ares_send.c", + "ares_strcasecmp.c", + "ares_strdup.c", + "ares_strerror.c", + "ares_timeout.c", + "ares_version.c", + "ares_writev.c", + "bitncmp.c", + "inet_net_pton.c", + "inet_ntop.c", + "windows_port.c", + ], + hdrs = [ + "ares_config.h", + "ares.h", + "ares_build.h", + "ares_data.h", + "ares_dns.h", + "ares_getenv.h", + "ares_getopt.h", + "ares_inet_net_pton.h", + "ares_iphlpapi.h", + "ares_ipv6.h", + "ares_library_init.h", + "ares_llist.h", + "ares_nowarn.h", + "ares_platform.h", + "ares_private.h", + "ares_rules.h", + "ares_setup.h", + "ares_strcasecmp.h", + "ares_strdup.h", + "ares_version.h", + "ares_writev.h", + "bitncmp.h", + "nameser.h", + "setup_once.h", + ], + copts = [ + "-DHAVE_CONFIG_H", + ], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "config", + srcs = glob(["**/*"]), + outs = ["ares_config.h"], + cmd = "pushd external/cares_git ; ./buildconf ; ./configure ; cp ares_config.h ../../$@", + visibility = ["//visibility:public"], +) + +genrule( + name = "ares_build", + srcs = [ + "ares_build.h.dist", + ], + outs = [ + "ares_build.h", + ], + cmd = "cp $(SRCS) $@", + visibility = ["//visibility:public"], +) +""" + + native.new_git_repository( + name = "cares_git", + remote = "https://github.com/c-ares/c-ares.git", + commit = "7691f773af79bf75a62d1863fd0f13ebf9dc51b1", # v1.12.0 + build_file_content = BUILD, + ) + + if bind: + native.bind( + name = "ares", + actual = "@cares_git//:ares", + ) def envoy_repositories(bind=True): libevent_repositories(bind) @@ -451,6 +572,7 @@ def envoy_repositories(bind=True): http_parser_repositories(bind) rapidjson_repositories(bind) nghttp2_repositories(bind) + ares_repositories(bind) BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") @@ -541,6 +663,7 @@ cc_library( alwayslink=1, deps = [ ":envoy-ratelimit-pb", + "//external:ares", "//external:libssl", "//external:nghttp2", "//external:spdlog", @@ -629,6 +752,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "02c6fc97b4c21d25ab596a25208fbe283e927f6a", + commit = "70e5d651b55d356770529e5bee9c6b2707d9cf21", # 3/1/2017 build_file_content = BUILD, )